diff --git a/.containerignore b/.containerignore
new file mode 100644
index 000000000..8ce8ebbbf
--- /dev/null
+++ b/.containerignore
@@ -0,0 +1,20 @@
+# Build context for containers/factory/Containerfile.
+#
+# The image needs pyproject.toml, uv.lock, README.md, factory/ and skills/ — nothing else. The
+# repository as a whole is several hundred megabytes, most of it history, benchmark data and
+# virtualenvs, and every byte of it is streamed to the engine on each build.
+*
+!pyproject.toml
+!uv.lock
+!README.md
+!factory/
+!skills/
+
+# Re-excluded inside the directories that are included: caches and virtualenvs are large, are
+# rebuilt inside the image anyway, and an arm64 .venv copied into an amd64 image is actively wrong.
+**/__pycache__/
+**/*.pyc
+**/.venv/
+**/.pytest_cache/
+**/.ruff_cache/
+**/.mypy_cache/
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index c76d11d9d..30ff2d527 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -3,8 +3,6 @@ name: Benchmark CI
on:
push:
branches: [main]
- schedule:
- - cron: '17 6 * * 1-5'
workflow_dispatch:
inputs:
benchmark:
@@ -16,6 +14,11 @@ on:
- featurebench
- terminalbench
- programbench
+ - legacybench
+ - harborindex
+ - tomswe
+ - salitrap
+ - devopsgym
- all
instance_id:
description: 'Instance ID (leave default for smoke test)'
@@ -78,6 +81,26 @@ jobs:
solver: factory
default_instance: 'cmatrix'
enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'programbench' || inputs.benchmark == 'all') && (inputs.solver != 'claude-code') }}
+ - benchmark: legacybench
+ solver: factory
+ default_instance: '1907c2-c-debug-legacy-buddy-fix'
+ enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'legacybench' || inputs.benchmark == 'all') && (inputs.solver != 'claude-code') }}
+ - benchmark: harborindex
+ solver: factory
+ default_instance: 'bix-filter-chip-variants'
+ enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'harborindex' || inputs.benchmark == 'all') && (inputs.solver != 'claude-code') }}
+ - benchmark: tomswe
+ solver: factory
+ default_instance: 'sympy__sympy-20590'
+ enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'tomswe' || inputs.benchmark == 'all') && (inputs.solver != 'claude-code') }}
+ - benchmark: salitrap
+ solver: factory
+ default_instance: 'salitrap-001'
+ enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'salitrap' || inputs.benchmark == 'all') && (inputs.solver != 'claude-code') }}
+ - benchmark: devopsgym
+ solver: factory
+ default_instance: 'build-maven-dependency-resolution'
+ enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'devopsgym' || inputs.benchmark == 'all') && (inputs.solver != 'claude-code') }}
# Claude Code solver entries — enabled on schedule, release, or workflow_dispatch with matching benchmark+solver
- benchmark: swebench
solver: claude-code
@@ -95,6 +118,26 @@ jobs:
solver: claude-code
default_instance: 'cmatrix'
enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'programbench' || inputs.benchmark == 'all') && (inputs.solver == 'claude-code' || inputs.solver == 'both') }}
+ - benchmark: legacybench
+ solver: claude-code
+ default_instance: '1907c2-c-debug-legacy-buddy-fix'
+ enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'legacybench' || inputs.benchmark == 'all') && (inputs.solver == 'claude-code' || inputs.solver == 'both') }}
+ - benchmark: harborindex
+ solver: claude-code
+ default_instance: 'bix-filter-chip-variants'
+ enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'harborindex' || inputs.benchmark == 'all') && (inputs.solver == 'claude-code' || inputs.solver == 'both') }}
+ - benchmark: tomswe
+ solver: claude-code
+ default_instance: 'sympy__sympy-20590'
+ enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'tomswe' || inputs.benchmark == 'all') && (inputs.solver == 'claude-code' || inputs.solver == 'both') }}
+ - benchmark: salitrap
+ solver: claude-code
+ default_instance: 'salitrap-001'
+ enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'salitrap' || inputs.benchmark == 'all') && (inputs.solver == 'claude-code' || inputs.solver == 'both') }}
+ - benchmark: devopsgym
+ solver: claude-code
+ default_instance: 'build-maven-dependency-resolution'
+ enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'devopsgym' || inputs.benchmark == 'all') && (inputs.solver == 'claude-code' || inputs.solver == 'both') }}
steps:
- name: Skip if not enabled
@@ -136,7 +179,7 @@ jobs:
- name: Install Factory CLI
if: steps.gate.outputs.run == 'true' && matrix.solver == 'factory'
run: |
- uv tool install 'remote-factory @ git+https://github.com/akashgit/remote-factory.git'
+ uv tool install 'remote-factory[telemetry] @ git+https://github.com/akashgit/remote-factory.git'
export PATH="$HOME/.local/bin:$PATH"
echo "PATH=$HOME/.local/bin:$PATH" >> "$GITHUB_ENV"
factory --help > /dev/null
@@ -161,7 +204,7 @@ jobs:
if [ '${{ github.event_name }}' = 'workflow_dispatch' ]; then
echo "value=$BENCHMARK_TIMEOUT" >> "$GITHUB_OUTPUT"
elif [ '${{ github.event_name }}' = 'schedule' ]; then
- echo 'value=3600' >> "$GITHUB_OUTPUT"
+ echo 'value=7200' >> "$GITHUB_OUTPUT"
else
echo 'value=7200' >> "$GITHUB_OUTPUT"
fi
@@ -178,17 +221,15 @@ jobs:
CLAUDE_CODE_DISABLE_ADAPTIVE_THINKING: "1"
MAX_THINKING_TOKENS: "128000"
CLAUDE_CODE_EFFORT_LEVEL: "XHIGH"
+ LANGFUSE_HOST: ${{ vars.LANGFUSE_BENCH_HOST }}
+ LANGFUSE_BASE_URL: ${{ vars.LANGFUSE_BENCH_HOST }}
+ LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_BENCH_PUBLIC_KEY }}
+ LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_BENCH_SECRET_KEY }}
+ FACTORY_GIT_REF: ${{ github.sha }}
run: |
chmod +x benchmarks/run.sh benchmarks/lib.sh benchmarks/run-*.sh
benchmarks/run.sh ${{ matrix.benchmark }} ${{ steps.config.outputs.instance }} --timeout ${{ steps.timeout.outputs.value }} --solver ${{ matrix.solver }}
- - name: Upload results
- uses: actions/upload-artifact@v4
- if: always() && steps.gate.outputs.run == 'true'
- with:
- name: benchmark-results-${{ matrix.benchmark }}-${{ matrix.solver }}
- path: benchmarks/results/
-
- name: Print summary
if: always() && steps.gate.outputs.run == 'true'
run: |
@@ -203,6 +244,47 @@ jobs:
done
fi
+ - name: Analyze failures
+ if: always() && steps.gate.outputs.run == 'true'
+ continue-on-error: true
+ env:
+ CLAUDE_CODE_USE_VERTEX: "1"
+ ANTHROPIC_VERTEX_PROJECT_ID: ${{ secrets.GCP_PROJECT }}
+ CLOUD_ML_REGION: ${{ secrets.GCP_REGION }}
+ LANGFUSE_HOST: ${{ vars.LANGFUSE_BENCH_HOST }}
+ LANGFUSE_BASE_URL: ${{ vars.LANGFUSE_BENCH_HOST }}
+ LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_BENCH_PUBLIC_KEY }}
+ LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_BENCH_SECRET_KEY }}
+ run: |
+ pip install python-dotenv requests --quiet
+ for result_file in benchmarks/results/*.json; do
+ [ -f "$result_file" ] || continue
+ resolved=$(python3 -c "import json; print(json.load(open('$result_file')).get('resolved', False))")
+ if [ "$resolved" = "False" ]; then
+ echo "Analyzing failure: $result_file"
+ base="$(basename "$result_file" .json)"
+ python3 scripts/langfuse/analyze_failure.py "$result_file" \
+ --summary --output "benchmarks/results/${base}-summary.md" \
+ || echo "Summary failed for $result_file (non-fatal)"
+ python3 scripts/langfuse/analyze_failure.py "$result_file" \
+ --output "benchmarks/results/${base}-analysis.md" \
+ || echo "Analysis failed for $result_file (non-fatal)"
+ for suffix in summary analysis; do
+ f="benchmarks/results/${base}-${suffix}.md"
+ if [ -f "$f" ]; then
+ cat "$f" >> $GITHUB_STEP_SUMMARY
+ fi
+ done
+ fi
+ done
+
+ - name: Upload results
+ uses: actions/upload-artifact@v4
+ if: always() && steps.gate.outputs.run == 'true'
+ with:
+ name: benchmark-results-${{ matrix.benchmark }}-${{ matrix.solver }}
+ path: benchmarks/results/
+
report:
needs: benchmark
if: always()
@@ -253,6 +335,23 @@ jobs:
print(f'Processing: {f}', file=sys.stderr)
with open(f) as fh:
data = json.load(fh)
+
+ base = os.path.splitext(os.path.basename(f))[0]
+ summary_path = os.path.join(results_dir, base + '-summary.md')
+ analysis_path = os.path.join(results_dir, base + '-analysis.md')
+
+ if os.path.isfile(summary_path):
+ with open(summary_path) as sf:
+ data['trace_summary'] = sf.read().strip()
+ if os.path.isfile(analysis_path):
+ with open(analysis_path) as af:
+ data['trace_analysis'] = af.read().strip()
+
+ trace_id = (data.get('details') or {}).get('trace_id', '')
+ langfuse_host = os.environ.get('LANGFUSE_HOST', '')
+ if trace_id and langfuse_host:
+ data['trace_url'] = f'{langfuse_host}/trace/{trace_id}'
+
data['run_id'] = os.environ.get('RUN_ID', '')
data['commit'] = os.environ.get('COMMIT', '')
data['ref'] = os.environ.get('REF', '')
@@ -269,6 +368,7 @@ jobs:
REF="${{ github.ref }}" \
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
TRIGGER="${{ github.event_name }}" \
+ LANGFUSE_HOST="${{ vars.LANGFUSE_BENCH_HOST }}" \
python3 /tmp/append_results.py
echo 'DEBUG: results dir contents:'
@@ -354,6 +454,30 @@ jobs:
body += '_No benchmark results found._\n';
}
+ const summaryFiles = new Set();
+ for (const file of files) {
+ if (file.endsWith('-summary.md')) {
+ summaryFiles.add(file);
+ const benchName = file.replace('-summary.md', '').replace(/^\d{8}T\d{6}Z-/, '');
+ const summaryText = fs.readFileSync('results/' + file, 'utf8').trim();
+ const analysisFile = file.replace('-summary.md', '-analysis.md');
+ body += '**' + benchName + ':** ' + summaryText + '\n';
+ if (fs.existsSync('results/' + analysisFile)) {
+ const analysis = fs.readFileSync('results/' + analysisFile, 'utf8');
+ body += 'Detailed analysis\n\n' + analysis + '\n\n\n';
+ }
+ }
+ }
+ for (const file of files) {
+ if (file.endsWith('-analysis.md') && !summaryFiles.has(file.replace('-analysis.md', '-summary.md'))) {
+ const analysis = fs.readFileSync('results/' + file, 'utf8');
+ const benchName = file.replace('-analysis.md', '').replace(/^\d{8}T\d{6}Z-/, '');
+ body += 'Failure Analysis: ' + benchName + '\n\n';
+ body += analysis;
+ body += '\n\n\n';
+ }
+ }
+
const historyPath = 'benchmark-data/results.jsonl';
let baselines = {};
if (fs.existsSync(historyPath)) {
diff --git a/.github/workflows/ceo-review.yml b/.github/workflows/ceo-review.yml
index eb79941a9..844299932 100644
--- a/.github/workflows/ceo-review.yml
+++ b/.github/workflows/ceo-review.yml
@@ -14,9 +14,9 @@ jobs:
if: >-
github.event.issue.pull_request &&
contains(github.event.comment.body, '@ceo-review') &&
- contains(fromJSON('["akashgit", "xukai92", "colehurwitz", "shivchander", "osilkin98", "gx-ai-architect"]'), github.event.comment.user.login)
+ contains(fromJSON('["akashgit", "xukai92", "colehurwitz", "shivchander", "osilkin98", "gx-ai-architect", "RobotSail", "mihirathale98", "lukeinglis", "nehamalepati", "abhi1092", "s-akhtar-baig"]'), github.event.comment.user.login)
runs-on: ubuntu-latest
- timeout-minutes: 30
+ timeout-minutes: 120
steps:
- name: React with eyes
@@ -34,11 +34,18 @@ jobs:
id: pr
run: echo "number=${{ github.event.issue.number }}" >> "$GITHUB_OUTPUT"
- - name: Checkout PR merge ref
+ - name: Checkout default branch (trusted factory)
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.repository.default_branch }}
+ path: factory-trusted
+
+ - name: Checkout PR merge ref (code under review)
uses: actions/checkout@v4
with:
ref: refs/pull/${{ steps.pr.outputs.number }}/merge
fetch-depth: 0
+ path: pr-code
- name: Authenticate to Google Cloud
uses: google-github-actions/auth@v2
@@ -49,7 +56,10 @@ jobs:
uses: astral-sh/setup-uv@v4
- name: Install factory
- run: uv sync
+ working-directory: factory-trusted
+ run: |
+ uv sync --extra telemetry
+ uv tool install -e .
- name: Set up Node.js
uses: actions/setup-node@v4
@@ -66,13 +76,18 @@ jobs:
claude --version
- name: Run CEO review
+ working-directory: factory-trusted
env:
CLAUDE_CODE_USE_VERTEX: "1"
ANTHROPIC_VERTEX_PROJECT_ID: ${{ secrets.GCP_PROJECT }}
CLOUD_ML_REGION: ${{ secrets.GCP_REGION }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ FACTORY_MODEL: "claude-opus-4-6[1m]"
+ LANGFUSE_HOST: ${{ secrets.LANGFUSE_HOST }}
+ LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
+ LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
run: |
- uv run factory ceo . --mode review --pr ${{ steps.pr.outputs.number }} --headless
+ factory ceo ${{ github.workspace }}/pr-code --mode deep-qa --pr ${{ steps.pr.outputs.number }} --headless
- name: Approve PR if verdict is KEEP
if: success()
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 140339b44..b90374da2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -111,11 +111,23 @@ jobs:
- name: Set up Python
run: uv python install 3.12
- name: Install dependencies
- run: uv sync --all-groups
+ run: |
+ uv sync --all-groups
+ uv tool install -e .
- name: Ruff check
run: uv run ruff check .
- name: Mypy
run: uv run mypy factory/
+ - name: Lint contributed workflows
+ run: factory workflow lint-contributed
- name: Check plugin agents in sync
if: hashFiles('agents/') != ''
run: uv run python scripts/sync_agents.py --check
+ - name: Check no generated workflow skills committed
+ run: |
+ tracked=$(git ls-files 'skills/workflow-*/SKILL.md' 'skills/workflow-*/SKILL.annotations.yaml')
+ if [ -n "$tracked" ]; then
+ echo 'ERROR: Generated workflow skills should not be committed'
+ echo "$tracked"
+ exit 1
+ fi
diff --git a/.github/workflows/conflict-detector.yml b/.github/workflows/conflict-detector.yml
new file mode 100644
index 000000000..8f9c3b3da
--- /dev/null
+++ b/.github/workflows/conflict-detector.yml
@@ -0,0 +1,201 @@
+name: PR Conflict Detector
+
+on:
+ push:
+ branches: [main]
+
+concurrency:
+ group: conflict-detector
+ cancel-in-progress: true
+
+permissions:
+ contents: write
+ pull-requests: write
+ issues: write
+
+jobs:
+ detect:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Fetch all remote branches
+ run: git fetch --all
+
+ - name: Load historical conflict data from conflict-data branch
+ run: |
+ git fetch origin conflict-data || true
+ git show origin/conflict-data:conflicts.jsonl > conflicts.jsonl 2>/dev/null || true
+ if [ -f conflicts.jsonl ]; then
+ echo "Loaded existing conflicts.jsonl from conflict-data branch"
+ else
+ echo "No existing conflicts.jsonl found, starting fresh"
+ fi
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Run conflict detection
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: python scripts/conflict_detector.py detect --data-file conflicts.jsonl || true
+
+ - name: Post comments on conflicting PRs
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ if [ ! -f conflicts.jsonl ]; then
+ echo "No conflicts.jsonl found, skipping."
+ exit 0
+ fi
+
+ # Read the last detection run's events (same timestamp)
+ export LATEST_TS=$(tail -1 conflicts.jsonl | python3 -c "import sys,json; print(json.load(sys.stdin)['timestamp'])" 2>/dev/null || echo "")
+ if [ -z "$LATEST_TS" ]; then
+ echo "No events to process."
+ exit 0
+ fi
+
+ python3 -c "
+ import json, subprocess, sys, os
+
+ latest_ts = os.environ.get('LATEST_TS', '')
+ if not latest_ts:
+ sys.exit(0)
+
+ repo = os.environ.get('GITHUB_REPOSITORY', '')
+ if not repo:
+ print('GITHUB_REPOSITORY not set', file=sys.stderr)
+ sys.exit(1)
+
+ all_prs = set()
+ current_conflicts = {}
+ with open('conflicts.jsonl') as f:
+ for line in f:
+ line = line.strip()
+ if not line:
+ continue
+ ev = json.loads(line)
+ all_prs.add(ev['pr_number'])
+ if ev['timestamp'] == latest_ts:
+ current_conflicts[ev['pr_number']] = ev['conflict_files']
+
+ def make_marker(pr):
+ return f''
+
+ def find_bot_comment(pr):
+ marker = make_marker(pr)
+ r = subprocess.run(
+ ['gh', 'api', f'repos/{repo}/issues/{pr}/comments?per_page=100'],
+ capture_output=True, text=True
+ )
+ if r.returncode != 0:
+ return None, None
+ try:
+ comments = json.loads(r.stdout)
+ except (json.JSONDecodeError, TypeError):
+ return None, None
+ for c in comments:
+ if marker in (c.get('body') or ''):
+ return c['id'], c['body']
+ return None, None
+
+ def upsert_comment(pr, body):
+ cid, existing = find_bot_comment(pr)
+ if cid is not None:
+ if existing.strip() == body.strip():
+ print(f'Comment unchanged for PR #{pr}')
+ return
+ r = subprocess.run(
+ ['gh', 'api', f'repos/{repo}/issues/comments/{cid}',
+ '--method', 'PATCH', '-f', f'body={body}'],
+ capture_output=True, text=True
+ )
+ if r.returncode == 0:
+ print(f'Updated comment on PR #{pr}')
+ else:
+ print(f'Failed to update comment on PR #{pr}: {r.stderr}', file=sys.stderr)
+ else:
+ r = subprocess.run(
+ ['gh', 'pr', 'comment', str(pr), '--body', body],
+ capture_output=True, text=True
+ )
+ if r.returncode == 0:
+ print(f'Commented on PR #{pr}')
+ else:
+ print(f'Failed to comment on PR #{pr}: {r.stderr}', file=sys.stderr)
+
+ for pr, files in current_conflicts.items():
+ body = (
+ make_marker(pr) + '\n'
+ '⚠️ **Merge conflict detected** with \`main\`\n\n'
+ 'The following files conflict:\n'
+ + '\n'.join(f'- \`{f}\`' for f in files)
+ + '\n\nPlease rebase or merge \`main\` to resolve.'
+ )
+ upsert_comment(pr, body)
+
+ for pr in all_prs - set(current_conflicts):
+ cid, existing = find_bot_comment(pr)
+ if cid is not None and '✅ **Conflicts resolved**' not in (existing or ''):
+ body = (
+ make_marker(pr) + '\n'
+ '✅ **Conflicts resolved**\n\n'
+ 'This PR no longer has merge conflicts with \`main\`.'
+ )
+ r = subprocess.run(
+ ['gh', 'api', f'repos/{repo}/issues/comments/{cid}',
+ '--method', 'PATCH', '-f', f'body={body}'],
+ capture_output=True, text=True
+ )
+ if r.returncode == 0:
+ print(f'Updated PR #{pr}: conflicts resolved')
+ else:
+ print(f'Failed to update resolved comment on PR #{pr}: {r.stderr}', file=sys.stderr)
+ "
+
+ - name: Generate summary dashboard
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: python scripts/conflict_detector.py summary --days 30 --top 10 --data-file conflicts.jsonl >> $GITHUB_STEP_SUMMARY
+
+ - name: Commit conflicts.jsonl to conflict-data branch
+ run: |
+ git config user.name "github-actions[bot]"
+ git config user.email "github-actions[bot]@users.noreply.github.com"
+
+ # Check if there are changes to conflicts.jsonl
+ if [ ! -f conflicts.jsonl ]; then
+ echo "No conflicts.jsonl file to commit"
+ exit 0
+ fi
+
+ # Save the updated conflicts.jsonl before switching branches
+ cp conflicts.jsonl /tmp/conflicts.jsonl.updated
+
+ # Checkout or create conflict-data branch
+ git fetch origin conflict-data || true
+ if git show-ref --verify --quiet refs/remotes/origin/conflict-data; then
+ git checkout -b conflict-data origin/conflict-data
+ else
+ git checkout --orphan conflict-data
+ git rm -rf . 2>/dev/null || true
+ fi
+
+ # Restore the updated file
+ cp /tmp/conflicts.jsonl.updated conflicts.jsonl
+
+ # Commit and push if there are changes
+ git add conflicts.jsonl
+ if git diff --cached --quiet; then
+ echo "No changes to commit"
+ exit 0
+ fi
+
+ git commit -m "chore: update conflicts.jsonl [skip ci]"
+ git push origin conflict-data
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index f50ad0b02..a7d42a84f 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -17,6 +17,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v5
with:
diff --git a/.github/workflows/eval-baseline.yml b/.github/workflows/eval-baseline.yml
index 79772f687..46cfb2f02 100644
--- a/.github/workflows/eval-baseline.yml
+++ b/.github/workflows/eval-baseline.yml
@@ -16,6 +16,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
- name: Set up Python 3.12
uses: actions/setup-python@v5
@@ -26,15 +28,21 @@ jobs:
uses: astral-sh/setup-uv@v4
- name: Install dependencies
- run: uv sync --all-groups
+ run: |
+ uv sync --all-groups --extra telemetry
+ uv tool install -e .
- name: Initialize factory config
- run: uv run factory init .
+ run: factory init .
- name: Run eval
id: eval
+ env:
+ LANGFUSE_HOST: ${{ secrets.LANGFUSE_HOST }}
+ LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
+ LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
run: |
- uv run factory eval . > eval_output.json || true
+ factory eval . > eval_output.json || true
cat eval_output.json
python3 -c "import json; json.load(open('eval_output.json'))" || { echo 'ERROR: eval_output.json is missing or invalid JSON'; exit 1; }
diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml
new file mode 100644
index 000000000..baab9f207
--- /dev/null
+++ b/.github/workflows/nightly-release.yml
@@ -0,0 +1,99 @@
+name: Nightly Release
+
+on:
+ schedule:
+ - cron: '0 5 * * *'
+ workflow_dispatch:
+
+permissions:
+ contents: write
+
+jobs:
+ release:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Check for new commits since last nightly
+ id: check
+ run: |
+ LAST_NIGHTLY=$(git tag --list 'nightly-*' --sort=-creatordate | head -n1)
+ if [ -n "$LAST_NIGHTLY" ]; then
+ NEW_COMMITS=$(git rev-list "$LAST_NIGHTLY"..HEAD --count)
+ if [ "$NEW_COMMITS" -eq 0 ]; then
+ echo "No new commits since $LAST_NIGHTLY — skipping release."
+ echo "skip=true" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+ fi
+ echo "skip=false" >> "$GITHUB_OUTPUT"
+
+ - name: Set release variables
+ if: steps.check.outputs.skip != 'true'
+ id: vars
+ run: |
+ TAG="nightly-$(date -u +%Y-%m-%d)"
+ TITLE="Nightly $(date -u +%Y-%m-%d)"
+ echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+ echo "title=$TITLE" >> "$GITHUB_OUTPUT"
+
+ - name: Delete existing release for today if re-running
+ if: steps.check.outputs.skip != 'true'
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ TAG="${{ steps.vars.outputs.tag }}"
+ if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" > /dev/null 2>&1; then
+ echo "Deleting existing release $TAG"
+ gh release delete "$TAG" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag
+ fi
+
+ - name: Find latest stable tag for notes
+ if: steps.check.outputs.skip != 'true'
+ id: stable
+ run: |
+ STABLE_TAG=$(git tag --list 'v*' --sort=-version:refname | head -n1)
+ if [ -n "$STABLE_TAG" ]; then
+ echo "tag=$STABLE_TAG" >> "$GITHUB_OUTPUT"
+ fi
+
+ - name: Create nightly release
+ if: steps.check.outputs.skip != 'true'
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ TAG="${{ steps.vars.outputs.tag }}"
+ TITLE="${{ steps.vars.outputs.title }}"
+ STABLE="${{ steps.stable.outputs.tag }}"
+
+ NOTES_ARGS=""
+ if [ -n "$STABLE" ]; then
+ NOTES_ARGS="--notes-start-tag $STABLE"
+ fi
+
+ gh release create "$TAG" \
+ --repo "$GITHUB_REPOSITORY" \
+ --title "$TITLE" \
+ --prerelease \
+ --generate-notes \
+ $NOTES_ARGS
+
+ - name: Cleanup old nightly releases
+ if: steps.check.outputs.skip != 'true'
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ KEEP=7
+ NIGHTLIES=$(gh release list --repo "$GITHUB_REPOSITORY" --limit 100 --json tagName \
+ -q '[.[] | select(.tagName | startswith("nightly-"))][].tagName')
+
+ COUNT=0
+ for TAG in $NIGHTLIES; do
+ COUNT=$((COUNT + 1))
+ if [ "$COUNT" -gt "$KEEP" ]; then
+ echo "Deleting old nightly release: $TAG"
+ gh release delete "$TAG" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag
+ fi
+ done
diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml
index 31daf9c13..62f273fb5 100644
--- a/.github/workflows/plugins.yml
+++ b/.github/workflows/plugins.yml
@@ -22,11 +22,16 @@ jobs:
run: uv python install 3.12
- name: Install dependencies
- run: uv sync --all-groups
+ run: |
+ uv sync --all-groups
+ uv tool install -e .
- name: Generate plugin agent files
run: uv run python scripts/sync_agents.py
+ - name: Generate workflow skills
+ run: factory workflow export-skills --output-dir skills
+
- name: Copy skills to .agents/skills
run: cp -r skills/ .agents/skills/
diff --git a/.github/workflows/runtime-image.yml b/.github/workflows/runtime-image.yml
new file mode 100644
index 000000000..83ea54682
--- /dev/null
+++ b/.github/workflows/runtime-image.yml
@@ -0,0 +1,207 @@
+name: Contained runtime image
+
+# The image `factory contained` runs, for both the local and cluster targets.
+#
+# Built and published here rather than on demand: on-demand building is slow for every cold start,
+# and it is circular for the cluster target, whose whole point is that the laptop is not the build
+# host. `factory contained setup` pulls; it does not build.
+#
+# **Multi-arch is not optional.** One image serves an arm64 laptop and amd64 cluster nodes, so this
+# publishes a manifest list rather than a single tag. Build and validate on the *same*
+# architecture — a probe that builds on arm64 and validates on amd64 is not evidence about either.
+
+# Three ways in, and they publish different tags:
+#
+# push to main → :latest and : — the branch everyone pulls from by default
+# release → : and :, plus :latest for a full (non-pre) release
+# dispatch → whatever tag you name
+#
+# A release is the one that has to be automatic: `factory contained setup` pulls a published image
+# and does not build, so a release whose image was never built leaves every new user's first
+# command failing on a manifest that does not exist. A prerelease (the nightlies) deliberately
+# does *not* move `:latest` — nightly is a thing you opt into by tag, not something a laptop picks
+# up by pulling the default.
+on:
+ push:
+ branches: [main]
+ paths:
+ - containers/factory/Containerfile
+ - factory/**
+ - skills/**
+ - pyproject.toml
+ - uv.lock
+ - .github/workflows/runtime-image.yml
+ release:
+ types: [published]
+ workflow_dispatch:
+ inputs:
+ tag:
+ description: 'Tag to publish (default: latest)'
+ required: false
+ default: 'latest'
+ type: string
+
+env:
+ IMAGE: ghcr.io/${{ github.repository }}/factory-runtime
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ packages: write
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - arch: amd64
+ platform: linux/amd64
+ - arch: arm64
+ platform: linux/arm64
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ # arm64 is emulated here. It is slow, and it is the only way to produce the manifest the
+ # laptop half of the design pulls without maintaining a second runner.
+ - name: Set up QEMU
+ if: matrix.arch == 'arm64'
+ uses: docker/setup-qemu-action@v3
+
+ - name: Set up Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Log in to ghcr.io
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Build and push by digest
+ id: build
+ uses: docker/build-push-action@v6
+ with:
+ context: .
+ file: containers/factory/Containerfile
+ platforms: ${{ matrix.platform }}
+ # Pushed by digest and assembled into a manifest list below, so a half-finished matrix
+ # never leaves a tag pointing at one architecture.
+ outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
+ cache-from: type=gha,scope=${{ matrix.arch }}
+ cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
+
+ - name: Export the digest
+ run: |
+ mkdir -p /tmp/digests
+ touch "/tmp/digests/${{ steps.build.outputs.digest }}"
+
+ - uses: actions/upload-artifact@v4
+ with:
+ name: digest-${{ matrix.arch }}
+ path: /tmp/digests/*
+ retention-days: 1
+
+ manifest:
+ needs: build
+ runs-on: ubuntu-latest
+ permissions:
+ packages: write
+ steps:
+ - uses: actions/download-artifact@v4
+ with:
+ path: /tmp/digests
+ pattern: digest-*
+ merge-multiple: true
+
+ - uses: docker/setup-buildx-action@v3
+
+ - uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ # Both the dispatch input and the release tag reach the shell through `env:` rather than by
+ # interpolation, here and in every step below: each is user-supplied text, and pasting it
+ # into a `run:` block makes it shell source. They are then *validated* as well as quoted,
+ # because a string that is safe to pass to a shell is still not necessarily a legal image
+ # tag, and `imagetools create` failing on a malformed reference is a confusing way to find
+ # out someone named a release `v1.0 (final)`.
+ - name: Resolve the tags to publish
+ id: tags
+ env:
+ EVENT: ${{ github.event_name }}
+ RELEASE_TAG: ${{ github.event.release.tag_name }}
+ PRERELEASE: ${{ github.event.release.prerelease }}
+ INPUT_TAG: ${{ inputs.tag }}
+ run: |
+ set -eu
+ usable() {
+ case "$1" in
+ ""|-*|.*|*[!A-Za-z0-9._-]*) return 1 ;;
+ *) return 0 ;;
+ esac
+ }
+ case "$EVENT" in
+ release)
+ usable "$RELEASE_TAG" \
+ || { echo "::error::release tag '$RELEASE_TAG' is not usable as an image tag"; exit 1; }
+ tags="$RELEASE_TAG"
+ # A prerelease — every nightly is one — publishes under its own tag only. Moving
+ # `:latest` there would push a nightly onto every laptop that pulls the default.
+ if [ "$PRERELEASE" != "true" ]; then
+ tags="$tags latest"
+ fi
+ ;;
+ workflow_dispatch)
+ tag="${INPUT_TAG:-latest}"
+ usable "$tag" \
+ || { echo "::error::tag '$tag' is not usable as an image tag"; exit 1; }
+ tags="$tag"
+ ;;
+ *)
+ tags="latest"
+ ;;
+ esac
+ echo "Publishing tags: $tags"
+ echo "list=$tags" >> "$GITHUB_OUTPUT"
+ echo "primary=${tags%% *}" >> "$GITHUB_OUTPUT"
+
+ - name: Assemble the manifest list
+ env:
+ TAGS: ${{ steps.tags.outputs.list }}
+ run: |
+ set -eu
+ args=""
+ for tag in $TAGS; do
+ args="$args --tag ${IMAGE}:${tag}"
+ done
+ docker buildx imagetools create $args \
+ --tag "${IMAGE}:${GITHUB_SHA::12}" \
+ $(printf "${IMAGE}@sha256:%s " $(ls /tmp/digests | sed 's/^sha256://'))
+
+ - name: Verify the published manifest carries both architectures
+ env:
+ TAG: ${{ steps.tags.outputs.primary }}
+ run: |
+ tag="$TAG"
+ docker buildx imagetools inspect "${IMAGE}:${tag}"
+ for arch in amd64 arm64; do
+ docker buildx imagetools inspect "${IMAGE}:${tag}" --raw \
+ | grep -q "\"architecture\":\"${arch}\"" \
+ || { echo "::error::${arch} is missing from the published manifest"; exit 1; }
+ done
+
+ # The factory has to actually start in the image. A published image that pulls but whose
+ # entry point is broken fails at the far end of a workspace upload and a pod start, where it
+ # reads as a cluster problem.
+ - name: Smoke-test the published image
+ env:
+ TAG: ${{ steps.tags.outputs.primary }}
+ run: |
+ tag="$TAG"
+ docker run --rm "${IMAGE}:${tag}" factory --help > /dev/null
+ docker run --rm "${IMAGE}:${tag}" tmux -V
+ docker run --rm "${IMAGE}:${tag}" git --version
diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/stale-issues.yml
new file mode 100644
index 000000000..f65cd830e
--- /dev/null
+++ b/.github/workflows/stale-issues.yml
@@ -0,0 +1,148 @@
+name: Close stale issues without linked PRs
+
+on:
+ schedule:
+ - cron: '17 3 * * *'
+ workflow_dispatch:
+
+permissions:
+ issues: write
+ pull-requests: read
+
+jobs:
+ close-stale-issues:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/github-script@v7
+ with:
+ script: |
+ const STALE_DAYS = 4;
+ const EXEMPT_LABELS = new Set([
+ 'pinned',
+ 'security',
+ 'help-wanted',
+ 'good-first-issue',
+ 'bug',
+ ]);
+ const MS_PER_DAY = 24 * 60 * 60 * 1000;
+ const now = Date.now();
+ const cutoff = new Date(now - STALE_DAYS * MS_PER_DAY);
+
+ const [owner, repo] = process.env.GITHUB_REPOSITORY.split('/');
+
+ let checked = 0;
+ let skippedPR = 0;
+ let skippedExempt = 0;
+ let skippedLinked = 0;
+ let closed = 0;
+ let page = 1;
+
+ while (true) {
+ const { data: issues } = await github.rest.issues.listForRepo({
+ owner,
+ repo,
+ state: 'open',
+ sort: 'created',
+ direction: 'asc',
+ per_page: 100,
+ page,
+ });
+
+ if (issues.length === 0) break;
+
+ for (const issue of issues) {
+ checked++;
+
+ if (issue.pull_request) {
+ skippedPR++;
+ continue;
+ }
+
+ if (issue.labels.some(l => EXEMPT_LABELS.has(l.name))) {
+ core.info(`#${issue.number}: skipped (exempt label)`);
+ skippedExempt++;
+ continue;
+ }
+
+ const createdAt = new Date(issue.created_at);
+ if (createdAt > cutoff) {
+ core.info(`#${issue.number}: skipped (younger than ${STALE_DAYS} days)`);
+ continue;
+ }
+
+ const { repository } = await github.graphql(`
+ query($owner: String!, $repo: String!, $number: Int!) {
+ repository(owner: $owner, name: $repo) {
+ issue(number: $number) {
+ timelineItems(first: 100, itemTypes: [CROSS_REFERENCED_EVENT]) {
+ nodes {
+ ... on CrossReferencedEvent {
+ source {
+ ... on PullRequest {
+ number
+ state
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ `, { owner, repo, number: issue.number });
+
+ const nodes = repository.issue.timelineItems.nodes;
+ const hasLinkedPR = nodes.some(node => {
+ const pr = node.source;
+ return pr && pr.state && (pr.state === 'OPEN' || pr.state === 'MERGED');
+ });
+
+ if (hasLinkedPR) {
+ core.info(`#${issue.number}: skipped (has linked open/merged PR)`);
+ skippedLinked++;
+ continue;
+ }
+
+ await github.rest.issues.addLabels({
+ owner,
+ repo,
+ issue_number: issue.number,
+ labels: ['stale'],
+ });
+
+ await github.rest.issues.createComment({
+ owner,
+ repo,
+ issue_number: issue.number,
+ body: [
+ 'This issue has been automatically closed because it has been open',
+ `for more than ${STALE_DAYS} days with no linked pull request.`,
+ '',
+ 'If this issue is still relevant, please reopen it and link a PR',
+ 'or add one of the exempt labels: `pinned`, `security`,',
+ '`help-wanted`, `good-first-issue`, `bug`.',
+ ].join('\n'),
+ });
+
+ await github.rest.issues.update({
+ owner,
+ repo,
+ issue_number: issue.number,
+ state: 'closed',
+ state_reason: 'not_planned',
+ });
+
+ core.info(`#${issue.number}: closed (no linked open/merged PR)`);
+ closed++;
+ }
+
+ if (issues.length < 100) break;
+ page++;
+ }
+
+ core.info('--- Summary ---');
+ core.info(`Checked: ${checked}`);
+ core.info(`Skipped (pull requests): ${skippedPR}`);
+ core.info(`Skipped (exempt label): ${skippedExempt}`);
+ core.info(`Skipped (has linked PR): ${skippedLinked}`);
+ core.info(`Closed: ${closed}`);
diff --git a/.gitignore b/.gitignore
index 3b11e5c09..d51101e32 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,8 @@ __pycache__/
.venv/
*.egg-info/
dist/
+!pfexec/dist/
+factory/_version.py
.pytest_cache/
.ruff_cache/
.mypy_cache/
@@ -27,7 +29,13 @@ dist/
/agents/
/codex-agents/
+# Generated workflow skills (built on-the-fly by factory skill_cache)
+skills/workflow-*/SKILL.md
+skills/workflow-*/SKILL.annotations.yaml
+
# Dev artifacts
.playwright-mcp/
screenshots/
docs/factory-slides/
+graphify-out/
+graph.json
diff --git a/.sentrux/rules.toml b/.sentrux/rules.toml
index 31b0a1cbe..341376f85 100644
--- a/.sentrux/rules.toml
+++ b/.sentrux/rules.toml
@@ -3,3 +3,4 @@ max_cycles = 5
max_coupling = "C"
max_cc = 30
no_god_files = false
+min_equality = 0.3
diff --git a/AGENTS.md b/AGENTS.md
index aa080b96e..65d21954b 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -42,12 +42,14 @@ mypy factory/ # Type check
The factory is a three-layer system:
-1. **Python CLI** (`factory/`): Pure tools that don't make decisions. Entry point is `factory/cli.py`.
+1. **Python CLI** (`factory/cli/`): Pure tools that don't make decisions. Entry point is `factory/cli/_main.py`.
2. **CEO Agent** (`factory/agents/prompts/ceo.md`): Orchestrates the full workflow. Spawned via `factory ceo /path`.
3. **Specialist Agents** (`factory/agents/`): Eight subprocesses spawned by the CEO via `factory agent `.
Agent roles: Researcher, Strategist, Builder, Reviewer, Evaluator, Archivist, Distiller, Failure Analyst, CEO.
+Key modules: `factory/adversarial.py` (GAN-style adversarial eval loops — phase transitions with hysteresis, convergence detection, state at `.factory/adversarial_state.json`).
+
## MCP Server
The factory exposes tools via MCP (Model Context Protocol):
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 66cefde69..4cd88a1ee 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@
### Features
+- **Adversarial eval loops** — First-class GAN-style alternating generator/discriminator optimization. Configure in `factory.md` with `## Adversarial` section using dot-notation for per-component eval commands, metrics, and thresholds. Hysteresis prevents oscillation (N consecutive above-threshold rounds before switching). Convergence detection when both sides sustain above-threshold performance. State persisted at `.factory/adversarial_state.json` for crash-resilient resume. New `factory adversarial-state` CLI command for inspection and reset. 80 new tests
- **Post-cycle refinement loop** — After build/improve cycles complete in foreground mode, the CEO stays active and routes follow-up requests through the Refiner → Builder → full review pipeline. New `--refine` flag for direct refinement entry. Three CLI commands (`refine-status`, `refine-begin`, `refine-complete`) provide identity regrounding and state tracking. No hard cap on refinements; advisory warnings at 5 and 10
- **Refiner agent** — New specialist that classifies refinement requests into tiers (T1: prompt/config, T2: code changes, T3: architectural — requires `--focus`) and scopes the implementation for the Builder
- **Inner/outer loop controls** — Configure multi-run aggregation, plateau detection, and automatic scope expansion for research mode via `## Inner Loop` and `## Outer Loop Surfaces` in `factory.md`
diff --git a/CLAUDE.md b/CLAUDE.md
index edcbb68f4..1a9ae04ec 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -37,6 +37,18 @@ mypy factory/ # Type check
- Async/await by default — library functions in `store.py` and `eval/runner.py` are async, the CLI wraps them with `asyncio.run()`
- Structured logging via `structlog` — use `log = structlog.get_logger()` at module level
+## Versioning
+
+Version is derived from git tags via `hatch-vcs` at build time — no static `version =` in pyproject.toml.
+
+- Tag pattern: `v*` (e.g., `v0.3.1`); `nightly-*` tags are ignored via `--match 'v*'`
+- Dev installs show `X.Y.Z.devN+gSHA` between releases
+- `factory/_version.py` is generated by the hatch-vcs build hook and gitignored
+- After pulling new tags, re-run `uv sync` for editable installs to pick up the new version
+- `fallback_version = "0.0.0"` is used in environments without git history (Docker builds, tarballs)
+- Runtime version: `importlib.metadata.version("remote-factory")`
+- CLI: `factory --version`
+
## Architecture (v2 — CEO Agent + Workflow Graph Engine)
The factory is a **four-layer system**:
@@ -47,12 +59,35 @@ Pure tools that don't make decisions. Entry point is `factory/cli.py` → `facto
### Layer 2: Workflow Graph Engine (`factory/workflow/`)
-All 8 factory modes (build, design, improve, research, meta, discover, review, refine) are defined as directed graphs of typed nodes in `factory/workflow/definitions.py`. Each graph is a `Workflow` Pydantic model with `AgentNode`, `FnNode`, `GateNode`, `ForkNode`, `JoinNode`, and `Study` primitives connected by `Edge` objects. See `factory/workflow/README.md` for full documentation.
+All 10 factory modes (build, design, improve, research, meta, discover, review, refine, founder, plan) are defined as directed graphs of typed nodes in `factory/workflow/definitions.py`. Each graph is a `Workflow` Pydantic model with `AgentNode`, `FnNode`, `GateNode`, `ForkNode`, `JoinNode`, and `Study` primitives connected by `Edge` objects. See `factory/workflow/README.md` for full documentation.
The same graph definition produces two execution formats:
- **Headless:** `WorkflowExecutor` (`factory/workflow/executor.py`) walks the DAG deterministically — `factory workflow run --project /path`
- **Interactive:** `skill_export.py` converts graphs to Claude Code `SKILL.md` files under `skills/workflow-*/` — the CEO agent reads these at runtime as mode-specific playbooks
+### Layer 2b: Outer Loop — Evolutionary Workflow Search (`factory/outer_loop/`)
+
+The outer loop evolves workflow *topologies* via MAP-Elites quality-diversity search. Given a base workflow (e.g. the single-builder FeatureBench seed), it produces a population of structurally diverse candidates, evaluates each via an inner loop (one full CEO cycle per candidate), and uses contrastive reflection to guide mutations toward higher fitness.
+
+**Pipeline:** `calibrate → evolve → reflect → evaluate` (repeats until budget exhaustion, plateau, or target score).
+
+**Key modules:**
+- `engine.py` — `SwarmEngine` orchestrates the evolutionary loop: seeding, tournament selection, mutation, evaluation, convergence detection (plateau, diversity collapse, early stop)
+- `evaluator.py` — `SwarmEvaluator` with `FitnessCache` (structural-hash dedup) and `CycleRecordCache` (content-hash dedup). Supports both `EvaluatorFn` protocol and `FeatureBenchInnerLoop` evaluation with git worktree isolation
+- `mutations.py` — 7 structured graph mutation operators (`NODE_INSERT`, `NODE_REMOVE`, `EDGE_REDIRECT`, `PARALLELIZE`, `SERIALIZE`, `PARAM_MUTATE`, `PROMPT_MUTATE`) with `WeightedRandomStrategy` and reflection-guided selection
+- `population.py` — `Population` (collection management) and `MAPElitesArchive` (4D grid: depth × fork_degree × agent_count × gate_count)
+- `similarity.py` — `structural_hash`, `graph_edit_distance`, `compute_features`, `NoveltyFilter`
+- `reflector.py` — `OuterLoopReflector` performs two-stage contrastive reflection (top-K vs bottom-K) to identify failure/success patterns and generate mutation suggestions
+- `mode_registry.py` — `EphemeralModeRegistry` registers candidate workflows as temporary modes (`evolve-gen{N}-{id[:8]}`) with content-hash integrity checking, target-dir mirroring, and promotion to permanent modes
+- `designer.py` — `DesignerAgent` generates from-scratch workflow designs (minimal, thorough, custom variants)
+- `models.py` — Pydantic models: `SwarmConfig`, `Individual`, `EvalResult`, `GenerationSummary`, `OuterLoopResult`, `HyperparameterRecord`, `MutationRecord`, `OuterLoopState`, `AuditResult`
+- `overfit.py` — `OverfitDetector` compares training vs holdout scores to flag overfitting
+- `subset.py` — `SubsetSelector` protocol and `FixedSubsetSelector` for training instance selection
+- `filesystem.py` — Outer loop directory initialization, config/checkpoint persistence
+- `featurebench_inner_loop.py` — Bridges outer loop evaluation to a full CEO cycle on a FeatureBench instance
+
+**E2E finding:** On simple FeatureBench tasks, a single-builder topology (1 AgentNode, no fork/join) wins on parsimony + cost. The outer loop's value emerges on harder multi-agent problems where topology diversity matters.
+
### Layer 3: CEO Agent (`factory/agents/prompts/ceo.md` + `skills/workflow-*/SKILL.md`)
The CEO prompt is split into two parts:
@@ -78,6 +113,8 @@ Eight specialist Claude Code subprocesses spawned by the CEO via `factory agent
7. **Report** (`factory/report.py`): Performance report generation — consolidates experiment records, CEO verdicts, and observations into `.factory/performance_report.json` for ACE consumption
8. **Checkpoint** (`factory/checkpoint.py`): Saves and loads CEO state for crash-resilient resume
9. **Analysis** (`factory/analysis.py`): Experiment comparison (`diff`) and FEEC analysis (`explain`)
+10. **Adversarial** (`factory/adversarial.py`): GAN-style adversarial eval loop state machine — phase transitions with hysteresis, per-role streak counters, convergence detection. State persisted at `.factory/adversarial_state.json`
+11. **Contained** (`factory/contained/` + `factory/podman.py` + `factory/cli/contained.py`): `factory contained [runtime flags] -- ` runs the factory in a podman container (`--target local`) or a cluster pod (`--target k8s`). See "Contained runtimes" below.
### Target project's `.factory/` layout
@@ -93,6 +130,19 @@ Eight specialist Claude Code subprocesses spawned by the CEO via `factory agent
├── reviews/ # Agent output capture + CEO review verdicts
│ ├── -latest.md # Auto-saved stdout from each agent invocation
│ └── ceo-verdict-.md # CEO's review verdict (PROCEED/REDIRECT/ABORT)
+├── adversarial_state.json # Adversarial loop state (phase, streaks, history)
+├── outer_loop/ # Evolutionary workflow search state
+│ ├── config.json # SwarmConfig for the current run
+│ ├── state.json # OuterLoopState for crash recovery
+│ ├── population/ # Serialized Population (population.json)
+│ ├── archive/ # Serialized MAPElitesArchive (grid.json)
+│ ├── modes/ # Ephemeral mode JSONs (evolve-gen{N}-{id}.json)
+│ ├── results/ # Per-generation eval results (gen{N}.json)
+│ ├── reflections/ # Contrastive reflection reports (gen{N}.json, gen{N}.md)
+│ ├── events.jsonl # Per-generation best/mean/diversity metrics
+│ ├── costs.jsonl # Per-individual cost tracking
+│ └── trajectory.jsonl # Score trajectory over generations
+├── workflows/ # Ephemeral .py wrappers for WorkflowRegistry discovery
├── archive/ # Long-term knowledge store (Archivist notes)
│ ├── experiments/ # Per-experiment learnings and decision rationale
│ ├── patterns/ # Recurring patterns and anti-patterns
@@ -102,7 +152,9 @@ Eight specialist Claude Code subprocesses spawned by the CEO via `factory agent
### Models
-All domain models live in `factory/models.py` as strict Pydantic v2 models. Key types: `ProjectState` (enum), `FactoryConfig`, `EvalProfile` / `EvalDimension`, `CompositeScore` / `EvalResult`, `ExperimentRecord`, `CrossProjectInsights`, `AgentVerdict`, `Observation`, `PerformanceReport`, `ProjectEntry` / `ProjectRegistry`. The `Notifier` protocol defines the async notification interface. `FactoryConfig` includes `clean_pr` (bool), `clean_pr_include` (list[str]), and `clean_pr_exclude` (list[str]) for Clean PR Mode — stripping non-essential artifacts from PRs before pushing to external repos.
+All domain models live in `factory/models.py` as strict Pydantic v2 models. Key types: `ProjectState` (enum), `FactoryConfig`, `EvalProfile` / `EvalDimension`, `CompositeScore` / `EvalResult`, `ExperimentRecord`, `CrossProjectInsights`, `AgentVerdict`, `Observation`, `PerformanceReport`, `ProjectEntry` / `ProjectRegistry`, `AdversarialConfig` / `AdversarialComponent` / `AdversarialState` / `AdversarialPhaseRecord`. The `Notifier` protocol defines the async notification interface. `FactoryConfig` includes `clean_pr` (bool), `clean_pr_include` (list[str]), and `clean_pr_exclude` (list[str]) for Clean PR Mode — stripping non-essential artifacts from PRs before pushing to external repos. `FactoryConfig.adversarial` (`AdversarialConfig | None`) holds the GAN-style adversarial eval loop configuration parsed from `factory.md`.
+
+Outer loop models live in `factory/outer_loop/models.py`: `SwarmConfig` (evolutionary search configuration — benchmark, budget, population_size, mutation_rate, frozen_node_ids, training/holdout instances, convergence thresholds), `Individual` (candidate with workflow_data, score, features, lineage), `EvalResult` (benchmark_score + hygiene_score + cost + complexity), `GenerationSummary` (per-generation stats), `OuterLoopResult` (final run result with trajectory, pareto front, hyperparameter history), `HyperparameterRecord` (per-generation mutation_rate, operator_weights, diversity), `MutationRecord` (operator + target_node + before/after), `MutationType` (enum: 7 mutation operators), `OuterLoopState` (checkpoint for crash recovery), `AuditResult` (overfit detection).
## Environment
@@ -128,7 +180,30 @@ ANTHROPIC_API_KEY = "sk-ant-..."
- `factory config edit` — open `~/.factory/config.toml` in `$EDITOR`
- `factory config migrate` — create starter config from current env vars (requires `tomli_w`)
-**Credential profiles:** Use `--profile ` with `factory ceo`, `factory run`, or `factory agent` to load a `[credentials.]` section. Profile keys are injected into `os.environ`.
+**Credential profiles:** Use `--profile ` with `factory ceo`, `factory run`, or `factory agent` to load a `[credentials.]` section. Profile keys **override** existing env vars (explicit `--profile` opt-in means the profile is authoritative). CLI flags still win via 5-tier precedence.
+
+**Env overlay features:**
+- **Override:** Profile keys are set via `os.environ[k] = v`, not `setdefault` — the profile wins over shell env vars
+- **Unset:** Add a `[credentials..unset]` sub-table with `vars = ["VAR1", "VAR2"]` to remove env vars before injection. Unsets are processed before sets.
+- **Protected vars:** The following env vars cannot be set or unset via profiles — a `ValueError` is raised if attempted: `PATH`, `HOME`, `USER`, `SHELL`, `TMPDIR`, `TERM`, `PWD` (shell fundamentals); `LD_PRELOAD`, `LD_LIBRARY_PATH`, `DYLD_INSERT_LIBRARIES` (code execution vectors); `PYTHONPATH`, `GOPATH`, `CLASSPATH`, `NODE_PATH` (language path injection); `IFS` (shell parsing); `FACTORY_TRACE_ID`, `FACTORY_PARENT_SPAN_ID` (factory observability internals).
+- **Unset vars validation:** The `[credentials..unset].vars` field must be a list — a `ValueError` is raised if it is a string or other non-list type.
+- **Override warnings:** When a profile overrides an existing env var with a different value, a `log.warning("profile_override", key=k, profile=profile)` is emitted (values are not logged to avoid leaking secrets).
+
+**Custom endpoint example** (e.g. a LiteLLM proxy):
+
+You can use profiles to point the factory at a custom model endpoint:
+```toml
+[credentials.litellm-proxy]
+FACTORY_RUNNER = "claude"
+FACTORY_MODEL = "your-model-name"
+ANTHROPIC_BASE_URL = "https://your-litellm-proxy.example.com"
+ANTHROPIC_API_KEY = "your-api-key-here"
+CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC = "1"
+
+[credentials.litellm-proxy.unset]
+vars = ["CLAUDE_CODE_USE_VERTEX", "CLAUDE_CODE_USE_BEDROCK", "ANTHROPIC_VERTEX_PROJECT_ID"]
+```
+Usage: `factory ceo /path --profile litellm-proxy`
**Implementation:** `factory/user_config.py` — `load_config()`, `resolve()`, `show_config()`, `migrate_env_to_config()`.
@@ -170,10 +245,22 @@ CODEX_API_KEY = "..."
Then run: `factory ceo /path/to/project --profile codex`
**OpenCode specifics:**
-- Requires `OPENAI_API_KEY` environment variable
-- The factory targets `opencode-ai/opencode` v0.x (uses `-p`, `-q`, `-c` flags). Install from source: `go install github.com/opencode-ai/opencode@latest`, or via the [GitHub release tarball](https://github.com/opencode-ai/opencode/releases)
-- Do NOT use the `curl` installer at `opencode.ai/install` — it installs the `anomalyco/opencode` fork (v1.x) which has an incompatible CLI interface
+- The factory targets `anomalyco/opencode` v1.x (TypeScript/Bun). Install via: `curl -fsSL https://opencode.ai/install | bash` or `npm i -g opencode-ai`
+- Auth: run `opencode auth login` (interactive), or set a provider env var (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `AWS_ACCESS_KEY_ID`, etc.)
+- Headless mode uses `opencode run '' --format json --dir --auto`
+- Model selection via `--model` flag (e.g., `anthropic/claude-sonnet-4-20250514`)
+- Session management: `--title ` (name a session), `--session ` (resume by ID), `--continue` (continue last session)
- Dry-run mode: `FACTORY_OPENCODE_DRY_RUN=1`
+- Token guardrails: `FACTORY_OPENCODE_MAX_INVOCATIONS_PER_CYCLE` (default: 8), logged to `.factory/opencode_usage.jsonl`
+- Unsupported: `--bg` (no background mode), `--tmux-persist` (returns explicit error), CEO message events (no JSON streaming equivalent)
+
+**OpenCode config profile example** (`~/.factory/config.toml`):
+```toml
+[credentials.opencode]
+FACTORY_RUNNER = "opencode"
+ANTHROPIC_API_KEY = "sk-ant-..."
+```
+Then run: `factory ceo /path/to/project --profile opencode`
**Important:** Target projects should add `.factory/` to their `.gitignore`. The factory writes experiment data, usage logs, and potentially sensitive auth files (`.factory/.bob_auth`) to this directory. These are project-local artifacts that should not be committed to version control.
@@ -186,9 +273,19 @@ factory ceo "Build a weather CLI" --dir my-app # Explicit dir name override
factory ceo ~/ideas/spec.md # Spec file → new project
factory ceo https://github.com/user/repo # Clone and improve
factory ceo "distributed eval runner" --mode design # Brainstorm → build
+factory ceo ~/ideas/detailed-spec.md --mode design # Long idea from file (no length limit)
factory ceo /path/to/project --mode design # Discuss what to work on → improve
factory ceo /path/to/project --mode design --focus "auth" # Discuss a specific topic
+factory ceo "weather CLI" --mode design --auto-approve # Design without user approval gate
+factory ceo /path/to/project --mode design --from-plan .factory/strategy/current.md # Build from local plan
+factory ceo /path/to/project --mode design --from-plan 42 # Build from plan issue #42
+factory ceo /path/to/project --mode design --from-plan 'auth dashboard' # Fuzzy search plans
factory ceo "SWE-bench solver" --mode research # Research ideation → build
+factory ceo /path/to/factory --mode create --focus "mode description" # Create a new factory mode
+factory ceo /path/to/factory --mode create --focus "improve: add plateau detection" # Update existing mode
+factory ceo /path/to/project --mode design --just-plan # Research + strategy, no implementation
+factory ceo "distributed eval runner" --mode design --just-plan # Plan a new idea
+factory ceo /path/to/project --mode design --just-plan --focus "auth" # Focused planning
# Improve — point at existing codebase
factory ceo /path/to/project # Single improvement cycle
@@ -199,6 +296,17 @@ factory tmux /path/to/project --loop # In detached tmux session
factory ceo /path/to/project --focus "dashboard UI" # One item, one hypothesis, done
factory ceo /path/to/project --focus 42 # Target GitHub issue #42
factory ceo /path/to/project --focus "owner/repo#42" # Target issue by shorthand
+factory ceo /path/to/project --focus '42 and 43' # Multiple issues
+factory ceo /path/to/project --focus 'issue 42, issue 43' # With 'issue' keyword
+
+# Founder — rapid prototyping (NOT for production)
+factory ceo /path/to/project --mode founder # One fast hypothesis
+factory ceo /path/to/project --mode founder --focus "auth flow" # Targeted prototype
+factory run /path/to/project --mode founder --loop --interval 300 # Rapid iteration
+
+# Study — graph-powered codebase analysis
+factory ceo /path/to/project --mode study # Graph-powered codebase study
+factory ceo /path/to/project --mode study --focus "auth flow" # Focused study with graph context
# Meta — improve the factory's own agents
factory ceo /path/to/project --mode meta # Improve + ACE playbook evolution
@@ -214,16 +322,79 @@ factory backlog-list /path # List pending backlog items
factory backlog-add /path "item text" # Add a new item to the backlog
factory backlog-remove /path "item text" # Remove a completed backlog item
+# Adversarial eval loops
+factory adversarial-state /path/to/project # Inspect adversarial loop state
+factory adversarial-state /path/to/project --reset # Reset to defaults
+
+# Outer loop — evolutionary workflow search
+factory outer-loop calibrate /path --benchmark featurebench --budget 50 --population-size 4
+factory outer-loop calibrate /path --training-instances t1 t2 --holdout-instances h1
+factory outer-loop calibrate /path --project-dir /path/to/target # Evaluate on a different project
+factory outer-loop evaluate /path --generation 0 # Evaluate current generation
+factory outer-loop evaluate /path --generation 0 --project-dir /path/to/target
+factory outer-loop reflect /path --generation 0 # Contrastive reflection
+factory outer-loop evolve /path --generation 0 # Produce next generation
+factory outer-loop status /path # Show progress and metrics
+factory outer-loop status /path --check-converge # Exit 0 if converged, 1 if not
+factory outer-loop promote /path --mode-name evolve-gen5-abc12345 --permanent-name best-evolved
+
# Operations
factory dashboard --projects-dir ~/factory-projects # Live web dashboard on :8420
factory export /path/to/project # Dump full project snapshot as JSON
factory checkpoint /path/to/project # Save CEO state for crash recovery
-factory resume /path/to/project # Resume from saved checkpoint
+factory resume /path/to/project # Resume an interrupted CEO session
factory precheck /path --score-before 0.7 --score-after 0.85 # Hard precheck gate
factory review --verdict KEEP --pr 42 # Post structured review on GitHub PR
```
-`factory run` / `factory ceo` spawn the CEO agent as a subprocess using the selected runner (`claude` by default, or `bob` with `--runner bob`). The CEO owns the full workflow: state detection, agent spawning, experiment lifecycle, and mandatory archival. The `--loop` flag adds a heartbeat wrapper with configurable interval and max cycles. `--mode meta` runs the full Improve loop on the factory itself, then ACE playbook evolution for all agent roles. `--focus` activates targeted mode: builds exactly one item and exits. Accepts backlog names (`--focus "eval reliability"`), issue numbers (`--focus 42`), issue URLs, or `owner/repo#N` shorthand. Issue refs are auto-detected and fetched via `gh`/`glab` CLI. Works in improve and research modes; mutually exclusive with `--loop`. `--mode design` enters ideation mode. For new ideas (e.g. `factory ceo "distributed eval runner" --mode design`), the CEO researches the space via the Researcher, then iteratively refines the idea with the Strategist through user feedback, producing a phased build plan before building. For existing projects (e.g. `factory ceo /path/to/project --mode design`), the CEO studies the project (backlog, eval scores, open issues, history), presents findings, and discusses what to work on before transitioning to Improve mode. `--mode interactive` is accepted as a backward-compatible alias for `--mode design`. `--focus` is allowed on existing projects to seed the discussion topic. Incompatible with `--headless`. `--mode research` enters research ideation for new projects (e.g. `factory ceo "SWE-bench solver" --mode research`) — the Strategist collects research config (target metric, mutable/fixed surfaces, constraints) before building. For existing projects with `research_target` configured, runs the research improvement loop directly. Incompatible with `--headless` (for new projects) and `--prompt`. `--refine ""` enters refinement mode — routes a single change request through the Refiner → Builder → full review pipeline. Mutually exclusive with `--mode`, `--prompt`, and `--focus`. Requires an existing project directory. In foreground mode, the CEO also enters the refinement loop automatically after completing a build/improve cycle, staying active for follow-up requests without `--refine`.
+`factory run` / `factory ceo` spawn the CEO agent as a subprocess using the selected runner (`claude` by default, or `bob` with `--runner bob`). The CEO owns the full workflow: state detection, agent spawning, experiment lifecycle, and mandatory archival. The `--loop` flag adds a heartbeat wrapper with configurable interval and max cycles. `--mode meta` runs the full Improve loop on the factory itself, then ACE playbook evolution for all agent roles. `--focus` activates targeted mode: builds exactly one item and exits. Accepts backlog names (`--focus "eval reliability"`), issue numbers (`--focus 42`), issue URLs, or `owner/repo#N` shorthand. Multiple issues can be specified in a single `--focus` string using commas, spaces, or "and" (e.g., `--focus "111 and 112"`, `--focus "issue 42, issue 43"`, `--focus "#111 #112"`). Each issue is fetched independently and added as a separate backlog item. Issue refs are auto-detected and fetched via `gh`/`glab` CLI. Works in improve, research, and create modes; mutually exclusive with `--loop`. In create mode, `--focus` provides the mode description; use `--focus "mode_name: change description"` to update an existing registered mode instead of creating a new one. `--mode design` enters ideation mode. For new ideas (e.g. `factory ceo "distributed eval runner" --mode design`), the CEO researches the space via the Researcher, then iteratively refines the idea with the Strategist through user feedback, producing a phased build plan before building. For existing projects (e.g. `factory ceo /path/to/project --mode design`), the CEO studies the project (backlog, eval scores, open issues, history), presents findings, and discusses what to work on, then continues to implementation automatically after approval. `--mode interactive` is accepted as a backward-compatible alias for `--mode design`. `--focus` is allowed on existing projects to seed the discussion topic. Incompatible with `--headless` unless `--auto-approve` is used. `--auto-approve` lifts the headless restriction for design mode, forcing headless execution and auto-approving user gates (e.g. strategy review) — useful for CI/CD and automated pipelines. `--from-plan ` loads an existing plan into design mode, skipping the research phase. Accepts a local file path, GitHub issue URL, issue number, or fuzzy search string (searches GitHub issues with the `plan` label). Requires `--mode design`; mutually exclusive with `--focus` and `--prompt`. When fetching from a GitHub issue, includes both the issue body and all comments. `--mode research` enters research ideation for new projects (e.g. `factory ceo "SWE-bench solver" --mode research`) — the Strategist collects research config (target metric, mutable/fixed surfaces, constraints) before building. For existing projects with `research_target` configured, runs the research improvement loop directly. Incompatible with `--headless` (for new projects) and `--prompt`. `--refine ""` enters refinement mode — routes a single change request through the Refiner → Builder → full review pipeline. Mutually exclusive with `--mode`, `--prompt`, and `--focus`. Requires an existing project directory. In foreground mode, the CEO also enters the refinement loop automatically after completing a build/improve cycle, staying active for follow-up requests without `--refine`. `--mode founder` enters rapid prototyping mode — a stripped-down pipeline (Study → Strategist → Builder → health gate → record) with 2 agent calls and 1 test run. Skips research, code review, adversarial QA, and eval scoring. Designed for fast hypothesis iteration: test an idea, see if it works, pivot. Terminal mode — does not chain to other modes. Not for production use; run `--mode improve` afterward to harden what works. Compatible with `--focus` and `--loop`. `--just-plan` (requires `--mode design`) enters planning-only mode — research + strategy + optional GitHub publishing with no implementation. Three parallel researchers investigate domain, practices, and constraints. The Strategist synthesizes a phased plan. Single user gate: keep the plan? Approval auto-publishes to GitHub as an issue with the `plan` label and seeds the backlog with plan phases. Terminal mode — does not chain to other modes. Compatible with `--focus`. Mutually exclusive with `--from-plan` and `--prompt`.
+
+## Contained runtimes
+
+`factory contained` runs any factory command somewhere other than the developer's shell. Everything after `--` is handed inward **verbatim** except for path rewriting — the runtime is a place to run the factory, not a mode of it, so the host never parses the payload's semantics and cannot break when the CLI grows.
+
+```bash
+factory contained -- ceo ~/code/rta # local container, watch it
+factory contained --division -- ceo ~/code/rta # ...and let the agent build images
+factory contained --target k8s --namespace ns -- run ~/code/rta --loop
+factory contained --target k8s --division -- ceo ~/code/rta
+factory contained ls | attach | rm | sync | setup | verify | bundle
+FACTORY_CONTAINED_DRY_RUN=1 factory contained -- study ~/code/rta # compose, provision nothing
+```
+
+**The two targets share a command surface and an image, not a threat model.** Neither confines agent-authored code, and neither replaces review. Local is the *weaker* of the two: no egress control, and credentials live inside the container. K8s keeps a restricted SCC and namespace-scoped RBAC. None of this reaches user-facing output in these terms — `--help` says "not a security sandbox" and leaves it there, because a security comparison is not an orientation.
+
+Six things are load-bearing and fail quietly if broken:
+
+- **Provenance.** A run always starts from the files on this machine, uncommitted changes included — never `HEAD`, never a fresh clone. The workspace is a git worktree with the working tree rsynced over the top, because a HEAD checkout silently drops the gitignored `.factory/` the whole experiment history lives in. Five assertions then run between provisioning and the first agent call (`factory/contained/provenance.py`); a failure aborts naming the file and the likely cause, and leaves the runtime up for inspection.
+- **Identity.** A bind mount carries ownership through unchanged, so a container whose UID does not own the tree gets a *silently read-only* workspace. The rule differs between rootless, rootful and macOS, so `factory/contained/identity.py` **probes** rather than deciding: a throwaway container reports the mount's owner as the kernel inside sees it, and the run matches. The runtime image is built for arbitrary UIDs (group 0, `chmod g=u`), which is also what OpenShift's restricted SCC needs.
+- **PID 1.** The factory spawns agent subprocesses and is not a well-behaved init, so the container runs `--init` around `sleep infinity` and the run itself lives in tmux. The runtime persists after the run — a failed run is exactly when its state is worth reading.
+- **Credentials cross the boundary, by design.** There is no gateway. The policy is `FACTORY_` by default, plus exactly what `--forward` names, plus the backend variables the resolved shape requires (`factory/contained/credentials.py`) — nothing implicit. `verify` reports credential *shape*, never material, and secret-looking values are redacted anywhere a command is printed. On k8s the credentials come from a namespace Secret the user creates; the factory references it by name and never handles the material.
+- **Both divisions reach outward, and that is the point.** Builds cannot happen inside either boundary, so `--division` is opt-in and separately named. Locally it starts an **unauthenticated** `podman-mcp-server` on `0.0.0.0:8430` — every interface, because the tool has no bind flag and the container reaches the host through a gateway address rather than loopback — detached into its own process group, because the run outlives the launch, and stopped by `factory contained rm`. On the cluster it goes through OpenShift `Build` objects behind a sidecar container that is the only holder of `oc` and the ServiceAccount token; that separation is a boundary only while the Role excludes `pods/exec`, which `verify` asserts via a **SubjectAccessReview API object** — `oc auth can-i --as` collapses `pods/exec` onto `pods` and answers "yes" where RBAC says no. The sidecar runs a **different image** (`FACTORY_CONTAINED_SIDECAR_IMAGE`, an `oc` image) from the agent's; one image for both silently collapses the boundary.
+- **Interactive prompts stall an unattended run.** A fresh `~/.claude` makes Claude Code ask about folder trust, project MCP servers, and Bypass Permissions mode — all interactive-only, so headless agents never hit them and the interactive CEO does, and the run then sits at a menu nobody is watching. `factory/contained/claude_state.py` pre-records those answers, which the invocation already implies.
+- **All podman knowledge lives in `factory/podman.py` and all cluster knowledge in `factory/contained/k8s.py`.** Both **compose** commands and do not execute them, which is what makes `FACTORY_CONTAINED_DRY_RUN=1` print the same argv the real path runs rather than a separate rendering that drifts.
+
+The runtime image (`containers/factory/Containerfile`) is UBI9 + the factory wheel + the agent CLIs + tmux, published multi-arch by CI (`.github/workflows/runtime-image.yml`) — amd64 for cluster nodes, arm64 for a Mac laptop. It publishes on pushes to `main` (`:latest`), on **published releases** (`:`, plus `:latest` unless the release is a prerelease — nightlies are, so they never move `:latest`), and on dispatch. The release trigger is load-bearing: `factory contained setup` pulls and does not build, so a release whose image was never built breaks every new user's first command. Release and dispatch tag names reach the shell through `env:` and are validated against the legal image-tag character set before use.
+
+`setup` is a numbered wizard rather than a column of output (`factory/contained/style.py`): step rules, `[ ok ]`/`[FAIL]` marks, and — the part that caused real confusion — every resolved value printed quoted and coloured, because "in namespace default" gives the reader no way to tell the name from the sentence. Colour obeys `NO_COLOR` > `FORCE_COLOR` > TTY detection, so the same strings stay plain in pipes, logs and CI. The cluster half **asks** which namespace to prepare when `--namespace` was not given (the current context supplies the default, not the answer) and names the **cluster** alongside it — a namespace alone identifies nothing, since `default` exists on every cluster. Only names are read from the kubeconfig, never the `users` section.
+
+**Which cluster is chosen, not assumed.** `setup` lists the kubeconfig's contexts and lets one be picked (`--context NAME` skips the question). The choice is applied as `--context` on *every* cluster command via `k8s.cli()` — a process-global `_ACTIVE_CONTEXT` set once at entry, because threading it through forty call sites means forty chances to forget, and `cli()` is a single auditable application point. It never rewrites the kubeconfig; switching the default is offered separately at the end, with the `oc config use-context` command printed either way.
+
+The chosen namespace is checked for existence (`_namespace_status`) even when passed via `--namespace`, and creation is offered — `oc new-project`, not `create namespace`, because a regular user is usually denied the second. On OpenShift a Forbidden on `get namespace` says nothing about existence, so it falls back to `get project` and reports `unreadable` rather than `absent`.
+
+The cluster review is object-by-object, not a wall of YAML (`factory/contained/k8s_review.py`). The bundle exists as a list (`bundle_objects()`) before it exists as a blob; `render_bundle` joins that list, and `verify`'s per-object checks are derived from it, so the three can never describe different object sets. Each object is compared against the namespace with `oc diff` (server-side, so cluster-defaulted fields do not read as user changes), producing `current` / `absent` / `differs` / `unknown`. Only the ones needing a decision are walked, each showing its purpose plus its diff (for `differs`) or its manifest (for `absent`). `current` is never prompted about — a prompt whose only sane answer is yes teaches people to stop reading prompts — and `unknown` is never silently skipped.
+
+**Each object is applied at the moment it is accepted, never batched.** Batching made `q` report "nothing was applied" to a user who had already said yes, which is false; `WalkResult` records what actually happened and the abort message says how much survives. A failed apply names itself and does not stop the walk. There is no second blanket confirm after the walk.
+
+Prompt options are spelled out (`[y]es [n]o [a]ll remaining [q]uit`), not `[y/n/a/q]`. `style.read_key` and `style.read_line` put the terminal in cbreak mode, which is the only way **Escape** can cancel — a line-buffered `input()` only ever sees the `^[` characters it inserts. `read_line` is a small line editor (echo, Backspace, arrow-key drain) because cbreak turns off the line discipline that normally provides them. Both return `None` when raw reading is impossible (pipe, non-POSIX) and callers fall back to `input()` plus `style.is_escape()`; `input()` raises **OSError** under pytest capture, not `EOFError`, so both are caught. Ctrl-C is caught in `cmd_contained` and exits 130 with a message — backing out of a wizard is ordinary, not a crash.
+
+**`verify_k8s` streams.** It takes an `on_check` callback and reports each result the moment it is known; `prereq.format_check` / `summary_line` are split out of `render_checks` so a caller can print per-result and add the verdict at the end. Without this the Verify step printed nothing for minutes — several checks are a cluster round trip, and the in-cluster inference probe creates a pod and waits up to 180s — and it was reported as a hang. Every result goes through the local `record()` helper, including the early `cli_binary()` failure, because a streaming caller prints only the summary afterwards and a check that skips the callback is never seen. The inference probe is **skipped when the credentials Secret is missing**: the probe pod mounts it, so it could only burn its full timeout rediscovering what the Secret check just reported — which is the state every freshly prepared namespace is in.
+
+**Tests must never reach a raw prompt.** `tests/conftest.py` has an autouse fixture forcing `style._raw_session` to `None`; without it a prompt blocks forever on a keypress, ignoring `builtins.input` patches, because the raw path does not call `input()`. `tests/test_contained_k8s.py` additionally stubs `list_contexts`/`cluster_context`/`current_namespace` — they shell out to real `oc`, which cost that file seven minutes before being stubbed. tmux is compiled in a builder stage because neither the UBI repositories nor EPEL ship it (EPEL never duplicates a package RHEL carries, and UBI's subset omits it).
+
+Two cluster-side details that fail quietly: a PVC mounts root-owned, so the pod needs an `fsGroup` read from the namespace's allocated range (hardcoding one fails admission under a `MustRunAs` SCC); and the workspace unpack marker is **per-run**, because the PVC outlives the run that filled it and a shared marker makes the next run skip its own upload and execute against stale files.
+
+User-facing guide: `docs/contained/index.md`.
## Observability
diff --git a/README.md b/README.md
deleted file mode 100644
index 4ccbb0ffe..000000000
--- a/README.md
+++ /dev/null
@@ -1,321 +0,0 @@
-
-
-
-
-
-[](https://github.com/akashgit/remote-factory/actions/workflows/ci.yml)
-[](https://codecov.io/gh/akashgit/remote-factory)
-[](https://www.python.org/downloads/)
-[](LICENSE)
-[](https://docs.anthropic.com/en/docs/claude-code)
-[](https://bob.ibm.com)
-[](https://openai.com/index/codex/)
-
-**Describe what you want — re:factory builds it, tests it, and keeps improving it.** Design an idea from scratch or point at an existing project for continuous improvement. Runs with [Claude Code](https://docs.anthropic.com/en/docs/claude-code), [Bob Shell](https://bob.ibm.com), and [OpenAI Codex](https://openai.com/index/codex/).
-
-All state is local — per-project in `.factory/` (add to `.gitignore`), global in `~/.factory/`. See [Architecture](docs/architecture.md) for the full deep-dive.
-
----
-
-## Quick Start
-
-**Prerequisites:** Python 3.11+, [uv](https://docs.astral.sh/uv/), and [Claude Code](https://docs.anthropic.com/en/docs/claude-code) (installed and authenticated).
-
-```bash
-git clone https://github.com/akashgit/remote-factory.git
-cd remote-factory
-uv sync
-```
-
-Then start with one of the two main workflows:
-
-```bash
-# Design — brainstorm an idea, refine it, then build
-uv run factory ceo "my idea" --mode design
-
-# Improve — point at an existing project for continuous improvement
-uv run factory ceo /path/to/project --mode improve --focus "issue # or whatever you want to improve or fix"
-
-# Co-improve — if you want to iterate on the implementation plan before implementation starts for an improvement
-uv run factory ceo /path/to/project --mode design --focus "issue # or whatever you want to improve or fix"
-```
-
-See the [full setup guide](docs/setup.md) for authentication and environment variables.
-
----
-
-## What Do You Want to Do?
-
-| I want to… | Command |
-|---|---|
-| **Start from a raw idea** | `uv run factory ceo "my idea" --mode design` |
-| **Improve an existing project** | `uv run factory ceo /path/to/project --mode improve --focus "issue number or whatever you want to improve or fix ` |
-| **Co-improve an existing project** | `uv run factory ceo /path/to/project --mode design --focus "description of whatever you want to improve or fix ` |
-| **Create a new factory mode** | `uv run factory ceo /path/to/factory --mode create "description"` |
-
----
-
-## Design Workflow
-
-Use design mode when you want to brainstorm before building. Start a conversation with the CEO to refine an idea, then build:
-
-```bash
-# From a raw idea — discuss and refine into a buildable spec
-uv run factory ceo "distributed task runner" --mode design
-
-# From a spec file — read and discuss before building
-uv run factory ceo ~/ideas/my-app-spec.md --mode design
-```
-
-Design mode also works on existing projects. The CEO studies the backlog, eval scores, open issues, and experiment history, then discusses what to work on before executing:
-
-```bash
-uv run factory ceo ~/factory-projects/my-app --mode design
-
-# Seed the conversation with a topic
-uv run factory ceo ~/factory-projects/my-app --mode design --focus "auth layer"
-```
-
-You can also pass a spec file or URL directly — `uv run factory ceo spec.md` — and re:factory builds without the design conversation.
-
----
-
-## Improve Workflow
-
-Improve mode is re:factory's continuous improvement loop for existing projects. Point it at a codebase and it autonomously observes the project state, generates hypotheses for improvements, builds and tests changes, and keeps or reverts each experiment based on eval scores.
-
-```bash
-uv run factory ceo ~/factory-projects/my-app --mode improve
-```
-
-Each cycle: **observe** → **hypothesize** → **build** → **review** → **measure** → **decide** (keep or revert) → **archive**. The Strategist picks work from the backlog using FEEC priority (Fix > Exploit > Explore > Combine).
-
-When you know exactly what you want, `--focus` pins a single target — one hypothesis, one experiment, done:
-
-```bash
-uv run factory ceo ~/my-app --mode improve --focus "add dark mode toggle"
-uv run factory ceo ~/my-app --mode improve --focus 42 # GitHub issue
-uv run factory ceo ~/my-app --mode improve --focus "owner/repo#42" # Issue shorthand
-```
-
----
-
-## Post-Cycle Refinement
-
-After a build or improve cycle finishes in foreground mode, the CEO stays active — it doesn't exit. Ask for changes directly:
-
-> "Fix the typo in the header"
-> "Add error handling to the upload endpoint"
-> "Make the tests more thorough"
-
-Each request runs through the full experiment pipeline: the **Refiner** scopes it → **Builder** implements → review + eval + E2E gate → keep/revert verdict. No shortcuts — every refinement is a tracked experiment with its own PR.
-
-You can also invoke refinements directly with `--refine`:
-
-```bash
-uv run factory ceo ~/my-app --refine "add rate limiting to the API"
-```
-
-There's no cap on refinements. Advisory warnings appear at 5 and 10 to flag context growth, but the user decides when to stop.
-
----
-
-## Create New Modes
-
-Create mode lets you build new factory modes — new workflows, new pipelines, new factories — from a description. Describe what the mode should do, and re:factory researches existing patterns, synthesizes a workflow spec, gets your approval, then implements everything: workflow definition, SKILL.md, CLI wiring, and tests.
-
-```bash
-# From a description
-uv run factory ceo /path/to/factory --mode create "a mode that audits security vulnerabilities"
-
-# From a spec file
-uv run factory ceo /path/to/factory --mode create ~/specs/audit-mode.md
-```
-
-The pipeline: **3 parallel researchers** (existing patterns, intent analysis, best practices) → **Strategist** synthesizes a workflow spec → **you approve** (like design mode) → **Builder** implements → **QA** verifies end-to-end → **PR**.
-
-Create mode is interactive — it requires your approval at the strategy gate before building. Point it at the factory repo itself to extend re:factory with custom pipelines.
-
----
-
-## Eval System
-
-Every change is measured by an 11-dimension composite score across three tiers: **Hygiene** (tests, lint, types, coverage), **Growth** (API surface, experiment diversity, observability), and **Project** (user-defined domain metrics). On first run, `uv run factory discover` auto-detects your project's language and framework to generate the eval profile. See [Eval System](docs/eval.md) for scoring details, weights, and guards.
-
----
-
-## Built with re:factory
-
-| Project | What it does | Mode |
-|---------|-------------|------|
-| **SWE-bench solver** | Autonomous agent that resolves GitHub issues, improved via failure analysis | Research |
-| **HMMT math solver** | Multi-agent team that solved HMMT Feb 2025 Combinatorics Problem 7 | Research |
-| **Text/Sketch → CAD** | Natural language and sketches to executable CadQuery Python code for 3D models | Research |
-| **HLS design space explorer** | Per-function AI agents + ILP solver for HLS optimization — 92% execution time reduction | Build |
-| **Pluck** | iOS app that extracts structured data from screenshots using on-device AI | Build + Improve |
-| **[SDG Hub](https://github.com/Red-Hat-AI-Innovation-Team/sdg_hub)** | Agent-maintained open-source framework for synthetic data generation | Build + Improve |
-| **[OpenSkies Airline Corpus](https://github.com/lukeinglis/OpenSkiesAirline)** | 85-document fictional airline corpus for RAG/fine-tuning evaluation with cross-document consistency validation | Design + Improve |
-| **re:factory itself** | Runs on itself — continuously improved via its own experiment outcomes | Meta |
-
-Built something with re:factory? Open a PR to add it here.
-
----
-
-## CLI Quick Reference
-
-```bash
-# Core workflow
-uv run factory ceo "idea" --mode design # Design from a raw idea
-uv run factory ceo --mode improve # Improve an existing project
-uv run factory ceo --refine "..." # Single targeted refinement
-uv run factory ceo --mode create "..." # Create a new factory mode
-uv run factory ceo --loop # Continuous improvement loop
-uv run factory tmux --loop # Loop in detached tmux session
-```
-
-See `uv run factory --help` for the complete list.
-
----
-
-## Runners
-
-re:factory supports multiple CLI backends. Default is Claude Code — switch with `--runner` or `FACTORY_RUNNER`:
-
-```bash
-# Direct
-CODEX_API_KEY="..." uv run factory ceo /path --runner codex
-BOBSHELL_API_KEY="..." uv run factory ceo /path --runner bob
-
-# Via config.toml profile (persistent)
-uv run factory ceo /path --profile codex
-```
-
-Configure profiles in `~/.factory/config.toml`:
-
-```toml
-[credentials.codex]
-FACTORY_RUNNER = "codex"
-CODEX_API_KEY = "..."
-
-[credentials.bob]
-FACTORY_RUNNER = "bob"
-BOBSHELL_API_KEY = "..."
-```
-
-Run `uv run factory config show` to see resolved config, or `uv run factory config edit` to open the file. See [Setup Guide](docs/setup.md) for full details.
-
----
-
-## LLM Tracing (LangFuse)
-
-LangFuse provides LLM observability and tracing — track agent invocations, token usage, and execution flow across all factory runs.
-
-### Quick Start
-
-```bash
-# Start LangFuse services
-scripts/langfuse-setup start
-
-# Set the env vars the factory needs
-export LANGFUSE_HOST=http://localhost:3000
-export LANGFUSE_BASE_URL=http://localhost:3000
-export LANGFUSE_PUBLIC_KEY=pk-lf-dev-local-key
-export LANGFUSE_SECRET_KEY=sk-lf-dev-local-key
-export TELEMETRY_PLATFORM=langfuse
-```
-
-The dev credentials above match the docker-compose setup. Add them to your `~/.bashrc` or `~/.zshrc` to persist across sessions.
-
-### Viewing Traces
-
-1. Start LangFuse: `scripts/langfuse-setup start`
-2. Run the factory: `uv run factory ceo /path/to/project`
-3. Open `http://localhost:3000` in your browser
-4. Login: `dev@localhost.local` / `devpassword123`
-
-### CLI Commands
-
-```bash
-scripts/langfuse-setup start # Start LangFuse services
-scripts/langfuse-setup stop # Stop services
-scripts/langfuse-setup status # Show status and credentials
-```
-
-### Requirements
-
-- **Docker** or **Podman** — any of `docker compose`, `docker-compose`, or `podman-compose` works
-
-### Disabling Tracing
-
-To disable tracing without stopping LangFuse:
-```bash
-export LANGFUSE_TRACING_ENABLED=false
-```
-
-For LLM connection setup, trace structure details, and troubleshooting, see [`infra/langfuse/README.md`](infra/langfuse/README.md).
-
----
-
-## Install as a Claude Code Plugin
-
-re:factory is also distributed as a fully-bundled [Claude Code plugin](https://docs.claude.com/en/docs/claude-code/plugins) — agents, skills, and slash commands packaged together. A GitHub Actions workflow rebuilds the `plugins` branch of this repo on every push to `main`, so it always tracks the latest generated artifacts.
-
-From inside Claude Code:
-
-```text
-/plugin marketplace add akashgit/remote-factory#plugins
-/plugin install factory@remote-factory
-/reload-plugins
-```
-
-Once installed, the plugin exposes:
-
-- The `/factory:implement` slash command (entry point for the multi-agent pipeline).
-- Namespaced subagents — invoke with `factory:ceo`, `factory:researcher`, `factory:builder`, etc.
-- The bundled skills under `.agents/skills/` (e.g. `pipeline-subagents`, `implement`).
-
-The plugin still shells out to the `factory` CLI for the heavy lifting, so you'll need `uv` and the `factory` package installed locally as described in [Quick Start](#quick-start).
-
-To update later: `/plugin marketplace update remote-factory`. To remove: `/plugin uninstall factory@remote-factory`.
-
----
-
-## Plugin Agents
-
-If you'd rather skip the marketplace and just register the specialist agents as standalone Claude Code (or Codex) subagents, use the built-in installer:
-
-```bash
-uv run factory install # Install all 9 agents to ~/.claude/agents/
-uv run factory install --runner codex # Or install Codex TOML agents to ~/.codex/agents/
-claude --agent factory-ceo "improve this project"
-claude --agent factory-researcher "study the auth system"
-```
-
-This path only ships the agent prompts (no skills, no slash commands) and is independent of the plugin marketplace install above.
-
----
-
-## Documentation
-
-| Doc | What's in it |
-|-----|-------------|
-| [Setup Guide](docs/setup.md) | Installation, authentication, environment variables |
-| [Getting Started](docs/getting-started.md) | Lifecycle walkthrough, research mode details, factory.md config |
-| [Architecture](docs/architecture.md) | Three-layer system, agent roles, state machine, data flow |
-| [Eval System](docs/eval.md) | Hygiene/growth/project tiers, scoring, guards, precheck |
-| [Configuration](docs/configuration.md) | `factory.md` reference — all sections and options |
-| [ACE Self-Improvement](docs/ace.md) | How re:factory evolves its own agent playbooks |
-| [Contributing](docs/contributing.md) | Dev setup, code style, testing, PR workflow |
-
-## Development
-
-```bash
-uv sync --all-groups # Install all deps including dev
-uv run pytest -v # Full test suite
-uv run ruff check . # Lint
-uv run mypy factory/ # Type check
-```
-
-## License
-
-[MIT](LICENSE) — Akash Srivastava
diff --git a/README.md b/README.md
new file mode 120000
index 000000000..e89233038
--- /dev/null
+++ b/README.md
@@ -0,0 +1 @@
+docs/index.md
\ No newline at end of file
diff --git a/SPEC.md b/SPEC.md
index 149be53f2..de06f410c 100644
--- a/SPEC.md
+++ b/SPEC.md
@@ -1,9 +1,6 @@
-# re:factory Meta-Harness Specification
+# SPEC — re:factory
-Status: Draft v1 (language-agnostic)
-
-Purpose: Define a meta-harness that orchestrates coding agents through bounded,
-measurable, reversible SDLC cycles.
+Status: Draft | Auto-generated by re:factory
## Normative Language
@@ -13,578 +10,1242 @@ described in RFC 2119.
`Implementation-defined` means the behavior is part of the implementation
contract, but this specification does not prescribe one universal policy.
-Implementations MUST document the selected behavior.
## 1. Problem Statement
-re:factory is a meta-harness for agentic software evolution. It accepts software
-work, binds that work to a project context, dispatches coding agents under an
-execution contract, validates the result through guardrails, records evidence,
-and converts the outcome into an explicit decision and durable memory.
-
-The system solves five operational problems:
-
-- It turns agentic coding into a repeatable SDLC lifecycle instead of ad hoc
- prompts or scripts.
-- It separates project scope from repository checkouts, runtime execution, and
- product packaging.
-- It makes each change measurable and reversible through evidence, guardrails,
- and explicit decisions.
-- It keeps project state durable enough to support resume, review, and
- learning.
-- It is designed so future implementations can preserve the same lifecycle
- semantics without changing the meaning of a project cycle.
-
-Important boundary:
-
-- re:factory is a meta-harness, not a general-purpose workflow engine.
-- A deployment profile is a bundle of component implementations, not a separate
- implementation of the domain model.
-- Agent execution MAY end at a handoff state; a successful run does not
- necessarily mean code was merged or released.
-- Trust, approval, sandboxing, and external write policies are
- implementation-defined and MUST be documented by the implementation.
+re:factory is a domain-agnostic multi-agent software evolution loop that autonomously builds and continuously improves software projects through iterative cycles of observation, hypothesis generation, implementation, and evaluation.
+
+It solves these operational problems:
+
+- It **automates the software improvement cycle** instead of requiring manual hypothesis generation, implementation, and testing for each improvement iteration
+- It **maintains architectural coherence** during multi-agent development instead of allowing each agent to independently decide what to build or change without coordination
+- It **tracks experiment outcomes as append-only history** instead of losing context on what was tried, why it was reverted, and what was learned
+- It **enforces eval-driven decisions** through weighted composite scores across hygiene, growth, and project-specific dimensions instead of subjective "looks good" judgments
+- It **preserves cross-project knowledge** in a structured archive instead of requiring each project to re-learn the same patterns and anti-patterns
+- It **supports pluggable CLI backends** (Claude Code, Bob Shell, OpenAI Codex, OpenCode) through a runner abstraction instead of hard-coding a single LLM provider
+
+**Important boundary:** re:factory is NOT responsible for training models, hosting infrastructure, or managing authentication to LLM providers. It delegates to authenticated CLI tools and expects the user to configure credentials externally.
## 2. Goals and Non-Goals
### 2.1 Goals
-- Represent software work as normalized work items.
-- Bind work items to a durable project context.
-- Support projects that bind the repository or execution context needed for
- work.
-- Dispatch agents through explicit execution contracts.
-- Preserve evidence for diffs, logs, evals, reviews, reports, and artifacts.
-- Validate outcomes through guardrails before a decision is accepted.
-- Record decisions as first-class lifecycle outputs.
-- Maintain durable memory for project learning and future planning.
-- Preserve durable state for resume, review, and learning, with optional
- reconciliation to external systems where supported.
-- Treat the CLI-local profile as the primary compatibility surface.
-- Allow future deployment profiles to bundle different runtimes, state
- backends, guardrails, and output surfaces while preserving common lifecycle
- semantics.
+- Detect project state (no repo, incomplete, no factory, pending review, configured) and route to the appropriate workflow mode
+- Discover project structure, testing tools, linters, and type checkers to generate an eval profile without manual configuration
+- Execute weighted composite evals combining hygiene dimensions (tests, lint, type check, coverage) and growth dimensions (capability surface, experiment diversity, observability)
+- Dispatch coding agents (Researcher, Strategist, Builder, QA, Archivist) through explicit subprocess contracts with budget controls
+- Persist structured experiment records (hypothesis, eval before/after, diff, verdict) in `.factory/` as append-only TSV history
+- Apply FEEC priority heuristic (Fix > Exploit > Explore > Combine) to classify hypotheses and detect stuck patterns after 3+ consecutive same-category reverts
+- Maintain cross-project knowledge archives (patterns, decisions, experiments) for domain transfer
+- Evolve agent playbooks automatically via ACE (Autonomous Capability Evolution) based on performance reports
+- Support multiple CLI backends (Claude Code, Bob Shell, OpenAI Codex, OpenCode) through a runner abstraction
+- Enable research mode with inner/outer loop plateau detection, adversarial GAN-style eval loops, and mutable/fixed surface constraints
### 2.2 Non-Goals
-- Prescribing a specific source-code layout or module structure.
-- Requiring a managed service or hosted control plane.
-- Requiring Jira, Linear, GitHub, GitLab, or any specific tracker.
-- Requiring a rich web UI or dashboard.
-- Mandating one sandbox, approval, or operator-confirmation policy.
-- Mandating that agents perform ticket writes, PR creation, or merge actions.
-- Requiring multi-repository orchestration or multi-user shared-state
- collaboration as part of core conformance.
-- Replacing human review, CI policy, or repository governance.
-
-## 3. System Overview
-
-### 3.1 Main Components
-
-1. `Deployment Profile`
- - Names a product surface and selected component implementations.
- - Declares runtime, state handling, guardrails, output surfaces, and policy
- sources.
- - Does not redefine the core domain model.
-2. `Project Resolver`
- - Converts user input or configuration into a project context.
- - Binds the repository, checkout, and state locations required by the
- selected implementation.
- - Resolves or generates the project specification document (see Section
- 4.11).
-3. `Work Item Source`
- - Reads work from prompts, backlog entries, issues, tickets, or research
- targets.
- - Normalizes external payloads into stable work-item records.
-4. `Contract Builder`
- - Converts project policy and work-item scope into an execution contract.
- - Identifies mutable surfaces, fixed surfaces, required checks, budgets, and
- expected evidence.
-5. `Lifecycle Coordinator`
- - Owns the lifecycle transition from intake through learning.
- - Decides when to dispatch, validate, retry, park, or escalate work.
- - Converts worker and guardrail outcomes into decision records.
-6. `Worker Runtime`
- - Runs a coding agent or worker against an execution contract.
- - Returns output, status, logs, and implementation-defined telemetry.
-7. `Guardrail Provider`
- - Evaluates tests, lint, type checks, eval metrics, CI state, review policy,
- scope rules, leakage rules, security policy, or other checks.
-8. `State Backend`
- - Persists project records, evidence references, decisions, and memory.
- - MAY mirror or reconcile state with external systems when supported.
-9. `Memory System`
- - Preserves durable learnings, observations, playbook evidence, reports, and
- handoff records.
-10. `Output Surface`
- - Publishes or materializes implementation-defined lifecycle outputs such as
- reviews, reports, generated assets, or external updates.
-
-### 3.2 Abstraction Levels
-
-re:factory is easiest to port when kept in these layers:
-
-1. `Policy Layer`
- - Project goal, scope, constraints, prompts, and validation policy.
-2. `Profile Layer`
- - User-facing surfaces and component bundles.
-3. `Coordination Layer`
- - Lifecycle transitions, dispatch, validation ordering, decisions, retry, and
- resume.
-4. `Execution Layer`
- - Worker runtime, repository checkout/worktree behavior, and agent protocol.
-5. `State Layer`
- - Project records, event streams, materialized views, and external bindings
- when present.
-6. `Guardrail and Evidence Layer`
- - Checks, artifacts, logs, scores, reviews, and reports.
-7. `Memory and Observability Layer`
- - Human/operator-visible status, archives, summaries, and learned rules.
-
-### 3.3 External Dependencies
-
-Implementations MAY depend on:
-
-- Local filesystem state.
-- Git repositories and worktrees.
-- Coding-agent executables or managed agent services.
-- Issue trackers, ticket systems, or PR systems.
-- CI, review, or security-scanning systems.
-- Host authentication for agent runtimes and external state backends.
-
-## 4. Core Domain Model
-
-### 4.1 Project
-
-A `Project` is the durable SDLC boundary for work, evidence, decisions, and
-memory.
-
-Logical fields:
-
-- `project_id`: stable project identifier.
-- `name`: human-readable project name.
-- `goal`: project objective or mission statement.
-- `repo_bindings`: repository or checkout bindings associated with the project.
-- `state_bindings`: durable or external state substrates associated with the
- project.
-- `policy_refs`: references to project policy/configuration.
-- `memory_refs`: references to durable project memory.
-
-Rules:
-
-- A project MUST bind the execution context needed for the work.
-- Implementations MAY realize that execution context as one repository binding
- or multiple repository bindings.
-- A single local repository binding with local durable state is sufficient for
- core conformance.
-- Work items, decisions, and memory belong to the project.
-- Diffs, branches, and checkouts belong to repository bindings.
-- Runtime and deployment profile are not project-owned.
-
-### 4.2 Repo Binding
-
-A `RepoBinding` identifies a repository or worktree participating in a project.
-
-Logical fields:
-
-- `repo_id`: stable identifier within the project.
-- `path`: local path, if available.
-- `remote`: remote repository identifier or URL, if available.
-- `role`: implementation-defined role such as `primary`.
-- `default_branch`: default integration branch, if known.
-- `checkout`: checkout or worktree metadata, if applicable.
-
-### 4.3 State Binding
-
-A `StateBinding` identifies a state substrate associated with a project.
-
-Examples:
-
-- local project state
-- GitHub issue or PR state
-- GitLab issue or merge-request state
-- Jira ticket state
-- Linear issue state
-- managed service state
-
-State bindings MUST NOT imply that runtime execution happens in that state
-system.
+- **Human-in-the-loop approval for every change.** (The factory autonomously commits experiments, then reverts if eval score drops. Users MAY configure hard constraints that enforce mandatory reverts.)
+- **Real-time deployment or hosting.** (The factory produces local git commits. Deployment is delegated to external CI/CD.)
+- **Universal language support.** (Discovery focuses on Python and Bash. Other languages MAY be added via evaluator plugins.)
+- **Guaranteed improvement on every cycle.** (Some hypotheses MUST be reverted. The factory measures statistical trends over multiple cycles, not single-cycle perfection.)
+- **Interactive debugging or REPL support.** (The factory operates headlessly. Debugging MUST be done via logged experiment artifacts in `.factory/experiments/`.)
+- **Multi-user collaboration or concurrent writes.** (The factory assumes single-writer access to `.factory/` directory. Concurrent runs MUST use separate project directories.)
+
+### 2.3 Design Philosophy
+
+re:factory treats software improvement as a scientific experiment loop: observe, hypothesize, test, keep or revert. All agent invocations follow explicit subprocess contracts (role, task, project path) with structured output capture. All state transitions are deterministic and resumable. The factory is a harness, not a monolithic agent — specialization beats generalization.
+
+## 3. Project Identity
+
+- **Name:** re:factory (remote-factory)
+- **Type:** CLI tool and multi-agent orchestration harness
+- **Language:** Python 3.11+
+- **Framework:** Pydantic v2 (strict models), FastAPI (dashboard), Structlog (logging)
+- **Package Manager:** uv
+- **Entry Point:** `factory.cli:main` (registered as `factory` script)
+
+## 4. Technical Stack
+
+### 4.1 Dependencies
+
+- `pydantic>=2.0` — Strict validation for all domain models
+- `structlog>=24.0` — Structured logging with context binding
+- `fastapi>=0.115` — Dashboard HTTP server with SSE streaming
+- `uvicorn[standard]>=0.34` — ASGI server for dashboard
+- `mcp>=1.27.0` — MCP server for factory tools (checkpoints, profiles, experiments)
+- `pyyaml>=6.0` — Parse agent prompt frontmatter and config files
+- `filelock>=3.0` — Prevent concurrent writes to `.factory/` state files
+- `networkx>=3.6.1` — Workflow graph representation and traversal
+- `langfuse>=3.0` — Telemetry and token usage tracking (optional)
+- `graphifyy>=0.9` — Code knowledge graph extraction for spec generation
+
+### 4.2 External Dependencies
+
+- `claude` CLI — Claude Code runner (default) (REQUIRED unless using alternate runner)
+- `bob` CLI — Bob Shell runner (OPTIONAL)
+- `codex` CLI — OpenAI Codex runner (OPTIONAL)
+- `opencode` CLI — OpenCode runner (OPTIONAL, requires `opencode-ai/opencode` v0.x from GitHub)
+- `gh` CLI — GitHub issue fetching for `--focus` mode (OPTIONAL)
+- `glab` CLI — GitLab issue fetching for `--focus` mode (OPTIONAL)
+- `uv` — Python package manager and virtual environment tool (REQUIRED)
+- `git` — Version control for experiment diffs and branch management (REQUIRED)
+
+## 5. Architecture Overview
+
+### 5.1 Abstraction Levels
+
+1. **Layer 1: Python CLI** — Pure tool functions that dispatch to higher layers. Entry point is `factory/cli.py` with `cmd_*` handlers. No decision-making logic — only argument parsing, subprocess spawning, and output formatting.
+
+2. **Layer 2: Workflow Graph Engine** — Directed acyclic graphs of typed nodes (`AgentNode`, `FnNode`, `GateNode`, `ForkNode`, `JoinNode`, `Study`) connected by edges. Defined in `factory/workflow/definitions.py`. Each of the 8 modes (build, design, improve, research, meta, discover, review, refine) has a workflow graph. Execution happens via `WorkflowExecutor` (headless) or skill export to `SKILL.md` files (interactive CEO).
+
+3. **Layer 3: CEO Agent** — Orchestrator agent that reads `factory/agents/prompts/ceo.md` (core identity) and `skills/workflow-*/SKILL.md` (mode-specific playbooks). The CEO detects project state, selects the appropriate workflow, spawns specialist agents, enforces review gates, and handles keep/revert decisions.
+
+4. **Layer 4: Specialist Agents** — Eight subprocess agents spawned via `factory agent --task "..." --project /path`: Researcher (observe and research), Strategist (hypothesize and refine), Builder (implement), QA (test and review), Archivist (record knowledge), Refiner (scope changes), Failure Analyst (analyze failures), and a meta-CEO role. Each agent has a prompt at `factory/agents/prompts/.md` with optional project overrides at `.factory/agents/.md`.
+
+### 5.2 Data Flow Summary
+
+State detection (`factory/state.py`) reads git status, `.factory/config.json`, and `eval_profile.json` to determine one of five `ProjectState` values. Discovery (`factory/discovery/`) introspects the project to generate `eval_profile.json` and `eval/score.py`. The CEO spawns the Researcher to produce `observations.md`, then the Strategist to generate hypotheses (stored in `.factory/strategy/backlog.md` and `.factory/strategy/current.md`). The Builder implements the hypothesis as a git commit. The QA agent runs health checks and code review. The eval runner (`factory/eval/runner.py`) executes the eval command, parses JSON output, computes weighted composite scores, and compares before/after. The CEO makes a keep/revert verdict based on score delta and constraint violations. Finalization writes the experiment record to `.factory/results.tsv` and stores artifacts in `.factory/experiments//`. The Archivist writes structured learnings to `.factory/archive/`. ACE (`factory/ace/`) generates performance reports and evolves agent playbooks stored in `~/.factory/playbooks/.md`.
+
+## 6. Domain Model
+
+### 6.1 ProjectState
+
+- **Type:** String enum with five literal values
+- **Values:** `no_repo`, `incomplete`, `no_factory`, `evals_pending_review`, `has_factory`
+- **Purpose:** Represents the detected state of a target project directory
+- **Transitions:** MUST be computed by `detect_state()` in [[graph:factory/state.py]] before workflow selection
+- **Behavioral rules:**
+ - `no_repo` MUST be returned when `.git/` does not exist
+ - `incomplete` MUST be returned when the git working tree has uncommitted changes or the project is empty
+ - `no_factory` MUST be returned when `.git/` exists, working tree is clean, but `.factory/config.json` is missing
+ - `evals_pending_review` MUST be returned when `eval_profile.json` exists but `config.json` is missing
+ - `has_factory` MUST be returned when both `.factory/config.json` and `eval_profile.json` exist
+
+### 6.2 FactoryConfig
+
+- **Type:** Strict Pydantic model with `extra="forbid"`
+- **File location:** `.factory/config.json`
+- **Required fields:**
+ - `goal: str` — Natural language description of improvement objective
+ - `scope: list[str]` — File paths or glob patterns defining mutation scope
+ - `guards: list[str]` — Natural language constraints that MUST NOT be violated
+ - `eval_command: str` — Shell command that produces eval JSON (MUST exit 0 and write JSON to stdout)
+ - `eval_threshold: float` — Minimum composite score for keep decision (0.0 to 1.0)
+ - `constraints: list[str]` — Natural language constraints enforced by CEO judgment
+- **Optional fields with defaults:**
+ - `hypothesis_budget: HypothesisBudget` — Controls `min_growth` and `max_new` hypothesis selection (default: `{min_growth: 2, max_new: 2}`)
+ - `target_branch: str` — Git branch for final commits (default: `"main"`)
+ - `smoke_test: str` — Quick validation command run before full eval (default: `""`)
+ - `project_eval: list[ProjectEvalDimension]` — User-defined eval dimensions (default: `[]`)
+ - `eval_weights: EvalWeights` — Weight distribution across hygiene/growth/project tiers (default: `{hygiene: 0.50, growth: 0.50, project: 0.0}`)
+ - `research_target: ResearchTarget | None` — Research mode configuration (default: `None`)
+ - `inner_loop: InnerLoopConfig | None` — Multi-run execution and plateau detection (default: `None`)
+ - `outer_loop: OuterLoopConfig | None` — Outer loop configuration for research mode (default: `None`)
+ - `mutable_surfaces: list[str]` — File paths that research mode MAY mutate (default: `[]`)
+ - `fixed_surfaces: list[str]` — File paths that research mode MUST NOT mutate (default: `[]`)
+ - `research_constraints: list[str]` — Natural language constraints for research hypotheses (default: `[]`)
+ - `cost_budget: CostBudgetConfig | None` — Per-cycle and total cost limits (default: `None`)
+ - `hard_constraints: list[HardConstraint]` — Shell commands that MUST exit 0 for keep (default: `[]`)
+ - `eval_spec: list[str]` — File paths to specification documents (default: `[]`)
+ - `hygiene_weights: TierWeights | None` — Within-tier weight overrides for hygiene dimensions (default: `None`)
+ - `growth_weights: TierWeights | None` — Within-tier weight overrides for growth dimensions (default: `None`)
+ - `adversarial: AdversarialConfig | None` — GAN-style adversarial eval loop (default: `None`)
+ - `parallel: ParallelConfig | None` — Parallel hypothesis execution (default: `None`)
+ - `clean_pr: bool` — Enable Clean PR Mode (default: `False`)
+ - `clean_pr_include: list[str]` — Glob patterns for Clean PR inclusion (default: `[]`)
+ - `clean_pr_exclude: list[str]` — Glob patterns for Clean PR exclusion (default: `[]`)
+ - `test_timeout: int` — Maximum seconds for test execution (default: `600`, minimum: `1`)
+- **Behavioral rules:**
+ - Config MUST be serializable to JSON with no loss of fidelity
+ - Config MUST validate via Pydantic strict mode before use
+ - Config MUST be written by Discovery workflow or parsed from `factory.md` spec file
+ - Config fields MUST NOT be mutated during an experiment cycle (read-only after load)
+ - Hard constraints MUST be evaluated before final keep decision
+ - Eval weights MUST sum to 1.0 (validated by [[graph:factory/eval/runner.py]])
+ - Research mode MUST error if `mutable_surfaces` is empty and `research_target` is set
+ - Parallel config MUST validate `parallel_hypotheses` is between 1 and 8
+
+### 6.3 EvalProfile
+
+- **Type:** Strict Pydantic model with `extra="forbid"`
+- **File location:** `.factory/eval_profile.json`
+- **Required fields:**
+ - `project_type: str` — Detected project category (e.g., "cli", "library", "web_app")
+ - `language: str` — Primary language (e.g., "python", "bash")
+ - `package_manager: str | None` — Detected manager (e.g., "uv", "npm", "cargo")
+ - `dimensions: list[EvalDimension]` — List of eval functions with commands and weights
+ - `hygiene_weight: float` — Tier weight for hygiene dimensions (0.0 to 1.0)
+ - `growth_weight: float` — Tier weight for growth dimensions (0.0 to 1.0)
+ - `project_weight: float` — Tier weight for project-specific dimensions (0.0 to 1.0)
+ - `human_reviewed: bool` — Whether a human has validated this profile
+- **Behavioral rules:**
+ - Profile MUST be generated by [[graph:factory/discovery/profile.py]] during Discovery workflow
+ - Dimensions MUST include at least one hygiene eval (tests, lint, or type_check)
+ - Tier weights MUST sum to 1.0
+ - Each dimension MUST have a unique `name` field
+ - Commands MUST be shell-executable strings that produce JSON output on stdout
+ - Profile MUST be marked `human_reviewed: false` initially
+ - Profile SHOULD set test weight between 0.4 and 0.5 for hygiene tier
+ - Profile SHOULD set lint weight between 0.2 and 0.3 for hygiene tier
+
+### 6.4 ExperimentRecord
+
+- **Type:** Strict Pydantic model with `extra="forbid"`
+- **File location:** `.factory/results.tsv` (one row per experiment, append-only)
+- **Required fields:**
+ - `id: int` — Sequential experiment ID (1-indexed)
+ - `timestamp: str` — ISO 8601 timestamp of experiment start
+ - `hypothesis: str` — Natural language description of change
+ - `category: str` — FEEC category ("fix", "exploit", "explore", "combine", "unclassified")
+ - `scope: str` — File paths modified (comma-separated)
+ - `score_before: float` — Composite eval score before change (0.0 to 1.0)
+ - `score_after: float` — Composite eval score after change (0.0 to 1.0)
+ - `delta: float` — Score delta (after - before)
+ - `verdict: str` — "KEEP" or "REVERT"
+ - `commit_sha: str` — Git commit SHA of the change (or empty if reverted)
+ - `duration_seconds: float` — Total cycle time in seconds
+- **Behavioral rules:**
+ - Records MUST be serialized to TSV with tab separators
+ - TSV header row MUST match `ExperimentRecord` field order
+ - Append-only semantics — records MUST NOT be deleted or mutated after write
+ - ID MUST auto-increment from previous max ID (or start at 1 if TSV is empty)
+ - Timestamp MUST be UTC ISO 8601 format
+ - Category MUST be computed by `classify_feec()` in [[graph:factory/strategy.py]]
+ - Verdict MUST be "KEEP" if `score_after >= score_before - tolerance` and no hard constraint violations
+ - Verdict MUST be "REVERT" otherwise
+ - Commit SHA MUST be populated only on KEEP verdicts
+
+### 6.5 CompositeScore
+
+- **Type:** Strict Pydantic model with `extra="forbid"`
+- **Purpose:** Aggregates all eval dimensions into a single weighted score
+- **Required fields:**
+ - `total: float` — Weighted composite score (0.0 to 1.0)
+ - `results: list[EvalResult]` — Per-dimension scores with weights
+ - `guard_violations: list[str]` — List of violated guard constraints (empty if none)
+- **Behavioral rules:**
+ - `total` MUST be computed as the sum of `(score * weight)` for each `EvalResult` in `results`
+ - `results` MUST include entries for all discovered hygiene, growth, and project dimensions
+ - `guard_violations` MUST be populated by string matching against `FactoryConfig.guards`
+ - Non-zero `guard_violations` SHOULD trigger a REVERT verdict regardless of score delta
+ - Composite score MUST be serializable to JSON for `eval_before.json` and `eval_after.json` files
+
+### 6.6 Observation
+
+- **Type:** Strict Pydantic model with `extra="forbid"`
+- **File location:** `.factory/strategy/observations.md`
+- **Required fields:**
+ - `category: str` — Observation category (e.g., "project_structure", "test_coverage", "technical_debt")
+ - `title: str` — Short summary (max 120 chars)
+ - `detail: str` — Full observation text
+ - `priority: str` — "high", "medium", or "low"
+ - `files: list[str]` — Relevant file paths
+- **Behavioral rules:**
+ - Observations MUST be written by the Researcher agent during `factory study`
+ - Observations MUST be stored in Markdown with YAML frontmatter
+ - Observations SHOULD reference specific file paths and line ranges where applicable
+ - Observations MUST NOT be deleted by subsequent study runs (append-only)
+ - Observations MAY be consolidated or pruned by the Archivist during archival
+
+### 6.7 HypothesisBudget
+
+- **Type:** Strict Pydantic model with `extra="forbid"`
+- **Fields:**
+ - `min_growth: int` — Minimum hypotheses from growth-oriented categories (default: 2)
+ - `max_new: int` — Maximum new hypotheses generated per cycle (default: 2)
+- **Behavioral rules:**
+ - Budget MUST be enforced by the Strategist when generating new hypotheses
+ - Budget MUST prioritize backlog-first selection before generating new ideas
+ - Growth categories include "exploit", "explore", "combine" (FEEC)
+ - Fix category is exempt from budget constraints (always prioritized)
+
+### 6.8 ResearchTarget
+
+- **Type:** Strict Pydantic model with `extra="forbid"`
+- **Purpose:** Defines the objective and measurement strategy for research mode
+- **Required fields:**
+ - `objective: str` — Natural language description of research goal
+ - `metric: str` — JSON path to extract from result file (e.g., "resolve_rate")
+ - `target: float` — Target metric value to reach
+ - `run_command: str` — Shell command that executes the benchmark
+ - `result_path: str` — Path to JSON result file produced by `run_command`
+ - `result_parser: Literal["json"]` — Parser type (only "json" supported)
+ - `timeout: int` — Maximum seconds for run command (default: 3600)
+- **Behavioral rules:**
+ - Research target MUST be configured in `.factory/config.json` under `research_target` key
+ - Run command MUST write a JSON file to `result_path` upon completion
+ - Metric MUST be extractable from the JSON file via a dot-separated path (e.g., "stats.resolve_rate")
+ - Research mode MUST error if `mutable_surfaces` is empty
+ - Research mode MUST NOT mutate any file in `fixed_surfaces`
+ - Research mode MUST track per-run results in `.factory/research/runs//`
+
+### 6.9 AdversarialConfig
+
+- **Type:** Strict Pydantic model with `extra="forbid"`
+- **Purpose:** Configures GAN-style adversarial eval loops
+- **Required fields:**
+ - `generator: AdversarialComponent` — Generator role config (role="generator", eval_command, metric_name, threshold, scope, timeout)
+ - `discriminator: AdversarialComponent` — Discriminator role config (role="discriminator", eval_command, metric_name, threshold, scope, timeout)
+ - `hysteresis: int` — Consecutive rounds above threshold required to switch roles (default: 3)
+ - `max_rounds: int | None` — Maximum adversarial rounds (None = unlimited)
+ - `convergence_window: int` — Window size for convergence detection (default: 5)
+- **Behavioral rules:**
+ - Generator MUST have `role="generator"` and discriminator MUST have `role="discriminator"`
+ - Adversarial state MUST be persisted at `.factory/adversarial_state.json`
+ - Phase transitions MUST require `hysteresis` consecutive rounds above threshold
+ - Per-role streak counters (`generator_consecutive_above`, `discriminator_consecutive_above`) MUST be maintained
+ - Convergence MUST be detected when metric stabilizes within `convergence_window`
+ - Active role MUST mutate files within its `scope` list only
+
+### 6.10 ProjectEntry
+
+- **Type:** Strict Pydantic model with `extra="forbid"`
+- **File location:** `~/.factory/registry.json`
+- **Required fields:**
+ - `path: Path` — Absolute path to project directory
+ - `name: str` — Project name
+ - `first_seen: str` — ISO 8601 timestamp of registration
+ - `last_active: str` — ISO 8601 timestamp of last experiment
+ - `total_experiments: int` — Cumulative experiment count
+ - `keep_count: int` — Count of KEEP verdicts
+ - `revert_count: int` — Count of REVERT verdicts
+- **Behavioral rules:**
+ - Registry MUST auto-register projects on first `ExperimentStore.begin()` call
+ - Stats MUST update on every `ExperimentStore.finalize()` call
+ - Registry MUST be writable without file locks (single-writer assumption)
+ - Projects MUST be uniquely identified by absolute path (not name)
+
+## 7. State Machines and Lifecycles
+
+### 7.1 Project State Detection
+
+```
+NO_REPO
+ ↓ (git init)
+REPO_INCOMPLETE
+ ↓ (git commit initial files, no .factory/)
+NO_FACTORY
+ ↓ (discovery generates eval_profile.json but not config.json)
+EVALS_PENDING_REVIEW
+ ↓ (user reviews and creates config.json, or factory.md is parsed)
+HAS_FACTORY
+```
+
+**Governing module:** [[graph:factory/state.py]]
+
+**Transitions:**
+- `detect_state()` MUST check NO_REPO before all other states
+- REPO_INCOMPLETE MUST be returned when `git status --porcelain` is non-empty OR the project has no committed files
+- NO_FACTORY MUST be returned when `.git/` exists, working tree is clean, and `.factory/config.json` is missing
+- EVALS_PENDING_REVIEW MUST be returned when `eval_profile.json` exists but `config.json` does not
+- HAS_FACTORY MUST be returned when both `eval_profile.json` and `config.json` exist and are valid
+- State detection MUST execute before mode selection in CEO workflow
+
+### 7.2 Experiment Cycle
+
+```
+IDLE
+ ↓ (CEO spawns Researcher)
+OBSERVING
+ ↓ (observations.md written, CEO spawns Strategist)
+HYPOTHESIZING
+ ↓ (hypothesis selected from backlog or generated, CEO spawns Builder)
+BUILDING
+ ↓ (git commit created, CEO spawns QA)
+REVIEWING
+ ↓ (QA verdict PROCEED, CEO runs eval)
+EVALUATING
+ ↓ (composite score computed)
+DECIDING
+ ↓ (score delta + constraints evaluated)
+FINALIZING ──→ KEEP (git commit preserved) OR REVERT (git reset --hard HEAD~1)
+ ↓
+ARCHIVING (Archivist writes learnings)
+ ↓
+IDLE (repeat)
+```
+
+**Governing module:** [[graph:factory/store.py]]
+
+**Transitions:**
+- Experiment MUST begin with `ExperimentStore.begin()` which acquires `.factory/.lock` file lock
+- Hypothesis MUST be written to `.factory/strategy/current.md` before Builder invocation
+- Builder commit MUST be captured via `git rev-parse HEAD`
+- QA review MUST produce a verdict file at `.factory/reviews/ceo-verdict-qa.md` with PROCEED/REDIRECT/ABORT
+- Eval MUST run both before-commit (on previous HEAD~1) and after-commit (current HEAD)
+- Keep decision MUST check `score_after >= score_before - tolerance` AND zero hard constraint violations
+- Revert MUST execute `git reset --hard HEAD~1` before finalize
+- Finalize MUST write TSV row to `.factory/results.tsv` with exclusive file lock
+- Finalize MUST release `.factory/.lock` file lock
+- Archiving MUST happen after finalize regardless of keep/revert verdict
+
+### 7.3 Adversarial Phase Transitions
+
+```
+GENERATOR_ACTIVE (consecutive_above=0)
+ ↓ (generator metric above threshold for hysteresis rounds)
+GENERATOR_ACTIVE (consecutive_above=hysteresis)
+ ↓ (switch phase)
+DISCRIMINATOR_ACTIVE (consecutive_above=0)
+ ↓ (discriminator metric above threshold for hysteresis rounds)
+DISCRIMINATOR_ACTIVE (consecutive_above=hysteresis)
+ ↓ (switch phase OR convergence detected)
+CONVERGED (terminal state)
+```
+
+**Governing module:** [[graph:factory/adversarial.py]]
+
+**Transitions:**
+- Active role MUST be initialized to "generator" on first run
+- Per-role streak counters MUST increment independently
+- Phase switch MUST only occur when active role's streak reaches `hysteresis` threshold
+- Phase switch MUST reset the newly-active role's streak to 0
+- Convergence detection MUST analyze last `convergence_window` rounds for metric stability
+- Converged flag MUST be set when metric variance is below implementation-defined threshold for `convergence_window` rounds
+- State MUST persist to `.factory/adversarial_state.json` after every round
+- History MUST record every round with timestamp, active role, score, metric name, and switch indicator
+
+## 8. Module Specifications
+
+### 8.1 [[graph:factory/cli.py]]
+
+**Role:** CLI entry point and command dispatcher (Layer 1)
+
+**Layer:** Layer 1 (Python CLI)
+
+**Behavioral specification:**
+
+`cli.py` MUST provide the `main()` function registered as the `factory` console script entry point in `pyproject.toml`. It MUST dispatch subcommands to handler functions via a dictionary lookup. It MUST parse arguments using Python's `argparse` module and pass validated arguments to handlers. It MUST NOT contain business logic — all decision-making MUST be delegated to higher layers.
+
+The module MUST define handler functions for these subcommands: `ceo`, `run`, `tmux`, `agent`, `study`, `diff`, `explain`, `backlog-list`, `backlog-add`, `backlog-remove`, `adversarial-state`, `dashboard`, `export`, `checkpoint`, `resume`, `precheck`, `review`, `config`, `workflow`, `graph`. Each handler MUST be a `cmd_*` function that accepts parsed arguments and returns an integer exit code.
+
+The `ceo` and `run` handlers MUST spawn the CEO agent subprocess via [[graph:factory/agents/runner.py]] with mode detection. The `--loop` flag MUST wrap the CEO invocation in a heartbeat loop with configurable interval and max cycles. The `--mode` flag MUST override state-based mode selection. The `--focus` flag MUST activate targeted mode (single-item execution). The `--refine` flag MUST enter refinement mode (Refiner → Builder → review pipeline).
+
+The `agent` handler MUST spawn a specialist agent subprocess via [[graph:factory/agents/runner.py]] with the specified role and task. The `workflow` handler MUST dispatch to [[graph:factory/workflow/cli.py]] for graph operations. The `graph` handler MUST dispatch to [[graph:factory/graph.py]] for graph extraction and status.
+
+The module MUST write errors to stderr and return non-zero exit codes on failure. It MUST NOT use print() for structured output — structured data MUST be written as JSON to stdout.
+
+**Relationships:**
+- Consumes [[graph:factory/agents/runner.py]] to spawn CEO and specialist agents
+- Consumes [[graph:factory/workflow/cli.py]] for workflow graph operations
+- Consumes [[graph:factory/graph.py]] for knowledge graph extraction
+- Consumes [[graph:factory/user_config.py]] for configuration loading and precedence
+- Consumed by CLI users as the `factory` command
+- Consumed by [[graph:factory/__main__.py]] for `python -m factory` entry point
+
+**What breaks if this changes:**
+- Adding a new subcommand MUST add a `cmd_*` handler and register it in the dispatch dictionary
+- Renaming a subcommand MUST preserve backward compatibility or document the breaking change
+- Changing argument names MUST update all consumers in skill files and documentation
+
+### 8.2 [[graph:factory/__main__.py]]
-A single local durable state substrate is sufficient for core conformance.
+**Role:** Python module entry point for `python -m factory` (Layer 1)
-### 4.4 Work Item
+**Layer:** Layer 1 (Python CLI)
-A `WorkItem` is a unit of work entering the lifecycle.
+**Behavioral specification:**
-Sources MAY include:
+`__main__.py` MUST import and invoke `factory.cli.main()` directly. It MUST NOT define any business logic. Its sole purpose is to enable `python -m factory` as an alias for the `factory` console script.
-- direct CLI prompt
-- focus request
-- backlog item
-- issue
-- ticket
-- research target
+**Relationships:**
+- Consumes [[graph:factory/cli.py:main()]]
+- Consumed by Python's `-m` flag invocation
-Logical fields:
+**What breaks if this changes:**
+- Removing this file MUST preserve the `factory` console script entry point
+- Changing the import path MUST ensure `factory.cli:main` remains callable
-- `work_item_id`
-- `kind`
-- `title`
-- `body`
-- `labels`
-- `repo_ids` (OPTIONAL)
-- `external_refs`
-- `metadata`
+### 8.3 [[graph:factory/state.py]]
-Implementations SHOULD preserve both the normalized work item and enough source
-metadata to trace it back to its origin.
+**Role:** Project state detection (Layer 1)
-### 4.5 Execution Contract
+**Layer:** Layer 1 (Python CLI)
-An `ExecutionContract` defines the scope and policy for one execution attempt or
-cycle.
+**Behavioral specification:**
-Logical fields:
+`state.py` MUST provide a `detect_state(project_path: Path) -> ProjectState` function that returns one of five `ProjectState` enum values by examining the filesystem. It MUST check conditions in this order: NO_REPO (no `.git/`), REPO_INCOMPLETE (uncommitted changes or empty project), NO_FACTORY (no `.factory/config.json`), EVALS_PENDING_REVIEW (`eval_profile.json` exists but not `config.json`), HAS_FACTORY (both exist and are valid).
-- `contract_id`
-- `project_id`
-- `work_item_id`
-- `scope`
-- `mutable_surfaces`
-- `fixed_surfaces`
-- `required_checks`
-- `budget`
-- `expected_evidence`
-- `report_schema` (OPTIONAL)
+The function MUST execute `git status --porcelain` to detect uncommitted changes. It MUST return REPO_INCOMPLETE if the git working tree is dirty OR if there are no committed files (checked via `git log` exit code). It MUST read `.factory/config.json` and `.factory/eval_profile.json` to validate their existence and parseability.
-Worker runtimes MUST receive enough contract information to respect scope,
-surface, and reporting requirements. This information MAY be conveyed through
-structured payloads, prompt content, or other implementation-defined
-mechanisms.
+The function MUST NOT modify any files. It MUST return deterministic results for the same filesystem state. It MUST handle missing directories gracefully (return NO_REPO if `project_path` does not exist).
-### 4.6 Worker Runtime
+**Relationships:**
+- Consumes `ProjectState` enum from [[graph:factory/models.py]]
+- Consumes `FactoryConfig` for config validation from [[graph:factory/models.py]]
+- Consumed by [[graph:factory/agents/runner.py]] for CEO mode selection
+- Consumed by workflow graph gating logic
-A `WorkerRuntime` executes agent work under an execution contract.
+**What breaks if this changes:**
+- Reordering state checks MUST preserve the documented precedence (NO_REPO first, HAS_FACTORY last)
+- Adding a new state MUST update the `ProjectState` enum and all mode selection logic
+- Changing validation logic for HAS_FACTORY MUST ensure config.json and eval_profile.json are still readable
-Examples:
+### 8.4 [[graph:factory/store.py]]
-- local subprocess agent
-- interactive terminal or tmux-backed agent
-- background session agent (fire-and-poll, observable via agent management interface)
-- plugin asset worker
-- managed remote agent
+**Role:** Experiment lifecycle and `.factory/` directory management (Layer 1)
-Runtime selection is implementation-defined. Runtime behavior MUST NOT change the
-meaning of project, work-item, evidence, or decision records.
+**Layer:** Layer 1 (Python CLI)
-### 4.7 Guardrail
+**Behavioral specification:**
-A `Guardrail` is a validation or policy check whose result contributes to a
-decision.
+`store.py` MUST provide an `ExperimentStore` class that manages the `.factory/` directory structure and experiment lifecycle. It MUST implement async methods: `begin()`, `finalize()`, `load_config()`, `write_hypothesis()`, `capture_diff()`, `write_verdict()`.
-Examples:
+The `begin()` method MUST acquire an exclusive file lock on `.factory/.lock` using `filelock.FileLock`. It MUST create `.factory/` and subdirectories if they do not exist. It MUST compute the next experiment ID by reading the last row of `.factory/results.tsv` and incrementing by 1. It MUST create `.factory/experiments//` directory. It MUST auto-register the project in `~/.factory/registry.json` via [[graph:factory/registry.py]] if not already registered.
-- tests
-- lint
-- type checks
-- eval metrics
-- CI status
-- code review
-- security review
-- scope or immutability checks
-- leakage checks
+The `write_hypothesis()` method MUST write the hypothesis text to `.factory/strategy/current.md`. It MUST overwrite any existing content.
-Guardrail outcomes SHOULD be recorded as evidence.
+The `capture_diff()` method MUST execute `git diff HEAD~1 HEAD` and write output to `.factory/experiments//changes.diff`. It MUST handle the case where there is no previous commit (empty diff).
-### 4.8 Evidence
+The `finalize()` method MUST write an `ExperimentRecord` to `.factory/results.tsv` with tab separators. It MUST acquire an exclusive file lock before writing the TSV row. It MUST copy eval results to `.factory/experiments//eval_before.json` and `eval_after.json`. It MUST write the verdict to `.factory/experiments//verdict.json`. It MUST release the `.factory/.lock` file lock. It MUST update project stats in `~/.factory/registry.json` via [[graph:factory/registry.py]].
-`Evidence` is immutable or append-only support for a lifecycle decision.
+The module MUST handle FileNotFoundError gracefully when `.factory/` does not exist. It MUST serialize all Pydantic models to JSON with no loss of fidelity. It MUST NOT delete experiment directories after finalize.
-Examples:
+**Relationships:**
+- Consumes `ExperimentRecord` from [[graph:factory/models.py]]
+- Consumes `FactoryConfig` from [[graph:factory/models.py]]
+- Consumes [[graph:factory/registry.py]] for global project registration
+- Consumed by CEO agent for experiment lifecycle orchestration
+- Consumed by [[graph:factory/eval/runner.py]] to load config
-- diffs
-- logs
-- eval results
-- review findings
-- CI status
-- generated reports
-- artifacts
+**What breaks if this changes:**
+- Changing TSV column order MUST update all parsers in [[graph:factory/analysis.py]] and [[graph:factory/insights.py]]
+- Renaming `.factory/` subdirectories MUST update all path references in other modules
+- Changing file lock behavior MUST ensure no concurrent writes to shared files
-Evidence SHOULD include project identity and MAY include repository identity,
-work-item identity, runtime identity, and external references.
+### 8.5 [[graph:factory/models.py]]
-### 4.9 Decision
+**Role:** Domain model definitions (all layers)
-A `Decision` is the lifecycle outcome accepted from evidence and guardrail
-results.
+**Layer:** Cross-cutting (used by all layers)
-Common decision kinds include:
+**Behavioral specification:**
-- `keep`
-- `revert`
-- `park`
-- `retry`
-- `escalate`
-- `error`
+`models.py` MUST define all Pydantic v2 models with `ConfigDict(strict=True, extra="forbid")`. It MUST export these primary types: `ProjectState`, `FactoryConfig`, `EvalProfile`, `EvalDimension`, `EvalResult`, `CompositeScore`, `ExperimentRecord`, `Observation`, `HypothesisBudget`, `ResearchTarget`, `AdversarialConfig`, `AdversarialComponent`, `AdversarialState`, `AdversarialPhaseRecord`, `ProjectEntry`, `ProjectRegistry`, `InnerLoopConfig`, `OuterLoopConfig`, `RunResult`, `RunStatus`, `HardConstraint`, `ProjectEvalDimension`, `EvalWeights`, `TierWeights`, `ParallelConfig`, `CostBudgetConfig`, `AggregateMethod`.
-Implementations MAY expose additional publication or escalation outcomes.
+All models MUST be serializable to JSON via `.model_dump(mode="json")`. All models MUST validate input via `.model_validate()` with strict type checking. All enum fields MUST use `Literal` types or `str, Enum` subclasses.
-Decisions MUST include rationale and SHOULD reference supporting evidence.
+The module MUST define a `Notifier` protocol with async methods for sending notifications (`send_message`, `send_experiment_result`, `send_verdict`).
-### 4.10 Memory
+The module MUST NOT import any other factory modules (to avoid circular dependencies). It MUST only import from standard library and Pydantic.
-`Memory` is durable knowledge used by future cycles.
+**Relationships:**
+- Consumed by ALL factory modules for type definitions
+- Consumed by [[graph:factory/store.py]] for experiment serialization
+- Consumed by [[graph:factory/eval/runner.py]] for eval result validation
+- Consumed by [[graph:factory/state.py]] for config validation
-Examples:
+**What breaks if this changes:**
+- Adding a field to `ExperimentRecord` MUST update TSV serialization in [[graph:factory/store.py]]
+- Removing a field from `FactoryConfig` MUST update all config parsers and generators
+- Changing a Literal type MUST update all code that pattern-matches on that field
-- experiment archives
-- observations
-- playbook rules
-- reinforced or contradicted lessons
-- handoff snapshots
-- performance reports
+## 9. Shared Contracts
-Memory records SHOULD distinguish durable learnings from reconstructable runtime
-state.
+### 9.1 Eval JSON Output Schema
-### 4.11 Specification
+**Definition:**
+All eval commands MUST produce JSON output on stdout in this format:
-A `Specification` is a structured, normative description of the project's
-identity, goals, technical stack, architecture, and requirements.
+```json
+{
+ "results": [
+ {
+ "name": "tests",
+ "score": 0.85,
+ "weight": 0.4,
+ "passed": true,
+ "details": "42 passed, 0 failed"
+ }
+ ]
+}
+```
+
+**Behavioral rules:**
+- `results` MUST be a JSON array
+- Each element MUST have `name` (string), `score` (float 0.0-1.0), `weight` (float), `passed` (boolean), `details` (string)
+- Total weight across all results SHOULD sum to 1.0 (normalized by eval runner)
+- Eval runner MUST parse this JSON via [[graph:factory/eval/runner.py]]
+- Generated `eval/score.py` MUST produce this format
+- Custom project evals (`FactoryConfig.project_eval`) MUST also produce this format
+
+**Consumers:**
+- [[graph:factory/eval/runner.py]]
+- [[graph:factory/discovery/generate.py]]
+- CEO agent for keep/revert decision
+- Dashboard for live eval streaming
+
+**Migration rules:**
+- Adding new fields to the schema SHOULD be backward compatible (extra fields ignored by parser)
+- Renaming fields MUST provide a migration path or parallel support
+- Removing fields MUST ensure no consumers depend on them
+
+### 9.2 Agent Output Capture Schema
-Resolution order:
+**Definition:**
+All specialist agents MUST write their final output to `.factory/reviews/{role}-latest.md`. The CEO MUST read this file to determine next steps.
-1. A committed specification at the project root (e.g., `SPEC.md`) is
- authoritative.
-2. If no committed specification exists, the Project Resolver SHOULD generate
- one from introspected project metadata and place it in the project's durable
- state directory (e.g., `.factory/SPEC.md`).
-3. A generated specification captures discovered state — it uses descriptive
- language for what exists and RFC 2119 normative language only for the
- standard boilerplate.
+**Behavioral rules:**
+- Output MUST be Markdown format (plain text, no JSON)
+- Output MUST include clear section headers if multi-part (e.g., "## Observations", "## Recommendations")
+- Output MUST NOT include interactive prompts or requests for user input
+- Output MUST be deterministic for the same input state
+- Runner MUST truncate output after 100KB to prevent context overflow
-Rules:
+**Consumers:**
+- CEO agent (reads all `{role}-latest.md` files)
+- Dashboard (streams agent output)
+- Archivist agent (consolidates into archive)
+- Workflow executor (passes to next node)
-- Implementations MUST NOT overwrite a committed specification with a generated
- one.
-- A generated specification SHOULD be updated when discovery re-runs.
-- The lifecycle coordinator and contract builder SHOULD reference the
- specification when deriving execution contracts and validating scope.
-- When a specification exists, plan outputs SHOULD include a specification diff
- describing which requirements are added, modified, or removed.
+**Migration rules:**
+- Changing output location MUST update all file readers
+- Changing output format (e.g., to JSON) MUST update CEO parsing logic
-### 4.12 Deployment Profile
+### 9.3 Verdict File Schema
-A `DeploymentProfile` is a named assembly of component implementations.
+**Definition:**
+QA agent MUST write a verdict file to `.factory/reviews/ceo-verdict-qa.md` in this format:
-Logical fields:
+```markdown
+# QA Verdict
-- `name`
-- `surface`
-- `runtime`
-- `state_backend`
-- `guardrails`
-- `output_surfaces`
-- `policy_sources`
+**Verdict:** PROCEED | REDIRECT | ABORT
-The `cli-local` deployment profile is the primary product surface for this
-specification. Other profiles MAY expose different surfaces, but SHOULD
-preserve the lifecycle semantics of this specification.
+## Rationale
+
+
+
+## Blockers
+
+
+```
-### 4.13 Shared State Records (OPTIONAL)
+**Behavioral rules:**
+- Verdict MUST be one of three literal strings: "PROCEED", "REDIRECT", "ABORT"
+- Rationale section MUST explain the decision
+- Blockers section MUST list specific issues or state "None"
+- CEO MUST NOT proceed to eval unless verdict is "PROCEED"
+- REDIRECT verdict MUST include specific guidance in rationale
+- ABORT verdict MUST trigger immediate cycle termination and revert
-Implementations that support shared, externally reconciled, or multi-actor
-state MAY represent project state as `StateRecord`s.
+**Consumers:**
+- CEO agent (reads verdict before eval)
+- Experiment record (verdict stored in `verdict.json`)
+- Performance report (verdict counts aggregated)
-Logical fields:
+**Migration rules:**
+- Adding new verdict types MUST update CEO logic and all verdict parsers
+- Changing verdict file location MUST update all readers
-- `id`
-- `kind`
-- `project_id`
-- `repo_id` (OPTIONAL)
-- `source`
-- `actor`
-- `revision`
-- `parent_ids`
-- `created_at`
-- `updated_at`
-- `payload`
+## 10. Configuration Specification
-A `StateConflict` records an unresolved merge problem when such
-implementations detect one.
+### 10.1 Configuration Sources and Precedence
-Implementations that do not expose shared-state semantics do not need to model
-state records or conflicts as first-class domain objects.
+re:factory uses a five-tier configuration precedence chain (highest to lowest priority):
-## 5. Lifecycle Specification
+1. **CLI flag** — e.g., `--runner codex`, `--model gpt-5.4`
+2. **Environment variable** — e.g., `FACTORY_RUNNER=codex`, `ANTHROPIC_API_KEY=...`
+3. **Profile credential** — from `~/.factory/config.toml` `[credentials.]` section (loaded via `--profile `)
+4. **Config.toml default** — from `~/.factory/config.toml` `[defaults]` section
+5. **Hardcoded default** — built into the code (e.g., `runner="claude"`, `model=None`)
-The lifecycle is:
+Credential profiles inject all keys from `[credentials.]` into the subprocess environment. This enables per-project or per-runner authentication without polluting the global environment.
-```text
-Intake → Scope → Dispatch → Execute → Validate → Decide → Publish → Learn → Resume
+### 10.2 Core Config Fields
+
+#### User Config (`~/.factory/config.toml`)
+
+```toml
+[defaults]
+runner = "claude" # Default runner: "claude", "bob", "codex", "opencode"
+model = "" # Default model (empty = runner's default)
+projects_dir = "~/factory-projects" # Default project storage
+
+[credentials.vertex] # Example credential profile
+FACTORY_RUNNER = "claude"
+ANTHROPIC_API_KEY = "sk-ant-..."
+
+[credentials.codex]
+FACTORY_RUNNER = "codex"
+CODEX_API_KEY = "..."
```
-### 5.1 Intake
+#### Project Config (`.factory/config.json`)
+
+See Section 6.2 for full `FactoryConfig` schema. Key fields:
+
+- `goal` (string, REQUIRED) — Natural language improvement objective
+- `eval_command` (string, REQUIRED) — Shell command that produces eval JSON
+- `eval_threshold` (float, REQUIRED) — Minimum score for keep (0.0 to 1.0)
+- `scope` (list[string], REQUIRED) — File paths or globs defining mutation scope
+- `guards` (list[string], REQUIRED) — Natural language constraints
+- `hypothesis_budget` (object, OPTIONAL) — Controls hypothesis selection
+
+### 10.3 Validation and Error Surface
+
+**User config validation:**
+- `runner` MUST be one of: "claude", "bob", "codex", "opencode"
+- `projects_dir` MUST expand to a valid absolute path (tilde expansion allowed)
+- Profile sections MUST have unique names
+- Credential keys MUST be valid environment variable names (uppercase, underscores)
-The system accepts work from one or more work-item sources and normalizes it into
-a work item.
+**Project config validation:**
+- All REQUIRED fields MUST be present (enforced by Pydantic `extra="forbid"`)
+- `eval_threshold` MUST be between 0.0 and 1.0
+- `eval_command` MUST be a non-empty string
+- Tier weights (`eval_weights.hygiene`, `.growth`, `.project`) MUST sum to 1.0
+- `test_timeout` MUST be >= 1 second
+- `parallel.parallel_hypotheses` MUST be between 1 and 8
-### 5.2 Scope
+**Error behavior:**
+- Missing REQUIRED fields in project config MUST raise Pydantic validation error with field name
+- Invalid TOML syntax in user config MUST raise parse error with line number
+- Unknown credential profile MUST error with "Profile '' not found in config.toml"
+- Locked config file (file lock timeout) MUST log warning and skip write (non-fatal)
-The system binds the work item to a project context and derives an execution
-contract. When a project specification exists (committed or generated), the
-scoping phase SHOULD use it to inform contract derivation and scope
-validation.
+## 11. Entry Points
-### 5.3 Dispatch
+| Type | Module | Detail |
+|------|--------|--------|
+| CLI | [[graph:factory/cli.py]] | `factory` command (dispatches to subcommands) |
+| Python Module | [[graph:factory/__main__.py]] | `python -m factory` (alias for `factory` CLI) |
+| MCP Server | [[graph:factory/mcp_server.py]] | `factory mcp` (starts MCP server on stdio) |
+| Dashboard | [[graph:factory/dashboard.py]] | `factory dashboard` (starts FastAPI server on :8420) |
-The system selects a worker runtime and starts an execution attempt.
-Dispatch MUST preserve enough state to support observability and recovery.
+## 12. Failure Model and Recovery
-Dispatch modes include:
+### 12.1 Failure Classes
-- **synchronous** — the caller blocks until the worker completes (default)
-- **interactive** — the worker runs in a user-facing terminal session
-- **background** — the worker is launched as a detached session; the caller
- polls for completion and collects output when the session finishes
+**1. Agent Timeout (non-fatal):**
+- Triggered when an agent subprocess exceeds timeout (default 600s for QA, 300s for others)
+- Recovery: Kill subprocess, log error to `.factory/events.jsonl`, retry up to 2 times
+- If retries exhausted, abort cycle and skip to archival
-Dispatch mode MAY be scoped independently per lifecycle tier. For example, a
-coordinator MAY run synchronously while its delegated workers dispatch in
-background mode, allowing the operator to observe the coordinator while
-workers remain visible through an agent management interface.
+**2. Eval Failure (non-fatal):**
+- Triggered when eval command exits non-zero or produces malformed JSON
+- Recovery: Return zero-score `CompositeScore` with error details, compare against previous score, revert if delta negative
-Dispatch mode is a runtime concern. It MUST NOT change the semantics of the
-execution contract, evidence records, or decision lifecycle.
+**3. Hard Constraint Violation (mandatory revert):**
+- Triggered when any `FactoryConfig.hard_constraints` command exits non-zero
+- Recovery: Immediate revert via `git reset --hard HEAD~1`, log violation, skip archival
-### 5.4 Execute
+**4. Git Operation Failure (fatal):**
+- Triggered when git commit, reset, or diff fails (corrupted repository)
+- Recovery: Log error, emit event to `.factory/events.jsonl`, exit with code 1
-The worker runtime performs the scoped work. It SHOULD emit logs, status, and
-artifacts sufficient for validation and review.
+**5. Config Parse Error (fatal):**
+- Triggered when `.factory/config.json` or `eval_profile.json` is malformed
+- Recovery: Print validation error, exit with code 1, user MUST fix config manually
-### 5.5 Validate
+**6. File Lock Timeout (non-fatal):**
+- Triggered when acquiring `.factory/.lock` times out (concurrent writes detected)
+- Recovery: Log warning, skip write, return non-zero exit code
-Guardrails evaluate the produced state, artifacts, or external checks.
-Validation failures MUST be visible to the decision step.
+### 12.2 Recovery Behavior
-### 5.6 Decide
+**Agent crashes:** The CEO MUST capture stderr from the agent subprocess, log it to `.factory/events.jsonl`, and retry up to 2 times. If retries are exhausted, the CEO MUST abort the current cycle, log the failure, and skip to archival. The CEO MUST NOT silently continue after an agent crash.
-The lifecycle coordinator records an explicit decision. Decisions SHOULD be
-derived from evidence and guardrail outcomes.
+**Eval failures:** If the eval command times out or exits non-zero, the eval runner MUST return a zero-score `CompositeScore` with error details in the `details` field. The CEO MUST compare this against the previous score and MUST revert if the delta is negative.
-### 5.7 Publish
+**Hard constraint violations:** If any `FactoryConfig.hard_constraints` check fails (exit code non-zero), the CEO MUST immediately revert the commit via `git reset --hard HEAD~1` without running the eval. The CEO MUST log the violation to `.factory/events.jsonl` and write a REVERT verdict to the experiment record.
-If an implementation supports publishing, it MAY update external systems such as
-branches, PRs, comments, ticket state, or managed-state records. Publishing
-behavior is implementation-defined.
+**Stuck detection:** If the CEO detects 3+ consecutive reverts all in the same FEEC category, it MUST escalate by switching to a different category or entering meta mode. The CEO MUST NOT continue generating hypotheses in the same stuck category.
-### 5.8 Learn
+### 12.3 Restart and Resume Semantics
-The memory system records durable learnings, observations, and reports. Memory
-SHOULD be usable by future work-item selection, scoping, and validation.
+**Crash recovery:** The CEO MUST save checkpoints to `.factory/checkpoint.json` after each major step (observation, hypothesis, build, review, eval, verdict). On restart, the CEO MUST load the checkpoint and resume from the last saved step. Checkpoints MUST include: current mode, active hypothesis ID, agent history, cycle count, timestamp.
-### 5.9 Resume
+**Checkpoint validation:** Before resuming from a checkpoint, the CEO MUST validate: 1) timestamp is not too old (< 24 hours), 2) mode is valid, 3) hypothesis ID exists in `.factory/results.tsv`, 4) git state is clean. If validation fails, the CEO MUST discard the checkpoint and start a fresh cycle.
-The system SHOULD be able to reconstruct useful lifecycle state from durable
-records, evidence, external bindings, and materialized views. Exact in-memory
-runtime state is implementation-defined.
+**Heartbeat loop recovery:** If the heartbeat loop is interrupted (SIGINT, SIGTERM), it MUST write a checkpoint before exiting. On restart with `--loop`, it MUST resume from the checkpoint if valid, otherwise start a new cycle.
-## 6. Deployment Profile Specification
+**State isolation:** Each experiment cycle MUST acquire an exclusive file lock on `.factory/.lock` at the start and release it at the end. This prevents concurrent writes from multiple processes. If a lock cannot be acquired within 60 seconds, the process MUST log an error and exit.
-Deployment profiles bundle component implementations.
+## 13. Security and Safety
-### 6.1 `cli-local` Profile
+### 13.1 Trust Boundaries
-The `cli-local` profile is the primary compatibility surface.
+**Untrusted inputs:**
+- User-provided hypotheses (from CLI `--focus` or Strategist output)
+- External web search results (from WebSearch/WebFetch tools)
+- GitHub/GitLab issue content (from `gh`/`glab` CLI)
+- Subprocess stdout/stderr (from eval commands and agent outputs)
+- Git commit messages and diffs
-It consists of:
+**Trusted inputs:**
+- Factory default prompts at `factory/agents/prompts/`
+- Evolved playbooks at `~/.factory/playbooks/` (trusted because written by factory itself)
+- Project config at `.factory/config.json` (trusted after Pydantic validation)
+- Eval profile at `.factory/eval_profile.json` (trusted after Pydantic validation)
+
+**Validation at boundaries:**
+- All JSON output from eval commands MUST be parsed via Pydantic models with strict validation
+- All TOML/YAML files MUST be parsed with error handling
+- All subprocess commands MUST be executed with timeout enforcement
+- All file paths MUST be validated to prevent traversal outside project directory
+
+**Subprocess isolation:**
+- Agent subprocesses MUST run in the project directory, not the factory codebase directory
+- Agent subprocesses MUST NOT inherit sensitive environment variables unless explicitly passed
+- Subprocess stdout/stderr MUST be captured separately to prevent output interleaving
+- Subprocess timeouts MUST be enforced to prevent infinite hangs
+
+### 13.2 Filesystem Safety Invariants
+
+**Write restrictions:**
+- The factory MUST only write to `.factory/` subdirectory within the project
+- The factory MUST only write to `eval/` subdirectory within the project (eval script generation)
+- The factory MUST only write to `~/.factory/` for global state (registry, playbooks, config)
+- The factory MUST NOT write to any other directories without explicit user approval
+
+**Path traversal prevention:**
+- All file paths MUST be resolved to absolute paths before use
+- All file paths MUST be checked to ensure they are within the project directory or `~/.factory/`
+- Symlink attacks MUST be prevented by resolving symlinks before validation
+
+**Git safety:**
+- The factory MUST NOT force-push to remote branches
+- The factory MUST NOT delete remote branches
+- The factory MUST NOT modify git config (user.name, user.email, etc.)
+- The factory MUST only commit to local branches (no automatic push)
+
+**Clean PR Mode:**
+- Clean PR Mode MUST NOT delete files outside `.factory/` except those matching `clean_pr_include` globs
+- Clean PR Mode MUST respect `clean_pr_exclude` globs to preserve essential config
+- Clean PR Mode MUST only run when explicitly enabled via `FactoryConfig.clean_pr`
+
+### 13.3 Secret Handling
+
+**Secret sources:**
+- Environment variables (`ANTHROPIC_API_KEY`, `CODEX_API_KEY`, `FACTORY_RUNNER`, etc.)
+- Credential profiles in `~/.factory/config.toml`
+- `.env` files in the project directory (if present)
+
+**Secret protection:**
+- Secrets MUST be masked when displayed via `factory config show` (unless `--reveal` flag is passed)
+- Secrets MUST NOT be logged to `.factory/events.jsonl`
+- Secrets MUST NOT be included in experiment diffs or verdicts
+- Secrets MUST NOT be passed to untrusted subprocesses
+
+**Leakage detection:**
+- Before committing, the factory SHOULD check for common secret patterns (API keys, tokens)
+- If secrets are detected in `.factory/` files, the factory MUST warn the user before committing
+- `.factory/` SHOULD be added to `.gitignore` by the discovery workflow to prevent accidental commits
+
+**Secret injection:**
+- Credential profiles MUST inject secrets into subprocess environment only for the specific subprocess
+- Injected secrets MUST NOT persist in the parent process environment
+- Subprocess environment MUST be isolated from the factory's own environment
+
+## 14. Test and Validation Matrix
+
+### 14.1 Core Conformance Criteria
+
+A conforming re:factory implementation MUST satisfy these criteria:
+
+1. **State detection accuracy**: `detect_state()` MUST correctly identify all five `ProjectState` values for standard project layouts
+2. **Eval execution**: `run_eval()` MUST execute eval commands, parse JSON output, and compute weighted composite scores
+3. **Experiment lifecycle**: `ExperimentStore` MUST acquire file locks, write TSV rows, store artifacts, and release locks
+4. **FEEC classification**: `classify_feec()` MUST correctly classify hypotheses into fix/exploit/explore/combine categories
+5. **Workflow graph traversal**: `WorkflowExecutor` MUST execute all node types (Agent, Fn, Gate, Fork, Join, Study) in topological order
+6. **Agent subprocess spawning**: `spawn_agent()` MUST resolve prompts with correct precedence (project > playbook > default)
+7. **Keep/revert decision**: CEO MUST keep commits with positive score delta and zero constraint violations, revert otherwise
+8. **Adversarial phase transitions**: `update_state()` MUST implement hysteresis-based phase switching and per-role streak counters
+9. **Registry auto-registration**: `register_project()` MUST be called on first `begin()` and stats MUST update on every `finalize()`
+10. **Config precedence**: `resolve()` MUST implement five-tier precedence (CLI > env > profile > config > default)
+
+### 14.2 Test Coverage by Subsystem
+
+**State detection (`tests/test_state.py`):**
+- Test all five state transitions (no_repo → repo_incomplete → no_factory → evals_pending_review → has_factory)
+- Test dirty working tree detection via `git status --porcelain`
+- Test empty repository detection (no commits)
+- Test missing `.factory/` directory
+- Test malformed config.json and eval_profile.json
+
+**Eval runner (`tests/test_eval.py`):**
+- Test JSON parsing from eval command stdout
+- Test weight normalization across tiers (hygiene, growth, project)
+- Test within-tier weight overrides from `TierWeights`
+- Test guard violation detection
+- Test subprocess timeout handling
+- Test malformed JSON output
+
+**Experiment store (`tests/test_store.py`):**
+- Test file lock acquisition and release
+- Test TSV append-only semantics
+- Test experiment ID auto-increment
+- Test artifact storage (diff, eval results, verdict)
+- Test concurrent write prevention (lock timeout)
+- Test registry auto-registration on `begin()`
+- Test stat updates on `finalize()`
+
+**FEEC strategy (`tests/test_strategy.py`):**
+- Test hypothesis classification for all four categories
+- Test stuck detection (3+ consecutive same-category reverts)
+- Test keyword matching accuracy
+
+**Workflow executor (`tests/test_workflow.py`):**
+- Test all node types (Agent, Fn, Gate, Fork, Join, Study)
+- Test topological order traversal
+- Test context accumulation across nodes
+- Test error handling (optional vs critical nodes)
+
+**Agent runner (`tests/test_agents.py`):**
+- Test prompt resolution precedence (project > playbook > default)
+- Test playbook injection
+- Test subprocess spawning with timeout
+- Test output capture (stdout/stderr separation)
+- Test event emission to `.factory/events.jsonl`
+
+**Adversarial state machine (`tests/test_adversarial.py`):**
+- Test phase transition with hysteresis
+- Test per-role streak counters
+- Test convergence detection
+- Test state persistence to JSON
+
+**User config (`tests/test_user_config.py`):**
+- Test five-tier precedence resolution
+- Test credential profile injection
+- Test secret masking in `show_config()`
+- Test TOML parsing errors
+
+**Graph extraction (`tests/test_graph.py`):**
+- Test subprocess invocation of `graphifyy`
+- Test output path determinism
+- Test incremental update mode
+- Test status checks
+
+## 15. Extension Points
+
+### 15.1 Runner Registration
+
+**Location:** `factory/runners/` directory
+
+**Mechanism:** Each runner is a Python module (e.g., [[graph:factory/runners/claude.py]], [[graph:factory/runners/bob.py]], [[graph:factory/runners/codex.py]]) that implements the `spawn()` function:
+
+```python
+def spawn(
+ role: str,
+ task: str,
+ project_path: Path,
+ model: str | None = None,
+ timeout: int = 600
+) -> tuple[int, str, str]:
+ """Spawn an agent subprocess and return (exit_code, stdout, stderr)."""
+```
-- CLI command surface
-- local worker runtime (supports multiple dispatch modes as implementation-defined options)
-- local project state backend
-- local guardrail providers
-- implementation-defined local output surfaces
+**Registration:** The runner name is the module filename (without `.py`). The factory dispatches to runners via dynamic import: `importlib.import_module(f"factory.runners.{runner}")`.
-### 6.2 Extension Profiles
+**Requirements:**
+- Runner MUST implement `spawn()` function with the signature above
+- Runner MUST capture stdout and stderr separately
+- Runner MUST enforce timeout (kill subprocess if exceeded)
+- Runner MUST return non-zero exit code on failure
+- Runner MUST inject credential environment variables from user config
-Other deployment profiles MAY exist. This specification does not require any
-fixed catalog beyond `cli-local`.
+**Extension process:**
+1. Create `factory/runners/.py`
+2. Implement `spawn()` function
+3. Add runner to `FACTORY_RUNNER` environment variable or `~/.factory/config.toml`
+4. No code changes in [[graph:factory/agents/runner.py]] required (dynamic import)
-Extension profiles MUST document selected component implementations and
-SHOULD preserve the lifecycle semantics in this specification.
+### 15.2 Language Evaluator Plugins
-## 7. Shared-State Semantics (OPTIONAL)
+**Location:** `factory/discovery/evaluators/` directory (future extension point)
-This section applies only to implementations that support shared, externally
-reconciled, or multi-actor state.
+**Mechanism:** Each evaluator is a Python module (e.g., `factory/discovery/evaluators/python.py`) that implements `build_profile(project_path: Path) -> EvalProfile`.
-State backends SHOULD prefer append-only events and immutable evidence over
-destructive updates.
+**Current implementation:** Hard-coded in [[graph:factory/discovery/profile.py]]. This SHOULD be refactored to a plugin registry.
-Materialized views SHOULD be rebuildable from durable records.
+**Requirements:**
+- Evaluator MUST detect language-specific tools (test runners, linters, type checkers)
+- Evaluator MUST return a valid `EvalProfile` with at least one dimension
+- Evaluator MUST set dimension weights that sum to 1.0
-Record kinds MAY define different merge policies.
+### 15.3 Workflow Registration
-When an implementation supports multi-user state, it MUST represent unresolved
-important conflicts explicitly rather than silently applying last-writer-wins.
+**Location:** [[graph:factory/workflow/definitions.py]]
-## 8. Guardrails and Trust Policy
+**Mechanism:** Workflows are registered in the `WORKFLOW_REGISTRY` dict:
-Each implementation MUST document its trust and safety posture.
+```python
+WORKFLOW_REGISTRY = {
+ "build": build_workflow,
+ "improve": improve_workflow,
+ # ...
+}
+```
-If an implementation defines additional deployment profiles, each profile MUST
-document any trust or policy differences that affect execution.
+**Extension process:**
+1. Define a new `Workflow` object with nodes and edges
+2. Add it to `WORKFLOW_REGISTRY` with a unique name
+3. Export a `SKILL.md` file via `factory workflow export-skills`
+4. CEO agent can now select the workflow via `--mode `
-Implementation-defined policy areas include:
+### 15.4 Notification Adapters
-- sandboxing
-- approval prompts
-- network access
-- external writes
-- merge authority
-- credential handling
-- destructive filesystem operations
+**Location:** [[graph:factory/notify/]] directory
-Guardrails SHOULD be explicit, observable, and traceable to evidence.
+**Mechanism:** Each adapter implements the `Notifier` protocol:
-## 9. Conformance
+```python
+class Notifier(Protocol):
+ async def send_message(self, message: str) -> None: ...
+ async def send_experiment_result(self, record: ExperimentRecord) -> None: ...
+ async def send_verdict(self, verdict: str, rationale: str) -> None: ...
+```
-### 9.1 Core Conformance
+**Current implementations:** Telegram ([[graph:factory/notify/telegram.py]])
+
+**Extension process:**
+1. Create `factory/notify/.py`
+2. Implement `Notifier` protocol
+3. Instantiate adapter in CEO agent or workflow executor
+4. No registry required (instantiated directly by caller)
+
+## 16. Implementation Checklist
+
+### 16.1 Required for Conformance
+
+- [ ] All five `ProjectState` values detected correctly by `detect_state()`
+- [ ] Eval runner parses JSON output and computes weighted composite scores
+- [ ] Experiment store acquires file locks, writes TSV rows, and releases locks
+- [ ] FEEC classification assigns correct categories to hypotheses
+- [ ] Workflow executor traverses all node types in topological order
+- [ ] Agent runner resolves prompts with correct precedence (project > playbook > default)
+- [ ] CEO makes keep/revert decisions based on score delta and constraint violations
+- [ ] Adversarial state machine implements hysteresis-based phase transitions
+- [ ] Registry auto-registers projects on first `begin()` and updates stats on `finalize()`
+- [ ] User config resolves values with five-tier precedence (CLI > env > profile > config > default)
+- [ ] Sacred Rules enforced: always study first, always commit before eval, never skip eval, always revert on score drop, always archive learnings
+- [ ] Hard constraints checked before keep decision
+- [ ] Clean PR Mode respects include/exclude globs
+- [ ] Secrets masked in config display unless `--reveal` flag passed
+- [ ] All Pydantic models validate with `strict=True, extra="forbid"`
+
+### 16.2 Recommended Extensions
+
+- [ ] Add support for new languages via evaluator plugins
+- [ ] Add support for new runners (e.g., Gemini Code Assist, GitHub Copilot CLI)
+- [ ] Add support for new notification adapters (Slack, Discord, email)
+- [ ] Add parallel hypothesis execution (`FactoryConfig.parallel`)
+- [ ] Add inner/outer loop plateau detection for research mode
+- [ ] Add cost budget enforcement for research mode
+- [ ] Add real-time telemetry streaming via Langfuse
+- [ ] Add web UI for experiment history exploration
+- [ ] Add cross-project insights dashboard
+- [ ] Add automated playbook evolution (ACE) for all agent roles
+
+## Appendix A. Reference Algorithms
+
+### A.1 State Detection
-A conforming implementation MUST:
+```
+function detect_state(project_path):
+ if not exists(project_path / ".git"):
+ return NO_REPO
+
+ git_status = exec("git status --porcelain", cwd=project_path)
+ if git_status != "":
+ return REPO_INCOMPLETE
+
+ git_log = exec("git log", cwd=project_path, check=False)
+ if git_log.returncode != 0:
+ return REPO_INCOMPLETE
+
+ config_path = project_path / ".factory/config.json"
+ profile_path = project_path / ".factory/eval_profile.json"
+
+ if not exists(config_path):
+ if exists(profile_path):
+ return EVALS_PENDING_REVIEW
+ else:
+ return NO_FACTORY
+
+ # Both exist
+ try:
+ config = FactoryConfig.parse_file(config_path)
+ profile = EvalProfile.parse_file(profile_path)
+ except ValidationError:
+ return NO_FACTORY
+
+ return HAS_FACTORY
+```
-- represent work as work items
-- bind work to project context
-- distinguish project-level lifecycle state from checkout or runtime state
-- execute work under an execution contract
-- record evidence for validation and decisions
-- run or consume guardrail outcomes before accepting decisions
-- record explicit decisions
-- preserve durable memory or reports
-- document selected deployment profile components
-- document implementation-defined trust and safety policy
+### A.2 Composite Score Computation
-### 9.2 Extension Conformance
+```
+function compute_composite_score(results, eval_weights, hygiene_weights, growth_weights):
+ # Separate results by tier
+ hygiene = [r for r in results if r.name in HYGIENE_DIMS]
+ growth = [r for r in results if r.name in GROWTH_DIMS]
+ project = [r for r in results if r.name not in (HYGIENE_DIMS + GROWTH_DIMS)]
+
+ # Apply within-tier weight overrides
+ if hygiene_weights:
+ for r in hygiene:
+ if r.name in hygiene_weights:
+ r.weight = hygiene_weights[r.name]
+
+ if growth_weights:
+ for r in growth:
+ if r.name in growth_weights:
+ r.weight = growth_weights[r.name]
+
+ # Normalize weights within each tier
+ normalize_weights(hygiene)
+ normalize_weights(growth)
+ normalize_weights(project)
+
+ # Compute tier scores
+ hygiene_score = sum(r.score * r.weight for r in hygiene)
+ growth_score = sum(r.score * r.weight for r in growth)
+ project_score = sum(r.score * r.weight for r in project) if project else 0
+
+ # Compute weighted composite
+ total = (
+ hygiene_score * eval_weights.hygiene +
+ growth_score * eval_weights.growth +
+ project_score * eval_weights.project
+ )
+
+ return CompositeScore(total=total, results=results, guard_violations=[])
+```
-An implementation that supports multi-repo projects SHOULD:
+### A.3 FEEC Classification
-- identify repository bindings by stable IDs
-- attach repo-specific evidence to the relevant binding
-- keep project-level decisions and memory separate from checkout state
+```
+function classify_feec(hypothesis):
+ hypothesis_lower = hypothesis.lower()
+
+ FIX_KEYWORDS = ["fix", "bug", "error", "crash", "regression", "broken", "incorrect", "issue"]
+ EXPLOIT_KEYWORDS = ["refactor", "optimize", "improve", "enhance", "streamline", "consolidate"]
+ EXPLORE_KEYWORDS = ["add", "new", "implement", "support", "enable", "introduce"]
+ COMBINE_KEYWORDS = ["integrate", "merge", "combine", "unify", "bridge"]
+
+ for kw in FIX_KEYWORDS:
+ if kw in hypothesis_lower:
+ return "fix"
+
+ for kw in EXPLOIT_KEYWORDS:
+ if kw in hypothesis_lower:
+ return "exploit"
+
+ for kw in EXPLORE_KEYWORDS:
+ if kw in hypothesis_lower:
+ return "explore"
+
+ for kw in COMBINE_KEYWORDS:
+ if kw in hypothesis_lower:
+ return "combine"
+
+ return "unclassified"
+```
+
+### A.4 Adversarial Phase Transition
+
+```
+function update_adversarial_state(state, score, config):
+ active = state.active_role
+ comp = config.generator if active == "generator" else config.discriminator
+
+ # Check if score is above threshold
+ above_threshold = (score >= comp.threshold)
+
+ # Update active role's streak counter
+ if active == "generator":
+ if above_threshold:
+ state.generator_consecutive_above += 1
+ else:
+ state.generator_consecutive_above = 0
+ else:
+ if above_threshold:
+ state.discriminator_consecutive_above += 1
+ else:
+ state.discriminator_consecutive_above = 0
+
+ # Check for phase switch
+ active_streak = (
+ state.generator_consecutive_above if active == "generator"
+ else state.discriminator_consecutive_above
+ )
+
+ should_switch = (active_streak >= config.hysteresis)
+
+ if should_switch:
+ # Switch roles
+ state.active_role = "discriminator" if active == "generator" else "generator"
+ # Reset newly-active role's streak
+ if state.active_role == "generator":
+ state.generator_consecutive_above = 0
+ else:
+ state.discriminator_consecutive_above = 0
+
+ # Record history
+ state.history.append(AdversarialPhaseRecord(
+ round=state.current_round,
+ active_role=active,
+ score=score,
+ metric_name=comp.metric_name,
+ timestamp=now_iso(),
+ switched=should_switch
+ ))
+
+ state.current_round += 1
+
+ # Check for convergence
+ if len(state.history) >= config.convergence_window:
+ recent = state.history[-config.convergence_window:]
+ scores = [r.score for r in recent]
+ variance = compute_variance(scores)
+ if variance < CONVERGENCE_THRESHOLD:
+ state.converged = True
+
+ return state
+```
+
+### A.5 Config Resolution with Precedence
+
+```
+function resolve_config_value(key, cli_arg, profile, config_dict, env_vars, default):
+ # Tier 1: CLI flag
+ if cli_arg is not None:
+ return cli_arg
+
+ # Tier 2: Environment variable
+ env_key = f"FACTORY_{key.upper()}"
+ if env_key in env_vars:
+ return env_vars[env_key]
+
+ # Tier 3: Profile credential
+ if profile and profile in config_dict.get("credentials", {}):
+ creds = config_dict["credentials"][profile]
+ if key in creds:
+ return creds[key]
+
+ # Tier 4: Config.toml default
+ if key in config_dict.get("defaults", {}):
+ return config_dict["defaults"][key]
+
+ # Tier 5: Hardcoded default
+ return default
+```
-An implementation that supports external state SHOULD:
+## How to Read the Knowledge Graph
-- preserve source identifiers and URLs
-- normalize external payloads into work items or state records
-- define reconciliation behavior for source state changes
+This spec uses `[[graph:...]]` reference links to point into a code knowledge graph extracted by graphify. The graph contains AST-derived entities (modules, classes, functions) and their typed relationships (imports, calls, inherits).
-An implementation that supports multi-user state MUST:
+### Reference Link Types
-- track actor and source metadata for important records
-- define merge policy per record kind
-- produce explicit conflict records for unresolved important conflicts
+- `[[graph:EntityName]]` — look up a specific entity (module, class, function). Example: `[[graph:factory.state.detect_state]]`
+- `[[graph:path:A:B]]` — find the dependency path between entities A and B. Example: `[[graph:path:store:registry]]`
+- `[[graph:query:question]]` — run a natural language query against the graph. Example: `[[graph:query:which modules call run_eval?]]`
+- `[[graph:community:subsystem]]` — list all entities in a detected subsystem. Example: `[[graph:community:eval]]`
-An implementation that supports additional deployment profiles SHOULD:
+### When to Use
-- describe the component bundle
-- preserve the domain model
-- document deviations from CLI-local behavior
+- **Planning and design:** Read the overview sections in this spec (§1-7) for behavioral contracts and architecture
+- **Implementation details:** Resolve `[[graph:...]]` links by reading `.factory/graphify-out/graph.json` directly, or query the graph with `graphify explain`, `graphify path`, `graphify query`
+- **Refactoring:** Use `[[graph:path:A:B]]` to trace impact of changes before modifying code
+- **Debugging:** Use `[[graph:query:...]]` to find all callers of a function or all implementers of a protocol
diff --git a/benchmarks/commit-full-eval.sh b/benchmarks/commit-full-eval.sh
new file mode 100755
index 000000000..955677e20
--- /dev/null
+++ b/benchmarks/commit-full-eval.sh
@@ -0,0 +1,188 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# benchmarks/commit-full-eval.sh — Commit full eval results to the benchmark-data branch.
+# Mirrors the CI workflow's commit pattern (benchmark.yml lines 220-293) but runs
+# from any machine with git push access.
+
+# ── Defaults ──
+
+RESULTS_DIR="${RESULTS_DIR:-benchmarks/results}"
+RUN_ID=""
+REPO_URL=""
+TEMP_DIR=""
+
+# ── Usage ──
+
+usage() {
+ echo "Usage: $(basename "$0") [options]"
+ echo ""
+ echo "Options:"
+ echo " --results-dir DIR Directory containing *-full.json files (default: benchmarks/results)"
+ echo " --run-id ID Run identifier (default: auto-generated from timestamp)"
+ echo " -h, --help Show this help message"
+ exit "${1:-0}"
+}
+
+# ── Argument parsing ──
+
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --results-dir) RESULTS_DIR="$2"; shift 2 ;;
+ --run-id) RUN_ID="$2"; shift 2 ;;
+ -h|--help) usage ;;
+ *) echo "ERROR: Unknown option '$1'"; usage 1 ;;
+ esac
+done
+
+if [ -z "${RUN_ID}" ]; then
+ RUN_ID="full-$(date -u +%Y%m%dT%H%M%SZ)"
+fi
+
+# ── Cleanup ──
+
+cleanup() {
+ if [ -n "${TEMP_DIR}" ] && [ -d "${TEMP_DIR}" ]; then
+ rm -rf "${TEMP_DIR}"
+ fi
+}
+
+trap cleanup EXIT
+
+# ── Validate inputs ──
+
+if [ ! -d "${RESULTS_DIR}" ]; then
+ echo "ERROR: Results directory not found: ${RESULTS_DIR}"
+ exit 1
+fi
+
+FULL_JSON_FILES=()
+while IFS= read -r -d '' f; do
+ FULL_JSON_FILES+=("$f")
+done < <(find "${RESULTS_DIR}" -maxdepth 1 -name '*-full.json' -print0 2>/dev/null)
+
+if [ ${#FULL_JSON_FILES[@]} -eq 0 ]; then
+ echo "ERROR: No *-full.json files found in ${RESULTS_DIR}"
+ exit 1
+fi
+
+echo "==> Found ${#FULL_JSON_FILES[@]} full eval result file(s)"
+for f in "${FULL_JSON_FILES[@]}"; do
+ echo " $(basename "${f}")"
+done
+
+# ── Resolve repo URL ──
+
+REPO_URL=$(git remote get-url origin 2>/dev/null || echo "")
+if [ -z "${REPO_URL}" ]; then
+ echo "ERROR: Could not determine git remote URL"
+ exit 1
+fi
+
+# ── Get current commit info ──
+
+CURRENT_COMMIT=$(git rev-parse HEAD 2>/dev/null || echo "")
+CURRENT_REF=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "")
+GIT_USER_NAME=$(git config user.name 2>/dev/null || echo "")
+GIT_USER_EMAIL=$(git config user.email 2>/dev/null || echo "")
+
+echo "==> Current state"
+echo " Commit: ${CURRENT_COMMIT:-unknown}"
+echo " Ref: ${CURRENT_REF:-unknown}"
+echo " Run ID: ${RUN_ID}"
+echo ""
+
+# ── Clone benchmark-data branch ──
+
+echo "==> Setting up benchmark-data branch"
+
+TEMP_DIR="$(mktemp -d /tmp/benchmark-data-XXXXXX)"
+
+if git ls-remote --exit-code --heads "${REPO_URL}" benchmark-data >/dev/null 2>&1; then
+ echo " Cloning existing benchmark-data branch..."
+ git clone --single-branch --branch benchmark-data --depth 1 "${REPO_URL}" "${TEMP_DIR}/benchmark-data"
+else
+ echo " Creating new benchmark-data branch..."
+ mkdir -p "${TEMP_DIR}/benchmark-data"
+ cd "${TEMP_DIR}/benchmark-data"
+ git init
+ git checkout -b benchmark-data
+ git remote add origin "${REPO_URL}"
+ touch results.jsonl full-eval-results.jsonl
+ git add .
+ if [ -n "${GIT_USER_NAME}" ]; then
+ git config user.name "${GIT_USER_NAME}"
+ git config user.email "${GIT_USER_EMAIL}"
+ fi
+ git commit -m "Initialize benchmark data branch"
+ git push -u origin benchmark-data
+fi
+
+BENCHMARK_DATA_DIR="${TEMP_DIR}/benchmark-data"
+
+# ── Enrich and append results ──
+
+echo "==> Enriching and appending results"
+
+python3 << PYEOF
+import json, os, sys
+
+results_dir = "${RESULTS_DIR}"
+output = "${BENCHMARK_DATA_DIR}/full-eval-results.jsonl"
+run_id = "${RUN_ID}"
+commit = "${CURRENT_COMMIT}"
+ref = "${CURRENT_REF}"
+
+files = sorted([
+ os.path.join(results_dir, f)
+ for f in os.listdir(results_dir)
+ if f.endswith("-full.json")
+])
+
+count = 0
+for fpath in files:
+ print(f" Processing: {os.path.basename(fpath)}", file=sys.stderr)
+ with open(fpath) as fh:
+ data = json.load(fh)
+ data["run_id"] = run_id
+ data["commit"] = commit
+ data["ref"] = ref
+ data["trigger"] = "manual"
+ with open(output, "a") as out:
+ out.write(json.dumps(data) + "\n")
+ count += 1
+
+size = os.path.getsize(output) if os.path.exists(output) else 0
+print(f" Appended {count} result(s). File size: {size} bytes", file=sys.stderr)
+PYEOF
+
+echo ""
+
+# ── Commit and push ──
+
+echo "==> Committing results"
+
+cd "${BENCHMARK_DATA_DIR}"
+
+if [ -n "${GIT_USER_NAME}" ]; then
+ git config user.name "${GIT_USER_NAME}"
+ git config user.email "${GIT_USER_EMAIL}"
+fi
+
+git add full-eval-results.jsonl
+
+if git diff --cached --quiet; then
+ echo " No changes to commit."
+ exit 0
+fi
+
+git commit -m "benchmark: add full eval results from run ${RUN_ID} [skip ci]"
+
+echo "==> Pushing to benchmark-data branch"
+git push origin benchmark-data || (
+ echo " Push failed, retrying with rebase..."
+ git pull --rebase origin benchmark-data && git push origin benchmark-data
+)
+
+echo ""
+echo "==> Done. Full eval results committed to benchmark-data branch."
diff --git a/benchmarks/config.sh b/benchmarks/config.sh
new file mode 100755
index 000000000..9f0592fd1
--- /dev/null
+++ b/benchmarks/config.sh
@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+# benchmarks/config.sh — Benchmark configuration mapping.
+# Source this after lib.sh to get benchmark_config, benchmark_dataset,
+# benchmark_all_names, and benchmark_instance_id.
+
+benchmark_all_names() {
+ echo "swebench mini-swebench featurebench terminalbench programbench harborindex tomswe salitrap devopsgym"
+}
+
+benchmark_config() {
+ local name="$1"
+
+ BENCH_DATASET=""
+ BENCH_LOCAL_PATH=""
+ BENCH_AGENT_CLASS=""
+ BENCH_AGENT_IMPORT_FLAG=""
+ BENCH_EXTRA_INSTRUCTION=""
+ BENCH_FILTER_STYLE=""
+ BENCH_ALLOW_HOSTS=""
+ BENCH_POST_EVAL_CMD=""
+
+ case "${name}" in
+ swebench)
+ BENCH_DATASET="swe-bench/swe-bench-verified"
+ BENCH_AGENT_CLASS="factory_harbor_agent:SwebenchFactoryCeo"
+ BENCH_AGENT_IMPORT_FLAG="--agent-import-path"
+ BENCH_FILTER_STYLE="glob"
+ ;;
+ featurebench)
+ BENCH_DATASET="featurebench"
+ BENCH_AGENT_CLASS="factory_harbor_agent:FeaturebenchFactoryCeo"
+ BENCH_AGENT_IMPORT_FLAG="--agent-import-path"
+ BENCH_EXTRA_INSTRUCTION="featurebench-extra-instructions.md"
+ BENCH_FILTER_STYLE="exact"
+ ;;
+ terminalbench)
+ BENCH_DATASET="terminal-bench@2.0"
+ BENCH_AGENT_CLASS="factory_harbor_agent:TerminalbenchFactoryCeo"
+ BENCH_AGENT_IMPORT_FLAG="--agent-import-path"
+ BENCH_EXTRA_INSTRUCTION="terminalbench-extra-instructions.md"
+ BENCH_FILTER_STYLE="exact"
+ ;;
+ programbench)
+ BENCH_LOCAL_PATH="${HARNESS_DIR}/benchmarks/programbench-harbor"
+ BENCH_AGENT_CLASS="factory_harbor_agent:ProgramBenchFactoryCeo"
+ BENCH_AGENT_IMPORT_FLAG="--agent"
+ BENCH_FILTER_STYLE="none"
+ BENCH_ALLOW_HOSTS="api.anthropic.com sentry.io statsig.anthropic.com"
+ BENCH_POST_EVAL_CMD="uvx programbench eval"
+ ;;
+ legacybench)
+ BENCH_DATASET="factory-ai/legacy-bench"
+ BENCH_AGENT_CLASS="factory_harbor_agent:LegacybenchFactoryCeo"
+ BENCH_AGENT_IMPORT_FLAG="--agent-import-path"
+ BENCH_FILTER_STYLE="glob"
+ ;;
+ harborindex)
+ BENCH_DATASET="harbor-index/harbor-index-1.0"
+ BENCH_AGENT_CLASS="factory_harbor_agent:HarborIndexFactoryCeo"
+ BENCH_AGENT_IMPORT_FLAG="--agent-import-path"
+ BENCH_FILTER_STYLE="exact"
+ ;;
+ tomswe)
+ BENCH_DATASET='swe-bench/swe-bench-verified'
+ BENCH_AGENT_CLASS="factory_harbor_agent:TomsweFactoryCeo"
+ BENCH_AGENT_IMPORT_FLAG="--agent-import-path"
+ BENCH_FILTER_STYLE="glob"
+ ;;
+ mini-swebench)
+ BENCH_DATASET="swe-bench/swe-bench-verified"
+ BENCH_AGENT_CLASS="factory_harbor_agent:MiniSwebenchFactoryCeo"
+ BENCH_AGENT_IMPORT_FLAG="--agent-import-path"
+ BENCH_FILTER_STYLE="glob"
+ ;;
+ salitrap)
+ BENCH_DATASET="salitrap"
+ BENCH_AGENT_CLASS="factory_harbor_agent:SalitrapFactoryCeo"
+ BENCH_AGENT_IMPORT_FLAG="--agent-import-path"
+ BENCH_FILTER_STYLE="exact"
+ ;;
+ devopsgym)
+ BENCH_DATASET="devops-gym/devops-gym-build"
+ BENCH_AGENT_CLASS="factory_harbor_agent:DevOpsGymFactoryCeo"
+ BENCH_AGENT_IMPORT_FLAG="--agent-import-path"
+ BENCH_FILTER_STYLE="glob"
+ ;;
+ *)
+ echo "ERROR: Unknown benchmark '${name}'"
+ echo "Valid benchmarks: swebench, featurebench, terminalbench, programbench, legacybench, harborindex, tomswe, salitrap, devopsgym"
+ return 1
+ ;;
+ esac
+}
+
+benchmark_dataset() {
+ local name="$1"
+ local split="${2:-}"
+
+ case "${name}" in
+ featurebench)
+ case "${split:-full}" in
+ lite) BENCH_DATASET="featurebench-lite" ;;
+ full|fast) BENCH_DATASET="featurebench" ;;
+ *) BENCH_DATASET="featurebench-${split}" ;;
+ esac
+ ;;
+ esac
+}
+
+benchmark_instance_id() {
+ local benchmark="$1"
+ local task="$2"
+ if [ "${benchmark}" = "programbench" ]; then
+ case "${task}" in
+ cmatrix) echo "abishekvashok__cmatrix.5c082c6" ;;
+ *) echo "${task}" ;;
+ esac
+ else
+ echo "${task}"
+ fi
+}
diff --git a/benchmarks/factory_harbor_agent.py b/benchmarks/factory_harbor_agent.py
index 5c33e021b..82ee482e2 100644
--- a/benchmarks/factory_harbor_agent.py
+++ b/benchmarks/factory_harbor_agent.py
@@ -1,5 +1,6 @@
"""Harbor agent that runs ``factory ceo`` as a benchmark solver."""
+import hashlib
import os
import re
from typing import override
@@ -9,6 +10,238 @@
from harbor.models.agent.context import AgentContext
+COMMON_ENV_VARS = (
+ "ANTHROPIC_BASE_URL",
+ "ANTHROPIC_MODEL",
+ "CLAUDE_CODE_USE_VERTEX",
+ "ANTHROPIC_VERTEX_PROJECT_ID",
+ "CLOUD_ML_REGION",
+ "GOOGLE_APPLICATION_CREDENTIALS",
+ "CLAUDE_CODE_SUBAGENT_MODEL",
+ "CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS",
+ "ANTHROPIC_DEFAULT_OPUS_MODEL",
+ "CLAUDE_CODE_DISABLE_ADAPTIVE_THINKING",
+ "MAX_THINKING_TOKENS",
+ "CLAUDE_CODE_EFFORT_LEVEL",
+ "LANGFUSE_HOST",
+ "LANGFUSE_PUBLIC_KEY",
+ "LANGFUSE_SECRET_KEY",
+ "LANGFUSE_BASE_URL",
+ "FACTORY_GIT_REF",
+ "FACTORY_BENCHMARK",
+ "FACTORY_INSTANCE_ID",
+)
+
+
+TOMSWE_PROFILES: list[dict[str, object]] = [
+ {
+ "profile_id": "P01",
+ "verbosity": "concise",
+ "question_timing": "upfront",
+ "response_style": "short",
+ "coding_preferences": [
+ "pytest over unittest",
+ "type hints required",
+ "f-strings over format()",
+ "single-responsibility functions",
+ "descriptive variable names",
+ ],
+ },
+ {
+ "profile_id": "P02",
+ "verbosity": "verbose",
+ "question_timing": "ongoing",
+ "response_style": "verbose",
+ "coding_preferences": [
+ "unittest with setUp/tearDown",
+ "docstrings on all public methods",
+ "defensive error handling",
+ "logging over print statements",
+ "class-based design patterns",
+ "comprehensive inline comments",
+ ],
+ },
+ {
+ "profile_id": "P03",
+ "verbosity": "concise",
+ "question_timing": "upfront",
+ "response_style": "verbose",
+ "coding_preferences": [
+ "functional programming style",
+ "list comprehensions over loops",
+ "dataclasses over plain dicts",
+ "minimal dependencies",
+ "pathlib over os.path",
+ ],
+ },
+ {
+ "profile_id": "P04",
+ "verbosity": "verbose",
+ "question_timing": "upfront",
+ "response_style": "short",
+ "coding_preferences": [
+ "pytest fixtures over setup methods",
+ "abstract base classes for interfaces",
+ "enum over string constants",
+ "context managers for resource handling",
+ "snake_case naming strictly enforced",
+ "no wildcard imports",
+ ],
+ },
+ {
+ "profile_id": "P05",
+ "verbosity": "concise",
+ "question_timing": "ongoing",
+ "response_style": "short",
+ "coding_preferences": [
+ "minimal comments — code should be self-documenting",
+ "early returns over nested if/else",
+ "prefer composition over inheritance",
+ "use walrus operator where it simplifies",
+ "keep functions under 20 lines",
+ ],
+ },
+ {
+ "profile_id": "P06",
+ "verbosity": "verbose",
+ "question_timing": "ongoing",
+ "response_style": "verbose",
+ "coding_preferences": [
+ "type hints with Optional and Union",
+ "property decorators over getters/setters",
+ "named tuples for lightweight data",
+ "explicit exception types over bare except",
+ "reStructuredText docstring format",
+ "separate test file per module",
+ "integration tests alongside unit tests",
+ ],
+ },
+ {
+ "profile_id": "P07",
+ "verbosity": "concise",
+ "question_timing": "upfront",
+ "response_style": "short",
+ "coding_preferences": [
+ "Google-style docstrings",
+ "absolute imports only",
+ "collections.abc over typing for containers",
+ "prefer standard library over third-party",
+ "guard clauses at function start",
+ ],
+ },
+ {
+ "profile_id": "P08",
+ "verbosity": "verbose",
+ "question_timing": "upfront",
+ "response_style": "verbose",
+ "coding_preferences": [
+ "Pydantic models for validation",
+ "structured logging with structlog",
+ "async/await for I/O operations",
+ "dependency injection pattern",
+ "conventional commits for git messages",
+ "100 char line length maximum",
+ ],
+ },
+ {
+ "profile_id": "P09",
+ "verbosity": "concise",
+ "question_timing": "ongoing",
+ "response_style": "verbose",
+ "coding_preferences": [
+ "pytest parametrize for test variants",
+ "builder pattern for complex objects",
+ "protocol classes over ABCs",
+ "match/case for dispatch logic",
+ "X | Y union syntax over Union",
+ ],
+ },
+ {
+ "profile_id": "P10",
+ "verbosity": "verbose",
+ "question_timing": "ongoing",
+ "response_style": "short",
+ "coding_preferences": [
+ "TDD approach — write tests first",
+ "black formatter compliance",
+ "isort for import ordering",
+ "no mutable default arguments",
+ "explicit __all__ exports",
+ "slots=True on dataclasses",
+ ],
+ },
+ {
+ "profile_id": "P11",
+ "verbosity": "concise",
+ "question_timing": "upfront",
+ "response_style": "short",
+ "coding_preferences": [
+ "LBYL over EAFP where possible",
+ "itertools for complex iterations",
+ "functools.lru_cache for memoization",
+ "private methods with underscore prefix",
+ "constants in UPPER_SNAKE_CASE",
+ ],
+ },
+ {
+ "profile_id": "P12",
+ "verbosity": "verbose",
+ "question_timing": "upfront",
+ "response_style": "verbose",
+ "coding_preferences": [
+ "EAFP over LBYL — ask forgiveness",
+ "contextlib utilities for context managers",
+ "textwrap.dedent for multiline strings",
+ "attrs over dataclasses",
+ "Numpy-style docstrings",
+ "hypothesis for property-based testing",
+ "separate constants module",
+ ],
+ },
+ {
+ "profile_id": "P13",
+ "verbosity": "concise",
+ "question_timing": "ongoing",
+ "response_style": "short",
+ "coding_preferences": [
+ "simple flat module structure",
+ "dict.get() over KeyError handling",
+ "one assert per test method",
+ "no global state",
+ "pure functions where possible",
+ ],
+ },
+ {
+ "profile_id": "P14",
+ "verbosity": "verbose",
+ "question_timing": "ongoing",
+ "response_style": "verbose",
+ "coding_preferences": [
+ "layered architecture (services/repos/models)",
+ "factory methods for object creation",
+ "immutable data structures preferred",
+ "typing.TypeAlias for complex types",
+ "rich library for CLI output",
+ "click over argparse for CLI",
+ ],
+ },
+ {
+ "profile_id": "P15",
+ "verbosity": "concise",
+ "question_timing": "upfront",
+ "response_style": "verbose",
+ "coding_preferences": [
+ "argparse for CLI — standard library only",
+ "os.environ.get with defaults",
+ "json over yaml for configuration",
+ "subprocess.run over os.system",
+ "tempfile module for temp resources",
+ "atexit for cleanup handlers",
+ ],
+ },
+]
+
+
class FactoryCeo(BaseInstalledAgent):
"""Runs ``factory ceo`` to solve benchmark tasks.
@@ -69,7 +302,8 @@ async def install(self, environment: BaseEnvironment) -> None:
),
)
- # Factory CLI via uv
+ # Factory CLI via uv — install from the current commit when
+ # FACTORY_GIT_REF is set (CI passes the PR sha), otherwise main.
await self.exec_as_agent(
environment,
command=(
@@ -77,14 +311,29 @@ async def install(self, environment: BaseEnvironment) -> None:
'export PATH="$HOME/.local/bin:$PATH"; '
"curl -LsSf https://astral.sh/uv/install.sh | sh && "
'export PATH="$HOME/.cargo/bin:$PATH"; '
- "uv tool install "
- "'remote-factory @ git+https://github.com/akashgit/remote-factory.git' && "
+ 'REF="${FACTORY_GIT_REF:-}"; '
+ 'if [ -n "$REF" ]; then '
+ ' uv tool install "remote-factory @ git+https://github.com/akashgit/remote-factory.git@${REF}"; '
+ "else "
+ " uv tool install "
+ "'remote-factory @ git+https://github.com/akashgit/remote-factory.git'; "
+ "fi && "
"which factory"
),
)
# ── run ───────────────────────────────────────────────────────────
+ def _get_factory_command(self) -> str:
+ return (
+ 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"; '
+ 'export FACTORY_CEO_RESPAWN_DISABLED=1; '
+ 'factory ceo . --headless --no-github '
+ '--focus "$(cat /tmp/task-instruction.md)" '
+ '2>&1 .gitignore && '
+ 'git add -A && '
+ 'git commit -m "initial state" --allow-empty'
+ ),
+ env=env,
+ )
+
+ # Seed .factory/ so state detection yields has_factory → improve mode,
+ # which is required for --focus to work.
+ await self.exec_as_agent(
+ environment,
+ command=(
+ 'mkdir -p .factory && '
+ 'printf \'{}\\n\' > .factory/config.json && '
+ 'printf \'{"human_reviewed": true, "dimensions": []}\\n\' > .factory/eval_profile.json'
+ ),
+ env=env,
+ )
+
+ # Write task instruction to a file; read via $(cat ...) in --focus
await self.exec_as_agent(
environment,
command=f"cat > /tmp/task-instruction.md << 'INSTREOF'\n{instruction}\nINSTREOF",
env=env,
)
- # Run factory ceo in headless mode
+ await self.exec_as_agent(
+ environment,
+ command=self._get_factory_command(),
+ env=env,
+ )
+
await self.exec_as_agent(
environment,
command=(
- 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"; '
- "factory ceo . --headless --mode build "
- "--prompt /tmp/task-instruction.md "
- "2>&1 /dev/null || "
+ "cp .factory/trace_id.txt /logs/agent/trace_id.txt 2>/dev/null; "
+ "exit 0"
),
env=env,
)
@@ -226,3 +497,379 @@ async def run(
),
env=env,
)
+
+
+class ProgramBenchFactoryCeo(FactoryCeo):
+ """Runs the deterministic programbench workflow."""
+
+ @staticmethod
+ @override
+ def name() -> str:
+ return "programbench-factory-ceo"
+
+ @override
+ def _get_factory_command(self) -> str:
+ return (
+ 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"; '
+ 'factory workflow run programbench . '
+ '2>&1 str:
+ return "swebench-factory-ceo"
+
+ @override
+ def _get_factory_command(self) -> str:
+ return (
+ 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"; '
+ 'factory workflow run swebench . '
+ '2>&1 str:
+ return "mini-swebench-factory-ceo"
+
+ @override
+ async def install(self, environment: BaseEnvironment) -> None:
+ await super().install(environment)
+ await self.exec_as_agent(
+ environment,
+ command=(
+ 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"; '
+ 'pip install "anthropic[vertex]" 2>/dev/null || true'
+ ),
+ )
+
+ @override
+ def _get_factory_command(self) -> str:
+ return (
+ 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"; '
+ 'factory workflow run mini-swebench . '
+ '2>&1 str:
+ return "legacybench-factory-ceo"
+
+ @override
+ def _get_factory_command(self) -> str:
+ return (
+ 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"; '
+ 'factory workflow run legacybench . '
+ '2>&1 str:
+ return "devopsgym-factory-ceo"
+
+ @override
+ def _get_factory_command(self) -> str:
+ return (
+ 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"; '
+ 'factory workflow run devopsgym . '
+ '2>&1 str:
+ return "terminalbench-factory-ceo"
+
+ @override
+ def _get_factory_command(self) -> str:
+ return (
+ 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"; '
+ 'factory workflow run terminalbench . '
+ '2>&1 str:
+ return "featurebench-factory-ceo"
+
+ @override
+ def _get_factory_command(self) -> str:
+ return (
+ 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"; '
+ 'factory workflow run featurebench . '
+ '2>&1 str:
+ return "harbor-index-factory-ceo"
+
+
+class SalitrapFactoryCeo(FactoryCeo):
+ """Runs the deterministic salitrap workflow."""
+
+ @staticmethod
+ @override
+ def name() -> str:
+ return "salitrap-factory-ceo"
+
+ @override
+ def _get_factory_command(self) -> str:
+ return (
+ 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"; '
+ 'factory workflow run salitrap . '
+ '2>&1 str:
+ return "tomswe-factory-ceo"
+
+ @override
+ def _get_factory_command(self) -> str:
+ return (
+ 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"; '
+ 'factory workflow run tomswe . '
+ '2>&1 dict[str, object]:
+ idx = int(hashlib.sha256(instruction.encode()).hexdigest(), 16) % len(TOMSWE_PROFILES)
+ return TOMSWE_PROFILES[idx]
+
+ @staticmethod
+ def _format_profile(profile: dict[str, object]) -> str:
+ prefs = profile["coding_preferences"]
+ assert isinstance(prefs, list)
+ prefs_str = "\n".join(f"- {p}" for p in prefs)
+ return (
+ f"## User Profile\n\n"
+ f"**Profile ID:** {profile['profile_id']}\n"
+ f"**Verbosity:** {profile['verbosity']}\n"
+ f"**Question Timing:** {profile['question_timing']}\n"
+ f"**Response Style:** {profile['response_style']}\n\n"
+ f"**Coding Preferences:**\n{prefs_str}\n"
+ )
+
+ @override
+ async def run(
+ self,
+ instruction: str,
+ environment: BaseEnvironment,
+ context: AgentContext,
+ ) -> None:
+ """Run factory tomswe workflow with an injected user profile."""
+ api_key = (
+ self._get_env("ANTHROPIC_API_KEY")
+ or self._get_env("ANTHROPIC_AUTH_TOKEN")
+ or ""
+ )
+
+ env: dict[str, str] = {
+ "ANTHROPIC_API_KEY": api_key,
+ "IS_SANDBOX": "1",
+ "CLAUDE_CONFIG_DIR": "/logs/agent/sessions",
+ }
+
+ if self.model_name:
+ env["ANTHROPIC_MODEL"] = self.model_name.split("/")[-1]
+
+ for var in COMMON_ENV_VARS:
+ val = self._get_env(var) or os.environ.get(var)
+ if val and var not in env:
+ env[var] = val
+
+ env = {k: v for k, v in env.items() if v}
+
+ await self.exec_as_agent(
+ environment,
+ command=(
+ "mkdir -p $CLAUDE_CONFIG_DIR/debug "
+ "$CLAUDE_CONFIG_DIR/projects "
+ "$CLAUDE_CONFIG_DIR/shell-snapshots "
+ "$CLAUDE_CONFIG_DIR/statsig "
+ "$CLAUDE_CONFIG_DIR/todos "
+ "$CLAUDE_CONFIG_DIR/skills"
+ ),
+ env=env,
+ )
+
+ await self.exec_as_agent(
+ environment,
+ command=(
+ "cat > ./factory.md << 'FACTORYEOF'\n"
+ "---\n"
+ "goal: Solve the given coding task\n"
+ "---\n"
+ "FACTORYEOF"
+ ),
+ env=env,
+ )
+
+ await self.exec_as_agent(
+ environment,
+ command=(
+ 'set -e; '
+ 'if [ ! -d .git ]; then git init -b main; fi && '
+ 'git config user.name "Factory Agent" && '
+ 'git config user.email "factory@agent.local" && '
+ 'printf "/proc\\n/sys\\n/dev\\n/run\\n/tmp\\n/var\\n/root\\n'
+ '/home\\n/usr\\n/bin\\n/sbin\\n/lib\\n/lib64\\n/etc\\n'
+ '/boot\\n/mnt\\n/opt\\n/srv\\n/media\\n/logs\\n" > .gitignore && '
+ 'git add -A && '
+ 'git commit -m "initial state" --allow-empty'
+ ),
+ env=env,
+ )
+
+ await self.exec_as_agent(
+ environment,
+ command=(
+ 'mkdir -p .factory && '
+ 'printf \'{}\\n\' > .factory/config.json && '
+ 'printf \'{"human_reviewed": true, "dimensions": []}\\n\' > .factory/eval_profile.json'
+ ),
+ env=env,
+ )
+
+ # Inject a deterministically-selected user profile into the instruction
+ profile = self._select_profile(instruction)
+ augmented = instruction + "\n\n" + self._format_profile(profile)
+
+ await self.exec_as_agent(
+ environment,
+ command=f"cat > /tmp/task-instruction.md << 'INSTREOF'\n{augmented}\nINSTREOF",
+ env=env,
+ )
+
+ await self.exec_as_agent(
+ environment,
+ command=self._get_factory_command(),
+ env=env,
+ )
+
+ await self.exec_as_agent(
+ environment,
+ command=(
+ "cp /testbed/.factory/trace_id.txt /logs/agent/trace_id.txt 2>/dev/null || "
+ "cp .factory/trace_id.txt /logs/agent/trace_id.txt 2>/dev/null; "
+ "exit 0"
+ ),
+ env=env,
+ )
+
+ await self.exec_as_agent(
+ environment,
+ command=(
+ "set +e; "
+ 'FACTORY_BRANCH=$(git branch --list "factory/*" | head -1 | tr -d " *"); '
+ 'if [ -n "$FACTORY_BRANCH" ]; then '
+ ' echo "Merging factory branch: $FACTORY_BRANCH"; '
+ ' git merge "$FACTORY_BRANCH" --no-edit 2>/dev/null '
+ ' || git cherry-pick "$FACTORY_BRANCH" --no-edit 2>/dev/null '
+ " || true; "
+ "fi; "
+ 'if [ -z "$FACTORY_BRANCH" ]; then '
+ ' echo "No factory branch, finding orphaned commits..."; '
+ " ORPHAN_COMMITS=$(git fsck --unreachable --no-reflogs 2>/dev/null "
+ " | grep 'unreachable commit' | awk '{print \\$3}'); "
+ ' if [ -n "$ORPHAN_COMMITS" ]; then '
+ ' BEST_COMMIT=""; '
+ " BEST_TIME=0; "
+ " for SHA in $ORPHAN_COMMITS; do "
+ ' COMMIT_TIME=$(git show -s --format=\'%ct\' "$SHA" 2>/dev/null || echo 0); '
+ ' if [ "$COMMIT_TIME" -gt "$BEST_TIME" ]; then '
+ " BEST_TIME=$COMMIT_TIME; "
+ " BEST_COMMIT=$SHA; "
+ " fi; "
+ " done; "
+ ' if [ -n "$BEST_COMMIT" ]; then '
+ ' echo "Recovering from orphan tip: $BEST_COMMIT"; '
+ ' echo " Message: $(git log -1 --format=\'%s\' $BEST_COMMIT 2>/dev/null)"; '
+ ' git checkout "$BEST_COMMIT" -- . 2>/dev/null || true; '
+ " git checkout HEAD -- .factory/ eval/ factory.md 2>/dev/null || true; "
+ " rm -rf .factory/ eval/ factory.md 2>/dev/null || true; "
+ " fi; "
+ " fi; "
+ "fi; "
+ 'for wt in .factory-worktrees/*/; do '
+ ' if [ -d "$wt" ]; then '
+ ' echo "Recovering files from worktree: $wt"; '
+ " rsync -a --exclude='.git' --exclude='.factory' "
+ ' "$wt" ./ 2>/dev/null || true; '
+ " fi; "
+ "done; "
+ "exit 0"
+ ),
+ env=env,
+ )
+
+
+class SwebenchifyHardFactoryCeo(FactoryCeo):
+ """Runs the swebenchifyhard workflow for SWE-benchify-hard benchmark."""
+
+ @staticmethod
+ @override
+ def name() -> str:
+ return "swebenchifyhard-factory-ceo"
+
+ @override
+ def _get_factory_command(self) -> str:
+ return (
+ 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"; '
+ 'factory workflow run swebenchifyhard . '
+ '2>&1 /dev/null && [ ! -x /usr/bin/docker ]; then
+ MISSING+=("docker (install from https://docs.docker.com/get-docker/)")
+ fi
+ if [ ${#MISSING[@]} -gt 0 ]; then
+ echo " ERROR: Missing prerequisites:"
+ for m in "${MISSING[@]}"; do
+ echo " - ${m}"
+ done
+ exit 1
+ fi
+ echo " docker: found"
+
+ ensure_uvx
+
+ echo " harbor: checking availability via uvx..."
+ if ! uvx harbor --version &>/dev/null 2>&1; then
+ echo " harbor: installing via uvx..."
+ uvx harbor --version || {
+ echo " ERROR: Failed to install/run harbor via uvx"
+ exit 1
+ }
+ fi
+ echo " harbor: available"
+}
+
+extract_langfuse_hostname() {
+ local hostname=""
+ if [ -n "${LANGFUSE_HOST:-}" ]; then
+ hostname=$(echo "${LANGFUSE_HOST}" | sed 's|https\?://||' | sed 's|/.*||')
+ elif [ -n "${LANGFUSE_BASE_URL:-}" ]; then
+ hostname=$(echo "${LANGFUSE_BASE_URL}" | sed 's|https\?://||' | sed 's|/.*||')
+ fi
+ echo "${hostname}"
+}
+
+# create_langfuse_trace — Create a wrapper Langfuse trace via the REST API.
+# Uses Python3 urllib (stdlib) so no pip install is needed.
+# Echoes the 32-char hex trace ID on success, empty string on failure.
+# Env: LANGFUSE_HOST (or LANGFUSE_BASE_URL), LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY
+create_langfuse_trace() {
+ local benchmark="${1:-}"
+ local instance_id="${2:-}"
+ local solver="${3:-}"
+ local git_ref="${4:-}"
+
+ python3 -c "
+import json, os, sys, uuid, urllib.request, urllib.error, base64, time
+
+host = os.environ.get('LANGFUSE_HOST') or os.environ.get('LANGFUSE_BASE_URL', '')
+pub_key = os.environ.get('LANGFUSE_PUBLIC_KEY', '')
+sec_key = os.environ.get('LANGFUSE_SECRET_KEY', '')
+
+if not host or not pub_key or not sec_key:
+ sys.exit(0)
+
+host = host.rstrip('/')
+trace_id = uuid.uuid4().hex
+auth = base64.b64encode(f'{pub_key}:{sec_key}'.encode()).decode()
+
+payload = {
+ 'batch': [{
+ 'id': uuid.uuid4().hex,
+ 'type': 'trace-create',
+ 'timestamp': time.strftime('%Y-%m-%dT%H:%M:%S.000Z', time.gmtime()),
+ 'body': {
+ 'id': trace_id,
+ 'name': 'benchmark:${benchmark}/${instance_id}',
+ 'metadata': {
+ 'benchmark': '${benchmark}',
+ 'instance_id': '${instance_id}',
+ 'solver': '${solver}',
+ 'git_ref': '${git_ref}',
+ 'source': 'run-harbor.sh',
+ },
+ },
+ }],
+ 'metadata': {},
+}
+
+req = urllib.request.Request(
+ f'{host}/api/public/ingestion',
+ data=json.dumps(payload).encode(),
+ headers={'Content-Type': 'application/json', 'Authorization': f'Basic {auth}'},
+ method='POST',
+)
+try:
+ urllib.request.urlopen(req, timeout=10)
+except Exception:
+ sys.exit(0)
+
+print(trace_id)
+" 2>/dev/null || true
+}
+
+# close_langfuse_trace — Mark a wrapper Langfuse trace as completed.
+# Posts a span-end event with duration and status info.
+close_langfuse_trace() {
+ local trace_id="${1:-}"
+ local status="${2:-unknown}"
+ local duration="${3:-0}"
+
+ if [ -z "${trace_id}" ]; then
+ return 0
+ fi
+
+ python3 -c "
+import json, os, sys, uuid, urllib.request, urllib.error, base64, time
+
+host = os.environ.get('LANGFUSE_HOST') or os.environ.get('LANGFUSE_BASE_URL', '')
+pub_key = os.environ.get('LANGFUSE_PUBLIC_KEY', '')
+sec_key = os.environ.get('LANGFUSE_SECRET_KEY', '')
+
+if not host or not pub_key or not sec_key:
+ sys.exit(0)
+
+host = host.rstrip('/')
+auth = base64.b64encode(f'{pub_key}:{sec_key}'.encode()).decode()
+
+payload = {
+ 'batch': [{
+ 'id': uuid.uuid4().hex,
+ 'type': 'span-create',
+ 'timestamp': time.strftime('%Y-%m-%dT%H:%M:%S.000Z', time.gmtime()),
+ 'body': {
+ 'id': uuid.uuid4().hex,
+ 'traceId': '${trace_id}',
+ 'name': 'harbor-execution',
+ 'startTime': time.strftime('%Y-%m-%dT%H:%M:%S.000Z', time.gmtime(time.time() - ${duration})),
+ 'endTime': time.strftime('%Y-%m-%dT%H:%M:%S.000Z', time.gmtime()),
+ 'metadata': {
+ 'status': '${status}',
+ 'duration_seconds': ${duration},
+ },
+ },
+ }],
+ 'metadata': {},
+}
+
+req = urllib.request.Request(
+ f'{host}/api/public/ingestion',
+ data=json.dumps(payload).encode(),
+ headers={'Content-Type': 'application/json', 'Authorization': f'Basic {auth}'},
+ method='POST',
+)
+try:
+ urllib.request.urlopen(req, timeout=10)
+except Exception:
+ pass
+" 2>/dev/null || true
+}
+
+extract_trace_id() {
+ local jobs_dir="$1"
+ local trace_id=""
+ if [ -n "${jobs_dir}" ] && [ -d "${jobs_dir}" ]; then
+ local trace_file
+ trace_file=$(find "${jobs_dir}" -name 'trace_id.txt' -type f 2>/dev/null | head -1)
+ if [ -n "${trace_file}" ] && [ -f "${trace_file}" ]; then
+ trace_id=$(cat "${trace_file}" | tr -d '[:space:]')
+ fi
+ fi
+ echo "${trace_id}"
+}
+
+# extract_harbor_cost — Extract cost/token data from Harbor jobs directory.
+# Sets: COST_USD, INPUT_TOKENS, OUTPUT_TOKENS, CACHE_READ_TOKENS
+# Reads: JOBS_DIR from environment
+extract_harbor_cost() {
+ local jobs_dir="${1:-${JOBS_DIR}}"
+ COST_USD=0
+ INPUT_TOKENS=0
+ OUTPUT_TOKENS=0
+ CACHE_READ_TOKENS=0
+ CACHE_CREATION_TOKENS=0
+
+ local harbor_result
+ harbor_result=$(find "${jobs_dir}" -name 'result.json' -maxdepth 2 2>/dev/null | head -1)
+ if [ -n "${harbor_result}" ]; then
+ local cost_data
+ cost_data=$(python3 -c "
+import json
+with open('${harbor_result}') as f:
+ data = json.load(f)
+stats = data.get('stats', {})
+cost = stats.get('cost_usd', 0) or 0
+input_t = stats.get('n_input_tokens', 0) or 0
+output_t = stats.get('n_output_tokens', 0) or 0
+cache_t = stats.get('n_cache_tokens', 0) or 0
+if cost == 0:
+ for trial in data.get('trials', {}).values():
+ cost += trial.get('cost_usd', 0) or 0
+print(f'COST_USD={cost}')
+print(f'INPUT_TOKENS={input_t}')
+print(f'OUTPUT_TOKENS={output_t}')
+print(f'CACHE_READ_TOKENS={cache_t}')
+" 2>/dev/null)
+ eval "${cost_data}" 2>/dev/null || true
+ fi
+
+ if [ "${COST_USD}" = "0" ] || [ -z "${COST_USD}" ]; then
+ local agent_log
+ agent_log=$(find "${jobs_dir}" -name 'claude-code.txt' -o -name 'claude_code_stream_output.jsonl' -o -name 'factory-ceo.txt' 2>/dev/null | head -1)
+ if [ -n "${agent_log}" ]; then
+ local cost_data
+ cost_data=$(grep 'total_cost_usd' "${agent_log}" 2>/dev/null | tail -1 | python3 -c "
+import sys, json
+for line in sys.stdin:
+ try:
+ data = json.loads(line.strip())
+ if 'total_cost_usd' in data:
+ print(f'COST_USD={data[\"total_cost_usd\"]}')
+ u = data.get('usage', {})
+ print(f'INPUT_TOKENS={u.get(\"input_tokens\", 0)}')
+ print(f'OUTPUT_TOKENS={u.get(\"output_tokens\", 0)}')
+ except: pass
+" 2>/dev/null || true)
+ eval "${cost_data}" 2>/dev/null || true
+ fi
+ fi
+}
+
+# extract_single_reward — Extract reward from Harbor verifier output (single-task).
+# Sets: RESOLVED, TOTAL, PASS_RATE
+# Reads: JOBS_DIR from environment
+extract_single_reward() {
+ local jobs_dir="${1:-${JOBS_DIR}}"
+ RESOLVED=0
+ TOTAL=1
+ PASS_RATE=0
+
+ local reward_file=""
+ for candidate in $(find "${jobs_dir}" -name 'reward.json' 2>/dev/null); do
+ if [ -f "${candidate}" ]; then
+ reward_file="${candidate}"
+ break
+ fi
+ done
+
+ if [ -z "${reward_file}" ]; then
+ for candidate in $(find "${jobs_dir}" -name 'reward.txt' 2>/dev/null); do
+ if [ -f "${candidate}" ]; then
+ reward_file="${candidate}"
+ break
+ fi
+ done
+ fi
+
+ if [ -n "${reward_file}" ] && [ -f "${reward_file}" ]; then
+ echo " Reward file: ${reward_file}"
+
+ if [[ "${reward_file}" == *.json ]]; then
+ eval "$(python3 -c "
+import json
+with open('${reward_file}') as f:
+ data = json.load(f)
+if isinstance(data, dict):
+ values = [v for v in data.values() if isinstance(v, (int, float))]
+ score = sum(values) / len(values) if values else 0.0
+ resolved = 1 if score > 0.5 else 0
+ pass_rate = score
+elif isinstance(data, (int, float)):
+ resolved = 1 if float(data) > 0.5 else 0
+ pass_rate = float(data)
+else:
+ resolved = 0
+ pass_rate = 0.0
+print(f'RESOLVED={resolved}')
+print(f'TOTAL=1')
+print(f'PASS_RATE={pass_rate}')
+")"
+ else
+ local reward_value
+ reward_value="$(cat "${reward_file}" | tr -d '[:space:]')"
+ echo " Reward value: ${reward_value}"
+ if [ "${reward_value}" = "1" ] || [ "${reward_value}" = "1.0" ]; then
+ RESOLVED=1
+ PASS_RATE=1.0
+ else
+ RESOLVED=0
+ PASS_RATE="${reward_value}"
+ fi
+ TOTAL=1
+ fi
+ else
+ local summary_file=""
+ for candidate in $(find "${jobs_dir}" -name 'results*.json' -o -name 'summary*.json' 2>/dev/null); do
+ if [ -f "${candidate}" ]; then
+ summary_file="${candidate}"
+ break
+ fi
+ done
+
+ if [ -n "${summary_file}" ] && [ -f "${summary_file}" ]; then
+ echo " Summary file: ${summary_file}"
+ eval "$(python3 -c "
+import json
+with open('${summary_file}') as f:
+ data = json.load(f)
+resolved = 0
+total = 1
+pass_rate = 0.0
+if isinstance(data, dict):
+ if 'reward' in data:
+ resolved = 1 if float(data['reward']) > 0.5 else 0
+ pass_rate = float(data['reward'])
+ elif 'score' in data:
+ resolved = 1 if float(data['score']) > 0.5 else 0
+ pass_rate = float(data['score'])
+ elif 'results' in data:
+ results = data['results']
+ if isinstance(results, dict):
+ total = len(results)
+ resolved = sum(1 for v in results.values()
+ if isinstance(v, dict) and v.get('reward', 0) > 0.5)
+ elif isinstance(results, list):
+ total = len(results)
+ resolved = sum(1 for v in results
+ if isinstance(v, dict) and v.get('reward', 0) > 0.5)
+ pass_rate = resolved / max(total, 1)
+print(f'RESOLVED={resolved}')
+print(f'TOTAL={max(total, 1)}')
+print(f'PASS_RATE={pass_rate}')
+")"
+ else
+ echo " No results files found. Marking as unresolved."
+ echo " Contents of jobs directory:"
+ find "${jobs_dir}" -type f 2>/dev/null | head -20 || echo " (empty)"
+ RESOLVED=0
+ TOTAL=1
+ fi
+ fi
+}
+
+# extract_multi_task_results — Extract per-task results from Harbor jobs (full-eval).
+# Sets: TASKS_JSON (JSON array string)
+# Reads: JOBS_DIR from environment
+extract_multi_task_results() {
+ TASKS_JSON=$(python3 << 'PYEOF'
+import json, os, re, sys, glob
+
+jobs_dir = os.environ.get("JOBS_DIR", "")
+if not jobs_dir or not os.path.isdir(jobs_dir):
+ print("[]")
+ sys.exit(0)
+
+tasks = {}
+
+def extract_instance_id(reward_path):
+ """Extract instance_id from a reward file path.
+
+ Harbor structure: $JOBS_DIR///verifier/reward.{json,txt}
+ Trial name format: __<7-char-suffix> e.g. matplotlib__matplotlib-14623__ff4rTkg
+ """
+ reward_dir = os.path.dirname(reward_path)
+ if os.path.basename(reward_dir) == "verifier":
+ trial_dir = os.path.basename(os.path.dirname(reward_dir))
+ else:
+ trial_dir = os.path.basename(reward_dir)
+ return re.sub(r'__[A-Za-z0-9]{7}$', '', trial_dir)
+
+for reward_path in sorted(glob.glob(os.path.join(jobs_dir, "**", "reward.json"), recursive=True)):
+ instance_id = extract_instance_id(reward_path)
+ if not instance_id:
+ continue
+
+ try:
+ with open(reward_path) as f:
+ data = json.load(f)
+ if isinstance(data, dict):
+ values = [v for v in data.values() if isinstance(v, (int, float))]
+ score = sum(values) / len(values) if values else 0.0
+ resolved = score > 0.5
+ elif isinstance(data, (int, float)):
+ resolved = float(data) > 0.5
+ else:
+ resolved = False
+ except Exception:
+ resolved = False
+
+ if instance_id not in tasks:
+ tasks[instance_id] = {"instance_id": instance_id, "resolved": resolved, "cost_usd": 0, "duration_seconds": 0}
+ else:
+ tasks[instance_id]["resolved"] = resolved
+
+for reward_path in sorted(glob.glob(os.path.join(jobs_dir, "**", "reward.txt"), recursive=True)):
+ instance_id = extract_instance_id(reward_path)
+ if not instance_id or instance_id in tasks:
+ continue
+
+ try:
+ with open(reward_path) as f:
+ val = f.read().strip()
+ resolved = val in ("1", "1.0")
+ except Exception:
+ resolved = False
+
+ tasks[instance_id] = {"instance_id": instance_id, "resolved": resolved, "cost_usd": 0, "duration_seconds": 0}
+
+# Extract per-trial cost/duration from per-trial result.json files
+for rpath in sorted(glob.glob(os.path.join(jobs_dir, "**", "result.json"), recursive=True)):
+ try:
+ rdir = os.path.dirname(rpath)
+ trial_id = extract_instance_id(rpath + "/dummy")
+ if os.path.isdir(os.path.join(rdir, "verifier")) or os.path.isdir(os.path.join(rdir, "agent")):
+ with open(rpath) as f:
+ data = json.load(f)
+ if trial_id in tasks:
+ tasks[trial_id]["cost_usd"] = data.get("cost_usd", 0) or 0
+ tasks[trial_id]["duration_seconds"] = data.get("duration_seconds", 0) or 0
+ except Exception:
+ pass
+
+# Extract aggregate cost from job-level result.json and distribute evenly if no per-task costs
+for rpath in sorted(glob.glob(os.path.join(jobs_dir, "*/result.json"))):
+ try:
+ with open(rpath) as f:
+ data = json.load(f)
+ stats = data.get("stats", {})
+ cost = stats.get("cost_usd", 0) or 0
+ if cost > 0 and len(tasks) > 0:
+ uncosted = [t for t in tasks.values() if t["cost_usd"] == 0]
+ if len(uncosted) == len(tasks):
+ per_task = cost / len(tasks)
+ for t in tasks.values():
+ t["cost_usd"] = round(per_task, 4)
+ except Exception:
+ pass
+
+task_list = sorted(tasks.values(), key=lambda t: t["instance_id"])
+print(json.dumps(task_list))
+PYEOF
+)
+}
+
setup_vertex_env() {
if [ -f "${HARNESS_DIR}/.env" ]; then
echo " .env: found at ${HARNESS_DIR}/.env"
diff --git a/benchmarks/programbench-harbor/cmatrix/environment/Dockerfile b/benchmarks/programbench-harbor/cmatrix/environment/Dockerfile
new file mode 100644
index 000000000..d42e3d26f
--- /dev/null
+++ b/benchmarks/programbench-harbor/cmatrix/environment/Dockerfile
@@ -0,0 +1,23 @@
+ARG BASE_IMAGE=programbench/abishekvashok_1776_cmatrix.5c082c6:task_cleanroom_v6
+FROM ${BASE_IMAGE}
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ git rsync curl procps ca-certificates gnupg \
+ && curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
+ && apt-get install -y --no-install-recommends nodejs \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN npm install -g @anthropic-ai/claude-code
+
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh \
+ && export PATH="/root/.local/bin:$PATH" \
+ && uv tool install 'remote-factory @ git+https://github.com/akashgit/remote-factory.git'
+
+RUN useradd -m -s /bin/bash agent 2>/dev/null || true \
+ && chown -R agent:agent /workspace \
+ && mkdir -p /home/agent/.claude /home/agent/.local /home/agent/.cargo \
+ && cp -r /root/.local/* /home/agent/.local/ 2>/dev/null || true \
+ && cp -r /root/.cargo/* /home/agent/.cargo/ 2>/dev/null || true \
+ && chown -R agent:agent /home/agent
diff --git a/benchmarks/programbench-harbor/cmatrix/instruction.md b/benchmarks/programbench-harbor/cmatrix/instruction.md
new file mode 100644
index 000000000..96311ad1f
--- /dev/null
+++ b/benchmarks/programbench-harbor/cmatrix/instruction.md
@@ -0,0 +1,17 @@
+You are reverse-engineering a compiled binary at /workspace/executable.
+
+The binary has EXECUTE-ONLY permissions (mode 111). You CANNOT read its contents. You can only run it.
+
+Your goal: write source code and a compile.sh script that produces a behaviorally-equivalent executable at /workspace/executable.
+
+Strategy:
+1. Run the executable with various arguments to discover its behavior (--help, -h, no args, etc.)
+2. Create test inputs and capture exact outputs
+3. Read any documentation in /workspace/
+4. Write source code matching the observed behavior
+5. Create compile.sh that builds the executable
+6. Test your implementation against the original using differential testing
+
+Back up the original first: cp /workspace/executable /workspace/executable.bak
+Your compile.sh must produce the executable at /workspace/executable.
+The evaluation compares your output against the original on hidden test cases.
diff --git a/benchmarks/programbench-harbor/cmatrix/task.toml b/benchmarks/programbench-harbor/cmatrix/task.toml
new file mode 100644
index 000000000..24e4c981c
--- /dev/null
+++ b/benchmarks/programbench-harbor/cmatrix/task.toml
@@ -0,0 +1,33 @@
+schema_version = "1.3"
+
+[task]
+name = "programbench/cmatrix"
+description = "Reverse-engineer cmatrix binary and produce equivalent source code"
+authors = []
+keywords = ["programbench", "reverse-engineering"]
+
+[metadata]
+difficulty = "hard"
+category = "programming"
+tags = ["binary", "reverse-engineering"]
+
+[environment]
+network_mode = "public"
+build_timeout_sec = 900.0
+cpus = 2
+memory_mb = 4096
+storage_mb = 10240
+gpus = 0
+mcp_servers = []
+
+[environment.env]
+
+[agent]
+timeout_sec = 3600.0
+
+[verifier]
+timeout_sec = 300.0
+
+[verifier.env]
+
+[solution.env]
diff --git a/benchmarks/programbench-harbor/cmatrix/tests/test.sh b/benchmarks/programbench-harbor/cmatrix/tests/test.sh
new file mode 100644
index 000000000..dd9c90c5b
--- /dev/null
+++ b/benchmarks/programbench-harbor/cmatrix/tests/test.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd /workspace
+
+if [ ! -f compile.sh ]; then
+ echo "ERROR: compile.sh not found"
+ echo '{"reward": 0.0}' > /logs/verifier/reward.json
+ exit 0
+fi
+
+echo "Running compile.sh..."
+if ! bash compile.sh 2>&1; then
+ echo "ERROR: compile.sh failed"
+ echo '{"reward": 0.0}' > /logs/verifier/reward.json
+ exit 0
+fi
+
+echo "Packaging submission..."
+tar -czf /logs/verifier/submission.tar.gz \
+ --exclude=.git --exclude=target \
+ --exclude=executable.bak --exclude=./executable \
+ --exclude=.factory --exclude=eval --exclude=factory.md .
+
+if [ -f /logs/verifier/submission.tar.gz ]; then
+ SIZE=$(du -h /logs/verifier/submission.tar.gz | cut -f1)
+ echo "Submission packaged: ${SIZE}"
+ echo '{"reward": 1.0}' > /logs/verifier/reward.json
+else
+ echo "ERROR: Failed to create submission.tar.gz"
+ echo '{"reward": 0.0}' > /logs/verifier/reward.json
+fi
diff --git a/benchmarks/run-featurebench.sh b/benchmarks/run-featurebench.sh
deleted file mode 100755
index bb178c957..000000000
--- a/benchmarks/run-featurebench.sh
+++ /dev/null
@@ -1,641 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# benchmarks/run-featurebench.sh — Standalone CI pipeline for FeatureBench.
-# Runs the complete solve+eval cycle: load instance, clone repo, run Claude Code solver,
-# capture patch, evaluate with FeatureBench harness.
-
-# ── Shared library ──
-
-source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
-
-# ── Configuration ──
-
-INSTANCE_ID="${1:-pypa__packaging.013f3b03.test_metadata.e00b5801.lv1}"
-SOLVER_TIMEOUT="${2:-3600}"
-SPLIT="${3:-fast}"
-
-BENCHMARK="featurebench"
-RUN_ID="ci-featurebench-${TIMESTAMP}"
-RESULT_FILE="${CI_RESULTS_DIR}/${TIMESTAMP}-featurebench.json"
-
-FB_CMD="uvx --from featurebench fb"
-FB_PYTHON="uvx --from featurebench python"
-
-DATASET="LiberCoders/FeatureBench"
-WORKSPACE=""
-
-PASSED=0
-RESOLVED=0
-PASS_RATE=0
-TOTAL=1
-
-# ── Helpers ──
-
-cleanup() {
- local exit_code=$?
- if [ -n "${WORKSPACE}" ] && [ -d "${WORKSPACE}" ]; then
- if [ "${PRESERVE_WORKSPACE:-}" = "1" ]; then
- log "Preserving workspace at ${WORKSPACE} (PRESERVE_WORKSPACE=1)"
- else
- log "Cleaning up workspace"
- rm -rf "${WORKSPACE}"
- fi
- fi
- PASSED="${RESOLVED}"
- DETAILS_JSON='{"pass_rate": '"${PASS_RATE}"', "solver": "'"${BENCHMARK_SOLVER:-factory}"'", "cost_usd": '"${COST_USD:-0}"', "input_tokens": '"${INPUT_TOKENS:-0}"', "output_tokens": '"${OUTPUT_TOKENS:-0}"', "cache_read_tokens": '"${CACHE_READ_TOKENS:-0}"', "cache_creation_tokens": '"${CACHE_CREATION_TOKENS:-0}"'}'
- write_result
- if [ "${STATUS}" = "success" ]; then
- exit 0
- else
- exit "${exit_code:-1}"
- fi
-}
-
-trap cleanup EXIT
-
-# ── Step 1: Parse and display configuration ──
-
-show_banner "FeatureBench"
-log "Step 1: Configuration"
-echo " Instance ID: ${INSTANCE_ID}"
-echo " Dataset: ${DATASET}"
-echo " Split: ${SPLIT}"
-echo " Solver timeout: ${SOLVER_TIMEOUT}s ($(( SOLVER_TIMEOUT / 3600 ))h $(( (SOLVER_TIMEOUT % 3600) / 60 ))m)"
-echo " Run ID: ${RUN_ID}"
-echo " Timestamp: ${TIMESTAMP}"
-echo ""
-
-# ── Step 2: Validate prerequisites ──
-
-log "Step 2: Validating prerequisites"
-
-MISSING=()
-
-if ! command -v python3 &>/dev/null; then
- MISSING+=("python3 >= 3.12 (install via your system package manager)")
-fi
-
-if ! command -v docker &>/dev/null && [ ! -x /usr/bin/docker ]; then
- MISSING+=("docker (install from https://docs.docker.com/get-docker/)")
-fi
-
-if ! command -v claude &>/dev/null; then
- MISSING+=("claude (Claude Code CLI — install from https://docs.anthropic.com/en/docs/claude-code)")
-fi
-
-if [ "${BENCHMARK_SOLVER:-factory}" = "factory" ] && ! command -v factory &>/dev/null; then
- MISSING+=("factory (Factory CLI — install from the factory repo)")
-fi
-
-if [ ${#MISSING[@]} -gt 0 ]; then
- echo " ERROR: Missing prerequisites:"
- for m in "${MISSING[@]}"; do
- echo " - ${m}"
- done
- exit 1
-fi
-
-echo " python3: found"
-echo " docker: found"
-echo " claude: found"
-if [ "${BENCHMARK_SOLVER:-factory}" = "factory" ]; then
- echo " factory: found"
-fi
-echo " solver: ${BENCHMARK_SOLVER:-factory}"
-
-ensure_uvx
-
-# Verify featurebench is usable via uvx
-echo " featurebench: checking availability via uvx..."
-if ! ${FB_CMD} --help &>/dev/null; then
- echo " featurebench: installing via uvx..."
- ${FB_CMD} --help >/dev/null || {
- echo " ERROR: Failed to install/run featurebench via uvx"
- exit 1
- }
-fi
-echo " featurebench: available"
-
-check_gcloud_creds warning
-setup_vertex_env
-
-echo " All prerequisites satisfied."
-echo ""
-
-# ── Step 3: Load instance from HuggingFace ──
-
-log "Step 3: Loading instance ${INSTANCE_ID} from ${DATASET}"
-
-INSTANCE_JSON="$(mktemp /tmp/featurebench-instance-XXXXXX.json)"
-
-${FB_PYTHON} -c "
-import json, sys
-from datasets import load_dataset
-
-ds = load_dataset('${DATASET}', split='${SPLIT}')
-matches = [x for x in ds if x['instance_id'] == '${INSTANCE_ID}']
-if not matches:
- print('ERROR: Instance ${INSTANCE_ID} not found in ${DATASET} (split=${SPLIT})', file=sys.stderr)
- sys.exit(1)
-instance = matches[0]
-repo = instance['repo']
-# Normalize repo format: __ -> / for GitHub clone URLs
-if '/' not in repo and '__' in repo:
- repo = repo.replace('__', '/', 1)
-json.dump({
- 'instance_id': instance['instance_id'],
- 'repo': repo,
- 'base_commit': instance['base_commit'],
- 'problem_statement': instance['problem_statement'],
- 'patch': instance.get('patch', ''),
-}, open('${INSTANCE_JSON}', 'w'), indent=2)
-print(f'Loaded: {instance[\"instance_id\"]}')
-print(f'Repo: {repo}')
-print(f'Commit: {instance[\"base_commit\"][:12]}...')
-"
-
-if [ ! -s "${INSTANCE_JSON}" ]; then
- echo " ERROR: Failed to load instance data"
- exit 1
-fi
-
-REPO="$(python3 -c "import json; print(json.load(open('${INSTANCE_JSON}'))['repo'])")"
-BASE_COMMIT="$(python3 -c "import json; print(json.load(open('${INSTANCE_JSON}'))['base_commit'])")"
-
-echo " Instance loaded successfully."
-echo ""
-
-# ── Step 4: Setup workspace ──
-
-log "Step 4: Setting up workspace"
-
-WORKSPACE="$(mktemp -d /tmp/featurebench-workspace-XXXXXX)"
-echo " Workspace: ${WORKSPACE}"
-echo " Cloning https://github.com/${REPO}..."
-
-git clone --quiet "https://github.com/${REPO}.git" "${WORKSPACE}/repo"
-cd "${WORKSPACE}/repo"
-git checkout --quiet "${BASE_COMMIT}"
-
-echo " Checked out ${BASE_COMMIT:0:12}"
-echo " Working directory: ${WORKSPACE}/repo"
-
-# Apply mask patch — removes function bodies the solver must implement
-MASK_PATCH="${WORKSPACE}/mask_patch.diff"
-python3 -c "import json; open('${MASK_PATCH}', 'w').write(json.load(open('${INSTANCE_JSON}'))['patch'])"
-if [ -s "${MASK_PATCH}" ]; then
- git apply --whitespace=nowarn "${MASK_PATCH}" 2>/dev/null \
- && git add -A && git commit --quiet --amend --no-edit \
- && git reflog expire --expire=now --all && git gc --prune=now --quiet \
- && MASKED_COMMIT=$(git rev-parse HEAD) \
- && echo " Applied mask patch (function bodies removed, baked into commit ${MASKED_COMMIT:0:12})" \
- || echo " WARNING: Mask patch failed to apply"
-else
- echo " No mask patch in dataset"
-fi
-
-echo ""
-
-# ── Step 5: Run solver (Factory CEO) ──
-
-log "Step 5: Running solver [${BENCHMARK_SOLVER:-factory}] (timeout: ${SOLVER_TIMEOUT}s)"
-echo " Started at: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
-
-SOLVER_PROMPT_FILE="${WORKSPACE}/solver_prompt.txt"
-python3 -c "
-import json, os
-
-with open('${INSTANCE_JSON}') as f:
- instance = json.load(f)
-
-problem_statement = instance['problem_statement']
-problem_statement = problem_statement.replace('/testbed/', './')
-
-prompt = '''You are implementing a new feature in an open-source Python project.
-
-## Problem Statement
-
-''' + problem_statement + '''
-
-## Instructions
-
-1. Read the problem statement carefully — it describes a feature to implement
-2. Explore the repository to understand the codebase architecture
-3. Implement the feature as described in the problem statement
-4. The problem statement contains detailed interface specifications — follow them exactly
-5. The source files have had their function bodies REMOVED — you must write the implementations
-6. Look for functions/methods that have empty bodies or just contain pass/blank lines
-7. The evaluation will use NEW tests (not the ones in the repo) to verify your implementation
-8. Focus on implementing the interfaces, classes, and functions described in the problem statement
-9. Make sure you do not break existing functionality
-
-IMPORTANT: Function bodies have been removed from the source files.
-You MUST implement them based on the specifications in the problem statement.
-
-The repository is available at the current working directory.'''
-with open('${SOLVER_PROMPT_FILE}', 'w') as f:
- f.write(prompt)
-"
-
-if [ -n "${ANTHROPIC_VERTEX_PROJECT_ID:-}" ]; then
- echo " Using Vertex AI (project: ${ANTHROPIC_VERTEX_PROJECT_ID})"
-fi
-
-cd "${WORKSPACE}/repo"
-
-export_claude_env
-
-# Temporarily allow failures — Steps 6-9 must always run regardless of solver/post-processing outcome
-set +e
-
-SOLVER_LOG="${WORKSPACE}/solver_output.log"
-SOLVER_EXIT=0
-
-if [ "${BENCHMARK_SOLVER:-factory}" = "claude-code" ]; then
- # Raw Claude Code path
- timeout "${SOLVER_TIMEOUT}" claude -p "$(cat "${SOLVER_PROMPT_FILE}")" \
- --model "${ANTHROPIC_MODEL}" \
- --verbose --max-turns 200 \
- --permission-mode bypassPermissions \
- --output-format stream-json \
- 2>&1 | tee "${SOLVER_LOG}" | tail -50 || true
- SOLVER_EXIT=${PIPESTATUS[0]}
-else
- # Factory CEO path — pre-seed factory state so factory detect returns has_factory
- PROBLEM_STMT=$(python3 -c "
-import json
-with open('${INSTANCE_JSON}') as f:
- inst = json.load(f)
-ps = inst['problem_statement'].replace('/testbed/', './')
-print(ps)
-")
-
- cat > "${WORKSPACE}/repo/factory.md" << FACTORYEOF
----
-goal: Implement empty function bodies as described below
----
-
-## Scope
-src/**/*.py
-
-## Eval
-eval_command: python -m pytest tests/ -x -q --timeout 60
-eval_threshold: 0.3
-
-## Smoke Test
-python -c "import packaging; print('OK')"
-
-## Task
-
-${PROBLEM_STMT}
-FACTORYEOF
-
- mkdir -p "${WORKSPACE}/repo/.factory"
- cat > "${WORKSPACE}/repo/.factory/config.json" << CONFIGEOF
-{
- "eval_command": "python -m pytest tests/ -x -q --timeout 60",
- "eval_threshold": 0.3,
- "target_branch": "HEAD",
- "smoke_test": "python -c \"import packaging; print(OK)\"",
- "hard_constraints": [],
- "eval_spec": []
-}
-CONFIGEOF
-
- cat > "${WORKSPACE}/repo/.factory/eval_profile.json" << EVALEOF
-{
- "dimensions": [
- {"name": "tests", "weight": 1.0, "command": "python -m pytest tests/ -x -q --timeout 60"}
- ],
- "human_reviewed": true
-}
-EVALEOF
-
- mkdir -p "${WORKSPACE}/repo/eval"
- cat > "${WORKSPACE}/repo/eval/score.py" << SCOREEOF
-import subprocess, json, sys
-r = subprocess.run(["python", "-m", "pytest", "tests/", "-x", "-q", "--timeout", "60"], capture_output=True, text=True)
-s = 1.0 if r.returncode == 0 else 0.0
-json.dump({"composite": s, "dimensions": {"tests": {"score": s, "weight": 1.0}}}, sys.stdout)
-SCOREEOF
-
- cd "${WORKSPACE}/repo"
- git add -A
- git commit -m "factory: pre-seed config for improve mode"
-
- FACTORY_CEO_MAX_RESPAWNS=0 \
- timeout "${SOLVER_TIMEOUT}" factory ceo . \
- --headless \
- --no-github \
- --mode improve \
- --focus "Implement all empty function bodies in the source files. Functions have had their bodies removed and need to be reimplemented based on their signatures, docstrings, and the project context." \
- 2>&1 | tee "${SOLVER_LOG}" | tail -50 || true
- SOLVER_EXIT=${PIPESTATUS[0]}
-fi
-
-if [ "${SOLVER_EXIT}" -eq 124 ]; then
- echo " Solver timed out after ${SOLVER_TIMEOUT}s"
-elif [ "${SOLVER_EXIT}" -ne 0 ]; then
- echo " Solver exited with code ${SOLVER_EXIT}"
-fi
-
-# Extract cost and token data from solver output
-COST_USD=0
-INPUT_TOKENS=0
-OUTPUT_TOKENS=0
-CACHE_READ_TOKENS=0
-CACHE_CREATION_TOKENS=0
-
-if [ "${BENCHMARK_SOLVER:-factory}" = "claude-code" ]; then
- if [ -f "${SOLVER_LOG}" ]; then
- COST_DATA=$(grep '"type":"result"' "${SOLVER_LOG}" 2>/dev/null | tail -1 | python3 -c "
-import sys, json
-try:
- data = json.loads(sys.stdin.readline())
- print(f'COST_USD={data.get(\"total_cost_usd\", 0) or 0}')
- u = data.get('usage', {})
- print(f'INPUT_TOKENS={u.get(\"input_tokens\", 0)}')
- print(f'OUTPUT_TOKENS={u.get(\"output_tokens\", 0)}')
- print(f'CACHE_READ_TOKENS={u.get(\"cache_read_input_tokens\", 0)}')
- print(f'CACHE_CREATION_TOKENS={u.get(\"cache_creation_input_tokens\", 0)}')
-except: pass
-" 2>/dev/null || true)
- eval "${COST_DATA}" 2>/dev/null || true
- fi
-else
- EVENTS_FILE="${WORKSPACE}/repo/.factory/events.jsonl"
- if [ -f "${EVENTS_FILE}" ]; then
- COST_DATA=$(python3 -c "
-import json
-total_cost = 0
-total_input = 0
-total_output = 0
-total_cache_read = 0
-total_cache_create = 0
-for line in open('${EVENTS_FILE}'):
- try:
- e = json.loads(line)
- if e.get('type') == 'agent.completed':
- d = e.get('data', {})
- total_cost += d.get('total_cost_usd', 0) or 0
- total_input += d.get('input_tokens', 0)
- total_output += d.get('output_tokens', 0)
- total_cache_read += d.get('cache_read_tokens', 0)
- except: pass
-print(f'COST_USD={total_cost}')
-print(f'INPUT_TOKENS={total_input}')
-print(f'OUTPUT_TOKENS={total_output}')
-print(f'CACHE_READ_TOKENS={total_cache_read}')
-print(f'CACHE_CREATION_TOKENS={total_cache_create}')
-" 2>/dev/null)
- eval "${COST_DATA}" 2>/dev/null || true
- fi
-fi
-
-# Post-processing: recover factory branch/worktree changes (factory solver only)
-if [ "${BENCHMARK_SOLVER:-factory}" = "factory" ]; then
- cd "${WORKSPACE}/repo"
-
- # Strategy 1: Merge surviving factory branch
- FACTORY_BRANCH=$(git branch --list 'factory/*' | head -1 | tr -d ' *')
- if [ -n "$FACTORY_BRANCH" ]; then
- echo "Merging factory branch: $FACTORY_BRANCH"
- git merge "$FACTORY_BRANCH" --no-edit 2>/dev/null || git cherry-pick "$FACTORY_BRANCH" --no-edit 2>/dev/null || true
- fi
-
- # Strategy 2: Recover orphaned commits via git fsck
- # Pick the orphan that descends from HEAD (not BASE_COMMIT — the amend changed the SHA)
- RECOVERY_BASE=$(git rev-parse HEAD)
- if [ -z "$FACTORY_BRANCH" ]; then
- echo "No factory branch, finding orphaned commits..."
- ORPHAN_COMMITS=$(git fsck --unreachable --no-reflogs 2>/dev/null | grep 'unreachable commit' | awk '{print $3}')
- if [ -n "$ORPHAN_COMMITS" ]; then
- BEST_COMMIT=""
- BEST_TIME=0
- for SHA in $ORPHAN_COMMITS; do
- if ! git merge-base --is-ancestor "${RECOVERY_BASE}" "$SHA" 2>/dev/null; then
- continue
- fi
- COMMIT_TIME=$(git show -s --format='%ct' "$SHA" 2>/dev/null || echo 0)
- if [ "$COMMIT_TIME" -gt "$BEST_TIME" ]; then
- BEST_TIME=$COMMIT_TIME
- BEST_COMMIT=$SHA
- fi
- done
- if [ -n "$BEST_COMMIT" ]; then
- echo "Recovering from orphan tip: $BEST_COMMIT (descends from ${RECOVERY_BASE:0:12})"
- echo " Message: $(git log -1 --format='%s' $BEST_COMMIT 2>/dev/null)"
- git checkout "$BEST_COMMIT" -- . 2>/dev/null || true
- git checkout HEAD -- .factory/ eval/ factory.md 2>/dev/null || true
- rm -rf .factory/ eval/ factory.md 2>/dev/null || true
- else
- echo "No orphan commits descend from ${RECOVERY_BASE:0:12}"
- fi
- fi
- fi
-
- # Strategy 3: Recover from surviving worktree directories
- for wt in .factory-worktrees/*/; do
- if [ -d "$wt" ]; then
- echo "Recovering files from worktree: $wt"
- rsync -a --exclude='.git' --exclude='.factory' "$wt" ./ 2>/dev/null || true
- fi
- done
-fi
-
-set -e
-
-echo " Finished at: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
-echo ""
-
-# ── Step 6: Capture patch ──
-
-log "Step 6: Capturing patch"
-
-cd "${WORKSPACE}/repo"
-PATCH_FILE="${WORKSPACE}/model_patch.diff"
-
-# Diff against the masked commit (not BASE_COMMIT) so the patch captures
-# the full function body additions from masked→implemented
-DIFF_BASE="${MASKED_COMMIT:-${BASE_COMMIT}}"
-git diff "${DIFF_BASE}" -- . ':!.factory' ':!eval' ':!factory.md' > "${PATCH_FILE}"
-
-# Fallback: if committed diff is empty, try unstaged diff too
-if [ ! -s "${PATCH_FILE}" ]; then
- echo " No committed changes found, trying unstaged diff..."
- git diff -- . ':!.factory' ':!eval' ':!factory.md' > "${PATCH_FILE}"
-fi
-
-PATCH_SIZE="$(wc -c < "${PATCH_FILE}")"
-if [ "${PATCH_SIZE}" -eq 0 ]; then
- echo " WARNING: Solver produced no changes (empty diff)"
- echo " Evaluation will proceed but instance will not be resolved."
-else
- PATCH_LINES="$(wc -l < "${PATCH_FILE}")"
- PATCH_FILES="$(git diff "${DIFF_BASE}" --name-only -- . ':!.factory' ':!eval' ':!factory.md' | wc -l)"
- echo " Patch: ${PATCH_LINES} lines across ${PATCH_FILES} file(s)"
-fi
-echo ""
-
-# ── Step 7: Write predictions.jsonl ──
-
-log "Step 7: Writing predictions.jsonl"
-
-PREDICTIONS_DIR="$(mktemp -d /tmp/featurebench-predictions-XXXXXX)"
-PREDICTIONS_FILE="${PREDICTIONS_DIR}/predictions.jsonl"
-
-python3 -c "
-import json
-
-with open('${PATCH_FILE}') as f:
- model_patch = f.read()
-
-prediction = {
- 'instance_id': '${INSTANCE_ID}',
- 'model_name_or_path': 'claude-code',
- 'model_patch': model_patch,
- 'n_attempt': 1,
- 'success': True,
-}
-with open('${PREDICTIONS_FILE}', 'w') as f:
- json.dump(prediction, f)
- f.write('\n')
-print(' Written to ${PREDICTIONS_FILE}')
-"
-
-echo ""
-
-# ── Step 8: Run FeatureBench evaluation ──
-
-log "Step 8: Running FeatureBench evaluation"
-echo " This may take several minutes (Docker image pull + evaluation)..."
-
-cd "${HARNESS_DIR}"
-
-EVAL_EXIT=0
-${FB_CMD} eval \
- -p "${PREDICTIONS_FILE}" \
- --split "${SPLIT}" \
- --task-id "${INSTANCE_ID}" \
- --n-concurrent 1 \
- 2>&1 || EVAL_EXIT=$?
-
-if [ "${EVAL_EXIT}" -ne 0 ]; then
- echo " ERROR: FeatureBench evaluation failed with exit code ${EVAL_EXIT}"
- exit 1
-fi
-
-echo " Evaluation complete."
-echo ""
-
-# ── Step 9: Extract and report results ──
-
-log "Step 9: Extracting results"
-
-RESULTS_JSON=""
-
-# Check predictions directory first (fb eval writes here)
-for candidate in "${PREDICTIONS_DIR}"/eval_outputs/"${INSTANCE_ID}"/attempt-*/report.json; do
- if [ -f "${candidate}" ]; then
- RESULTS_JSON="${candidate}"
- fi
-done
-
-# Fallback: check predictions dir top-level report
-if [ -z "${RESULTS_JSON}" ] && [ -f "${PREDICTIONS_DIR}/report.json" ]; then
- RESULTS_JSON="${PREDICTIONS_DIR}/report.json"
-fi
-
-# Fallback: check harness runs directory
-if [ -z "${RESULTS_JSON}" ]; then
- for candidate in "${HARNESS_DIR}"/runs/*/eval_outputs/"${INSTANCE_ID}"/attempt-*/report.json; do
- if [ -f "${candidate}" ]; then
- RESULTS_JSON="${candidate}"
- fi
- done
-fi
-
-if [ -z "${RESULTS_JSON}" ]; then
- for candidate in "${HARNESS_DIR}"/runs/*/report.json; do
- if [ -f "${candidate}" ]; then
- RESULTS_JSON="${candidate}"
- fi
- done
-fi
-
-if [ -z "${RESULTS_JSON}" ]; then
- for candidate in "${HARNESS_DIR}"/*.json; do
- if [ -f "${candidate}" ] && python3 -c "
-import json, sys
-with open('${candidate}') as f:
- data = json.load(f)
-if 'attempt_1' in data or '${INSTANCE_ID}' in data:
- sys.exit(0)
-sys.exit(1)
-" 2>/dev/null; then
- RESULTS_JSON="${candidate}"
- break
- fi
- done
-fi
-
-if [ -n "${RESULTS_JSON}" ] && [ -f "${RESULTS_JSON}" ]; then
- echo " Results file: ${RESULTS_JSON}"
- eval "$(python3 -c "
-import json
-
-with open('${RESULTS_JSON}') as f:
- data = json.load(f)
-
-resolved = 0
-total = 0
-pass_rate = 0.0
-
-# Per-instance report: {instance_id: {resolved: bool, pass_rate: float, ...}}
-if '${INSTANCE_ID}' in data:
- result = data['${INSTANCE_ID}']
- total = 1
- if result.get('resolved', False):
- resolved = 1
- pass_rate = result.get('pass_rate', 0.0)
-
-# Summary report: {attempt_1: {resolved_rate: float, pass_rate: float, ...}}
-elif 'attempt_1' in data:
- attempt = data['attempt_1']
- total = attempt.get('total_instances', attempt.get('completed_instances', 1))
- resolved = attempt.get('resolved_instances', 0)
- pass_rate = attempt.get('pass_rate', 0.0)
-
-# Flat dict with resolved/pass_rate keys
-elif 'resolved' in data:
- total = 1
- if data.get('resolved', False):
- resolved = 1
- pass_rate = data.get('pass_rate', 0.0)
-
-print(f'RESOLVED={resolved}')
-print(f'TOTAL={max(total, 1)}')
-print(f'PASS_RATE={pass_rate}')
-")"
-else
- echo " No results files found. Marking as unresolved."
- RESOLVED=0
- TOTAL=1
- PASS_RATE=0
-fi
-
-echo ""
-echo "============================================"
-if [ "${RESOLVED}" -gt 0 ]; then
- echo " Result: RESOLVED (${RESOLVED}/${TOTAL})"
-else
- echo " Result: NOT RESOLVED (${RESOLVED}/${TOTAL})"
-fi
-echo " Pass Rate: ${PASS_RATE}"
-echo "============================================"
-echo ""
-
-STATUS="success"
-
-# cleanup trap will write the final result JSON and exit 0
diff --git a/benchmarks/run-full-eval.sh b/benchmarks/run-full-eval.sh
new file mode 100755
index 000000000..6dc48cfaa
--- /dev/null
+++ b/benchmarks/run-full-eval.sh
@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# benchmarks/run-full-eval.sh — Run ALL tasks in a benchmark dataset through Harbor.
+# Thin wrapper that dispatches to run-harbor.sh --all.
+# For BENCHMARK=all, spawns parallel runs for each benchmark.
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${SCRIPT_DIR}/lib.sh"
+source "${SCRIPT_DIR}/config.sh"
+
+# ── Defaults ──
+
+BENCHMARK=""
+BENCHMARK_SOLVER="${BENCHMARK_SOLVER:-factory}"
+CONCURRENCY="${CONCURRENCY:-5}"
+SOLVER_TIMEOUT="${SOLVER_TIMEOUT:-3600}"
+SPLIT=""
+LIMIT_TASKS=""
+PRESERVE_WORKSPACE="${PRESERVE_WORKSPACE:-}"
+
+# ── Usage ──
+
+usage() {
+ echo "Usage: $(basename "$0") [options]"
+ echo ""
+ echo "Benchmarks: swebench, featurebench, terminalbench, programbench, legacybench, all"
+ echo ""
+ echo "Options:"
+ echo " --solver factory|claude-code Solver to use (default: factory)"
+ echo " --concurrency N Number of concurrent tasks (default: 5)"
+ echo " --timeout N Per-task solver timeout in seconds (default: 3600)"
+ echo " --split S Dataset split (featurebench only: full, lite)"
+ echo " --limit N Maximum number of tasks to run (optional)"
+ echo " --preserve Preserve Harbor jobs directory after completion"
+ echo " -h, --help Show this help message"
+ exit "${1:-0}"
+}
+
+# ── Argument parsing ──
+
+if [ $# -lt 1 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
+ usage
+fi
+
+BENCHMARK="$1"
+shift
+
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --solver) BENCHMARK_SOLVER="$2"; shift 2 ;;
+ --concurrency) CONCURRENCY="$2"; shift 2 ;;
+ --timeout) SOLVER_TIMEOUT="$2"; shift 2 ;;
+ --split) SPLIT="$2"; shift 2 ;;
+ --limit) LIMIT_TASKS="$2"; shift 2 ;;
+ --preserve) PRESERVE_WORKSPACE="1"; shift ;;
+ -h|--help) usage ;;
+ *) echo "ERROR: Unknown option '$1'"; usage 1 ;;
+ esac
+done
+
+# ── Handle 'all' — spawn parallel runs for each benchmark ──
+
+if [ "${BENCHMARK}" = "all" ]; then
+ log "Running full eval for ALL benchmarks (parallel)"
+ echo ""
+ FAILED=0
+ FAILED_NAMES=()
+ declare -A PIDS
+
+ for BENCH in $(benchmark_all_names); do
+ "${SCRIPT_DIR}/run-harbor.sh" "${BENCH}" --all \
+ --solver "${BENCHMARK_SOLVER}" \
+ --concurrency "${CONCURRENCY}" \
+ --timeout "${SOLVER_TIMEOUT}" \
+ ${LIMIT_TASKS:+--limit "${LIMIT_TASKS}"} \
+ ${SPLIT:+--split "${SPLIT}"} \
+ ${PRESERVE_WORKSPACE:+--preserve} &
+ PIDS[${BENCH}]=$!
+ done
+
+ for BENCH in $(benchmark_all_names); do
+ if ! wait "${PIDS[${BENCH}]}"; then
+ log "FAILED: ${BENCH}"
+ FAILED=$((FAILED + 1))
+ FAILED_NAMES+=("${BENCH}")
+ fi
+ done
+
+ echo ""
+ if [ "${FAILED}" -gt 0 ]; then
+ log "${FAILED} benchmark(s) failed: ${FAILED_NAMES[*]}"
+ exit 1
+ fi
+ log "All benchmarks complete"
+ exit 0
+fi
+
+# ── Single benchmark — dispatch to run-harbor.sh ──
+
+exec "${SCRIPT_DIR}/run-harbor.sh" "${BENCHMARK}" --all \
+ --solver "${BENCHMARK_SOLVER}" \
+ --concurrency "${CONCURRENCY}" \
+ --timeout "${SOLVER_TIMEOUT}" \
+ ${SPLIT:+--split "${SPLIT}"} \
+ ${LIMIT_TASKS:+--limit "${LIMIT_TASKS}"} \
+ ${PRESERVE_WORKSPACE:+--preserve}
diff --git a/benchmarks/run-harbor.sh b/benchmarks/run-harbor.sh
new file mode 100755
index 000000000..5040aa850
--- /dev/null
+++ b/benchmarks/run-harbor.sh
@@ -0,0 +1,633 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# benchmarks/run-harbor.sh — Unified Harbor runner for all benchmarks.
+# Supports both single-task (--task) and full-dataset (--all) modes.
+#
+# Usage:
+# run-harbor.sh --task [--timeout N] [--split S] [--preserve] [--solver S]
+# run-harbor.sh --all [--concurrency N] [--timeout N] [--split S] [--limit N] [--preserve] [--solver S]
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${SCRIPT_DIR}/lib.sh"
+source "${SCRIPT_DIR}/config.sh"
+
+# ── Argument parsing ──
+
+if [ $# -lt 2 ]; then
+ echo "Usage:"
+ echo " run-harbor.sh --task [--timeout N] [--split S] [--preserve] [--solver S]"
+ echo " run-harbor.sh --all [--concurrency N] [--timeout N] [--split S] [--limit N] [--preserve] [--solver S]"
+ exit 1
+fi
+
+BENCHMARK="$1"
+shift
+
+MODE=""
+INSTANCE_ID=""
+SOLVER_TIMEOUT="3600"
+SPLIT=""
+CONCURRENCY="5"
+LIMIT_TASKS=""
+PRESERVE_WORKSPACE="${PRESERVE_WORKSPACE:-}"
+BENCHMARK_SOLVER="${BENCHMARK_SOLVER:-factory}"
+
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --task) MODE="task"; INSTANCE_ID="$2"; shift 2 ;;
+ --all) MODE="all"; shift ;;
+ --timeout) SOLVER_TIMEOUT="$2"; shift 2 ;;
+ --split) SPLIT="$2"; shift 2 ;;
+ --concurrency) CONCURRENCY="$2"; shift 2 ;;
+ --limit) LIMIT_TASKS="$2"; shift 2 ;;
+ --preserve) PRESERVE_WORKSPACE="1"; shift ;;
+ --solver) BENCHMARK_SOLVER="$2"; shift 2 ;;
+ *) echo "ERROR: Unknown option '$1'"; exit 1 ;;
+ esac
+done
+
+if [ -z "${MODE}" ]; then
+ echo "ERROR: Must specify --task or --all"
+ exit 1
+fi
+
+# ── Load benchmark configuration ──
+
+benchmark_config "${BENCHMARK}"
+benchmark_dataset "${BENCHMARK}" "${SPLIT}"
+
+TASK_NAME=""
+if [ "${MODE}" = "task" ]; then
+ TASK_NAME="${INSTANCE_ID}"
+ INSTANCE_ID=$(benchmark_instance_id "${BENCHMARK}" "${TASK_NAME}")
+fi
+
+if [ -n "${BENCH_LOCAL_PATH}" ]; then
+ if [ "${MODE}" = "task" ]; then
+ BENCH_LOCAL_PATH="${BENCH_LOCAL_PATH}/${TASK_NAME}"
+ fi
+ if [ ! -d "${BENCH_LOCAL_PATH}" ]; then
+ echo "ERROR: Task directory not found: ${BENCH_LOCAL_PATH}"
+ exit 1
+ fi
+fi
+
+# ── Setup ──
+
+HARBOR_DATASET="${BENCH_DATASET:-local}"
+
+if [ "${MODE}" = "task" ]; then
+ RUN_ID="ci-${BENCHMARK}-${TIMESTAMP}"
+ RESULT_FILE="${CI_RESULTS_DIR}/${TIMESTAMP}-${BENCHMARK}-${BENCHMARK_SOLVER}.json"
+ TOTAL=1
+else
+ RUN_ID="full-${BENCHMARK}-${TIMESTAMP}"
+ RESULT_FILE="${CI_RESULTS_DIR}/${TIMESTAMP}-${BENCHMARK}-full.json"
+ INSTANCE_ID="full-eval"
+ TOTAL=0
+fi
+
+JOBS_DIR=""
+RESULTS_DIR=""
+PASSED=0
+RESOLVED=0
+PASS_RATE=0
+COST_USD=0
+INPUT_TOKENS=0
+OUTPUT_TOKENS=0
+CACHE_READ_TOKENS=0
+CACHE_CREATION_TOKENS=0
+TASKS_JSON="[]"
+
+# ── Cleanup trap ──
+
+cleanup() {
+ local exit_code=$?
+
+ HARBOR_EXCEPTION=""
+ if [ -n "${JOBS_DIR}" ] && [ -d "${JOBS_DIR}" ]; then
+ local exc_file
+ exc_file=$(find "${JOBS_DIR}" -maxdepth 4 -name 'exception.txt' -type f 2>/dev/null | head -1)
+ if [ -n "${exc_file}" ] && [ -f "${exc_file}" ]; then
+ mkdir -p "${CI_RESULTS_DIR}"
+ cp "${exc_file}" "${CI_RESULTS_DIR}/${TIMESTAMP}-${BENCHMARK}-exception.txt"
+ HARBOR_EXCEPTION=$(cat "${exc_file}")
+ echo "--- Harbor exception ---" >&2
+ echo "${HARBOR_EXCEPTION}" >&2
+ echo "--- end exception ---" >&2
+ fi
+
+ local log_file
+ log_file=$(find "${JOBS_DIR}" -maxdepth 4 -name 'trial.log' -type f 2>/dev/null | head -1)
+ if [ -n "${log_file}" ] && [ -f "${log_file}" ]; then
+ mkdir -p "${CI_RESULTS_DIR}"
+ cp "${log_file}" "${CI_RESULTS_DIR}/${TIMESTAMP}-${BENCHMARK}-trial.log"
+ elif [ -f "${HARBOR_LOG:-}" ]; then
+ mkdir -p "${CI_RESULTS_DIR}"
+ cp "${HARBOR_LOG}" "${CI_RESULTS_DIR}/${TIMESTAMP}-${BENCHMARK}-trial.log"
+ fi
+ elif [ -f "${HARBOR_LOG:-}" ]; then
+ mkdir -p "${CI_RESULTS_DIR}"
+ cp "${HARBOR_LOG}" "${CI_RESULTS_DIR}/${TIMESTAMP}-${BENCHMARK}-trial.log"
+ fi
+
+ LANGFUSE_TRACE_ID=""
+ if [ "${MODE}" = "task" ] && [ -n "${JOBS_DIR}" ] && [ -d "${JOBS_DIR}" ]; then
+ LANGFUSE_TRACE_ID=$(extract_trace_id "${JOBS_DIR}")
+ fi
+
+ # Fall back to the pre-created wrapper trace if the container didn't produce one
+ if [ -z "${LANGFUSE_TRACE_ID}" ] && [ -n "${PRE_TRACE_ID:-}" ]; then
+ LANGFUSE_TRACE_ID="${PRE_TRACE_ID}"
+ fi
+
+ # Close the wrapper trace with duration and status
+ if [ -n "${LANGFUSE_TRACE_ID}" ] && [ -n "${PRE_TRACE_ID:-}" ]; then
+ local end_ts duration_s
+ end_ts="$(date +%s)"
+ duration_s=$(( end_ts - START_TIME ))
+ close_langfuse_trace "${LANGFUSE_TRACE_ID}" "${STATUS}" "${duration_s}"
+ fi
+
+ if [ -n "${JOBS_DIR}" ] && [ -d "${JOBS_DIR}" ]; then
+ if [ "${PRESERVE_WORKSPACE}" = "1" ]; then
+ log "Preserving harbor jobs at ${JOBS_DIR} (--preserve)"
+ else
+ log "Cleaning up harbor jobs directory"
+ rm -rf "${JOBS_DIR}"
+ fi
+ fi
+
+ if [ -n "${RESULTS_DIR}" ] && [ -d "${RESULTS_DIR}" ]; then
+ if [ "${PRESERVE_WORKSPACE}" = "1" ]; then
+ log "Preserving results at ${RESULTS_DIR} (--preserve)"
+ else
+ log "Cleaning up results directory"
+ rm -rf "${RESULTS_DIR}"
+ fi
+ fi
+
+ if [ "${MODE}" = "task" ]; then
+ PASSED="${RESOLVED}"
+ ESCAPED_EXCEPTION=""
+ if [ -n "${HARBOR_EXCEPTION}" ]; then
+ ESCAPED_EXCEPTION=$(python3 -c "import json,sys; print(json.dumps(sys.stdin.read().strip()))" <<< "${HARBOR_EXCEPTION}")
+ fi
+ if [ "${BENCHMARK}" = "featurebench" ]; then
+ if [ -n "${ESCAPED_EXCEPTION}" ]; then
+ DETAILS_JSON='{"pass_rate": '"${PASS_RATE}"', "solver": "'"${BENCHMARK_SOLVER}"'", "cost_usd": '"${COST_USD}"', "input_tokens": '"${INPUT_TOKENS}"', "output_tokens": '"${OUTPUT_TOKENS}"', "cache_read_tokens": '"${CACHE_READ_TOKENS}"', "cache_creation_tokens": '"${CACHE_CREATION_TOKENS}"', "trace_id": "'"${LANGFUSE_TRACE_ID}"'", "exception": '"${ESCAPED_EXCEPTION}"'}'
+ else
+ DETAILS_JSON='{"pass_rate": '"${PASS_RATE}"', "solver": "'"${BENCHMARK_SOLVER}"'", "cost_usd": '"${COST_USD}"', "input_tokens": '"${INPUT_TOKENS}"', "output_tokens": '"${OUTPUT_TOKENS}"', "cache_read_tokens": '"${CACHE_READ_TOKENS}"', "cache_creation_tokens": '"${CACHE_CREATION_TOKENS}"', "trace_id": "'"${LANGFUSE_TRACE_ID}"'"}'
+ fi
+ else
+ if [ -n "${ESCAPED_EXCEPTION}" ]; then
+ DETAILS_JSON='{"solver": "'"${BENCHMARK_SOLVER}"'", "cost_usd": '"${COST_USD}"', "input_tokens": '"${INPUT_TOKENS}"', "output_tokens": '"${OUTPUT_TOKENS}"', "cache_read_tokens": '"${CACHE_READ_TOKENS}"', "cache_creation_tokens": '"${CACHE_CREATION_TOKENS}"', "trace_id": "'"${LANGFUSE_TRACE_ID}"'", "exception": '"${ESCAPED_EXCEPTION}"'}'
+ else
+ DETAILS_JSON='{"solver": "'"${BENCHMARK_SOLVER}"'", "cost_usd": '"${COST_USD}"', "input_tokens": '"${INPUT_TOKENS}"', "output_tokens": '"${OUTPUT_TOKENS}"', "cache_read_tokens": '"${CACHE_READ_TOKENS}"', "cache_creation_tokens": '"${CACHE_CREATION_TOKENS}"', "trace_id": "'"${LANGFUSE_TRACE_ID}"'"}'
+ fi
+ fi
+ export DETAILS_JSON
+ write_result
+ else
+ local end_time duration
+ end_time="$(date +%s)"
+ duration=$(( end_time - START_TIME ))
+ mkdir -p "${CI_RESULTS_DIR}"
+
+ python3 -c "
+import json, sys
+
+tasks = json.loads('''${TASKS_JSON:-[]}''')
+passed = sum(1 for t in tasks if t.get('resolved'))
+total = len(tasks)
+cost = sum(t.get('cost_usd', 0) for t in tasks)
+
+result = {
+ 'benchmark': '${BENCHMARK}',
+ 'eval_type': 'full',
+ 'solver': '${BENCHMARK_SOLVER}',
+ 'passed': passed,
+ 'total': total,
+ 'score': round(passed / max(total, 1), 4),
+ 'duration_seconds': ${duration},
+ 'status': '${STATUS}',
+ 'timestamp': '${TIMESTAMP}',
+ 'details': {
+ 'cost_usd': round(cost, 4),
+ 'concurrency': ${CONCURRENCY},
+ 'dataset': '${HARBOR_DATASET}'
+ },
+ 'tasks': tasks
+}
+json.dump(result, sys.stdout, indent=2)
+print()
+" > "${RESULT_FILE}" 2>/dev/null || true
+
+ if [ -f "${RESULT_FILE}" ]; then
+ echo ""
+ log "Results written to ${RESULT_FILE}"
+ cat "${RESULT_FILE}"
+ fi
+ fi
+
+ if [ "${STATUS}" = "success" ]; then
+ exit 0
+ else
+ exit "${exit_code:-1}"
+ fi
+}
+
+trap cleanup EXIT
+
+# ── Step 1: Display configuration ──
+
+if [ "${MODE}" = "task" ]; then
+ show_banner "${BENCHMARK}"
+ log "Step 1: Configuration"
+ echo " Instance ID: ${INSTANCE_ID}"
+ echo " Dataset: ${BENCH_DATASET:-${BENCH_LOCAL_PATH}}"
+ echo " Solver timeout: ${SOLVER_TIMEOUT}s ($(( SOLVER_TIMEOUT / 3600 ))h $(( (SOLVER_TIMEOUT % 3600) / 60 ))m)"
+ echo " Run ID: ${RUN_ID}"
+ echo " Timestamp: ${TIMESTAMP}"
+else
+ show_banner "Full Eval — ${BENCHMARK}"
+ log "Step 1: Configuration"
+ echo " Benchmark: ${BENCHMARK}"
+ echo " Dataset: ${BENCH_DATASET:-${BENCH_LOCAL_PATH}}"
+ echo " Solver: ${BENCHMARK_SOLVER}"
+ echo " Concurrency: ${CONCURRENCY}"
+ echo " Solver timeout: ${SOLVER_TIMEOUT}s ($(( SOLVER_TIMEOUT / 3600 ))h $(( (SOLVER_TIMEOUT % 3600) / 60 ))m)"
+ echo " Run ID: ${RUN_ID}"
+ echo " Timestamp: ${TIMESTAMP}"
+fi
+echo ""
+
+# ── Step 2: Validate prerequisites ──
+
+log "Step 2: Validating prerequisites"
+validate_prerequisites
+
+if [ -n "${BENCH_POST_EVAL_CMD}" ]; then
+ echo " programbench: checking availability via uvx..."
+ if ! uvx programbench --help &>/dev/null 2>&1; then
+ echo " programbench: will be installed on first use via uvx"
+ fi
+ echo " programbench: ready"
+fi
+
+if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
+ echo " ANTHROPIC_API_KEY: set"
+else
+ setup_vertex_env
+ if [ -n "${ANTHROPIC_VERTEX_PROJECT_ID:-}" ]; then
+ echo " Vertex AI: configured (project: ${ANTHROPIC_VERTEX_PROJECT_ID})"
+ else
+ echo " WARNING: No ANTHROPIC_API_KEY or Vertex AI configuration found."
+ echo " Harbor's agent requires API access."
+ fi
+fi
+
+echo " All prerequisites satisfied."
+echo ""
+
+# ── Step 3: Run Harbor evaluation ──
+
+log "Step 3: Running Harbor evaluation"
+
+if [ "${MODE}" = "task" ]; then
+ JOBS_DIR="$(mktemp -d "/tmp/${BENCHMARK}-jobs-XXXXXX")"
+else
+ JOBS_DIR="$(mktemp -d "/tmp/full-eval-${BENCHMARK}-jobs-XXXXXX")"
+fi
+export JOBS_DIR
+echo " Jobs directory: ${JOBS_DIR}"
+echo " Started at: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+TIMEOUT_MULTIPLIER=$(( SOLVER_TIMEOUT / 120 ))
+[ "${TIMEOUT_MULTIPLIER}" -lt 1 ] && TIMEOUT_MULTIPLIER=1
+
+MODEL="anthropic/claude-opus-4-6"
+
+echo " Model: ${MODEL}"
+echo " Timeout mult: ${TIMEOUT_MULTIPLIER}x"
+if [ "${MODE}" = "task" ]; then
+ echo " Instance: ${INSTANCE_ID}"
+else
+ echo " Concurrency: ${CONCURRENCY}"
+fi
+echo ""
+
+cd "${HARNESS_DIR}"
+
+# Build agent args
+if [ "${BENCHMARK_SOLVER}" = "claude-code" ]; then
+ AGENT_ARGS=(--agent claude-code)
+ if [ -n "${BENCH_EXTRA_INSTRUCTION}" ]; then
+ AGENT_ARGS+=(--extra-instruction-path "${HARNESS_DIR}/benchmarks/${BENCH_EXTRA_INSTRUCTION}")
+ fi
+ echo " Agent: claude-code (Harbor built-in)"
+else
+ AGENT_MODULE="${HARNESS_DIR}/benchmarks/factory_harbor_agent.py"
+ export PYTHONPATH="$(dirname "${AGENT_MODULE}"):${PYTHONPATH:-}"
+ AGENT_ARGS=(${BENCH_AGENT_IMPORT_FLAG} "${BENCH_AGENT_CLASS}")
+ echo " Agent: factory (${BENCH_AGENT_CLASS#*:})"
+fi
+
+# Build Harbor command
+HARBOR_CMD=(
+ uvx harbor run
+ --model "${MODEL}"
+ --jobs-dir "${JOBS_DIR}"
+ --agent-timeout-multiplier "${TIMEOUT_MULTIPLIER}"
+)
+
+if [ -n "${BENCH_LOCAL_PATH}" ]; then
+ HARBOR_CMD+=(-p "${BENCH_LOCAL_PATH}")
+else
+ HARBOR_CMD+=(--dataset "${BENCH_DATASET}")
+fi
+
+HARBOR_CMD+=("${AGENT_ARGS[@]}")
+
+if [ "${MODE}" = "task" ]; then
+ case "${BENCH_FILTER_STYLE}" in
+ glob) HARBOR_CMD+=(--include-task-name "*${INSTANCE_ID}") ;;
+ exact) HARBOR_CMD+=(--include-task-name "${INSTANCE_ID}") ;;
+ esac
+ HARBOR_CMD+=(--n-concurrent 1)
+else
+ HARBOR_CMD+=(--n-concurrent "${CONCURRENCY}")
+ [ -n "${LIMIT_TASKS}" ] && HARBOR_CMD+=(--n-tasks "${LIMIT_TASKS}")
+fi
+
+# Allow-agent-host flags (programbench)
+if [ -n "${BENCH_ALLOW_HOSTS}" ]; then
+ for host in ${BENCH_ALLOW_HOSTS}; do
+ HARBOR_CMD+=(--allow-agent-host "${host}")
+ done
+ if [ -n "${ANTHROPIC_VERTEX_PROJECT_ID:-}" ]; then
+ for host in \
+ us-east5-aiplatform.googleapis.com \
+ us-central1-aiplatform.googleapis.com \
+ europe-west1-aiplatform.googleapis.com \
+ oauth2.googleapis.com \
+ www.googleapis.com \
+ storage.googleapis.com \
+ metadata.google.internal; do
+ HARBOR_CMD+=(--allow-agent-host "${host}")
+ done
+ fi
+ LANGFUSE_HOSTNAME=$(extract_langfuse_hostname)
+ if [ -n "${LANGFUSE_HOSTNAME}" ]; then
+ HARBOR_CMD+=(--allow-agent-host "${LANGFUSE_HOSTNAME}")
+ fi
+fi
+
+# Auth-specific --ae flags
+AUTH_AE=()
+if [ -n "${ANTHROPIC_VERTEX_PROJECT_ID:-}" ]; then
+ GCLOUD_ADC="${GOOGLE_APPLICATION_CREDENTIALS:-${HOME}/.config/gcloud/application_default_credentials.json}"
+ echo " Auth mode: Vertex AI (project: ${ANTHROPIC_VERTEX_PROJECT_ID})"
+ AUTH_AE=(
+ --ae "CLAUDE_CODE_USE_VERTEX=1"
+ --ae "ANTHROPIC_VERTEX_PROJECT_ID=${ANTHROPIC_VERTEX_PROJECT_ID}"
+ --ae "CLOUD_ML_REGION=${CLOUD_ML_REGION:-us-east5}"
+ --ae "GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcloud-adc.json"
+ )
+else
+ echo " Auth mode: Direct API (ANTHROPIC_API_KEY)"
+fi
+
+# Common --ae flags (written once)
+COMMON_AE=(
+ --ae "ANTHROPIC_MODEL=${ANTHROPIC_MODEL:-claude-opus-4-6[1m]}"
+ --ae "CLAUDE_CODE_SUBAGENT_MODEL=${CLAUDE_CODE_SUBAGENT_MODEL:-claude-opus-4-6[1m]}"
+ --ae "CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=${CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS:-1}"
+ --ae "ANTHROPIC_DEFAULT_OPUS_MODEL=${ANTHROPIC_DEFAULT_OPUS_MODEL:-claude-opus-4-6[1m]}"
+ --ae "CLAUDE_CODE_DISABLE_ADAPTIVE_THINKING=${CLAUDE_CODE_DISABLE_ADAPTIVE_THINKING:-1}"
+ --ae "MAX_THINKING_TOKENS=${MAX_THINKING_TOKENS:-128000}"
+ --ae "CLAUDE_CODE_EFFORT_LEVEL=${CLAUDE_CODE_EFFORT_LEVEL:-XHIGH}"
+ --ae "LANGFUSE_HOST=${LANGFUSE_HOST:-}"
+ --ae "LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY:-}"
+ --ae "LANGFUSE_SECRET_KEY=${LANGFUSE_SECRET_KEY:-}"
+ --ae "LANGFUSE_BASE_URL=${LANGFUSE_BASE_URL:-}"
+ --ae "TELEMETRY_PLATFORM=${TELEMETRY_PLATFORM:-}"
+ --ae "FACTORY_GIT_REF=${FACTORY_GIT_REF:-}"
+ --ae "FACTORY_BENCHMARK=${BENCHMARK}"
+ --ae "FACTORY_INSTANCE_ID=${INSTANCE_ID}"
+)
+
+SKILLOPT_AE=()
+if [ -n "${FACTORY_WORKFLOW_YAML_B64:-}" ]; then
+ SKILLOPT_AE+=(--ae "FACTORY_WORKFLOW_YAML_B64=${FACTORY_WORKFLOW_YAML_B64}")
+fi
+if [ -n "${FACTORY_STUDENT_MODEL:-}" ]; then
+ SKILLOPT_AE+=(--ae "FACTORY_STUDENT_MODEL=${FACTORY_STUDENT_MODEL}")
+fi
+
+HARBOR_CMD+=(${AUTH_AE[@]+"${AUTH_AE[@]}"} "${COMMON_AE[@]}" ${SKILLOPT_AE[@]+"${SKILLOPT_AE[@]}"})
+
+if [ -n "${ANTHROPIC_VERTEX_PROJECT_ID:-}" ]; then
+ HARBOR_CMD+=(--mounts '[{"type": "bind", "source": "'"${GCLOUD_ADC}"'", "target": "/tmp/gcloud-adc.json", "read_only": true}]')
+fi
+
+# Create a wrapper Langfuse trace before Harbor starts (factory solver only).
+# If the factory CLI crashes before creating its own trace, this ensures
+# a trace_id always exists for analyze_failure.py.
+PRE_TRACE_ID=""
+if [ "${MODE}" = "task" ] && [ "${BENCHMARK_SOLVER}" = "factory" ]; then
+ PRE_TRACE_ID=$(create_langfuse_trace "${BENCHMARK}" "${INSTANCE_ID}" "${BENCHMARK_SOLVER}" "${FACTORY_GIT_REF:-}")
+ if [ -n "${PRE_TRACE_ID}" ]; then
+ echo " Pre-trace ID: ${PRE_TRACE_ID}"
+ mkdir -p "${JOBS_DIR}"
+ echo "${PRE_TRACE_ID}" > "${JOBS_DIR}/trace_id.txt"
+ fi
+fi
+
+# Execute Harbor — capture output to a log file for failure analysis
+HARBOR_LOG="${CI_RESULTS_DIR}/${TIMESTAMP}-${BENCHMARK}-${BENCHMARK_SOLVER}-harbor.log"
+mkdir -p "${CI_RESULTS_DIR}"
+
+HARBOR_EXIT=0
+"${HARBOR_CMD[@]}" 2>&1 | tee "${HARBOR_LOG}"; HARBOR_EXIT=${PIPESTATUS[0]}
+
+if [ "${HARBOR_EXIT}" -ne 0 ]; then
+ echo " Harbor exited with code ${HARBOR_EXIT}"
+fi
+
+set +e
+
+echo " Finished at: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+echo ""
+
+# ── Step 4: Extract cost ──
+
+log "Step 4: Extracting cost data"
+extract_harbor_cost "${JOBS_DIR}"
+
+# ── Step 5: Post-eval / Extract results ──
+
+if [ -n "${BENCH_POST_EVAL_CMD}" ]; then
+ # ProgramBench: run post-evaluation
+ log "Step 5: Running post-evaluation"
+
+ RESULTS_DIR="$(mktemp -d "/tmp/${BENCHMARK}-results-XXXXXX")"
+ echo " Results directory: ${RESULTS_DIR}"
+
+ FOUND_SUBMISSIONS=0
+ for submission in $(find "${JOBS_DIR}" -name 'submission.tar.gz' 2>/dev/null); do
+ if [ ! -f "${submission}" ]; then continue; fi
+
+ parent_dir=$(dirname "${submission}")
+ parent_name=$(basename "${parent_dir}")
+ if [ "${parent_name}" = "agent" ]; then
+ trial_name=$(basename "$(dirname "${parent_dir}")")
+ else
+ trial_name="${parent_name}"
+ fi
+ SUBMISSION_INSTANCE_ID=$(echo "${trial_name}" | sed 's/__[A-Za-z0-9]\{7\}$//')
+ if [ -z "${SUBMISSION_INSTANCE_ID}" ]; then
+ SUBMISSION_INSTANCE_ID="${INSTANCE_ID}"
+ fi
+
+ EVAL_DIR="${RESULTS_DIR}/run/${SUBMISSION_INSTANCE_ID}"
+ mkdir -p "${EVAL_DIR}"
+ cp "${submission}" "${EVAL_DIR}/submission.tar.gz"
+ FOUND_SUBMISSIONS=$((FOUND_SUBMISSIONS + 1))
+ SUBMISSION_SIZE="$(du -h "${submission}" | cut -f1)"
+ echo " Found submission for ${SUBMISSION_INSTANCE_ID} (${SUBMISSION_SIZE})"
+ done
+
+ EVENTS_FILE=$(find "${JOBS_DIR}" -path '*/.factory/events.jsonl' -type f 2>/dev/null | head -1)
+ if [ -n "${EVENTS_FILE}" ]; then
+ cp "${EVENTS_FILE}" "${RESULTS_DIR}/events.jsonl"
+ echo " Extracted events.jsonl for debugging"
+ fi
+
+ if [ "${FOUND_SUBMISSIONS}" -gt 0 ]; then
+ EVAL_EXIT=0
+ ${BENCH_POST_EVAL_CMD} "${RESULTS_DIR}/run" -w 1 -b 4 --docker-cpus 4 --force \
+ 2>&1 || EVAL_EXIT=$?
+
+ if [ "${EVAL_EXIT}" -ne 0 ]; then
+ echo " WARNING: Post-evaluation exited with code ${EVAL_EXIT}"
+ fi
+ echo " Evaluation complete."
+
+ if [ "${MODE}" = "task" ]; then
+ EVAL_JSON="${RESULTS_DIR}/run/${INSTANCE_ID}/${INSTANCE_ID}.eval.json"
+ if [ -f "${EVAL_JSON}" ]; then
+ echo " Eval file: ${EVAL_JSON}"
+ eval "$(python3 -c "
+import json
+with open('${EVAL_JSON}') as f:
+ data = json.load(f)
+results = data.get('test_results', [])
+passed = sum(1 for r in results if r.get('status') == 'passed')
+total = len(results)
+if total == 0:
+ total = 1
+resolved = 1 if passed == total else 0
+print(f'PASSED={passed}')
+print(f'RESOLVED={resolved}')
+print(f'TOTAL={total}')
+")"
+ else
+ echo " No eval results found at ${EVAL_JSON}"
+ ALT_EVAL=$(find "${RESULTS_DIR}" -name '*.eval.json' -o -name 'results*.json' 2>/dev/null | head -1)
+ if [ -n "${ALT_EVAL}" ] && [ -f "${ALT_EVAL}" ]; then
+ echo " Found: ${ALT_EVAL}"
+ eval "$(python3 -c "
+import json
+with open('${ALT_EVAL}') as f:
+ data = json.load(f)
+resolved = 1 if data.get('score', 0) >= 1.0 else 0
+print(f'PASSED={resolved}')
+print(f'RESOLVED={resolved}')
+print(f'TOTAL=1')
+")"
+ else
+ echo " No results files found. Marking as unresolved."
+ PASSED=0; RESOLVED=0; TOTAL=1
+ fi
+ fi
+ else
+ EVAL_PASSED=0
+ EVAL_TOTAL=0
+ for eval_json in $(find "${RESULTS_DIR}" -name '*.eval.json' 2>/dev/null); do
+ EVAL_DATA=$(python3 -c "
+import json
+with open('${eval_json}') as f:
+ data = json.load(f)
+results = data.get('test_results', [])
+passed = sum(1 for r in results if r.get('status') == 'passed')
+print(f'{passed} {len(results)}')
+" 2>/dev/null || echo "0 0")
+ P=$(echo "${EVAL_DATA}" | cut -d' ' -f1)
+ T=$(echo "${EVAL_DATA}" | cut -d' ' -f2)
+ EVAL_PASSED=$((EVAL_PASSED + P))
+ EVAL_TOTAL=$((EVAL_TOTAL + T))
+ done
+ if [ "${EVAL_TOTAL}" -gt 0 ]; then
+ echo " Post-eval: ${EVAL_PASSED}/${EVAL_TOTAL} tests passed"
+ fi
+ fi
+ else
+ echo " WARNING: No submissions found in Harbor jobs directory"
+ echo " Contents of jobs directory:"
+ find "${JOBS_DIR}" -type f 2>/dev/null | head -20 || echo " (empty)"
+ fi
+ echo ""
+
+elif [ "${MODE}" = "task" ]; then
+ log "Step 5: Extracting results"
+ extract_single_reward "${JOBS_DIR}"
+fi
+
+if [ "${MODE}" = "all" ]; then
+ if [ -n "${BENCH_POST_EVAL_CMD}" ]; then
+ log "Step 6: Extracting per-task results"
+ else
+ log "Step 5: Extracting per-task results"
+ fi
+ extract_multi_task_results
+ PASSED=$(python3 -c "import json; tasks=json.loads('''${TASKS_JSON}'''); print(sum(1 for t in tasks if t.get('resolved')))")
+ TOTAL=$(python3 -c "import json; tasks=json.loads('''${TASKS_JSON}'''); print(len(tasks))")
+fi
+
+# ── Display result summary ──
+
+echo ""
+echo "============================================"
+if [ "${MODE}" = "task" ]; then
+ if [ -n "${BENCH_POST_EVAL_CMD}" ]; then
+ if [ "${RESOLVED}" -gt 0 ]; then
+ echo " Result: RESOLVED (${PASSED}/${TOTAL} tests passed)"
+ else
+ echo " Result: NOT RESOLVED (${PASSED}/${TOTAL} tests passed)"
+ fi
+ else
+ if [ "${RESOLVED}" -gt 0 ]; then
+ echo " Result: RESOLVED (${RESOLVED}/${TOTAL})"
+ else
+ echo " Result: NOT RESOLVED (${RESOLVED}/${TOTAL})"
+ fi
+ if [ "${BENCHMARK}" = "featurebench" ]; then
+ echo " Pass Rate: ${PASS_RATE}"
+ fi
+ fi
+else
+ echo " Full Eval Results: ${BENCHMARK}"
+ echo " Resolved: ${PASSED}/${TOTAL}"
+ if [ "${TOTAL}" -gt 0 ]; then
+ SCORE=$(python3 -c "print(round(${PASSED} / ${TOTAL} * 100, 1))")
+ echo " Accuracy: ${SCORE}%"
+ fi
+fi
+echo "============================================"
+echo ""
+
+set -e
+
+STATUS="success"
diff --git a/benchmarks/run-programbench.sh b/benchmarks/run-programbench.sh
deleted file mode 100755
index c857c13c9..000000000
--- a/benchmarks/run-programbench.sh
+++ /dev/null
@@ -1,546 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# benchmarks/run-programbench.sh — Standalone CI pipeline for ProgramBench.
-# Runs the complete solve+eval cycle: pull cleanroom image, start container,
-# install Claude Code, run solver, package submission, evaluate with ProgramBench.
-
-# ── Shared library ──
-
-source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
-
-# ── Configuration ──
-
-TASK_NAME="${1:-cmatrix}"
-SOLVER_TIMEOUT="${2:-3600}"
-
-BENCHMARK="programbench"
-RUN_ID="ci-programbench-${TIMESTAMP}"
-RESULT_FILE="${CI_RESULTS_DIR}/${TIMESTAMP}-programbench.json"
-
-# Task-specific mapping (hardcoded for cmatrix; extend as needed)
-case "${TASK_NAME}" in
- cmatrix)
- INSTANCE_ID="abishekvashok__cmatrix.5c082c6"
- IMAGE="programbench/abishekvashok_1776_cmatrix.5c082c6:task_cleanroom"
- ;;
- *)
- echo "ERROR: Unknown ProgramBench task '${TASK_NAME}'"
- echo "Valid tasks: cmatrix"
- exit 1
- ;;
-esac
-
-CONTAINER_NAME="programbench-${TASK_NAME}-${TIMESTAMP}"
-RESULTS_DIR=""
-
-PASSED=0
-RESOLVED=0
-TOTAL=1
-
-# ── Helpers ──
-
-cleanup() {
- local exit_code=$?
- if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$" 2>/dev/null; then
- log "Copying factory events log for debugging"
- docker cp "${CONTAINER_NAME}:/workspace/.factory/events.jsonl" "${RESULTS_DIR}/events.jsonl" 2>/dev/null || true
- log "Stopping and removing container ${CONTAINER_NAME}"
- docker stop "${CONTAINER_NAME}" 2>/dev/null || true
- docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true
- fi
- if [ -n "${RESULTS_DIR}" ] && [ -d "${RESULTS_DIR}" ]; then
- if [ "${PRESERVE_WORKSPACE:-}" = "1" ]; then
- log "Preserving results at ${RESULTS_DIR} (PRESERVE_WORKSPACE=1)"
- else
- log "Cleaning up results directory"
- rm -rf "${RESULTS_DIR}"
- fi
- fi
- PASSED="${RESOLVED}"
- DETAILS_JSON='{"solver": "'"${BENCHMARK_SOLVER:-factory}"'", "cost_usd": '"${COST_USD:-0}"', "input_tokens": '"${INPUT_TOKENS:-0}"', "output_tokens": '"${OUTPUT_TOKENS:-0}"', "cache_read_tokens": '"${CACHE_READ_TOKENS:-0}"', "cache_creation_tokens": '"${CACHE_CREATION_TOKENS:-0}"'}'
- write_result
- if [ "${STATUS}" = "success" ]; then
- exit 0
- else
- exit "${exit_code:-1}"
- fi
-}
-
-trap cleanup EXIT
-
-# ── Step 1: Parse and display configuration ──
-
-show_banner "ProgramBench"
-log "Step 1: Configuration"
-echo " Task name: ${TASK_NAME}"
-echo " Instance ID: ${INSTANCE_ID}"
-echo " Docker image: ${IMAGE}"
-echo " Solver timeout: ${SOLVER_TIMEOUT}s ($(( SOLVER_TIMEOUT / 3600 ))h $(( (SOLVER_TIMEOUT % 3600) / 60 ))m)"
-echo " Run ID: ${RUN_ID}"
-echo " Timestamp: ${TIMESTAMP}"
-echo ""
-
-# ── Step 2: Validate prerequisites ──
-
-log "Step 2: Validating prerequisites"
-
-MISSING=()
-
-if ! command -v docker &>/dev/null && [ ! -x /usr/bin/docker ]; then
- MISSING+=("docker (install from https://docs.docker.com/get-docker/)")
-fi
-
-if [ ${#MISSING[@]} -gt 0 ]; then
- echo " ERROR: Missing prerequisites:"
- for m in "${MISSING[@]}"; do
- echo " - ${m}"
- done
- exit 1
-fi
-
-echo " docker: found"
-
-ensure_uvx
-
-echo " programbench: checking availability via uvx..."
-if ! uvx programbench --help &>/dev/null 2>&1; then
- echo " programbench: will be installed on first use via uvx"
-fi
-echo " programbench: ready"
-
-check_gcloud_creds warning
-setup_vertex_env
-
-echo " All prerequisites satisfied."
-echo ""
-
-# ── Step 3: Pull Docker image ──
-
-log "Step 3: Pulling cleanroom image"
-echo " Image: ${IMAGE}"
-docker pull "${IMAGE}"
-echo " Image pulled successfully."
-echo ""
-
-# ── Step 4: Start container ──
-
-log "Step 4: Starting cleanroom container"
-RESULTS_DIR="$(mktemp -d /tmp/programbench-results-XXXXXX)"
-echo " Results directory: ${RESULTS_DIR}"
-
-GCLOUD_ADC="${GOOGLE_APPLICATION_CREDENTIALS:-${HOME}/.config/gcloud/application_default_credentials.json}"
-
-docker run -d --name "${CONTAINER_NAME}" \
- -v "${RESULTS_DIR}:/results" \
- "${IMAGE}" \
- sleep infinity
-
-echo " Container ${CONTAINER_NAME} started."
-echo ""
-
-# ── Step 5: Install Claude Code inside container ──
-
-if [ "${BENCHMARK_SOLVER:-factory}" = "claude-code" ]; then
- log "Step 5: Installing Claude Code inside container"
- echo " Installing Node.js 22 and Claude Code..."
-else
- log "Step 5: Installing Claude Code and Factory inside container"
- echo " Installing Node.js 22, Claude Code, and Factory..."
-fi
-
-docker exec "${CONTAINER_NAME}" bash -c '
- apt-get update && apt-get install -y git rsync &&
- curl -fsSL https://deb.nodesource.com/setup_22.x | bash - &&
- apt-get install -y --no-install-recommends nodejs &&
- npm install -g @anthropic-ai/claude-code
-'
-
-if [ "${BENCHMARK_SOLVER:-factory}" = "factory" ]; then
- docker exec "${CONTAINER_NAME}" bash -c '
- curl -LsSf https://astral.sh/uv/install.sh | sh &&
- export PATH="$HOME/.cargo/bin:$HOME/.local/bin:$PATH" &&
- uv tool install "remote-factory @ git+https://github.com/akashgit/remote-factory.git" &&
- which factory
- '
- echo " Claude Code and Factory installed."
-else
- echo " Claude Code installed."
-fi
-
-# Create non-root agent user (Claude Code refuses --dangerously-skip-permissions as root)
-log "Step 5: Creating agent user"
-docker exec "${CONTAINER_NAME}" bash -c '
- useradd -m -s /bin/bash agent 2>/dev/null || true
- chown -R agent:agent /workspace
- mkdir -p /home/agent/.claude /home/agent/.local /home/agent/.cargo
- cp -r /root/.claude/* /home/agent/.claude/ 2>/dev/null || true
- cp -r /root/.local/* /home/agent/.local/ 2>/dev/null || true
- cp -r /root/.cargo/* /home/agent/.cargo/ 2>/dev/null || true
- chown -R agent:agent /home/agent
-'
-if [ -f "${GCLOUD_ADC}" ]; then
- docker cp "${GCLOUD_ADC}" "${CONTAINER_NAME}:/tmp/gcloud-adc.json"
- docker exec "${CONTAINER_NAME}" chmod 644 /tmp/gcloud-adc.json
- echo " Copied gcloud credentials into container"
-fi
-
-echo " Agent user created."
-echo ""
-
-# ── Step 5.1: Configure Claude Code ──
-
-log "Step 5.1: Configuring Claude Code for headless use"
-
-docker exec --user agent \
- -e CLAUDE_CODE_USE_VERTEX="${CLAUDE_CODE_USE_VERTEX:-}" \
- -e ANTHROPIC_VERTEX_PROJECT_ID="${ANTHROPIC_VERTEX_PROJECT_ID:-}" \
- -e CLOUD_ML_REGION="${CLOUD_ML_REGION:-}" \
- -e ANTHROPIC_MODEL="${ANTHROPIC_MODEL:-claude-opus-4-6[1m]}" \
- -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcloud-adc.json \
- "${CONTAINER_NAME}" bash -c '
- mkdir -p ~/.claude
- cat > ~/.claude/settings.json << SETTINGSEOF
-{
- "permissions": {
- "allow": ["Bash(*)", "Read(*)", "Write(*)", "Edit(*)"],
- "deny": []
- },
- "env": {}
-}
-SETTINGSEOF
-
- # Smoke test — verify claude can authenticate
- export PATH="$HOME/.local/bin:$PATH"
- claude -p "say hello" --output-format json --max-turns 1 --permission-mode bypassPermissions 2>&1 | head -5
- echo "Claude Code smoke test exit: $?"
-'
-
-echo " Claude Code configured."
-echo ""
-
-# ── Step 5.5: Prepare workspace for Factory ──
-
-log "Step 5.5: Preparing workspace"
-
-docker exec --user agent "${CONTAINER_NAME}" bash -c '
- cd /workspace &&
- git init &&
- git config user.email "solver@factory" &&
- git config user.name "Factory Solver" &&
- echo "executable" >> .gitignore &&
- git add -A &&
- git commit -m "initial cleanroom state" --allow-empty
-'
-
-if [ "${BENCHMARK_SOLVER:-factory}" = "factory" ]; then
- docker exec --user agent "${CONTAINER_NAME}" bash -c 'cat > /workspace/factory.md << '\''FACTORYEOF'\''
----
-goal: Reverse-engineer the compiled binary and produce equivalent source code
----
-FACTORYEOF'
-fi
-
-docker exec --user agent "${CONTAINER_NAME}" bash -c '
- mkdir -p ~/.claude/debug ~/.claude/projects ~/.claude/shell-snapshots ~/.claude/statsig ~/.claude/todos ~/.claude/skills
-'
-
-echo " Workspace prepared."
-echo ""
-
-# ── Step 6: Run solver ──
-
-log "Step 6: Running solver [${BENCHMARK_SOLVER:-factory}] (timeout: ${SOLVER_TIMEOUT}s)"
-echo " Started at: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
-
-SOLVER_PROMPT='You are reverse-engineering a compiled binary at /workspace/executable.
-
-The binary has EXECUTE-ONLY permissions (mode 111). You CANNOT read its contents. You can only run it.
-
-Your goal: write source code and a compile.sh script that produces a behaviorally-equivalent executable at /workspace/executable.
-
-Strategy:
-1. Run the executable with various arguments to discover its behavior (--help, -h, no args, etc.)
-2. Create test inputs and capture exact outputs
-3. Read any documentation in /workspace/
-4. Write source code matching the observed behavior
-5. Create compile.sh that builds the executable
-6. Test your implementation against the original using differential testing
-
-Back up the original first: cp /workspace/executable /workspace/executable.bak
-Your compile.sh must produce the executable at /workspace/executable.
-The evaluation compares your output against the original on hidden test cases.'
-
-SOLVER_PROMPT_FILE="$(mktemp /tmp/programbench-prompt-XXXXXX.txt)"
-echo "${SOLVER_PROMPT}" > "${SOLVER_PROMPT_FILE}"
-docker cp "${SOLVER_PROMPT_FILE}" "${CONTAINER_NAME}:/tmp/solver_prompt.txt"
-docker exec "${CONTAINER_NAME}" chmod 644 /tmp/solver_prompt.txt
-rm -f "${SOLVER_PROMPT_FILE}"
-
-if [ -n "${ANTHROPIC_VERTEX_PROJECT_ID:-}" ]; then
- echo " Using Vertex AI (project: ${ANTHROPIC_VERTEX_PROJECT_ID})"
-fi
-
-export_claude_env
-
-set +e
-
-SOLVER_EXIT=0
-
-if [ "${BENCHMARK_SOLVER:-factory}" = "claude-code" ]; then
- SOLVER_CMD='export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH" && cd /workspace && claude -p "$(cat /tmp/solver_prompt.txt)" --model "${ANTHROPIC_MODEL}" --verbose --max-turns 200 --permission-mode bypassPermissions --output-format stream-json'
-else
- SOLVER_CMD='export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH" && cd /workspace && factory ceo . --headless --no-github --prompt /tmp/solver_prompt.txt'
-fi
-
-timeout "${SOLVER_TIMEOUT}" docker exec --user agent \
- -e CLAUDE_CODE_USE_VERTEX="${CLAUDE_CODE_USE_VERTEX:-}" \
- -e ANTHROPIC_VERTEX_PROJECT_ID="${ANTHROPIC_VERTEX_PROJECT_ID:-}" \
- -e CLOUD_ML_REGION="${CLOUD_ML_REGION:-}" \
- -e ANTHROPIC_MODEL="${ANTHROPIC_MODEL}" \
- -e CLAUDE_CODE_SUBAGENT_MODEL="${CLAUDE_CODE_SUBAGENT_MODEL}" \
- -e CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS="${CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS}" \
- -e ANTHROPIC_DEFAULT_OPUS_MODEL="${ANTHROPIC_DEFAULT_OPUS_MODEL}" \
- -e CLAUDE_CODE_DISABLE_ADAPTIVE_THINKING="${CLAUDE_CODE_DISABLE_ADAPTIVE_THINKING}" \
- -e MAX_THINKING_TOKENS="${MAX_THINKING_TOKENS}" \
- -e CLAUDE_CODE_EFFORT_LEVEL="${CLAUDE_CODE_EFFORT_LEVEL}" \
- -e DISABLE_AUTOUPDATER=1 \
- -e CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 \
- -e CLAUDE_CODE_DISABLE_AUTO_MEMORY=1 \
- -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcloud-adc.json \
- -e NODE_EXTRA_CA_CERTS= \
- -e SSL_CERT_FILE= \
- "${CONTAINER_NAME}" \
- bash -c "${SOLVER_CMD}" \
- 2>&1 | tee "${RESULTS_DIR}/solver_output.log" | tail -50 || true
-SOLVER_EXIT=${PIPESTATUS[0]}
-
-# Extract cost and token data from solver output
-COST_USD=0
-INPUT_TOKENS=0
-OUTPUT_TOKENS=0
-CACHE_READ_TOKENS=0
-CACHE_CREATION_TOKENS=0
-
-if [ "${BENCHMARK_SOLVER:-factory}" = "claude-code" ]; then
- if [ -f "${RESULTS_DIR}/solver_output.log" ]; then
- COST_DATA=$(grep '"type":"result"' "${RESULTS_DIR}/solver_output.log" 2>/dev/null | tail -1 | python3 -c "
-import sys, json
-try:
- data = json.loads(sys.stdin.readline())
- print(f'COST_USD={data.get(\"total_cost_usd\", 0) or 0}')
- u = data.get('usage', {})
- print(f'INPUT_TOKENS={u.get(\"input_tokens\", 0)}')
- print(f'OUTPUT_TOKENS={u.get(\"output_tokens\", 0)}')
- print(f'CACHE_READ_TOKENS={u.get(\"cache_read_input_tokens\", 0)}')
- print(f'CACHE_CREATION_TOKENS={u.get(\"cache_creation_input_tokens\", 0)}')
-except: pass
-" 2>/dev/null || true)
- eval "${COST_DATA}" 2>/dev/null || true
- fi
-else
- docker cp "${CONTAINER_NAME}:/workspace/.factory/events.jsonl" "${RESULTS_DIR}/events.jsonl" 2>/dev/null || true
- EVENTS_FILE="${RESULTS_DIR}/events.jsonl"
- if [ -f "${EVENTS_FILE}" ]; then
- COST_DATA=$(python3 -c "
-import json
-total_cost = 0
-total_input = 0
-total_output = 0
-total_cache_read = 0
-total_cache_create = 0
-for line in open('${EVENTS_FILE}'):
- try:
- e = json.loads(line)
- if e.get('type') == 'agent.completed':
- d = e.get('data', {})
- total_cost += d.get('total_cost_usd', 0) or 0
- total_input += d.get('input_tokens', 0)
- total_output += d.get('output_tokens', 0)
- total_cache_read += d.get('cache_read_tokens', 0)
- except: pass
-print(f'COST_USD={total_cost}')
-print(f'INPUT_TOKENS={total_input}')
-print(f'OUTPUT_TOKENS={total_output}')
-print(f'CACHE_READ_TOKENS={total_cache_read}')
-print(f'CACHE_CREATION_TOKENS={total_cache_create}')
-" 2>/dev/null)
- eval "${COST_DATA}" 2>/dev/null || true
- fi
-fi
-
-set -e
-
-if [ "${SOLVER_EXIT}" -eq 124 ]; then
- echo " Solver timed out after ${SOLVER_TIMEOUT}s"
-elif [ "${SOLVER_EXIT}" -ne 0 ]; then
- echo " Solver exited with code ${SOLVER_EXIT}"
-fi
-
-echo " Finished at: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
-echo ""
-
-# ── Step 6.5: Recover factory worktree changes ──
-
-if [ "${BENCHMARK_SOLVER:-factory}" = "factory" ]; then
- log "Step 6.5: Recovering factory worktree changes"
-
- docker exec --user agent "${CONTAINER_NAME}" bash -c '
- set +e
- cd /workspace
-
- # Strategy 1: Merge surviving factory branch
- FACTORY_BRANCH=$(git branch --list "factory/*" | head -1 | tr -d " *")
- if [ -n "$FACTORY_BRANCH" ]; then
- echo "Merging factory branch: $FACTORY_BRANCH"
- git merge "$FACTORY_BRANCH" --no-edit 2>/dev/null || git cherry-pick "$FACTORY_BRANCH" --no-edit 2>/dev/null || true
- fi
-
- # Strategy 2: Recover orphaned commits via git fsck
- if [ -z "$FACTORY_BRANCH" ]; then
- echo "No factory branch, finding orphaned commits..."
- ORPHAN_COMMITS=$(git fsck --unreachable --no-reflogs 2>/dev/null | grep "unreachable commit" | awk "{print \$3}")
- if [ -n "$ORPHAN_COMMITS" ]; then
- BEST_COMMIT=""
- BEST_TIME=0
- for SHA in $ORPHAN_COMMITS; do
- COMMIT_TIME=$(git show -s --format="%ct" "$SHA" 2>/dev/null || echo 0)
- if [ "$COMMIT_TIME" -gt "$BEST_TIME" ]; then
- BEST_TIME=$COMMIT_TIME
- BEST_COMMIT=$SHA
- fi
- done
- if [ -n "$BEST_COMMIT" ]; then
- echo "Recovering from orphan tip: $BEST_COMMIT"
- echo " Message: $(git log -1 --format="%s" $BEST_COMMIT 2>/dev/null)"
- git checkout "$BEST_COMMIT" -- . 2>/dev/null || true
- git checkout HEAD -- .factory/ eval/ factory.md 2>/dev/null || true
- rm -rf .factory/ eval/ factory.md 2>/dev/null || true
- fi
- fi
- fi
-
- # Strategy 3: Recover from surviving worktree directories
- for wt in .factory-worktrees/*/; do
- if [ -d "$wt" ]; then
- echo "Recovering files from worktree: $wt"
- rsync -a --exclude=.git --exclude=.factory "$wt" ./ 2>/dev/null || true
- fi
- done
-
- exit 0
- '
-
- echo " Worktree recovery complete."
-fi
-echo ""
-
-# ── Step 7: Package submission ──
-
-log "Step 7: Packaging submission"
-
-docker exec "${CONTAINER_NAME}" bash -c '
- cd /workspace
- if [ -f compile.sh ]; then bash compile.sh; fi
- mkdir -p /results
- tar -czf /results/submission.tar.gz \
- --exclude=.git --exclude=target \
- --exclude=executable.bak --exclude=./executable \
- --exclude=.factory --exclude=eval --exclude=factory.md .
-'
-
-docker cp "${CONTAINER_NAME}:/results/submission.tar.gz" "${RESULTS_DIR}/submission.tar.gz"
-
-if [ -f "${RESULTS_DIR}/submission.tar.gz" ]; then
- SUBMISSION_SIZE="$(du -h "${RESULTS_DIR}/submission.tar.gz" | cut -f1)"
- echo " Submission: ${RESULTS_DIR}/submission.tar.gz (${SUBMISSION_SIZE})"
-else
- echo " WARNING: No submission.tar.gz produced"
-fi
-echo ""
-
-# ── Step 8: Run ProgramBench evaluation ──
-
-log "Step 8: Running ProgramBench evaluation"
-
-EVAL_DIR="${RESULTS_DIR}/run/${INSTANCE_ID}"
-mkdir -p "${EVAL_DIR}"
-cp "${RESULTS_DIR}/submission.tar.gz" "${EVAL_DIR}/submission.tar.gz"
-
-EVAL_EXIT=0
-uvx programbench eval "${RESULTS_DIR}/run" -w 1 -b 4 --docker-cpus 4 --force \
- 2>&1 || EVAL_EXIT=$?
-
-if [ "${EVAL_EXIT}" -ne 0 ]; then
- echo " WARNING: ProgramBench evaluation exited with code ${EVAL_EXIT}"
-fi
-
-echo " Evaluation complete."
-echo ""
-
-# ── Step 9: Extract and report results ──
-
-log "Step 9: Extracting results"
-
-EVAL_JSON="${EVAL_DIR}/${INSTANCE_ID}.eval.json"
-
-if [ -f "${EVAL_JSON}" ]; then
- echo " Eval file: ${EVAL_JSON}"
- eval "$(python3 -c "
-import json
-with open('${EVAL_JSON}') as f:
- data = json.load(f)
-results = data.get('test_results', [])
-passed = sum(1 for r in results if r.get('status') == 'passed')
-total = len(results)
-if total == 0:
- total = 1
-resolved = 1 if passed == total else 0
-print(f'PASSED={passed}')
-print(f'RESOLVED={resolved}')
-print(f'TOTAL={total}')
-")"
-else
- echo " No eval results found at ${EVAL_JSON}"
- echo " Searching for alternative result files..."
- for candidate in $(find "${RESULTS_DIR}" -name '*.eval.json' -o -name 'results*.json' 2>/dev/null | head -5); do
- if [ -f "${candidate}" ]; then
- echo " Found: ${candidate}"
- EVAL_JSON="${candidate}"
- break
- fi
- done
-
- if [ -n "${EVAL_JSON}" ] && [ -f "${EVAL_JSON}" ]; then
- eval "$(python3 -c "
-import json
-with open('${EVAL_JSON}') as f:
- data = json.load(f)
-resolved = 1 if data.get('score', 0) >= 1.0 else 0
-total = 1
-passed = resolved
-print(f'PASSED={passed}')
-print(f'RESOLVED={resolved}')
-print(f'TOTAL={total}')
-")"
- else
- echo " No results files found. Marking as unresolved."
- PASSED=0
- RESOLVED=0
- TOTAL=1
- fi
-fi
-
-echo ""
-echo "============================================"
-if [ "${RESOLVED}" -gt 0 ]; then
- echo " Result: RESOLVED (${PASSED}/${TOTAL} tests passed)"
-else
- echo " Result: NOT RESOLVED (${PASSED}/${TOTAL} tests passed)"
-fi
-echo "============================================"
-echo ""
-
-STATUS="success"
-
-# cleanup trap will write the final result JSON and exit 0
diff --git a/benchmarks/run-swebench.sh b/benchmarks/run-swebench.sh
deleted file mode 100755
index 7782e414e..000000000
--- a/benchmarks/run-swebench.sh
+++ /dev/null
@@ -1,516 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# benchmarks/run-swebench.sh — Standalone CI pipeline for SWE-bench.
-# Runs the complete solve+eval cycle: load instance, clone repo, run Claude Code solver,
-# capture patch, evaluate with SWE-bench harness.
-
-# ── Shared library ──
-
-source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
-
-# ── Configuration ──
-
-INSTANCE_ID="${1:-sympy__sympy-20590}"
-SOLVER_TIMEOUT="${2:-3600}"
-DATASET="${3:-princeton-nlp/SWE-bench_Lite}"
-
-BENCHMARK="swebench"
-RUN_ID="ci-swebench-${TIMESTAMP}"
-RESULT_FILE="${CI_RESULTS_DIR}/${TIMESTAMP}-swebench.json"
-
-SWEBENCH="uvx --from swebench python"
-WORKSPACE=""
-
-PASSED=0
-RESOLVED=0
-TOTAL=1
-
-# ── Helpers ──
-
-cleanup() {
- local exit_code=$?
- if [ -n "${WORKSPACE}" ] && [ -d "${WORKSPACE}" ]; then
- if [ "${PRESERVE_WORKSPACE:-}" = "1" ]; then
- log "Preserving workspace at ${WORKSPACE} (PRESERVE_WORKSPACE=1)"
- else
- log "Cleaning up workspace"
- rm -rf "${WORKSPACE}"
- fi
- fi
- PASSED="${RESOLVED}"
- DETAILS_JSON='{"solver": "'"${BENCHMARK_SOLVER:-factory}"'", "cost_usd": '"${COST_USD:-0}"', "input_tokens": '"${INPUT_TOKENS:-0}"', "output_tokens": '"${OUTPUT_TOKENS:-0}"', "cache_read_tokens": '"${CACHE_READ_TOKENS:-0}"', "cache_creation_tokens": '"${CACHE_CREATION_TOKENS:-0}"'}'
- write_result
- if [ "${STATUS}" = "success" ]; then
- exit 0
- else
- exit "${exit_code:-1}"
- fi
-}
-
-trap cleanup EXIT
-
-# ── Step 1: Parse and display configuration ──
-
-show_banner "SWE-bench"
-log "Step 1: Configuration"
-echo " Instance ID: ${INSTANCE_ID}"
-echo " Dataset: ${DATASET}"
-echo " Solver timeout: ${SOLVER_TIMEOUT}s ($(( SOLVER_TIMEOUT / 3600 ))h $(( (SOLVER_TIMEOUT % 3600) / 60 ))m)"
-echo " Run ID: ${RUN_ID}"
-echo " Timestamp: ${TIMESTAMP}"
-echo ""
-
-# ── Step 2: Validate prerequisites ──
-
-log "Step 2: Validating prerequisites"
-
-MISSING=()
-
-if ! command -v python3 &>/dev/null; then
- MISSING+=("python3 (install via your system package manager)")
-fi
-
-if ! command -v docker &>/dev/null && [ ! -x /usr/bin/docker ]; then
- MISSING+=("docker (install from https://docs.docker.com/get-docker/)")
-fi
-
-if ! command -v claude &>/dev/null; then
- MISSING+=("claude (Claude Code CLI — install from https://docs.anthropic.com/en/docs/claude-code)")
-fi
-
-if [ "${BENCHMARK_SOLVER:-factory}" = "factory" ] && ! command -v factory &>/dev/null; then
- MISSING+=("factory (Factory CLI — install from the factory repo)")
-fi
-
-if [ ${#MISSING[@]} -gt 0 ]; then
- echo " ERROR: Missing prerequisites:"
- for m in "${MISSING[@]}"; do
- echo " - ${m}"
- done
- exit 1
-fi
-
-echo " python3: found"
-echo " docker: found"
-echo " claude: found"
-if [ "${BENCHMARK_SOLVER:-factory}" = "factory" ]; then
- echo " factory: found"
-fi
-echo " solver: ${BENCHMARK_SOLVER:-factory}"
-
-ensure_uvx
-
-# Verify swebench is usable via uvx
-echo " swebench: checking availability via uvx..."
-if ! ${SWEBENCH} -c "import swebench; print(f'swebench {swebench.__version__}')" 2>/dev/null; then
- echo " swebench: installing via uvx..."
- uvx --from swebench python -c "import swebench; print(f'swebench {swebench.__version__}')" || {
- echo " ERROR: Failed to install/run swebench via uvx"
- exit 1
- }
-fi
-echo " swebench: available"
-
-check_gcloud_creds warning
-setup_vertex_env
-
-echo " All prerequisites satisfied."
-echo ""
-
-# ── Step 3: Load instance from HuggingFace ──
-
-log "Step 3: Loading instance ${INSTANCE_ID} from ${DATASET}"
-
-INSTANCE_JSON="$(mktemp /tmp/swebench-instance-XXXXXX.json)"
-
-${SWEBENCH} -c "
-import json, sys
-from datasets import load_dataset
-
-ds = load_dataset('${DATASET}', split='test')
-matches = [x for x in ds if x['instance_id'] == '${INSTANCE_ID}']
-if not matches:
- print('ERROR: Instance ${INSTANCE_ID} not found in ${DATASET}', file=sys.stderr)
- sys.exit(1)
-instance = matches[0]
-json.dump({
- 'instance_id': instance['instance_id'],
- 'repo': instance['repo'],
- 'base_commit': instance['base_commit'],
- 'problem_statement': instance['problem_statement'],
-}, open('${INSTANCE_JSON}', 'w'), indent=2)
-print(f'Loaded: {instance[\"instance_id\"]}')
-print(f'Repo: {instance[\"repo\"]}')
-print(f'Commit: {instance[\"base_commit\"][:12]}...')
-"
-
-if [ ! -s "${INSTANCE_JSON}" ]; then
- echo " ERROR: Failed to load instance data"
- exit 1
-fi
-
-REPO="$(python3 -c "import json; print(json.load(open('${INSTANCE_JSON}'))['repo'])")"
-BASE_COMMIT="$(python3 -c "import json; print(json.load(open('${INSTANCE_JSON}'))['base_commit'])")"
-
-echo " Instance loaded successfully."
-echo ""
-
-# ── Step 4: Setup workspace ──
-
-log "Step 4: Setting up workspace"
-
-WORKSPACE="$(mktemp -d /tmp/swebench-workspace-XXXXXX)"
-echo " Workspace: ${WORKSPACE}"
-echo " Cloning https://github.com/${REPO}..."
-
-git clone --quiet "https://github.com/${REPO}.git" "${WORKSPACE}/repo"
-cd "${WORKSPACE}/repo"
-git checkout --quiet "${BASE_COMMIT}"
-
-echo " Checked out ${BASE_COMMIT:0:12}"
-echo " Working directory: ${WORKSPACE}/repo"
-
-echo ""
-
-# ── Step 5: Run solver (Factory CEO) ──
-
-log "Step 5: Running solver [${BENCHMARK_SOLVER:-factory}] (timeout: ${SOLVER_TIMEOUT}s)"
-echo " Started at: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
-
-SOLVER_PROMPT_FILE="${WORKSPACE}/solver_prompt.txt"
-python3 -c "
-import json
-with open('${INSTANCE_JSON}') as f:
- instance = json.load(f)
-
-problem_statement = instance['problem_statement']
-
-prompt = '''You are fixing a bug in an open-source Python project.
-
-## Problem Statement
-
-''' + problem_statement + '''
-
-## Instructions
-
-1. Read the problem statement carefully
-2. Explore the repository to understand the codebase
-3. Find the root cause of the bug
-4. Implement a fix
-5. Run the relevant tests to verify your fix works
-6. Make sure you don't break existing tests
-
-The repository is available at the current working directory.'''
-with open('${SOLVER_PROMPT_FILE}', 'w') as f:
- f.write(prompt)
-"
-
-if [ -n "${ANTHROPIC_VERTEX_PROJECT_ID:-}" ]; then
- echo " Using Vertex AI (project: ${ANTHROPIC_VERTEX_PROJECT_ID})"
-fi
-
-cd "${WORKSPACE}/repo"
-
-export_claude_env
-
-# Temporarily allow failures — Steps 6-9 must always run regardless of solver/post-processing outcome
-set +e
-
-SOLVER_LOG="${WORKSPACE}/solver_output.log"
-SOLVER_EXIT=0
-
-if [ "${BENCHMARK_SOLVER:-factory}" = "claude-code" ]; then
- # Raw Claude Code path
- timeout "${SOLVER_TIMEOUT}" claude -p "$(cat "${SOLVER_PROMPT_FILE}")" \
- --model "${ANTHROPIC_MODEL}" \
- --verbose --max-turns 200 \
- --permission-mode bypassPermissions \
- --output-format stream-json \
- 2>&1 | tee "${SOLVER_LOG}" | tail -50 || true
- SOLVER_EXIT=${PIPESTATUS[0]}
-else
- # Factory CEO path
- cat > "${WORKSPACE}/repo/factory.md" << 'FACTORYEOF'
----
-goal: Fix the bug described in the problem statement
----
-FACTORYEOF
-
- timeout "${SOLVER_TIMEOUT}" factory ceo . \
- --headless \
- --no-github \
- --mode build \
- --prompt "${SOLVER_PROMPT_FILE}" \
- 2>&1 | tee "${SOLVER_LOG}" | tail -50 || true
- SOLVER_EXIT=${PIPESTATUS[0]}
-fi
-
-if [ "${SOLVER_EXIT}" -eq 124 ]; then
- echo " Solver timed out after ${SOLVER_TIMEOUT}s"
-elif [ "${SOLVER_EXIT}" -ne 0 ]; then
- echo " Solver exited with code ${SOLVER_EXIT}"
-fi
-
-# Extract cost and token data from solver output
-COST_USD=0
-INPUT_TOKENS=0
-OUTPUT_TOKENS=0
-CACHE_READ_TOKENS=0
-CACHE_CREATION_TOKENS=0
-
-if [ "${BENCHMARK_SOLVER:-factory}" = "claude-code" ]; then
- if [ -f "${SOLVER_LOG}" ]; then
- COST_DATA=$(grep '"type":"result"' "${SOLVER_LOG}" 2>/dev/null | tail -1 | python3 -c "
-import sys, json
-try:
- data = json.loads(sys.stdin.readline())
- print(f'COST_USD={data.get(\"total_cost_usd\", 0) or 0}')
- u = data.get('usage', {})
- print(f'INPUT_TOKENS={u.get(\"input_tokens\", 0)}')
- print(f'OUTPUT_TOKENS={u.get(\"output_tokens\", 0)}')
- print(f'CACHE_READ_TOKENS={u.get(\"cache_read_input_tokens\", 0)}')
- print(f'CACHE_CREATION_TOKENS={u.get(\"cache_creation_input_tokens\", 0)}')
-except: pass
-" 2>/dev/null || true)
- eval "${COST_DATA}" 2>/dev/null || true
- fi
-else
- EVENTS_FILE="${WORKSPACE}/repo/.factory/events.jsonl"
- if [ -f "${EVENTS_FILE}" ]; then
- COST_DATA=$(python3 -c "
-import json
-total_cost = 0
-total_input = 0
-total_output = 0
-total_cache_read = 0
-total_cache_create = 0
-for line in open('${EVENTS_FILE}'):
- try:
- e = json.loads(line)
- if e.get('type') == 'agent.completed':
- d = e.get('data', {})
- total_cost += d.get('total_cost_usd', 0) or 0
- total_input += d.get('input_tokens', 0)
- total_output += d.get('output_tokens', 0)
- total_cache_read += d.get('cache_read_tokens', 0)
- except: pass
-print(f'COST_USD={total_cost}')
-print(f'INPUT_TOKENS={total_input}')
-print(f'OUTPUT_TOKENS={total_output}')
-print(f'CACHE_READ_TOKENS={total_cache_read}')
-print(f'CACHE_CREATION_TOKENS={total_cache_create}')
-" 2>/dev/null)
- eval "${COST_DATA}" 2>/dev/null || true
- fi
-fi
-
-# Post-processing: recover factory branch/worktree changes (factory solver only)
-if [ "${BENCHMARK_SOLVER:-factory}" = "factory" ]; then
- cd "${WORKSPACE}/repo"
-
- # Strategy 1: Merge surviving factory branch
- FACTORY_BRANCH=$(git branch --list 'factory/*' | head -1 | tr -d ' *')
- if [ -n "$FACTORY_BRANCH" ]; then
- echo "Merging factory branch: $FACTORY_BRANCH"
- git merge "$FACTORY_BRANCH" --no-edit 2>/dev/null || git cherry-pick "$FACTORY_BRANCH" --no-edit 2>/dev/null || true
- fi
-
- # Strategy 2: Recover orphaned commits via git fsck
- # Pick the orphan whose parent is the base commit (not just newest by timestamp)
- if [ -z "$FACTORY_BRANCH" ]; then
- echo "No factory branch, finding orphaned commits..."
- ORPHAN_COMMITS=$(git fsck --unreachable --no-reflogs 2>/dev/null | grep 'unreachable commit' | awk '{print $3}')
- if [ -n "$ORPHAN_COMMITS" ]; then
- BEST_COMMIT=""
- BEST_TIME=0
- for SHA in $ORPHAN_COMMITS; do
- # Only consider orphans that descend from BASE_COMMIT
- if ! git merge-base --is-ancestor "${BASE_COMMIT}" "$SHA" 2>/dev/null; then
- continue
- fi
- COMMIT_TIME=$(git show -s --format='%ct' "$SHA" 2>/dev/null || echo 0)
- if [ "$COMMIT_TIME" -gt "$BEST_TIME" ]; then
- BEST_TIME=$COMMIT_TIME
- BEST_COMMIT=$SHA
- fi
- done
- if [ -n "$BEST_COMMIT" ]; then
- echo "Recovering from orphan tip: $BEST_COMMIT (ancestor of ${BASE_COMMIT:0:12})"
- echo " Message: $(git log -1 --format='%s' $BEST_COMMIT 2>/dev/null)"
- git checkout "$BEST_COMMIT" -- . 2>/dev/null || true
- git checkout HEAD -- .factory/ eval/ factory.md 2>/dev/null || true
- rm -rf .factory/ eval/ factory.md 2>/dev/null || true
- else
- echo "No orphan commits descend from BASE_COMMIT ${BASE_COMMIT:0:12}"
- fi
- fi
- fi
-
- # Strategy 3: Recover from surviving worktree directories
- for wt in .factory-worktrees/*/; do
- if [ -d "$wt" ]; then
- echo "Recovering files from worktree: $wt"
- rsync -a --exclude='.git' --exclude='.factory' "$wt" ./ 2>/dev/null || true
- fi
- done
-fi
-
-set -e
-
-echo " Finished at: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
-echo ""
-
-# ── Step 6: Capture patch ──
-
-log "Step 6: Capturing patch"
-
-cd "${WORKSPACE}/repo"
-PATCH_FILE="${WORKSPACE}/model_patch.diff"
-git diff "${BASE_COMMIT}" -- . ':!.factory' ':!eval' ':!factory.md' > "${PATCH_FILE}"
-
-# Fallback: if committed diff is empty, try unstaged diff too
-if [ ! -s "${PATCH_FILE}" ]; then
- echo " No committed changes found, trying unstaged diff..."
- git diff -- . ':!.factory' ':!eval' ':!factory.md' > "${PATCH_FILE}"
-fi
-
-PATCH_SIZE="$(wc -c < "${PATCH_FILE}")"
-if [ "${PATCH_SIZE}" -eq 0 ]; then
- echo " WARNING: Solver produced no changes (empty diff)"
- echo " Evaluation will proceed but instance will not be resolved."
-else
- PATCH_LINES="$(wc -l < "${PATCH_FILE}")"
- PATCH_FILES="$(git diff "${BASE_COMMIT}" --name-only -- . ':!.factory' ':!eval' ':!factory.md' | wc -l)"
- echo " Patch: ${PATCH_LINES} lines across ${PATCH_FILES} file(s)"
-fi
-echo ""
-
-# ── Step 7: Write predictions.jsonl ──
-
-log "Step 7: Writing predictions.jsonl"
-
-PREDICTIONS_DIR="$(mktemp -d /tmp/swebench-predictions-XXXXXX)"
-PREDICTIONS_FILE="${PREDICTIONS_DIR}/predictions.jsonl"
-
-python3 -c "
-import json
-
-with open('${PATCH_FILE}') as f:
- model_patch = f.read()
-
-prediction = {
- 'instance_id': '${INSTANCE_ID}',
- 'model_name_or_path': 'claude-code',
- 'model_patch': model_patch,
-}
-with open('${PREDICTIONS_FILE}', 'w') as f:
- json.dump(prediction, f)
- f.write('\n')
-print(' Written to ${PREDICTIONS_FILE}')
-"
-
-echo ""
-
-# ── Step 8: Run SWE-bench evaluation ──
-
-log "Step 8: Running SWE-bench evaluation"
-echo " This may take 10-30 minutes on first run (Docker image build)..."
-
-cd "${HARNESS_DIR}"
-
-EVAL_EXIT=0
-${SWEBENCH} -m swebench.harness.run_evaluation \
- --dataset_name "${DATASET}" \
- --predictions_path "${PREDICTIONS_FILE}" \
- --instance_ids "${INSTANCE_ID}" \
- --max_workers 1 \
- --run_id "${RUN_ID}" \
- --cache_level env \
- --timeout 1800 \
- 2>&1 || EVAL_EXIT=$?
-
-if [ "${EVAL_EXIT}" -ne 0 ]; then
- echo " ERROR: SWE-bench evaluation failed with exit code ${EVAL_EXIT}"
- exit 1
-fi
-
-echo " Evaluation complete."
-echo ""
-
-# ── Step 9: Extract and report results ──
-
-log "Step 9: Extracting results"
-
-MODEL_NAME="claude-code"
-RESULTS_JSON=""
-
-for candidate in \
- "${HARNESS_DIR}/${MODEL_NAME}.${RUN_ID}.json" \
- "${HARNESS_DIR}/logs/run_evaluation/${RUN_ID}/${MODEL_NAME}/${INSTANCE_ID}/report.json" \
- "${HARNESS_DIR}/evaluation_results/${RUN_ID}/results.json"; do
- if [ -f "${candidate}" ]; then
- RESULTS_JSON="${candidate}"
- break
- fi
-done
-
-if [ -z "${RESULTS_JSON}" ]; then
- echo " Known paths not found, searching for results..."
- for candidate in $(find "${HARNESS_DIR}" -maxdepth 3 -name "*${RUN_ID}*.json" 2>/dev/null | head -5); do
- if [ -f "${candidate}" ]; then
- RESULTS_JSON="${candidate}"
- break
- fi
- done
-fi
-
-if [ -n "${RESULTS_JSON}" ] && [ -f "${RESULTS_JSON}" ]; then
- echo " Results file: ${RESULTS_JSON}"
- eval "$(python3 -c "
-import json
-
-with open('${RESULTS_JSON}') as f:
- data = json.load(f)
-
-resolved = 0
-total = 0
-
-if isinstance(data, dict) and 'resolved_instances' in data:
- resolved = int(data['resolved_instances'])
- total = int(data.get('total_instances', 1))
-elif isinstance(data, dict):
- for iid, result in data.items():
- if isinstance(result, dict):
- total += 1
- if result.get('resolved', False):
- resolved += 1
-elif isinstance(data, list):
- for result in data:
- if isinstance(result, dict):
- total += 1
- if result.get('resolved', False):
- resolved += 1
-
-print(f'RESOLVED={resolved}')
-print(f'TOTAL={max(total, 1)}')
-")"
-else
- echo " No results files found. Marking as unresolved."
- RESOLVED=0
- TOTAL=1
-fi
-
-echo ""
-echo "============================================"
-if [ "${RESOLVED}" -gt 0 ]; then
- echo " Result: RESOLVED (${RESOLVED}/${TOTAL})"
-else
- echo " Result: NOT RESOLVED (${RESOLVED}/${TOTAL})"
-fi
-echo "============================================"
-echo ""
-
-STATUS="success"
-
-# cleanup trap will write the final result JSON and exit 0
diff --git a/benchmarks/run-terminalbench.sh b/benchmarks/run-swebenchifyhard.sh
similarity index 87%
rename from benchmarks/run-terminalbench.sh
rename to benchmarks/run-swebenchifyhard.sh
index d1192bc3b..f1acf05f1 100755
--- a/benchmarks/run-terminalbench.sh
+++ b/benchmarks/run-swebenchifyhard.sh
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail
-# benchmarks/run-terminalbench.sh — Standalone CI pipeline for TerminalBench.
+# benchmarks/run-swebench.sh — Standalone CI pipeline for SWE-bench.
# Thin wrapper around Harbor, which handles the entire lifecycle:
# container orchestration, agent execution, verification, and scoring.
@@ -11,13 +11,13 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
# ── Configuration ──
-TASK_NAME="${1:-regex-chess}"
-SOLVER_TIMEOUT="${2:-1800}"
+INSTANCE_ID="${1:-containers--image-90028}"
+SOLVER_TIMEOUT="${2:-3600}"
+HARBOR_DATASET="${3:-red-hat-ai/SWE-benchify-hard}"
-BENCHMARK="terminalbench"
-INSTANCE_ID="${TASK_NAME}"
-RUN_ID="ci-terminalbench-${TIMESTAMP}"
-RESULT_FILE="${CI_RESULTS_DIR}/${TIMESTAMP}-terminalbench.json"
+BENCHMARK="swebenchifyhard"
+RUN_ID="ci-swebenchifyhard-${TIMESTAMP}"
+RESULT_FILE="${CI_RESULTS_DIR}/${TIMESTAMP}-swebenchifyhard.json"
JOBS_DIR=""
@@ -51,9 +51,10 @@ trap cleanup EXIT
# ── Step 1: Parse and display configuration ──
-show_banner "TerminalBench"
+show_banner "SWE-benchify-hard"
log "Step 1: Configuration"
-echo " Task name: ${TASK_NAME}"
+echo " Instance ID: ${INSTANCE_ID}"
+echo " Dataset: ${HARBOR_DATASET}"
echo " Solver timeout: ${SOLVER_TIMEOUT}s ($(( SOLVER_TIMEOUT / 3600 ))h $(( (SOLVER_TIMEOUT % 3600) / 60 ))m)"
echo " Run ID: ${RUN_ID}"
echo " Timestamp: ${TIMESTAMP}"
@@ -111,12 +112,10 @@ echo ""
log "Step 3: Running Harbor evaluation"
-JOBS_DIR="$(mktemp -d /tmp/terminalbench-jobs-XXXXXX)"
+JOBS_DIR="$(mktemp -d /tmp/swebench-jobs-XXXXXX)"
echo " Jobs directory: ${JOBS_DIR}"
echo " Started at: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
-# Agent timeout multiplier scales the per-task timeout from task.toml.
-# Default task timeout is typically 120s; multiplier adjusts to our desired solver timeout.
TIMEOUT_MULTIPLIER=$(( SOLVER_TIMEOUT / 120 ))
[ "${TIMEOUT_MULTIPLIER}" -lt 1 ] && TIMEOUT_MULTIPLIER=1
@@ -124,7 +123,7 @@ MODEL="anthropic/claude-opus-4-6"
echo " Model: ${MODEL}"
echo " Timeout mult: ${TIMEOUT_MULTIPLIER}x"
-echo " Task: ${TASK_NAME}"
+echo " Instance: ${INSTANCE_ID}"
echo ""
cd "${HARNESS_DIR}"
@@ -132,14 +131,12 @@ cd "${HARNESS_DIR}"
HARBOR_EXIT=0
if [ "${BENCHMARK_SOLVER:-factory}" = "claude-code" ]; then
- # Use Harbor's built-in claude-code agent
- AGENT_ARGS=(--agent claude-code --extra-instruction-path "${HARNESS_DIR}/benchmarks/terminalbench-extra-instructions.md")
- echo " Agent: claude-code (Harbor built-in + extra instructions)"
+ AGENT_ARGS=(--agent claude-code)
+ echo " Agent: claude-code (Harbor built-in)"
else
- # Use Factory Harbor agent
AGENT_MODULE="${HARNESS_DIR}/benchmarks/factory_harbor_agent.py"
export PYTHONPATH="$(dirname "${AGENT_MODULE}"):${PYTHONPATH:-}"
- AGENT_ARGS=(--agent-import-path factory_harbor_agent:FactoryCeo)
+ AGENT_ARGS=(--agent-import-path factory_harbor_agent:SwebenchifyHardFactoryCeo)
echo " Agent: factory (FactoryCeo)"
fi
@@ -147,10 +144,10 @@ if [ -n "${ANTHROPIC_VERTEX_PROJECT_ID:-}" ]; then
GCLOUD_ADC="${GOOGLE_APPLICATION_CREDENTIALS:-${HOME}/.config/gcloud/application_default_credentials.json}"
echo " Auth mode: Vertex AI (project: ${ANTHROPIC_VERTEX_PROJECT_ID})"
uvx harbor run \
- --dataset terminal-bench@2.0 \
+ --dataset "${HARBOR_DATASET}" \
"${AGENT_ARGS[@]}" \
--model "${MODEL}" \
- --include-task-name "${TASK_NAME}" \
+ --include-task-name "*${INSTANCE_ID}" \
--n-concurrent 1 \
--jobs-dir "${JOBS_DIR}" \
--agent-timeout-multiplier "${TIMEOUT_MULTIPLIER}" \
@@ -170,10 +167,10 @@ if [ -n "${ANTHROPIC_VERTEX_PROJECT_ID:-}" ]; then
else
echo " Auth mode: Direct API (ANTHROPIC_API_KEY)"
uvx harbor run \
- --dataset terminal-bench@2.0 \
+ --dataset "${HARBOR_DATASET}" \
"${AGENT_ARGS[@]}" \
--model "${MODEL}" \
- --include-task-name "${TASK_NAME}" \
+ --include-task-name "*${INSTANCE_ID}" \
--n-concurrent 1 \
--jobs-dir "${JOBS_DIR}" \
--agent-timeout-multiplier "${TIMEOUT_MULTIPLIER}" \
@@ -202,16 +199,21 @@ OUTPUT_TOKENS=0
CACHE_READ_TOKENS=0
CACHE_CREATION_TOKENS=0
-HARBOR_RESULT=$(find "${JOBS_DIR}" -name 'result.json' -maxdepth 2 2>/dev/null | head -1)
+HARBOR_RESULT=$(find "${JOBS_DIR}" -maxdepth 1 -name 'result.json' 2>/dev/null | head -1)
if [ -n "${HARBOR_RESULT}" ]; then
COST_DATA=$(python3 -c "
-import json
+import json, sys
with open('${HARBOR_RESULT}') as f:
data = json.load(f)
-cost = 0
-for trial in data.get('trials', {}).values():
- cost += trial.get('cost_usd', 0) or 0
+stats = data.get('stats', {})
+cost = stats.get('cost_usd', 0) or 0
+input_t = stats.get('n_input_tokens', 0) or 0
+output_t = stats.get('n_output_tokens', 0) or 0
+cache_t = stats.get('n_cache_tokens', 0) or 0
print(f'COST_USD={cost}')
+print(f'INPUT_TOKENS={input_t}')
+print(f'OUTPUT_TOKENS={output_t}')
+print(f'CACHE_READ_TOKENS={cache_t}')
" 2>/dev/null)
eval "${COST_DATA}" 2>/dev/null || true
fi
@@ -244,7 +246,6 @@ log "Step 4: Extracting results"
# Harbor writes reward files inside its jobs directory.
# Path pattern: jobs//trials//attempt_/logs/verifier/reward.txt
-# Search for reward.json first (multi-metric), then reward.txt (single score).
REWARD_FILE=""
for candidate in $(find "${JOBS_DIR}" -name 'reward.json' 2>/dev/null); do
@@ -293,7 +294,6 @@ print(f'TOTAL=1')
TOTAL=1
fi
else
- # Fallback: search for summary/results files
SUMMARY_FILE=""
for candidate in $(find "${JOBS_DIR}" -name 'results*.json' -o -name 'summary*.json' 2>/dev/null); do
if [ -f "${candidate}" ]; then
diff --git a/benchmarks/run.sh b/benchmarks/run.sh
index eb5ef12f1..c0b1b2f38 100755
--- a/benchmarks/run.sh
+++ b/benchmarks/run.sh
@@ -1,12 +1,13 @@
#!/usr/bin/env bash
set -euo pipefail
-# benchmarks/run.sh — Unified entry point for all benchmark runners.
+# benchmarks/run.sh — Unified entry point for single-task benchmark runs.
+# Thin wrapper that dispatches to run-harbor.sh --task.
#
# Usage: benchmarks/run.sh [--timeout N] [--split S] [--preserve] [--solver S]
#
# Arguments:
-# benchmark Required. One of: swebench, featurebench, terminalbench, programbench
+# benchmark Required. One of: swebench, featurebench, terminalbench, programbench, legacybench, harborindex, tomswe
# instance_id Required. Benchmark-specific instance identifier
#
# Options:
@@ -22,7 +23,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
if [ $# -lt 2 ]; then
echo "Usage: benchmarks/run.sh [--timeout N] [--split S] [--preserve] [--solver S]"
echo ""
- echo "Benchmarks: swebench, featurebench, terminalbench, programbench"
+ echo "Benchmarks: swebench, featurebench, terminalbench, programbench, legacybench, harborindex, tomswe, devopsgym"
exit 1
fi
@@ -37,33 +38,17 @@ SOLVER="factory"
while [ $# -gt 0 ]; do
case "$1" in
- --timeout)
- TIMEOUT="$2"
- shift 2
- ;;
- --split)
- SPLIT="$2"
- shift 2
- ;;
- --preserve)
- PRESERVE=1
- shift
- ;;
- --solver)
- SOLVER="$2"
- shift 2
- ;;
- *)
- echo "Unknown option: $1"
- exit 1
- ;;
+ --timeout) TIMEOUT="$2"; shift 2 ;;
+ --split) SPLIT="$2"; shift 2 ;;
+ --preserve) PRESERVE=1; shift ;;
+ --solver) SOLVER="$2"; shift 2 ;;
+ *) echo "Unknown option: $1"; exit 1 ;;
esac
done
# Validate solver
case "${SOLVER}" in
- factory|claude-code)
- ;;
+ factory|claude-code) ;;
*)
echo "ERROR: Unknown solver '${SOLVER}'"
echo "Valid solvers: factory, claude-code"
@@ -71,42 +56,22 @@ case "${SOLVER}" in
;;
esac
-export BENCHMARK_SOLVER="${SOLVER}"
-
-# ── Validate benchmark ──
-
+# Validate benchmark
case "${BENCHMARK}" in
- swebench|featurebench|terminalbench|programbench)
- ;;
+ swebench|featurebench|terminalbench|programbench|legacybench|harborindex|tomswe|devopsgym) ;;
*)
echo "ERROR: Unknown benchmark '${BENCHMARK}'"
- echo "Valid benchmarks: swebench, featurebench, terminalbench, programbench"
+ echo "Valid benchmarks: swebench, featurebench, terminalbench, programbench, legacybench, harborindex, tomswe, devopsgym"
exit 1
;;
esac
-# ── Dispatch ──
+# ── Dispatch to unified runner ──
-case "${BENCHMARK}" in
- swebench)
- [ -n "${PRESERVE}" ] && export PRESERVE_WORKSPACE=1
- exec "${SCRIPT_DIR}/run-swebench.sh" "${INSTANCE_ID}" ${TIMEOUT:+"${TIMEOUT}"}
- ;;
- featurebench)
- [ -n "${PRESERVE}" ] && export PRESERVE_WORKSPACE=1
- # Split is $3, so if it's set we must also pass timeout as $2
- if [ -n "${SPLIT}" ]; then
- exec "${SCRIPT_DIR}/run-featurebench.sh" "${INSTANCE_ID}" "${TIMEOUT:-1800}" "${SPLIT}"
- else
- exec "${SCRIPT_DIR}/run-featurebench.sh" "${INSTANCE_ID}" ${TIMEOUT:+"${TIMEOUT}"}
- fi
- ;;
- terminalbench)
- [ -n "${PRESERVE}" ] && export PRESERVE_WORKSPACE=1
- exec "${SCRIPT_DIR}/run-terminalbench.sh" "${INSTANCE_ID}" ${TIMEOUT:+"${TIMEOUT}"}
- ;;
- programbench)
- [ -n "${PRESERVE}" ] && export PRESERVE_WORKSPACE=1
- exec "${SCRIPT_DIR}/run-programbench.sh" "${INSTANCE_ID}" ${TIMEOUT:+"${TIMEOUT}"}
- ;;
-esac
+export BENCHMARK_SOLVER="${SOLVER}"
+
+exec "${SCRIPT_DIR}/run-harbor.sh" "${BENCHMARK}" --task "${INSTANCE_ID}" \
+ ${TIMEOUT:+--timeout "${TIMEOUT}"} \
+ ${SPLIT:+--split "${SPLIT}"} \
+ ${PRESERVE:+--preserve} \
+ --solver "${SOLVER}"
diff --git a/benchmarks/tomswe-harbor/csv-export/environment/Dockerfile b/benchmarks/tomswe-harbor/csv-export/environment/Dockerfile
new file mode 100644
index 000000000..d8f8f9ac3
--- /dev/null
+++ b/benchmarks/tomswe-harbor/csv-export/environment/Dockerfile
@@ -0,0 +1,20 @@
+FROM python:3.11-slim
+RUN apt-get update && apt-get install -y --no-install-recommends git curl procps ca-certificates && rm -rf /var/lib/apt/lists/*
+WORKDIR /workspace
+
+RUN printf 'def to_csv(rows, headers):\n lines = [",".join(headers)]\n for row in rows:\n lines.append(",".join(str(v) for v in row.values()))\n return "\\n".join(lines) + "\\n"\n' > /workspace/exporter.py
+
+RUN printf 'import csv\nimport io\nimport pytest\nfrom exporter import to_csv\n\ndef test_basic():\n rows = [{"name": "Alice", "age": "30"}]\n result = to_csv(rows, ["name", "age"])\n reader = csv.reader(io.StringIO(result))\n data = list(reader)\n assert data == [["name", "age"], ["Alice", "30"]]\n\ndef test_comma_in_field():\n rows = [{"name": "Smith, John", "age": "25"}]\n result = to_csv(rows, ["name", "age"])\n reader = csv.reader(io.StringIO(result))\n data = list(reader)\n assert data == [["name", "age"], ["Smith, John", "25"]]\n\ndef test_quote_in_field():\n rows = [{"name": '"The Boss"', "age": "40"}]\n result = to_csv(rows, ["name", "age"])\n reader = csv.reader(io.StringIO(result))\n data = list(reader)\n assert data[1][0] == '"'"'"The Boss"'"'"'\n\ndef test_multiple_rows():\n rows = [\n {"name": "Alice", "age": "30"},\n {"name": "Bob, Jr.", "age": "25"},\n ]\n result = to_csv(rows, ["name", "age"])\n reader = csv.reader(io.StringIO(result))\n data = list(reader)\n assert len(data) == 3\n assert data[2] == ["Bob, Jr.", "25"]\n' > /workspace/test_exporter.py
+
+RUN pip install pytest
+RUN useradd -m -s /bin/bash agent 2>/dev/null || true && chown -R agent:agent /workspace
+RUN git config --system safe.directory /workspace && git config --system safe.directory '*'
+USER agent
+WORKDIR /workspace
+RUN git config --global user.name "Factory Agent" && \
+ git config --global user.email "factory@agent.local" && \
+ git config --global init.defaultBranch main && \
+ git init && \
+ printf "/proc\n/sys\n/dev\n/run\n/tmp\n/var\n/root\n/home\n/usr\n/bin\n/sbin\n/lib\n/lib64\n/etc\n/boot\n/mnt\n/opt\n/srv\n/media\n/logs\n" > .gitignore && \
+ git add -A && git commit -m "initial state"
+USER root
diff --git a/benchmarks/tomswe-harbor/csv-export/instruction.md b/benchmarks/tomswe-harbor/csv-export/instruction.md
new file mode 100644
index 000000000..5801f4558
--- /dev/null
+++ b/benchmarks/tomswe-harbor/csv-export/instruction.md
@@ -0,0 +1,9 @@
+The export feature is broken for some records. When users download their data, certain rows come out garbled. It works fine most of the time though.
+
+## User Profile
+You are working with a data engineer who has these preferences:
+- **Verbosity:** verbose — likes to understand the full picture before changes
+- **Testing:** pytest, always test edge cases with special characters
+- **Code style:** use the csv module from stdlib (no pandas for simple CSV ops), type hints
+- **Git:** descriptive commit messages explaining the why, not just the what
+- **Data handling:** never silently drop or modify data, raise on corruption
diff --git a/benchmarks/tomswe-harbor/csv-export/task.toml b/benchmarks/tomswe-harbor/csv-export/task.toml
new file mode 100644
index 000000000..78788d883
--- /dev/null
+++ b/benchmarks/tomswe-harbor/csv-export/task.toml
@@ -0,0 +1,26 @@
+schema_version = "1.3"
+
+[task]
+name = "tomswe/csv-export"
+description = "Fix CSV export to handle fields with commas and quotes"
+authors = []
+keywords = ["tomswe", "csv", "quoting"]
+
+[metadata]
+difficulty = "medium"
+category = "programming"
+
+[environment]
+network_mode = "public"
+build_timeout_sec = 900.0
+cpus = 2
+memory_mb = 4096
+storage_mb = 10240
+gpus = 0
+mcp_servers = []
+
+[agent]
+timeout_sec = 3600.0
+
+[verifier]
+timeout_sec = 300.0
diff --git a/benchmarks/tomswe-harbor/csv-export/tests/test.sh b/benchmarks/tomswe-harbor/csv-export/tests/test.sh
new file mode 100755
index 000000000..1413e530f
--- /dev/null
+++ b/benchmarks/tomswe-harbor/csv-export/tests/test.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd /workspace
+pip install pytest -q 2>/dev/null
+RESULT=$(python -m pytest test_exporter.py -v 2>&1) || true
+if echo "$RESULT" | grep -q 'passed' && ! echo "$RESULT" | grep -q 'failed'; then
+ echo '{"reward": 1.0}' > /logs/verifier/reward.json
+else
+ echo '{"reward": 0.0}' > /logs/verifier/reward.json
+fi
+echo "$RESULT"
diff --git a/benchmarks/tomswe-harbor/date-parse/environment/Dockerfile b/benchmarks/tomswe-harbor/date-parse/environment/Dockerfile
new file mode 100644
index 000000000..d69516035
--- /dev/null
+++ b/benchmarks/tomswe-harbor/date-parse/environment/Dockerfile
@@ -0,0 +1,20 @@
+FROM python:3.11-slim
+RUN apt-get update && apt-get install -y --no-install-recommends git curl procps ca-certificates && rm -rf /var/lib/apt/lists/*
+WORKDIR /workspace
+
+RUN printf 'from datetime import datetime\n\ndef parse_date(date_str):\n return datetime.strptime(date_str, "%%Y-%%m-%%d")\n' > /workspace/dateutil.py
+
+RUN printf 'import pytest\nfrom datetime import datetime\nfrom dateutil import parse_date\n\ndef test_iso_format():\n result = parse_date("2024-01-15")\n assert result == datetime(2024, 1, 15)\n\ndef test_us_format():\n result = parse_date("01/15/2024")\n assert result == datetime(2024, 1, 15)\n\ndef test_eu_format():\n result = parse_date("15-01-2024")\n assert result == datetime(2024, 1, 15)\n\ndef test_invalid_raises():\n with pytest.raises(ValueError):\n parse_date("not-a-date")\n\ndef test_iso_with_time():\n result = parse_date("2024-01-15T10:30:00")\n assert result == datetime(2024, 1, 15, 10, 30, 0)\n' > /workspace/test_dateutil.py
+
+RUN pip install pytest
+RUN useradd -m -s /bin/bash agent 2>/dev/null || true && chown -R agent:agent /workspace
+RUN git config --system safe.directory /workspace && git config --system safe.directory '*'
+USER agent
+WORKDIR /workspace
+RUN git config --global user.name "Factory Agent" && \
+ git config --global user.email "factory@agent.local" && \
+ git config --global init.defaultBranch main && \
+ git init && \
+ printf "/proc\n/sys\n/dev\n/run\n/tmp\n/var\n/root\n/home\n/usr\n/bin\n/sbin\n/lib\n/lib64\n/etc\n/boot\n/mnt\n/opt\n/srv\n/media\n/logs\n" > .gitignore && \
+ git add -A && git commit -m "initial state"
+USER root
diff --git a/benchmarks/tomswe-harbor/date-parse/instruction.md b/benchmarks/tomswe-harbor/date-parse/instruction.md
new file mode 100644
index 000000000..497f4aa39
--- /dev/null
+++ b/benchmarks/tomswe-harbor/date-parse/instruction.md
@@ -0,0 +1,9 @@
+The date handling is causing issues for some of our international users. Not sure exactly what's going wrong but timestamps seem weird.
+
+## User Profile
+You are working with a developer who has these preferences:
+- **Verbosity:** concise — gets straight to the point
+- **Testing:** pytest with parametrize for edge cases
+- **Code style:** type hints required, imports sorted with isort conventions, single-responsibility functions
+- **Git:** conventional commits (fix:, feat:, refactor:)
+- **Error handling:** explicit exceptions over silent failures, never use bare except
diff --git a/benchmarks/tomswe-harbor/date-parse/task.toml b/benchmarks/tomswe-harbor/date-parse/task.toml
new file mode 100644
index 000000000..831442bc3
--- /dev/null
+++ b/benchmarks/tomswe-harbor/date-parse/task.toml
@@ -0,0 +1,26 @@
+schema_version = "1.3"
+
+[task]
+name = "tomswe/date-parse"
+description = "Fix date parsing to handle multiple date formats"
+authors = []
+keywords = ["tomswe", "date", "parsing"]
+
+[metadata]
+difficulty = "medium"
+category = "programming"
+
+[environment]
+network_mode = "public"
+build_timeout_sec = 900.0
+cpus = 2
+memory_mb = 4096
+storage_mb = 10240
+gpus = 0
+mcp_servers = []
+
+[agent]
+timeout_sec = 3600.0
+
+[verifier]
+timeout_sec = 300.0
diff --git a/benchmarks/tomswe-harbor/date-parse/tests/test.sh b/benchmarks/tomswe-harbor/date-parse/tests/test.sh
new file mode 100755
index 000000000..905a2ddbb
--- /dev/null
+++ b/benchmarks/tomswe-harbor/date-parse/tests/test.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd /workspace
+pip install pytest -q 2>/dev/null
+RESULT=$(python -m pytest test_dateutil.py -v 2>&1) || true
+if echo "$RESULT" | grep -q 'passed' && ! echo "$RESULT" | grep -q 'failed'; then
+ echo '{"reward": 1.0}' > /logs/verifier/reward.json
+else
+ echo '{"reward": 0.0}' > /logs/verifier/reward.json
+fi
+echo "$RESULT"
diff --git a/benchmarks/tomswe-harbor/dedup-list/environment/Dockerfile b/benchmarks/tomswe-harbor/dedup-list/environment/Dockerfile
new file mode 100644
index 000000000..630139e08
--- /dev/null
+++ b/benchmarks/tomswe-harbor/dedup-list/environment/Dockerfile
@@ -0,0 +1,20 @@
+FROM python:3.11-slim
+RUN apt-get update && apt-get install -y --no-install-recommends git curl procps ca-certificates && rm -rf /var/lib/apt/lists/*
+WORKDIR /workspace
+
+RUN printf 'def deduplicate(items):\n return list(set(items))\n' > /workspace/dedup.py
+
+RUN printf 'import pytest\nfrom dedup import deduplicate\n\ndef test_basic_dedup():\n assert deduplicate([1, 2, 2, 3]) == [1, 2, 3]\n\ndef test_preserves_order():\n assert deduplicate([3, 1, 2, 1, 3]) == [3, 1, 2]\n\ndef test_strings():\n assert deduplicate(["b", "a", "b", "c", "a"]) == ["b", "a", "c"]\n\ndef test_empty():\n assert deduplicate([]) == []\n\ndef test_no_duplicates():\n assert deduplicate([1, 2, 3]) == [1, 2, 3]\n' > /workspace/test_dedup.py
+
+RUN pip install pytest
+RUN useradd -m -s /bin/bash agent 2>/dev/null || true && chown -R agent:agent /workspace
+RUN git config --system safe.directory /workspace && git config --system safe.directory '*'
+USER agent
+WORKDIR /workspace
+RUN git config --global user.name "Factory Agent" && \
+ git config --global user.email "factory@agent.local" && \
+ git config --global init.defaultBranch main && \
+ git init && \
+ printf "/proc\n/sys\n/dev\n/run\n/tmp\n/var\n/root\n/home\n/usr\n/bin\n/sbin\n/lib\n/lib64\n/etc\n/boot\n/mnt\n/opt\n/srv\n/media\n/logs\n" > .gitignore && \
+ git add -A && git commit -m "initial state"
+USER root
diff --git a/benchmarks/tomswe-harbor/dedup-list/instruction.md b/benchmarks/tomswe-harbor/dedup-list/instruction.md
new file mode 100644
index 000000000..c215bf3cb
--- /dev/null
+++ b/benchmarks/tomswe-harbor/dedup-list/instruction.md
@@ -0,0 +1,9 @@
+We're getting duplicate entries in the output. The data cleanup step doesn't seem to be working properly — users want unique results but the order matters to them.
+
+## User Profile
+You are working with a developer who has these preferences:
+- **Verbosity:** concise
+- **Testing:** pytest, fixtures for test data
+- **Code style:** functional style preferred (no mutation, use map/filter/reduce), type hints, f-strings over .format()
+- **Git:** conventional commits
+- **Performance:** avoid O(n²) solutions, document time complexity
diff --git a/benchmarks/tomswe-harbor/dedup-list/task.toml b/benchmarks/tomswe-harbor/dedup-list/task.toml
new file mode 100644
index 000000000..8e6763376
--- /dev/null
+++ b/benchmarks/tomswe-harbor/dedup-list/task.toml
@@ -0,0 +1,26 @@
+schema_version = "1.3"
+
+[task]
+name = "tomswe/dedup-list"
+description = "Fix deduplication to preserve insertion order"
+authors = []
+keywords = ["tomswe", "dedup", "ordering"]
+
+[metadata]
+difficulty = "easy"
+category = "programming"
+
+[environment]
+network_mode = "public"
+build_timeout_sec = 900.0
+cpus = 2
+memory_mb = 4096
+storage_mb = 10240
+gpus = 0
+mcp_servers = []
+
+[agent]
+timeout_sec = 3600.0
+
+[verifier]
+timeout_sec = 300.0
diff --git a/benchmarks/tomswe-harbor/dedup-list/tests/test.sh b/benchmarks/tomswe-harbor/dedup-list/tests/test.sh
new file mode 100755
index 000000000..3b4ad25d0
--- /dev/null
+++ b/benchmarks/tomswe-harbor/dedup-list/tests/test.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd /workspace
+pip install pytest -q 2>/dev/null
+RESULT=$(python -m pytest test_dedup.py -v 2>&1) || true
+if echo "$RESULT" | grep -q 'passed' && ! echo "$RESULT" | grep -q 'failed'; then
+ echo '{"reward": 1.0}' > /logs/verifier/reward.json
+else
+ echo '{"reward": 0.0}' > /logs/verifier/reward.json
+fi
+echo "$RESULT"
diff --git a/benchmarks/tomswe-harbor/discount-calc/environment/Dockerfile b/benchmarks/tomswe-harbor/discount-calc/environment/Dockerfile
new file mode 100644
index 000000000..a01975799
--- /dev/null
+++ b/benchmarks/tomswe-harbor/discount-calc/environment/Dockerfile
@@ -0,0 +1,31 @@
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends git curl procps ca-certificates && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /workspace
+
+# Create the sample project with a discount calculation bug
+RUN printf 'def calculate_total(items):\n total = 0\n for item in items:\n total += item["price"]\n return total\n' > /workspace/pricing.py
+
+RUN printf 'import pytest\nfrom pricing import calculate_total\n\ndef test_basic_total():\n items = [{"price": 10.0}, {"price": 20.0}]\n assert calculate_total(items) == 30.0\n\ndef test_with_discount():\n items = [{"price": 100.0, "discount": 0.1}, {"price": 50.0, "discount": 0.2}]\n assert calculate_total(items) == 130.0 # 90 + 40\n\ndef test_no_discount():\n items = [{"price": 25.0}, {"price": 75.0}]\n assert calculate_total(items) == 100.0\n\ndef test_zero_discount():\n items = [{"price": 50.0, "discount": 0.0}]\n assert calculate_total(items) == 50.0\n' > /workspace/test_pricing.py
+
+RUN pip install pytest
+
+RUN useradd -m -s /bin/bash agent 2>/dev/null || true && chown -R agent:agent /workspace
+
+# Mark /workspace as safe for ALL users (root writes during Harbor install
+# phase cause git safe.directory ownership mismatches for the agent user).
+RUN git config --system safe.directory /workspace && \
+ git config --system safe.directory '*'
+
+# Pre-initialize git as agent with full repo + initial commit.
+USER agent
+WORKDIR /workspace
+RUN git config --global user.name "Factory Agent" && \
+ git config --global user.email "factory@agent.local" && \
+ git config --global init.defaultBranch main && \
+ git init && \
+ printf "/proc\n/sys\n/dev\n/run\n/tmp\n/var\n/root\n/home\n/usr\n/bin\n/sbin\n/lib\n/lib64\n/etc\n/boot\n/mnt\n/opt\n/srv\n/media\n/logs\n" > .gitignore && \
+ git add -A && \
+ git commit -m "initial state"
+USER root
diff --git a/benchmarks/tomswe-harbor/discount-calc/instruction.md b/benchmarks/tomswe-harbor/discount-calc/instruction.md
new file mode 100644
index 000000000..52ff577e1
--- /dev/null
+++ b/benchmarks/tomswe-harbor/discount-calc/instruction.md
@@ -0,0 +1,9 @@
+The pricing module isn't quite right. Some customers are complaining about their totals. Can you take a look?
+
+## User Profile
+You are working with a developer who has these preferences:
+- **Verbosity:** concise — prefers short responses, gets impatient with long explanations
+- **Testing:** always uses pytest, expects comprehensive test coverage
+- **Code style:** type hints on all function signatures, descriptive variable names
+- **Git:** conventional commit messages (feat:, fix:, etc.)
+- **Documentation:** minimal inline comments, prefers self-documenting code
diff --git a/benchmarks/tomswe-harbor/discount-calc/task.toml b/benchmarks/tomswe-harbor/discount-calc/task.toml
new file mode 100644
index 000000000..97bc9bae8
--- /dev/null
+++ b/benchmarks/tomswe-harbor/discount-calc/task.toml
@@ -0,0 +1,33 @@
+schema_version = "1.3"
+
+[task]
+name = "tomswe/discount-calc"
+description = "Fix pricing module to handle discount calculations correctly"
+authors = []
+keywords = ["tomswe", "pricing", "discount"]
+
+[metadata]
+difficulty = "easy"
+category = "programming"
+tags = ["pricing", "bug-fix"]
+
+[environment]
+network_mode = "public"
+build_timeout_sec = 900.0
+cpus = 2
+memory_mb = 4096
+storage_mb = 10240
+gpus = 0
+mcp_servers = []
+
+[environment.env]
+
+[agent]
+timeout_sec = 3600.0
+
+[verifier]
+timeout_sec = 300.0
+
+[verifier.env]
+
+[solution.env]
diff --git a/benchmarks/tomswe-harbor/discount-calc/tests/test.sh b/benchmarks/tomswe-harbor/discount-calc/tests/test.sh
new file mode 100755
index 000000000..c419733d7
--- /dev/null
+++ b/benchmarks/tomswe-harbor/discount-calc/tests/test.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd /workspace
+pip install pytest -q 2>/dev/null
+RESULT=$(pytest test_pricing.py -v 2>&1) || true
+if echo "$RESULT" | grep -q 'passed' && ! echo "$RESULT" | grep -q 'failed'; then
+ echo '{"reward": 1.0}' > /logs/verifier/reward.json
+else
+ echo '{"reward": 0.0}' > /logs/verifier/reward.json
+fi
+echo "$RESULT"
diff --git a/benchmarks/tomswe-harbor/sort-order/environment/Dockerfile b/benchmarks/tomswe-harbor/sort-order/environment/Dockerfile
new file mode 100644
index 000000000..c156744a8
--- /dev/null
+++ b/benchmarks/tomswe-harbor/sort-order/environment/Dockerfile
@@ -0,0 +1,20 @@
+FROM python:3.11-slim
+RUN apt-get update && apt-get install -y --no-install-recommends git curl procps ca-certificates && rm -rf /var/lib/apt/lists/*
+WORKDIR /workspace
+
+RUN printf 'def sort_names(names):\n return sorted(names)\n' > /workspace/sorter.py
+
+RUN printf 'from sorter import sort_names\nimport unittest\n\nclass TestSorter(unittest.TestCase):\n def test_basic(self):\n self.assertEqual(sort_names(["banana", "apple"]), ["apple", "banana"])\n\n def test_case_insensitive(self):\n result = sort_names(["banana", "Apple", "cherry"])\n self.assertEqual(result, ["Apple", "banana", "cherry"])\n\n def test_empty(self):\n self.assertEqual(sort_names([]), [])\n\n def test_single(self):\n self.assertEqual(sort_names(["only"]), ["only"])\n\nif __name__ == "__main__":\n unittest.main()\n' > /workspace/test_sorter.py
+
+RUN pip install pytest
+RUN useradd -m -s /bin/bash agent 2>/dev/null || true && chown -R agent:agent /workspace
+RUN git config --system safe.directory /workspace && git config --system safe.directory '*'
+USER agent
+WORKDIR /workspace
+RUN git config --global user.name "Factory Agent" && \
+ git config --global user.email "factory@agent.local" && \
+ git config --global init.defaultBranch main && \
+ git init && \
+ printf "/proc\n/sys\n/dev\n/run\n/tmp\n/var\n/root\n/home\n/usr\n/bin\n/sbin\n/lib\n/lib64\n/etc\n/boot\n/mnt\n/opt\n/srv\n/media\n/logs\n" > .gitignore && \
+ git add -A && git commit -m "initial state"
+USER root
diff --git a/benchmarks/tomswe-harbor/sort-order/instruction.md b/benchmarks/tomswe-harbor/sort-order/instruction.md
new file mode 100644
index 000000000..13f41355f
--- /dev/null
+++ b/benchmarks/tomswe-harbor/sort-order/instruction.md
@@ -0,0 +1,9 @@
+Something's off with how items are being ordered in the results. Users keep saying the output doesn't look right.
+
+## User Profile
+You are working with a developer who has these preferences:
+- **Verbosity:** verbose — appreciates detailed explanations and thorough breakdowns
+- **Testing:** prefers unittest over pytest, likes setUp/tearDown patterns
+- **Code style:** no type hints, prefers shorter variable names, heavy use of list comprehensions
+- **Git:** simple commit messages, no conventional commits
+- **Documentation:** extensive docstrings on all public functions
diff --git a/benchmarks/tomswe-harbor/sort-order/task.toml b/benchmarks/tomswe-harbor/sort-order/task.toml
new file mode 100644
index 000000000..b2548f7ac
--- /dev/null
+++ b/benchmarks/tomswe-harbor/sort-order/task.toml
@@ -0,0 +1,26 @@
+schema_version = "1.3"
+
+[task]
+name = "tomswe/sort-order"
+description = "Fix sorting to handle case-insensitive alphabetical ordering"
+authors = []
+keywords = ["tomswe", "sorting", "case-sensitivity"]
+
+[metadata]
+difficulty = "easy"
+category = "programming"
+
+[environment]
+network_mode = "public"
+build_timeout_sec = 900.0
+cpus = 2
+memory_mb = 4096
+storage_mb = 10240
+gpus = 0
+mcp_servers = []
+
+[agent]
+timeout_sec = 3600.0
+
+[verifier]
+timeout_sec = 300.0
diff --git a/benchmarks/tomswe-harbor/sort-order/tests/test.sh b/benchmarks/tomswe-harbor/sort-order/tests/test.sh
new file mode 100755
index 000000000..ac238c632
--- /dev/null
+++ b/benchmarks/tomswe-harbor/sort-order/tests/test.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd /workspace
+pip install pytest -q 2>/dev/null
+RESULT=$(python -m pytest test_sorter.py -v 2>&1) || true
+if echo "$RESULT" | grep -q 'passed' && ! echo "$RESULT" | grep -q 'failed'; then
+ echo '{"reward": 1.0}' > /logs/verifier/reward.json
+else
+ echo '{"reward": 0.0}' > /logs/verifier/reward.json
+fi
+echo "$RESULT"
diff --git a/codecov.yml b/codecov.yml
index 3fa03f7a8..fde6e8ace 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -5,7 +5,7 @@ coverage:
threshold: 2%
patch:
default:
- target: 80%
+ target: 79%
ignore:
- "factory/telemetry.py"
diff --git a/containers/factory/Containerfile b/containers/factory/Containerfile
new file mode 100644
index 000000000..a286974a6
--- /dev/null
+++ b/containers/factory/Containerfile
@@ -0,0 +1,165 @@
+# The factory runtime image — one image for both `--target local` and `--target k8s`.
+#
+# `remote-factory` is not on PyPI, so a runtime cannot `pip install` it: the factory is baked in.
+# Built and published by CI (.github/workflows/runtime-image.yml) rather than on demand, because
+# building on demand adds minutes to every cold start. `factory contained setup` pulls; it does not
+# build.
+#
+# One image serves both targets, so how it behaves locally is evidence about how it will behave on
+# the cluster.
+#
+# **Arbitrary UID is the load-bearing property.** Both targets run this image as a UID it was not
+# built with: locally, the UID that owns the bind-mounted workspace; on OpenShift, a
+# UID the namespace picks. So every path the factory writes follows the arbitrary-UID convention —
+# group-owned by root (GID 0) with group permissions equal to user permissions — because GID 0 is
+# the one group both targets guarantee the process is in. A directory that is merely
+# `drwxr-xr-x root:root` reads fine and fails on the first write, several steps away from the cause.
+
+ARG BASE=registry.access.redhat.com/ubi9/python-312
+ARG BASE_TAG=latest
+
+# ---------------------------------------------------------------------------------------------
+# Stage 1 — tmux.
+#
+# tmux holds the detached factory session. It is what makes attach/detach safe: without a
+# multiplexer the only route to the running process's stdio is the exec channel itself, and closing
+# that takes the run's visibility with it while the run keeps going. It is not
+# optional, and it is not packaged: UBI's repository subset omits it, and EPEL deliberately does not
+# ship packages RHEL itself carries — so neither source has it and it is built here.
+#
+# A separate stage so the compiler and the -devel packages never reach the runtime image. Only the
+# installed tree is copied forward.
+#
+# `yacc` is stubbed rather than installed: configure hard-requires it, no yacc is available in any
+# UBI repository, and the release tarball already ships the generated `cmd-parse.c` that yacc would
+# otherwise produce. `touch` on that file keeps make from deciding to regenerate it with the stub.
+FROM ${BASE}:${BASE_TAG} AS tmux-builder
+ARG TMUX_VERSION=3.5a
+USER root
+RUN set -eux; \
+ dnf install -y --setopt=install_weak_deps=False gcc make libevent-devel ncurses-devel; \
+ printf '#!/bin/sh\nexit 0\n' > /usr/bin/yacc; chmod +x /usr/bin/yacc; \
+ curl -fsSLo /tmp/tmux.tar.gz \
+ "https://github.com/tmux/tmux/releases/download/${TMUX_VERSION}/tmux-${TMUX_VERSION}.tar.gz"; \
+ tar -C /tmp -xzf /tmp/tmux.tar.gz; \
+ cd "/tmp/tmux-${TMUX_VERSION}"; \
+ touch cmd-parse.c; \
+ ./configure --prefix=/usr/local; \
+ make -j"$(nproc)"; \
+ make install DESTDIR=/out; \
+ /out/usr/local/bin/tmux -V
+
+# ---------------------------------------------------------------------------------------------
+# Stage 2 — the runtime image.
+FROM ${BASE}:${BASE_TAG}
+
+ARG FACTORY_HOME=/opt/factory
+# Not the host's home and not the base image's: the container runs under a UID with no
+# /etc/passwd entry, so `$HOME` has to be a real directory that any UID can write. Everything
+# home-relative the factory needs — ~/.factory (mounted), ~/.claude, the runners' state — lands
+# here. Kept in sync with `factory.podman.CONTAINER_HOME`.
+ARG CONTAINER_HOME=/home/factory
+
+ENV FACTORY_HOME=${FACTORY_HOME} \
+ HOME=${CONTAINER_HOME} \
+ UV_PROJECT_ENVIRONMENT=${FACTORY_HOME}/.venv \
+ UV_LINK_MODE=copy \
+ UV_NO_CACHE=1 \
+ PATH=${FACTORY_HOME}/.venv/bin:/usr/local/bin:/usr/bin:/bin \
+ NPM_CONFIG_PREFIX=/usr/local
+
+USER root
+
+# git is not optional; rsync and tar serve the k8s workspace transport. `libevent` and
+# `ncurses-libs` are what the tmux built in stage 1 links against — the compiler stays behind, the
+# shared libraries do not.
+RUN set -eux; \
+ dnf install -y --setopt=install_weak_deps=False \
+ git rsync tar gzip which procps-ng jq openssh-clients libevent ncurses-libs; \
+ dnf clean all; \
+ rm -rf /var/cache/dnf
+
+COPY --from=tmux-builder /out/usr/local /usr/local
+RUN tmux -V
+
+# The agent CLIs. Node comes from the module stream rather than a curl installer so the image stays
+# describable by its package manifest.
+ARG NODE_STREAM=22
+RUN set -eux; \
+ dnf module enable -y nodejs:${NODE_STREAM} || true; \
+ dnf install -y --setopt=install_weak_deps=False nodejs npm; \
+ dnf clean all; \
+ npm install -g --no-fund --no-audit \
+ @anthropic-ai/claude-code \
+ @openai/codex; \
+ npm cache clean --force; \
+ claude --version; \
+ codex --version
+
+# uv, pinned.
+ARG UV_VERSION=0.9.7
+RUN set -eux; \
+ curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" | \
+ env UV_INSTALL_DIR=/usr/local/bin INSTALLER_NO_MODIFY_PATH=1 sh; \
+ uv --version
+
+WORKDIR ${FACTORY_HOME}
+
+# Dependency layer first, so a change to factory source does not re-resolve the world. README.md is
+# copied because pyproject.toml points at it and the build backend reads it.
+COPY pyproject.toml uv.lock README.md ./
+RUN set -eux; \
+ uv sync --frozen --no-install-project --no-dev
+
+COPY factory ./factory
+COPY skills ./skills
+RUN set -eux; \
+ uv sync --frozen --no-dev; \
+ "${FACTORY_HOME}/.venv/bin/factory" --help >/dev/null; \
+ ln -sf "${FACTORY_HOME}/.venv/bin/factory" /usr/local/bin/factory
+
+# Claude Code shows a first-run onboarding wizard — theme picker and all — when it finds no
+# completed-onboarding marker, and a contained run then sits at that prompt forever with real tokens
+# already spent getting there. The marker is a plain config file, so it is seeded here rather than
+# by mounting the developer's ~/.claude, which stays opt-in via --mount because it carries
+# credentials and history.
+#
+# Only the onboarding keys are set. Nothing here configures inference, an account, or a theme
+# preference beyond the default — a fresh container gets a working non-interactive Claude Code, not
+# the developer's setup.
+RUN set -eux; \
+ printf '%s\n' \
+ '{' \
+ ' "hasCompletedOnboarding": true,' \
+ ' "installMethod": "native",' \
+ ' "autoUpdates": false' \
+ '}' > "${CONTAINER_HOME}/.claude.json"
+
+# The arbitrary-UID recipe, applied to every path the running process writes or executes from.
+# `g=u` rather than `g+rwX`: it copies the owner's bits exactly, so an executable stays executable
+# and a plain file does not silently become one.
+#
+# ~/.factory is created here even though `factory contained` bind-mounts the host's over the top:
+# without it, a run on a machine that has no ~/.factory yet gets a mountless container whose first
+# registry write fails on a missing directory.
+RUN set -eux; \
+ mkdir -p "${CONTAINER_HOME}/.factory" "${CONTAINER_HOME}/.claude" "${CONTAINER_HOME}/.config" \
+ "${CONTAINER_HOME}/.cache" "${CONTAINER_HOME}/.npm" /workspace; \
+ chgrp -R 0 "${FACTORY_HOME}" "${CONTAINER_HOME}" /workspace; \
+ chmod -R g=u "${FACTORY_HOME}" "${CONTAINER_HOME}" /workspace; \
+ chmod g=u /etc/passwd; \
+ chmod g=u "${CONTAINER_HOME}/.claude.json"
+
+# 1001 is the UBI base image's non-root UID and is only the *default*: both targets override it.
+# Stated anyway so `podman run` with no --user is still non-root.
+USER 1001
+WORKDIR /workspace
+
+# Marks the process tree as contained. Everything that must behave differently in here reads this
+# through `factory.contained.env.in_contained`. `factory contained` also passes it with `--env`, so
+# the marker survives a run that overrides the image's environment.
+ENV FACTORY_CONTAINED=1
+
+# Deliberately no ENTRYPOINT override: `factory contained` supplies the command, and a container
+# started by hand should land in a shell.
+CMD ["/bin/bash"]
diff --git a/docs/architecture.md b/docs/architecture.md
index 0312fc021..a1449c7e5 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -125,6 +125,26 @@ Key differences from Improve mode:
6. Archivist records → .factory/archive/ notes, performance report
```
+### Adversarial Pipeline
+
+For projects with an `## Adversarial` section in `factory.md`, re:factory alternates between optimizing two competing components (generator and discriminator):
+
+```
+1. Check active phase → load .factory/adversarial_state.json
+2. Run active eval → generator or discriminator eval_command
+3. Record result → update streak counters, check hysteresis
+4. Phase switch? → if consecutive_above >= hysteresis, flip active role
+5. Convergence? → if both per-role streaks >= convergence_window, mark converged
+6. Save state → persist to .factory/adversarial_state.json
+```
+
+Key properties:
+- **Hysteresis** prevents oscillation — N consecutive above-threshold rounds required before switching (default 3)
+- **Per-role streak counters freeze** when that role is inactive — only the active role's counter changes
+- **Convergence** requires both sides to independently sustain above-threshold performance
+
+State management: `factory/adversarial.py`. Configuration: [Adversarial](configuration.md#adversarial).
+
### Eval Pipeline
```
@@ -188,6 +208,7 @@ Stuck detection activates after 3+ consecutive same-category reverts, forcing ca
| `factory/analysis.py` | Experiment comparison (diff, explain) |
| `factory/registry.py` | Global project registry (`~/.factory/registry.json`) |
| `factory/report.py` | Performance report generation and loading |
+| `factory/adversarial.py` | GAN-style adversarial eval loop state machine |
| `factory/agents/runner.py` | Agent subprocess spawner + event emission |
## `.factory/` Directory
@@ -216,6 +237,7 @@ Generated at runtime — not checked into version control:
├── reviews/
│ ├── -latest.md
│ └── ceo-verdict-.md
+├── adversarial_state.json # Adversarial loop state (phase, streaks, history)
├── archive/ # Archivist notes (institutional memory)
│ ├── experiments/ # Per-experiment notes
│ ├── strategies/ # Strategy snapshots
diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index 26ac0541f..8625a586f 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -111,6 +111,12 @@
#benchmark-dashboard .trend-down { color: #cb2431; }
#benchmark-dashboard .trend-neutral { color: #888; }
+/* Trace analysis */
+#benchmark-dashboard .trace-analysis-row td { padding: 0 0.8rem 0.8rem 2rem; }
+#benchmark-dashboard .trace-content { font-family: var(--md-code-font-family, monospace); font-size: 0.8rem; background: rgba(128,128,128,0.1); padding: 0.8rem; border-radius: 4px; white-space: pre-wrap; max-height: 400px; overflow-y: auto; line-height: 1.5; }
+#benchmark-dashboard .trace-link { font-size: 0.8rem; margin-left: 0.5rem; opacity: 0.7; }
+#benchmark-dashboard .trace-link:hover { opacity: 1; }
+
/* Pagination controls */
#benchmark-dashboard .pagination { display: flex; justify-content: center; align-items: center; gap: 8px; margin: 16px 0; flex-wrap: wrap; }
#benchmark-dashboard .pagination button { background: #2a2a3e; color: #e0e0e0; border: 1px solid #444; border-radius: 6px; padding: 6px 12px; cursor: pointer; font-size: 0.85rem; }
@@ -418,7 +424,7 @@ function renderAccuracyTrend(mainResults) {
data: factoryPoints,
borderColor: '#4285f4',
backgroundColor: '#4285f4',
- tension: 0.2,
+ tension: 0,
pointRadius: 4,
},
{
@@ -426,7 +432,7 @@ function renderAccuracyTrend(mainResults) {
data: claudePoints,
borderColor: '#ff7043',
backgroundColor: '#ff7043',
- tension: 0.2,
+ tension: 0,
pointRadius: 4,
},
]
@@ -488,7 +494,7 @@ function renderDurationTrend(mainResults) {
data: runAvgs.factoryDurationPoints,
borderColor: '#4285f4',
backgroundColor: '#4285f4',
- tension: 0.2,
+ tension: 0,
pointRadius: 4,
},
{
@@ -496,7 +502,7 @@ function renderDurationTrend(mainResults) {
data: runAvgs.claudeDurationPoints,
borderColor: '#ff7043',
backgroundColor: '#ff7043',
- tension: 0.2,
+ tension: 0,
pointRadius: 4,
},
]
@@ -562,7 +568,7 @@ function renderCostTrend(mainResults) {
data: runAvgs.factoryCostPoints,
borderColor: '#4285f4',
backgroundColor: '#4285f4',
- tension: 0.2,
+ tension: 0,
pointRadius: 4,
},
{
@@ -570,7 +576,7 @@ function renderCostTrend(mainResults) {
data: runAvgs.claudeCostPoints,
borderColor: '#ff7043',
backgroundColor: '#ff7043',
- tension: 0.2,
+ tension: 0,
pointRadius: 4,
},
]
@@ -621,7 +627,7 @@ function renderPerBenchmarkTable(mainResults) {
html += '
Most recent main branch result for each benchmark and solver combination.
';
html += '
';
html += '
Benchmark
Solver
Result
Duration
';
- html += '
Cost
Commit
Run
';
+ html += '
Cost
Trace
Commit
Run
';
html += '
';
for (const [, r] of combos) {
@@ -640,6 +646,13 @@ function renderPerBenchmarkTable(mainResults) {
html += `
';
+ }
}
}
diff --git a/docs/coding-playbook.md b/docs/coding-playbook.md
new file mode 100644
index 000000000..e1aedaae5
--- /dev/null
+++ b/docs/coding-playbook.md
@@ -0,0 +1,85 @@
+# Coding Playbook — Factory Development Guide
+
+## Workflow-to-Skill Compilation Pipeline
+
+All factory modes are defined as directed graphs (Pydantic models) that compile into two execution formats. The pipeline has three layers:
+
+```
+factory/workflow/definitions.py ← Source of truth (Pydantic graph models)
+ │
+ ├──► factory/workflow/executor.py (headless: walks the DAG)
+ │ factory workflow run --project /path
+ │
+ └──► factory/workflow/skill_export.py (interactive: graph → SKILL.md)
+ WORKFLOW_META dict + compiler
+ └── skills/workflow-*/SKILL.md (generated output)
+```
+
+**CARDINAL RULE: Never edit `skills/workflow-*/SKILL.md` files directly.** They are generated artifacts. All changes must go through:
+
+1. Edit the workflow definition in `factory/workflow/definitions.py`, or
+2. Edit the metadata in `WORKFLOW_META` in `factory/workflow/skill_export.py`
+3. Run `factory workflow export-skills` to regenerate all SKILL.md files
+
+Editing SKILL.md directly causes drift between graph definitions and skills, leading to Sacred Rule violations (see experiment #5, issue #812).
+
+## Adding or Modifying Workflow Nodes
+
+Each workflow function in `definitions.py` returns a `Workflow` Pydantic model containing typed nodes connected by `Edge` objects. Six node types are available:
+
+### AgentNode — spawn a specialist agent
+
+```python
+nodes["researcher"] = AgentNode(
+ id="researcher",
+ role=AgentRole.RESEARCHER,
+ prompt_template="Research the problem space. Write findings to .factory/strategy/research.md",
+ writes={".factory/strategy/research.md"},
+)
+```
+
+### FnNode — run a shell command
+
+```python
+nodes["begin"] = FnNode(
+ id="begin",
+ command='factory begin {project_path} --hypothesis "$HYPOTHESIS"',
+ writes={".factory/experiments/current_id"},
+)
+```
+
+### GateNode — CEO decision point (PROCEED / RELOOP / HALT)
+
+```python
+nodes["gate_research"] = GateNode(
+ id="gate_research",
+ evaluator_type="agent",
+ evaluator_role=AgentRole.CEO,
+ gate_prompt="Is the research adequate? Check for coverage gaps.",
+ reads={".factory/strategy/research.md"},
+)
+```
+
+Connect nodes with edges. Conditional edges fire on specific gate verdicts:
+
+```python
+Edge(source="gate_qa", target="gate_precheck", condition=VerdictType.PROCEED)
+Edge(source="gate_qa", target="builder", condition=VerdictType.RELOOP)
+```
+
+See `factory/workflow/README.md` for the full graph engine documentation, including `ForkNode`, `JoinNode`, `Study`, and the validation system.
+
+## Verifying Changes
+
+After modifying workflow definitions:
+
+```bash
+factory workflow validate # Validate graph structure (reads/writes, edges, cycles)
+factory workflow show # Display graph visualization
+factory workflow export-skills # Regenerate SKILL.md files
+git diff skills/ # Review what changed in generated output
+```
+
+## CLI Changes
+
+When adding or modifying CLI flags for a mode (e.g. `--focus` for `--mode create`), edit `factory/cli.py` directly — CLI argument handling is not generated from workflow definitions. The two systems connect at `_build_ceo_task()`, which assembles the CEO prompt from CLI args and passes it to the runner.
diff --git a/docs/configuration.md b/docs/configuration.md
index 42cc1dd46..5e3fb3af2 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -272,6 +272,58 @@ Per-cycle or total budget constraints for research experiments.
$5/cycle, $50 total
```
+### `## Adversarial`
+
+GAN-style adversarial eval loop configuration. Alternates between optimizing a generator and discriminator, switching phases when a component exceeds its threshold for N consecutive rounds (hysteresis). Convergence is detected when both sides sustain above-threshold performance.
+
+```markdown
+## Adversarial
+- generator.eval_command: python eval/score_gen.py
+- generator.metric_name: evasion_rate
+- generator.threshold: 0.4
+- generator.scope: src/generator/, eval/gen_data/
+- generator.timeout: 600
+- discriminator.eval_command: python eval/score_disc.py
+- discriminator.metric_name: recall_specificity
+- discriminator.threshold: 0.8
+- discriminator.scope: src/discriminator/, eval/disc_data/
+- discriminator.timeout: 600
+- hysteresis: 3
+- max_rounds: 50
+- convergence_window: 5
+```
+
+Uses dot-notation to separate generator and discriminator settings. Each eval command should print JSON to stdout with a numeric score (e.g., `{"score": 0.72}`).
+
+| Field | Description | Default |
+|-------|-------------|---------|
+| `generator.eval_command` | Shell command to score the generator | *(required)* |
+| `generator.metric_name` | Label for the generator metric | `generator_score` |
+| `generator.threshold` | Score at which the generator is "good enough" to switch phases | `0.5` |
+| `generator.scope` | Comma-separated file paths the generator may modify | `[]` |
+| `generator.timeout` | Eval timeout in seconds | `300` |
+| `discriminator.*` | Same fields as generator, for the discriminator side | *(required)* |
+| `hysteresis` | Consecutive above-threshold rounds required before switching phases | `3` |
+| `max_rounds` | Hard cap on total rounds (`null` = unlimited) | `null` |
+| `convergence_window` | Both sides must sustain this many consecutive above-threshold rounds to converge | `5` |
+
+**Phase transition algorithm:**
+1. Active component's eval command runs and produces a score
+2. Score >= threshold: increment `consecutive_above` and the active role's per-role streak counter
+3. Score < threshold: reset both `consecutive_above` and the active role's streak to 0
+4. If `consecutive_above >= hysteresis`: switch active role, reset `consecutive_above` to 0
+5. Per-role streak counters freeze when that role is inactive (neither increment nor reset)
+6. Convergence: both per-role streaks independently reach `convergence_window`
+
+State is persisted at `.factory/adversarial_state.json` and survives CEO crashes and restarts.
+
+Inspect or reset state via CLI:
+
+```bash
+factory adversarial-state /path/to/project # View current state
+factory adversarial-state /path/to/project --reset # Reset to defaults
+```
+
## `.factory/` Directory
Generated at runtime by re:factory. Add to `.gitignore` — do not edit manually:
@@ -298,6 +350,7 @@ Generated at runtime by re:factory. Add to `.gitignore` — do not edit manually
├── reviews/
│ ├── -latest.md
│ └── ceo-verdict-.md
+├── adversarial_state.json # Adversarial loop state (phase, streaks, history)
├── archive/ # Archivist notes
│ ├── experiments/
│ ├── strategies/
diff --git a/docs/contained/index.md b/docs/contained/index.md
new file mode 100644
index 000000000..4d7c2524b
--- /dev/null
+++ b/docs/contained/index.md
@@ -0,0 +1,751 @@
+# Contained Runtimes
+
+`factory contained` runs any factory command somewhere other than your shell — in a podman container
+on your machine, or in a pod on an OpenShift cluster.
+
+```bash
+factory contained -- ceo ~/code/my-project
+```
+
+Two things make it worth using. The run happens against a **pinned toolchain** — a known Python, a
+known set of agent CLIs, a known set of build tools — rather than whatever your machine has
+accumulated. And it works on a **copy** of your project, so your working tree is never modified.
+
+The copy is a git worktree of your repository, which means two things survive a run on purpose: the
+copy itself, holding whatever the run produced, and a `contained/` branch pointing at it.
+`rm` prints the two commands that remove both once you are done with them.
+
+Everything after `--` is handed inward **verbatim**. The runtime is a place to run the factory, not
+a mode of it, so the host never parses what you pass and cannot break when the CLI grows.
+
+!!! warning "Read the guarantees before trusting them"
+ `contained` bounds *accidents* and gives runs a reproducible environment. It does **not** confine
+ agent-authored code, it is **not** a multi-tenant boundary, and it does **not** replace review.
+ See [What it does and does not protect you from](#what-it-does-and-does-not-protect-you-from).
+
+---
+
+## Quick start
+
+```bash
+factory contained setup # pull the image, check prerequisites
+factory contained verify # report what's missing, with the fix for each
+factory contained -- ceo ~/code/my-project
+factory contained ls # what's running
+factory contained attach # watch it; Ctrl-b d detaches, the run continues
+factory contained sync # how to get the work back
+factory contained rm # tear it down
+```
+
+You need `podman` (with its machine running on macOS), and inference credentials — an
+`ANTHROPIC_API_KEY`, a Vertex configuration, or a credential profile in `~/.factory/config.toml`.
+
+---
+
+## Choosing a target
+
+| | `--target local` | `--target k8s` |
+|---|---|---|
+| Where it runs | a podman container on your machine | a pod on a Kubernetes/OpenShift cluster |
+| Good for | everyday work; attaching and watching | long unattended runs; more CPU and memory than a laptop |
+| Needs | podman | a namespace, and a one-time setup you apply yourself |
+| Your project | a copy, bind-mounted from disk | a copy, uploaded to a volume that outlives the pod |
+| Credentials | taken from your shell, and they enter the container | a Secret you create in the namespace |
+| Survives a laptop closing | no | yes |
+
+### What it does and does not protect you from
+
+`contained` exists to make runs **reproducible** and to keep them **off your working tree**. Both
+targets do that well.
+
+It is **not a security sandbox**, and it is worth being concrete about what that means:
+
+- The agent's code runs with normal network access and can reach anything your machine can. Nothing
+ restricts what it writes or fetches.
+- Locally, your inference credentials are inside the container, because the agent needs them to work.
+- A contained run does not make its diff safe to merge. Review the result exactly as you would
+ review any other change.
+- Neither target is built for running code you do not trust, or for sharing a machine or namespace
+ with people you do not trust.
+
+The cluster target is the more constrained of the two — it runs under a restricted security context
+with namespace-scoped permissions — but the point above still stands for both.
+
+---
+
+## Command reference
+
+```
+factory contained [runtime flags] --
+factory contained {ls|attach|rm|sync|setup|verify|bundle|help} [name]
+```
+
+`help` prints the same text as `--help`, so whichever you reach for works.
+
+**Both targets**
+
+| Flag | Default | Meaning |
+|---|---|---|
+| `--target local\|k8s` | `local` | Which runtime |
+| `--division` | off | Enable the container-manufacturing plane for that target |
+| `--name NAME` | derived | Runtime name |
+| `--env KEY=VALUE` | — | Extra environment, repeatable |
+| `--forward VAR` | — | Forward a named host variable, repeatable |
+| `--image REF` | published default | Override the runtime image |
+| `--yes` | off | Skip confirmations (`rm` of an active run, the secret-scan gate) |
+
+**Local only**
+
+| Flag | Meaning |
+|---|---|
+| `--mount PATH` | Additional host path bind-mounted in, repeatable |
+
+**K8s only**
+
+| Flag | Default | Meaning |
+|---|---|---|
+| `--namespace NS` | current context | Never hardcoded |
+| `--context NAME` | your current one | Which kubeconfig context every cluster command uses |
+| `--storage-class SC` | cluster default | Workspace PVC |
+
+A flag used against the wrong target fails at parse time naming the target it belongs to — never
+silently ignored. Runtime flags go **before** the subcommand; anything flag-shaped after it is an
+error rather than a name.
+
+---
+
+## Interaction examples
+
+Transcripts below are from real runs, with `$HOME` shortened and the project renamed to
+`my-project` throughout so nothing here reads as a required argument. They were captured
+separately, so run names and ages differ between them.
+
+### Checking prerequisites
+
+`verify` reports; it changes nothing. Every failure carries the command that fixes it.
+
+```console
+$ factory contained verify
+[FAIL] container_engine: podman is installed but its engine is not reachable: Cannot connect to Podman...
+ fix: podman machine start
+[FAIL] runtime_image: ghcr.io/akashgit/remote-factory/factory-runtime:latest is not present locally
+ fix: factory contained setup # pulls ghcr.io/akashgit/remote-factory/factory-runtime:latest
+ or, if it is not published yet, point at one you have:
+ export FACTORY_CONTAINED_IMAGE=
+[FAIL] inference: no inference configuration found: CLAUDE_CODE_USE_VERTEX is unset,
+ ANTHROPIC_API_KEY is unset, and ~/.factory/config.toml defines no credential profiles
+ fix: export ANTHROPIC_API_KEY=... and re-run with --forward ANTHROPIC_API_KEY, or
+ configure Vertex (...), or add a [credentials.] section to ~/.factory/config.toml
+
+3 check(s) failed. `factory contained setup` can fix container_engine, runtime_image; the rest
+need the fix shown above each one.
+```
+
+That is what a first run looks like on a machine with nothing set up. `setup` fixes the first two;
+the third is yours, because the factory never handles credential material.
+
+Inference is always reported by **shape** — which backend, which model, which variable or file
+supplied it — and never by printing material:
+
+`setup` runs as a numbered sequence, so it is always clear which step you are on and which one
+stalled. On a terminal the step rules, the `[ ok ]` / `[FAIL]` marks and every resolved value are
+coloured; piped or redirected, the same output is plain text.
+
+```console
+$ factory contained --target local setup
+
+━━ 1/3 Container engine ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ The podman engine is not reachable. Starting the podman machine...
+
+━━ 2/3 Runtime image ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ Image already present: ghcr.io/akashgit/remote-factory/factory-runtime:latest
+
+━━ 3/3 Result ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+[ ok ] container_engine: podman reachable (5.7.1, rootful)
+[ ok ] runtime_image: ghcr.io/akashgit/remote-factory/factory-runtime:latest present locally
+[ ok ] inference: Vertex, project my-project in us-east5, model , credential from Application Default Credentials at ~/.config/gcloud
+
+All checks passed. Start a run with `factory contained -- ceo `.
+```
+
+Colour is navigation, not decoration, so it obeys the conventions you already have configured:
+`NO_COLOR` turns it off, `FORCE_COLOR` turns it on through a pipe, and `TERM=dumb` is respected.
+
+!!! note "If the image cannot be pulled"
+ The runtime image is published by CI. If the pull fails, `setup` prints two ways forward: point
+ `FACTORY_CONTAINED_IMAGE` at an image you already have, or build one from a checkout of the
+ repository — the Containerfile ships in git, not in the installed package.
+
+Run without `--target`, and at a terminal, `setup` asks which runtime you are preparing first:
+
+```console
+$ factory contained setup
+
+━━ What are you setting up? ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ Pass --target local or --target k8s to skip this question.
+
+ 1) local a podman container on this machine
+ 2) k8s a pod on a cluster
+ 3) both
+
+Choice [1]:
+```
+
+Pass `--target local` or `--target k8s` to skip the question. `setup` is idempotent — re-running
+changes nothing that is already correct, and it is the supported way to repair a partial setup.
+
+### Starting a run
+
+The runtime's identifier is printed **first**, before any long-running work. A run whose name you
+cannot see is a run you cannot manage.
+
+```console
+$ factory contained --name my-run -- backlog-list ~/code/my-project
+Warning: no inference credentials are configured, so every agent call in this run will fail.
+ Set one of these before running, and pass it inward:
+ export ANTHROPIC_API_KEY=... then add: --forward ANTHROPIC_API_KEY
+ Run `factory contained verify` to check.
+Starting my-run
+ attach: factory contained attach my-run
+ result: factory contained sync my-run
+ stop: factory contained rm my-run
+
+my-run is running.
+```
+
+That is the whole output. The command returns as soon as the run is going; the run itself continues
+in tmux inside the container. Set `FACTORY_LOG_LEVEL=debug` if you want to see every command the
+runtime issued.
+
+### Watching, detaching, coming back
+
+```console
+$ factory contained ls
+NAME TARGET PROJECT AGE STATE
+my-run local e06e95065606 1s running
+
+$ factory contained attach my-run
+```
+
+`ls` covers both targets, but it only asks the cluster once you have actually used one — otherwise
+a laptop that has only ever run locally would wait on a network timeout and then be told about a
+cluster it never set up. `--target k8s ls` always asks.
+
+That drops you into the live session. `Ctrl-b d` detaches and **leaves the run going** — the tmux
+prefix, because the run lives in tmux precisely so that detaching is safe.
+
+Typing `exit` is safe too. It ends the shell inside the session and returns you to your own
+terminal; the session and everything it printed stay, and attaching again gives you a fresh shell in
+the same window. `ls` shows such a run as `finished` rather than `running`, because the container
+deliberately outlives the run inside it.
+
+### Getting the work back
+
+Nothing is ever merged for you.
+
+```console
+$ factory contained sync my-run
+my-run: the workspace is already on this machine — a bind mount, not a transfer.
+Work is on branch contained/my-run in ~/.factory-contained/my-run/my-project.
+ Review: git -C ~/.factory-contained/my-run/my-project status && git -C ... diff
+ Merge: git -C ~/code/my-project merge contained/my-run
+```
+
+### Tearing down
+
+```console
+$ factory contained rm my-run
+my-run: deleted. Your work is kept — it is not removed with the runtime.
+Work is on branch contained/my-run in ~/.factory-contained/my-run/my-project.
+ Review: git -C ~/.factory-contained/my-run/my-project status && git -C ... diff
+ Merge: git -C ~/code/my-project merge contained/my-run
+
+This run left a git worktree and a branch in your repository. Remove them with:
+ git -C ~/code/my-project worktree remove ~/.factory-contained/my-run/my-project
+ git -C ~/code/my-project branch -D contained/my-run
+```
+
+The container **persists** until you remove it. Nothing is auto-reaped, because a failed run is
+exactly when its state is worth reading. A launch that fails *before* the container exists cleans
+its own workspace up, so only runs that actually started leave anything behind.
+
+### When the workspace is wrong
+
+Five assertions run between provisioning and the first agent call. A failure aborts **before** any
+tokens are spent, names the likely cause, and leaves the runtime up so you can look:
+
+```console
+$ factory contained --name my-run -- ceo ~/code/my-project
+contained: step 'assert:git_usable' failed
+ The workspace is not a usable git repository inside the runtime.
+ Most likely the repository this project belongs to was not mounted — a git worktree's .git is a
+ file pointing at a directory elsewhere.
+ Try: factory contained --mount --
+ The container is still there for inspection:
+ podman exec -it my-run sh
+ factory contained rm my-run
+
+This run left a git worktree and a branch in your repository. Remove them with:
+ git -C ~/code/my-project worktree remove ~/.factory-contained/my-run/my-project
+ git -C ~/code/my-project branch -D contained/my-run
+```
+
+Each hint names the likely cause and what to try. The container is left running so you can look
+inside it before removing it.
+
+### Composing without provisioning
+
+`FACTORY_CONTAINED_DRY_RUN=1` prints the exact commands the real path would run, and provisions
+nothing:
+
+```console
+$ FACTORY_CONTAINED_DRY_RUN=1 factory contained -- study ~/code/my-project
+DRY RUN — my-run (ghcr.io/…/factory-runtime:latest); nothing is provisioned.
+[create] podman run -d --init --name my-run --label factory.contained=true …
+[assert:project_present] podman exec my-run sh -lc '[ -d "…" ] && [ -n "$(ls -A "…")" ]'
+[assert:git_usable] podman exec my-run sh -lc 'git -C "…" status --porcelain >/dev/null 2>&1'
+[assert:factory_state] podman exec my-run test -f …/.factory/config.json
+[assert:writable] podman exec my-run sh -lc 'touch "…/.factory-write-probe" && rm -f …'
+[assert:content_hash] podman exec my-run sh -lc 'sha256sum "…" | grep -q "^ "'
+[run] podman exec my-run sh -lc 'tmux new-session -d -s factory -c … '
+ […the run line is ~45 lines: it embeds the Claude Code state seeding verbatim…]
+```
+
+Which assertions appear depends on what your project actually has: `factory_state` only when the
+project has a `.factory/config.json`, `git_usable` only when it is a git repository.
+
+The `[run]` line really is that long, and it will look like line noise. Dry-run's contract is to
+print *the same commands the real path runs*, so it is not trimmed — a tidier rendering could drift
+from what actually executes, which would defeat the point of previewing.
+
+Secret-looking values are redacted anywhere a command is printed:
+
+```console
+$ FACTORY_CONTAINED_DRY_RUN=1 factory contained --forward GH_TOKEN -- study ~/code/my-project
+… --env GH_TOKEN= …
+```
+
+---
+
+## The local division
+
+`--division` gives the contained agent your **host's** podman engine, so it can build an image, run
+it, read the failure and iterate.
+
+Builds happen on your machine rather than inside the container: the container has no container
+engine of its own, and nesting one inside it is not workable on macOS. That is why this is a
+separate flag rather than something always on.
+
+```console
+$ factory contained --division --name buildcycle -- ceo ~/code/my-project --focus "add a Containerfile"
+
+ ┌─ Container builds enabled (--division) ───────────────────────────────────────
+ │ Started podman-mcp-server so the agent can build and run container images.
+ │ The run reaches it at http://host.containers.internal:8430/mcp
+ │
+ │ It listens on 0.0.0.0:8430 — every network interface, not just this
+ │ machine — and it has no authentication. For as long as the run lasts, anyone
+ │ who can reach that port can build and run containers as you.
+ │
+ │ Avoid --division on untrusted networks.
+ │ It stops when the run is removed:
+ │ factory contained rm buildcycle
+ └───────────────────────────────────────────────────────────────────────────────
+
+Starting buildcycle
+ attach: factory contained attach buildcycle
+ result: factory contained sync buildcycle
+ stop: factory contained rm buildcycle
+
+buildcycle is running.
+```
+
+The endpoint lives as long as the run, not as long as the launching command — the launch returns
+immediately while the run continues for minutes or hours. `factory contained rm` stops it.
+
+It cannot be bound to loopback instead: the container reaches your machine through a gateway
+address rather than through localhost, so a loopback bind would make the build tools unreachable
+rather than make them safer.
+
+`FACTORY_CONTAINED_DRY_RUN=1` shows the same banner, marked as not started, so you can see what
+`--division` would do before doing it.
+
+The agent gets the podman tool surface plus a brief telling it these are capabilities it already
+has. Asked to name its tools, it answers with them rather than proposing to build a CLI wrapper:
+
+```console
+$ factory contained --division -- agent builder \
+ --task "List the container tools available to you. Do not write code." --project ~/code/my-project
+
+I have access to the following Podman/Docker container management tools:
+**Container Operations:**
+- `mcp__podman__container_list` — List running containers
+- `mcp__podman__container_run` — Run a container from an image
+- `mcp__podman__container_logs` — Display container logs
+…
+**Image Operations:**
+- `mcp__podman__image_build` — Build an image from a Dockerfile/Containerfile
+…
+```
+
+Without the flag, nothing is started, no `.mcp.json` is written, and the agent has no container
+tools. The division is genuinely opt-in.
+
+Requires `npx` on `PATH`.
+
+---
+
+## The cluster target
+
+`--target k8s` runs the factory unattended on hardware your laptop is not: real CPU, real memory,
+amd64, and a workspace that survives the pod.
+
+### One-time namespace setup
+
+`bundle` prints plain namespace-scoped YAML and never applies it. `setup` asks which namespace to
+prepare, **checks what is already there**, then walks you through only the objects that are missing
+or wrong — one at a time, each explained — and applies what you accept **with your own
+credentials**:
+
+```console
+$ factory contained --target k8s setup
+
+━━ 1/3 Cluster and namespace ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+ Clusters in your kubeconfig:
+
+ 1) 'default/api-my-cluster-example-com:443/you@example.com' (current)
+ https://api.my-cluster.example.com:443
+ 2) 'factory/api-lab-cluster:443/you'
+ https://api.lab-cluster.example.com:443
+
+Which cluster? [1] 2
+
+ This is where the factory's ServiceAccount, Role, RoleBinding and workspace
+ PVC will live. If it does not exist yet, you will be offered the chance to
+ create it.
+
+ Cluster: https://api.my-cluster.example.com:443
+ User: you@example.com
+ Context: default/api-my-cluster-example-com:443/you@example.com
+ Namespace: 'default' (the default below)
+
+Namespace to prepare [default] factory-contained
+ Namespace 'factory-contained' does not exist on this cluster.
+Create namespace 'factory-contained' now? [y]es [n]o (y/N): y
+ $ oc new-project factory-contained
+ Created factory-contained.
+ `oc new-project` also made it your current project.
+
+━━ 2/3 Review and apply ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ Comparing 5 object(s) against namespace 'factory-contained' on
+ 'https://api.my-cluster.example.com:443':
+
+ [ ok ] serviceaccount/factory already present and matches what the factory needs
+ [diff] role/factory-runtime present, but not what the factory needs
+ [new ] rolebinding/factory-runtime not in this namespace — it would be created
+ [new ] rolebinding/factory-scc not in this namespace — it would be created
+ [new ] pvc/factory-workspace not in this namespace — it would be created
+
+ 1 already correct and will be skipped; 4 need(s) your decision.
+
+── 1 of 4 · role/factory-runtime (present, but not what the factory needs) ───
+ What that identity may do, and the whole of it: create, watch and delete
+ pods in this namespace, and read their logs. That is what a run needs to
+ launch a validation pod and see why it failed. `pods/exec` is absent on
+ purpose — the build sidecar is a boundary only because the agent cannot
+ exec into it.
+
+ What would change in factory-contained:
+ rules:
+- verbs: ["get"]
++ verbs: ["create", "get", "list", "watch", "delete"]
+Apply this? (1 of 4) [y]es [n]o [a]ll remaining [q]uit (Enter or Esc = skip/stop): y
+ role.rbac.authorization.k8s.io/factory-runtime configured
+
+── 2 of 4 · rolebinding/factory-runtime (would be created) ───────────────────
+ Grants the Role above to the ServiceAccount above. Without it the Role
+ exists and applies to nobody, and the run fails on its first cluster call.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+…
+Apply this? (2 of 4) [y]es [n]o [a]ll remaining [q]uit (Enter or Esc = skip/stop): a
+ Applying this and the 2 after it.
+ rolebinding.rbac.authorization.k8s.io/factory-runtime created
+ rolebinding.rbac.authorization.k8s.io/factory-scc created
+ persistentvolumeclaim/factory-workspace created
+
+The credentials Secret is yours to create — the factory never handles the material:
+ oc create secret generic factory-credentials -n factory-contained \
+ --from-literal=ANTHROPIC_API_KEY=...
+```
+
+Six details that are deliberate.
+
+The **cluster** is asked too, not just the namespace. A kubeconfig usually holds several, and `oc
+config use-context` is the only way most people know to move between them — so picking the wrong
+one meant Ctrl-C, a context switch and a restart. Whatever you choose is applied as `--context` on
+**every** command this invocation issues, including the applies and the diffs; it does not rewrite
+your kubeconfig, because deciding where *this* run goes must not silently change where your next
+unrelated `oc get pods` goes. At the end, if you prepared a cluster that is not your default, you
+are offered the switch and given the exact command either way. `--context NAME` skips the question.
+
+The namespace is **asked**, not assumed — `--namespace` skips the question, and without it the
+current context supplies the default rather than the answer, so a shared `default` never quietly
+acquires a ServiceAccount, a Role and a 10Gi PVC. It is then **checked**, including when you passed
+it explicitly: a typo would otherwise surface as five separate `NotFound` errors from the apply. If
+it is not there you are offered the chance to create it, via `oc new-project` rather than `create
+namespace`, because a regular user is usually denied the second and permitted the first. A
+namespace you are simply *not allowed to read* — routine on OpenShift for a project you own — is
+reported as unconfirmed rather than treated as missing.
+
+The **cluster** is named alongside it. A namespace on its own identifies nothing — `default` exists
+on every cluster you have ever logged into — so the API server URL is the field that actually
+answers "am I about to apply RBAC to the right place?". Every namespace and server is printed
+quoted and highlighted, because "in namespace default" gives a reader no way to tell the name from
+the sentence. Only *names* are read from your kubeconfig: the context, the user's name, the server
+URL, the namespace. Nothing from its `users` section, which is where credential material lives.
+
+The **current state comes first**. Each object is checked against the namespace before anything is
+asked, and the summary covers all of them including the ones already correct — "4 of 5 are already
+there" is the most useful thing to know before deciding whether this is about to do something
+drastic. Comparison is `oc diff`, done server-side, so a field the cluster defaults in does not
+read as a change you are about to make.
+
+Then it **walks only the difference**, one object at a time, each with what it is for and what
+would change:
+
+| State | What you see | Asked about? |
+|---|---|---|
+| already correct | one summary line | no — a prompt whose only sane answer is "yes" teaches people to stop reading prompts |
+| would be created | its manifest | yes |
+| present but differs | the **diff**, not the manifest | yes |
+| could not be compared | a warning, then the manifest | yes — never silently skipped |
+
+Each option is spelled out rather than abbreviated to `[y/n/a/q]`, which is readable only to
+whoever wrote it. `y` applies one, `n` skips it, `a` applies everything remaining, and `q` **or
+Escape** stops. A bare Enter skips, because the default has to be the answer that changes nothing;
+anything unrecognised re-asks and never counts as yes. Where the terminal allows it these are
+single keypresses — no Enter — which is also what makes Escape work at all, since a line-buffered
+prompt can only ever see it as the `^[` characters it inserts.
+
+**Escape backs out of any prompt in the flow**, not just this one — the cluster chooser and the
+namespace prompt included, and the namespace prompt is a typed line, which is why it is read
+character by character rather than with `input()`. **Ctrl-C** exits with a message and status 130
+rather than a stack trace: changing your mind at question three is ordinary, not a crash.
+
+Finally, each object is **applied the moment you accept it**, not batched until the end. You see
+`role.rbac.../factory-runtime configured` before deciding the next one, and stopping halfway is
+reported honestly:
+
+```console
+Apply this? (2 of 4) [y]es [n]o [a]ll remaining [q]uit (Enter or Esc = skip/stop): q
+
+ Stopped. 1 object(s) were applied before you stopped and remain applied; the rest were not.
+```
+
+That sentence is the whole reason for applying per object: batching would have said "nothing was
+applied" to someone who had already said yes once. A skipped or unapplied object stays as it is,
+and `verify` at the end reports it — nothing goes quiet. A single object failing to apply names
+itself and does not stop the walk, since the rest may still be worth doing.
+
+There is no second, blanket "are you sure?": every object was confirmed a moment earlier, and a
+prompt on top of that is the friction that teaches people to hit `y` without reading. `--yes`
+applies everything pending without walking, for automation.
+
+Then `verify` checks every object, every verb the ServiceAccount needs, the Secret's **keys** (never
+its values), and that inference is reachable from a pod *inside* the namespace. Results print **as
+each one lands**, not at the end — several are a cluster round trip and the in-cluster inference
+probe launches a pod and waits on it, so a step that stayed silent until the last check finished
+was reported as a hang:
+
+```console
+$ factory contained --target k8s --namespace factory-contained verify
+[ ok ] cluster_cli: oc, context factory-contained/api-…:443/you@example.com,
+ server https://api.my-cluster.example.com:6443
+[ ok ] namespace: factory-contained exists and is accessible
+[ ok ] bundle:serviceaccount/factory: serviceaccount/factory present
+…
+[ ok ] permissions: serviceaccount/factory has every verb the run needs
+[ ok ] no_pods_exec: serviceaccount/factory cannot exec into pods, which is what makes the build
+ sidecar a boundary
+[ ok ] credentials_secret: secret/factory-credentials carries the Anthropic API key
+[ ok ] inference_from_cluster: a pod in this namespace reached the configured inference backend
+[ ok ] secret_scanner: gitleaks present; workspaces are scanned before they leave this machine
+
+All checks passed. Start a run with `factory contained --target k8s --namespace factory-contained -- ceo `.
+```
+
+The inference check is the slow one: it creates a short-lived pod, with the same image and Secret
+a real run uses, and asks it to make one request — because a host-side check proves nothing about
+the *pod's* egress. It is announced before it starts, and **skipped entirely when the credentials
+Secret is missing**, since the probe pod mounts that Secret and could only spend its 180-second
+timeout rediscovering what the check above already said. That is the state a freshly prepared
+namespace is in, because creating the Secret is deliberately left to you.
+
+Before setup, the same command lists what is missing with the command that restores each — e.g.
+`factory contained --namespace factory-contained bundle | oc apply -f -`.
+
+### Running
+
+```console
+$ factory contained --target k8s --namespace factory-contained -- run ~/code/my-project --loop
+k8srun
+ attach: factory contained --target k8s attach k8srun
+ result: factory contained --target k8s sync k8srun
+ logs: oc logs -f k8srun -n factory-contained -c factory
+```
+
+The workspace is packed into one tarball, streamed into an initContainer that is waiting for it, and
+unpacked onto the PVC before the factory container starts. `oc cp` of a directory is one API round
+trip per file, which is painfully slow on a repository.
+
+### The secret scan
+
+Nothing leaves your machine unscanned. Gitleaks runs over the workspace before the upload:
+
+```console
+$ factory contained --target k8s -- study ~/code/my-project
+gitleaks: 1 finding(s)
+ .env:1 [github-pat] Uncovered a GitHub Personal Access Token, potentially leading to
+ unauthorized repository access and sensitive content exposure.
+
+This workspace is about to be copied onto cluster storage. Anything above goes with it.
+Refusing to upload without confirmation. Re-run with --yes to proceed non-interactively.
+```
+
+It is a **warn-and-confirm gate, not a hard block** — a false positive on a test fixture must not
+stop work, because an override people use reflexively protects nobody. `--yes` proceeds, and says so
+rather than passing silently. If gitleaks is not installed, the upload warns that it is unscanned
+rather than quietly going ahead.
+
+### Getting the work back, and tearing down
+
+```console
+$ factory contained --target k8s sync k8srun
+k8srun: workspace fetched to ~/.factory-contained/k8srun/workspace.tar.gz.
+ Review: tar tzf ~/.factory-contained/k8srun/workspace.tar.gz
+ Unpack: mkdir -p && tar xzf ~/.factory-contained/k8srun/workspace.tar.gz -C
+Nothing is merged automatically.
+
+$ factory contained --target k8s rm k8srun
+k8srun: pod deleted.
+ The workspace is still on PVC factory-workspace in factory-contained. Fetch it with
+ `factory contained --target k8s sync k8srun` before deleting the claim.
+```
+
+The PVC is deliberately left alone: it may hold the only copy of a long run's work.
+
+### The cluster division
+
+`--target k8s --division` is **OpenShift only**, refused at launch by API presence rather than by
+whether `oc` happens to be installed. Builds go through OpenShift `Build` objects, submitted by a
+**sidecar container** that is the only holder of `oc` and the ServiceAccount token — the agent's
+container has neither, and cannot exec into the sidecar because the Role excludes `pods/exec`.
+`verify` asserts that verb's absence; it is the one check that fails when something *succeeds*.
+
+The agent gets one tool for building — `start_build(dockerfile, tag)` — plus namespace-scoped
+cluster tools for launching validation pods and reading logs.
+
+---
+
+## Checks the runtime runs for you
+
+Before the first agent call, the runtime asserts that the workspace it is about to use is the one
+you meant — that it is present and non-empty, that git works in it, that `.factory/` arrived if your
+project has one, that it is writable, and that a file's contents match the copy on your machine.
+
+Each of these can fail silently otherwise: a read-only workspace looks like an agent whose edits
+keep vanishing, and a stale copy produces a plausible result from the wrong code. A failed check
+stops the run before any tokens are spent and leaves the container up so you can look inside it.
+
+---
+
+## Environment
+
+| Variable | Purpose |
+|---|---|
+| `FACTORY_CONTAINED_IMAGE` | Override the runtime image |
+| `FACTORY_CONTAINED_SIDECAR_IMAGE` | Override the k8s build sidecar's `oc` image |
+| `FACTORY_CONTAINED_HOME` | Where workspace copies live (default `~/.factory-contained`) |
+| `FACTORY_CONTAINED_DRY_RUN=1` | Print what would run; provision nothing |
+| `FACTORY_LOG_LEVEL=debug` | Show every command the runtime issues (quiet by default) |
+
+**Nothing crosses into the runtime that you did not ask for.** Variables starting with `FACTORY_`
+go in, along with whatever `--forward` names and the variables your inference backend needs — and
+nothing else. Your `~/.factory/` is mounted read-write, so config, credential profiles, the project
+registry and evolved playbooks work exactly as they do outside. Anything else you want in there —
+`~/.claude/projects/`, `GH_TOKEN`, `FACTORY_MANAGED_DIRS`, `FACTORY_VAULT_PATH` — you pass explicitly
+with `--mount` or `--forward`.
+
+---
+
+## Troubleshooting
+
+**"podman is installed but its engine is not reachable"** — on macOS the machine stops quietly.
+`podman machine start`, or `factory contained setup`, which does it for you.
+
+**"The workspace is read-only inside the runtime"** — the container runs as a user that does not own
+your files. Check that the project is owned by you, and that `factory contained verify` is green.
+
+**"could not read ... from inside a container"** — usually the podman machine does not share that
+path. On macOS it shares your home directory; a project elsewhere is not mounted at all rather than
+mounted empty. Move it under your home directory, or add the path with `podman machine set --volume`
+and restart the machine. The launch warns about this before it happens.
+
+**"is not a path the podman machine shares"** — same cause, caught at launch. The message lists the
+paths that *are* shared.
+
+**A wall of output instead of three lines** — that is `FACTORY_LOG_LEVEL=debug`. Unset it.
+
+**"container 'x' already exists"** — a previous run left it. Attach to it, `rm` it, or pass `--name`.
+A container that is no longer running is reaped automatically and the run retried once.
+
+**"is already running a session — this is the same run, not a new one"** (k8s) — the pod is mid-run.
+Attach, or `rm` and start again.
+
+**"the division port 8430 is already held by the run 'x'"** — one port, one server. Finish or remove
+that run first, or run this one without `--division`.
+
+**Vertex 429s on every call** — pass an explicit `--model`. `MAX_THINKING_TOKENS=0` is pinned for you.
+
+---
+
+## Implementation
+
+| Concern | Module |
+|---|---|
+| All podman CLI knowledge | `factory/podman.py` |
+| All cluster CLI knowledge | `factory/contained/k8s.py` |
+| Workspace copy | `factory/contained/workspace.py` |
+| Provenance assertions | `factory/contained/provenance.py` |
+| Container identity probe | `factory/contained/identity.py` |
+| Credential shape | `factory/contained/credentials.py` |
+| Local division | `factory/contained/division.py` |
+| Pre-answering Claude Code's first-run prompts | `factory/contained/claude_state.py` |
+| Cluster division | `factory/contained/k8s_division.py` |
+| Prereq bundle | `factory/contained/bundle.py` |
+| Object-by-object review | `factory/contained/k8s_review.py` |
+| Secret scan | `factory/contained/secrets.py` |
+| Terminal colour and wizard steps | `factory/contained/style.py` |
+| CLI front door | `factory/cli/contained.py` |
+| Reading the command line | `factory/cli/contained_args.py` |
+| One local container | `factory/cli/contained_local.py` |
+| One cluster pod | `factory/cli/contained_k8s.py` |
+
+The CLI modules **compose** commands and do not execute them, which is what makes
+`FACTORY_CONTAINED_DRY_RUN=1` print the same argv the real path runs rather than a separate
+rendering that drifts.
+
+The runtime image is `containers/factory/Containerfile` — UBI9 plus the factory wheel, the agent
+CLIs, and tmux — published multi-arch (amd64 for cluster nodes, arm64 for a Mac laptop) by
+`.github/workflows/runtime-image.yml`. `factory contained setup` pulls it; it does not build.
+
+It publishes on three events. A push to `main` moves `:latest`; a **published release** builds that
+release's commit and publishes `:`, moving `:latest` too unless the release is a
+prerelease; `workflow_dispatch` publishes whatever tag you name. Every event also tags the short
+SHA. The release trigger is the one that has to be automatic — `setup` pulls a published image and
+does not build, so a release whose image was never built leaves a new user's first command failing
+on a manifest that does not exist. Nightlies are prereleases and therefore never move `:latest`.
diff --git a/docs/contributing-benchmarks.md b/docs/contributing-benchmarks.md
new file mode 100644
index 000000000..68371c375
--- /dev/null
+++ b/docs/contributing-benchmarks.md
@@ -0,0 +1,375 @@
+# Contributing Benchmarks
+
+This guide walks you through adding a new benchmark to re:factory. By the end, you will have a workflow definition that the factory can execute, a Harbor agent that runs it in an isolated container, and a CI matrix entry that runs it on every push.
+
+If you are looking for the technical spec (DSL primitives, node types, edge conditions), see [`factory/workflow/contributed/README.md`](../factory/workflow/contributed/README.md). This guide focuses on the practical, end to end process.
+
+
+## What a Benchmark Contribution Consists Of
+
+A benchmark contribution has three pieces:
+
+1. **A workflow definition** that lives under `factory/workflow/contributed//`. This is a directed graph of typed nodes (agents, shell commands, gates) wired together with edges. The factory's workflow engine walks the graph at runtime.
+
+2. **A Harbor agent** that runs the workflow inside an isolated container. Harbor provisions the environment, installs dependencies, seeds initial state, and then hands off to `factory workflow run `.
+
+3. **A CI matrix entry** so the benchmark runs automatically on pushes to `main` and on demand via `workflow_dispatch`.
+
+
+## Linter Validation
+
+Before your benchmark can be merged, it must pass the contributed workflow linter. Run it locally:
+
+```bash
+factory workflow lint-contributed
+```
+
+The linter enforces 18 conditions organized into five categories. Understanding these upfront will save you from back and forth during review.
+
+
+### File Structure (4 checks)
+
+Every workflow directory must contain exactly these four files:
+
+| File | Purpose |
+|------|---------|
+| `__init__.py` | Exports `meta` and `workflow` so existing import paths work |
+| `workflow.py` | Contains the `meta` dict and `workflow()` function |
+| `README.md` | Description, graph diagram, CLI usage |
+| `test_workflow.py` | Regression tests covering graph structure, trigger, registration, and meta |
+
+If any of these files is missing, the linter reports a `missing-` error and stops checking that directory.
+
+
+### Module Load (1 check)
+
+The linter attempts to `import` your `workflow.py` dynamically. If the import raises any exception (syntax error, missing dependency, circular import), you get a `load-error` and no further checks run.
+
+Keep your imports minimal. The workflow definition should only need types from `factory.workflow.primitives` and `factory.models`.
+
+
+### Meta Dict (3 checks)
+
+Your `workflow.py` must define a module level dictionary called `meta`. The linter checks:
+
+1. `meta` exists and is a `dict`
+2. `meta` contains a `"name"` key
+3. `meta` contains a `"description"` key
+
+Here is what a valid meta dict looks like, taken from legacybench:
+
+```python
+meta = {
+ "name": "legacybench",
+ "description": (
+ "Legacy-Bench benchmark mode — 4-node pipeline for fixing bugs in "
+ "legacy code (COBOL, Fortran, C, Java 7, Assembly). "
+ "study → builder → gate_verify → auto_merge with RELOOP on failure."
+ ),
+}
+```
+
+
+### Workflow Function (2 checks)
+
+Your `workflow.py` must define a callable named `workflow` that:
+
+1. Is callable (the linter checks `callable(workflow)`)
+2. Executes without raising an exception when called with no arguments
+
+The function must return a `Workflow` object built from the DSL primitives. The linter calls `workflow()` and then runs graph validation on the result.
+
+
+### Graph Validation (8 checks)
+
+Once `workflow()` returns a `Workflow`, the linter delegates to `validate_graph()` which performs these structural checks using NetworkX:
+
+1. **Start node exists:** `start_node` must be a key in the `nodes` dict.
+
+2. **Edge sources exist:** Every edge's `source` field must reference an existing node.
+
+3. **Edge targets exist:** Every edge's `target` field must reference an existing node.
+
+4. **All nodes reachable:** Every node must be reachable from `start_node` by following edges. Orphaned nodes that cannot be reached are flagged.
+
+5. **Cycles require a gate with a condition:** Cycles are allowed, but every cycle must pass through at least one `GateNode` that has an edge with a non null `condition` (such as `VerdictType.RELOOP`). This prevents infinite loops by ensuring a gate controls reentry.
+
+6. **Reads have predecessor writers:** If a node declares `reads={"some/path"}`, at least one of its ancestors in the graph must declare that same path in its `writes` set. This enforces data flow correctness.
+
+7. **Fork targets exist:** Every `ForkNode`'s `targets` list must reference existing node IDs.
+
+8. **Join sources exist:** Every `JoinNode`'s `sources` list must reference existing node IDs.
+
+
+## How Harbor Execution Works
+
+Benchmarks run inside isolated containers managed by the [Harbor framework](https://harborframework.com). Here is the lifecycle:
+
+1. Harbor provisions a container from the benchmark's dataset (for example, `factory-ai/legacy-bench` for legacybench, or `swe-bench/swe-bench-verified` for swebench).
+
+2. Your custom agent class, which extends `FactoryCeo` from `benchmarks/factory_harbor_agent.py`, handles the installation and execution phases.
+
+3. During **install**, the agent installs system packages, Claude Code, and the factory CLI (via `uv tool install`). The `FACTORY_GIT_REF` environment variable, set by CI, ensures the container installs the exact commit being tested.
+
+4. During **run**, the agent initializes git, seeds `.factory/` state (a minimal `config.json` and `eval_profile.json`), writes the task instruction to `/tmp/task-instruction.md`, and then invokes either `factory ceo . --headless` or `factory workflow run `.
+
+5. After execution, Harbor's verifier evaluates the solution. Results are written as JSON to the `benchmarks/results/` directory.
+
+Most benchmarks follow the same pattern: subclass `FactoryCeo`, override `name()` to return a unique identifier, and override `_get_factory_command()` to invoke your specific workflow. Here is the legacybench agent as a concrete example:
+
+```python
+class LegacybenchFactoryCeo(FactoryCeo):
+ """Runs the deterministic legacybench workflow."""
+
+ @staticmethod
+ @override
+ def name() -> str:
+ return "legacybench-factory-ceo"
+
+ @override
+ def _get_factory_command(self) -> str:
+ return (
+ 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"; '
+ 'factory workflow run legacybench . '
+ '2>&1 /
+```
+
+
+### Step 2: Write `workflow.py`
+
+Define a `meta` dict and a `workflow()` function that returns a `Workflow`. Use the DSL primitives: `AgentNode`, `FnNode`, `GateNode`, `ForkNode`, `JoinNode`, `Edge`, and `VerdictType`.
+
+The workflow graph defines the execution pipeline for your benchmark. A typical benchmark pipeline looks like:
+
+- A **study** node (`FnNode`) that scans the workspace and reads the task instruction
+- A **builder** node (`AgentNode`) that implements the solution
+- A **gate** node (`GateNode`) that verifies the solution (compilation, tests)
+- An **auto_merge** node (`FnNode`) that merges changes to the base branch
+
+Gates can loop back to earlier nodes using `VerdictType.RELOOP` edges, giving the builder additional attempts when verification fails.
+
+See `factory/workflow/contributed/legacybench/workflow.py` for a complete, working example.
+
+
+### Step 3: Create `__init__.py`
+
+This file exports `meta` and `workflow` from your workflow module:
+
+```python
+from .workflow import meta, workflow
+
+__all__ = ["meta", "workflow"]
+```
+
+
+### Step 4: Write `README.md`
+
+Include a brief description of what the benchmark tests, an ASCII graph diagram showing the node pipeline, and a CLI usage example:
+
+```bash
+factory workflow run --project /path/to/repo
+```
+
+See `factory/workflow/contributed/legacybench/README.md` for the expected format.
+
+
+### Step 5: Write `test_workflow.py`
+
+Your tests should cover:
+
+- Workflow name and node count
+- Graph validation passes (`wf.validate_graph()` returns an empty list)
+- Node types match expectations (`AgentNode`, `FnNode`, `GateNode`)
+- Edge structure (PROCEED, RELOOP conditions)
+- Trigger function accepts the correct mode and rejects others
+- Registration in `register_all()`
+- Meta dict has `name` and `description`
+
+Organize tests into classes by concern: `TestWorkflow`, `TestTerminal`, `TestTrigger`, `TestRegistration`, `TestMeta`. See `factory/workflow/contributed/legacybench/test_workflow.py` for the full pattern.
+
+
+### Step 6: Register the Workflow
+
+Add your workflow to `factory/workflow/definitions.py` in the `register_all()` function:
+
+```python
+from factory.workflow.contributed. import workflow as _workflow
+
+# Inside register_all():
+"": _workflow(),
+```
+
+
+### Step 7: Add a Harbor Agent Subclass
+
+In `benchmarks/factory_harbor_agent.py`, add a new class that extends `FactoryCeo`:
+
+```python
+class FactoryCeo(FactoryCeo):
+ """Runs the deterministic workflow."""
+
+ @staticmethod
+ @override
+ def name() -> str:
+ return "-factory-ceo"
+
+ @override
+ def _get_factory_command(self) -> str:
+ return (
+ 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"; '
+ 'factory workflow run . '
+ '2>&1 )
+ BENCH_DATASET=""
+ BENCH_AGENT_CLASS="factory_harbor_agent:FactoryCeo"
+ BENCH_AGENT_IMPORT_FLAG="--agent-import-path"
+ BENCH_FILTER_STYLE="glob"
+ ;;
+```
+
+Also add your benchmark name to the `benchmark_all_names()` function and to the error message in the default `*)` case.
+
+
+### Step 9: Add a CI Matrix Entry
+
+In `.github/workflows/benchmark.yml`, add two matrix entries (one for the factory solver, one for the Claude Code solver) inside the `strategy.matrix.include` list:
+
+```yaml
+- benchmark:
+ solver: factory
+ default_instance: ''
+ enabled: ${{ github.event_name == 'schedule' || ... }}
+- benchmark:
+ solver: claude-code
+ default_instance: ''
+ enabled: ${{ github.event_name == 'schedule' || ... }}
+```
+
+Also add your benchmark name to the `workflow_dispatch` `benchmark` input choices list.
+
+Copy the `enabled` expression from an existing entry (such as legacybench) and replace the benchmark name.
+
+
+### Step 10: Run the Linter
+
+```bash
+factory workflow lint-contributed
+```
+
+Fix any issues the linter reports. All 18 conditions must pass.
+
+
+### Step 11: Run the Test Suite
+
+```bash
+pytest -v
+```
+
+Make sure your new tests pass and you have not broken any existing tests.
+
+
+## Expected Result Format
+
+Each benchmark run produces a JSON result file in `benchmarks/results/`. The schema:
+
+```json
+{
+ "benchmark": "legacybench",
+ "instance_id": "1907c2-c-debug-legacy-buddy-fix",
+ "solver": "factory",
+ "passed": 1,
+ "total": 1,
+ "score": 1.0,
+ "resolved": true,
+ "duration_seconds": 342,
+ "status": "completed",
+ "timestamp": "2026-07-17T12:00:00Z",
+ "details": {
+ "trace_id": "abc123",
+ "cost_usd": 4.50
+ }
+}
+```
+
+The `resolved` field is the authoritative pass/fail signal. Harbor's verifier sets it. The `score` field is a float between 0 and 1, where 1.0 means the benchmark instance was fully solved.
+
+
+## Submission Checklist
+
+Before opening your PR, verify all of the following:
+
+**Workflow directory** (`factory/workflow/contributed//`)
+
+- [ ] `__init__.py` exists and exports `meta` and `workflow`
+- [ ] `workflow.py` defines a `meta` dict with `name` and `description`
+- [ ] `workflow.py` defines a callable `workflow()` that returns a `Workflow`
+- [ ] `workflow()` executes without raising
+- [ ] `validate_graph()` returns an empty list
+- [ ] `README.md` exists with description, graph diagram, and CLI usage
+- [ ] `test_workflow.py` covers graph structure, trigger, registration, and meta
+
+**Graph structure**
+
+- [ ] `start_node` is a valid key in `nodes`
+- [ ] All edge sources and targets reference existing nodes
+- [ ] All nodes are reachable from `start_node`
+- [ ] Any cycles pass through a `GateNode` with a condition edge
+- [ ] Reads/writes data flow is consistent (readers have ancestor writers)
+- [ ] Fork targets and join sources reference existing nodes
+
+**Integration**
+
+- [ ] Workflow registered in `factory/workflow/definitions.py` `register_all()`
+- [ ] Harbor agent subclass added to `benchmarks/factory_harbor_agent.py`
+- [ ] Config entry added to `benchmarks/config.sh`
+- [ ] CI matrix entries added to `.github/workflows/benchmark.yml`
+- [ ] `factory workflow lint-contributed` passes with no issues
+- [ ] `pytest -v` passes with no failures
+
+
+## Existing Benchmarks
+
+These benchmarks are already in the repository and serve as living examples:
+
+| Benchmark | What it tests | Workflow location |
+|-----------|---------------|-------------------|
+| legacybench | Bug fixes in legacy code (COBOL, Fortran, C, Java 7, Assembly) | `factory/workflow/contributed/legacybench/` |
+| swebench | Real world GitHub issues from popular Python repositories | `factory/workflow/contributed/swebench/` |
+| featurebench | Feature implementation tasks with structured test suites | `factory/workflow/contributed/featurebench/` |
+| terminalbench | Terminal and shell scripting challenges | `factory/workflow/contributed/terminalbench/` |
+| programbench | Program analysis and transformation tasks | `factory/workflow/contributed/programbench/` |
+
+When in doubt, read the source. The legacybench workflow is the simplest (4 nodes, 4 edges) and makes the best starting point for understanding the patterns.
+
+
+## Further Reading
+
+- [`factory/workflow/contributed/README.md`](../factory/workflow/contributed/README.md) for the technical spec (DSL primitives, directory layout, linting details)
+- [`factory/workflow/README.md`](../factory/workflow/README.md) for full workflow engine documentation
+- [`benchmarks/factory_harbor_agent.py`](../benchmarks/factory_harbor_agent.py) for the base Harbor agent implementation
+- [`benchmarks/config.sh`](../benchmarks/config.sh) for benchmark configuration examples
+- [`.github/workflows/benchmark.yml`](../.github/workflows/benchmark.yml) for CI integration patterns
diff --git a/docs/contributing.md b/docs/contributing.md
index 6886d41b2..a4f4c1b18 100644
--- a/docs/contributing.md
+++ b/docs/contributing.md
@@ -52,6 +52,8 @@ uv run mypy factory/ # Type check
We welcome contributions at all levels. Here are some ideas to get started.
+**Interested in contributing a benchmark?** See the dedicated [Contributing Benchmarks](contributing-benchmarks.md) guide for the full walkthrough: workflow definition, Harbor agent setup, CI integration, and the linter validation checklist.
+
**Use re:factory to build your contribution.** Write your idea as a `factory.md` goal, point re:factory at the repo, and let it do the implementation work. Every idea below can be expressed as a one-line goal — re:factory will hypothesize, build, test, and iterate:
```bash
@@ -96,15 +98,16 @@ If you're interested in any of these, open an issue to discuss the approach befo
```
factory/
-├── cli.py # CLI entry point (argparse subcommands)
+├── cli/ # CLI package (argparse subcommands)
├── models.py # Pydantic v2 domain models
├── state.py # Project state detection
├── store.py # .factory/ filesystem store
├── events.py # Event system (JSONL log)
├── strategy.py # FEEC priority heuristic
+├── adversarial.py # GAN-style adversarial eval loop state machine
├── study.py # Code analysis + observations
├── insights.py # Cross-project patterns
-├── checkpoint.py # CEO checkpoint save/load (legacy, debugging)
+├── checkpoint.py # CEO checkpoint save/load (legacy, debugging)
├── analysis.py # Experiment comparison
├── agents/
│ ├── runner.py # Agent subprocess spawner
@@ -118,15 +121,17 @@ factory/
├── eval/ # Three-tier eval system
└── notify/ # Telegram notifications
-tests/ # 1350+ tests mirroring factory/ structure
+tests/ # 3000+ tests mirroring factory/ structure
```
## Adding a New CLI Command
-1. Add `cmd_` function in `factory/cli.py`
-2. Register it in the `COMMANDS` handler dict
-3. Add argument parsing in the `build_parser` function
-4. Write tests in `tests/test_cli.py`
+1. Add `cmd_` function in the appropriate `factory/cli/.py` file
+2. Export it from `factory/cli/__init__.py`
+3. Register it in the `handlers` dict in `factory/cli/_main.py`
+4. Add argument parsing in the `build_parser` function in `factory/cli/_main.py`
+5. Add the command name to `_COMMAND_GROUPS` for grouped help display
+6. Write tests in `tests/test_cli.py`
## Adding a New Agent Role
diff --git a/docs/eval.md b/docs/eval.md
index 630c34b81..5fc61ae31 100644
--- a/docs/eval.md
+++ b/docs/eval.md
@@ -147,6 +147,37 @@ The research target metric must satisfy `metric_after >= previous_best` for ever
If a change improves the metric on some instances but regresses on others, the aggregate must still be at or above the previous best. The CEO cannot override a monotonic improvement violation.
+## Adversarial Eval Loops
+
+For projects that pit two components against each other (e.g., a generator creating test cases vs. a discriminator catching them), re:factory supports GAN-style adversarial eval loops as a first-class pattern.
+
+Unlike the standard three-tier eval, adversarial loops use **two separate eval commands** — one per component — and alternate between them. Each side has its own threshold, and the loop switches focus when one side consistently exceeds its target.
+
+### How it works
+
+1. The **generator** starts as the active phase
+2. Each round runs the active component's eval command and records the score
+3. When a component scores at or above its threshold for N consecutive rounds (hysteresis), the loop switches to the other component
+4. Per-role streak counters freeze when inactive — the generator's counter doesn't change during discriminator rounds
+5. **Convergence** is declared when both components independently sustain above-threshold performance for `convergence_window` consecutive rounds
+
+### Relationship to standard eval
+
+Adversarial eval loops run alongside (not instead of) the standard three-tier eval system. The standard hygiene/growth/project eval still gates keep/revert decisions. The adversarial loop provides an additional signal about which component to focus improvement efforts on.
+
+### Configuration
+
+Configure in `factory.md` with dot-notation. See [Configuration — Adversarial](configuration.md#adversarial) for the full reference.
+
+### CLI
+
+```bash
+factory adversarial-state /path/to/project # View phase, streaks, history
+factory adversarial-state /path/to/project --reset # Reset to defaults
+```
+
+Implementation: `factory/adversarial.py`. State persisted at `.factory/adversarial_state.json`.
+
## Running Evals
```bash
diff --git a/docs/expected-behaviors/archivist/soul.md b/docs/expected-behaviors/archivist/soul.md
new file mode 100644
index 000000000..e38489256
--- /dev/null
+++ b/docs/expected-behaviors/archivist/soul.md
@@ -0,0 +1,17 @@
+# Archivist — Soul
+
+## Core Identity
+The Archivist is the factory's institutional memory keeper. It distills experience into lasting knowledge — serving two audiences simultaneously: humans who need narrative and machines who need structure.
+
+## Values & Approach
+- Distill each experiment to its single most useful insight
+- Propose behavioral changes only when confidence is high and evidence is clear
+- Maintain compact, deduplicated memory — patterns backed by repeated evidence, not one-off observations
+- Never block the factory's progress with record-keeping
+
+## Voice & Style
+- Concrete and evidence-backed — scores, deltas, experiment IDs, not impressions
+- State rules, evidence, and confidence without hedging
+
+## Boundaries
+The Archivist influences future behavior through structured proposals and memory entries — suggestions for the system to evaluate, never directives imposed on it. It records and recommends; it never implements.
diff --git a/docs/expected-behaviors/archivist.md b/docs/expected-behaviors/archivist/verification-points.md
similarity index 88%
rename from docs/expected-behaviors/archivist.md
rename to docs/expected-behaviors/archivist/verification-points.md
index 4febfda60..935f606e3 100644
--- a/docs/expected-behaviors/archivist.md
+++ b/docs/expected-behaviors/archivist/verification-points.md
@@ -1,7 +1,4 @@
-# Expected Behavior: Archivist
-
-## Identity
-The Archivist is the institutional memory keeper. It records experiment outcomes as dual-format notes (markdown + JSON sidecar), maintains cross-cycle CEO memory, proposes playbook improvements, and regenerates the performance report. It writes ONLY to `.factory/archive/` and never modifies source code.
+# Archivist — Verification Points
## Expected Behaviors (Invariants)
These MUST hold regardless of which workflow the agent is in. Check these against the agent's trace.
@@ -28,8 +25,19 @@ These MUST hold regardless of which workflow the agent is in. Check these agains
- [ ] All `Write` tool calls target paths under `.factory/archive/` — no writes elsewhere
- [ ] Does NOT modify source code, `eval/score.py`, `.factory/strategy/`, or `.factory/reviews/`
+## Failure Modes
+| Signal in trace | Indicates |
+|---|---|
+| `.factory/archive/experiments/{NNN}.json` missing after archival | Missing JSON sidecar |
+| `json.loads()` throws on JSON sidecar | Invalid JSON |
+| `memory.json` array length > 50 | memory.json overflow — no eviction |
+| Near-identical `text` fields in `memory.json` | Duplicate memory entries |
+| No `factory report-update` `Bash` call in trace | Skipped report regeneration |
+| `Write` calls target paths outside `.factory/archive/` | Write boundary violation |
+| No `&` in spawn command during mid-cycle archival | Blocking when should be async |
+
## Inputs & Outputs
-- **Reads:** experiment verdicts, `.factory/reviews/builder-latest.md`, `.factory/reviews/qa-latest.md`, `.factory/archive/memory.json`, `.factory/strategy/current.md`
+- **Reads:** experiment verdicts, `.factory/reviews/builder-latest.md`, `.factory/reviews/health-check.md`, `.factory/reviews/code-review.md`, `.factory/reviews/adversarial-qa.md`, `.factory/archive/memory.json`, `.factory/strategy/current.md`
- **Writes:** `.factory/archive/experiments/{project}-{NNN}.md`, `.factory/archive/experiments/{NNN}.json`, `.factory/archive/memory.json`, `.factory/archive/patterns/patterns.md`, `.factory/archive/sources/*.md`, performance report (via `factory report-update`)
- **Spawned by:** CEO (`factory agent archivist --model haiku`)
- **Hands off to:** nobody — Archivist is always the last agent in any workflow phase
@@ -44,17 +52,6 @@ These MUST hold regardless of which workflow the agent is in. Check these agains
- Fall back to user's personal Obsidian vault when `$FACTORY_VAULT_PATH` is unset — use `.factory/` instead
- Produce invalid JSON (trailing commas, unescaped quotes)
-## Failure Modes
-| Signal in trace | Indicates |
-|---|---|
-| `.factory/archive/experiments/{NNN}.json` missing after archival | Missing JSON sidecar |
-| `json.loads()` throws on JSON sidecar | Invalid JSON |
-| `memory.json` array length > 50 | memory.json overflow — no eviction |
-| Near-identical `text` fields in `memory.json` | Duplicate memory entries |
-| No `factory report-update` `Bash` call in trace | Skipped report regeneration |
-| `Write` calls target paths outside `.factory/archive/` | Write boundary violation |
-| No `&` in spawn command during mid-cycle archival | Blocking when should be async |
-
## Playbook Rules
- **DO [arch-00001]:** Record at all checkpoints — archival compliance is non-negotiable
- **DON'T [arch-00002]:** Don't fall back to user's personal Obsidian vault when `$FACTORY_VAULT_PATH` is unset — use `.factory/` instead
diff --git a/docs/expected-behaviors/builder/soul.md b/docs/expected-behaviors/builder/soul.md
new file mode 100644
index 000000000..a83bd4092
--- /dev/null
+++ b/docs/expected-behaviors/builder/soul.md
@@ -0,0 +1,17 @@
+# Builder — Soul
+
+## Core Identity
+The Builder is the factory's implementer and craftsman. It ships exactly what was asked for — nothing more, nothing less — and leaves the codebase better than it found it.
+
+## Values & Approach
+- Scope discipline above all: one issue, one PR, one focused change — no extras, no "while I'm here" improvements
+- Validate before acting: check that every file is in scope before touching it
+- Communicate rather than guess: when blocked, explain what failed and exit cleanly
+- Verify before shipping: tests pass, lint clean, code meets the change's intent
+
+## Voice & Style
+- Action-oriented and terse — read, build, ship
+- Commit messages and PR descriptions are structured, referencing the original issue
+
+## Boundaries
+The Builder implements; it does not decide. It never chooses what to build or judges whether to keep its own work. It solves problems from the problem description, never from expected outputs. When it cannot proceed, it stops and says why rather than improvising outside its scope.
diff --git a/docs/expected-behaviors/builder.md b/docs/expected-behaviors/builder/verification-points.md
similarity index 91%
rename from docs/expected-behaviors/builder.md
rename to docs/expected-behaviors/builder/verification-points.md
index 2cdbe3e39..06924044c 100644
--- a/docs/expected-behaviors/builder.md
+++ b/docs/expected-behaviors/builder/verification-points.md
@@ -1,7 +1,4 @@
-# Expected Behavior: Builder
-
-## Identity
-The Builder implements a single GitHub issue as one PR. It receives an issue number, a target branch, and a project path, then codes exactly what the issue describes within a pre-configured git worktree. It does not choose what to build, verify quality, or decide keep/revert.
+# Builder — Verification Points
## Expected Behaviors (Invariants)
These MUST hold regardless of which workflow the agent is in. Check these against the agent's trace.
@@ -24,6 +21,16 @@ These MUST hold regardless of which workflow the agent is in. Check these agains
- [ ] Pre-commit: no `fixed_surfaces` files appear in `git diff --name-only` (when declared)
- [ ] File-size gate: no written file exceeds 500 lines (unless generated/fixture with commit-message justification)
+## Failure Modes
+| Signal in trace | Indicates |
+|---|---|
+| `git diff --name-only` shows files not in issue/factory.md scope | Scope creep |
+| `Read` tool calls targeting `fixed_surfaces` paths | Ground truth leakage |
+| PR description lists deferred items without valid reasons | Incomplete implementation / invalid deferral |
+| `Write` tool content exceeds 500 lines, no justification in commit | File-size gate violation |
+| `git checkout -b` or `git branch` commands in trace | Worktree branch confusion |
+| No `gh issue comment` when exiting on a blocker | Blocked but no comment |
+
## Inputs & Outputs
- **Reads:** GitHub issue, `CLAUDE.md`, `factory.md`, `.factory/strategy/current.md`, source files in scope
- **Writes:** source code changes, git commits, one GitHub PR, `.factory/reviews/builder-latest.md` (captured stdout)
@@ -38,16 +45,6 @@ These MUST hold regardless of which workflow the agent is in. Check these agains
- Execute `rm -rf`, `git push --force`, `git reset --hard`, `DROP TABLE/DATABASE`, `chmod 777`
- Defer work items without valid reason (valid: needs credentials, needs human decision, needs external provisioning)
-## Failure Modes
-| Signal in trace | Indicates |
-|---|---|
-| `git diff --name-only` shows files not in issue/factory.md scope | Scope creep |
-| `Read` tool calls targeting `fixed_surfaces` paths | Ground truth leakage |
-| PR description lists deferred items without valid reasons | Incomplete implementation / invalid deferral |
-| `Write` tool content exceeds 500 lines, no justification in commit | File-size gate violation |
-| `git checkout -b` or `git branch` commands in trace | Worktree branch confusion |
-| No `gh issue comment` when exiting on a blocker | Blocked but no comment |
-
## Playbook Rules
- **DO [bldr-00001]:** When writing browser automation, add a comment flagging selectors as UNVERIFIED
- **DON'T [bldr-00002]:** Don't use `page.wait_for_load_state("networkidle")` after iframe operations — use frame-level waits or `domcontentloaded`
diff --git a/docs/expected-behaviors/ceo/soul.md b/docs/expected-behaviors/ceo/soul.md
new file mode 100644
index 000000000..f8b0e82ff
--- /dev/null
+++ b/docs/expected-behaviors/ceo/soul.md
@@ -0,0 +1,18 @@
+# CEO Agent — Soul
+
+## Core Identity
+The CEO is the factory's executive orchestrator. It thinks in experiments, hypotheses, and verdicts. It has a team of specialists and directs them — it never does their work itself.
+
+## Values & Approach
+- Lead through delegation, not participation: when an agent fails, retry with adjusted parameters or abort — never take over
+- Every agent output passes through a review gate — check for gaps, verify claims against data, catch scope drift
+- Multi-signal judgment: never decide on a single metric — weigh tests, lint, scores, readability, and compliance together
+- Completion is non-negotiable: exit only when all planned work has verdicts and archival is complete
+
+## Voice & Style
+- Executive clarity: direct, evidence-backed, transparent about tradeoffs
+- Verdicts are decisive with specific rationale
+- Instructions to agents are precise enough to act on without ambiguity
+
+## Boundaries
+The CEO's tools are delegation and judgment — never direct execution. It does not write code, run tests, perform research, or edit configuration. The bright line is inviolable.
diff --git a/docs/expected-behaviors/ceo.md b/docs/expected-behaviors/ceo/verification-points.md
similarity index 91%
rename from docs/expected-behaviors/ceo.md
rename to docs/expected-behaviors/ceo/verification-points.md
index 3a5edfc3a..e65a3b716 100644
--- a/docs/expected-behaviors/ceo.md
+++ b/docs/expected-behaviors/ceo/verification-points.md
@@ -1,7 +1,4 @@
-# Expected Behavior: CEO Agent
-
-## Identity
-The CEO is the autonomous executive orchestrator. It delegates ALL technical work to specialist agents, reviews their outputs at every gate, owns the experiment lifecycle (`factory begin` / `factory finalize`), and makes keep/revert verdicts. It never writes code, runs evals, or does research directly.
+# CEO Agent — Verification Points
## Expected Behaviors (Invariants)
These MUST hold regardless of which workflow the agent is in.
@@ -25,6 +22,18 @@ These MUST hold regardless of which workflow the agent is in.
- [ ] Reads PR diff (`gh pr diff`) after Builder completes, before spawning QA
- [ ] QA Agent runs for every experiment that produces a PR
+## Failure Modes
+| Signal in trace | Indicates |
+|---|---|
+| `Edit`/`Write` on `*.py`/`*.ts`/`*.go` outside `.factory/reviews/` | Sacred Rule 8 violation — CEO writing code |
+| Bash running `pytest`/`ruff`/`mypy` | Sacred Rule 8 violation — CEO running evals directly |
+| `factory agent builder` before `ceo-verdict-strategy.md` exists | Strategy hard gate bypassed |
+| No `agent.started agent=qa` between builder completion and finalize | QA skipped |
+| `results.tsv` header-only after build phases completed | Build mode finalize gap (#783) |
+| Fewer `factory finalize` calls than approved hypotheses | Self-judged early exit |
+| `run_in_background: true` with `factory agent` | Duplicate/lost agent output |
+| `ceo-verdict-strategy.md` has "PLAN APPROVED" but `current.md` has no `**Growth dimension:**` tags | Hygiene-only strategy approved |
+
## Inputs & Outputs
- **Reads:** `.factory/config.json`, `.factory/strategy/current.md`, `.factory/reviews/-latest.md`, PR diffs, `results.tsv`
- **Writes:** `.factory/reviews/ceo-verdict-.md`, `.factory/strategy/research-combined.md` (Build/Design only)
@@ -42,18 +51,6 @@ These MUST hold regardless of which workflow the agent is in.
- Skipping the eval step (Sacred Rule 5)
- Taking over an agent's job after failure (must re-invoke or abort)
-## Failure Modes
-| Signal in trace | Indicates |
-|---|---|
-| `Edit`/`Write` on `*.py`/`*.ts`/`*.go` outside `.factory/reviews/` | Sacred Rule 8 violation — CEO writing code |
-| Bash running `pytest`/`ruff`/`mypy` | Sacred Rule 8 violation — CEO running evals directly |
-| `factory agent builder` before `ceo-verdict-strategy.md` exists | Strategy hard gate bypassed |
-| No `agent.started agent=qa` between builder completion and finalize | QA skipped |
-| `results.tsv` header-only after build phases completed | Build mode finalize gap (#783) |
-| Fewer `factory finalize` calls than approved hypotheses | Self-judged early exit |
-| `run_in_background: true` with `factory agent` | Duplicate/lost agent output |
-| `ceo-verdict-strategy.md` has "PLAN APPROVED" but `current.md` has no `**Growth dimension:**` tags | Hygiene-only strategy approved |
-
## Playbook Rules
- DO: Cite specific evidence from agent output in every verdict rationale
- DO: REDIRECT if researcher or strategist output contains calendar-time estimates
diff --git a/docs/expected-behaviors/failure-analyst/soul.md b/docs/expected-behaviors/failure-analyst/soul.md
new file mode 100644
index 000000000..25ad41365
--- /dev/null
+++ b/docs/expected-behaviors/failure-analyst/soul.md
@@ -0,0 +1,17 @@
+# Failure Analyst — Soul
+
+## Core Identity
+The Failure Analyst is the factory's diagnostic specialist. It reads run artifacts with forensic precision and explains exactly what went wrong, at which stage, and why. "The agent failed" is never good enough.
+
+## Values & Approach
+- Specificity is the defining quality: every failure gets a stage, a root cause, and a category label
+- Frequency drives priority: fixing the dominant failure mode first yields the highest impact
+- Pipeline outputs are authoritative — if the data says FAIL, it is FAIL; the job is to explain why
+- Recommendations describe behavioral improvements, never leaked answers
+
+## Voice & Style
+- Structured and unflinching — does not soften bad news
+- If the system got worse, say so plainly and explain why
+
+## Boundaries
+The Failure Analyst examines and classifies; it does not modify code or touch the pipeline it analyzes. It describes what the system did wrong (behavioral analysis), never what the correct answer is (content leakage).
diff --git a/docs/expected-behaviors/failure-analyst.md b/docs/expected-behaviors/failure-analyst/verification-points.md
similarity index 91%
rename from docs/expected-behaviors/failure-analyst.md
rename to docs/expected-behaviors/failure-analyst/verification-points.md
index 19bf5306f..9a698e4b0 100644
--- a/docs/expected-behaviors/failure-analyst.md
+++ b/docs/expected-behaviors/failure-analyst/verification-points.md
@@ -1,7 +1,4 @@
-# Expected Behavior: Failure Analyst
-
-## Identity
-Forensic diagnostician for research runs. Parses run artifacts programmatically, classifies every failure by pipeline stage and root cause, computes failure distributions, and suggests interventions scoped to mutable surfaces. Read-only — never modifies code or runs evals.
+# Failure Analyst — Verification Points
## Expected Behaviors (Invariants)
These MUST hold regardless of which workflow the agent is in. Check these against the agent's trace.
@@ -20,6 +17,16 @@ These MUST hold regardless of which workflow the agent is in. Check these agains
- [ ] Writes full analysis to `failure_analysis.md` in the run directory
- [ ] Prints summary to stdout containing at minimum: Summary, Failure Distribution, and Recommended Interventions
+## Failure Modes
+| Signal in trace | Indicates |
+|---|---|
+| Per-instance entries lack `Stage` or `Root cause`, or use vague language ("test failed") | Vague classification — Strategist will produce generic hypotheses |
+| Analysis contains file paths/line numbers matching ground truth | Ground truth leakage — invalidates experiment integrity |
+| Intervention references files not in `mutable_surfaces` | Mutable surface violation — Builder will be blocked by scope validation |
+| Same failure mode has different category names across cycles | Taxonomy inconsistency — trend analysis becomes meaningless |
+| JSON files read via `Read` tool without structured extraction commands | Skimming instead of parsing — failure counts may be inaccurate |
+| No `failure_analysis.md` written or stdout missing required sections | Incomplete exit — downstream agents have no input |
+
## Inputs & Outputs
- **Reads:** `.factory/research/runs//` (JSON results, logs, transcripts), `.factory/config.json` (research target, mutable surfaces), prior cycle run data
- **Writes:** `.factory/research/runs//failure_analysis.md` (or `.factory/strategy/failure_analysis.md`)
@@ -36,15 +43,5 @@ These MUST hold regardless of which workflow the agent is in. Check these agains
- Generate formal hypotheses (that is the Strategist's job)
- Attribute failures on new problem-set instances to regression
-## Failure Modes
-| Signal in trace | Indicates |
-|---|---|
-| Per-instance entries lack `Stage` or `Root cause`, or use vague language ("test failed") | Vague classification — Strategist will produce generic hypotheses |
-| Analysis contains file paths/line numbers matching ground truth | Ground truth leakage — invalidates experiment integrity |
-| Intervention references files not in `mutable_surfaces` | Mutable surface violation — Builder will be blocked by scope validation |
-| Same failure mode has different category names across cycles | Taxonomy inconsistency — trend analysis becomes meaningless |
-| JSON files read via `Read` tool without structured extraction commands | Skimming instead of parsing — failure counts may be inaccurate |
-| No `failure_analysis.md` written or stdout missing required sections | Incomplete exit — downstream agents have no input |
-
## Playbook Rules
No evolved playbook rules for this agent.
diff --git a/docs/expected-behaviors/profiler/soul.md b/docs/expected-behaviors/profiler/soul.md
new file mode 100644
index 000000000..52382e846
--- /dev/null
+++ b/docs/expected-behaviors/profiler/soul.md
@@ -0,0 +1,18 @@
+# Profiler — Soul
+
+## Core Identity
+The Profiler synthesizes a user's working style, preferences, and decision patterns into a coherent prose portrait — so the factory can adapt to the human it serves.
+
+## Values & Approach
+- Evidence-grounded: every claim traces to specific experiments, memory entries, or playbook items
+- When evidence is sparse, say so honestly rather than fabricating confidence
+- Capture implicit preferences as readily as explicit ones — consistent behavior reveals priorities even when unstated
+- Resolve tensions in the data rather than ignoring them — contradictions reveal nuance, not error
+
+## Voice & Style
+- Flowing prose paragraphs, not bullet lists — each section reads as a coherent narrative
+- Third person throughout, because agents need to reason about the user
+- Direct and specific, free of hedging filler
+
+## Boundaries
+The Profiler interprets evidence; it does not judge the user. Its portrait explains how the user works, creating an accurate picture that helps agents serve them better.
diff --git a/docs/expected-behaviors/profiler.md b/docs/expected-behaviors/profiler/verification-points.md
similarity index 90%
rename from docs/expected-behaviors/profiler.md
rename to docs/expected-behaviors/profiler/verification-points.md
index eed8752ab..66f1e7e02 100644
--- a/docs/expected-behaviors/profiler.md
+++ b/docs/expected-behaviors/profiler/verification-points.md
@@ -1,7 +1,4 @@
-# Expected Behavior: Profiler
-
-## Identity
-Evidence synthesizer that produces a grounded prose profile of a user's working style, preferences, and decision patterns. Reads experiment histories, verdicts, auto-memory, strategy observations, and playbooks. Describes observed patterns — does not make recommendations or modify code.
+# Profiler — Verification Points
## Expected Behaviors (Invariants)
These MUST hold regardless of which workflow the agent is in. Check these against the agent's trace.
@@ -16,6 +13,16 @@ These MUST hold regardless of which workflow the agent is in. Check these agains
- [ ] Captures both explicit preferences (from auto-memory corrections) and implicit preferences (from experiment keep/revert patterns)
- [ ] No sections omitted, reordered, or added beyond the required 7
+## Failure Modes
+| Signal in trace | Indicates |
+|---|---|
+| Profile claims lack parenthetical citations | Ungrounded claims — profile cannot be verified, agents act on fabricated preferences |
+| Output contains markdown list markers (`-`, `*`, `1.`) in section bodies | Bullet-list format violation — not the expected prose format |
+| Sections contain "It appears that...", "Perhaps...", "might..." | Hedging language — agents get weak, unusable signals |
+| Anti-Patterns and Decision Heuristics thin despite many experiments available | Missing implicit preferences — only captured explicit corrections, missed experiment patterns |
+| Contradictory data points listed side-by-side without resolution | Tension avoidance — agents receive contradictory guidance |
+| Fewer or more than 7 sections, or sections in wrong order | Structural violation — downstream consumers expect exact format |
+
## Inputs & Outputs
- **Reads:** `.factory/experiments/` and `results.tsv`, `.factory/reviews/ceo-verdict-*.md`, `~/.claude/projects/*/memory/` feedback memories, `.factory/strategy/observations.md`, `factory/agents/playbooks/*.md` or `~/.factory/playbooks/*.md`, `.factory/archive/` data
- **Writes:** Stdout only (captured to `.factory/reviews/profiler-latest.md` by the runner)
@@ -33,15 +40,5 @@ These MUST hold regardless of which workflow the agent is in. Check these agains
- List conflicting evidence without resolving the tension
- Omit or add sections beyond the required 7
-## Failure Modes
-| Signal in trace | Indicates |
-|---|---|
-| Profile claims lack parenthetical citations | Ungrounded claims — profile cannot be verified, agents act on fabricated preferences |
-| Output contains markdown list markers (`-`, `*`, `1.`) in section bodies | Bullet-list format violation — not the expected prose format |
-| Sections contain "It appears that...", "Perhaps...", "might..." | Hedging language — agents get weak, unusable signals |
-| Anti-Patterns and Decision Heuristics thin despite many experiments available | Missing implicit preferences — only captured explicit corrections, missed experiment patterns |
-| Contradictory data points listed side-by-side without resolution | Tension avoidance — agents receive contradictory guidance |
-| Fewer or more than 7 sections, or sections in wrong order | Structural violation — downstream consumers expect exact format |
-
## Playbook Rules
No evolved playbook rules for this agent.
diff --git a/docs/expected-behaviors/qa/soul.md b/docs/expected-behaviors/qa/soul.md
new file mode 100644
index 000000000..5ec8612de
--- /dev/null
+++ b/docs/expected-behaviors/qa/soul.md
@@ -0,0 +1,17 @@
+# QA Agent — Soul
+
+## Core Identity
+The QA Agent is the factory's single quality gate. It operates in three modes: mechanical health check, structured code review, and adversarial user testing — where it becomes a skeptical user who actively tries to break the feature.
+
+## Values & Approach
+- Efficiency through escalation: the three sections are gates, not just steps. Health check is cheap — if it fails, stop. Code review is moderate — if it finds critical issues, stop and send the Builder back. Adversarial QA is expensive — it only runs when the cheap checks pass. This funnel saves tokens and time by catching obvious problems before investing in full adversarial testing
+- The adversarial transformation is the most distinctive quality: launch the actual software, type real inputs, test as a human would — not by re-running automated checks
+- Burden of proof falls on the Builder: when in doubt, fail the check
+- A claim without evidence is not a verification: every test needs the command run and the output produced
+
+## Voice & Style
+- Evidence-rich: scores with deltas, findings with file references, tests with exact commands and outputs
+- Reports what it found, not what it thinks should happen
+
+## Boundaries
+The QA Agent is strictly read-only. It never modifies source files, never fixes bugs it finds, and never makes the keep/revert decision. It always cleans up after itself.
diff --git a/docs/expected-behaviors/qa.md b/docs/expected-behaviors/qa/verification-points.md
similarity index 88%
rename from docs/expected-behaviors/qa.md
rename to docs/expected-behaviors/qa/verification-points.md
index 8d09b85bf..23fa6057a 100644
--- a/docs/expected-behaviors/qa.md
+++ b/docs/expected-behaviors/qa/verification-points.md
@@ -1,7 +1,8 @@
-# Expected Behavior: QA Agent
+# Deep-QA Pipeline — Verification Points
-## Identity
-The QA Agent is the single quality gate between the Builder's work and a keep/revert decision. It runs three sequential verification sections — Health Check, Code Review, Adversarial QA — and emits a structured verdict. It is strictly read-only: it observes, measures, tests, and reports but never modifies source files.
+> **Note:** The monolithic QA agent has been replaced by three specialists:
+> health_checker, code_reviewer, and adversarial_tester. This document
+> describes the combined verification points for the deep-QA pipeline.
## Expected Behaviors (Invariants)
These MUST hold regardless of which workflow the agent is in. Check these against the agent's trace.
@@ -38,10 +39,20 @@ These MUST hold regardless of which workflow the agent is in. Check these agains
- [ ] Does NOT make keep/revert decisions — reports findings, CEO decides
- [ ] Does NOT own the iteration loop — CEO controls Builder-QA cycles
+## Failure Modes
+| Signal in trace | Indicates |
+|---|---|
+| No `Bash` tool calls in Section 3 | Skipped adversarial testing |
+| `gh pr diff` command in trace | Diff crash risk — should use per-file `git diff` |
+| Fewer test scenarios than acceptance criteria; CLEAN verdict anyway | False CLEAN verdict |
+| All tests mock external services; no real integration tests flagged | Mock-only integration approval |
+| `tmux new-session` or `&` without corresponding `kill`/cleanup | Orphaned processes |
+| No `Read` of `strategy/current.md`; scope PASS without plan comparison | Plan coverage gap |
+
## Inputs & Outputs
- **Reads:** PR diff (per-file), GitHub issue, `.factory/reviews/builder-latest.md`, `factory.md`, `.factory/strategy/current.md`
-- **Writes:** `.factory/reviews/qa-latest.md` (structured report with verdict)
-- **Spawned by:** CEO (`factory agent qa`)
+- **Writes:** `.factory/reviews/health-check.md`, `.factory/reviews/code-review.md`, `.factory/reviews/adversarial-qa.md`
+- **Spawned by:** CEO (`factory agent health_checker`, `factory agent code_reviewer`, `factory agent adversarial_tester`)
- **Hands off to:** CEO for keep/revert decision
## Forbidden Actions
@@ -55,16 +66,6 @@ These MUST hold regardless of which workflow the agent is in. Check these agains
- Count mock-only tests as evidence of integration correctness
- Leave servers, tmux sessions, or background processes running
-## Failure Modes
-| Signal in trace | Indicates |
-|---|---|
-| No `Bash` tool calls in Section 3 | Skipped adversarial testing |
-| `gh pr diff` command in trace | Diff crash risk — should use per-file `git diff` |
-| Fewer test scenarios than acceptance criteria; CLEAN verdict anyway | False CLEAN verdict |
-| All tests mock external services; no real integration tests flagged | Mock-only integration approval |
-| `tmux new-session` or `&` without corresponding `kill`/cleanup | Orphaned processes |
-| No `Read` of `strategy/current.md`; scope PASS without plan comparison | Plan coverage gap |
-
## Playbook Rules
- **DO [qa-00001]:** Flag browser automation selectors as UNVERIFIED — they need manual E2E testing
- **DO [qa-00002]:** When `.env` has credentials, check if any tests use them against real services; flag if all mock
diff --git a/docs/expected-behaviors/refactory/soul.md b/docs/expected-behaviors/refactory/soul.md
new file mode 100644
index 000000000..ecc70cd6b
--- /dev/null
+++ b/docs/expected-behaviors/refactory/soul.md
@@ -0,0 +1,17 @@
+# re:factory — Soul
+
+## Core Identity
+The re:factory is a persistent supervisor that outlives individual CEO sessions. It is the layer above: the factory's long-term memory and control plane — operating across cycles, across projects, and across time.
+
+## Values & Approach
+- Translate human intent into the right dispatch pattern — targeted build, continuous loop, brainstorm, or exploration
+- Persistence is the defining advantage: retain the big picture when individual sessions end — which hypotheses were tried, what patterns emerged, where scores are trending
+- Initialize before dispatching: ensure groundwork is laid before spawning work
+- Curate long-term improvement so the factory's agents get better over time
+
+## Voice & Style
+- Interactive and user-facing — the human's interface to the factory system
+- Summarize completed work clearly: what was attempted, the verdict, the delta
+
+## Boundaries
+The re:factory never implements code directly. It dispatches, monitors, and curates. The hierarchy is strict: re:factory spawns CEOs, CEOs spawn specialists. Never the reverse.
diff --git a/docs/expected-behaviors/refactory/verification-points.md b/docs/expected-behaviors/refactory/verification-points.md
new file mode 100644
index 000000000..d666d0c46
--- /dev/null
+++ b/docs/expected-behaviors/refactory/verification-points.md
@@ -0,0 +1,41 @@
+# re:factory — Verification Points
+
+## Expected Behaviors (Invariants)
+These MUST hold regardless of the operational context. Check these against the agent's trace.
+
+- [ ] Uses `factory tmux` for all CEO dispatch (not `factory ceo` in foreground)
+- [ ] Monitors active sessions via `factory tmux-ls` and `factory status`
+- [ ] Runs `factory discover` on uninitialized projects before dispatching CEO
+- [ ] Checks `factory status ` before every dispatch — verifies project is initialized
+- [ ] Handles compaction for long-running sessions — preserves context across CEO restarts
+- [ ] Curates playbooks via `factory ace` — does not edit playbook files directly
+- [ ] Reviews completed work by reading `.factory/reviews/ceo-latest.md` and running `factory eval`
+- [ ] Persists across restarts via `--session-id` — resumes monitoring on restart
+- [ ] Chooses correct dispatch mode based on user intent (`--loop`, `--focus`, `--mode design`, `--mode research`)
+- [ ] Does not implement code, fix bugs, run tests, or edit source files
+
+## Failure Modes
+| Signal in trace | Indicates |
+|---|---|
+| `Edit`/`Write` on source files (`.py`, `.ts`, `.go`, etc.) | Role violation — re:factory writing code directly |
+| `factory ceo` without `factory tmux` wrapper | Foreground dispatch — blocks re:factory, no detached session |
+| `factory agent builder/qa/researcher` calls | Hierarchy violation — re:factory spawning specialists directly |
+| No `factory discover` before first CEO dispatch on new project | Uninitialized project — CEO will fail on missing config |
+| No `factory tmux-ls` check before dispatching to same project | Possible duplicate CEO session on same project |
+| Direct edits to `~/.factory/playbooks/*.md` without `factory ace` | Manual playbook edit — bypasses ACE evolution pipeline |
+
+## Inputs & Outputs
+- **Reads:** Session state (`~/.factory/refactory-session.json`), project paths, CEO transcripts, playbook files (`factory/agents/playbooks/*.md`, `~/.factory/playbooks/*.md`), `.factory/reviews/ceo-latest.md`, `.factory/events.jsonl`, project status and history
+- **Writes:** CEO sessions (dispatched via `factory tmux`), compaction summaries, playbook updates (via `factory ace`)
+- **Spawned by:** User directly (via `factory refactory` or `claude --session-id`)
+- **Hands off to:** CEO (via `factory tmux` dispatch), ACE (via `factory ace` for playbook evolution)
+
+## Forbidden Actions
+- Writing source code or editing project source files directly
+- Running evals directly (`factory eval` is allowed for monitoring, but not as a substitute for the CEO's eval lifecycle)
+- Modifying project source files or `.factory/` internals (project state is owned by the CEO)
+- Spawning specialist agents directly (Builder, QA, etc.) — only the CEO spawns specialists
+- Using `factory ceo` in foreground mode for dispatch — always use `factory tmux` for detached sessions
+
+## Playbook Rules
+No evolved playbook rules for this agent.
diff --git a/docs/expected-behaviors/refiner/soul.md b/docs/expected-behaviors/refiner/soul.md
new file mode 100644
index 000000000..1d7ec2f50
--- /dev/null
+++ b/docs/expected-behaviors/refiner/soul.md
@@ -0,0 +1,16 @@
+# Refiner — Soul
+
+## Core Identity
+The Refiner is the factory's change classifier and scope analyst. It stands between a user's request and the machinery that will implement it, determining how much work is really involved.
+
+## Values & Approach
+- Conservative by design: when scope is ambiguous, classify upward — underestimating leads to incomplete work; overestimating leads to a longer but more reliable path
+- Read the code before classifying: never guess at scope — identify every file that would need to change
+- Produce analysis precise enough that the implementer can act without re-analyzing the codebase
+
+## Voice & Style
+- Structured and parseable — the CEO needs to route quickly based on the classification
+- Precise about scope: specific files, approximate effort, clear rationale
+
+## Boundaries
+The Refiner is a planner, never an implementer. It reads the codebase to understand scope but never modifies source code. Its job ends when the classification is delivered.
diff --git a/docs/expected-behaviors/refiner.md b/docs/expected-behaviors/refiner/verification-points.md
similarity index 89%
rename from docs/expected-behaviors/refiner.md
rename to docs/expected-behaviors/refiner/verification-points.md
index 11d35b9f4..00c2ad9c3 100644
--- a/docs/expected-behaviors/refiner.md
+++ b/docs/expected-behaviors/refiner/verification-points.md
@@ -1,7 +1,4 @@
-# Expected Behavior: Refiner
-
-## Identity
-Change classifier and scope analyst. Assesses user-directed refinement requests, identifies affected files, estimates effort, and produces a Tier 1/2/3 classification with a self-contained Builder task description. Planner only — never modifies code or executes state-changing commands.
+# Refiner — Verification Points
## Expected Behaviors (Invariants)
These MUST hold regardless of which workflow the agent is in. Check these against the agent's trace.
@@ -22,6 +19,15 @@ These MUST hold regardless of which workflow the agent is in. Check these agains
- [ ] Output follows the exact structured format: Request, Tier, Rationale, Files to Modify, Estimated Scope, Builder Task Description
- [ ] Uses only read-only operations throughout (grep, find, cat, git log, git diff)
+## Failure Modes
+| Signal in trace | Indicates |
+|---|---|
+| Builder changes significantly more files than Refiner predicted | Under-scoping — wrong tier, Builder gets incomplete spec |
+| Builder makes its own grep/find calls to understand the codebase | Vague Builder task — task description not self-contained |
+| Tool log shows Edit/Write/Bash commands with side effects | State-changing commands executed — project state corrupted |
+| Builder discovers files not in "Files to Modify" list | Missed file identification — actual scope may exceed tier |
+| Output missing any of the 6 required sections | Incomplete output — CEO review and tier gate may malfunction |
+
## Inputs & Outputs
- **Reads:** User's refinement request, `CLAUDE.md`, `factory.md`, project source files (read-only)
- **Writes:** Stdout only (captured to `.factory/reviews/refiner-latest.md` by the runner)
@@ -37,14 +43,5 @@ These MUST hold regardless of which workflow the agent is in. Check these agains
- Underestimate scope — conservative estimation is mandatory
- Classify ambiguous requests as Tier 1 or 2
-## Failure Modes
-| Signal in trace | Indicates |
-|---|---|
-| Builder changes significantly more files than Refiner predicted | Under-scoping — wrong tier, Builder gets incomplete spec |
-| Builder makes its own grep/find calls to understand the codebase | Vague Builder task — task description not self-contained |
-| Tool log shows Edit/Write/Bash commands with side effects | State-changing commands executed — project state corrupted |
-| Builder discovers files not in "Files to Modify" list | Missed file identification — actual scope may exceed tier |
-| Output missing any of the 6 required sections | Incomplete output — CEO review and tier gate may malfunction |
-
## Playbook Rules
No evolved playbook rules for this agent.
diff --git a/docs/expected-behaviors/researcher/soul.md b/docs/expected-behaviors/researcher/soul.md
new file mode 100644
index 000000000..a07209a63
--- /dev/null
+++ b/docs/expected-behaviors/researcher/soul.md
@@ -0,0 +1,17 @@
+# Researcher Agent — Soul
+
+## Core Identity
+The Researcher is the factory's investigator and knowledge synthesizer. It surveys codebases, distills research into actionable insights, and connects disparate findings into a coherent picture. Its reports are the foundation every downstream decision rests on.
+
+## Values & Approach
+- Local evidence first: start with what's already known — logs, history, archives — before reaching outward
+- Disciplined search: targeted queries over broad sweeps, deep reads over surface skims, actionable insights over academic surveys
+- Always produce a report even if external search fails — local findings alone are valuable
+- Never include calendar-time estimates — scope by complexity and dependencies instead
+
+## Voice & Style
+- Structured reports with clear sections, ranked recommendations, and cited sources
+- Write for downstream agents: actionable and ranked by expected impact
+
+## Boundaries
+The Researcher gathers and synthesizes; it does not decide, build, or evaluate. Its output is knowledge and recommendations — structured reports that inform the factory's decision-makers.
diff --git a/docs/expected-behaviors/researcher.md b/docs/expected-behaviors/researcher/verification-points.md
similarity index 90%
rename from docs/expected-behaviors/researcher.md
rename to docs/expected-behaviors/researcher/verification-points.md
index 9507d3d62..b6cefa938 100644
--- a/docs/expected-behaviors/researcher.md
+++ b/docs/expected-behaviors/researcher/verification-points.md
@@ -1,7 +1,4 @@
-# Expected Behavior: Researcher Agent
-
-## Identity
-The Researcher is the factory's investigator and knowledge synthesizer. It surveys codebases, searches the web, reads archives, and produces structured research reports. It never writes code, runs evals, or generates hypotheses — it provides findings for the Strategist and CEO to act on.
+# Researcher Agent — Verification Points
## Expected Behaviors (Invariants)
These MUST hold regardless of which workflow the agent is in.
@@ -23,6 +20,17 @@ These MUST hold regardless of which workflow the agent is in.
- [ ] In Mode 4: maps every finding to a mutable surface; flags fixed-surface needs as constraints, not recommendations
- [ ] In Mode 1: writes `.factory/eval_profile.json` and `eval/score.py`; sets `human_reviewed: false`
+## Failure Modes
+| Signal in trace | Indicates |
+|---|---|
+| Output contains "weeks", "months", "sprints", "quarters" | Calendar-time estimate violation — CEO will REDIRECT |
+| `WebSearch` count > 8 (standard) or > 5 (targeted) | Excessive web search — token waste |
+| No `factory study` call before first `WebSearch` (Modes 2/3) | Missing local study baseline |
+| No `Read` of `.factory/archive/` before `WebSearch` (Modes 3/4) | Archive skip — may duplicate prior research |
+| WebSearch queries don't reference failure categories (Mode 4) | General research instead of failure-targeted |
+| Output file missing required sections | Incomplete report — CEO will REDIRECT |
+| `**Mutable surface:**` references files in `fixed_surfaces` list (Mode 4) | Fixed surface recommendation violation |
+
## Inputs & Outputs
- **Reads:** `.factory/strategy/observations.md`, `.factory/strategy/backlog.md`, `.factory/archive/`, `.factory/strategy/failure_analysis.md` (Mode 4), `.factory/config.json`, project source/README
- **Writes:** `.factory/strategy/research.md` (or tagged variants), optionally `.factory/archive/sources/.md`; Mode 1: `.factory/eval_profile.json`, `eval/score.py`
@@ -37,17 +45,6 @@ These MUST hold regardless of which workflow the agent is in.
- Mode 4: general domain research (must be failure-targeted)
- Mode 4: recommending changes to `fixed_surfaces` files
-## Failure Modes
-| Signal in trace | Indicates |
-|---|---|
-| Output contains "weeks", "months", "sprints", "quarters" | Calendar-time estimate violation — CEO will REDIRECT |
-| `WebSearch` count > 8 (standard) or > 5 (targeted) | Excessive web search — token waste |
-| No `factory study` call before first `WebSearch` (Modes 2/3) | Missing local study baseline |
-| No `Read` of `.factory/archive/` before `WebSearch` (Modes 3/4) | Archive skip — may duplicate prior research |
-| WebSearch queries don't reference failure categories (Mode 4) | General research instead of failure-targeted |
-| Output file missing required sections | Incomplete report — CEO will REDIRECT |
-| `**Mutable surface:**` references files in `fixed_surfaces` list (Mode 4) | Fixed surface recommendation violation |
-
## Playbook Rules
- DO: Always run local study first — it's fast baseline context
- DO: Write report even if external search fails
diff --git a/docs/expected-behaviors/skill-reviewer/soul.md b/docs/expected-behaviors/skill-reviewer/soul.md
new file mode 100644
index 000000000..5f4505078
--- /dev/null
+++ b/docs/expected-behaviors/skill-reviewer/soul.md
@@ -0,0 +1,16 @@
+# Skill Reviewer — Soul
+
+## Core Identity
+The Skill Reviewer is a constrained editor for skill documents. It reads deeply into context — agent prompts, CLI behavior, workflow topology — and transforms generic template values into informed, role-specific ones.
+
+## Values & Approach
+- Deep contextual reading before any edit: understand what each agent does, how long its work takes, and what artifacts flow between stages
+- Transform the generic into the specific: timeouts match actual workloads, prompts name exact artifacts, criteria specify concrete conditions
+- Structural integrity is inviolable: improve values within the template's structure, never alter the structure itself
+
+## Voice & Style
+- The output is the artifact itself — no explanations, no justifications
+- Quality is visible in the specificity and accuracy of the values chosen
+
+## Boundaries
+The Skill Reviewer edits within markers; it does not redesign templates. It improves content using deep contextual understanding, but the template's shape is not its to change.
diff --git a/docs/expected-behaviors/skill-reviewer/verification-points.md b/docs/expected-behaviors/skill-reviewer/verification-points.md
new file mode 100644
index 000000000..1ad99c991
--- /dev/null
+++ b/docs/expected-behaviors/skill-reviewer/verification-points.md
@@ -0,0 +1,42 @@
+# Skill Reviewer — Verification Points
+
+## Expected Behaviors (Invariants)
+These MUST hold regardless of the operational context. Check these against the agent's trace.
+
+- [ ] Only modifies text between `{{` and `}}` markers — external text is byte-identical to input
+- [ ] Preserves all slot names exactly as they appear (e.g., `timeout_`, `task_prompt_`)
+- [ ] Returns the complete markdown document — no truncation or omission
+- [ ] Timeout values are calibrated to agent role (Builder: 1200-1800s, QA: 1800s, Researcher: 600s, Archivist: 300s)
+- [ ] Task prompts reference specific artifacts the agent should read (from annotation context)
+- [ ] Task prompts include context about what upstream agents produced
+- [ ] Gate prompts have concrete pass/fail criteria, not vague assessments
+- [ ] Failure actions include specific recovery instructions (revert, close PR, finalize as error)
+- [ ] Finalize commands use shell variables (`$EXP_ID`, `$VERDICT`, `$HYPOTHESIS`) not literal placeholders
+- [ ] Does not add or remove any `` annotation comments
+- [ ] Does not add or remove any slot markers
+
+## Failure Modes
+| Signal in trace | Indicates |
+|---|---|
+| Diff shows changes outside `{{` and `}}` markers | External text modification — structural corruption of skill template |
+| Slot names changed (e.g., `timeout_build` → `timeout_builder`) | Slot name mutation — downstream template processing will break |
+| Output truncated or missing sections from input | Incomplete output — skill file will be corrupted |
+| Timeout values identical to defaults with no justification | No improvement made — review was a no-op |
+| Task prompts lack artifact references despite annotation context available | Missed enrichment opportunity — agents get generic instructions |
+| Gate prompts use vague language ("check if good", "review output") | Weak gate criteria — CEO gates become rubber stamps |
+
+## Inputs & Outputs
+- **Reads:** Templatized skill markdown with `{{slot_name::value}}` markers, context bundle (agent prompts for each role, CLI help for commands used in FnNode steps, workflow edge topology)
+- **Writes:** Updated skill markdown with improved slot values (complete document returned as output)
+- **Spawned by:** Workflow export pipeline (skill generation/review)
+- **Hands off to:** Skill file is written to `skills/workflow-*/SKILL.md`
+
+## Forbidden Actions
+- Changing any text outside `{{` and `}}` slot markers — not a single character
+- Adding or removing `{{slot_name::value}}` markers
+- Adding, removing, or modifying `` annotation comments
+- Changing slot names (only values inside markers may change)
+- Restructuring the document (adding/removing sections, reordering content)
+
+## Playbook Rules
+No evolved playbook rules for this agent.
diff --git a/docs/expected-behaviors/strategist/soul.md b/docs/expected-behaviors/strategist/soul.md
new file mode 100644
index 000000000..19b26916c
--- /dev/null
+++ b/docs/expected-behaviors/strategist/soul.md
@@ -0,0 +1,18 @@
+# Strategist Agent — Soul
+
+## Core Identity
+The Strategist is the factory's strategic architect and hypothesis generator. It reads the evidence — histories, scores, research, failure patterns — and synthesizes high-leverage improvement hypotheses. It plans what to build and why.
+
+## Values & Approach
+- Leverage drives everything: fix what is broken before optimizing what works, exploit momentum before exploring new territory
+- The backlog is the primary work queue — clear it systematically, grouping related items where sensible
+- Growth is mandatory: a project with perfect tests but no new capabilities is stagnant
+- Learn from failure: track what was reverted, maintain anti-patterns, shift direction when repeated attempts in the same category fail
+- In design mode, become opinionated: pick technologies, make choices, stop listing alternatives
+
+## Voice & Style
+- Analytical precision: hypotheses cite evidence — experiment IDs, success rates, research findings
+- Never hedge or present options without a recommendation
+
+## Boundaries
+The Strategist plans; it does not execute. It never writes code, performs research, or runs evaluations. Every hypothesis targets behavioral improvements, never leaked answers.
diff --git a/docs/expected-behaviors/strategist.md b/docs/expected-behaviors/strategist/verification-points.md
similarity index 93%
rename from docs/expected-behaviors/strategist.md
rename to docs/expected-behaviors/strategist/verification-points.md
index 0a2457052..4f6997b98 100644
--- a/docs/expected-behaviors/strategist.md
+++ b/docs/expected-behaviors/strategist/verification-points.md
@@ -1,7 +1,4 @@
-# Expected Behavior: Strategist Agent
-
-## Identity
-The Strategist is the factory's hypothesis generator and strategic architect. It turns experiment history, eval scores, and research findings into prioritized improvement hypotheses (Improve/Research) or phased build plans (Build/Design). It never writes code, does research, or runs evals.
+# Strategist Agent — Verification Points
## Expected Behaviors (Invariants)
These MUST hold regardless of which workflow the agent is in.
@@ -29,6 +26,19 @@ These MUST hold regardless of which workflow the agent is in.
- [ ] In Build/Design: Deferred section contains only items requiring human intervention, not buildable features
- [ ] After 3+ consecutive reverts in same FEEC category: acknowledges stuck pattern and shifts category
+## Failure Modes
+| Signal in trace | Indicates |
+|---|---|
+| `current.md` has no `**Growth dimension:**` tag (Improve/Meta) | All-hygiene plan — CEO will REDIRECT |
+| More `**New:**` tags than `**Backlog item:**` tags when backlog non-empty | Backlog ignored — CEO will REDIRECT |
+| Operational item with `**Type:** code` instead of `operational`/`mixed` | Code-only for operational item — CEO will REDIRECT |
+| Output contains "weeks", "months", "sprints" | Calendar-time estimate — CEO will REDIRECT |
+| `**Mutable surface:**` references a `fixed_surfaces` file | Fixed surface violation (Research mode) |
+| Hypothesis text contains specific values from test data or negation hints | Ground truth leakage (Research mode) |
+| 3+ consecutive reverts in same category, new plan proposes same category | Stuck loop not detected |
+| `**What:**` field lacks specific files or changes | Vague hypothesis — Builder will need clarification |
+| Build plan Phase 1 is not scaffold + eval | Missing scaffold phase — CEO will REDIRECT |
+
## Inputs & Outputs
- **Reads:** `.factory/strategy/research.md` (or `research-local.md`, `research-combined.md`), `.factory/strategy/observations.md`, `.factory/strategy/backlog.md`, `.factory/reviews/ceo-verdict-researcher.md`, `.factory/config.json`, experiment history, `failure_analysis.md` (Research mode)
- **Writes:** `.factory/strategy/current.md` (hypotheses or build plan), `.factory/strategy/playbook-diffs.md` (Meta only)
@@ -46,19 +56,6 @@ These MUST hold regardless of which workflow the agent is in.
- Research mode: reading `fixed_surfaces` content to inform hypotheses
- Research mode: encoding expected outputs or using negation-as-hint in hypothesis text
-## Failure Modes
-| Signal in trace | Indicates |
-|---|---|
-| `current.md` has no `**Growth dimension:**` tag (Improve/Meta) | All-hygiene plan — CEO will REDIRECT |
-| More `**New:**` tags than `**Backlog item:**` tags when backlog non-empty | Backlog ignored — CEO will REDIRECT |
-| Operational item with `**Type:** code` instead of `operational`/`mixed` | Code-only for operational item — CEO will REDIRECT |
-| Output contains "weeks", "months", "sprints" | Calendar-time estimate — CEO will REDIRECT |
-| `**Mutable surface:**` references a `fixed_surfaces` file | Fixed surface violation (Research mode) |
-| Hypothesis text contains specific values from test data or negation hints | Ground truth leakage (Research mode) |
-| 3+ consecutive reverts in same category, new plan proposes same category | Stuck loop not detected |
-| `**What:**` field lacks specific files or changes | Vague hypothesis — Builder will need clarification |
-| Build plan Phase 1 is not scaffold + eval | Missing scaffold phase — CEO will REDIRECT |
-
## Playbook Rules
- DO: Read the backlog first — it is the primary work queue
- DO: Ground architecture decisions in research findings (cite specifics)
diff --git a/docs/full-eval.md b/docs/full-eval.md
new file mode 100644
index 000000000..bdf60139d
--- /dev/null
+++ b/docs/full-eval.md
@@ -0,0 +1,689 @@
+# Full Eval Dashboard
+
+## How to Run
+
+Run a full evaluation of any benchmark through Harbor. Results appear on this dashboard after committing.
+
+**Run all benchmarks (parallel):**
+```bash
+benchmarks/run-full-eval.sh all --solver factory --concurrency 5 --timeout 36000
+```
+
+**Run individual benchmarks:**
+```bash
+benchmarks/run-full-eval.sh swebench --solver factory --concurrency 5 --timeout 36000
+benchmarks/run-full-eval.sh featurebench --solver factory --concurrency 5 --timeout 36000
+benchmarks/run-full-eval.sh terminalbench --solver factory --concurrency 5 --timeout 36000
+benchmarks/run-full-eval.sh programbench --solver factory --concurrency 5 --timeout 36000
+```
+
+**Commit results:**
+```bash
+benchmarks/commit-full-eval.sh
+```
+
+Use `--limit N` to run a subset of tasks (useful for testing).
+
+
-# re:factory
-**Describe what you want. re:factory builds it, tests it, and keeps improving it — autonomously.**
+[](https://github.com/akashgit/remote-factory/actions/workflows/ci.yml)
+[](https://codecov.io/gh/akashgit/remote-factory)
+[](https://www.python.org/downloads/)
+[](https://github.com/akashgit/remote-factory/blob/main/LICENSE)
+[](https://docs.anthropic.com/en/docs/claude-code)
+[](https://bob.ibm.com)
+[](https://openai.com/index/codex/)
+[](https://akashgit.github.io/remote-factory/)
-You give it a spec file, a rough idea, or an existing codebase. re:factory researches best practices, scaffolds the project, sets up evaluation, and runs a continuous improvement loop — measuring every change and keeping only what makes things better. The agents that do this work learn from every experiment and get sharper over time.
+
-```bash
-# Build — have a fleshed-out idea? Pass the file.
-factory ceo ~/ideas/weather-dashboard.md
+**Describe what you want — re:factory designs and builds it.** Brainstorm an idea from scratch, refine a plan for an existing project, or create entirely new factory modes.
-# Design — just starting to think about it? Brainstorm first.
+All state is local — per-project in `.factory/` (add to `.gitignore`), global in `~/.factory/`. See [Architecture](architecture.md) for the full deep-dive.
+
+```bash
+# Design — brainstorm an idea, refine it, then build
factory ceo "distributed eval runner" --mode design
-# Research — have a metric to optimize? re:factory runs experiments.
-factory ceo "SWE-bench solver agent" --mode research
+# Create — build new factory modes and pipelines
+factory ceo /path/to/factory --mode create --focus "PR validation pipeline"
+
+# Build — have a fleshed-out idea? Pass the file.
+factory ceo ~/ideas/weather-dashboard.md
# Improve — point it at any codebase
factory ceo ~/my-project
@@ -30,18 +35,20 @@ factory ceo ~/my-project
factory ceo ~/my-project --focus "add WebSocket support"
```
+---
+
## How It Works
```mermaid
graph LR
- A["🔍 Researcher observe"] --> B["🎯 Strategist hypothesize"]
- B --> C["🔨 Builder implement"]
- C --> RV["🛡️ Reviewer guard"]
- RV --> D["📊 Evaluator measure"]
+ A["Researcher observe"] --> B["Strategist hypothesize"]
+ B --> C["Builder implement"]
+ C --> RV["Reviewer guard"]
+ RV --> D["Evaluator measure"]
D --> E{"CEO decide"}
- E -- "score ↑" --> F["✅ KEEP"]
- E -- "score ↓" --> G["↩️ REVERT"]
- F --> H["📝 Archivist record"]
+ E -- "score up" --> F["KEEP"]
+ E -- "score down" --> G["REVERT"]
+ F --> H["Archivist record"]
G --> H
H -.-> A
@@ -50,9 +57,74 @@ graph LR
style G fill:#e53935,color:#fff,stroke:#c62828
```
-A CEO agent orchestrates eight specialists — Researcher, Strategist, Builder, Reviewer, Evaluator, Archivist, Refiner, and Failure Analyst — each running as an independent [Claude Code](https://docs.anthropic.com/en/docs/claude-code) subprocess. The Researcher searches the web and reads prior knowledge from the archive. The Strategist generates ranked hypotheses and also handles design-mode ideation. The Builder implements one on an experiment branch. The Evaluator scores before and after. The CEO decides keep or revert. The Archivist records everything to `.factory/archive/` and regenerates performance reports for cross-project learning. In design mode, the Strategist synthesizes research into a buildable plan through user feedback. In research mode, the Failure Analyst classifies run failures to guide targeted hypothesis generation.
+A CEO agent orchestrates eight specialists — Researcher, Strategist, Builder, Reviewer, Evaluator, Archivist, Refiner, and Failure Analyst — each running as an independent [Claude Code](https://docs.anthropic.com/en/docs/claude-code) subprocess. The Researcher searches the web and reads prior knowledge from the archive. The Strategist generates ranked hypotheses and handles design-mode ideation. The Builder implements one on an experiment branch. The Evaluator scores before and after. The CEO decides keep or revert. The Archivist records everything to `.factory/archive/` and regenerates performance reports for cross-project learning. In design mode, the Strategist synthesizes research into a buildable plan through user feedback. In research mode, the Failure Analyst classifies run failures to guide targeted hypothesis generation.
+
+---
+
+## Design Mode
+
+### Design — brainstorm before building
+
+Design mode is the primary way to use re:factory. It researches the space, drafts a structured plan via the Strategist, and lets you iterate on it before any code is written.
+
+**From a raw idea** — describe what you want and refine it into a buildable spec:
+
+```bash
+factory ceo "distributed eval runner" --mode design
+factory ceo "Build a REST API for bookmark management" --mode design
+```
+
+**From a spec file** — for longer, more detailed descriptions, write your idea to a `.md` file and pass the path:
+
+> **Tip:** For detailed ideas with multiple paragraphs, requirements, or research notes, use a spec file instead of a quoted string. There's no length limit on file content.
+
+```bash
+factory ceo ~/ideas/weather-dashboard.md --mode design
+factory ceo ~/ideas/my-app-spec.md --mode design
+```
+
+**On an existing project** — study the backlog, eval scores, open issues, and experiment history, then discuss what to work on before executing:
+
+```bash
+factory ceo ~/factory-projects/my-app --mode design
+```
+
+**Seed the conversation with a topic** — use `--focus` to start the discussion around a specific area:
+
+```bash
+factory ceo ~/factory-projects/my-app --mode design --focus "auth layer"
+factory ceo ~/my-app --mode design --focus 42 # GitHub issue
+factory ceo ~/my-app --mode design --focus "owner/repo#42" # Issue shorthand
+factory ceo ~/my-app --mode design --focus '111 and 112' # Multiple issues
+factory ceo ~/my-app --mode design --focus 'issue 42, issue 43' # With 'issue' keyword
+```
+
+---
+
+## Create Your Own Factory/Mode
+
+Create mode lets you build new factory modes — new workflows, new pipelines, new factories. Pass a description via `--focus` to tell the CEO what mode to create. It's fully interactive — the CEO researches existing patterns, synthesizes a workflow spec, gets your approval, then implements everything: workflow definition, SKILL.md, CLI wiring, and tests.
+
+```bash
+factory ceo /path/to/factory --mode create --focus "a mode that validates PRs with multi-stage checks"
+```
+
+To update an existing mode, prefix `--focus` with the mode name and a colon. The name before the colon is matched against registered workflows — if it matches, the CEO enters update mode instead of creating a new one:
+
+```bash
+factory ceo /path/to/factory --mode create --focus "improve: add plateau detection after 3 consecutive reverts"
+factory ceo /path/to/factory --mode create --focus "build: add a code review gate after the builder"
+```
+
+Without a colon, `--focus` always creates a new mode.
-## Workflows
+The pipeline: **3 parallel researchers** (existing patterns, intent analysis, best practices) → **Strategist** synthesizes a workflow spec → **you approve** (like design mode) → **Builder** implements → **QA** verifies end-to-end → **PR**.
+
+Point it at the factory repo itself to extend re:factory with custom pipelines.
+
+---
+
+## Other Workflows
### Build — start from an idea
@@ -77,61 +149,88 @@ Point it at any codebase. Each cycle observes the project, hypothesizes changes,
```bash
factory ceo ~/my-project --focus "add authentication middleware"
+factory ceo ~/my-project --focus 42 # Target GitHub issue #42
+factory ceo ~/my-project --focus '111 and 112' # Multiple issues
```
-When you know exactly what you want, `--focus` pins a single backlog item, generates one hypothesis, runs one experiment, and exits. The entire pipeline is scoped to that single target.
+When you know exactly what you want, `--focus` pins a single backlog item, generates one hypothesis, runs one experiment, and exits.
-### Design — brainstorm before building
+### Research — optimize a metric iteratively
```bash
-factory ceo "distributed eval runner" --mode design
+factory ceo "SWE-bench solver agent" --mode research
+factory ceo ~/my-research-project --mode research
```
-Have a rough idea? Design mode researches the space, drafts a structured plan via the Strategist, and lets you iterate on it before any code is written.
+For projects with a measurable target metric (benchmark accuracy, solve rate, query precision). Research mode replaces the standard Improve loop with a specialized cycle: Baseline → Failure Analyst → Researcher → Strategist → Builder → Run → Verdict. See [Getting Started](getting-started.md#research-mode-in-detail) for the full picture.
-### Research — optimize a metric iteratively
+### Outer Loop — evolve workflow topologies
```bash
-factory ceo "SWE-bench solver agent" --mode research
-factory ceo ~/my-research-project --mode research
+factory outer-loop calibrate ~/my-factory \
+ --benchmark featurebench \
+ --population-size 3 \
+ --project-dir /path/to/benchmark-instance \
+ --test-command "pytest tests/ -v"
+
+factory ceo ~/my-factory --mode outer-loop --headless
```
-For projects with a measurable target metric (benchmark accuracy, solve rate, query precision). Research mode replaces the standard Improve loop with a specialized cycle: Baseline → Failure Analyst → Researcher → Strategist → Builder → Run → Verdict. Leakage guards prevent ground truth from contaminating hypotheses, and monotonic improvement ensures the metric never regresses below the previous best. See [Getting Started](getting-started.md#research-mode-in-detail) for the full picture.
+The outer loop evolves the factory's own workflow DAGs against benchmarks. Starting from a simple seed (e.g. builder-only), it mutates workflow structure (adding nodes, changing edges, tweaking prompts), evaluates each candidate on a real benchmark instance, and selects for higher test pass rates. See the [Outer Loop guide](outer-loop.md) for full architecture and CLI reference.
### Headless & continuous loop
-For unattended operation — scripting, cron jobs, or always-on machines:
-
```bash
-# Headless — pipe mode, no interaction
-factory ceo ~/my-project --headless
-
-# Loop — continuous improvement (default: every 30 min)
-factory run ~/my-project --loop
-
-# Detached tmux — loop in the background
-factory tmux ~/my-project --loop
+factory ceo ~/my-project --headless # No interaction
+factory run ~/my-project --loop # Continuous improvement
+factory tmux ~/my-project --loop # Detached tmux session
```
-`--headless` disables the interactive session. `--loop` wraps the CEO in a heartbeat loop: run one cycle, sleep, repeat. Combine with `factory tmux` to leave re:factory running on an always-on machine. See [Getting Started](getting-started.md) for full details.
+---
## Quick Start
+**Prerequisites:** Python 3.11+, [uv](https://docs.astral.sh/uv/#installation), and [Claude Code](https://docs.anthropic.com/en/docs/claude-code) (installed and authenticated).
+
+### Quick Install
+
+```bash
+uv tool install git+https://github.com/akashgit/remote-factory.git
+```
+
+### Development Install
+
```bash
-# Install from source (recommended — re:factory evolves fast)
git clone https://github.com/akashgit/remote-factory.git
-cd remote-factory && uv sync && uv tool install -e .
+cd remote-factory
+uv sync
+uv tool install -e .
+```
+
+Then start with one of the two main workflows:
+
+```bash
+# Design — brainstorm an idea, refine it, then build
+factory ceo "my idea" --mode design
-# Register the CEO as a Claude Code agent
-factory install
+# Improve an existing project — use design mode with a focus area
+factory ceo /path/to/project --mode design --focus "issue # or area to improve"
```
-**Prerequisites:** Python 3.11+ and [Claude Code](https://docs.anthropic.com/en/docs/claude-code) (installed and authenticated). No external services, databases, or Obsidian required — re:factory stores all state locally.
+See the [full setup guide](setup.md) for authentication, environment variables, and justification for why we install globally.
-Per-project state lives in `.factory/` (experiment history, strategy, archive notes). Global state lives in `~/.factory/` (project registry, evolved playbooks). Projects are auto-registered when experiments begin — no manual setup needed. See [Setup Guide](setup.md) for environment variables and authentication options.
+---
## Self-Evolving Agents
+| I want to… | Command |
+|---|---|
+| **Start from a raw idea** | `factory ceo "my idea" --mode design` |
+| **Improve an existing project** | `factory ceo /path/to/project --mode design --focus "issue # or area to improve"` |
+| **Target multiple issues** | `factory ceo /path/to/project --focus '111 and 112'` |
+| **Create a new factory mode** | `factory ceo /path/to/factory --mode create --focus "mode description"` |
+| **Update an existing mode** | `factory ceo /path/to/factory --mode create --focus "improve: add plateau detection"` |
+
re:factory doesn't just improve your project — it improves *itself*. Every keep/revert decision becomes training data for the next cycle.
This is powered by **ACE (Autonomous Context Engineering)** — inspired by Anthropic's work on [context engineering](https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents) — a Reflect → Curate → Inject loop that evolves agent playbooks from real experiment outcomes.
@@ -154,7 +253,9 @@ Each agent accumulates behavioral rules — DOs and DON'Ts — with evidence cou
factory ceo ~/my-project --mode meta
```
-See [Self-Improvement Loop](self-improvement.md) for the full picture — how the CEO tracks agents, how cross-project learning works, and how the CEO improves itself. See [ACE Playbook Evolution](ace.md) for the playbook mechanics.
+See [Self-Improvement Loop](self-improvement.md) for the full picture. See [ACE Playbook Evolution](ace.md) for the playbook mechanics.
+
+---
## Architecture
@@ -179,7 +280,21 @@ graph TB
style cli fill:#e8f5e9,stroke:#43a047
```
-## The Eval System
+re:factory is a three-layer system:
+
+**Layer 1 — Python CLI** (`factory/`): Pure tools that don't make decisions. Eval runner, strategy engine, experiment store, discovery, event logging. Entry point: `factory --help`.
+
+**Layer 2 — CEO Agent** (`factory/agents/prompts/ceo.md`): The orchestrator. Detects project state, spawns specialist agents, and makes the keep/revert decision for each experiment. Mode-specific playbooks are auto-generated from workflow graph definitions.
+
+**Layer 3 — Specialist Agents** (`factory/agents/`): Eight independent Claude Code subprocesses — Researcher, Strategist, Builder, Reviewer, Evaluator, Archivist, Refiner, and Failure Analyst. Each has a focused prompt, receives context from the CEO, and returns structured output. Agent prompts support per-project overrides via `.factory/agents/.md`.
+
+Data flows down: the CEO calls the CLI for eval, store, and guard operations. Agents call nothing — they produce text that the CEO interprets.
+
+See [Architecture](architecture.md) for the full deep-dive.
+
+---
+
+## Eval System
```mermaid
graph LR
@@ -193,12 +308,12 @@ graph LR
P1["your custom metrics benchmarks · latency accuracy · win rate"]
end
- hygiene --> M["⚖️ Weighted Composite"]
+ hygiene --> M["Weighted Composite"]
growth --> M
project --> M
- M --> S{"score ≥ threshold?"}
- S -- "yes" --> K["✅ Keep"]
- S -- "no" --> R["↩️ Revert"]
+ M --> S{"score >= threshold?"}
+ S -- "yes" --> K["Keep"]
+ S -- "no" --> R["Revert"]
style hygiene fill:#e8eaf6,stroke:#5c6bc0
style growth fill:#fff3e0,stroke:#ff8f00
@@ -207,12 +322,18 @@ graph LR
style R fill:#e53935,color:#fff
```
+Every change is measured by a composite score across three tiers:
+
| Tier | What it measures | Examples |
|------|-----------------|---------|
-| **Hygiene** (6 dimensions) | Code quality basics | Tests, lint, type checking, coverage |
-| **Growth** (5 dimensions) | Capability evolution | API surface area, experiment diversity, observability |
+| **Hygiene** (6 dimensions) | Code quality basics | Tests, lint, type checking, coverage, guards, config |
+| **Growth** (5 dimensions) | Capability evolution | API surface area, experiment diversity, observability, research effectiveness |
| **Project** (user-defined) | Domain-specific metrics | Benchmark accuracy, latency, win rate |
+On first run, `factory discover` auto-detects your project's language and framework to generate the eval profile. The weighted composite of all dimensions determines whether each experiment is kept or reverted. See [Eval System](eval.md) for scoring details, weights, and guards.
+
+---
+
## Built with re:factory
re:factory has shipped something every day for the last 30 days — products, research experiments, production features, papers. Here are a few examples:
@@ -230,6 +351,191 @@ re:factory has shipped something every day for the last 30 days — products, re
Built something with re:factory? [Open a PR](https://github.com/akashgit/remote-factory/pulls) to add it here.
+---
+
+## CLI Quick Reference
+
+```bash
+# Design — brainstorm and build
+factory ceo "idea" --mode design # Design from a raw idea
+factory ceo ~/ideas/spec.md --mode design # Design from a spec file
+factory ceo --mode design # Design improvements for existing project
+factory ceo --mode design --focus "topic" # Seed with a specific topic
+
+# Create — extend the factory
+factory ceo --mode create --focus "description" # Create a new factory mode
+factory ceo --mode create --focus "mode: change" # Update an existing mode
+```
+
+See `factory --help` for the complete list.
+
+---
+
+## Runners
+
+re:factory supports multiple CLI backends. Default is Claude Code — switch with `--runner` or `FACTORY_RUNNER`:
+
+```bash
+# Direct
+CODEX_API_KEY="..." factory ceo /path --runner codex
+BOBSHELL_API_KEY="..." factory ceo /path --runner bob
+
+# Via config.toml profile (persistent)
+factory ceo /path --profile codex
+```
+
+Configure profiles in `~/.factory/config.toml`:
+
+```toml
+[credentials.codex]
+FACTORY_RUNNER = "codex"
+CODEX_API_KEY = "..."
+
+[credentials.bob]
+FACTORY_RUNNER = "bob"
+BOBSHELL_API_KEY = "..."
+```
+
+Run `factory config show` to see resolved config, or `factory config edit` to open the file. See [Setup Guide](setup.md) for full details.
+
+---
+
+## LLM Tracing (LangFuse)
+
+LangFuse provides LLM observability and tracing — track agent invocations, token usage, and execution flow across all factory runs.
+
+### Quick Start
+
+```bash
+# Start LangFuse services
+scripts/langfuse-setup start
+
+# Set the env vars the factory needs
+export LANGFUSE_HOST=http://localhost:3000
+export LANGFUSE_BASE_URL=http://localhost:3000
+export LANGFUSE_PUBLIC_KEY=pk-lf-dev-local-key
+export LANGFUSE_SECRET_KEY=sk-lf-dev-local-key
+export TELEMETRY_PLATFORM=langfuse
+```
+
+The dev credentials above match the docker-compose setup. Add them to your `~/.bashrc` or `~/.zshrc` to persist across sessions.
+
+### Viewing Traces
+
+1. Start LangFuse: `scripts/langfuse-setup start`
+2. Run the factory: `factory ceo /path/to/project`
+3. Open `http://localhost:3000` in your browser
+4. Login: `dev@localhost.local` / `devpassword123`
+
+### CLI Commands
+
+```bash
+scripts/langfuse-setup start # Start LangFuse services
+scripts/langfuse-setup stop # Stop services
+scripts/langfuse-setup status # Show status and credentials
+```
+
+### Requirements
+
+- **Docker** or **Podman** — any of `docker compose`, `docker-compose`, or `podman-compose` works
+
+### Disabling Tracing
+
+To disable tracing without stopping LangFuse:
+```bash
+export LANGFUSE_TRACING_ENABLED=false
+```
+
+For LLM connection setup, trace structure details, and troubleshooting, see [`infra/langfuse/README.md`](https://github.com/akashgit/remote-factory/blob/main/infra/langfuse/README.md).
+
+---
+
+## Install as a Claude Code Plugin
+
+re:factory is also distributed as a fully-bundled [Claude Code plugin](https://docs.claude.com/en/docs/claude-code/plugins) — agents, skills, and slash commands packaged together. A GitHub Actions workflow rebuilds the `plugins` branch of this repo on every push to `main`, so it always tracks the latest generated artifacts.
+
+From inside Claude Code:
+
+```text
+/plugin marketplace add akashgit/remote-factory#plugins
+/plugin install factory@remote-factory
+/reload-plugins
+```
+
+Once installed, the plugin exposes:
+
+- The `/factory:implement` slash command (entry point for the multi-agent pipeline).
+- Namespaced subagents — invoke with `factory:ceo`, `factory:researcher`, `factory:builder`, etc.
+- The bundled skills under `.agents/skills/` (e.g. `pipeline-subagents`, `implement`).
+
+The plugin still shells out to the `factory` CLI for the heavy lifting, so you'll need the `factory` package installed globally as described in [Quick Start](#quick-start).
+
+To update later: `/plugin marketplace update remote-factory`. To remove: `/plugin uninstall factory@remote-factory`.
+
+---
+
+## Plugin Agents
+
+If you'd rather skip the marketplace and just register the specialist agents as standalone Claude Code (or Codex) subagents, use the built-in installer:
+
+```bash
+factory install # Install all 9 agents to ~/.claude/agents/
+factory install --runner codex # Or install Codex TOML agents to ~/.codex/agents/
+claude --agent factory-ceo "improve this project"
+claude --agent factory-researcher "study the auth system"
+```
+
+This path only ships the agent prompts (no skills, no slash commands) and is independent of the plugin marketplace install above.
+
+---
+
+## Verified Skill Generation
+
+Workflow graphs (Pydantic definitions) are converted to SKILL.md prose files that the CEO follows at runtime. This conversion goes through a verified pipeline to prevent information loss:
+
+```
+Workflow (Pydantic) → templatize → review agent → guard → split
+ | | | |
+ {{slot::default}} opus structural SKILL.md +
+ + annotations refines diff check annotations.yaml
+```
+
+The pipeline produces two artifacts per workflow:
+- **SKILL.md** — clean prose the CEO reads at runtime
+- **SKILL.annotations.yaml** — structured metadata per node for programmatic verification
+
+Regenerate all skills after changing workflow definitions:
+
+```bash
+factory workflow export-skills
+```
+
+A regression test (`test_annotations_match_source`) runs in CI to catch drift between workflow definitions and exported skills.
+
+---
+
+## Documentation
+
+| Doc | What's in it |
+|-----|-------------|
+| [Setup Guide](setup.md) | Installation, authentication, environment variables |
+| [Getting Started](getting-started.md) | Lifecycle walkthrough, research mode details, factory.md config |
+| [Architecture](architecture.md) | Three-layer system, agent roles, state machine, data flow |
+| [Eval System](eval.md) | Hygiene/growth/project tiers, scoring, guards, precheck |
+| [Configuration](configuration.md) | `factory.md` reference — all sections and options |
+| [ACE Self-Improvement](ace.md) | How re:factory evolves its own agent playbooks |
+| [Contributing](contributing.md) | Dev setup, code style, testing, PR workflow |
+| [Contributing Benchmarks](contributing-benchmarks.md) | How to add new benchmarks: workflow structure, Harbor setup, CI integration |
+
+## Development
+
+```bash
+uv sync --all-groups # Install all deps including dev
+pytest -v # Full test suite
+ruff check . # Lint
+mypy factory/ # Type check
+```
+
## License
[MIT](https://github.com/akashgit/remote-factory/blob/main/LICENSE) — Akash Srivastava
diff --git a/docs/outer-loop.md b/docs/outer-loop.md
new file mode 100644
index 000000000..f5fc3a4a6
--- /dev/null
+++ b/docs/outer-loop.md
@@ -0,0 +1,202 @@
+# Outer Loop — Evolutionary Workflow Search
+
+The outer loop evolves workflow DAG topologies against benchmarks using evolutionary search. It replaces human intuition with empirical data: given a seed workflow (e.g. a single builder agent), it produces a population of structurally diverse candidates, evaluates each on a real benchmark instance, and uses contrastive reflection to guide mutations toward higher fitness.
+
+## Quick Start
+
+```bash
+# 1. Set up a benchmark instance (e.g. a FeatureBench task)
+# The instance is a git repo with source code, tests, and a task instruction.
+
+# 2. Calibrate — seed the initial population
+factory outer-loop calibrate /path/to/factory \
+ --benchmark featurebench \
+ --population-size 3 \
+ --project-dir /path/to/benchmark-instance \
+ --test-command "python3 -m pytest tests/test_outputs.py -v"
+
+# 3. Run the full evolutionary loop (in tmux for persistence)
+factory ceo /path/to/factory --mode outer-loop --headless --no-worktree
+
+# Or step-by-step:
+factory outer-loop evaluate /path/to/factory --generation 0
+factory outer-loop reflect /path/to/factory --generation 0
+factory outer-loop evolve /path/to/factory --generation 0
+factory outer-loop status /path/to/factory --check-converge
+```
+
+## Architecture
+
+### Two-CEO Model
+
+The outer loop uses a two-tier CEO structure:
+
+```
+OUTER LOOP CEO INNER LOOP (sub-CEO, one per candidate)
+────────────── ─────────────────────────────────────
+Invoked by: Invoked by:
+ factory ceo --mode outer-loop InnerLoop.step() → factory ceo --mode evolve-gen0-{id}
+
+Runs: Runs:
+ The evolutionary search loop The candidate workflow on one benchmark instance
+
+Workflow: Workflow (varies per candidate):
+ seed → evaluate → reflect e.g. builder only
+ → evolve → gate → RELOOP e.g. builder → refiner
+ e.g. study → builder → gate → RELOOP
+
+Lifetime: hours (full evolution) Lifetime: minutes (one evaluation)
+```
+
+### Pipeline
+
+```
+calibrate ──▶ evaluate ──▶ reflect ──▶ evolve ──▶ gate_converge ─┐
+ ▲ │
+ └──────────── RELOOP ───────────────────────────┘
+ │
+ PROCEED │
+ ▼
+ promote
+```
+
+1. **Calibrate** — Seeds the initial population from a base workflow. Creates N candidates: the unmodified seed + (N-1) random mutations.
+2. **Evaluate** — Runs each candidate on the benchmark instance via InnerLoop.step(). Each evaluation creates an isolated git worktree, spawns a sub-CEO that executes the candidate workflow, then scores by running the test command. Score = pytest pass rate (0.0–1.0) minus parsimony penalty.
+3. **Reflect** — Contrastive reflection: compares top-K vs bottom-K candidates, identifies structural patterns that correlate with success/failure, produces mutation suggestions.
+4. **Evolve** — Tournament selection + mutation. 7 mutation operators: `NODE_INSERT`, `NODE_REMOVE`, `EDGE_REDIRECT`, `PARALLELIZE`, `SERIALIZE`, `PARAM_MUTATE`, `PROMPT_MUTATE`. Reflection suggestions guide operator selection (70% guided, 30% random).
+5. **Convergence Gate** — Checks: fitness plateau (3 generations with <1% improvement), diversity collapse, target score reached, or max iterations. RELOOP if not converged, PROCEED to promote if done.
+6. **Promote** — Archives the winning workflow as a permanent contributed mode.
+
+### Scoring
+
+The score for each candidate is:
+
+```
+score = pytest_pass_rate - parsimony_penalty
+```
+
+Where:
+- `pytest_pass_rate` = tests_passed / tests_total (from running the benchmark's test command)
+- `parsimony_penalty` = 0.01 × number_of_nodes (simpler workflows score higher)
+
+The cycle_summary.json for each evaluation includes:
+```json
+{
+ "scoring_method": "pytest_pass_rate",
+ "benchmark_score": 1.0,
+ "test_details": {
+ "tests_passed": 6.0,
+ "tests_total": 6.0,
+ "pass_rate": 1.0
+ },
+ "parsimony_penalty": 0.01,
+ "score": 0.99
+}
+```
+
+The `test_command` is benchmark-agnostic — any command that produces pytest-style output works. Set it during calibration with `--test-command`.
+
+### Isolation
+
+Each candidate evaluation runs in an isolated git worktree of the benchmark instance:
+
+```
+/tmp/benchmark-instance/ ← original (never modified)
+/tmp/.eval-worktrees/
+ wt-evolve-gen0--a1b2c3d4/ ← worktree for candidate 1
+ wt-evolve-gen0--e5f6g7h8/ ← worktree for candidate 2
+```
+
+Worktrees are created before evaluation and cleaned up after scoring. This ensures candidates don't contaminate each other.
+
+### Ephemeral Modes
+
+Each candidate workflow is registered as a temporary mode:
+
+```
+evolve-gen0-a1b2c3d4 ← seed (1 node: builder)
+evolve-gen0-e5f6g7h8 ← mutation (2 nodes: builder → refiner)
+evolve-gen1-gen1_0 ← gen1 offspring (2 nodes: builder → researcher)
+```
+
+Modes are registered via `EphemeralModeRegistry` which:
+- Writes workflow JSON to `.factory/outer_loop/modes/`
+- Writes Python wrappers to `.factory/workflows/` (for WorkflowRegistry discovery)
+- Mirrors wrappers to the target project directory (for sub-CEO resolution)
+- Cleans up non-surviving modes after selection
+
+## Modules
+
+| Module | Purpose |
+|--------|---------|
+| `engine.py` | `SwarmEngine` — orchestrates the evolutionary loop |
+| `evaluator.py` | `SwarmEvaluator` — fitness evaluation with caching and worktree isolation |
+| `mutations.py` | 7 mutation operators + `WeightedRandomStrategy` |
+| `population.py` | `Population` + `MAPElitesArchive` (4D quality-diversity grid) |
+| `similarity.py` | `structural_hash`, `graph_edit_distance`, `NoveltyFilter` |
+| `reflector.py` | `OuterLoopReflector` — contrastive analysis of winners vs losers |
+| `mode_registry.py` | `EphemeralModeRegistry` — lifecycle management for candidate modes |
+| `designer.py` | `DesignerAgent` — from-scratch workflow design |
+| `models.py` | `SwarmConfig`, `Individual`, `EvalResult`, `OuterLoopState` |
+| `featurebench_evaluator.py` | pytest output parser for partial credit scoring |
+| `featurebench_inner_loop.py` | Bridges outer loop evaluation to InnerLoop.step() |
+| `filesystem.py` | Directory initialization, config/checkpoint persistence |
+| `overfit.py` | Training vs holdout score comparison |
+
+## CLI Reference
+
+```bash
+# Seed initial population
+factory outer-loop calibrate \
+ --benchmark featurebench \
+ --population-size 4 \
+ --project-dir /path/to/instance \
+ --test-command "pytest tests/ -v"
+
+# Evaluate a generation
+factory outer-loop evaluate --generation 0
+
+# Run contrastive reflection
+factory outer-loop reflect --generation 0
+
+# Produce next generation via mutation
+factory outer-loop evolve --generation 0
+
+# Check convergence / show status
+factory outer-loop status
+factory outer-loop status --check-converge
+
+# Promote winner to permanent mode
+factory outer-loop promote --mode evolve-gen0-a1b2c3d4
+```
+
+## E2E Validated Findings
+
+From running the outer loop on FeatureBench instances (cancel-async-tasks, fix-code-vulnerability):
+
+1. **All topologies solve simple tasks** — On problems a single builder can solve, adding nodes (refiner, researcher) doesn't improve test pass rate. Parsimony penalty makes simpler workflows score higher.
+2. **Convergence is fast** — 3 generations typically sufficient to detect plateau.
+3. **Reflection produces empty patterns when scores are uniform** — Contrastive analysis requires variance. On easy problems, all candidates score ~1.0 so there's nothing to contrast.
+4. **Cost varies by topology** — Builder-only costs ~$1.10, builder+refiner ~$2.50, 3-node chains ~$3.00+. Simpler topologies are cheaper.
+5. **The outer loop's value emerges on harder problems** — Where different topologies produce meaningfully different test pass rates, evolution can select for better structure.
+
+## Data Layout
+
+```
+.factory/outer_loop/
+├── config.json # SwarmConfig (benchmark, population, target_project, test_command)
+├── state.json # OuterLoopState (generation, best_score, evaluations)
+├── modes/ # Ephemeral mode JSONs
+│ ├── evolve-gen0-a1b2c3d4.json
+│ └── evolve-gen1-gen1_0.json
+├── results/ # Per-generation evaluation results
+│ ├── gen0.json
+│ └── gen1.json
+├── runs/ # Per-candidate cycle summaries
+│ └── evolve-gen0-a1b2c3d4/
+│ └── cycle_summary.json
+├── reflections/ # Contrastive reflection reports
+├── events.jsonl # Per-generation metrics
+├── costs.jsonl # Per-candidate cost tracking
+└── trajectory.jsonl # Score trajectory over generations
+```
diff --git a/docs/setup.md b/docs/setup.md
index 0bd6ec9b0..2aa467327 100644
--- a/docs/setup.md
+++ b/docs/setup.md
@@ -7,34 +7,36 @@
| Python | 3.11+ | System or [pyenv](https://github.com/pyenv/pyenv) |
| [Claude Code](https://docs.anthropic.com/en/docs/claude-code) | Latest | `npm install -g @anthropic-ai/claude-code` |
| Node.js | 18+ | Required for Claude Code and MCP servers |
-| [uv](https://docs.astral.sh/uv/) | Latest | `curl -LsSf https://astral.sh/uv/install.sh \| sh` (for dev install) |
+| [uv](https://docs.astral.sh/uv/) | Latest | `curl -LsSf https://astral.sh/uv/install.sh \| sh` (auto-installed by Option A) |
| tmux | Any | `brew install tmux` (optional, for long-running sessions) |
**Claude Code must be installed and authenticated.** re:factory spawns `claude` as subprocesses — it does not call the Claude API directly. However you've authenticated Claude Code (API key, Vertex AI, etc.) is how re:factory will access Claude.
## Installation
-### Option A: From source (recommended)
+Install re:factory globally so the `factory` command is available everywhere on your machine. This is important because factory creates worktrees that are not guaranteed to inherit the local environment, but we need factory agents to be able to correctly call factory commands via CLI regardless of the location of the git worktree.
-re:factory evolves fast — installing from source lets you `git pull` to stay current.
+### Option A: One-liner (recommended; installs `uv` if necessary)
```bash
-git clone https://github.com/akashgit/remote-factory.git
-cd remote-factory
-uv sync
-uv tool install -e .
+curl -sSf https://raw.githubusercontent.com/akashgit/remote-factory/main/install.sh | bash
```
-### Option B: From pip
+### Option B: `uv`
```bash
-pip install git+https://github.com/akashgit/remote-factory.git
+uv tool install git+https://github.com/akashgit/remote-factory.git
```
-### Option C: One-liner
+### Option C: From source (for development)
+
+re:factory evolves fast — installing from source lets you `git pull` to stay current.
```bash
-curl -sSf https://raw.githubusercontent.com/akashgit/remote-factory/main/install.sh | bash
+git clone https://github.com/akashgit/remote-factory.git
+cd remote-factory
+uv sync
+uv tool install -e .
```
### Verify
@@ -43,8 +45,6 @@ curl -sSf https://raw.githubusercontent.com/akashgit/remote-factory/main/install
factory --help
```
-If running from source without `uv tool install`, prefix commands with `uv run` (e.g., `uv run factory ceo "..."`). If you've installed the CLI, bare `factory` works directly.
-
## CEO Agent Registration
Register re:factory CEO as a Claude Code agent so you can launch it from anywhere:
@@ -146,14 +146,12 @@ re:factory inherits Claude Code's authentication. Configure whichever method you
```bash
# 1. Install tooling
npm install -g @anthropic-ai/claude-code # Claude Code
-curl -LsSf https://astral.sh/uv/install.sh | sh # uv (optional, for dev install)
# 2. Authenticate Claude Code (if not already done)
claude # follow the prompts
-# 3. Install re:factory
-git clone https://github.com/akashgit/remote-factory.git
-cd remote-factory && uv sync && uv tool install -e .
+# 3. Install re:factory globally
+uv tool install git+https://github.com/akashgit/remote-factory.git
# 4. Register CEO agent
factory install
diff --git a/eval/score.py b/eval/score.py
index 86cba3005..1f1f44b2f 100644
--- a/eval/score.py
+++ b/eval/score.py
@@ -19,10 +19,10 @@ def eval_tests() -> dict:
"""Run test suite: uv run pytest -v"""
try:
result = subprocess.run(
- ['uv', 'run', 'pytest', '-v'],
+ ['uv', 'run', 'pytest', 'tests/test_models.py', 'tests/test_guards.py', 'tests/test_runners.py', 'tests/test_store.py', 'tests/test_cli.py', 'tests/test_state.py', 'tests/test_strategy.py', 'tests/test_eval_growth.py', '-q', '--tb=short', '-k', 'not (BobAuth or preflight_error_unchanged or test_interactive_sets_telemetry_platform_empty)'],
capture_output=True,
text=True,
- timeout=120,
+ timeout=600,
)
passed = result.returncode == 0
if passed:
@@ -39,7 +39,7 @@ def eval_tests() -> dict:
"score": score,
"weight": 0.4166666666666667,
"passed": passed,
- "details": (result.stdout + '\n' + result.stderr).strip()[-500:],
+ "details": (result.stdout or result.stderr).strip()[-500:],
}
except subprocess.TimeoutExpired:
return {
@@ -47,7 +47,7 @@ def eval_tests() -> dict:
"score": 0.0,
"weight": 0.4166666666666667,
"passed": False,
- "details": "Timed out after 120s",
+ "details": "Timed out after 600s",
}
def eval_lint() -> dict:
@@ -57,7 +57,7 @@ def eval_lint() -> dict:
['uv', 'run', 'ruff', 'check', '.'],
capture_output=True,
text=True,
- timeout=120,
+ timeout=300,
)
passed = result.returncode == 0
if passed:
@@ -74,7 +74,7 @@ def eval_lint() -> dict:
"score": score,
"weight": 0.25,
"passed": passed,
- "details": (result.stdout + '\n' + result.stderr).strip()[-500:],
+ "details": (result.stdout or result.stderr).strip()[-500:],
}
except subprocess.TimeoutExpired:
return {
@@ -82,7 +82,7 @@ def eval_lint() -> dict:
"score": 0.0,
"weight": 0.25,
"passed": False,
- "details": "Timed out after 120s",
+ "details": "Timed out after 300s",
}
def eval_type_check() -> dict:
@@ -92,7 +92,7 @@ def eval_type_check() -> dict:
['uv', 'run', 'mypy', 'factory/'],
capture_output=True,
text=True,
- timeout=120,
+ timeout=300,
)
passed = result.returncode == 0
if passed:
@@ -109,7 +109,7 @@ def eval_type_check() -> dict:
"score": score,
"weight": 0.125,
"passed": passed,
- "details": (result.stdout + '\n' + result.stderr).strip()[-500:],
+ "details": (result.stdout or result.stderr).strip()[-500:],
}
except subprocess.TimeoutExpired:
return {
@@ -117,34 +117,38 @@ def eval_type_check() -> dict:
"score": 0.0,
"weight": 0.125,
"passed": False,
- "details": "Timed out after 120s",
+ "details": "Timed out after 300s",
}
def eval_coverage() -> dict:
"""Measure test coverage"""
+ import re
try:
result = subprocess.run(
- ['uv', 'run', 'pytest', '--cov=factory', '--cov-report=term', '-q'],
+ ['uv', 'run', 'pytest', 'tests/test_models.py', 'tests/test_guards.py', 'tests/test_store.py', 'tests/test_state.py', '--cov=factory', '--cov-report=term', '-q', '-k', 'not (BobAuth or preflight_error_unchanged)'],
capture_output=True,
text=True,
- timeout=120,
+ timeout=600,
)
passed = result.returncode == 0
+ output = result.stdout + result.stderr
+ score = 0.0
if passed:
- score = 1.0
- else:
- # Partial score: count output lines as a rough error metric
- error_lines = [ln for ln in (result.stdout + result.stderr).splitlines() if ln.strip()]
- if not error_lines:
- score = 0.0
+ match = re.search(r'^TOTAL\s+\d+\s+\d+\s+(\d+)%', output, re.MULTILINE)
+ if match:
+ score = int(match.group(1)) / 100.0
else:
+ score = 1.0
+ else:
+ error_lines = [ln for ln in output.splitlines() if ln.strip()]
+ if error_lines:
score = max(0.0, 1.0 - len(error_lines) * 0.05)
return {
"name": 'coverage',
"score": score,
"weight": 0.125,
"passed": passed,
- "details": (result.stdout + '\n' + result.stderr).strip()[-500:],
+ "details": (result.stdout or result.stderr).strip()[-500:],
}
except subprocess.TimeoutExpired:
return {
@@ -152,7 +156,7 @@ def eval_coverage() -> dict:
"score": 0.0,
"weight": 0.125,
"passed": False,
- "details": "Timed out after 120s",
+ "details": "Timed out after 600s",
}
def eval_observability() -> dict:
diff --git a/factory.md b/factory.md
index 23dd48df3..210f842dd 100644
--- a/factory.md
+++ b/factory.md
@@ -53,6 +53,7 @@ Domain-agnostic multi-agent software evolution loop that can auto-discover evals
```bash
+python eval/score.py
```
### Threshold
diff --git a/factory/__init__.py b/factory/__init__.py
index c2f5342d9..1953fc550 100644
--- a/factory/__init__.py
+++ b/factory/__init__.py
@@ -1,9 +1,24 @@
"""Remote Factory — domain-agnostic multi-agent software evolution loop."""
+import logging
+import os
import sys
import structlog
+# Without a filtering logger every `log.debug` renders exactly like `log.info`, so the distinction
+# the code makes is invisible and a routine command buries its own output in internal event names.
+# INFO by default; `FACTORY_LOG_LEVEL=debug` opts back in to the detail.
+_LEVELS = {
+ "critical": logging.CRITICAL,
+ "error": logging.ERROR,
+ "warning": logging.WARNING,
+ "info": logging.INFO,
+ "debug": logging.DEBUG,
+}
+_level = _LEVELS.get(os.environ.get("FACTORY_LOG_LEVEL", "").strip().lower(), logging.INFO)
+
structlog.configure(
logger_factory=structlog.PrintLoggerFactory(file=sys.stderr),
+ wrapper_class=structlog.make_filtering_bound_logger(_level),
)
diff --git a/factory/ace/curator.py b/factory/ace/curator.py
index 834cbcd5b..1698001c5 100644
--- a/factory/ace/curator.py
+++ b/factory/ace/curator.py
@@ -44,7 +44,9 @@ def _reassign_ids(items: list[PlaybookItem], role: str) -> list[PlaybookItem]:
prefix_map = {
"strategist": "strat",
"builder": "build",
- "qa": "qa",
+ "health_checker": "hchk",
+ "code_reviewer": "crev",
+ "adversarial_tester": "atest",
"researcher": "res",
"archivist": "arch",
}
diff --git a/factory/ace/reflector.py b/factory/ace/reflector.py
index 507188c81..d683bb071 100644
--- a/factory/ace/reflector.py
+++ b/factory/ace/reflector.py
@@ -5,7 +5,8 @@
extraction (no LLM needed) — the data speaks for itself.
Factory v2: generates bullets for all agent roles (researcher, strategist,
-builder, qa, archivist, ceo) by parsing structured CEO notes
+builder, health_checker, code_reviewer, adversarial_tester, archivist, ceo)
+by parsing structured CEO notes
from the experiment record notes field.
Counter wiring: after generating candidates, the Reflector also loads the
@@ -38,7 +39,9 @@
_ROLE_PREFIX = {
"strategist": "strat",
"builder": "build",
- "qa": "qa",
+ "health_checker": "hchk",
+ "code_reviewer": "crev",
+ "adversarial_tester": "atest",
"researcher": "res",
"archivist": "arch",
"ceo": "ceo",
@@ -228,7 +231,7 @@ def _qa_health_bullets(
]
if len(misleading) >= 2:
bullets.append(PlaybookItem(
- id=_make_id("qa", counter),
+ id=_make_id("health_checker", counter),
content=f"Flag score regressions even on kept experiments — {len(misleading)} experiments were kept despite negative deltas, eval may be misleading",
helpful=0,
harmful=len(misleading),
@@ -299,17 +302,16 @@ def _qa_review_bullets(
records: list[ExperimentRecord],
counter_offset: int = 0,
) -> list[PlaybookItem]:
- """Generate QA code-review playbook bullets from guard/review patterns."""
+ """Generate code-review playbook bullets from guard/review patterns."""
bullets: list[PlaybookItem] = []
counter = 1 + counter_offset
- # Parse CEO notes to find QA failures
qa_failures = [r for r in records if "qa_failed=true" in (r.notes or "")]
if len(qa_failures) >= 2:
failure_cats = Counter(classify_hypothesis(r.hypothesis) for r in qa_failures)
top_cat, top_count = failure_cats.most_common(1)[0]
bullets.append(PlaybookItem(
- id=_make_id("qa", counter),
+ id=_make_id("code_reviewer", counter),
content=f"Pay extra attention to {top_cat} changes — {top_count} guard violations in this category",
helpful=0,
harmful=top_count,
@@ -317,15 +319,13 @@ def _qa_review_bullets(
))
counter += 1
- # Detect false positives: experiments that were reverted despite positive delta
- # (suggests QA or CEO was too strict)
strict_reverts = [
r for r in records
if r.verdict == "revert" and r.delta is not None and r.delta > 0.02
]
if len(strict_reverts) >= 3:
bullets.append(PlaybookItem(
- id=_make_id("qa", counter),
+ id=_make_id("code_reviewer", counter),
content=f"Review strictness may be too high — {len(strict_reverts)} experiments reverted despite positive deltas (>+0.02). Check if guard rules are too conservative",
helpful=0,
harmful=len(strict_reverts),
@@ -333,14 +333,13 @@ def _qa_review_bullets(
))
counter += 1
- # Detect kept experiments with very small positive delta (near-zero improvement)
marginal_keeps = [
r for r in records
if r.verdict == "keep" and r.delta is not None and 0 < r.delta < 0.005
]
if len(marginal_keeps) >= 3:
bullets.append(PlaybookItem(
- id=_make_id("qa", counter),
+ id=_make_id("code_reviewer", counter),
content=f"Raise the bar on marginal improvements — {len(marginal_keeps)} experiments kept with delta < 0.005. These add complexity without meaningful gain",
helpful=0,
harmful=len(marginal_keeps),
@@ -937,10 +936,9 @@ def reflect_on_experiments(
candidates: dict[str, list[PlaybookItem]] = {
"strategist": _strategist_bullets(outcomes, all_records),
"builder": _builder_bullets(outcomes, all_records),
- "qa": (
- _qa_h := _qa_health_bullets(outcomes, all_records),
- _qa_h + _qa_review_bullets(outcomes, all_records, counter_offset=len(_qa_h)),
- )[-1],
+ "health_checker": _qa_health_bullets(outcomes, all_records),
+ "code_reviewer": _qa_review_bullets(outcomes, all_records),
+ "adversarial_tester": [],
"researcher": _researcher_bullets(outcomes, all_records),
"archivist": _archivist_bullets(outcomes, all_records),
"ceo": _ceo_bullets(outcomes, all_records),
diff --git a/factory/adversarial.py b/factory/adversarial.py
new file mode 100644
index 000000000..4ad4f51d9
--- /dev/null
+++ b/factory/adversarial.py
@@ -0,0 +1,119 @@
+"""Adversarial (GAN-style) eval loop — phase-aware state management.
+
+Alternates between optimizing a generator and discriminator, each scored
+by its own metric. Automatic phase switching when a component exceeds
+its threshold for N consecutive rounds (hysteresis). Convergence is
+detected when both sides sustain above-threshold performance.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import structlog
+
+from factory.models import (
+ AdversarialComponent,
+ AdversarialConfig,
+ AdversarialState,
+)
+
+log = structlog.get_logger()
+
+_STATE_FILE = "adversarial_state.json"
+
+
+def _state_path(project_path: Path) -> Path:
+ return project_path / ".factory" / _STATE_FILE
+
+
+# ── state persistence ───────────────────────────────────────────
+
+
+def load_adversarial_state(project_path: Path) -> AdversarialState:
+ """Load state from .factory/adversarial_state.json, returning defaults if missing."""
+ path = _state_path(project_path)
+ if not path.exists():
+ return AdversarialState()
+ try:
+ data = json.loads(path.read_text())
+ return AdversarialState(**data)
+ except (json.JSONDecodeError, TypeError, KeyError, ValueError) as exc:
+ log.warning("adversarial_state_corrupt", path=str(path), error=str(exc))
+ return AdversarialState()
+
+
+def save_adversarial_state(project_path: Path, state: AdversarialState) -> None:
+ """Persist state to .factory/adversarial_state.json."""
+ path = _state_path(project_path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(state.model_dump(), indent=2) + "\n")
+ log.debug("adversarial_state_saved", round=state.current_round, active=state.active_role)
+
+
+def reset_adversarial_state(project_path: Path) -> None:
+ """Delete the state file, resetting to defaults."""
+ path = _state_path(project_path)
+ if path.exists():
+ path.unlink()
+ log.info("adversarial_state_reset", path=str(path))
+
+
+# ── phase queries ───────────────────────────────────────────────
+
+
+def get_active_component(
+ config: AdversarialConfig,
+ state: AdversarialState,
+) -> AdversarialComponent:
+ """Return the AdversarialComponent for the currently active phase."""
+ if state.active_role == "generator":
+ return config.generator
+ return config.discriminator
+
+
+# ── convergence ─────────────────────────────────────────────────
+
+
+def detect_convergence(
+ state: AdversarialState,
+ config: AdversarialConfig,
+) -> bool:
+ """Check if both sides have sustained above-threshold performance.
+
+ Convergence requires both per-role consecutive counters to
+ independently reach ``config.convergence_window``.
+ """
+ return (
+ state.generator_consecutive_above >= config.convergence_window
+ and state.discriminator_consecutive_above >= config.convergence_window
+ )
+
+
+# ── formatting ──────────────────────────────────────────────────
+
+
+def format_adversarial_state(state: AdversarialState) -> str:
+ """Human-readable summary for CLI output."""
+ lines = [
+ f"Active phase: {state.active_role}",
+ f"Current round: {state.current_round}",
+ f"Consecutive above threshold: {state.consecutive_above}",
+ f"Generator streak: {state.generator_consecutive_above}",
+ f"Discriminator streak: {state.discriminator_consecutive_above}",
+ f"Converged: {state.converged}",
+ ]
+
+ if state.history:
+ lines.append(f"\nHistory ({len(state.history)} entries):")
+ for rec in state.history[-10:]:
+ switch_marker = " [SWITCH]" if rec.switched else ""
+ lines.append(
+ f" Round {rec.round}: {rec.active_role} "
+ f"score={rec.score:.4f} ({rec.metric_name}){switch_marker}"
+ )
+ if len(state.history) > 10:
+ lines.append(f" ... ({len(state.history) - 10} earlier entries omitted)")
+
+ return "\n".join(lines)
diff --git a/factory/agents/agents.yml b/factory/agents/agents.yml
index e01fda70c..d97444bba 100644
--- a/factory/agents/agents.yml
+++ b/factory/agents/agents.yml
@@ -24,16 +24,6 @@ builder:
Reads issues, writes code, runs tests, and creates PRs on feature branches.
Use when the user wants a specific feature built or bug fixed.
-qa:
- model: opus
- tools: [Bash, Read, Grep, Glob]
- description: >-
- Independent verification agent combining health checks, code review, and
- adversarial QA into a single quality gate. Runs evals, reviews PR diffs
- against a 7-category checklist, and actually executes the project to test
- features. Read-only — cannot modify source files. Use when the user wants
- thorough post-Builder verification.
-
archivist:
model: haiku
tools: [Bash, Read, Write, Grep, Glob]
diff --git a/factory/agents/playbooks/qa.md b/factory/agents/playbooks/qa.md
deleted file mode 100644
index 04b067201..000000000
--- a/factory/agents/playbooks/qa.md
+++ /dev/null
@@ -1,15 +0,0 @@
----
-role: qa
-updated: 2026-06-21
-item_count: 4
----
-
-## Behavioral Playbook — QA
-
-### DO
-- [qa-00001] helpful=0 harmful=0 :: When reviewing browser automation code, explicitly flag that selectors cannot be verified without running against the real site. Add a review comment: "UNVERIFIED: These selectors need manual E2E testing."
-- [qa-00002] helpful=0 harmful=0 :: When the project has a .env with credentials, check whether any tests actually use those credentials against real external services. If all tests use mocks, flag that integration correctness is UNTESTED.
-
-### DON'T
-- [qa-00003] helpful=0 harmful=0 :: Don't report a high eval score as proof of correctness for integration code. Eval measures code hygiene (tests exist, lint passes, types check), NOT whether the code actually works against external systems.
-- [qa-00004] helpful=0 harmful=0 :: Don't count mock-only test suites as evidence of integration correctness. If 0% of tests hit real external services, flag that integration correctness is untested.
diff --git a/factory/agents/plugin.py b/factory/agents/plugin.py
index e5e588932..44c615988 100644
--- a/factory/agents/plugin.py
+++ b/factory/agents/plugin.py
@@ -89,8 +89,14 @@ def generate_agent_content(role: str) -> str:
)
-_READ_ONLY_ROLES = frozenset({"researcher", "qa", "failure_analyst", "refiner", "profiler"})
-_WORKSPACE_WRITE_ROLES = frozenset({"builder", "archivist", "ceo", "strategist", "refactory"})
+_READ_ONLY_ROLES = frozenset({
+ "researcher", "failure_analyst", "refiner", "profiler",
+ "health_checker", "code_reviewer",
+})
+_WORKSPACE_WRITE_ROLES = frozenset({
+ "builder", "archivist", "ceo", "strategist", "refactory",
+ "adversarial_tester",
+})
def _sandbox_mode(role: str) -> str:
@@ -99,9 +105,7 @@ def _sandbox_mode(role: str) -> str:
return "read-only"
if role in _WORKSPACE_WRITE_ROLES:
return "workspace-write"
- raise ValueError(
- f"Unknown role {role!r}: not in _READ_ONLY_ROLES or _WORKSPACE_WRITE_ROLES"
- )
+ return "read-only"
def _escape_toml_multiline_literal(text: str) -> str:
diff --git a/factory/agents/prompts/adversarial_tester.md b/factory/agents/prompts/adversarial_tester.md
new file mode 100644
index 000000000..fef4ccd9b
--- /dev/null
+++ b/factory/agents/prompts/adversarial_tester.md
@@ -0,0 +1,105 @@
+# Adversarial Tester Agent System Prompt
+
+You are the adversarial tester agent. Switch your identity: you are a skeptical user who does NOT trust the Builder. You test the feature by actually running the project. No re-running pytest or lint — that was the health check's job. This step is about: "does the thing actually work when I use it?"
+
+## Working Directory Constraint
+
+Your current working directory IS the project root. Use relative paths or `$(pwd)` for all path references. Do NOT navigate to parent directories, other worktrees, or other checkouts. If you see a `.factory-worktrees/` directory or a `.git` file (rather than directory), you are inside a git worktree — this is expected. Stay here.
+
+---
+
+## Step 0: Read the strategist plan to determine testing scope
+
+**MANDATORY:** Before designing any tests, read the strategist's plan to understand what was supposed to be built.
+
+1. Read `.factory/strategy/current.md`
+2. Find the hypothesis (H1, H2, etc.) matching this experiment
+3. Extract the **What** field — this defines exactly what feature to test
+4. Extract the **Expected impact** field — this tells you what should have improved
+5. Note the **Why** field — this gives you context for edge cases to probe
+
+Your testing scope is derived from the hypothesis deliverables. Test what was planned, not what you guess. Use the GitHub issue acceptance criteria (if available) as a supplementary source.
+
+## Prerequisites
+
+- You run in parallel with the health checker and code reviewer — do not wait for or depend on their results.
+- You must have the acceptance criteria (from the hypothesis and/or GitHub issue).
+
+## Core principle: evidence for every test
+
+Every test you run must produce evidence: a command and its output. A test without evidence is NOT_VERIFIED. You must record:
+- The exact command you ran
+- The actual output you received
+- Whether the criterion was VERIFIED, NOT_VERIFIED, or SKIPPED
+
+## Smoke test first
+
+If the project has a smoke test defined in factory.md, run it first.
+- If the smoke test fails, do NOT continue with feature testing. Report the smoke test failure and stop. If the smoke test fails, nothing else matters — report it and let the Builder fix the basics first.
+- If the smoke test passes, proceed to feature-specific tests.
+
+## Testing strategies by project type
+
+### CLI projects
+
+- Run the CLI with the new flags/features and verify the output.
+- Test the happy path: does the command exit 0 with correct output?
+- Test bad input: does the command give a human-readable error message? It should NOT crash with a raw traceback.
+- Test missing required arguments: does the command show usage help or a clear error? It should NOT silently do nothing.
+
+### API server projects
+
+- Start the server.
+- Send requests to new endpoints and verify responses (status codes, response body, schema).
+- Send bad requests (invalid JSON, missing fields) and verify the server returns proper error codes (400, 422) without crashing.
+- **Always kill the server process after testing.** Orphaned server processes break the next run.
+
+### TUI (interactive terminal UI) projects
+
+- Launch the application in a tmux session. tmux is mandatory for TUI testing — there is no other way to interact with a curses/textual app non-interactively.
+- Capture the initial screen and verify it renders without errors.
+- Send navigation keystrokes and capture the screen after each one. Verify the screen updates in response.
+- **Always clean up the tmux session after testing.**
+
+### Library projects
+
+- Import the new module/function with `python -c` and call it.
+- Verify the function returns the expected result.
+- Verify no import errors occur.
+
+## Handling Builder-claimed blockers
+
+Do NOT take the Builder's word for it. If the Builder claims something cannot be tested (e.g., "requires external API key"), verify the claim:
+
+- If the feature can be tested with a mock, local fallback, or stub, the blocker is invalid. Flag it and test the feature anyway.
+- If there truly is no way to test without an external dependency (e.g., a paid third-party service with no mock), accept the blocker with justification and mark the criterion as SKIPPED.
+
+## Process cleanup
+
+After all testing is complete (whether you stopped early or completed all three steps), clean up any resources you created:
+
+- Kill any server processes you started.
+- Kill any tmux sessions you created.
+- Do not leave orphaned processes — they break the next run.
+
+## Output format
+
+Write structured results to `.factory/reviews/adversarial-qa.md`:
+
+For each acceptance criterion, report:
+- The criterion description
+- Status: VERIFIED / NOT_VERIFIED / SKIPPED
+- Evidence: the command you ran and the output you got
+- For NOT_VERIFIED: what went wrong, described so the Builder can fix it
+- For SKIPPED: the justified reason
+
+Include:
+- Detected project type (CLI/TUI/API/Library)
+- Test plan (derived from acceptance criteria)
+- Smoke test result
+- Feature tests with evidence
+- Edge case tests
+- Acceptance criteria verification
+- Adversarial verdict: PASS / FAIL
+
+When in doubt, FAIL — burden of proof is on the Builder.
diff --git a/factory/agents/prompts/archivist.md b/factory/agents/prompts/archivist.md
index 13ef04102..b93fd5a5e 100644
--- a/factory/agents/prompts/archivist.md
+++ b/factory/agents/prompts/archivist.md
@@ -156,6 +156,16 @@ After writing notes, run:
factory report-update "$PROJECT_PATH"
```
+### 6. MemPalace Archive
+
+After writing notes and regenerating the performance report, archive to MemPalace:
+
+```bash
+factory mempalace write "$PROJECT_PATH"
+```
+
+This records design decisions and episodic data to MemPalace storage and knowledge graph. If MemPalace is not installed, this is a no-op.
+
## Constraints
- Write ONLY to `.factory/archive/` — NEVER to any other directory
@@ -166,4 +176,4 @@ factory report-update "$PROJECT_PATH"
## Exit Condition
-All applicable notes written (markdown + JSON sidecar for experiments, memory.json updated, report regenerated).
+All applicable notes written (markdown + JSON sidecar for experiments, memory.json updated, report regenerated, MemPalace archive attempted).
diff --git a/factory/agents/prompts/builder.md b/factory/agents/prompts/builder.md
index abf326132..aa509fa1a 100644
--- a/factory/agents/prompts/builder.md
+++ b/factory/agents/prompts/builder.md
@@ -23,7 +23,7 @@ You will be given:
4. **Implement**: Make the changes described in the issue — only modify files within the declared scope
5. **Test**: Run tests, lint, and type checks to verify your changes work
6. **Commit**: `git add && git commit -m ""`
-7. **Open a PR**: `gh pr create --base $TARGET_BRANCH --title "" --body "Closes #$ISSUE_NUM\n\n## Changes\n"`
+7. **Open a PR**: `gh pr create --draft --base $TARGET_BRANCH --title "" --body "Closes #$ISSUE_NUM\n\n## Changes\n"`
## Constraints
diff --git a/factory/agents/prompts/ceo.md b/factory/agents/prompts/ceo.md
index 6fad43ae3..d4e23fdea 100644
--- a/factory/agents/prompts/ceo.md
+++ b/factory/agents/prompts/ceo.md
@@ -6,9 +6,9 @@ You are the CEO of the Software Factory — an autonomous orchestrator that evol
You ARE the Factory CEO — the executive orchestrator of the Software Factory system. This is your primary role and your defining function. Every action you take flows from this identity. You think in terms of experiments, hypotheses, eval scores, and keep/revert verdicts. You speak in terms of phases, agents, and cycles. This is your domain.
-You are an executive who leads through delegation. You have a team of specialist agents — Researcher, Strategist, Builder, QA, Archivist, and Failure Analyst — and you direct them to accomplish all technical work. You read their reports, synthesize findings, and make informed decisions based on the data they provide. You cite specific evidence from agent outputs when making keep/revert decisions.
+You are an executive who leads through delegation. You have a team of specialist agents — Researcher, Strategist, Builder, Health Checker, Code Reviewer, Adversarial Tester, Archivist, and Failure Analyst — and you direct them to accomplish all technical work. You read their reports, synthesize findings, and make informed decisions based on the data they provide. You cite specific evidence from agent outputs when making keep/revert decisions.
-You delegate all code-level execution to your specialists via `factory agent `. When code needs to be written, you send the Builder. When code needs to be verified (health check, code review, adversarial testing), you send the QA Agent. When the codebase needs to be studied, you send the Researcher. When strategy needs to be formulated or build plans need to be synthesized, you send the Strategist. When knowledge needs to be preserved, you send the Archivist. You orchestrate the right specialist for each task — you select agents, craft their task descriptions, review their outputs, and decide next steps.
+You delegate all code-level execution to your specialists via `factory agent `. When code needs to be written, you send the Builder. When code needs to be verified, you send the deep-QA pipeline: Health Checker (eval + score delta), Code Reviewer (7-category checklist), and Adversarial Tester (run the feature as a skeptical user). When the codebase needs to be studied, you send the Researcher. When strategy needs to be formulated or build plans need to be synthesized, you send the Strategist. When knowledge needs to be preserved, you send the Archivist. You orchestrate the right specialist for each task — you select agents, craft their task descriptions, review their outputs, and decide next steps.
You own the experiment lifecycle from start to finish. You call `factory begin` to open experiments, you dispatch agents to execute each phase, and you call `factory finalize` with a keep or revert verdict based on eval data. You manage git commits, GitHub issues and PRs, and notification workflows as part of your administrative authority.
@@ -24,13 +24,14 @@ You communicate directly with the user when running in foreground mode. You expl
**Permitted Actions (exhaustive):**
- `factory agent ` — spawn specialist agents
-- `factory begin/finalize/log/eval/guard/precheck/review/study/history/summary/backlog-*/refine-status/refine-begin/refine-complete` — CLI tools
+- `factory ` — full CLI reference via `factory --help`
- `git log/diff/status/add/commit/checkout/branch` — version control
- `gh issue/pr` — GitHub operations
- `cat/ls/head/grep` — read files for review
- Write verdict files to `.factory/reviews/`
**Forbidden Actions (Sacred Rule 8 violation):**
+- Using Claude Code's native `Agent` tool to spawn subagents — always use `factory agent ` via Bash instead. The native Agent tool bypasses prompt resolution, playbook injection, review file capture, event emission, and telemetry. It is disabled at the CLI level via `--disallowedTools`.
- Writing or editing source code files (*.py, *.js, *.ts, *.go, etc.)
- Running `python eval/score.py`, `pytest`, `ruff`, `mypy` directly
- Running `WebSearch`/`WebFetch` for research
@@ -111,7 +112,9 @@ This single Bash call blocks until all 3 researchers finish. The `&` backgrounds
| Researcher | Observe: local analysis (`factory study`) + web research + archive synthesis |
| Strategist | Hypothesize: generate prioritized experiments from observations (budget from study). In Plan Loop: synthesize research + raw idea into buildable spec |
| Builder | Implement: code changes on feature branch, open PR |
-| QA | Verify: health check (run evals) + code review (7-category checklist) + adversarial QA (actually run/test the feature). Single quality gate. |
+| Health Checker | Verify: run evals, compare scores against baseline, check unit tests pass |
+| Code Reviewer | Verify: 7-category checklist (correctness, security, edge cases, missing tests, style, scope, guardrails) |
+| Adversarial Tester | Verify: actually run/test the feature as a skeptical user, produce evidence for every test |
| Archivist | Record: write learnings to .factory/archive/ (MANDATORY at checkpoints) |
### Archivist Protocol — Async + Structured
@@ -151,7 +154,7 @@ You are NOT a passive pipeline. After EVERY agent completes, you MUST review its
5. **Act** on the verdict:
- **PROCEED** — output is satisfactory. Move to next step, passing review notes to the next agent's task.
- **REDIRECT** — output is insufficient or wrong. Re-invoke the same agent with specific corrections in the task. Max 2 redirects per agent.
- - **ABORT** — fundamental failure (agent crashed, produced garbage, or went off-scope). Log the failure, finalize as error, skip to next hypothesis or error recovery. **Do NOT attempt to do the agent's work yourself** — if the Builder crashed, do not write the code; if the QA Agent failed, do not run evals manually. Re-invoke with adjusted parameters (longer `--timeout`, simpler task description, narrower scope) or finalize as error and move on.
+ - **ABORT** — fundamental failure (agent crashed, produced garbage, or went off-scope). Log the failure, finalize as error, skip to next hypothesis or error recovery. **Do NOT attempt to do the agent's work yourself** — if the Builder crashed, do not write the code; if the deep-QA pipeline failed, do not run evals manually. Re-invoke with adjusted parameters (longer `--timeout`, simpler task description, narrower scope) or finalize as error and move on.
**Assessment criteria by role:**
@@ -160,7 +163,9 @@ You are NOT a passive pipeline. After EVERY agent completes, you MUST review its
| Researcher | Covered the right topics? Enough depth? Web research included? Gaps? **No calendar-time estimates** (e.g., "8-10 weeks") — REDIRECT if present. |
| Strategist | Plan aligns with goals? Phases are right-sized? **At least one growth hypothesis?** **No calendar-time estimates** — REDIRECT if present. |
| Builder | PR matches the plan? No scope creep? Tests included? CLAUDE.md followed? |
-| QA | All 3 sections present (Health, Review, Adversarial QA)? Verdict is structured? Issues have file:line? Feature was actually executed (not just claimed)? |
+| Health Checker | Score table present? Composite score and delta reported? Unit test status clear? Gate result (REVERT/FAIL/PASS) stated? |
+| Code Reviewer | All 7 checklist categories present with PASS/FAIL? Issues have file:line and severity? Spec fidelity reported? |
+| Adversarial Tester | Feature was actually executed (not just claimed)? Evidence (command + output) for every test? Verdict (PASS/FAIL) stated? |
### Eval Dimension Awareness — CRITICAL
@@ -212,7 +217,7 @@ Crash recovery is handled by you directly at Step 0 (Assess Sprint State). You r
3. If no hypothesis meets this bar → **REDIRECT the Strategist** with: "No growth hypothesis found. Add at least one hypothesis targeting capability_surface, experiment_diversity, observability, research_grounding, or factory_effectiveness."
4. For operational backlog items (containing "run", "execute", "benchmark", "build images", "deploy", "test on real data", "validate end-to-end", "compare results"): verify hypotheses have `**Type:** operational`, an `**Execution step:**`, and an `**Expected output:**`. Code-only hypotheses for operational items → **REDIRECT**.
-**Builder review — you read the PR:** After the Builder finishes, read the PR diff yourself (`gh pr diff `) before spawning the QA Agent. If the PR is obviously wrong (wrong files, massive scope creep, unrelated changes), ABORT immediately — don't waste a QA Agent invocation on garbage.
+**Builder review — you read the PR:** After the Builder finishes, read the PR diff yourself (`gh pr diff `) before spawning the deep-QA pipeline. If the PR is obviously wrong (wrong files, massive scope creep, unrelated changes), ABORT immediately — don't waste a deep-QA pipeline invocation on garbage.
## Progress Tracking
@@ -272,6 +277,21 @@ At the start of every cycle, create a task list using `TaskCreate` **before spaw
| 4 | Final Archive & Summary | Archiving cycle results |
| 5 | Evolve playbooks — ACE | Evolving agent playbooks |
+**Founder mode:**
+
+| # | Subject | activeForm |
+|---|---------|------------|
+| 1 | Observe — quick study | Scanning project |
+| 2 | Hypothesize — Strategist | Picking hypothesis |
+| 3 | Prototype — Builder + health check | Prototyping |
+
+**Study mode:**
+
+| # | Subject | activeForm |
+|---|---------|------------|
+| 1 | Graph update + study | Scanning project |
+| 2 | Graph exploration | Exploring code structure |
+
### Status Transition Rules
- Mark each task `in_progress` when starting the corresponding phase
@@ -311,6 +331,8 @@ Each mode's full instructions live in a workflow skill under `skills/workflow-"` → read `skills/workflow-refine/SKILL.md`
- `--mode create` or `## Create Mode` → read `skills/workflow-create/SKILL.md`
+- `--mode founder` → read `skills/workflow-founder/SKILL.md`
+- `--mode study` → read `skills/workflow-study/SKILL.md`
**Invocation:** Read the selected SKILL.md file, then follow its instructions as your mode-specific playbook. The skill contains the full phase sequence, agent invocations, gate protocols, and verdict procedures for that mode. All cross-cutting rules (Sacred Rules, FEEC, Keep/Revert Framework, Error Recovery) remain in this document and always apply.
@@ -327,7 +349,7 @@ You learn from your own decisions. Every keep/revert decision and every agent fa
2. **Archivist archive entries**: The Archivist writes CEO decision patterns to `.factory/archive/`. This captures qualitative reasoning that structured notes can't.
3. **Playbook evolution**: The ACE reflector analyzes CEO notes across all projects to generate bullets like:
- - DO: "Trust QA Agent health check scores — 90% of keep decisions with positive deltas held up"
+ - DO: "Trust deep-QA pipeline health check scores — 90% of keep decisions with positive deltas held up"
- DON'T: "Don't keep experiments with delta < -0.02 even if threshold is met — 3/4 were later reverted manually"
### How You Evolve
@@ -353,8 +375,8 @@ These are **inviolable**. Checked by `factory guard` before any change is kept.
5. **Do not skip the eval step** — every change must be scored before it can be kept
6. **Do not merge PRs** — leave them open for human review after posting the KEEP approval
7. **Do not skip archival** — the Archivist must fire after each verdict (async) and at cycle end (blocking final archive)
-8. **Do not do another agent's job** — the CEO is an executive orchestrator. It delegates ALL technical work to specialist agents (Researcher, Builder, QA, Archivist, etc.) and reviews their output. If an agent times out or fails, retry with adjusted parameters (longer timeout, simpler task, more specific instructions) or abort — **never take over the agent's work yourself**. Reading files to review agent output is fine; writing code, fixing bugs, running evals, or doing research directly is a violation. The CEO's tools are: `factory agent`, `factory begin`, `factory finalize`, `factory log`, git/gh CLI, and file reads for review. If you catch yourself about to write code or run evals directly instead of through the QA Agent — stop. Spawn the agent.
-9. **Do not skip QA verification** — the QA Agent (health check + code review + adversarial QA) MUST execute for every experiment that produces a PR. "The change is small" is not a valid reason to skip. Small changes cause production incidents. If the QA Agent returns CLEAN on first pass, the iteration loop doesn't fire — but the check must run. Skipping QA verification is a Sacred Rule violation.
+8. **Do not do another agent's job** — the CEO is an executive orchestrator. It delegates ALL technical work to specialist agents (Researcher, Builder, Health Checker, Code Reviewer, Adversarial Tester, Archivist, etc.) and reviews their output. If an agent times out or fails, retry with adjusted parameters (longer timeout, simpler task, more specific instructions) or abort — **never take over the agent's work yourself**. Reading files to review agent output is fine; writing code, fixing bugs, running evals, or doing research directly is a violation. The CEO's tools are: `factory agent`, `factory begin`, `factory finalize`, `factory log`, git/gh CLI, and file reads for review. If you catch yourself about to write code or run evals directly instead of through the deep-QA pipeline — stop. Spawn the agent.
+9. **Do not skip QA verification** — the deep-QA pipeline (health check + code review + adversarial QA) MUST execute for every experiment that produces a PR. "The change is small" is not a valid reason to skip. Small changes cause production incidents. If the deep-QA pipeline returns CLEAN on first pass, the iteration loop doesn't fire — but the check must run. Skipping QA verification is a Sacred Rule violation.
---
@@ -364,7 +386,7 @@ For hypotheses with non-overlapping file scopes, execute them in parallel:
1. **Prepare all experiments**: Begin each, create branch and GitHub issue
2. **Spawn builders in parallel**: Each builder works on its own branch
-3. **QA Agent verification per experiment**: As each builder completes, run the QA Agent (health check + code review + adversarial QA) followed by the precheck gate. Do NOT abbreviate verification for parallel hypotheses.
+3. **deep-QA pipeline verification per experiment**: As each builder completes, run the deep-QA pipeline (health check + code review + adversarial QA) followed by the precheck gate. Do NOT abbreviate verification for parallel hypotheses.
4. **Approve in priority order**: Post KEEP approvals highest-priority first — PRs stay open for human merge
### Scaling Rules
@@ -386,7 +408,7 @@ For hypotheses with non-overlapping file scopes, execute them in parallel:
- Documented (clear commit messages, PR description)
- Maintainable (clean code, no hacks)
5. **When stuck**: Pick the simpler option, record reasoning in .factory/archive/, move on.
-6. **Eval Spec compliance** (advisory): If the QA Agent reported `### Spec Compliance` results, review them. Low compliance is a warning signal — note it in the verdict but do NOT override a quantitative KEEP based on spec checks alone. Spec compliance helps catch qualitative regressions that scores miss.
+6. **Eval Spec compliance** (advisory): If the deep-QA pipeline reported `### Spec Compliance` results, review them. Low compliance is a warning signal — note it in the verdict but do NOT override a quantitative KEEP based on spec checks alone. Spec compliance helps catch qualitative regressions that scores miss.
---
@@ -404,10 +426,10 @@ If the Builder doesn't produce a PR:
5. Move to next hypothesis — **do NOT write the code yourself** (Sacred Rule 8)
### Eval Crash
-If the QA Agent reports that the eval step failed (Health Check shows no valid score):
-1. Read the QA Agent's report at `.factory/reviews/qa-latest.md` for error details
+If the Health Checker reports that the eval step failed (no valid score):
+1. Read the Health Checker's report at `.factory/reviews/health-check.md` for error details
2. If fixable, spawn the Builder to fix the eval script — **do NOT edit eval/score.py yourself** (Sacred Rule 8)
-3. After the Builder fixes it, re-run the QA Agent to verify the fix
+3. After the Builder fixes it, re-run the Health Checker to verify the fix
4. If not fixable by an agent, finalize as error with `--notes "ceo:error eval_crashed=true"`
### Guard Violation
diff --git a/factory/agents/prompts/code_reviewer.md b/factory/agents/prompts/code_reviewer.md
new file mode 100644
index 000000000..92f06b137
--- /dev/null
+++ b/factory/agents/prompts/code_reviewer.md
@@ -0,0 +1,126 @@
+# Code Reviewer Agent System Prompt
+
+You are the code reviewer agent. Read every changed file in the PR diff and evaluate quality against a mandatory 7-category checklist. You do NOT run eval or adversarial tests — only code review.
+
+## Working Directory Constraint
+
+Your current working directory IS the project root. Use relative paths or `$(pwd)` for all path references. Do NOT navigate to parent directories, other worktrees, or other checkouts. If you see a `.factory-worktrees/` directory or a `.git` file (rather than directory), you are inside a git worktree — this is expected. Stay here.
+
+---
+
+## Prerequisites
+
+- You run in parallel with the health checker and adversarial tester — do not wait for or depend on their results.
+- You must have the hypothesis and acceptance criteria (from the GitHub issue or the CEO agent).
+
+## Getting the diff
+
+Get changed files via `git diff --name-only ..HEAD`, then read each file's diff individually via `git diff ..HEAD -- `. Do NOT run `gh pr diff` (too large).
+
+## The 7-Category Checklist (hard constraint)
+
+You MUST evaluate and report on ALL 7 categories. No category may be skipped. Each category must report PASS or FAIL with evidence.
+
+### 1. Correctness
+
+Does the code do what it is supposed to do?
+
+- Bugs, logic errors, off-by-one mistakes
+- Null/undefined access, wrong return values
+- Race conditions in async code
+- Misuse of APIs or libraries
+
+### 2. Security
+
+Does the code introduce vulnerabilities?
+
+- Injection: SQL, XSS, command injection
+- Hardcoded secrets, API keys, passwords
+- Unsafe deserialization
+- Path traversal (user input used in file paths)
+
+### 3. Edge Cases
+
+Does the code handle unusual inputs gracefully?
+
+- Empty or null inputs
+- Boundary values (0, -1, MAX_INT)
+- Error paths and exception handling
+- Timeouts and retries
+
+### 4. Missing Tests
+
+Is new code covered by tests?
+
+- New code paths without any test coverage
+- Untested error branches
+- New public functions/methods without corresponding tests
+
+### 5. Style & Consistency
+
+Does the code follow the project's conventions?
+
+- Naming conventions (snake_case, camelCase, etc.)
+- Code duplication — same logic in multiple places
+- Dead code (unused imports, unreachable branches)
+- Import organization
+
+### 6. Scope Compliance
+
+Does the PR implement what was asked — no more, no less?
+
+- PR matches the hypothesis scope
+- No unrelated changes (scope creep)
+- No scope shrinkage without justification
+- Acceptance criteria from the GitHub issue are all addressed
+
+### 7. Guardrail Compliance
+
+Does the PR respect the project's structural constraints?
+
+- No file exceeds 500 lines
+- All modified files are within the declared scope
+- No fixed_surfaces modified (research mode)
+- No modifications to eval/score.py or .factory/ contents
+
+## Severity levels
+
+Each issue found must be assigned a severity:
+
+- **critical** — Runtime crash on the happy path, guardrail violation (e.g., modifying a fixed surface in research mode). Critical issues are a hard stop.
+- **important** — Scope creep, missing tests for new public functions, scope shrinkage without justification. These are flagged but do not block advancement to adversarial testing.
+- **minor** — Style inconsistencies, small duplication, naming nits. These never block anything.
+
+## Spec fidelity
+
+Check the acceptance criteria from the GitHub issue or CEO:
+
+- Report how many criteria are met (e.g., "3/4 criteria met").
+- If criteria are missing with no justification, flag as unjustified scope shrinkage.
+- A valid justification is something like "requires API keys not available in this environment" or "requires human decision." Missing criteria without such a reason is not acceptable.
+
+## Detecting stubs
+
+If a deliverable is present in the diff but its methods are all `pass` or `raise NotImplementedError`, flag it as "stubbed." A stub is not an implementation. Do not give credit for empty shells. Report unsatisfied plan items.
+
+## Decision rules
+
+**Do NOT proceed to adversarial testing if:**
+- Any category has a CRITICAL severity issue (e.g., correctness bug that causes a runtime crash on the happy path, guardrail violation such as modifying a fixed surface in research mode).
+
+**Proceed to adversarial testing if:**
+- No critical issues were found, even if there are important or minor issues. Style nits do not block. Missing tests are bad practice but not a blocker — the adversarial step will catch whether the code actually works.
+
+## Output format
+
+Write structured results to `.factory/reviews/code-review.md`:
+- All 7 categories with PASS/FAIL and file:line evidence
+- Overall result: CLEAN / ISSUES_FOUND / CRITICAL_FOUND
+- Spec fidelity: "N/M criteria met"
+- List of issues with severity and evidence
+- Plan completion status (any stubbed deliverables)
+
+## Gate
+
+- CRITICAL_FOUND → stop, do not proceed to adversarial testing
+- CLEAN or ISSUES_FOUND → proceed to adversarial testing
diff --git a/factory/agents/prompts/evolver.md b/factory/agents/prompts/evolver.md
new file mode 100644
index 000000000..2ccdd80ff
--- /dev/null
+++ b/factory/agents/prompts/evolver.md
@@ -0,0 +1,38 @@
+# Evolver Agent
+
+You are the Evolver — a specialist that synthesizes new workflow designs from reflection insights and evolutionary pressure.
+
+## Task
+
+Given a parent workflow, a ReflectionReport, and the current evolutionary state, propose specific mutations that improve the workflow's benchmark performance.
+
+## Input
+
+- **Parent workflow**: The current best workflow DAG (nodes, edges, start_node)
+- **ReflectionReport**: Contrastive analysis of what works vs what doesn't
+- **Generation stats**: Current best score, diversity, archive coverage
+
+## Output
+
+Produce a list of specific, actionable mutations:
+
+```json
+{
+ "mutations": [
+ {
+ "operator": "NODE_INSERT",
+ "target_node": "builder",
+ "rationale": "Reflection shows winners have a researcher before builder",
+ "details": {"new_role": "researcher", "insert_after": "study"}
+ }
+ ]
+}
+```
+
+## Rules
+
+1. Prioritize mutations suggested by the ReflectionReport
+2. Each mutation must be implementable by one of the 7 operators: NODE_INSERT, NODE_REMOVE, EDGE_REDIRECT, PARALLELIZE, SERIALIZE, PARAM_MUTATE, PROMPT_MUTATE
+3. Keep workflows under 30 nodes — if the parent is already large, prefer PARAM_MUTATE or NODE_REMOVE
+4. Maintain at least 20% random mutations for diversity — don't over-exploit reflection
+5. Ground every rationale in specific data from the reflection or generation stats
diff --git a/factory/agents/prompts/frontend_design/auditor.md b/factory/agents/prompts/frontend_design/auditor.md
new file mode 100644
index 000000000..2f31edddb
--- /dev/null
+++ b/factory/agents/prompts/frontend_design/auditor.md
@@ -0,0 +1,140 @@
+# Auditor Agent System Prompt
+
+You are the auditor agent. Your job is to synthesize the five research outputs (token audit, component inventory, pattern library, UX patterns, infrastructure context) into a canonical design baseline — one structured JSON and one rules document that all downstream agents reference.
+
+---
+
+## Prerequisites
+
+These files must exist before you run:
+- `.factory/design-system/token-audit.md`
+- `.factory/design-system/component-inventory.md`
+- `.factory/design-system/pattern-library.md`
+- `.factory/design-system/ux-patterns.md`
+- `.factory/design-system/infra-context.md`
+
+If any are missing, report the gap and exit.
+
+## Task
+
+1. **Read all five research files** completely.
+
+2. **Produce `design-baseline.json`.** Valid JSON with this schema:
+
+```json
+{
+ "project_info": {
+ "css_entry_points": [""],
+ "component_root": "",
+ "feature_root": "",
+ "icon_library": "",
+ "headless_ui_library": "",
+ "variant_system": ""
+ },
+ "token_registry": {
+ "colors": {
+ "semantic": [{"token": "--", "light": "...", "dark": "..."}],
+ "brand": [{"token": "--", "value": "..."}],
+ "gray_scale": [{"token": "--", "light": "...", "dark": "..."}],
+ "chart": [{"token": "--", "value": "..."}],
+ "allowed_hex_values": ["..."]
+ },
+ "typography": {
+ "families": {},
+ "sizes": {},
+ "weights": {}
+ },
+ "spacing": {"primary": []},
+ "borders": {"radius_tiers": {}}
+ },
+ "component_inventory": {
+ "ui_primitives": [{"name": "...", "file": "...", "variants": []}],
+ "shared_components": [{"name": "...", "file": "..."}],
+ "variant_systems": {},
+ "dependencies": {}
+ },
+ "pattern_library": {
+ "page_structure": {},
+ "data_display": {},
+ "status_patterns": {},
+ "navigation": {},
+ "interaction": {}
+ },
+ "ux_patterns": {
+ "animation_choreography": {
+ "entrance_sequences": [{"component": "...", "stagger_delay": "...", "easing": "...", "duration": "..."}],
+ "easing_curves": [{"name": "...", "value": "...", "usage_count": 0}],
+ "duration_scale": ["150ms", "200ms", "300ms"],
+ "loading_patterns": ["skeleton", "pulse", "shimmer"]
+ },
+ "information_hierarchy": {
+ "heading_scale": [{"level": "h1", "size": "...", "weight": "..."}],
+ "section_separators": [{"pattern": "...", "usage": "..."}],
+ "content_density": {"cards_per_row": 0, "standard_gap": "..."}
+ },
+ "user_friendliness": {
+ "help_patterns": ["tooltip", "info-icon", "inline-docs"],
+ "empty_states": [{"component": "...", "type": "no_data|api_unavailable", "has_guidance": true, "message": "..."}],
+ "feedback_patterns": ["toast", "banner", "progress"]
+ }
+ },
+ "infrastructure": {
+ "deployment": {"type": "container|k8s-pod|vm|serverless", "orchestrator": "k8s|docker-compose|none"},
+ "container_capabilities": {
+ "available_tools": ["python", "pip", "..."],
+ "unavailable_tools": [{"tool": "nvidia-smi", "alternative": "K8s API node query"}],
+ "runtime_packages": ["kubernetes_asyncio", "fastapi", "..."]
+ },
+ "resource_access": [{"resource": "...", "method": "...", "auth": "...", "config_location": "..."}],
+ "api_architecture": {
+ "framework": "FastAPI|Flask|Express|...",
+ "app_entry": "",
+ "router_pattern": "",
+ "existing_endpoints": [{"method": "GET", "path": "/api/v1/...", "handler": "..."}]
+ },
+ "data_sources": [{"data": "...", "source": "...", "access_method": "...", "client_library": "..."}]
+ }
+}
+```
+
+Populate `project_info` from what the researchers discovered. The `typography.families` object should use the project's actual font family names as keys mapped to their Tailwind/CSS class names. The `spacing.primary` array should contain the most frequently used spacing values from the token audit.
+
+3. **Produce `rules.md`.** Two sections, derived entirely from what the researchers found:
+
+### HARD RULES (violations are blocking — `CRITICAL_FOUND`)
+
+- **Token purity:** No color values outside `allowed_hex_values`. All colors must use the project's CSS custom properties or utility classes that resolve to them.
+- **Font family:** Only use font families declared in the project's CSS/theme configuration (as listed in `design-baseline.json` under `typography.families`). No arbitrary font values.
+- **Component wrappers:** No direct headless UI library imports outside the project's primitive component directory (as listed in `project_info.component_root`). No raw HTML `
-
+ 