fix: mock ONNX embedding function in mempalace tests to eliminate flaky CI downloads

.containerignore

@@ -0,0 +1,20 @@
+# Build context for containers/factory/Containerfile.
+#
+# The image needs pyproject.toml, uv.lock, README.md, factory/ and skills/ — nothing else. The
+# repository as a whole is several hundred megabytes, most of it history, benchmark data and
+# virtualenvs, and every byte of it is streamed to the engine on each build.
+*
+!pyproject.toml
+!uv.lock
+!README.md
+!factory/
+!skills/
+
+# Re-excluded inside the directories that are included: caches and virtualenvs are large, are
+# rebuilt inside the image anyway, and an arm64 .venv copied into an amd64 image is actively wrong.
+**/__pycache__/
+**/*.pyc
+**/.venv/
+**/.pytest_cache/
+**/.ruff_cache/
+**/.mypy_cache/

.github/workflows/benchmark.yml

@@ -3,8 +3,6 @@ name: Benchmark CI
 on:
   push:
     branches: [main]
-  schedule:
-    - cron: '17 6 * * 1-5'
   workflow_dispatch:
     inputs:
       benchmark:
@@ -16,6 +14,11 @@ on:
           - featurebench
           - terminalbench
           - programbench
+          - legacybench
+          - harborindex
+          - tomswe
+          - salitrap
+          - devopsgym
           - all
       instance_id:
         description: 'Instance ID (leave default for smoke test)'
@@ -78,6 +81,26 @@ jobs:
             solver: factory
             default_instance: 'cmatrix'
             enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'programbench' || inputs.benchmark == 'all') && (inputs.solver != 'claude-code') }}
+          - benchmark: legacybench
+            solver: factory
+            default_instance: '1907c2-c-debug-legacy-buddy-fix'
+            enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'legacybench' || inputs.benchmark == 'all') && (inputs.solver != 'claude-code') }}
+          - benchmark: harborindex
+            solver: factory
+            default_instance: 'bix-filter-chip-variants'
+            enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'harborindex' || inputs.benchmark == 'all') && (inputs.solver != 'claude-code') }}
+          - benchmark: tomswe
+            solver: factory
+            default_instance: 'sympy__sympy-20590'
+            enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'tomswe' || inputs.benchmark == 'all') && (inputs.solver != 'claude-code') }}
+          - benchmark: salitrap
+            solver: factory
+            default_instance: 'salitrap-001'
+            enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'salitrap' || inputs.benchmark == 'all') && (inputs.solver != 'claude-code') }}
+          - benchmark: devopsgym
+            solver: factory
+            default_instance: 'build-maven-dependency-resolution'
+            enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'devopsgym' || inputs.benchmark == 'all') && (inputs.solver != 'claude-code') }}
           # Claude Code solver entries — enabled on schedule, release, or workflow_dispatch with matching benchmark+solver
           - benchmark: swebench
             solver: claude-code
@@ -95,6 +118,26 @@ jobs:
             solver: claude-code
             default_instance: 'cmatrix'
             enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'programbench' || inputs.benchmark == 'all') && (inputs.solver == 'claude-code' || inputs.solver == 'both') }}
+          - benchmark: legacybench
+            solver: claude-code
+            default_instance: '1907c2-c-debug-legacy-buddy-fix'
+            enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'legacybench' || inputs.benchmark == 'all') && (inputs.solver == 'claude-code' || inputs.solver == 'both') }}
+          - benchmark: harborindex
+            solver: claude-code
+            default_instance: 'bix-filter-chip-variants'
+            enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'harborindex' || inputs.benchmark == 'all') && (inputs.solver == 'claude-code' || inputs.solver == 'both') }}
+          - benchmark: tomswe
+            solver: claude-code
+            default_instance: 'sympy__sympy-20590'
+            enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'tomswe' || inputs.benchmark == 'all') && (inputs.solver == 'claude-code' || inputs.solver == 'both') }}
+          - benchmark: salitrap
+            solver: claude-code
+            default_instance: 'salitrap-001'
+            enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'salitrap' || inputs.benchmark == 'all') && (inputs.solver == 'claude-code' || inputs.solver == 'both') }}
+          - benchmark: devopsgym
+            solver: claude-code
+            default_instance: 'build-maven-dependency-resolution'
+            enabled: ${{ github.event_name == 'schedule' || github.event_name == 'push' || ((github.event_name != 'workflow_dispatch') || inputs.benchmark == 'devopsgym' || inputs.benchmark == 'all') && (inputs.solver == 'claude-code' || inputs.solver == 'both') }}
 
     steps:
       - name: Skip if not enabled
@@ -136,7 +179,7 @@ jobs:
       - name: Install Factory CLI
         if: steps.gate.outputs.run == 'true' && matrix.solver == 'factory'
         run: |
-          uv tool install 'remote-factory @ git+https://github.com/akashgit/remote-factory.git'
+          uv tool install 'remote-factory[telemetry] @ git+https://github.com/akashgit/remote-factory.git'
           export PATH="$HOME/.local/bin:$PATH"
           echo "PATH=$HOME/.local/bin:$PATH" >> "$GITHUB_ENV"
           factory --help > /dev/null
@@ -161,7 +204,7 @@ jobs:
           if [ '${{ github.event_name }}' = 'workflow_dispatch' ]; then
             echo "value=$BENCHMARK_TIMEOUT" >> "$GITHUB_OUTPUT"
           elif [ '${{ github.event_name }}' = 'schedule' ]; then
-            echo 'value=3600' >> "$GITHUB_OUTPUT"
+            echo 'value=7200' >> "$GITHUB_OUTPUT"
           else
             echo 'value=7200' >> "$GITHUB_OUTPUT"
           fi
@@ -178,17 +221,15 @@ jobs:
           CLAUDE_CODE_DISABLE_ADAPTIVE_THINKING: "1"
           MAX_THINKING_TOKENS: "128000"
           CLAUDE_CODE_EFFORT_LEVEL: "XHIGH"
+          LANGFUSE_HOST: ${{ vars.LANGFUSE_BENCH_HOST }}
+          LANGFUSE_BASE_URL: ${{ vars.LANGFUSE_BENCH_HOST }}
+          LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_BENCH_PUBLIC_KEY }}
+          LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_BENCH_SECRET_KEY }}
+          FACTORY_GIT_REF: ${{ github.sha }}
         run: |
           chmod +x benchmarks/run.sh benchmarks/lib.sh benchmarks/run-*.sh
           benchmarks/run.sh ${{ matrix.benchmark }} ${{ steps.config.outputs.instance }} --timeout ${{ steps.timeout.outputs.value }} --solver ${{ matrix.solver }}
 
-      - name: Upload results
-        uses: actions/upload-artifact@v4
-        if: always() && steps.gate.outputs.run == 'true'
-        with:
-          name: benchmark-results-${{ matrix.benchmark }}-${{ matrix.solver }}
-          path: benchmarks/results/
-
       - name: Print summary
         if: always() && steps.gate.outputs.run == 'true'
         run: |
@@ -203,6 +244,47 @@ jobs:
             done
           fi
 
+      - name: Analyze failures
+        if: always() && steps.gate.outputs.run == 'true'
+        continue-on-error: true
+        env:
+          CLAUDE_CODE_USE_VERTEX: "1"
+          ANTHROPIC_VERTEX_PROJECT_ID: ${{ secrets.GCP_PROJECT }}
+          CLOUD_ML_REGION: ${{ secrets.GCP_REGION }}
+          LANGFUSE_HOST: ${{ vars.LANGFUSE_BENCH_HOST }}
+          LANGFUSE_BASE_URL: ${{ vars.LANGFUSE_BENCH_HOST }}
+          LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_BENCH_PUBLIC_KEY }}
+          LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_BENCH_SECRET_KEY }}
+        run: |
+          pip install python-dotenv requests --quiet
+          for result_file in benchmarks/results/*.json; do
+            [ -f "$result_file" ] || continue
+            resolved=$(python3 -c "import json; print(json.load(open('$result_file')).get('resolved', False))")
+            if [ "$resolved" = "False" ]; then
+              echo "Analyzing failure: $result_file"
+              base="$(basename "$result_file" .json)"
+              python3 scripts/langfuse/analyze_failure.py "$result_file" \
+                --summary --output "benchmarks/results/${base}-summary.md" \
+                || echo "Summary failed for $result_file (non-fatal)"
+              python3 scripts/langfuse/analyze_failure.py "$result_file" \
+                --output "benchmarks/results/${base}-analysis.md" \
+                || echo "Analysis failed for $result_file (non-fatal)"
+              for suffix in summary analysis; do
+                f="benchmarks/results/${base}-${suffix}.md"
+                if [ -f "$f" ]; then
+                  cat "$f" >> $GITHUB_STEP_SUMMARY
+                fi
+              done
+            fi
+          done
+
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        if: always() && steps.gate.outputs.run == 'true'
+        with:
+          name: benchmark-results-${{ matrix.benchmark }}-${{ matrix.solver }}
+          path: benchmarks/results/
+
   report:
     needs: benchmark
     if: always()
@@ -253,6 +335,23 @@ jobs:
               print(f'Processing: {f}', file=sys.stderr)
               with open(f) as fh:
                   data = json.load(fh)
+
+              base = os.path.splitext(os.path.basename(f))[0]
+              summary_path = os.path.join(results_dir, base + '-summary.md')
+              analysis_path = os.path.join(results_dir, base + '-analysis.md')
+
+              if os.path.isfile(summary_path):
+                  with open(summary_path) as sf:
+                      data['trace_summary'] = sf.read().strip()
+              if os.path.isfile(analysis_path):
+                  with open(analysis_path) as af:
+                      data['trace_analysis'] = af.read().strip()
+
+              trace_id = (data.get('details') or {}).get('trace_id', '')
+              langfuse_host = os.environ.get('LANGFUSE_HOST', '')
+              if trace_id and langfuse_host:
+                  data['trace_url'] = f'{langfuse_host}/trace/{trace_id}'
+
               data['run_id'] = os.environ.get('RUN_ID', '')
               data['commit'] = os.environ.get('COMMIT', '')
               data['ref'] = os.environ.get('REF', '')
@@ -269,6 +368,7 @@ jobs:
           REF="${{ github.ref }}" \
           RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
           TRIGGER="${{ github.event_name }}" \
+          LANGFUSE_HOST="${{ vars.LANGFUSE_BENCH_HOST }}" \
           python3 /tmp/append_results.py
 
           echo 'DEBUG: results dir contents:'
@@ -354,6 +454,30 @@ jobs:
               body += '_No benchmark results found._\n';
             }
 
+            const summaryFiles = new Set();
+            for (const file of files) {
+              if (file.endsWith('-summary.md')) {
+                summaryFiles.add(file);
+                const benchName = file.replace('-summary.md', '').replace(/^\d{8}T\d{6}Z-/, '');
+                const summaryText = fs.readFileSync('results/' + file, 'utf8').trim();
+                const analysisFile = file.replace('-summary.md', '-analysis.md');
+                body += '**' + benchName + ':** ' + summaryText + '\n';
+                if (fs.existsSync('results/' + analysisFile)) {
+                  const analysis = fs.readFileSync('results/' + analysisFile, 'utf8');
+                  body += '<details><summary>Detailed analysis</summary>\n\n' + analysis + '\n</details>\n\n';
+                }
+              }
+            }
+            for (const file of files) {
+              if (file.endsWith('-analysis.md') && !summaryFiles.has(file.replace('-analysis.md', '-summary.md'))) {
+                const analysis = fs.readFileSync('results/' + file, 'utf8');
+                const benchName = file.replace('-analysis.md', '').replace(/^\d{8}T\d{6}Z-/, '');
+                body += '<details><summary>Failure Analysis: ' + benchName + '</summary>\n\n';
+                body += analysis;
+                body += '\n</details>\n\n';
+              }
+            }
+
             const historyPath = 'benchmark-data/results.jsonl';
             let baselines = {};
             if (fs.existsSync(historyPath)) {

.github/workflows/ceo-review.yml

@@ -14,9 +14,9 @@ jobs:
     if: >-
       github.event.issue.pull_request &&
       contains(github.event.comment.body, '@ceo-review') &&
-      contains(fromJSON('["akashgit", "xukai92", "colehurwitz", "shivchander", "osilkin98", "gx-ai-architect"]'), github.event.comment.user.login)
+      contains(fromJSON('["akashgit", "xukai92", "colehurwitz", "shivchander", "osilkin98", "gx-ai-architect", "RobotSail", "mihirathale98", "lukeinglis", "nehamalepati", "abhi1092", "s-akhtar-baig"]'), github.event.comment.user.login)
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 120
 
     steps:
       - name: React with eyes
@@ -34,11 +34,18 @@ jobs:
         id: pr
         run: echo "number=${{ github.event.issue.number }}" >> "$GITHUB_OUTPUT"
 
-      - name: Checkout PR merge ref
+      - name: Checkout default branch (trusted factory)
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          path: factory-trusted
+
+      - name: Checkout PR merge ref (code under review)
         uses: actions/checkout@v4
         with:
           ref: refs/pull/${{ steps.pr.outputs.number }}/merge
           fetch-depth: 0
+          path: pr-code
 
       - name: Authenticate to Google Cloud
         uses: google-github-actions/auth@v2
@@ -49,7 +56,10 @@ jobs:
         uses: astral-sh/setup-uv@v4
 
       - name: Install factory
-        run: uv sync
+        working-directory: factory-trusted
+        run: |
+          uv sync --extra telemetry
+          uv tool install -e .
 
       - name: Set up Node.js
         uses: actions/setup-node@v4
@@ -66,13 +76,18 @@ jobs:
           claude --version
 
       - name: Run CEO review
+        working-directory: factory-trusted
         env:
           CLAUDE_CODE_USE_VERTEX: "1"
           ANTHROPIC_VERTEX_PROJECT_ID: ${{ secrets.GCP_PROJECT }}
           CLOUD_ML_REGION: ${{ secrets.GCP_REGION }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          FACTORY_MODEL: "claude-opus-4-6[1m]"
+          LANGFUSE_HOST: ${{ secrets.LANGFUSE_HOST }}
+          LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
+          LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
         run: |
-          uv run factory ceo . --mode review --pr ${{ steps.pr.outputs.number }} --headless
+          factory ceo ${{ github.workspace }}/pr-code --mode deep-qa --pr ${{ steps.pr.outputs.number }} --headless
 
       - name: Approve PR if verdict is KEEP
         if: success()

.github/workflows/ci.yml

@@ -111,11 +111,23 @@ jobs:
       - name: Set up Python
         run: uv python install 3.12
       - name: Install dependencies
-        run: uv sync --all-groups
+        run: |
+          uv sync --all-groups
+          uv tool install -e .
       - name: Ruff check
         run: uv run ruff check .
       - name: Mypy
         run: uv run mypy factory/
+      - name: Lint contributed workflows
+        run: factory workflow lint-contributed
       - name: Check plugin agents in sync
         if: hashFiles('agents/') != ''
         run: uv run python scripts/sync_agents.py --check
+      - name: Check no generated workflow skills committed
+        run: |
+          tracked=$(git ls-files 'skills/workflow-*/SKILL.md' 'skills/workflow-*/SKILL.annotations.yaml')
+          if [ -n "$tracked" ]; then
+            echo 'ERROR: Generated workflow skills should not be committed'
+            echo "$tracked"
+            exit 1
+          fi