bench.yml

name: Bench

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
    # tagpr のリリース PR では run を作らない。
    #
    # tagpr は GITHUB_TOKEN で push するため、リリース PR の pull_request run は
    # GitHub の承認ゲートに掛かって action_required のまま一度も実行されない。PR を
    # マージするとその保留 run は failure として終わる。結果、リリースのたびに
    # 「一度も走っていない赤」が 3 本並び、本物の失敗が埋もれる。
    #
    # 承認ゲートはジョブ評価より前なので job の if: では止められず、pull_request の
    # branches: は base しか絞れないので head が tagpr-from-* かでも除外できない。
    # run の生成自体を止められるのは paths フィルタだけ。
    #
    # 下の 3 つは tagpr のリリース PR が触る全ファイル。paths-ignore は「変更ファイルが
    # 全部ここに一致したときだけスキップ」なので、通常の PR（src/ や pnpm-lock.yaml を
    # 含む）は従来どおり走る。逆に言うと、この 3 つだけを変える人手の PR も
    # スキップされる — CHANGELOG / バージョン番号だけの変更なので許容している。
    #
    # 完全な解は tagpr に PAT / GitHub App のトークンを渡して承認ゲート自体を
    # 回避すること。そうすればこの除外は不要になる。
    paths-ignore:
      - "CHANGELOG.md"
      - "package.json"
      - "src-tauri/tauri.conf.json"
  workflow_dispatch:
    inputs:
      mode:
        description: 'Bench mode (live / dry-run). Live requires SAKURA_AI_API_KEY secret.'
        required: false
        default: 'dry-run'
        type: choice
        options:
          - dry-run
          - live
      n:
        description: 'Independent samples per run (default: 3 in live, 1 in dry-run).'
        required: false
        default: ''
        type: string

# sticky-pull-request-comment needs write access on PRs to post the delta comment.
permissions:
  contents: read
  pull-requests: write

jobs:
  # 既存 (Phase μ-1): wiki pipeline benchmark
  bench:
    name: bench / wiki-pipeline
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0   # bench:compare main で git show が必要

      - uses: pnpm/action-setup@v4
        with:
          version: 9

      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: pnpm

      - run: pnpm install --frozen-lockfile

      - name: Bench Tier 1 unit tests
        run: pnpm vitest run bench/metrics.test.ts

      - name: Bench Tier 2 — full run (dry-run on PR, live on manual dispatch)
        env:
          # workflow_dispatch で live を選んだ場合のみ SAKURA_AI_API_KEY を渡す。
          # PR からは secret に触れないので自動的に dry-run になる。
          SAKURA_AI_API_KEY: ${{ github.event.inputs.mode == 'live' && secrets.SAKURA_AI_API_KEY || '' }}
          BENCH_MODE: ${{ github.event.inputs.mode || 'dry-run' }}
          BENCH_N: ${{ github.event.inputs.n || '' }}
          BENCH_PROFILE: baseline
          BENCH_OUTPUT: bench/latest-baseline.json
        run: pnpm bench:run

      - name: Compare against main
        if: github.event_name == 'pull_request'
        continue-on-error: true   # bench delta は block ではなく warning（spec §5 CI 統合）
        run: pnpm bench:compare main > bench/delta.md

      - name: Post delta comment
        if: github.event_name == 'pull_request' && hashFiles('bench/delta.md') != ''
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          path: bench/delta.md
          header: bench-delta

      - name: Upload bench output
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: bench-${{ github.event.inputs.mode || 'dry-run' }}-${{ github.run_number }}
          path: |
            bench/latest-baseline.json
            bench/delta.md
          if-no-files-found: ignore

  # Phase μ-3: adversarial probes (safety / robustness)
  # LLM call なしの dry-run 完結なので毎 PR で走らせる。
  adversarial:
    name: bench / adversarial
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: pnpm/action-setup@v4
        with:
          version: 9

      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: pnpm

      - run: pnpm install --frozen-lockfile

      - name: Run adversarial probes
        run: pnpm bench:adversarial | tee bench/adversarial-stdout.log

      - name: Generate adversarial summary
        if: always()
        run: |
          {
            echo "## Adversarial probes"
            echo ""
            echo "<details><summary>Click to expand probe results</summary>"
            echo ""
            echo '```'
            cat bench/adversarial-stdout.log
            echo '```'
            echo ""
            echo "</details>"
          } > bench/adversarial-summary.md

      - name: Post adversarial summary
        if: github.event_name == 'pull_request' && hashFiles('bench/adversarial-summary.md') != ''
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          path: bench/adversarial-summary.md
          header: bench-adversarial

      - name: Upload adversarial artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: bench-adversarial-${{ github.run_number }}
          path: |
            bench/results/adversarial-latest.json
            bench/adversarial-stdout.log
            bench/adversarial-summary.md
          if-no-files-found: ignore

  # Phase μ-3: migration fixture tests (data-loss は CI block)
  migration:
    name: bench / migration
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: pnpm/action-setup@v4
        with:
          version: 9

      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: pnpm

      - run: pnpm install --frozen-lockfile

      - name: Run migration fixtures
        env:
          # データロス系の失敗は CI を block する。これは破壊的変更検知の最後の砦。
          BENCH_MIGRATION_STRICT: 'true'
        run: pnpm test:migration | tee bench/migration-stdout.log

      - name: Generate migration summary
        if: always()
        run: |
          {
            echo "## Migration fixtures"
            echo ""
            echo "<details><summary>Click to expand migration results</summary>"
            echo ""
            echo '```'
            cat bench/migration-stdout.log
            echo '```'
            echo ""
            echo "</details>"
          } > bench/migration-summary.md

      - name: Post migration summary
        if: github.event_name == 'pull_request' && hashFiles('bench/migration-summary.md') != ''
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          path: bench/migration-summary.md
          header: bench-migration

      - name: Upload migration artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: bench-migration-${{ github.run_number }}
          path: |
            bench/results/migration-latest.json
            bench/migration-stdout.log
            bench/migration-summary.md
          if-no-files-found: ignore

  # Phase μ-3: performance regression test (warning, not block)
  performance:
    name: bench / performance
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: pnpm/action-setup@v4
        with:
          version: 9

      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: pnpm

      - run: pnpm install --frozen-lockfile

      - name: Run performance regression test
        # noisy なので block せず warning のみ。20%以上の悪化は summary に視覚化される。
        run: pnpm bench:performance | tee bench/performance-stdout.log

      - name: Generate performance summary
        if: always()
        run: |
          {
            echo "## Performance regression"
            echo ""
            echo "<details><summary>Click to expand performance results</summary>"
            echo ""
            echo '```'
            cat bench/performance-stdout.log
            echo '```'
            echo ""
            echo "</details>"
          } > bench/performance-summary.md

      - name: Post performance summary
        if: github.event_name == 'pull_request' && hashFiles('bench/performance-summary.md') != ''
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          path: bench/performance-summary.md
          header: bench-performance

      - name: Upload performance artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: bench-performance-${{ github.run_number }}
          path: |
            bench/results/performance-latest.json
            bench/performance/baseline.json
            bench/performance-stdout.log
            bench/performance-summary.md
          if-no-files-found: ignore