smriti/.github/workflows/perf-bench.yml at main · zero8dotdev/smriti · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
name: Perf Bench

concurrency:
  group: perf-${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  pull_request:
    branches: [main, dev]
    paths:
      - "src/**"
      - "qmd/src/**"
      - "scripts/bench-*.ts"
      - "bench/**"
      - ".github/workflows/perf-bench.yml"
  push:
    branches: [main, dev, "feature/**"]
    paths:
      - "src/**"
      - "qmd/src/**"
      - "scripts/bench-*.ts"
      - "bench/**"
      - ".github/workflows/perf-bench.yml"

jobs:
  bench:
    name: Bench / ci-small
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: write

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Setup Bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: latest

      - name: Install dependencies
        run: bun install

      - name: Run benchmark (no-llm)
        run: bun run scripts/bench-qmd.ts --profile ci-small --out bench/results/ci-small.json --no-llm

      - name: Run repeated benchmark (ci-small)
        run: bun run scripts/bench-qmd-repeat.ts --profiles ci-small --runs 3 --out bench/results/repeat-summary.json

      - name: Compare against baseline (non-blocking)
        run: bun run scripts/bench-compare.ts --baseline bench/baseline.ci-small.json --current bench/results/ci-small.json --threshold 0.2

      - name: Generate scorecard markdown
        run: bun run bench:scorecard > bench/results/scorecard.md

      - name: Add scorecard to run summary
        run: |
          echo "## Benchmark Scorecard (ci-small)" >> "$GITHUB_STEP_SUMMARY"
          echo "" >> "$GITHUB_STEP_SUMMARY"
          cat bench/results/scorecard.md >> "$GITHUB_STEP_SUMMARY"

      - name: Upsert sticky PR comment
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require("fs");
            const body = fs.readFileSync("bench/results/scorecard.md", "utf8");
            const marker = "<!-- bench-scorecard -->";
            const fullBody = `${marker}
            ## Benchmark Scorecard (ci-small)

            ${body}`;
            const { owner, repo } = context.repo;
            const issue_number = context.payload.pull_request.number;
            const comments = await github.paginate(github.rest.issues.listComments, {
              owner,
              repo,
              issue_number,
              per_page: 100,
            });
            const existing = comments.find((c) => c.body && c.body.includes(marker));
            if (existing) {
              await github.rest.issues.updateComment({
                owner,
                repo,
                comment_id: existing.id,
                body: fullBody,
              });
            } else {
              await github.rest.issues.createComment({
                owner,
                repo,
                issue_number,
                body: fullBody,
              });
            }

      - name: Upload benchmark artifact
        uses: actions/upload-artifact@v4
        with:
          name: bench-ci-small
          path: |
            bench/results/ci-small.json
            bench/results/repeat-summary.json
            bench/results/scorecard.md