pycon26-ftp-challenge/.github/workflows/evaluate.yml at main · mpage/pycon26-ftp-challenge · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
name: Evaluate Submission

on:
  pull_request:
    types:
      - opened
      - synchronize
  workflow_dispatch:
    inputs:
      pr_number:
        description: 'PR number to evaluate'
        required: true
        type: number

jobs:
  resolve_pr:
    runs-on: ubuntu-latest
    permissions:
      pull-requests: read
    outputs:
      pr_number: ${{ steps.resolve.outputs.pr_number }}
      pr_author: ${{ steps.resolve.outputs.pr_author }}
      head_sha: ${{ steps.resolve.outputs.head_sha }}
      html_url: ${{ steps.resolve.outputs.html_url }}
    steps:
      - name: Resolve PR details
        id: resolve
        uses: actions/github-script@v7
        with:
          script: |
            let pr;
            if (context.eventName === 'workflow_dispatch') {
              const { data } = await github.rest.pulls.get({
                owner: context.repo.owner,
                repo: context.repo.repo,
                pull_number: ${{ inputs.pr_number || 0 }},
              });
              pr = data;
            } else {
              pr = context.payload.pull_request;
            }
            core.setOutput('pr_number', pr.number);
            core.setOutput('pr_author', pr.user.login);
            core.setOutput('head_sha', pr.head.sha);
            core.setOutput('html_url', pr.html_url);

  validate_pr:
    needs: resolve_pr
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: read
    steps:
      - name: Validate changed files
        uses: actions/github-script@v7
        with:
          script: |
            const prAuthor = '${{ needs.resolve_pr.outputs.pr_author }}';
            const prNumber = ${{ needs.resolve_pr.outputs.pr_number }};
            const expectedFile = `submissions/${prAuthor}.py`;

            const files = await github.paginate(github.rest.pulls.listFiles, {
              owner: context.repo.owner,
              repo: context.repo.repo,
              pull_number: prNumber,
              per_page: 100,
            });

            const changedFiles = files.map((f) => f.filename);

            if (changedFiles.length !== 1) {
              core.setFailed(
                `PR must modify exactly one file (${expectedFile}), but it modifies ${changedFiles.length}: ${changedFiles.join(", ")}`,
              );
              return;
            }

            if (changedFiles[0] !== expectedFile) {
              core.setFailed(
                `PR must only modify ${expectedFile}, but it modifies ${changedFiles[0]}`,
              );
              return;
            }

            core.info(`PR correctly modifies only ${expectedFile}`);

  run_harness:
    needs: [resolve_pr, validate_pr]
    runs-on: self-hosted
    permissions:
      contents: read
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.resolve_pr.outputs.head_sha }}

      - name: Install uv
        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b

      - name: Precompile Python files
        run: |
          uv run --python 3.14.3t python -m compileall challenge/ submissions/

      - name: Save PR metadata
        run: |
          mkdir -p ${{ runner.temp }}/eval-results
          cat <<'OUTER' > ${{ runner.temp }}/eval-results/pr_metadata.json
          {
            "pr_number": ${{ needs.resolve_pr.outputs.pr_number }},
            "pr_author": "${{ needs.resolve_pr.outputs.pr_author }}",
            "head_sha": "${{ needs.resolve_pr.outputs.head_sha }}",
            "html_url": "${{ needs.resolve_pr.outputs.html_url }}"
          }
          OUTER

      - name: Run Harness
        timeout-minutes: 10
        run: |
          set -o errexit
          set -o pipefail
          uv run --python 3.14.3t challenge/harness.py --num-trials 3 --json submissions/${{ needs.resolve_pr.outputs.pr_author }}.py graphs | tee ${{ runner.temp }}/eval-results/results.json

      - name: Upload results
        uses: actions/upload-artifact@v4
        with:
          name: eval-results
          path: ${{ runner.temp }}/eval-results/