Add files via upload #46
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Score Submission | |
| on: | |
| pull_request_target: | |
| paths: | |
| - 'submissions/inbox/**' | |
| jobs: | |
| score: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Checkout base repo (has decrypt scripts, validation, etc.) | |
| - uses: actions/checkout@v4 | |
| # Fetch ONLY the submitted .enc file from the PR fork | |
| - name: Checkout PR submission file | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| path: pr-submission | |
| sparse-checkout: submissions/inbox | |
| sparse-checkout-cone-mode: true | |
| - name: Copy submission into workspace | |
| run: | | |
| echo "=== PR submission contents ===" | |
| find pr-submission/submissions/inbox -type f 2>/dev/null | |
| cp -r pr-submission/submissions/inbox/* submissions/inbox/ 2>/dev/null || true | |
| rm -rf pr-submission | |
| echo "=== Workspace submissions/inbox contents ===" | |
| find submissions/inbox -type f 2>/dev/null | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install dependencies | |
| run: pip install pandas scikit-learn cryptography | |
| - name: Detect team and decrypt | |
| id: team | |
| env: | |
| SUBMISSION_PRIVATE_KEY: ${{ secrets.SUBMISSION_PRIVATE_KEY }} | |
| run: | | |
| ENC=$(find submissions/inbox -name "*.enc" | head -1) | |
| if [ -z "$ENC" ]; then | |
| echo "::error::No .enc file found in submissions/inbox/" | |
| exit 1 | |
| fi | |
| TEAM=$(echo "$ENC" | sed 's|submissions/inbox/||' | cut -d'/' -f1) | |
| # If team is just the filename (no subfolder), derive from PR branch | |
| if [[ "$TEAM" == *.enc ]]; then | |
| TEAM=$(echo "${{ github.head_ref }}" | sed 's|submission/||' | sed 's|inbox/||' | sed 's|/.*||') | |
| fi | |
| python encryption/decrypt.py "$ENC" /tmp/predictions.csv | |
| echo "name=$TEAM" >> "$GITHUB_OUTPUT" | |
| - name: Enforce one-submission policy | |
| run: | | |
| TEAM="${{ steps.team.outputs.name }}" | |
| if grep -qi ",$TEAM," leaderboard/leaderboard.csv 2>/dev/null; then | |
| echo "::error::Team '$TEAM' has already submitted." | |
| exit 1 | |
| fi | |
| - name: Validate submission format | |
| run: python competition/validate_submission.py /tmp/predictions.csv data/public/test_data.csv | |
| - name: Evaluate | |
| id: eval | |
| run: | | |
| echo "${{ secrets.TEST_LABELS_CSV }}" > /tmp/test_labels.csv | |
| OUTPUT=$(python competition/evaluate.py /tmp/predictions.csv /tmp/test_labels.csv) | |
| echo "$OUTPUT" | |
| F1=$(echo "$OUTPUT" | grep MACRO_F1 | cut -d= -f2) | |
| AUC=$(echo "$OUTPUT" | grep AUROC | cut -d= -f2) | |
| echo "f1=$F1" >> "$GITHUB_OUTPUT" | |
| echo "auc=$AUC" >> "$GITHUB_OUTPUT" | |
| - name: Read optional meta.yaml | |
| id: meta | |
| run: | | |
| # Find meta.yaml anywhere in submissions/inbox | |
| echo "=== All files in submissions/inbox ===" | |
| find submissions/inbox -type f | |
| META=$(find submissions/inbox \( -name "meta.yaml" -o -name "meta.yml" \) | head -1) | |
| MODEL="" | |
| NOTES="" | |
| if [ -n "$META" ]; then | |
| echo "Found metadata file: $META" | |
| cat "$META" | |
| MODEL=$(awk -F': ' '/^[Mm]odel:/{print $2}' "$META" | head -1 | tr ',' ' ' | xargs) | |
| NOTES=$(awk -F': ' '/^[Nn]otes:/{print $2}' "$META" | head -1 | tr ',' ' ' | xargs) | |
| TYPE=$(awk -F': ' '/^[Tt]ype:/{print $2}' "$META" | head -1 | tr ',' ' ' | xargs) | |
| echo "Parsed MODEL=$MODEL" | |
| echo "Parsed NOTES=$NOTES" | |
| echo "Parsed TYPE=$TYPE" | |
| else | |
| echo "No meta.yaml found anywhere in submissions/inbox/" | |
| fi | |
| # Default type to human if not specified | |
| if [ -z "$TYPE" ]; then | |
| TYPE="human" | |
| fi | |
| echo "model=$MODEL" >> "$GITHUB_OUTPUT" | |
| echo "notes=$NOTES" >> "$GITHUB_OUTPUT" | |
| echo "type=$TYPE" >> "$GITHUB_OUTPUT" | |
| - name: Update leaderboard | |
| run: | | |
| TEAM="${{ steps.team.outputs.name }}" | |
| F1="${{ steps.eval.outputs.f1 }}" | |
| AUC="${{ steps.eval.outputs.auc }}" | |
| MODEL="${{ steps.meta.outputs.model }}" | |
| NOTES="${{ steps.meta.outputs.notes }}" | |
| TYPE="${{ steps.meta.outputs.type }}" | |
| TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") | |
| # Validate type value | |
| if [ -n "$TYPE" ] && [ "$TYPE" != "human" ] && [ "$TYPE" != "llm" ] && [ "$TYPE" != "human+llm" ]; then | |
| echo "::warning::Invalid type '$TYPE'. Must be human, llm, or human+llm. Defaulting to human." | |
| TYPE="human" | |
| fi | |
| # Append to leaderboard CSV (columns: timestamp_utc,team,model,type,score,notes) | |
| echo "${TIMESTAMP},${TEAM},${MODEL},${TYPE},${F1},${NOTES}" >> leaderboard/leaderboard.csv | |
| # Render leaderboard markdown | |
| python competition/render_leaderboard.py | |
| # Sync to docs for GitHub Pages | |
| cp leaderboard/leaderboard.csv docs/leaderboard.csv | |
| - name: Commit leaderboard update | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add leaderboard/ docs/leaderboard.csv | |
| git commit -m "🏆 Leaderboard update: ${{ steps.team.outputs.name }} (F1=${{ steps.eval.outputs.f1 }})" | |
| git push | |
| - name: Comment score on PR | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: `## ✅ Submission Scored Successfully\n\n| Metric | Score |\n|--------|-------|\n| **Macro F1** (leaderboard) | ${{ steps.eval.outputs.f1 }} |\n| **AUROC** | ${{ steps.eval.outputs.auc }} |\n| **Model** | ${{ steps.meta.outputs.model || 'not specified' }} |\n| **Type** | ${{ steps.meta.outputs.type || 'not specified' }} |\n\n🏆 The leaderboard has been updated. You may now close this PR.` | |
| }) |