Remove daily survers from outro pages #641
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: notebook-pr | |
| on: | |
| pull_request: | |
| branches: main | |
| paths: "**/*.ipynb" | |
| env: | |
| NB_KERNEL: python | |
| ORG: neuromatch | |
| NMA_REPO: NeuroAI_Course | |
| NMA_MAIN_BRANCH: main | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| notebooks: ${{ steps.set-matrix.outputs.notebooks }} | |
| has_notebooks: ${{ steps.set-matrix.outputs.has_notebooks }} | |
| commit_message: ${{ steps.get-commit.outputs.message }} | |
| materials_changed: ${{ steps.materials-check.outputs.materials_changed }} | |
| skip_ci: ${{ steps.check-skip.outputs.skip }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| fetch-depth: 0 | |
| ref: ${{ github.head_ref }} | |
| - name: Get commit message | |
| id: get-commit | |
| run: | | |
| msg=$(git log -1 --pretty=format:"%s") | |
| echo "message=$msg" >> $GITHUB_OUTPUT | |
| - name: Check skip ci | |
| id: check-skip | |
| env: | |
| COMMIT_MESSAGE: ${{ steps.get-commit.outputs.message }} | |
| run: | | |
| if [[ "$COMMIT_MESSAGE" == *"skip ci"* ]]; then | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "skip=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Setup Python environment | |
| if: steps.check-skip.outputs.skip != 'true' | |
| uses: ./.github/actions/setup-python-env | |
| - name: Install additional dependencies | |
| if: steps.check-skip.outputs.skip != 'true' | |
| run: | | |
| pip install jupyter_client==7.3.5 | |
| pip install jinja2==3.1.2 | |
| - name: Setup CI tools | |
| if: steps.check-skip.outputs.skip != 'true' | |
| uses: ./.github/actions/setup-ci-tools | |
| with: | |
| commit-message: ${{ steps.get-commit.outputs.message }} | |
| - name: Setup rendering dependencies | |
| if: steps.check-skip.outputs.skip != 'true' | |
| uses: ./.github/actions/setup-rendering-deps | |
| - name: Get changed files | |
| if: steps.check-skip.outputs.skip != 'true' | |
| id: changed-files | |
| uses: tj-actions/changed-files@v35 | |
| - name: Check if materials.yml changed | |
| id: materials-check | |
| if: steps.check-skip.outputs.skip != 'true' | |
| run: | | |
| if [[ "${{ steps.changed-files.outputs.all_changed_files }}" =~ materials\.yml ]]; then | |
| echo "materials_changed=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "materials_changed=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Set notebook matrix | |
| id: set-matrix | |
| if: steps.check-skip.outputs.skip != 'true' | |
| run: | | |
| # Get filtered notebook list | |
| nbs=$(python ci/select_notebooks.py ${{ steps.changed-files.outputs.all_changed_files }}) | |
| if [ -z "$nbs" ]; then | |
| echo "No notebooks to process" | |
| echo "has_notebooks=false" >> $GITHUB_OUTPUT | |
| echo "notebooks=[]" >> $GITHUB_OUTPUT | |
| else | |
| echo "Notebooks to process: $nbs" | |
| echo "has_notebooks=true" >> $GITHUB_OUTPUT | |
| # Convert space-separated list to JSON array | |
| json_array=$(echo "$nbs" | tr ' ' '\n' | jq -R . | jq -s -c .) | |
| echo "notebooks=$json_array" >> $GITHUB_OUTPUT | |
| fi | |
| process: | |
| needs: setup | |
| if: needs.setup.outputs.skip_ci != 'true' && needs.setup.outputs.has_notebooks == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| notebook: ${{ fromJson(needs.setup.outputs.notebooks) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| fetch-depth: 0 | |
| ref: ${{ github.head_ref }} | |
| - name: Setup Python environment | |
| uses: ./.github/actions/setup-python-env | |
| - name: Install additional dependencies | |
| run: | | |
| pip install jupyter_client==7.3.5 | |
| pip install jinja2==3.1.2 | |
| - name: Setup CI tools | |
| uses: ./.github/actions/setup-ci-tools | |
| with: | |
| commit-message: ${{ needs.setup.outputs.commit_message }} | |
| - name: Setup rendering dependencies | |
| uses: ./.github/actions/setup-rendering-deps | |
| - name: Process notebook | |
| env: | |
| COMMIT_MESSAGE: ${{ needs.setup.outputs.commit_message }} | |
| PYTHONUNBUFFERED: "1" | |
| run: | | |
| echo "::group::Processing ${{ matrix.notebook }}" | |
| echo "Started at: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" | |
| echo "Job timeout: 20 minutes" | |
| if [[ "$COMMIT_MESSAGE" == *"ci:check"* ]]; then | |
| execflag="--check-execution" | |
| echo "Mode: Check execution (no changes)" | |
| else | |
| execflag="--execute" | |
| echo "Mode: Execute and process" | |
| fi | |
| echo "::endgroup::" | |
| echo "::group::Notebook execution output" | |
| python ci/process_notebooks.py "${{ matrix.notebook }}" $execflag | |
| exit_code=$? | |
| echo "::endgroup::" | |
| echo "Completed at: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" | |
| exit $exit_code | |
| - name: Get tutorial directory | |
| id: get-dir | |
| run: | | |
| nb="${{ matrix.notebook }}" | |
| dir=$(dirname "$nb") | |
| nb_name=$(basename "$nb" .ipynb) | |
| echo "dir=$dir" >> $GITHUB_OUTPUT | |
| echo "nb_name=$nb_name" >> $GITHUB_OUTPUT | |
| # Use ___ as delimiter (won't appear in paths) so we can restore later | |
| safe_dir=$(echo "$dir" | sed 's|/|___|g') | |
| echo "artifact_name=${safe_dir}___${nb_name}" >> $GITHUB_OUTPUT | |
| - name: Upload processed tutorial directory | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: tutorial-${{ steps.get-dir.outputs.artifact_name }} | |
| path: | | |
| ${{ matrix.notebook }} | |
| ${{ steps.get-dir.outputs.dir }}/static/${{ steps.get-dir.outputs.nb_name }}* | |
| ${{ steps.get-dir.outputs.dir }}/solutions/${{ steps.get-dir.outputs.nb_name }}* | |
| ${{ steps.get-dir.outputs.dir }}/student/${{ steps.get-dir.outputs.nb_name }}* | |
| ${{ steps.get-dir.outputs.dir }}/instructor/${{ steps.get-dir.outputs.nb_name }}* | |
| if-no-files-found: ignore | |
| retention-days: 1 | |
| finalize: | |
| needs: [setup, process] | |
| if: always() && needs.setup.outputs.skip_ci != 'true' && needs.setup.outputs.has_notebooks == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| fetch-depth: 0 | |
| ref: ${{ github.head_ref }} | |
| - name: Setup Python environment | |
| uses: ./.github/actions/setup-python-env | |
| - name: Install additional dependencies | |
| run: | | |
| pip install jupyter_client==7.3.5 | |
| pip install jinja2==3.1.2 | |
| - name: Setup CI tools | |
| uses: ./.github/actions/setup-ci-tools | |
| with: | |
| commit-message: ${{ needs.setup.outputs.commit_message }} | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts/ | |
| - name: Detect failed notebooks | |
| id: detect-failures | |
| run: | | |
| expected='${{ needs.setup.outputs.notebooks }}' | |
| failed_nbs="" | |
| success_nbs="" | |
| echo "Checking which notebooks were successfully processed..." | |
| for nb in $(echo "$expected" | jq -r '.[]'); do | |
| nb_name=$(basename "$nb" .ipynb) | |
| # Check if artifact exists for this notebook | |
| if ls artifacts/tutorial-*"___${nb_name}" 1>/dev/null 2>&1; then | |
| echo " ✓ $nb - SUCCESS" | |
| success_nbs="$success_nbs $nb" | |
| else | |
| echo " ✗ $nb - FAILED (no artifact)" | |
| failed_nbs="$failed_nbs $nb" | |
| fi | |
| done | |
| # Trim leading spaces | |
| success_nbs="${success_nbs# }" | |
| failed_nbs="${failed_nbs# }" | |
| echo "success_notebooks=$success_nbs" >> $GITHUB_OUTPUT | |
| echo "failed_notebooks=$failed_nbs" >> $GITHUB_OUTPUT | |
| if [ -n "$failed_nbs" ]; then | |
| echo "has_failures=true" >> $GITHUB_OUTPUT | |
| echo "" | |
| echo "⚠️ Some notebooks failed to process: $failed_nbs" | |
| else | |
| echo "has_failures=false" >> $GITHUB_OUTPUT | |
| echo "" | |
| echo "✅ All notebooks processed successfully" | |
| fi | |
| - name: Collect failure context | |
| if: steps.detect-failures.outputs.has_failures == 'true' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| run_id=${{ github.run_id }} | |
| echo "## Notebook Processing Failure Context" > failure_context.md | |
| echo "" >> failure_context.md | |
| jobs_json=$(gh api repos/${{ github.repository }}/actions/runs/${run_id}/jobs) | |
| for nb in ${{ steps.detect-failures.outputs.failed_notebooks }}; do | |
| nb_name=$(basename "$nb" .ipynb) | |
| echo "### $nb" >> failure_context.md | |
| echo '```' >> failure_context.md | |
| job_id=$(echo "$jobs_json" | jq -r --arg nb "$nb" '.jobs[] | select(.name | contains($nb)) | .id') | |
| if [ -n "$job_id" ] && [ "$job_id" != "null" ]; then | |
| # Get logs from the "Notebook execution output" group | |
| gh api repos/${{ github.repository }}/actions/jobs/${job_id}/logs 2>/dev/null | \ | |
| awk ' | |
| /##\[group\]Notebook execution output/{capturing=1; next} | |
| capturing && /##\[endgroup\]/{exit} | |
| capturing && /##\[error\]/{print; exit} | |
| capturing && /##\[group\]/{exit} | |
| capturing && /Post job cleanup/{exit} | |
| capturing{print} | |
| ' | \ | |
| tail -100 >> failure_context.md || echo "Could not retrieve logs" >> failure_context.md | |
| else | |
| echo "Job not found for $nb" >> failure_context.md | |
| fi | |
| echo '```' >> failure_context.md | |
| echo "" >> failure_context.md | |
| done | |
| - name: Post failure context comment | |
| if: steps.detect-failures.outputs.has_failures == 'true' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| if [ -f failure_context.md ]; then | |
| gh pr comment ${{ github.event.pull_request.number }} --body-file failure_context.md | |
| fi | |
| - name: Restore processed files | |
| run: | | |
| echo "Restoring processed files from artifacts..." | |
| # Artifact name format: tutorial-tutorials___W1D5_Microcircuits___W1D5_Tutorial2 | |
| # The path is encoded with ___ as delimiter. Last segment is notebook name, rest is directory. | |
| # upload-artifact strips common path prefixes, so we need to reconstruct the target directory. | |
| for dir in artifacts/tutorial-*; do | |
| if [ -d "$dir" ]; then | |
| # Extract artifact name and parse it to get target directory | |
| artifact_name=$(basename "$dir" | sed 's/^tutorial-//') | |
| # Convert ___ to / to get full path, then extract directory | |
| full_path=$(echo "$artifact_name" | sed 's|___|/|g') | |
| tutorial_dir=$(dirname "$full_path") | |
| echo "Restoring from artifact $(basename "$dir") to: $tutorial_dir" | |
| # Copy all files to the correct tutorial directory | |
| # The artifact contains files with stripped prefixes (e.g., W1D5_Tutorial2.ipynb, student/, solutions/) | |
| mkdir -p "$tutorial_dir" | |
| cp -rv "$dir"/* "$tutorial_dir/" 2>/dev/null || true | |
| fi | |
| done | |
| echo "Restore complete." | |
| - name: Verify exercises | |
| env: | |
| COMMIT_MESSAGE: ${{ needs.setup.outputs.commit_message }} | |
| run: | | |
| nbs='${{ needs.setup.outputs.notebooks }}' | |
| # Convert JSON array to space-separated list | |
| nb_list=$(echo "$nbs" | jq -r '.[]' | tr '\n' ' ') | |
| python ci/verify_exercises.py $nb_list --c "$COMMIT_MESSAGE" | |
| - name: Make PR comment | |
| run: | | |
| nbs='${{ needs.setup.outputs.notebooks }}' | |
| nb_list=$(echo "$nbs" | jq -r '.[]' | tr '\n' ' ') | |
| branch=${{ github.event.pull_request.head.ref }} | |
| python ci/make_pr_comment.py $nb_list --branch $branch --o comment.txt | |
| - name: Update READMEs | |
| if: needs.setup.outputs.materials_changed == 'true' | |
| run: python ci/generate_tutorial_readmes.py | |
| - name: Remove unreferenced derivatives | |
| run: | | |
| python ci/find_unreferenced_content.py > to_remove.txt | |
| if [ -s to_remove.txt ]; then git rm --pathspec-from-file=to_remove.txt; fi | |
| - name: Clean up artifacts directory | |
| run: rm -rf artifacts/ | |
| - name: Revert failed notebooks to main | |
| if: steps.detect-failures.outputs.has_failures == 'true' && steps.detect-failures.outputs.success_notebooks != '' | |
| run: | | |
| failed_nbs="${{ steps.detect-failures.outputs.failed_notebooks }}" | |
| echo "Reverting failed notebooks to main branch version: $failed_nbs" | |
| # Revert each failed notebook to the main branch version | |
| for nb in $failed_nbs; do | |
| if git show origin/main:"$nb" > /dev/null 2>&1; then | |
| git checkout origin/main -- "$nb" | |
| echo " Reverted: $nb" | |
| else | |
| # Notebook is new in this PR, remove it | |
| git rm -f "$nb" 2>/dev/null || rm -f "$nb" | |
| echo " Removed (new file): $nb" | |
| fi | |
| done | |
| - name: Create follow-up PR for failed notebooks | |
| if: steps.detect-failures.outputs.has_failures == 'true' && steps.detect-failures.outputs.success_notebooks != '' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| failed_nbs="${{ steps.detect-failures.outputs.failed_notebooks }}" | |
| pr_number=${{ github.event.pull_request.number }} | |
| original_branch=${{ github.head_ref }} | |
| new_branch="reprocess-pr${pr_number}-$(date +%Y%m%d%H%M%S)" | |
| echo "Creating follow-up PR for failed notebooks: $failed_nbs" | |
| # Stash all local changes to keep new branch clean | |
| git stash --include-untracked | |
| # Create new branch from origin/main (has main's version of notebooks) | |
| git fetch origin main | |
| git checkout -b "$new_branch" origin/main | |
| # Copy failed notebooks from the original PR branch (unprocessed version) | |
| for nb in $failed_nbs; do | |
| git checkout "origin/$original_branch" -- "$nb" 2>/dev/null || echo "Note: $nb may be new in this PR" | |
| done | |
| # Commit the notebooks | |
| git add $failed_nbs | |
| git commit -m "Reprocess notebooks from PR #${pr_number} | |
| These notebooks failed to process in the original PR and need reprocessing: | |
| $(echo "$failed_nbs" | tr ' ' '\n' | sed 's/^/- /')" | |
| # Configure git to use GITHUB_TOKEN for authentication | |
| gh auth setup-git | |
| # Push the new branch | |
| git push origin "$new_branch" | |
| # Create the PR targeting the original branch and capture the URL | |
| new_pr_url=$(gh pr create \ | |
| --title "Reprocess: Failed notebooks from PR #${pr_number}" \ | |
| --body "## Auto-generated PR | |
| The following notebooks failed to process in PR #${pr_number} and need reprocessing: | |
| $(echo "$failed_nbs" | tr ' ' '\n' | sed 's/^/- /') | |
| This PR was automatically created to allow these notebooks to be processed independently. | |
| Original PR: #${pr_number}" \ | |
| --base "$original_branch" \ | |
| --head "$new_branch") | |
| echo "Created follow-up PR: $new_pr_url" | |
| # Add comment to original PR referencing the follow-up PR | |
| gh pr comment "$pr_number" --body "## Follow-up PR Created | |
| A follow-up PR has been created for the notebooks that failed to process: | |
| **Follow-up PR:** $new_pr_url | |
| **Branch:** \`$new_branch\` | |
| The failed notebooks have been reverted to their \`main\` branch versions in this PR. | |
| Once this PR is merged, the follow-up PR can be used to reprocess the failed notebooks independently." | |
| # Return to original branch and restore state | |
| git checkout "$original_branch" | |
| git stash pop || true | |
| - name: Commit post-processed files | |
| if: steps.detect-failures.outputs.success_notebooks != '' | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| git add '**/*.ipynb' | |
| git add '**/static/*.png' | |
| git add '**/solutions/*.py' | |
| git add '**/student/*.ipynb' | |
| git add '**/instructor/*.ipynb' | |
| git add '**/README.md' | |
| # Build commit message based on whether there were failures | |
| if [ "${{ steps.detect-failures.outputs.has_failures }}" == "true" ]; then | |
| msg="Process tutorial notebooks (some notebooks moved to follow-up PR)" | |
| else | |
| msg="Process tutorial notebooks" | |
| fi | |
| git diff-index --quiet HEAD || git commit -m "$msg" | |
| - name: Push post-processed files | |
| if: steps.detect-failures.outputs.success_notebooks != '' | |
| uses: ad-m/github-push-action@v0.6.0 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| branch: ${{ github.head_ref }} |