release.yml

name: daily-dataset

on:
  schedule:
    - cron: "0 6 * * *"
  workflow_dispatch:

concurrency:
  group: routesentinel-daily-dataset-${{ github.ref }}
  cancel-in-progress: false

permissions:
  contents: write

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: pip

      - name: Install system MRT parser
        run: sudo apt-get update && sudo apt-get install -y bgpdump

      - name: Install RouteSentinel
        run: python -m pip install ".[dev]"

      - name: Test
        run: pytest

      - name: Set date
        id: date
        run: |
          today="$(date -u +%F)"
          previous="$(date -u -d "${today} -1 day" +%F)"
          generated_at="$(date -u '+%Y-%m-%d %H:%M UTC')"
          {
            echo "today=${today}"
            echo "previous=${previous}"
            echo "generated_at=${generated_at}"
            echo "release_url=https://github.com/${GITHUB_REPOSITORY}/releases/tag/data-current"
          } >> "$GITHUB_OUTPUT"

      - name: Restore source cache
        uses: actions/cache@v4
        with:
          path: data/raw
          key: routesentinel-raw-${{ steps.date.outputs.today }}

      - name: Fetch BGP RIB and VRPs
        run: |
          mkdir -p data/raw data/normalized out
          routesentinel fetch --retries 3 https://data.ris.ripe.net/rrc00/latest-bview.gz data/raw/rrc00-latest-bview.gz
          routesentinel fetch --retries 3 https://data.ris.ripe.net/rrc01/latest-bview.gz data/raw/rrc01-latest-bview.gz
          routesentinel fetch --retries 3 https://data.ris.ripe.net/rrc10/latest-bview.gz data/raw/rrc10-latest-bview.gz
          routesentinel fetch --retries 3 https://console.rpki-client.org/vrps.json data/raw/vrps.json

      - name: Parse MRT
        run: |
          routesentinel parse-mrt data/raw/rrc00-latest-bview.gz data/normalized/rrc00.csv --collector rrc00
          routesentinel parse-mrt data/raw/rrc01-latest-bview.gz data/normalized/rrc01.csv --collector rrc01
          routesentinel parse-mrt data/raw/rrc10-latest-bview.gz data/normalized/rrc10.csv --collector rrc10

      - name: Build snapshot
        run: |
          routesentinel snapshot \
            --announcements data/normalized/rrc00.csv \
            --announcements data/normalized/rrc01.csv \
            --announcements data/normalized/rrc10.csv \
            --vrps data/raw/vrps.json \
            --out out

      - name: Download previous release state
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          mkdir -p previous
          gh release download "data-current" \
            --repo "${GITHUB_REPOSITORY}" \
            --pattern route-origin-status.csv \
            --dir previous || true

      - name: Build changelog
        run: |
          python scripts/build_changelog.py \
            --current-status out/route-origin-status.csv \
            --current-summary out/rpki-summary.json \
            --previous-status previous/route-origin-status.csv \
            --date "${{ steps.date.outputs.today }}" \
            --updated-at "${{ steps.date.outputs.generated_at }}" \
            --out-md out/changelog.md \
            --out-json out/daily-diff.json

      - name: Sync branch before README update
        run: git pull --rebase origin "${GITHUB_REF_NAME}"

      - name: Update README stats
        run: |
          python scripts/update_readme_stats.py \
            --summary out/rpki-summary.json \
            --date "${{ steps.date.outputs.today }}" \
            --release-url "${{ steps.date.outputs.release_url }}" \
            --updated-at "${{ steps.date.outputs.generated_at }}"

      - name: Commit README stats
        run: |
          if git diff --quiet README.md; then
            echo "README stats unchanged"
            exit 0
          fi
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git add README.md
          git commit -m "Update RouteSentinel snapshot stats"
          git push || {
            git pull --rebase origin "${GITHUB_REF_NAME}"
            git push
          }

      - name: Recreate rolling release (data-current)
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          target="$(git rev-parse HEAD)"
          gh release delete "data-current" \
            --repo "${GITHUB_REPOSITORY}" \
            --cleanup-tag \
            --yes || true
          gh release create "data-current" \
            --repo "${GITHUB_REPOSITORY}" \
            --target "${target}" \
            --title "RouteSentinel snapshot ${{ steps.date.outputs.today }}" \
            --notes-file out/changelog.md \
            --latest \
            out/rpki-summary.json \
            out/rpki-invalids.csv \
            out/route-origin-status.csv \
            out/rpki-covered-prefixes.csv \
            out/top-invalid-asns.csv \
            out/suspected-events.jsonl \
            out/daily-diff.json \
            out/changelog.md