crawler-scope.yml

name: CrawlerScope

on:
  workflow_dispatch: {}
  schedule:
    - cron: "23 */6 * * *"

permissions:
  contents: write

concurrency:
  group: crawler-scope-${{ github.ref }}
  cancel-in-progress: false

jobs:
  update:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Update crawler data
        env:
          CRAWLER_SCOPE_SNAPSHOT_RETENTION: "168"
          CRAWLER_SCOPE_HISTORY_RETENTION: "720"
          CRAWLER_SCOPE_USER_AGENT: "CrawlerScope/0.1 (+${{ github.server_url }}/${{ github.repository }}; GitHub Actions public-data collector)"
        run: python3 scripts/update.py

      - name: Commit updated data
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git add data config
          if git diff --cached --quiet; then
            echo "No dataset changes to commit."
          else
            git commit -m "Update CrawlerScope data"
            git push
          fi

      - name: Publish dataset release
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          stamp="$(python3 -c 'import json; print(json.load(open("data/current/crawlers.json"))["generatedAt"].replace(":", "").replace("-", "").replace("Z", "Z"))')"
          tag="dataset-current"   # rolling: одна страница релиза, апдейт на месте
          snapshot="$(ls -1 data/snapshots/*.json | sort | tail -n 1)"
          python3 scripts/release_notes.py > release-notes.md

          if gh release view "$tag" >/dev/null 2>&1; then
            gh release edit "$tag" \
              --title "CrawlerScope dataset ${stamp}" \
              --notes-file release-notes.md \
              --latest
            gh release upload "$tag" \
              data/current/crawlers.json \
              data/current/robots-ai.txt \
              data/current/nginx-ai-map.conf \
              data/history/summary.csv \
              "$snapshot" \
              --clobber
          else
            gh release create "$tag" \
              data/current/crawlers.json \
              data/current/robots-ai.txt \
              data/current/nginx-ai-map.conf \
              data/history/summary.csv \
              "$snapshot" \
              --title "CrawlerScope dataset ${stamp}" \
              --notes-file release-notes.md \
              --latest
          fi

      - name: Publish GitHub Pages branch
        run: |
          site_source="$RUNNER_TEMP/crawler-scope-site"
          pages_dir="$RUNNER_TEMP/crawler-scope-pages"

          rm -rf "$site_source" "$pages_dir"
          mkdir -p "$site_source/data"
          cp -R public/. "$site_source/"
          cp -R data/. "$site_source/data/"
          touch "$site_source/.nojekyll"

          git fetch origin gh-pages --depth=1 || true
          if git rev-parse --verify origin/gh-pages >/dev/null 2>&1; then
            git worktree add "$pages_dir" origin/gh-pages
          else
            git worktree add --detach "$pages_dir"
            git -C "$pages_dir" checkout --orphan gh-pages
            git -C "$pages_dir" rm -rf . || true
          fi

          find "$pages_dir" -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} +
          cp -R "$site_source"/. "$pages_dir"/

          git -C "$pages_dir" add -A
          if git -C "$pages_dir" diff --cached --quiet; then
            echo "No Pages changes to publish."
          else
            git -C "$pages_dir" commit -m "Publish CrawlerScope site"
            git -C "$pages_dir" push origin HEAD:gh-pages
          fi