scraper.py

#!/usr/bin/env python3
"""
Hyrox result scraper (season 1-8)

Scrapes the athlete detail pages and extracts AS MUCH AVAILABLE data as possible.
Not just the summary table, but all front-end available f_times. 
Data collected:
  - general info (name, bib, age group, nation)
  • event info (race, division)
  • judging decisions (bonus, penalty, disqual reason, info)
  • totals (ranks, overall time)
  • workout Summary, run / station times + places
  • race replay / splits, time of day, elapsed, diff

Column naming uses CSS class identifiers:
  detail-box-other  → f-time_XX (time), f-time_XX_place (rank)
  detail-box-splits → f-time_XX_tod, f-time_XX_elapsed, f-time_XX_diff
  other boxes       → f-<field> (single value)

Row 2 in each CSV contains the human readable labels.
"""

import os
import time
import requests
import pandas as pd
from bs4 import BeautifulSoup
from tqdm import tqdm
from concurrent.futures import ThreadPoolExecutor, as_completed
from collections import OrderedDict

# config
MAX_WORKERS = 100 # parallel request, 100 might be aggressive. 
RETRIES     = 3
TIMEOUT     = 4

# cols written to csv
META_COLS = ["race", "division", "gender"]

# domain routing
# seasons 1-4 live on the old domain (POST-based ajax2).
# seasons 5+ live on the new mikatiming domain (GET-based, needs XHR header).

def _base_url(season: int) -> str:
    if season <= 4:
        return f"https://results.hyrox.com/season-{season}/index.php"
    return f"https://hyrox.r.mikatiming.com/season-{season}/"

def _headers(season: int) -> dict:
    h = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"}
    if season >= 5:
        h["X-Requested-With"] = "XMLHttpRequest"
        h["Referer"] = _base_url(season)
    return h

# Discover domain structure: what events, groups, divisions are there
def discover_structure(season: int) -> list[dict]:
    base = _base_url(season)
    headers = _headers(season)
    print(f"\n--- mapping season {season} Structure ---")

    for attempt in range(1, RETRIES + 1):
        try:
            # retrieve top-level event groups
            if season <= 4:
                r = requests.post(base, data={
                    "content": "ajax2", "func": "getSearchFields",
                    "options[pid]": "start",
                }, headers=headers, timeout=TIMEOUT)
            else:
                r = requests.get(base, params={
                    "content": "ajax2", "func": "getSearchFields",
                    "options[pid]": "start",
                }, headers=headers, timeout=TIMEOUT)

            groups = r.json()["branches"]["lists"]["fields"]["event_main_group"]["data"]
            structure = []

            for g in groups:
                g_id, g_name = g["v"][0], g["v"][1]

                # retrieve divisions within group
                grp_params = {
                    "content": "ajax2", "func": "getSearchFields",
                    "options[b][lists][event_main_group]": g_id,
                    "options[pid]": "start",
                }
                if season <= 4:
                    dr = requests.post(base, data=grp_params, headers=headers, timeout=TIMEOUT)
                else:
                    dr = requests.get(base, params=grp_params, headers=headers, timeout=TIMEOUT)

                divs = dr.json()["branches"]["lists"]["fields"]["event"]["data"]
                for d in divs:
                    if d["v"]:
                        structure.append({
                            "group_name": g_name,
                            "div_name":   d["v"][1],
                            "div_code":   d["v"][0],
                        })

            print(f"  Found {len(structure)} divisions across {len(groups)} event groups")
            return structure

        except Exception as e:
            print(f"  discovery error (attempt {attempt}/{RETRIES}): {e}")
            if attempt < RETRIES:
                time.sleep(3)

    print(f"  failed after {RETRIES} attempts — skipping season.")
    return []


# Detail page scraper
# Fetches the athlete detail page and extracts ALL f- fields. Returns (data_dict, label_dict).
# Naming conventions
#   detail-box-other   f-time_XX, f-time_XX_place
#   detail-box-splits  f-time_XX_tod, f-time_XX_elapsed, f-time_XX_diff
#   everything else    f-<class> (single value)

def fetch_athlete_detail(profile_url: str, base_url: str, headers: dict) -> tuple[dict, dict]:
    if not profile_url:
        return {}, {}

    full_url = profile_url if profile_url.startswith("http") else base_url + profile_url

    for attempt in range(RETRIES):
        try:
            r = requests.get(full_url, headers=headers, timeout=TIMEOUT)
            soup = BeautifulSoup(r.text, "html.parser")
            data   = OrderedDict()
            labels = OrderedDict()

            for box in soup.select("div.detail-box"):
                box_id = box.get("id", "")
                table = box.find("table")
                if not table:
                    continue

                for tr in table.select("tr"):
                    row_f = [c for c in tr.get("class", []) if c.startswith("f-")]
                    if not row_f:
                        continue

                    fid = row_f[0]
                    th  = tr.select_one("th.desc, th")
                    lbl = th.get_text(strip=True) if th else ""
                    tds = tr.select("td")

                    if box_id == "detail-box-other":
                        # workout summary: col 1 = time, col 2 = place
                        data[fid]   = tds[0].get_text(strip=True) if tds else ""
                        labels[fid] = lbl
                        if len(tds) > 1:
                            pc = fid + "_place"
                            data[pc]   = tds[1].get_text(strip=True)
                            labels[pc] = lbl + " (Place)"

                    elif box_id == "detail-box-splits":
                        # race replay: time-of-day, elapsed, diff
                        for i, (suf, slbl) in enumerate([
                            ("_tod",     " (ToD)"),
                            ("_elapsed", " (Elapsed)"),
                            ("_diff",    " (Diff)"),
                        ]):
                            col = fid + suf
                            data[col]   = tds[i].get_text(strip=True) if i < len(tds) else ""
                            labels[col] = lbl + slbl

                    else:
                        # Single-value boxes: general, eventinfo, judges, totals
                        data[fid]   = tds[0].get_text(strip=True) if tds else ""
                        labels[fid] = lbl

            return data, labels

        except Exception:
            if attempt < RETRIES - 1:
                time.sleep(1)

    return {}, {}


# division scraper
# Scrapers all athletes for one division so we can traverse them later

def scrape_division(div_item: dict, sex_code: str, season: int) -> tuple[list[dict], dict]:
    base    = _base_url(season)
    headers = _headers(season)
    gender_label = {"M": "Men", "W": "Women", "X": "Mixed"}.get(sex_code, sex_code)

    athletes = []
    page = 1 # Result pages have pageination, we loop until we get an empty page
    while True:
        resp = requests.get(base, params={
            "pid": "list",
            "event": div_item["div_code"],
            "page": page,
            "num_results": 100,
            "search[sex]": sex_code,
        }, headers=headers, timeout=TIMEOUT)
        soup = BeautifulSoup(resp.text, "html.parser")

        items = soup.select("li.list-group-item.row")
        athlete_items = [li for li in items if li.select_one(".type-fullname a")]
        if not athlete_items:
            break

        for li in athlete_items:
            tag = li.select_one(".type-fullname a")
            athletes.append({
                "race":     div_item["group_name"],
                "division": div_item["div_name"],
                "gender":   gender_label,
                "_url":     tag.get("href") if tag else None,
            })
        page += 1

    if not athletes:
        return [], {}

    # fetch details in parallel, but merge sequentially to avoid conflicts.
    merged_labels = OrderedDict()

    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
        futures = {
            pool.submit(fetch_athlete_detail, a["_url"], base, headers): a
            for a in athletes
        }
        desc = f"   {div_item['div_name'][:10]} ({gender_label})"
        for fut in tqdm(as_completed(futures), total=len(athletes), desc=desc, leave=False):
            athlete = futures[fut]
            detail_data, detail_labels = fut.result()
            athlete.update(detail_data)
            for k, v in detail_labels.items():
                if k not in merged_labels:
                    merged_labels[k] = v

    return athletes, merged_labels



def save_csv(athletes: list[dict], labels: dict, csv_path: str):
    if not athletes:
        return

    detail_cols = []
    seen = set()
    for a in athletes:
        for k in a:
            if k not in META_COLS and k != "_url" and k not in seen:
                detail_cols.append(k)
                seen.add(k)

    columns = META_COLS + detail_cols

    # label row (first data row)
    label_row = {}
    for c in columns:
        if c in META_COLS:
            label_row[c] = c.title()
        else:
            label_row[c] = labels.get(c, c)

    # Athlete data rows
    rows = [label_row]
    for a in athletes:
        rows.append({c: a.get(c, "") for c in columns})

    df = pd.DataFrame(rows, columns=columns)
    df.to_csv(csv_path, index=False)


# main
if __name__ == "__main__":
    for season_idx in [3]:
        season_dir = f"data.nosync/season_{season_idx}"
        os.makedirs(season_dir, exist_ok=True)

        structure = discover_structure(season_idx)

        for sex in ["M", "W", "X"]:
            for item in structure:
                gender_label = {"M": "Men", "W": "Women", "X": "Mixed"}[sex]
                safe_name = (
                    f"{item['group_name']}_{item['div_name']}_{gender_label}"
                    .replace(" ", "_").replace("/", "-")
                )
                csv_path = os.path.join(season_dir, f"{safe_name}.csv")

                if os.path.exists(csv_path):
                    continue

                print(f"Season {season_idx} | {item['group_name']} | {item['div_name']} | {gender_label}")

                athletes, labels = scrape_division(item, sex, season_idx)
                if athletes:
                    save_csv(athletes, labels, csv_path)