sync_data.py

#!/usr/bin/env python3
"""Auto-discover NUSV projects and generate the website's project pages.

Discovery rule (self-service, no hardcoding):
  A repository in the NUSV organization is treated as a project when it has
  the GitHub topic "nusv-project". Infrastructure repos (.github,
  NUSV.github.io, nusv-lite-sync, ...) simply never carry that topic.

Per-project conventions (all optional):
  site.json in the repo root:
    {
      "name": "Syna",                       # display name (default: repo name, minus a trailing "-NUSV")
      "tagline": "one-liner",               # default: repository description
      "platforms": ["Android", "Windows"],  # default: []
      "docs": "/docs/syna.html",            # site-relative (leading /) or absolute URL
      "icon": "logo.png"                    # path in the repo, default: auto-discovered
    }
  Icon auto-discovery: any *.png in the repo root named like the repo,
  "logo*" or "icon*"; the last one alphabetically wins (Syna_logo_2.png
  beats Syna_logo.png).
  Screenshots: images under screenshots/ | docs/screenshots/ |
  assets/screenshots/ are rendered as a gallery on the project page.

Generated outputs:
  projects/<slug>.html  - one page per discovered project
  index.html            - the "Featured projects" grid (kept in sync via markers)
  warehouse.html        - the "Everything we ship" grid (kept in sync via markers)

Run: python3 tools/sync_data.py [--token TOKEN]
Scheduled by .github/workflows/sync.yml (every 6h + manual dispatch).
"""

import argparse
import datetime
import html
import json
import os
import posixpath
import re
import sys
import time
import urllib.request
import urllib.error

API = "https://api.github.com"
RAW = "https://raw.githubusercontent.com"
ORG = "NUSV"
PROJECT_TOPIC = "nusv-project"
REPOS_PER_PAGE = 100

# Output files the generator rewrites between markers.
INDEX_FEATURED = os.path.join(
    os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "index.html"
)
WAREHOUSE = os.path.join(
    os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "warehouse.html"
)
DOCS_INDEX = os.path.join(
    os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "docs.html"
)

FEAT_START = "<!-- FEATURED-GRID:start -->"
FEAT_END = "<!-- FEATURED-GRID:end -->"
WH_START = "<!-- WAREHOUSE-GRID:start -->"
WH_END = "<!-- WAREHOUSE-GRID:end -->"
MIRRORED_START = "<!-- MIRRORED-DOCS:start -->"
MIRRORED_END = "<!-- MIRRORED-DOCS:end -->"

# Icon names to look for in the repo root (lower-cased, prefix match).
ICON_PREFIXES = ("logo", "icon")

NAV_TMPL = """  <header class="nav">
    <div class="wrap nav-inner">
      <a class="brand" href="{p}index.html">
        <span class="brand-mark"><img src="{p}assets/img/nusv-logo.png" alt="NUSV" loading="lazy"></span>
        NUSV <small>United Science Vaca</small>
      </a>
      <nav class="nav-links">
        <a href="{p}index.html" data-i18n="nav.home">Home</a>
        <a href="{p}warehouse.html" data-i18n="nav.warehouse">Warehouse</a>
        <a href="{p}docs.html" data-i18n="nav.docs">Docs</a>
        <a href="{p}about.html" data-i18n="nav.about">About</a>
      </nav>
      <span class="nav-cta">
        <button class="lang-toggle" id="lang-toggle" data-lang="en" title="中文版即将推出">EN</button>
      </span>
    </div>
  </header>"""

FOOTER_TMPL = """  <footer class="footer">
    <div class="wrap footer-inner">
      <span>&copy; <span class="year"></span> <span data-i18n="footer.copyright">United Science Vaca (NUSV). Open source.</span></span>
      <span><a href="https://github.com/NUSV" target="_blank" rel="noopener" data-i18n="footer.github">GitHub</a> &middot; <a href="{p}warehouse.html" data-i18n="footer.warehouse">Warehouse</a> &middot; <a href="{p}docs.html" data-i18n="footer.docs">Docs</a></span>
    </div>
  </footer>"""


def nav(prefix=""):
    return NAV_TMPL.replace("{p}", prefix)


def footer(prefix=""):
    return FOOTER_TMPL.replace("{p}", prefix)


class HttpError(Exception):
    pass


def _urlopen(req, timeout=30, attempts=4):
    """urlopen with retries for flaky networks (RemoteDisconnected etc.)."""
    last = None
    for k in range(attempts):
        try:
            return urllib.request.urlopen(req, timeout=timeout)
        except urllib.error.HTTPError:
            raise
        except Exception as e:  # noqa: BLE001 - transient network errors
            last = e
            time.sleep(1.5 * (k + 1))
    raise last


def http_json(url, token):
    req = urllib.request.Request(url)
    if token:
        req.add_header("Authorization", "Bearer " + token)
    req.add_header("Accept", "application/vnd.github+json")
    req.add_header("User-Agent", "nusv-site-sync")
    try:
        with _urlopen(req) as r:
            return json.loads(r.read().decode("utf-8"))
    except urllib.error.HTTPError as e:
        if e.code in (404,):
            return None
        raise HttpError("%s -> %s" % (url, e))


def http_text(url, token):
    req = urllib.request.Request(url)
    if token:
        req.add_header("Authorization", "Bearer " + token)
    req.add_header("User-Agent", "nusv-site-sync")
    try:
        with _urlopen(req) as r:
            return r.read().decode("utf-8")
    except urllib.error.HTTPError as e:
        if e.code in (404, 403):
            return None
        raise HttpError("%s -> %s" % (url, e))


# --------------------------------------------------------------------------
# GitHub data
# --------------------------------------------------------------------------

def list_org_repos(token):
    out, page = [], 1
    while True:
        batch = http_json(
            "%s/orgs/%s/repos?per_page=%d&page=%d&type=public"
            % (API, ORG, REPOS_PER_PAGE, page),
            token,
        )
        if not batch:
            break
        out.extend(batch)
        if len(batch) < REPOS_PER_PAGE:
            break
        page += 1
    return out


def get_topics(repo_name, token):
    data = http_json(
        "%s/repos/%s/%s/topics" % (API, ORG, repo_name),
        token,
    )
    if not data:
        return []
    # The list endpoint can be served without the topics accept header value;
    # topics arrive under .names.
    return data.get("names", [])


def get_site_json(repo_name, branch, token):
    data = http_json(
        "%s/repos/%s/%s/contents/site.json?ref=%s" % (API, ORG, repo_name, branch),
        token,
    )
    if not data or not data.get("content"):
        return {}
    try:
        return json.loads(__import__("base64").b64decode(data["content"]).decode("utf-8"))
    except Exception:
        return {}


def get_root_files(repo_name, branch, token):
    data = http_json(
        "%s/repos/%s/%s/contents?ref=%s" % (API, ORG, repo_name, branch), token
    )
    return [f["name"] for f in data] if data else []


_TREE_CACHE = {}


def get_tree(repo_name, branch, token):
    key = (repo_name, branch)
    if key not in _TREE_CACHE:
        data = http_json(
            "%s/repos/%s/%s/git/trees/%s?recursive=1" % (API, ORG, repo_name, branch),
            token,
        )
        _TREE_CACHE[key] = data.get("tree", []) if data else []
    return _TREE_CACHE[key]


def get_screenshots(repo_name, branch, token):
    """Find screenshots anywhere under a known folder name."""
    out = []
    for t in get_tree(repo_name, branch, token):
        p = t.get("path", "")
        if not p.lower().endswith((".png", ".jpg", ".jpeg", ".webp", ".gif")):
            continue
        parts = p.lower().split("/")
        if any(x in ("screenshots", "screenshot") for x in parts):
            out.append(p)
    out.sort()
    return out


def get_doc_paths(repo_name, branch, token):
    """Important documents to mirror: root *.md (except README) + docs/**/*.md."""
    out = []
    for t in get_tree(repo_name, branch, token):
        if t.get("type") != "blob":
            continue
        p = t.get("path", "")
        if not p.lower().endswith(".md"):
            continue
        if "/" not in p:
            if p.lower() == "readme.md":
                continue
            out.append(p)
        elif p.lower().startswith("docs/"):
            out.append(p)
    out.sort(key=lambda x: (0 if "/" not in x else 1, x.lower()))
    return out


def doc_slug(path):
    """Slug for a repo document path.

    Generic basenames borrow the parent directory, so
    docs/tutorials/en.md -> "tutorials-en" and docs/tutorials/index.md ->
    "tutorials"; normal names stand alone (docs/security/shield-design.md ->
    "shield-design"). Version suffixes are trimmed."""
    parts = path.split("/")
    base = re.sub(r"\.md$", "", parts[-1], flags=re.I).lower()
    dirs = [p for p in parts[:-1] if p.lower() != "docs"]
    if base in ("index", "readme") and dirs:
        source = dirs[-1]
    elif base in ("en", "zh", "cn", "ja", "ko", "es", "fr", "de", "ru") and dirs:
        source = "%s-%s" % (dirs[-1], base)
    else:
        source = base
    slug = slugify(source)
    return re.sub(r"-v\d+(-\d+)+$", "", slug) or slug


def slugify(text):
    s = re.sub(r"[^a-z0-9]+", "-", (text or "").lower()).strip("-")
    return s or "doc"


def prettify_name(path):
    base = re.sub(r"\.md$", "", os.path.basename(path), flags=re.I)
    return base.replace("_", " ").replace("-", " ").strip()


def extract_h1(md):
    m = re.search(r"^#\s+(.+?)\s*$", md, re.M)
    return m.group(1).strip() if m else None


IMG_RE = re.compile(
    r'src="(https://raw\.githubusercontent\.com/NUSV/([^/"]+)/([^/"]+)/([^"]+))"'
)
_IMG_CACHE = {}
_IMG_TOTAL = [0]
IMG_MAX_FILE = 8 * 1024 * 1024
IMG_MAX_TOTAL = 120 * 1024 * 1024


def localize_images(text, token, root):
    """Download referenced raw.githubusercontent images into
    assets/mirror/<repo>/<path> and rewrite src to site-root paths.

    Keeps the remote URL when a download fails or exceeds the caps, so the
    page still works (just without the mirror benefit for that image)."""

    def repl(m):
        url, repo, branch, path = m.group(1), m.group(2), m.group(3), m.group(4)
        if url in _IMG_CACHE:
            return 'src="%s"' % _IMG_CACHE[url]
        rel = os.path.join("assets", "mirror", repo, path)
        dest = os.path.join(root, rel)
        local = "/" + rel.replace(os.sep, "/")
        if os.path.exists(dest):
            _IMG_CACHE[url] = local
            return 'src="%s"' % local
        if _IMG_TOTAL[0] > IMG_MAX_TOTAL:
            return m.group(0)
        try:
            req = urllib.request.Request(url)
            if token:
                req.add_header("Authorization", "Bearer " + token)
            req.add_header("User-Agent", "nusv-site-sync")
            with _urlopen(req, timeout=60) as r:
                data = r.read(IMG_MAX_FILE + 1)
            if len(data) > IMG_MAX_FILE:
                print("  img too large, kept remote: %s" % url)
                return m.group(0)
        except Exception as e:  # noqa: BLE001
            print("  img failed, kept remote: %s (%s)" % (url, e))
            return m.group(0)
        os.makedirs(os.path.dirname(dest), exist_ok=True)
        with open(dest, "wb") as f:
            f.write(data)
        _IMG_TOTAL[0] += len(data)
        _IMG_CACHE[url] = local
        return 'src="%s"' % local

    return IMG_RE.sub(repl, text)


def discover_icon(repo_name, branch, token, override=None):
    if override:
        return "%s/%s/%s/%s/%s" % (RAW, ORG, repo_name, branch, override.lstrip("/"))
    files = get_root_files(repo_name, branch, token)
    low = repo_name.lower()
    base = re.sub(r"-nusv$", "", low)  # "Syna-NUSV" -> "syna"
    cands = []
    for f in files:
        fl = f.lower()
        if not fl.endswith(".png"):
            continue
        if fl.startswith(ICON_PREFIXES) or fl.startswith(base) or fl.startswith(low):
            cands.append(f)
    if not cands:
        return None
    # last alphabetically: Syna_logo_2.png > Syna_logo.png
    best = max(cands)
    return "%s/%s/%s/%s/%s" % (RAW, ORG, repo_name, branch, best)


def fetch_releases(repo_name, token):
    data = http_json(
        "%s/repos/%s/%s/releases?per_page=8" % (API, ORG, repo_name), token
    )
    out = []
    for rel in data or []:
        out.append(
            {
                "tag": rel.get("tag_name", ""),
                "name": rel.get("name") or rel.get("tag_name", ""),
                "published": rel.get("published_at", ""),
                "html_url": rel.get("html_url", ""),
                "assets": [
                    {
                        "name": a.get("name", ""),
                        "size": a.get("size", 0),
                        "count": a.get("download_count", 0),
                        "url": a.get("browser_download_url", ""),
                    }
                    for a in rel.get("assets", [])
                ],
            }
        )
    return out


def fetch_repo_text(repo_name, branch, path, token):
    """Fetch a text file from a repo, preferring the contents API (fresh,
    not CDN-cached) and falling back to raw for files over the API's
    1 MB inline limit."""
    data = http_json(
        "%s/repos/%s/%s/contents/%s?ref=%s" % (API, ORG, repo_name, path, branch),
        token,
    )
    if data and data.get("content"):
        try:
            return __import__("base64").b64decode(data["content"]).decode("utf-8")
        except Exception:
            pass
    url = "%s/%s/%s/%s/%s" % (RAW, ORG, repo_name, branch, path)
    req = urllib.request.Request(url)
    if token:
        req.add_header("Authorization", "Bearer " + token)
    req.add_header("User-Agent", "nusv-site-sync")
    try:
        with _urlopen(req) as r:
            return r.read().decode("utf-8")
    except urllib.error.HTTPError as e:
        if e.code in (404, 403):
            return None
        raise


def fetch_readme(repo_name, branch, token):
    return fetch_repo_text(repo_name, branch, "README.md", token)


def get_mirror_manifest(token):
    """Manifest produced by NUSV/downloads (latest-release mirror).

    The manifest is published on the `mirror` branch (the workflow keeps the
    pipeline on `main` and force-pushes snapshots to `mirror`)."""
    data = http_json(
        "%s/repos/%s/downloads/contents/downloads.json?ref=mirror" % (API, ORG), token
    )
    if not data or not data.get("content"):
        return {}
    try:
        return json.loads(__import__("base64").b64decode(data["content"]).decode("utf-8"))
    except Exception:
        return {}


# --------------------------------------------------------------------------
# Markdown subset renderer
# --------------------------------------------------------------------------

def resolve_ref(repo_name, branch, raw):
    raw = raw.strip()
    if raw.startswith(("http://", "https://", "#", "mailto:", "data:")):
        return raw
    raw = raw.lstrip("./")
    return "https://github.com/%s/%s/%s/%s" % (ORG, repo_name, branch, raw)


def resolve_raw(repo_name, branch, raw):
    raw = raw.strip()
    if raw.startswith(("http://", "https://", "data:")):
        return raw
    raw = raw.lstrip("./")
    return "%s/%s/%s/%s/%s" % (RAW, ORG, repo_name, branch, raw)


def md_inline(text, repo_name, branch, link_map=None, base_dir=""):
    img_tokens = {}

    def repo_path(ref):
        ref = ref.strip().lstrip("./")
        if base_dir:
            return posixpath.normpath(posixpath.join(base_dir, ref))
        return ref

    def img_placeholder(m):
        alt, src = m.group(1), m.group(2)
        key = "IMGTOK%d" % len(img_tokens)
        img_tokens[key] = (
            '<img src="%s" alt="%s" loading="lazy" referrerpolicy="no-referrer">'
            % (html.escape(resolve_raw(repo_name, branch, repo_path(src)), quote=True),
               html.escape(alt or "", quote=True))
        )
        return key

    def raw_img(m):
        tag = m.group(0)
        attrs, src = [], None
        for am in re.finditer(
            r"""\b(src|width|height|alt)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))""",
            tag,
        ):
            k = am.group(1)
            val = am.group(2) if am.group(2) is not None else (
                am.group(3) if am.group(3) is not None else am.group(4)
            )
            if k == "src":
                src = html.escape(resolve_raw(repo_name, branch, repo_path(val)), quote=True)
            else:
                attrs.append('%s="%s"' % (k, html.escape(val, quote=True)))
        if src is None:
            return ""
        return '<img src="%s" loading="lazy" referrerpolicy="no-referrer" %s>' % (
            src,
            " ".join(attrs),
        )

    text = re.sub(r"<img\b[^>]*/?>", raw_img, text)
    text = re.sub(
        r"!\[([^\]]*)\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)", img_placeholder, text
    )
    text = re.sub(r"`([^`]+)`", lambda m: "<code>%s</code>" % html.escape(m.group(1)), text)
    text = re.sub(r"\*\*([^*]+)\*\*", lambda m: "<strong>%s</strong>" % m.group(1), text)
    text = re.sub(r"__([^_]+)__", lambda m: "<strong>%s</strong>" % m.group(1), text)
    text = re.sub(r"(?<!\*)\*([^*\n]+)\*(?!\*)", lambda m: "<em>%s</em>" % m.group(1), text)

    def link(m):
        lab, dst = m.group(1), m.group(2)
        if not dst.startswith(("http", "#", "mailto:")):
            key = repo_path(dst)
            if link_map and key in link_map:
                return '<a href="%s">%s</a>' % (
                    html.escape(link_map[key], quote=True),
                    lab,
                )
            return '<a href="%s">%s</a>' % (
                html.escape(resolve_ref(repo_name, branch, key), quote=True),
                lab,
            )
        return '<a href="%s" target="_blank" rel="noopener">%s</a>' % (
            html.escape(dst, quote=True),
            lab,
        )

    text = re.sub(r"\[([^\]]+)\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)", link, text)

    for key, val in img_tokens.items():
        text = text.replace(key, val)
    return text


def _norm(s):
    return re.sub(r"[^a-z0-9]", "", (s or "").lower())


def md_to_html(md, repo_name, branch, drop_h1_names=None, link_map=None, base_dir=""):
    """Render a pragmatic README subset to HTML.

    If drop_h1_names is given (normalized repo/display names), a leading
    "# <name>" heading is dropped because the page already shows that title.
    Any other README heading is preserved."""
    out = []
    lines = md.replace("\r\n", "\n").split("\n")
    i, n = 0, len(lines)
    first_h1_dropped = False
    drop_set = (
        {_norm(x) for x in drop_h1_names} if drop_h1_names is not None else None
    )

    def inline(t):
        return md_inline(t, repo_name, branch, link_map, base_dir)

    while i < n:
        line = lines[i]

        # Optionally drop the very first "# <repo title>" heading and the
        # badge row that follows it (both duplicate the generated page head).
        m0 = re.match(r"^#\s+(.+?)\s*$", line)
        if m0 and drop_set is not None and _norm(m0.group(1)) in drop_set:
            first_h1_dropped = True
            i += 1
            while i < n and lines[i].strip() == "":
                i += 1
            # Skip consecutive centered <p> blocks that only contain links or
            # images (badge rows / banners), plus surrounding blank lines.
            while i < n:
                if lines[i].strip() == "":
                    i += 1
                    continue
                if lines[i].strip().startswith('<p align="center">'):
                    i += 1
                    while i < n and not lines[i].strip() == "</p>":
                        i += 1
                    i += 1
                    continue
                break
            continue

        if line.startswith("```"):
            fence = []
            i += 1
            while i < n and not lines[i].startswith("```"):
                fence.append(lines[i])
                i += 1
            i += 1
            out.append("<pre class=\"doc-code\">%s</pre>" % html.escape("\n".join(fence)))
            continue

        m = re.match(r"^(#{1,6})\s+(.*)$", line)
        if m:
            lvl = len(m.group(1))
            out.append("<h%d>%s</h%d>" % (lvl, inline(m.group(2)), lvl))
            i += 1
            continue

        if re.match(r"^\s*(---+|\*\*\*+)\s*$", line):
            out.append("<hr>")
            i += 1
            continue

        if line.strip().startswith(("> ", ">")):
            quote = []
            while i < n and lines[i].strip().startswith(">"):
                quote.append(re.sub(r"^\s*>\s?", "", lines[i], count=1))
                i += 1
            inner = md_to_html(
                "\n".join(quote), repo_name, branch,
                drop_h1_names=list(drop_set) if drop_set is not None else None,
                link_map=link_map,
                base_dir=base_dir,
            )[0]
            out.append("<blockquote>%s</blockquote>" % inner)
            continue

        if re.match(r"^\s*[-*+]\s+", line) or re.match(r"^\s*\d+[.)]\s+", line):
            items = []
            ordered = bool(re.match(r"^\s*\d+[.)]\s+", line))
            while i < n:
                m2 = re.match(r"^\s*[-*+]\s+(.*)$", lines[i])
                m3 = re.match(r"^\s*\d+[.)]\s+(.*)$", lines[i])
                if ordered and m3:
                    items.append(m3.group(1))
                elif not ordered and m2:
                    items.append(m2.group(1))
                elif lines[i].strip() == "":
                    i += 1
                    continue
                else:
                    break
                i += 1
            tag = "ol" if ordered else "ul"
            out.append(
                "<%s>%s</%s>"
                % (
                    tag,
                    "".join("<li>%s</li>" % inline(it) for it in items),
                    tag,
                )
            )
            continue

        if line.startswith("|"):
            rows = []
            while i < n and lines[i].startswith("|"):
                rows.append(lines[i])
                i += 1
            if len(rows) >= 2:
                head = [c.strip() for c in rows[0].strip("|").split("|")]
                body = rows[2:]
                tbl = ["<table><thead><tr>"]
                tbl.extend("<th>%s</th>" % inline(c) for c in head)
                tbl.append("</tr></thead><tbody>")
                for r in body:
                    cells = [c.strip() for c in r.strip("|").split("|")]
                    tbl.append("<tr>")
                    tbl.extend("<td>%s</td>" % inline(c) for c in cells)
                    tbl.append("</tr>")
                tbl.append("</tbody></table>")
                out.append("".join(tbl))
                continue
            out.append(inline(line))
            i += 1
            continue

        if line.strip() == "":
            i += 1
            continue

        para = []
        while i < n and lines[i].strip() != "" and not lines[i].startswith(("#", "```", "|", "> ")):
            if re.match(r"^\s*[-*+]\s+", lines[i]) or re.match(r"^\s*\d+[.)]\s+", lines[i]):
                break
            para.append(lines[i])
            i += 1
        joined = " ".join(x.strip() for x in para)
        wrap = re.fullmatch(r"<p\b[^>]*>(.*)</p>", joined, re.S)
        if wrap:
            out.append('<div class="readme-img-block">%s</div>' % inline(wrap.group(1)))
        else:
            out.append("<p>%s</p>" % inline(joined))

    return "\n".join(out), first_h1_dropped


# --------------------------------------------------------------------------
# Rendering
# --------------------------------------------------------------------------

def human_size(num):
    num = num or 0
    for unit in ("B", "KB", "MB", "GB"):
        if num < 1024 or unit == "GB":
            return ("%.0f %s" % (num, unit)) if unit == "B" else ("%.1f %s" % (num, unit))
        num /= 1024.0
    return str(num)


def fmt_iso(iso):
    try:
        dt = datetime.datetime.fromisoformat(iso.replace("Z", "+00:00")).astimezone(
            datetime.timezone.utc
        )
        return dt.strftime("%Y-%m-%d")
    except Exception:
        return (iso or "")[:10]


PLATFORM_BY_EXT = {
    ".apk": ("Android APK", ""),
    ".dmg": ("macOS (.dmg)", ""),
    ".msi": ("Windows (.msi)", ""),
    ".exe": ("Windows (.exe)", ""),
    ".deb": ("Linux (.deb)", ""),
    ".jar": ("Server / other (.jar)", "muted"),
    ".ipa": ("iOS (.ipa)", "muted"),
}


def card_markup(proj, kind="featured"):
    """Markup for listing cards (home "featured" or warehouse)."""
    icon = proj["icon_url"] or proj["fallback_icon"]
    icon_html = (
        '<span class="card-icon"><img src="%s" alt="%s icon" loading="lazy"></span>\n            '
        % (html.escape(icon, quote=True), html.escape(proj["name"]))
    )
    chips = "".join(
        '<span class="chip chip-accent">%s</span>' % html.escape(p) for p in proj["platforms"]
    )
    chips += '<span class="chip">%s</span>' % html.escape(proj["license"] or "Open source")
    links = []
    if kind == "featured":
        links.append('<a class="linkbtn" href="projects/%s.html" data-i18n="card.project_page">Project page</a>'
                     % proj["slug"])
        if proj["docs"]:
            links.append('<a class="linkbtn linkbtn-ghost" href="%s" data-i18n="card.tech_docs">Technical docs</a>'
                         % html.escape(proj["docs"], quote=True))
        else:
            links.append('<a class="linkbtn linkbtn-ghost" href="%s" target="_blank" rel="noopener" data-i18n="card.repository">Repository</a>'
                         % html.escape(proj["repo_url"], quote=True))
    else:
        links.append('<a class="linkbtn" href="projects/%s.html" data-i18n="card.project_page">Project page</a>' % proj["slug"])
        if proj["docs"]:
            links.append('<a class="linkbtn linkbtn-ghost" href="%s" data-i18n="card.tech_docs">Technical docs</a>'
                         % html.escape(proj["docs"], quote=True))
        if proj["releases"]:
            links.append('<a class="linkbtn linkbtn-ghost" href="%s/releases" target="_blank" rel="noopener" data-i18n="card.releases">Releases</a>'
                         % proj["repo_url"])
        else:
            links.append('<a class="linkbtn linkbtn-ghost" href="%s" target="_blank" rel="noopener" data-i18n="card.source">Source</a>'
                         % proj["repo_url"])
    return (
        '<div class="card reveal">\n'
        "          %s\n"
        '          <h3>%s</h3>\n'
        '          <p class="tagline">%s</p>\n'
        '          <div class="meta">%s</div>\n'
        '          <div class="card-links">\n'
        '            %s\n'
        "          </div>\n"
        "        </div>"
        % (icon_html, html.escape(proj["name"]), html.escape(proj["tagline"]),
           chips, "\n            ".join(links))
    )


def featured_section(projects):
    return "\n".join(card_markup(p, "featured") for p in projects)


def warehouse_section(projects):
    return "\n".join(card_markup(p, "warehouse") for p in projects)


def mirrored_docs_section(projects):
    """Cards for the docs.html hub: mirrored documents per project, grouped."""
    cards = []
    for p in projects:
        docs = p.get("doc_pages") or []
        if not docs:
            continue
        icon = p["icon_url"] or p["fallback_icon"]
        cards.append(
            '<div class="card reveal">\n'
            '          <span class="card-icon"><img src="%s" alt="%s icon" loading="lazy"></span>\n'
            "          <h3>%s</h3>\n"
            "          %s\n"
            "        </div>"
            % (html.escape(icon, quote=True), html.escape(p["name"]),
               html.escape(p["name"]), grouped_doc_list(docs))
        )
    return "\n".join(cards)


DOC_CATEGORY_ORDER = ["Tutorials", "FAQ", "Security", "Audits", "Release notes", "Documents"]

DOC_CATEGORY_KEYS = {
    "Tutorials": "cat.tutorials",
    "FAQ": "cat.faq",
    "Security": "cat.security",
    "Audits": "cat.audits",
    "Release notes": "cat.release_notes",
    "Documents": "cat.documents",
}


def doc_category(path):
    low = path.lower()
    base = os.path.basename(low)
    if base.startswith("tutorial") or "/tutorials/" in low or "/tutorial/" in low:
        return "Tutorials"
    if base.startswith("faq") or "/faq/" in low:
        return "FAQ"
    if "audit" in base:
        return "Audits"
    if "shield" in base or base.startswith("security") or "/security/" in low:
        return "Security"
    if "changelog" in base or "release" in base:
        return "Release notes"
    return "Documents"


def grouped_doc_list(doc_pages):
    groups = {}
    for d in doc_pages:
        groups.setdefault(doc_category(d["path"]), []).append(d)
    parts = []
    for name in DOC_CATEGORY_ORDER:
        if name not in groups:
            continue
        items = "".join(
            '<li><a href="%s">%s</a></li>'
            % (html.escape(d["local"], quote=True), html.escape(d["title"]))
            for d in groups[name]
        )
        parts.append(
            '<div class="doc-group-title" data-i18n="%s">%s</div><ul class="doc-list">%s</ul>'
            % (DOC_CATEGORY_KEYS.get(name, "cat.documents"), html.escape(name), items)
        )
    return "".join(parts)


def replace_markers(path, start_marker, end_marker, content):
    with open(path, "r", encoding="utf-8") as f:
        src = f.read()
    if start_marker not in src or end_marker not in src:
        raise RuntimeError("markers missing in %s" % path)
    before = src.split(start_marker, 1)[0]
    after = src.split(end_marker, 1)[1]
    return before + start_marker + "\n" + content + "\n" + end_marker + after


def render_page(proj, readme_html, updated_utc):
    name, tagline = proj["name"], proj["tagline"]
    repo_url = proj["repo_url"]
    docs = proj["docs"]
    platforms, license_txt, stars, topics = (
        proj["platforms"], proj["license"], proj["stars"], proj["topics"]
    )
    icon = proj["icon_url"] or proj["fallback_icon"]

    chips = ["".join('<span class="chip chip-accent">%s</span>' % html.escape(p) for p in platforms)]
    if license_txt:
        chips.append('<span class="chip">%s</span>' % html.escape(license_txt))
    for t in topics[:5]:
        chips.append('<span class="chip">%s</span>' % html.escape(t))

    latest = proj["releases"][0] if proj["releases"] else None
    mirrored = proj.get("mirror") or {}

    def dl_row(asset, big=False):
        label, kind = PLATFORM_BY_EXT.get(
            os.path.splitext(asset["name"].lower())[1], ("Download", "muted")
        )
        dl = (
            '<span data-i18n="pg.downloads" data-i18n-args="%d">%d downloads</span>'
            % (asset["count"], asset["count"])
        ) if asset.get("count") else human_size(asset["size"])
        href = mirrored.get(asset["name"]) or asset["url"]
        return (
            '<a class="dl-btn%s %s" href="%s" target="_blank" rel="noopener">'
            '<span class="dl-label">%s</span>'
            '<span class="dl-sub">%s &middot; %s</span></a>'
            % (" big" if big else "", kind, html.escape(href, quote=True),
               html.escape(label), html.escape(asset["name"]), dl)
        )

    if latest:
        top = latest["assets"][:3] if latest["assets"] else []
        alln = len(latest["assets"])
        dl_latest = "".join(dl_row(a, big=True) for a in top)
        if alln == 0:
            dl_latest = (
                '<a class="dl-btn muted" href="%s" target="_blank" rel="noopener">'
                '<span class="dl-label" data-i18n="pg.no_assets">No downloadable assets</span>'
                '<span class="dl-sub" data-i18n="pg.open_release">open the release page</span></a>'
                % html.escape(latest["html_url"], quote=True)
            )
        elif alln > len(top):
            dl_latest += (
                '<a class="dl-btn muted" href="%s" target="_blank" rel="noopener">'
                '<span class="dl-label" data-i18n="pg.all_assets" data-i18n-args="%d">All %d assets</span>'
                '<span class="dl-sub" data-i18n="pg.open_release">open the release page</span></a>'
                % (html.escape(latest["html_url"], quote=True), alln, alln)
            )
        head_sub = (
            '<span class="rel-tag">%s</span><span class="rel-date">%s</span>'
            % (html.escape(latest["tag"]), fmt_iso(latest["published"]))
        )
    else:
        dl_latest = ""
        head_sub = '<span class="rel-tag" data-i18n="pg.no_releases">no releases yet</span>'

    rel_items = []
    for rel in proj["releases"][1:]:
        rel_items.append(
            '<a class="rel-item" href="%s" target="_blank" rel="noopener">'
            '<span class="rel-tag">%s</span>'
            '<span class="rel-date">%s</span>'
            '<span class="rel-assets">%d assets</span></a>'
            % (html.escape(rel["html_url"], quote=True), html.escape(rel["tag"]),
               fmt_iso(rel["published"]), len(rel["assets"]))
        )

    sidebar = []
    mirror_note = ""
    if mirrored:
        mirror_note = (
            '<div class="mirror-note" data-i18n="pg.mirror_note">'
            '<a href="/downloads/">Mirrored on nusv.github.io</a> '
            "&mdash; no VPN required.</div>"
        )
    sidebar.append(
        '<div class="card pg-side">'
        '<div class="side-title" data-i18n="pg.latest_release">Latest release</div>'
        '%s<div class="dl-group">%s%s</div></div>'
        % (head_sub, dl_latest, mirror_note)
    )
    if rel_items:
        sidebar.append(
            '<div class="card pg-side"><div class="side-title" data-i18n="pg.recent_versions">Recent versions</div>%s</div>'
            % "".join(rel_items)
        )
    info = []
    info.append('<div class="pg-info"><span data-i18n="pg.license">License</span><b>%s</b></div>' % html.escape(license_txt or "—"))
    info.append('<div class="pg-info"><span data-i18n="pg.stars">Stars</span><b>%d</b></div>' % stars)
    info.append('<div class="pg-info"><span data-i18n="pg.updated">Repository updated</span><b>%s</b></div>'
                % html.escape((proj.get("pushed_at") or "")[:10]))
    info.append('<div class="pg-info"><span data-i18n="pg.source">Source</span><a href="%s" target="_blank" rel="noopener">%s</a></div>'
                % (html.escape(repo_url, quote=True), html.escape(proj["repo"])))
    sidebar.append('<div class="card pg-side"><div class="side-title" data-i18n="pg.project_info">Project info</div>%s</div>'
                   % "".join(info))
    doc_pages = proj.get("doc_pages") or []
    if doc_pages:
        sidebar.append(
            '<div class="card pg-side"><div class="side-title" data-i18n="pg.documents">Documents</div>%s</div>'
            % grouped_doc_list(doc_pages)
        )

    # Screenshot gallery (optional, convention-based).
    gal = ""
    if proj["screenshots"]:
        imgs = "".join(
            '<a class="gal-item" href="%s" target="_blank" rel="noopener">'
            '<img src="%s" alt="Screenshot" loading="lazy" referrerpolicy="no-referrer"></a>'
            % (html.escape(resolve_raw(proj["repo"], proj["branch"], p), quote=True),
               html.escape(resolve_raw(proj["repo"], proj["branch"], p), quote=True))
            for p in proj["screenshots"][:8]
        )
        gal = '<h2 class="gal-title" data-i18n="pg.screenshots">Screenshots</h2><div class="gal">%s</div>' % imgs

    actions = ['<a class="btn btn-primary" href="#downloads" data-i18n="pg.download_latest">Download latest</a>']
    actions.append('<a class="btn btn-ghost" href="%s" target="_blank" rel="noopener" data-i18n="pg.github_repo">GitHub repository</a>' % html.escape(repo_url, quote=True))
    if docs:
        actions.append('<a class="btn btn-ghost" href="%s" data-i18n="pg.tech_docs">Technical docs</a>' % html.escape(docs, quote=True))

    page = """<!DOCTYPE html>
<html lang="en" data-i18n-title="title.project" data-i18n-args="{name}">
<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>%s — Downloads &amp; Introduction | United Science Vaca (NUSV)</title>
  <meta name="description" content="%s">
  <meta property="og:title" content="%s — NUSV">
  <meta property="og:description" content="%s">
  <meta property="og:image" content="%s">
  <meta property="og:type" content="website">
  <link rel="icon" type="image/png" href="../assets/img/nusv-logo.png">
  <link rel="stylesheet" href="../assets/css/style.css">
</head>
<body>

%s

  <main class="wrap">
    <nav class="pg-back"><a href="../warehouse.html" data-i18n="pg.back">&#8249; All projects</a></nav>

    <article class="pg">

      <div class="pg-head">
        <div class="pg-head-main">
          <span class="pg-icon"><img src="%s" alt="%s icon" loading="lazy"></span>
          <h1>%s</h1>
          <p class="lede">%s</p>
          <div class="meta pg-chips">%s</div>
          <div class="btn-row pg-actions">
            %s
          </div>
        </div>
      </div>

      <div class="pg-grid">
        <section class="pg-about">
          <div class="pg-updated"><span data-i18n="pg.readme_synced">README auto-synced from the repository</span> &middot; %s UTC</div>
          %s
          %s
        </section>
        <aside class="pg-sidecol" id="downloads">
          %s
        </aside>
      </div>

    </article>
  </main>

%s

  <script src="../assets/js/i18n.js"></script>
  <script src="../assets/js/main.js"></script>
</body>
</html>
""" % (
        html.escape(name),
        html.escape(tagline[:150], quote=True),
        html.escape(name),
        html.escape(tagline[:150], quote=True),
        html.escape(icon, quote=True),
        nav("../"),
        html.escape(icon, quote=True), html.escape(name), html.escape(name),
        html.escape(tagline),
        "".join(chips),
        "\n            ".join(actions),
        updated_utc,
        readme_html or "<p><em>No README yet — check the repository for details.</em></p>",
        gal,
        "".join(sidebar),
        footer("../"),
    )
    return page.replace('data-i18n-args="{name}"', 'data-i18n-args="%s"' % html.escape(name, quote=True))


def render_doc_page(proj, doc, updated_utc):
    """A mirrored markdown document rendered as a site page."""
    icon = proj["icon_url"] or proj["fallback_icon"]
    original = "%s/blob/%s/%s" % (proj["repo_url"], proj["branch"], doc["path"])
    page = """<!DOCTYPE html>
<html lang="en" data-i18n-title="title.doc" data-i18n-args="{doc}">
<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>%s — %s | United Science Vaca (NUSV)</title>
  <meta name="description" content="%s — mirrored from the %s repository.">
  <link rel="icon" type="image/png" href="../../assets/img/nusv-logo.png">
  <link rel="stylesheet" href="../../assets/css/style.css">
</head>
<body>

%s

  <main class="wrap">
    <nav class="pg-back"><a href="../../projects/%s.html">&#8249; %s</a></nav>

    <article class="doc mdoc">
      <div class="doc-top">
        <span class="pg-icon"><img src="%s" alt="%s icon" loading="lazy"></span>
        <h1>%s</h1>
        <p class="mdoc-meta"><span data-i18n="doc.mirrored_from">Mirrored from</span> <a href="%s" target="_blank" rel="noopener">%s/%s</a> &middot; <code>%s</code> &middot; <span data-i18n="doc.synced">synced</span> %s UTC &middot; <a href="%s" target="_blank" rel="noopener" data-i18n="doc.original">original on GitHub</a></p>
      </div>
      <section class="pg-about mdoc-body">
        %s
      </section>
    </article>
  </main>

%s

  <script src="../../assets/js/i18n.js"></script>
  <script src="../../assets/js/main.js"></script>
</body>
</html>
""" % (
        html.escape(doc["title"]), html.escape(proj["name"]),
        html.escape(doc["title"]), html.escape(proj["repo"]),
        nav("../../"),
        proj["slug"], html.escape(proj["name"]),
        html.escape(icon, quote=True), html.escape(proj["name"]),
        html.escape(doc["title"]),
        html.escape(proj["repo_url"], quote=True), ORG, html.escape(proj["repo"]),
        html.escape(doc["path"]),
        updated_utc,
        html.escape(original, quote=True),
        doc["body"],
        footer("../../"),
    )
    return page.replace('data-i18n-args="{doc}"', 'data-i18n-args="%s"' % html.escape(doc["title"], quote=True))




def discover_projects(token):
    repos = list_org_repos(token) or []
    if not repos:
        return None  # signal: treat as failure (never wipe pages)

    projects = []
    for r in sorted(repos, key=lambda x: (x.get("name") or "").lower()):
        name = r.get("name")
        if r.get("fork"):
            continue
        if PROJECT_TOPIC not in get_topics(name, token):
            continue

        branch = r.get("default_branch") or "main"
        conf = get_site_json(name, branch, token)

        display = conf.get("name") or re.sub(r"-NUSV$", "", name, flags=re.I) or name
        desc = r.get("description") or ""
        tagline = conf.get("tagline") or desc
        platforms = conf.get("platforms") or []
        docs = conf.get("docs") or None  # site-relative or absolute URL
        license_txt = ((r.get("license") or {}) or {}).get("spdx_id")
        icon_url = discover_icon(name, branch, token, conf.get("icon"))
        screens = get_screenshots(name, branch, token)
        doc_files = get_doc_paths(name, branch, token)
        releases = fetch_releases(name, token) or []

        projects.append(
            {
                "repo": name,
                "slug": conf.get("slug") or (
                    re.sub(r"[^a-z0-9-]+", "-", name.lower()).strip("-") or name.lower()
                ),
                "name": display,
                "tagline": tagline,
                "platforms": platforms,
                "docs": docs,
                "license": license_txt,
                "repo_url": "https://github.com/%s/%s" % (ORG, name),
                "icon_url": icon_url,
                "fallback_icon": "https://avatars.githubusercontent.com/u/314111571",
                "branch": branch,
                "releases": releases,
                "screenshots": screens,
                "doc_files": doc_files,
                "stars": r.get("stargazers_count", 0),
                "pushed_at": r.get("pushed_at", ""),
                "topics": conf.get("topics", []),
            }
        )
        print("discovered %s -> projects/%s.html" % (name, projects[-1]["slug"]))
    return projects


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--token", default=None)
    args = ap.parse_args()
    token = args.token or os.environ.get("NUSV_SYNC_TOKEN") or os.environ.get("GITHUB_TOKEN")

    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    outdir = os.path.join(root, "projects")
    os.makedirs(outdir, exist_ok=True)
    now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M")
    stamp_re = re.compile(r'<div class="pg-updated">.*?</div>', re.S)
    mdoc_stamp_re = re.compile(r"synced \d{4}-\d{2}-\d{2} \d{2}:\d{2} UTC")

    def strip_stamp(t):
        t = stamp_re.sub("", t)
        return mdoc_stamp_re.sub("synced UTC", t)

    projects = discover_projects(token)
    if projects is None:
        print("ERROR: could not list organization repositories", file=sys.stderr)
        sys.exit(1)
    if not projects:
        print("WARNING: no repositories carry topic '%s' — nothing generated" % PROJECT_TOPIC)

    # Attach mirrored download URLs (NUSV/downloads) when the mirror is
    # current for the project's latest release.
    manifest = get_mirror_manifest(token)
    mirror_repos = (manifest or {}).get("repos") or {}
    for proj in projects:
        proj["mirror"] = {}
        entry = mirror_repos.get(proj["repo"])
        if (
            entry
            and proj["releases"]
            and entry.get("tag") == proj["releases"][0]["tag"]
        ):
            proj["mirror"] = {
                a["name"]: a.get("mirror")
                for a in entry.get("assets", [])
                if a.get("mirror")
            }
            print("mirror matched for %s (%d asset(s))" % (proj["repo"], len(proj["mirror"])))

    changed = []
    slugs = {p["slug"] for p in projects}

    # -- mirrored documents (important .md files rendered as site pages) --
    docs_root = os.path.join(root, "mirror")
    os.makedirs(docs_root, exist_ok=True)
    generated_docs = set()

    for proj in projects:
        doc_pages = []
        used_slugs = set()
        link_map = {}

        # pass 1: decide slugs and build the link map up front, so documents
        # that link to each other can be rewritten in either direction.
        plan = []
        for path in proj.get("doc_files") or []:
            base_slug = doc_slug(path)
            slug = base_slug
            n = 2
            while slug in used_slugs:
                slug = "%s-%d" % (base_slug, n)
                n += 1
            used_slugs.add(slug)
            local = "/mirror/%s/%s.html" % (proj["slug"], slug)
            link_map[path] = local
            plan.append((path, slug, local))

        # pass 2: fetch and render
        for path, slug, local in plan:
            raw = fetch_repo_text(proj["repo"], proj["branch"], path, token)
            if raw is None:
                print("  doc missing: %s/%s" % (proj["repo"], path))
                continue
            title = extract_h1(raw) or prettify_name(path)
            body = md_to_html(
                raw, proj["repo"], proj["branch"],
                drop_h1_names=[title],
                link_map=link_map,
                base_dir=posixpath.dirname(path),
            )[0]
            doc = {"path": path, "title": title, "slug": slug, "local": local, "body": body}
            doc_pages.append(doc)
            rel = os.path.join("mirror", proj["slug"], slug + ".html")
            generated_docs.add(rel)
            page_html = localize_images(render_doc_page(proj, doc, now), token, root)
            dest = os.path.join(root, rel)
            os.makedirs(os.path.dirname(dest), exist_ok=True)
            old = ""
            if os.path.exists(dest):
                with open(dest, "r", encoding="utf-8") as f:
                    old = f.read()
            if strip_stamp(old) != strip_stamp(page_html):
                with open(dest, "w", encoding="utf-8") as f:
                    f.write(page_html)
                changed.append(rel)
            print("  doc %s -> %s" % (path, local))
        proj["doc_pages"] = doc_pages
        proj["link_map"] = link_map

    # -- project pages --
    for proj in projects:
        md = fetch_readme(proj["repo"], proj["branch"], token)
        drop_names = {proj["name"], proj["repo"], re.sub(r"-NUSV$", "", proj["repo"], flags=re.I)}
        readme_html = (
            md_to_html(
                md, proj["repo"], proj["branch"], drop_names,
                link_map=proj.get("link_map") or None,
            )[0]
            if md
            else None
        )
        page = localize_images(render_page(proj, readme_html, now), token, root)
        path = os.path.join(outdir, proj["slug"] + ".html")
        old = ""
        if os.path.exists(path):
            with open(path, "r", encoding="utf-8") as f:
                old = f.read()
        if strip_stamp(old) != strip_stamp(page):
            with open(path, "w", encoding="utf-8") as f:
                f.write(page)
            changed.append("projects/%s.html" % proj["slug"])
        print("synced %s" % proj["slug"])

    # -- remove pages for projects that are no longer discovered --
    for f in os.listdir(outdir):
        if not f.endswith(".html"):
            continue
        stem = f[:-5]
        if stem not in slugs:
            os.remove(os.path.join(outdir, f))
            changed.append("projects/%s (removed)" % stem)
            print("removed stale %s" % f)

    # -- remove stale mirrored doc pages (mirror/<slug>/ directories) --
    for entry in os.listdir(docs_root):
        d = os.path.join(docs_root, entry)
        if not os.path.isdir(d):
            continue
        for f in os.listdir(d):
            rel = os.path.relpath(os.path.join(d, f), root)
            if rel not in generated_docs:
                os.remove(os.path.join(d, f))
                changed.append(rel + " (removed)")
                print("removed stale %s" % rel)
        if not os.listdir(d):
            os.rmdir(d)

    # -- legacy location: mirrored docs used to live under docs/<slug>/ --
    legacy_root = os.path.join(root, "docs")
    for entry in os.listdir(legacy_root):
        d = os.path.join(legacy_root, entry)
        if not os.path.isdir(d):
            continue
        for f in os.listdir(d):
            os.remove(os.path.join(d, f))
            changed.append("docs/%s/%s (removed)" % (entry, f))
            print("removed legacy docs/%s/%s" % (entry, f))
        os.rmdir(d)

    # -- listing pages (marker regions) --
    def patch_file(path, start, end, section):
        try:
            patched = replace_markers(path, start, end, section)
        except RuntimeError as e:
            print("SKIP %s: %s" % (os.path.basename(path), e))
            return
        with open(path, "r", encoding="utf-8") as f:
            current = f.read()
        if patched != current:
            with open(path, "w", encoding="utf-8") as f:
                f.write(patched)
            changed.append(os.path.basename(path))

    patch_file(
        INDEX_FEATURED, FEAT_START, FEAT_END,
        localize_images(featured_section(projects), token, root),
    )
    patch_file(
        WAREHOUSE, WH_START, WH_END,
        localize_images(warehouse_section(projects), token, root),
    )
    patch_file(
        DOCS_INDEX, MIRRORED_START, MIRRORED_END,
        localize_images(mirrored_docs_section(projects), token, root),
    )

    print("changed: %s" % (", ".join(changed) if changed else "none"))
    sys.exit(0)


if __name__ == "__main__":
    main()