perf(dump): add --exclude so deploy-pages skips generating the game dump

.github/workflows/deploy-pages.yml

@@ -44,7 +44,12 @@ jobs:
       - name: Generate static JSON dump + openapi.json
         env:
           TECHAPI_DATA_DIR: ${{ github.workspace }}/TechAPI/data
-        run: python -m app.dump --output dump
+        # Games are not published here: ~962k per-record files exceed the fixed
+        # GitHub Pages deployment window (the deploy step times out at
+        # "syncing_files"). Skipping them at generation time also keeps this
+        # step from writing files that would only be deleted before upload.
+        # They remain available as versioned data in the TechAPI repo.
+        run: python -m app.dump --output dump --exclude games
 
       - uses: actions/setup-node@v4
         with:
@@ -63,19 +68,6 @@ jobs:
           mkdir -p _site
           cp -r site/dist/. _site/
           cp -r dump/. _site/
-          # Games remain available as versioned data in the TechAPI repo, but
-          # publishing ~1M per-record files exceeds the fixed GitHub Pages
-          # deployment window (the deploy step times out at "syncing_files").
-          # Exclude them from the published artifact to keep deploys reliable,
-          # mirroring the TechAPI homepage workflow.
-          rm -rf _site/v1/games
-          node <<'NODE'
-          const fs = require("fs");
-          const path = "_site/v1/index.json";
-          const manifest = JSON.parse(fs.readFileSync(path, "utf8"));
-          delete manifest.collections?.games;
-          fs.writeFileSync(path, JSON.stringify(manifest, null, 2) + "\n");
-          NODE
           touch _site/.nojekyll
 
       - uses: actions/upload-pages-artifact@v3

app/dump.py

@@ -39,6 +39,22 @@
 PAGE_LIMIT = 100  # API max page size (§7.3)
 
 
+def resolve_collections(exclude: list[str] | None = None) -> list[str]:
+    """Return the collections to dump, minus ``exclude``.
+
+    Unknown names raise instead of being ignored, so a typo in a workflow fails
+    loudly rather than silently dumping everything.
+    """
+    if not exclude:
+        return list(COLLECTIONS)
+    unknown = sorted(set(exclude) - set(COLLECTIONS))
+    if unknown:
+        raise ValueError(
+            f"unknown collection(s) {unknown}; valid names: {', '.join(COLLECTIONS)}"
+        )
+    return [resource for resource in COLLECTIONS if resource not in set(exclude)]
+
+
 def _write_json(path: Path, data: object) -> None:
     path.parent.mkdir(parents=True, exist_ok=True)
     path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
@@ -98,23 +114,39 @@ def generate(
     return counts
 
 
-def run(output_dir: Path = OUTPUT_DIR) -> None:
+def run(output_dir: Path = OUTPUT_DIR, exclude: list[str] | None = None) -> None:
     from sqlmodel import Session
 
     from app.database import create_db_and_tables, engine
     from app.main import app
     from app.seed import seed
 
+    collections = resolve_collections(exclude)
+
     create_db_and_tables()
     with Session(engine) as session:
         seed(session)
     with TestClient(app) as client:
-        counts = generate(client, output_dir)
+        counts = generate(client, output_dir, collections)
     total = sum(counts.values())
-    print(f"Dumped {total} records to {output_dir}: {counts}")
+    skipped = sorted(set(COLLECTIONS) - set(collections))
+    suffix = f" (skipped: {', '.join(skipped)})" if skipped else ""
+    print(f"Dumped {total} records to {output_dir}: {counts}{suffix}")
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Generate the TechAPI static JSON dump (§4.2)")
     parser.add_argument("--output", type=Path, default=OUTPUT_DIR, help="output directory")
-    run(parser.parse_args().output)
+    parser.add_argument(
+        "--exclude",
+        action="append",
+        default=[],
+        metavar="COLLECTION",
+        help=(
+            "collection to skip, repeatable (e.g. --exclude games). Useful when a "
+            "consumer does not publish a large collection: skipping it avoids "
+            "writing hundreds of thousands of files that are discarded anyway."
+        ),
+    )
+    args = parser.parse_args()
+    run(args.output, args.exclude)

tests/integration/test_dump.py

@@ -5,9 +5,10 @@
 import json
 from pathlib import Path
 
+import pytest
 from fastapi.testclient import TestClient
 
-from app.dump import generate
+from app.dump import COLLECTIONS, generate, resolve_collections
 from tests.integration.mobile_device_fixtures import ensure_mobile_device_fixtures
 
 
@@ -47,3 +48,19 @@ def test_dump_writes_scores_and_scored_count(client: TestClient, tmp_path: Path)
     cpus = manifest["collections"]["cpus"]
     assert isinstance(cpus["scored"], int)
     assert 0 <= cpus["scored"] <= cpus["count"]
+
+
+def test_resolve_collections_defaults_to_everything() -> None:
+    assert resolve_collections() == COLLECTIONS
+    assert resolve_collections([]) == COLLECTIONS
+
+
+def test_resolve_collections_drops_excluded_and_keeps_order() -> None:
+    resolved = resolve_collections(["games"])
+    assert "games" not in resolved
+    assert resolved == [c for c in COLLECTIONS if c != "games"]
+
+
+def test_resolve_collections_rejects_unknown_names() -> None:
+    with pytest.raises(ValueError, match="unknown collection"):
+        resolve_collections(["gmaes"])