From c8066eb2542c810d8e62abd9872512eaca2140a8 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Tue, 28 Jul 2026 13:58:42 +0900 Subject: [PATCH] perf(dump): add --exclude so deploy-pages skips the game dump `python -m app.dump` regenerates every collection. With ~962k games the Pages build spent hours writing per-record files that the assemble step then deleted before upload (the last deploy ran 5h26m). Add a repeatable `--exclude COLLECTION` flag: `generate()` already accepted a `collections` list, so this just threads it through `run()` and the CLI via a new `resolve_collections()` helper. Unknown names raise instead of being ignored, so a typo in a workflow fails loudly. deploy-pages now runs `--exclude games`, which drops the wasted generation work and makes the post-hoc `rm -rf _site/v1/games` + manifest edit unnecessary (the manifest is written without games to begin with). --- .github/workflows/deploy-pages.yml | 20 +++++---------- app/dump.py | 40 +++++++++++++++++++++++++++--- tests/integration/test_dump.py | 19 +++++++++++++- 3 files changed, 60 insertions(+), 19 deletions(-) diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index 4b0987c..2c1a95a 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -44,7 +44,12 @@ jobs: - name: Generate static JSON dump + openapi.json env: TECHAPI_DATA_DIR: ${{ github.workspace }}/TechAPI/data - run: python -m app.dump --output dump + # Games are not published here: ~962k per-record files exceed the fixed + # GitHub Pages deployment window (the deploy step times out at + # "syncing_files"). Skipping them at generation time also keeps this + # step from writing files that would only be deleted before upload. + # They remain available as versioned data in the TechAPI repo. + run: python -m app.dump --output dump --exclude games - uses: actions/setup-node@v4 with: @@ -63,19 +68,6 @@ jobs: mkdir -p _site cp -r site/dist/. _site/ cp -r dump/. _site/ - # Games remain available as versioned data in the TechAPI repo, but - # publishing ~1M per-record files exceeds the fixed GitHub Pages - # deployment window (the deploy step times out at "syncing_files"). - # Exclude them from the published artifact to keep deploys reliable, - # mirroring the TechAPI homepage workflow. - rm -rf _site/v1/games - node <<'NODE' - const fs = require("fs"); - const path = "_site/v1/index.json"; - const manifest = JSON.parse(fs.readFileSync(path, "utf8")); - delete manifest.collections?.games; - fs.writeFileSync(path, JSON.stringify(manifest, null, 2) + "\n"); - NODE touch _site/.nojekyll - uses: actions/upload-pages-artifact@v3 diff --git a/app/dump.py b/app/dump.py index 36c3ac7..71a83ff 100644 --- a/app/dump.py +++ b/app/dump.py @@ -39,6 +39,22 @@ PAGE_LIMIT = 100 # API max page size (§7.3) +def resolve_collections(exclude: list[str] | None = None) -> list[str]: + """Return the collections to dump, minus ``exclude``. + + Unknown names raise instead of being ignored, so a typo in a workflow fails + loudly rather than silently dumping everything. + """ + if not exclude: + return list(COLLECTIONS) + unknown = sorted(set(exclude) - set(COLLECTIONS)) + if unknown: + raise ValueError( + f"unknown collection(s) {unknown}; valid names: {', '.join(COLLECTIONS)}" + ) + return [resource for resource in COLLECTIONS if resource not in set(exclude)] + + def _write_json(path: Path, data: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") @@ -98,23 +114,39 @@ def generate( return counts -def run(output_dir: Path = OUTPUT_DIR) -> None: +def run(output_dir: Path = OUTPUT_DIR, exclude: list[str] | None = None) -> None: from sqlmodel import Session from app.database import create_db_and_tables, engine from app.main import app from app.seed import seed + collections = resolve_collections(exclude) + create_db_and_tables() with Session(engine) as session: seed(session) with TestClient(app) as client: - counts = generate(client, output_dir) + counts = generate(client, output_dir, collections) total = sum(counts.values()) - print(f"Dumped {total} records to {output_dir}: {counts}") + skipped = sorted(set(COLLECTIONS) - set(collections)) + suffix = f" (skipped: {', '.join(skipped)})" if skipped else "" + print(f"Dumped {total} records to {output_dir}: {counts}{suffix}") if __name__ == "__main__": parser = argparse.ArgumentParser(description="Generate the TechAPI static JSON dump (§4.2)") parser.add_argument("--output", type=Path, default=OUTPUT_DIR, help="output directory") - run(parser.parse_args().output) + parser.add_argument( + "--exclude", + action="append", + default=[], + metavar="COLLECTION", + help=( + "collection to skip, repeatable (e.g. --exclude games). Useful when a " + "consumer does not publish a large collection: skipping it avoids " + "writing hundreds of thousands of files that are discarded anyway." + ), + ) + args = parser.parse_args() + run(args.output, args.exclude) diff --git a/tests/integration/test_dump.py b/tests/integration/test_dump.py index 9143848..5f0e9c2 100644 --- a/tests/integration/test_dump.py +++ b/tests/integration/test_dump.py @@ -5,9 +5,10 @@ import json from pathlib import Path +import pytest from fastapi.testclient import TestClient -from app.dump import generate +from app.dump import COLLECTIONS, generate, resolve_collections from tests.integration.mobile_device_fixtures import ensure_mobile_device_fixtures @@ -47,3 +48,19 @@ def test_dump_writes_scores_and_scored_count(client: TestClient, tmp_path: Path) cpus = manifest["collections"]["cpus"] assert isinstance(cpus["scored"], int) assert 0 <= cpus["scored"] <= cpus["count"] + + +def test_resolve_collections_defaults_to_everything() -> None: + assert resolve_collections() == COLLECTIONS + assert resolve_collections([]) == COLLECTIONS + + +def test_resolve_collections_drops_excluded_and_keeps_order() -> None: + resolved = resolve_collections(["games"]) + assert "games" not in resolved + assert resolved == [c for c in COLLECTIONS if c != "games"] + + +def test_resolve_collections_rejects_unknown_names() -> None: + with pytest.raises(ValueError, match="unknown collection"): + resolve_collections(["gmaes"])