diff --git a/.github/AGENTS.md b/.github/AGENTS.md index 1ca0336..2343ec3 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -32,3 +32,8 @@ GitHub Actions here must verify the native foundation without side effects. MovieLens, do not regenerate `results/serving_latency.*` in CI, and do not bind a public port. The load-test script is a measured local run, not a CI job. + Phase 10 re-rank tests (`python/tests/test_rerank.py`, picked up by + unittest discovery) use synthetic graphs and must not download + MovieLens, run `scripts/run_rerank_eval.py`, or regenerate + `results/rerank_eval.*` or `results/serving_latency.*`. Do not add + LightGBM to the CI image. diff --git a/AGENTS.md b/AGENTS.md index 7dfca50..a74bb67 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,8 @@ Phase 1 native foundation is verified. ADR-001 (MovieLens 100K), ADR-002 (matrix factorization), ADR-005 (uniform sampling without replacement), ADR-006 (PyG SAGEConv on native `graph_sampler` neighborhoods), ADR-007 (multi-seed MF vs GraphSAGE leaderboard), ADR-008 (demo CLI -serving), and ADR-009 (HTTP serving on precomputed embeddings) are +serving), ADR-009 (HTTP serving on precomputed embeddings), and +ADR-010 (two-stage retrieval and re-ranking) are accepted; MovieLens 1M is deferred. The verified slice includes the C++ CSR graph, seeded neighbor sampler, MovieLens 100K `u.data` parser, `graph_sampler` bindings, a naive Python reference sampler with native parity tests, @@ -23,7 +24,7 @@ ADR-003 leave-one-out prep, MovieLens 100K download and on-disk a native-backed mini-batch neighborhood helper, GraphSAGE training on that harness (PyG `SAGEConv`, native neighborhoods), and CI. -The current verified tip is Phase 9 / ADR-009. Phase 2 is delivered for +The current verified tip is Phase 10 / ADR-010. Phase 2 is delivered for MovieLens 100K: download + on-disk `processed/` prep, and timing charts (native `graph_sampler` versus the Python reference sampler, with stored JSON/SVG under `results/`; synthetic graph by default; optional @@ -43,9 +44,16 @@ PairScorer-compatible checkpoint, score candidate movie IDs, print top-K (`python/sagerec_serve.py`, `scripts/demo_recommend.py`). Phase 9 is delivered for the ADR-009 HTTP API: offline embedding export, FastAPI `GET /recommend` and `/healthz` over a NumPy top-K index, and measured -latency under `results/serving_latency.*`. Tiny synthetic +latency under `results/serving_latency.*`. Phase 10 is delivered for +ADR-010 two-stage retrieval and re-ranking: stage 1 is the existing +embedding top-N, stage 2 is a small NumPy MLP (retrieval score, train-only +popularity, genre affinity, and recency) trained on the ADR-003 validation +labels inside that pool. Offline comparison is +`results/rerank_eval.json`. `GET /recommend` accepts optional `rerank` +(default false) and the load test records p50/p95/p99 for both modes +(`serving-latency-v2`). Tiny synthetic GraphSAGE metrics remain protocol smoke and must stay separate from the -stored 100K numbers. Do not add MovieLens 1M, GCN, or batch-export +stored 100K numbers. Do not add MovieLens 1M, GCN, LightGBM, or batch-export paths. Do not invent metrics or hand-edit timings. ## Read order @@ -82,6 +90,10 @@ Accepted and recorded in `docs/decisions.md`: - ADR-009: HTTP serving on precomputed embeddings (FastAPI `GET /recommend` and `/healthz`). NumPy exact top-K. Not batch export. FAISS is not required. +- ADR-010: Two-stage retrieval and re-ranking. Stage 1 is ADR-009 top-N + (default 200). Stage 2 is a NumPy MLP on train-only features, optional + on `GET /recommend` (`rerank=false` by default). Not LightGBM, not + per-request sampling or training, and not batch export. Do not add node2vec unless a superseding ADR accepts it. diff --git a/Dockerfile b/Dockerfile index 248bd63..a551534 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ -# SageRec HTTP serving image (ADR-009). +# SageRec HTTP serving image (ADR-009, optional ADR-010 re-rank). # Serves a precomputed sagerec_serving_artifact. It does not train, # compile graph_sampler, or download MovieLens. Mount the artifact at -# /models/embeddings.npz (or set SAGEREC_SERVING_ARTIFACT). +# /models/embeddings.npz (or set SAGEREC_SERVING_ARTIFACT). Optional +# re-ranking needs SAGEREC_RERANK_ARTIFACT and rerank=true on the request. FROM python:3.12-slim WORKDIR /app @@ -9,7 +10,7 @@ WORKDIR /app COPY python/requirements-serve.txt /tmp/requirements-serve.txt RUN pip install --no-cache-dir -r /tmp/requirements-serve.txt -COPY python/sagerec_serving.py python/sagerec_api.py /app/ +COPY python/sagerec_serving.py python/sagerec_rerank.py python/sagerec_api.py /app/ ENV PYTHONPATH=/app \ SAGEREC_SERVING_ARTIFACT=/models/embeddings.npz diff --git a/Makefile b/Makefile index 3b6cd27..d3ae637 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ -# SageRec Phase 9 HTTP serving (ADR-009). +# SageRec HTTP serving (ADR-009) and optional re-rank (ADR-010). # Export needs PYTHONPATH to include the Release build (graph_sampler) and python/. # The API process only needs python/ plus a .npz artifact. +# Optional: RERANK=path/to/rerank.npz (requests still need rerank=true). PYTHON ?= python3 export PYTHONPATH := $(CURDIR)/build:$(CURDIR)/python:$(PYTHONPATH) @@ -21,7 +22,8 @@ export-embeddings: $(if $(TITLES),--titles $(TITLES),) serve: - $(PYTHON) scripts/serve_api.py --artifact $(ARTIFACT) --host $(HOST) --port $(PORT) + $(PYTHON) scripts/serve_api.py --artifact $(ARTIFACT) --host $(HOST) --port $(PORT) \ + $(if $(RERANK),--rerank $(RERANK),) loadtest: $(PYTHON) scripts/loadtest_serving.py @@ -35,5 +37,6 @@ docker-build: docker-run: docker run --rm -p $(PORT):8000 \ -v $(abspath $(ARTIFACT)):/models/embeddings.npz:ro \ + $(if $(RERANK),-v $(abspath $(RERANK)):/models/rerank.npz:ro -e SAGEREC_RERANK_ARTIFACT=/models/rerank.npz,) \ -e SAGEREC_SERVING_ARTIFACT=/models/embeddings.npz \ $(IMAGE) diff --git a/README.md b/README.md index 75b41e2..c456404 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,17 @@ **Problem:** GraphSAGE training needs fast, seeded neighbor sampling on a leakage-safe user–movie graph. A PyG `NeighborLoader` wrapper would hide the systems work. -**Result:** SageRec puts a real C++17 CSR sampler in the Python training loop, then compares GraphSAGE to implicit matrix factorization on the same MovieLens 100K protocol, then serves top-K from a checkpoint and from a precomputed-embedding HTTP API. +**Result:** SageRec puts a real C++17 CSR sampler in the Python training loop, then compares GraphSAGE to implicit matrix factorization on the same MovieLens 100K protocol, then serves top-K from a checkpoint and from a precomputed-embedding HTTP API, with an optional second-stage re-ranker. | 6-second scan | Stored evidence | | --- | --- | | Native C++ `graph_sampler` is **111.177×** faster than a matching Python reference (median) | [`results/sampler_timing.md`](results/sampler_timing.md) | | GraphSAGE **beats** same-protocol MF on single-seed ML-100K **Recall@10** (0.047720 vs 0.038176, seed 7) | [`results/gnn_vs_mf_movielens_100k.md`](results/gnn_vs_mf_movielens_100k.md) | +| Retrieval + re-rank **beats** retrieval-only on single-seed ML-100K **Recall@10** (0.053022 vs 0.038176; NDCG@10 0.026510 vs 0.020303; MAP@10 0.018500 vs 0.014950; seed 7) | [`results/rerank_eval.md`](results/rerank_eval.md) | | Demo CLI loads a checkpoint and prints top-K | [`scripts/demo_recommend.py`](scripts/demo_recommend.py) | -| HTTP `GET /recommend` p50 / p95 / p99 is **19.909990 / 59.100509 / 85.913701 ms** (617.951794 req/s) | [`results/serving_latency.md`](results/serving_latency.md) | +| HTTP `GET /recommend` retrieval-only p50 / p95 / p99 is **23.443567 / 66.870426 / 93.590268 ms** (534.012774 req/s); with re-rank **21.012950 / 66.638233 / 94.781428 ms** (568.753940 req/s) | [`results/serving_latency.md`](results/serving_latency.md) | -Those figures are copied from committed `results/` (and the existing README tables below). They are **not** rounded further, **not** SOTA, and **not** production latency. Sampler timing is a synthetic CSR on one machine. The Recall@10 pair is Phase 5 single-seed provenance (in-repo mean layers); the 5-seed PyG `SAGEConv` leaderboard is a separate table. +Those figures are copied from committed `results/` (and the existing README tables below). They are **not** rounded further, **not** SOTA, and **not** production latency. Sampler timing is a synthetic CSR on one machine. The GraphSAGE Recall@10 pair is Phase 5 single-seed provenance (in-repo mean layers); the 5-seed PyG `SAGEConv` leaderboard is a separate table. The re-rank row is a different comparison: implicit-MF retrieval versus the same top-200 pool after a NumPy MLP. It is not a GraphSAGE number. Stage-1 hit rate at 200 is 0.357370, so re-ranking never sees the other test items. Public source: [github.com/sathariels/SageRec](https://github.com/sathariels/SageRec). @@ -26,6 +27,7 @@ Public source: [github.com/sathariels/SageRec](https://github.com/sathariels/Sag 5. **5-seed leaderboard** — seeds 7, 11, 13, 17, 19; mean ± sample std (ADR-007). 6. **Demo CLI** — load a GraphSAGE or PairScorer checkpoint, score candidate movie IDs, print top-K (ADR-008). 7. **HTTP API** — export those embeddings once, then `GET /recommend` with exact NumPy top-K (ADR-009). +8. **Two-stage re-rank** — stage 1 keeps the top N candidates; a small NumPy MLP reorders them from train-only features (ADR-010). `rerank` defaults to off. ```mermaid flowchart LR @@ -43,6 +45,7 @@ flowchart LR G --> S[Embedding export] H --> S S --> T[HTTP recommend API] + T --> U[Optional NumPy re-rank] D --> J[C++ vs Python timing charts] D --> R[Python reference sampler] ``` @@ -181,7 +184,40 @@ curl -s 'http://127.0.0.1:8000/healthz' curl -s 'http://127.0.0.1:8000/recommend?user_id=0&k=10' ``` -`GET /healthz` returns 200 when the artifact loaded. `GET /recommend` returns `item_id`, `title`, and `score`. An unknown `user_id` is HTTP 404. A missing or non-integer `user_id`, or `k < 1`, is HTTP 422. +`GET /healthz` returns 200 when the artifact loaded (`rerank_loaded` says whether a re-rank artifact was also loaded). `GET /recommend` returns `item_id`, `title`, and `score`. An unknown `user_id` is HTTP 404. A missing or non-integer `user_id`, or `k < 1`, is HTTP 422. + +### Optional re-rank (ADR-010) + +`rerank` defaults to false, so the response order and scores stay the ADR-009 ranking. `rerank=true` asks the process to take the stage-1 top `n` (default 200, and `n` must be at least `k`) and reorder them with a NumPy MLP. The request does not sample neighbors and does not train. It needs a `sagerec_rerank_artifact` version 1 file whose user count, item count, and `stage1_model` match the serving artifact. Without that file, or with `n < k`, the status is HTTP 422. + +```bash +PYTHONPATH=python python3 scripts/serve_api.py \ + --artifact results/serving/embeddings.npz \ + --rerank results/serving/rerank.npz \ + --host 127.0.0.1 --port 8000 +curl -s 'http://127.0.0.1:8000/recommend?user_id=0&k=10&rerank=true&n=200' +``` + +Each re-ranked row still has `item_id`, `title`, and `score` (the re-rank score) and also `retrieval_score`. `make serve ARTIFACT=... RERANK=...` passes the same flag. The Docker image copies `python/sagerec_rerank.py`. Mount the artifact and set `SAGEREC_RERANK_ARTIFACT` when you want the second stage. + +Train and compare on MovieLens 100K (needs an existing ADR-003 `processed/` tree and `u.item`, extracted from the official zip when it is already under `data/raw/`): + +```bash +PYTHONPATH=build:python python3 scripts/run_rerank_eval.py \ + --processed-dir data/processed \ + --raw-dir data/raw \ + --rerank-out results/serving/rerank_movielens_100k.npz +``` + +That writes [`results/rerank_eval.json`](results/rerank_eval.json) and the markdown table rendered from it. Stage 1 is implicit MF (`movielens_100k_config`, seed 7), the same dot product this API serves. Stage 2 is fit on validation items inside the top-200 pool. Test metrics are Recall@10, NDCG@10, and MAP@10 for retrieval-only and retrieval plus re-rank on that same pool. The re-ranker cannot recover a test item that stage 1 missed. This is a single-seed measurement. It is not a GraphSAGE result and it does not replace the Phase 5 comparison or the ADR-007 leaderboard. `results/serving/` stays gitignored. Default CI does not run this script. + +Copied from [`results/rerank_eval.json`](results/rerank_eval.json). Protocol `rerank-eval-v1`. 943 eligible users. Stage-1 hit rate at 200 is 0.357370. The MLP saw 442 validation hits (the other users have no training label). Listwise loss moved from 5.181280 to 5.172139 (uniform over 200 items is about 5.30), so this is a light reordering, not a large fit. Retrieval-only Recall@10 and NDCG@10 match the stored single-seed MF file because N is at least 10 and the tie-break matches. Hardware: Intel(R) Xeon(R) Processor, 4 logical CPUs, Python 3.12.3, NumPy 1.26.4. + +| System | Recall@10 | NDCG@10 | MAP@10 | +| --- | ---: | ---: | ---: | +| retrieval only | 0.038176 | 0.020303 | 0.014950 | +| retrieval + re-rank | 0.053022 | 0.026510 | 0.018500 | +| delta (re-rank − retrieval) | 0.014846 | 0.006207 | 0.003550 | The same targets exist on the `Makefile` (`make export-embeddings CHECKPOINT=...`, `make serve ARTIFACT=...`, `make loadtest`, `make test-serving`, `make docker-build`). The image only runs the API: @@ -195,16 +231,14 @@ docker run --rm -p 8000:8000 \ ### Measured recommend latency -Copied from [`results/serving_latency.json`](results/serving_latency.json) (markdown rendering in [`results/serving_latency.md`](results/serving_latency.md)). Protocol `serving-latency-v1`. Local uvicorn, httpx client, 1000 measured `GET /recommend` calls after 100 warm-up calls, concurrency 16, `k=10`, NumPy linear percentiles. Catalog: synthetic standard-normal embeddings, seed 7, 943 users, 1682 items, dim 16, 20 seen items per user. Those counts match public MovieLens 100K cardinality so the dot product is a similar size. The embeddings are not a trained model, and no ratings were read. Hardware: Intel(R) Xeon(R) Processor, 4 logical CPUs, Linux 6.12.94+, Python 3.12.3, NumPy 1.26.4, FastAPI 0.159.0, uvicorn 0.54.0, httpx 0.28.1. +Copied from [`results/serving_latency.json`](results/serving_latency.json) (markdown rendering in [`results/serving_latency.md`](results/serving_latency.md)). Protocol `serving-latency-v2`. Local uvicorn, httpx client, 1000 measured `GET /recommend` calls per mode after 100 warm-up calls, concurrency 16, `k=10`, stage-1 N = 200, NumPy linear percentiles. Catalog: synthetic standard-normal embeddings, seed 7, 943 users, 1682 items, dim 16, 20 seen items per user. Those counts match public MovieLens 100K cardinality so the dot product is a similar size. The embeddings are not a trained model, and no ratings were read. The re-ranker timed here is a seeded synthetic MLP, not the MovieLens-trained weights. Hardware: Intel(R) Xeon(R) Processor, 4 logical CPUs, Linux 6.12.94+, Python 3.12.3, NumPy 1.26.4, FastAPI 0.159.0, uvicorn 0.54.0, httpx 0.28.1. Top-level `latency_ms` in the JSON duplicates retrieval-only. -| Statistic | Value | -| --- | ---: | -| p50 | 19.909990 ms | -| p95 | 59.100509 ms | -| p99 | 85.913701 ms | -| throughput | 617.951794 requests/s | +| Mode | p50 ms | p95 ms | p99 ms | requests/s | +| --- | ---: | ---: | ---: | ---: | +| retrieval_only | 23.443567 | 66.870426 | 93.590268 | 534.012774 | +| retrieval_plus_rerank | 21.012950 | 66.638233 | 94.781428 | 568.753940 | -This is one-machine HTTP latency, not a service-level objective and not a ranking-quality claim. Regenerate with `PYTHONPATH=python python3 scripts/loadtest_serving.py` (that overwrites the committed files; do not hand-edit them). +This is one-machine HTTP latency, not a service-level objective and not a ranking-quality claim. The MLP is a second in-process NumPy pass over 200 candidates. On this run the re-rank p50 is not higher than retrieval-only; the gap is inside HTTP variance, so it is not evidence that re-ranking is faster. Regenerate with `PYTHONPATH=python python3 scripts/loadtest_serving.py` (that overwrites the committed files; do not hand-edit them). API tests (pytest + `TestClient`, tiny fixture, no public port): @@ -244,22 +278,23 @@ If `files.grouplens.org` presents an expired TLS certificate, the downloader ret | Scoring protocol | [`python/sagerec_scoring.py`](python/sagerec_scoring.py) — `PairScorer` for baseline and GraphSAGE | | Training negatives | [`python/sagerec_negatives.py`](python/sagerec_negatives.py) — exclude known positives in the caller-supplied scope | | MF baseline | [`python/sagerec_baseline.py`](python/sagerec_baseline.py) — seeded NumPy logistic SGD | -| Ranking metrics | [`python/sagerec_metrics.py`](python/sagerec_metrics.py) — per-user then macro-averaged Recall@10 and NDCG@10 | +| Ranking metrics | [`python/sagerec_metrics.py`](python/sagerec_metrics.py) — per-user then macro-averaged Recall@10 and NDCG@10; binary AP@k for the ADR-010 MAP@10 comparison | | Mini-batch harness | [`python/sagerec_minibatch.py`](python/sagerec_minibatch.py) — train-only CSR + native multi-hop `sample_neighbors` | | GraphSAGE trainer | [`python/sagerec_graphsage.py`](python/sagerec_graphsage.py) — PyG `SAGEConv` + `PairScorer` on native samples | | Demo CLI | [`python/sagerec_serve.py`](python/sagerec_serve.py) + [`scripts/demo_recommend.py`](scripts/demo_recommend.py) — schema-v1 checkpoint load, candidate scores, top-K print (ADR-008) | | HTTP serving | [`python/sagerec_export.py`](python/sagerec_export.py), [`python/sagerec_serving.py`](python/sagerec_serving.py), [`python/sagerec_api.py`](python/sagerec_api.py) — offline embeddings, NumPy top-K, `GET /recommend` (ADR-009) | +| Two-stage re-rank | [`python/sagerec_rerank.py`](python/sagerec_rerank.py) — train-only features and a NumPy listwise MLP; optional `rerank` on `GET /recommend` (ADR-010) | | Comparison writer | [`python/sagerec_compare.py`](python/sagerec_compare.py) — MF vs GraphSAGE JSON, markdown table, SVG chart from stored results | | Multi-seed aggregator | [`python/sagerec_multiseed.py`](python/sagerec_multiseed.py) — per-seed rows plus mean±sample std (ADR-007) | | Sampler timing | [`python/sagerec_sampler_benchmark.py`](python/sagerec_sampler_benchmark.py) — native vs reference `sample_neighbors` JSON/SVG | Callers must pass **training-positive** `local_pairs()` into `BipartiteCSR`. The reference sampler reads CSR `offsets`/`neighbors`; it does not build graphs or ingest ratings files. -Layout: [`cpp/`](cpp/) native core, [`python/`](python/) prep/baseline/metrics/mini-batch/GraphSAGE/serve/HTTP/comparison/multi-seed/benchmark/tests, [`docs/`](docs/) contracts and ADRs, [`data/`](data/) schemas (no raw dataset), [`results/`](results/) for metrics and charts, [`scripts/`](scripts/) thin download/prep/MF/GraphSAGE/timing/multi-seed/demo-CLI/export/serve/load-test launchers. +Layout: [`cpp/`](cpp/) native core, [`python/`](python/) prep/baseline/metrics/mini-batch/GraphSAGE/serve/HTTP/re-rank/comparison/multi-seed/benchmark/tests, [`docs/`](docs/) contracts and ADRs, [`data/`](data/) schemas (no raw dataset), [`results/`](results/) for metrics and charts, [`scripts/`](scripts/) thin download/prep/MF/GraphSAGE/timing/multi-seed/demo-CLI/export/serve/load-test/re-rank launchers. -**In this repo:** C++ CSR + seeded sampler, MovieLens 100K parser/download/prep, native mini-batch GraphSAGE (`SAGEConv`), implicit MF, shared evaluator, single-seed and 5-seed comparisons, sampler timing charts, demo CLI, HTTP serving on precomputed embeddings, Release CMake + GitHub Actions CI. +**In this repo:** C++ CSR + seeded sampler, MovieLens 100K parser/download/prep, native mini-batch GraphSAGE (`SAGEConv`), implicit MF, shared evaluator, single-seed and 5-seed comparisons, sampler timing charts, demo CLI, HTTP serving on precomputed embeddings, optional NumPy re-ranker, Release CMake + GitHub Actions CI. -**Not in this repo:** PyG NeighborLoader / ClusterLoader (rejected for primary experiments), batch export, MovieLens 1M, GCN, node2vec. +**Not in this repo:** PyG NeighborLoader / ClusterLoader (rejected for primary experiments), batch export, MovieLens 1M, GCN, node2vec, LightGBM on the request path. The MF ranking smoke unittest is a tiny synthetic path. MovieLens 100K implicit-MF and GraphSAGE metrics (Recall@10 / NDCG@10) with split, seed, hyperparameter, and eligibility provenance are stored under [`results/`](results/). Phase 5 files are **single-seed** historical provenance. The ADR-007 leaderboard is a separate 5-seed SAGEConv measurement with mean±sample std. The MF JSON is not a GraphSAGE result. @@ -278,6 +313,7 @@ Recorded in [`docs/decisions.md`](docs/decisions.md): | ADR-007 | Accepted | 5-seed MovieLens 100K MF vs GraphSAGE leaderboard (seeds 7, 11, 13, 17, 19) | | ADR-008 | Accepted | Demo CLI (load checkpoint, score candidates, print top-K). HTTP prohibition superseded by ADR-009. Not batch export | | ADR-009 | Accepted | HTTP API on precomputed embeddings (`GET /recommend`, `/healthz`); NumPy top-K; FAISS not required | +| ADR-010 | Accepted | Two-stage retrieval and re-ranking: embedding top-N, then an optional NumPy MLP. `rerank` defaults to false. Not LightGBM | The native and Python reference samplers implement ADR-005 (full neighborhood when `k >= degree`; empty for isolated nodes or `k = 0`). Changing replacement policy requires a superseding ADR. @@ -300,7 +336,7 @@ ctest --test-dir build --output-on-failure --build-config Release Use `g++` (or another complete C++17 toolchain). A `c++` symlink that points at Clang without a discoverable `libstdc++` will fail at configure time. -CTest runs native CSR/sampler/parser tests, Python unittest discovery (bindings, sampler parity, leave-one-out leakage, download/prep fixtures, MF ranking smoke, mini-batch native sampling, GraphSAGE training smoke, sampler timing schema/parity, multi-seed aggregation, demo-CLI checkpoint/top-K), and pytest `python_serving_api` (FastAPI `TestClient`). Install `python/requirements-serve-test.txt` before that pytest target. After a successful build: +CTest runs native CSR/sampler/parser tests, Python unittest discovery (bindings, sampler parity, leave-one-out leakage, download/prep fixtures, MF ranking smoke, mini-batch native sampling, GraphSAGE training smoke, sampler timing schema/parity, multi-seed aggregation, demo-CLI checkpoint/top-K, re-rank features/MLP), and pytest `python_serving_api` (FastAPI `TestClient`, including optional `rerank`). Install `python/requirements-serve-test.txt` before that pytest target. After a successful build: ```bash PYTHONPATH=build:python python3 -m unittest discover -s python/tests -v @@ -313,6 +349,7 @@ PYTHONPATH=build:python python3 -m unittest python/tests/test_multiseed_gnn_vs_m PYTHONPATH=build:python python3 -m unittest python/tests/test_sampler_benchmark.py -v PYTHONPATH=build:python python3 -m unittest python/tests/test_serve_demo.py -v PYTHONPATH=build:python python3 -m pytest python/tests/test_serving_api.py -q +PYTHONPATH=build:python python3 -m unittest python/tests/test_rerank.py -v ``` Default CI does **not** download MovieLens. Set `SAGEREC_LIVE_MOVIELENS=1` only for the optional live-archive test. @@ -342,7 +379,8 @@ The mini-batch helper in [`python/sagerec_minibatch.py`](python/sagerec_minibatc - Native vs Python reference sampler timings are stored under `results/sampler_timing.json` (synthetic graph, median of measured repetitions). They are one-machine evidence, not a production latency or SOTA claim. - Production message passing is PyG `SAGEConv` on native mini-batches (ADR-006). PyG NeighborLoader is not the neighborhood source. - The ADR-008 demo CLI loads a local checkpoint and prints top-K. It is not a batch exporter and not a production latency claim. -- ADR-009 HTTP latency is stored under `results/serving_latency.json` (synthetic catalog, one machine, p50/p95/p99 and throughput). It is not a ranking-quality result and not a service-level objective. +- ADR-009 HTTP latency is stored under `results/serving_latency.json` (synthetic catalog, one machine, p50/p95/p99 and throughput for retrieval-only and for retrieval plus a synthetic re-ranker; protocol `serving-latency-v2`). It is not a ranking-quality result and not a service-level objective. Top-level latency duplicates the retrieval-only mode. +- The ADR-010 MovieLens comparison is stored under `results/rerank_eval.json` (single-seed implicit MF retrieval versus the same pool after a NumPy MLP). It is not GraphSAGE and does not replace Phase 5 or the 5-seed leaderboard. MAP is MAP@10. The re-ranker is trained on validation labels inside top-N (442 of 943 users) and cannot recover items stage 1 missed (hit rate at 200 is 0.357370). Listwise loss stayed near log(200). - This GitHub repository is the public homepage. Do not treat an Origin (or other private) URL as the project home. Acceptance criteria and component contracts: [`docs/project-requirements.md`](docs/project-requirements.md), [`docs/architecture.md`](docs/architecture.md). Contributors: read [`AGENTS.md`](AGENTS.md) before changing code. diff --git a/cpp/AGENTS.md b/cpp/AGENTS.md index 6412fab..9f645dc 100644 --- a/cpp/AGENTS.md +++ b/cpp/AGENTS.md @@ -64,8 +64,11 @@ support. Do not add MovieLens 1M code, downloads, or file-path helpers. and Phase 8 demo-CLI tests (missing/corrupt checkpoints, synthetic top-K; no MovieLens download). CTest `python_serving_api` runs pytest on - `python/tests/test_serving_api.py` (ADR-009 FastAPI TestClient, - synthetic artifact; no MovieLens download and no public port). + `python/tests/test_serving_api.py` (ADR-009 FastAPI TestClient plus + the ADR-010 `rerank` flag, synthetic artifact; no MovieLens download + and no public port). Unittest discovery also runs + `python/tests/test_rerank.py` (synthetic features and MLP; no + MovieLens download). PYTHONPATH includes the build directory and `python/` so `sagerec_*` modules import cleanly. diff --git a/data/AGENTS.md b/data/AGENTS.md index cfc9159..acb7e80 100644 --- a/data/AGENTS.md +++ b/data/AGENTS.md @@ -13,7 +13,9 @@ data and generated processed artifacts are not committed. - Native 100K parsing is `sagerec::parse_movielens_100k` on in-memory `u.data` text. Split assignment stays in `python/sagerec_prep.py` (ADR-003). Phase 2 download is `python/sagerec_download.py`; on-disk prep is - `python/sagerec_dataset.py`. Do not add MovieLens 1M. + `python/sagerec_dataset.py`. ADR-010 genre features read `u.item` + from `raw/` (extracted from the official zip; not committed). Do not + add MovieLens 1M. - The manifest schema is `processed/manifest.schema.json`. On-disk prep fills edition, split policy, eligibility/cold-start defaults, counts, source URL, checksum, and license. diff --git a/data/raw/AGENTS.md b/data/raw/AGENTS.md index aac18d3..a499ca6 100644 --- a/data/raw/AGENTS.md +++ b/data/raw/AGENTS.md @@ -6,7 +6,9 @@ tab-separated `u.data` (source user id, movie id, rating, timestamp). Phase 2 download (`python/sagerec_download.py` / `scripts/download_movielens_100k.py`) fetches the official GroupLens MovieLens 100K zip, verifies the published MD5 -when the default URL is used, and extracts `ml-100k/u.data`. Native parsing +when the default URL is used, and extracts `ml-100k/u.data`. The ADR-010 +re-rank script may also extract `ml-100k/u.item` from that same zip into +this gitignored directory for genre flags. Native parsing stays in-memory; it does not read this directory itself. Record source URL, dataset license reference, archive checksum, and extracted diff --git a/docs/AGENTS.md b/docs/AGENTS.md index da32d2c..280356f 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -12,10 +12,13 @@ decisions, evaluation protocol, and phased delivery plans. chronological leave-one-out), ADR-004 (matrix factorization), ADR-005 (uniform sampling without replacement), ADR-006 (PyG SAGEConv on native neighborhoods), ADR-007 (multi-seed MF vs GraphSAGE - leaderboard), ADR-008 (demo CLI serving), and ADR-009 (HTTP serving - on precomputed embeddings) are accepted. Do not add MovieLens 1M, - GCN, node2vec, or batch export. ADR-009 supersedes ADR-008's no-HTTP + leaderboard), ADR-008 (demo CLI serving), ADR-009 (HTTP serving + on precomputed embeddings), and ADR-010 (two-stage retrieval and + re-ranking) are accepted. Do not add MovieLens 1M, + GCN, node2vec, LightGBM, or batch export. ADR-009 supersedes ADR-008's no-HTTP clause; do not put sampling or training on the HTTP request path. + ADR-010 re-ranking is an optional second NumPy pass over precomputed + features. Default CI does not run the 100K re-rank job. Do not switch primary experiments to a PyG NeighborLoader. - Record consequential decisions with context, alternatives, rationale, and consequences. - Define metric and benchmark protocols before producing results. diff --git a/docs/architecture.md b/docs/architecture.md index 95f4966..0ea82e7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -246,6 +246,10 @@ does not use a PyG NeighborLoader. artifact and answer `GET /recommend` from FastAPI (ADR-009). Request time is a NumPy dot product. Native sampling runs at export, not per request. +11. Optionally re-rank the stage-1 top-N with a NumPy MLP trained on + train-only features (ADR-010). The MovieLens comparison uses implicit + MF as stage 1 and reports test Recall@10, NDCG@10, and MAP@10. + Request time stays a second NumPy pass; it does not train or sample. The GNN family is GraphSAGE (ADR-002). Production message passing is PyG `SAGEConv` (ADR-006) on CPU, still fed by native samples. Ranking on 100K @@ -311,9 +315,47 @@ The runtime image (`Dockerfile`) contains that app and the artifact loader only. `results/serving_latency.json` and `.md` are a measured local load test -(p50/p95/p99 and throughput) with CPU provenance. The committed catalog -is synthetic noise at MovieLens 100K user/item counts. It is not a -ranking-quality result and not a production latency claim. +(p50/p95/p99 and throughput) with CPU provenance. Protocol +`serving-latency-v2` records retrieval-only and retrieval-plus-re-rank. +Top-level `latency_ms` duplicates retrieval-only. The committed catalog +is synthetic noise at MovieLens 100K user/item counts, and the timed +re-ranker is a seeded synthetic MLP. It is not a ranking-quality result +and not a production latency claim. + +## Two-stage re-ranking (ADR-010) + +Stage 1 is the ADR-009 score +(`dot(user, item) + user_bias + item_bias`). Already-seen train +positives are removed. The stage keeps the top N unseen items (default +200, and N >= K). Ties stay score descending, then smaller item id. + +Stage 2 is `python/sagerec_rerank.py`: a NumPy MLP with one ReLU hidden +layer and a linear skip, +`relu(x W1 + b1) W2 + b2 + x w_linear`, on standardized features. +Feature order is retrieval score, train-only `log1p` item popularity, +train-only genre affinity (user mean genre vector dot the item's +`u.item` flags), and train-only item recency (min-max of the item's +latest train timestamp). Validation and test rows do not enter those +tables. The MLP is trained with listwise softmax on validation items +that stage 1 retrieved. Test pools drop the validation item. The +re-ranker cannot recover an item outside top-N. + +The published MovieLens comparison +(`python/sagerec_rerank_eval.py`, `results/rerank_eval.json`) uses +implicit MF (`movielens_100k_config`, seed 7) as stage 1 because that +is the dot product ADR-009 already serves. It reports Recall@10, +NDCG@10, and MAP@10 for retrieval-only and retrieval-plus-re-rank on +the same stage-1 pool. MAP@10 with one relevant item is `1/rank` when +the rank is at most 10, else 0. That file is single-seed and is not a +GraphSAGE result and not the ADR-007 leaderboard. + +`GET /recommend` adds `rerank` (default false) and `n`. With the flag +off the response order and scores match ADR-009. With the flag on, the +process loads `SAGEREC_RERANK_ARTIFACT` (`sagerec_rerank_artifact` +version 1), checks user/item counts and `stage1_model`, and returns +`score` (re-rank) plus `retrieval_score`. A missing artifact or `n < k` +is HTTP 422. Unknown users stay HTTP 404. The request does not import +PyTorch, PyG, LightGBM, or `graph_sampler`. ## Evaluation boundary @@ -333,7 +375,10 @@ The shared protocol for this slice: split is test. The evaluation positive stays in the candidate set. - Rank by score descending; ties break on smaller `movie_id` (the ADR-008 demo CLI uses this same tie-break). -- Report Recall@10 and NDCG@10 per user, then macro-average. +- Report Recall@10 and NDCG@10 per user, then macro-average. The + ADR-010 two-stage comparison also reports MAP@10 on the stage-1 pool + (one relevant item). That MAP does not replace the GNN-versus-MF + Recall@10 / NDCG@10 protocol. - Tiny synthetic unittest metrics are protocol verification, not a MovieLens 100K quality claim. Stored 100K numbers live under `results/` with provenance. Phase 5 single-seed files are diff --git a/docs/decisions.md b/docs/decisions.md index b7d74e0..9f6a466 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -16,6 +16,7 @@ choice, alternatives, rationale, and consequences. | ADR-007 | Published quality comparison | Multi-seed MF vs GraphSAGE with uncertainty / keep Phase 5 single-seed only | Accepted: 5-seed MovieLens 100K leaderboard (seeds include 7) | | ADR-008 | Production serving demo | Demo CLI / batch export / local HTTP API | Accepted: Demo CLI (HTTP prohibition superseded by ADR-009) | | ADR-009 | HTTP serving | Precomputed-embedding HTTP API / CLI only / per-request GraphSAGE | Accepted: FastAPI on offline embeddings (NumPy top-K) | +| ADR-010 | Two-stage ranking | Retrieval plus learned re-rank / retrieval only / LightGBM on the request | Accepted: optional NumPy MLP re-rank of the embedding top-N | ## Accepted @@ -448,6 +449,100 @@ checkpoint checks that do not bind a port. latency file, and does not bind a public port. - The ADR-008 demo CLI remains. Batch export stays rejected. MovieLens 1M, GCN, and node2vec remain deferred or rejected. + Optional re-ranking of the same embeddings is ADR-010. It does not + put sampling or training on the request path. + +### ADR-010: Two-stage retrieval and re-ranking + +- Date: 2026-10-23 +- Owner: Nithilan Kumaran +- Status: Accepted + +**Context:** ADR-009 serves `GET /recommend` by an exact NumPy dot product +over precomputed embeddings and returns top-K. A second ranker can reorder +a short candidate list using features the embedding does not see directly +(popularity, genre affinity, recency) without moving neighbor sampling or +training onto the request. The re-ranker has to stay inside the MovieLens +100K / ADR-003 protocol, keep held-out interactions out of its features, +and remain optional so the ADR-009 path still answers when no re-rank +artifact is loaded. LightGBM LambdaRank was the other concrete option. +MovieLens 1M, GCN, node2vec, batch export, and FAISS stay out of scope. + +**Choice:** Two stages. + +- **Stage 1** is the existing embedding retrieval. Score is + `dot(user, item) + user_bias + item_bias`. Already-seen train positives + are removed. The stage keeps the top **N** unseen items (default + **N = 200**, configurable, and **N >= K**). Tie-break stays score + descending, then smaller item id. +- **Stage 2** is a small NumPy MLP with one ReLU hidden layer and a linear + skip, trained with listwise softmax on that retrieved pool. It is not + LightGBM and not a PyTorch module, so the HTTP process stays free of + PyTorch, PyG, and `graph_sampler`. +- Features, in order: retrieval score, train-only item popularity + (`log1p` count), train-only genre affinity (user mean genre vector dot + the item's MovieLens `u.item` flags), and train-only item recency + (min-max of the item's latest train timestamp). Validation and test + interactions do not enter those tables. If genres are absent the + affinity column is zero and the run records `genres_available: false`. + The MovieLens script requires `u.item`. +- Training uses the ADR-003 split. The published stage-1 model is + implicit MF with `movielens_100k_config` (seed 7), which is the same + dot-product retrieval ADR-009 already serves. The MLP is fit on + validation items that stage 1 retrieved, with test positives removed + from those training pools. Test evaluation compares retrieval-only and + retrieval-plus-re-rank on the **same** stage-1 pool after removing + train and validation positives. Metrics are Recall@10, NDCG@10, and + MAP@10 (one relevant item: `1/rank` if the rank is <= 10, else 0). +- Serving: `GET /recommend` gains `rerank` (default **false**) and `n` + (stage-1 width, default 200 or the artifact's `stage1_n`). `rerank=true` + requires `SAGEREC_RERANK_ARTIFACT`. The loaded artifact's user/item + counts and `stage1_model` must match the serving artifact. The request + does not train, sample, or read MovieLens. The returned `score` is the + re-rank score; `retrieval_score` is also returned. Seen items stay + excluded. +- The load test records p50/p95/p99 and throughput for both modes. There + is no separate numeric SLO. The latency budget is that re-ranking is a + second in-process NumPy pass over N candidates using precomputed + tables, not a training or sampling step. + +**Alternatives:** Leave serving as retrieval only. Re-rank with LightGBM +LambdaRank on the request path. Recompute GraphSAGE per request and then +re-rank. Train the re-ranker on test labels. + +**Rationale:** The owner asked for two-stage retrieval and re-ranking on +2026-10-23. Keeping stage 1 as the ADR-009 dot product means the candidate +generator is already tested. A four-feature MLP is small enough to export +as NumPy weights and run inside the existing latency budget without a new +native library. LightGBM would put another native stack on the request +and in the serving image. Fitting on validation and reporting on test +keeps the held-out split honest. Default `rerank=false` preserves the +ADR-009 response order when the flag is off. + +**Consequences:** + +- `python/sagerec_rerank.py` owns features, the MLP, the re-rank + artifact (`sagerec_rerank_artifact` version 1), and the in-memory + two-stage eval. It does not import PyTorch, PyG, or `graph_sampler`. +- `python/sagerec_rerank_eval.py` and `scripts/run_rerank_eval.py` train + implicit MF on MovieLens 100K and write `results/rerank_eval.json` + (and the markdown rendered from it). That file is a single-seed + two-stage measurement. It does not replace + `results/mf_movielens_100k.json`, the Phase 5 comparison, or the + ADR-007 leaderboard, and it is not a GraphSAGE result. +- `python/sagerec_api.py` accepts optional `rerank` and `n`. Unknown + users stay HTTP 404. `rerank=true` without an artifact, or `n < k`, + is HTTP 422. +- `python/sagerec_loadtest.py` writes protocol `serving-latency-v2` + with both modes. The committed catalog is still synthetic noise at + MovieLens 100K cardinality, and the timed re-ranker is a seeded + synthetic MLP, not the MovieLens-trained weights. Do not hand-edit + those numbers. Top-level `latency_ms` duplicates retrieval-only. +- Default CI runs the synthetic feature, MLP, and API tests. It does + not download MovieLens, does not run the 100K re-rank job, and does + not regenerate `results/serving_latency.*`. +- Batch export, MovieLens 1M, GCN, node2vec, and FAISS remain rejected + or deferred. ## Proposals diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 55873d3..9c44283 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -4,8 +4,9 @@ ADR-001 (MovieLens 100K), ADR-002 (GraphSAGE), ADR-003 (per-user chronological leave-one-out), ADR-004 (matrix factorization), ADR-005 (uniform sampling without replacement), ADR-006 (PyG SAGEConv on native neighborhoods), ADR-007 (5-seed MF vs GraphSAGE leaderboard), -ADR-008 (demo CLI serving), and ADR-009 (HTTP serving on precomputed -embeddings) are accepted. MovieLens 1M remains +ADR-008 (demo CLI serving), ADR-009 (HTTP serving on precomputed +embeddings), and ADR-010 (two-stage retrieval and re-ranking) are +accepted. MovieLens 1M remains deferred. Phase 2 is delivered/complete for MovieLens 100K: download and on-disk `processed/` prep, and timing charts (native vs Python reference sampler, stored JSON/SVG). Phase 4 GraphSAGE training on the @@ -20,6 +21,10 @@ scoring, top-K print) with synthetic harness tests. Phase 9 is delivered for the ADR-009 HTTP API: offline embedding export, FastAPI `GET /recommend` and `/healthz`, NumPy top-K, pytest `TestClient` coverage, and a measured `results/serving_latency.*` load test. +Phase 10 is delivered for ADR-010: a NumPy MLP re-ranker on the stage-1 +top-N pool, MovieLens 100K retrieval-only versus retrieval-plus-re-rank +under `results/rerank_eval.json`, optional `rerank` on `GET /recommend`, +and load-test p50/p95/p99 for both modes (`serving-latency-v2`). ## Phase 1: Native foundation @@ -308,8 +313,9 @@ scope. The demo CLI stays. `python_serving_api`. Default CI does not download MovieLens, does not regenerate the latency file, and does not bind a public port. -Still not started: +Still not started in the Phase 9 slice (moved or still rejected): +- Optional re-ranking of the same embeddings (Phase 10 / ADR-010). - Batch export. - MovieLens 1M, GCN, node2vec, NeighborLoader. - FAISS as a required index. @@ -319,3 +325,49 @@ CI path, measured latency files committed from a real run, and docs match the artifact and HTTP contract. Do not invent or edit stored 100K quality result files. +## Phase 10: Two-stage retrieval and re-ranking + +Opened in the ADR-010 slice. Stage 1 stays the ADR-009 embedding +retriever. Stage 2 is optional. + +- `python/sagerec_rerank.py` builds train-only features (retrieval + score, `log1p` popularity, genre affinity from `u.item`, item + recency), trains a NumPy MLP with one ReLU hidden layer and a linear + skip by listwise softmax, and orders the stage-1 pool. It does not + import PyTorch, PyG, LightGBM, or `graph_sampler`. +- `python/sagerec_rerank_eval.py` and `scripts/run_rerank_eval.py` fit + implicit MF (`movielens_100k_config`, seed 7) as stage 1, fit the MLP + on validation items retrieved inside top-N (default 200), and compare + both systems on the test split of that same pool. Metrics are + Recall@10, NDCG@10, and MAP@10. Output is `results/rerank_eval.json` + and `.md`. This is a single-seed MF retrieval measurement. It does + not replace Phase 5 or the ADR-007 leaderboard and it is not a + GraphSAGE result. Training pools exclude the test item; test pools + exclude the validation item. The re-ranker cannot recover items + outside top-N. +- `GET /recommend` accepts `rerank` (default false) and `n` (stage-1 + width). `rerank=true` needs `SAGEREC_RERANK_ARTIFACT` whose + user/item counts and `stage1_model` match the serving artifact. + Unknown users stay 404. Missing artifact or `n < k` is 422. The + request does not train or sample. +- The load test writes protocol `serving-latency-v2` with p50/p95/p99 + and throughput for `retrieval_only` and `retrieval_plus_rerank`. + Top-level `latency_ms` duplicates retrieval-only. The committed + catalog and the timed MLP are synthetic. Do not hand-edit them. +- Tests: `python/tests/test_rerank.py` (unittest discovery) for MAP, + feature leakage, the MLP, and artifact order; API tests in + `python/tests/test_serving_api.py` for the flag, `n`, and startup + mismatch. Default CI does not download MovieLens, does not run the + 100K re-rank job, and does not regenerate `results/serving_latency.*`. + +Still rejected: + +- LightGBM (or any extra native library) on the request path. +- Batch export, MovieLens 1M, GCN, node2vec, NeighborLoader, FAISS. + +Exit condition: ADR-010 recorded, feature/MLP/API tests on the default +CI path, measured `results/rerank_eval.*` and regenerated +`results/serving_latency.*` committed from real runs, and docs match +the two-stage contract. Do not invent metrics or retcon Phase 5 or +Phase 7 files. + diff --git a/docs/project-requirements.md b/docs/project-requirements.md index a6cf4d6..0fa8f98 100644 --- a/docs/project-requirements.md +++ b/docs/project-requirements.md @@ -70,7 +70,17 @@ both recommendation quality and measured native sampling performance. train positives and return item id, title, and score. Unknown users return 404. Top-K is exact NumPy; FAISS is not required. - Record a measured load test (p50/p95/p99 and throughput) with hardware - provenance. Do not hand-edit those numbers. + provenance, for retrieval-only and for retrieval plus re-rank + (ADR-010). Do not hand-edit those numbers. +- Provide an optional second stage (ADR-010): stage 1 returns the top N + unseen items from the embedding scores (default N = 200, N >= K); + stage 2 is a small learned re-ranker using train-only features + (retrieval score, item popularity, genre affinity, recency). The + MovieLens comparison uses the ADR-003 split, trains on validation + labels inside the stage-1 pool, and reports test Recall@10, NDCG@10, + and MAP@10 for retrieval-only versus retrieval plus re-rank under + `results/rerank_eval.json`. `GET /recommend` keeps `rerank` off by + default. Do not put training or neighbor sampling on the request. - Do not add batch export unless a superseding ADR accepts it. ### Performance benchmark diff --git a/python/AGENTS.md b/python/AGENTS.md index 4135eaa..aae42e4 100644 --- a/python/AGENTS.md +++ b/python/AGENTS.md @@ -10,8 +10,10 @@ GraphSAGE training on that harness, the Phase 5 GNN-versus-MF comparison writer, the Phase 2 native-versus-reference sampler timing harness, Phase 6 PyG SAGEConv message passing on native samples, the ADR-007 multi-seed MF vs GraphSAGE aggregator, the ADR-008 demo -CLI (`sagerec_serve`), and the ADR-009 HTTP serving layer -(`sagerec_export`, `sagerec_serving`, `sagerec_api`, `sagerec_loadtest`). +CLI (`sagerec_serve`), the ADR-009 HTTP serving layer +(`sagerec_export`, `sagerec_serving`, `sagerec_api`, `sagerec_loadtest`), +and the ADR-010 two-stage re-ranker (`sagerec_rerank`, +`sagerec_rerank_eval`). ## Current slice @@ -41,7 +43,8 @@ CLI (`sagerec_serve`), and the ADR-009 HTTP serving layer (NumPy logistic SGD). It trains on `train_positive_pairs()` only. - `sagerec_metrics.py` is the shared ranking evaluator: per-user then macro-averaged Recall@10 and NDCG@10, ADR-003 eligibility, and candidate - filtering. + filtering. `average_precision_at_k` is the binary AP used by the + ADR-010 MAP@10 comparison. - `sagerec_minibatch.py` is the Phase 4 harness: train-only `BipartiteCSR` from local pairs and seeded multi-hop expansion via native `graph_sampler.sample_neighbors` (ADR-005). GraphSAGE training @@ -81,12 +84,24 @@ CLI (`sagerec_serve`), and the ADR-009 HTTP serving layer It must not import PyTorch, PyG, or `graph_sampler`. FAISS is not required. - `sagerec_api.py` is the FastAPI app: `GET /healthz` and - `GET /recommend`. Unknown users are HTTP 404. Do not sample or train + `GET /recommend`. Optional `rerank` (default false) and `n` select + the ADR-010 second stage. Unknown users are HTTP 404. `rerank=true` + without an artifact, or `n < k`, is HTTP 422. Do not sample or train on the request path. - `sagerec_loadtest.py` measures that endpoint and writes - `results/serving_latency.json` and `.md`. Do not hand-edit those - numbers. The default catalog is synthetic. Do not download MovieLens. -- Do not implement node2vec, MovieLens 1M, GCN, or batch export. + `results/serving_latency.json` and `.md` (protocol + `serving-latency-v2`, both modes). Do not hand-edit those + numbers. The default catalog and the timed MLP are synthetic. Do not + download MovieLens. +- `sagerec_rerank.py` owns train-only features, the NumPy listwise MLP, + the `sagerec_rerank_artifact` version 1 file, and in-memory two-stage + evaluation. It must not import PyTorch, PyG, LightGBM, or + `graph_sampler`. +- `sagerec_rerank_eval.py` trains implicit MF on MovieLens 100K and + writes `results/rerank_eval.json`. It requires `u.item`. It does not + start HTTP and does not overwrite Phase 5 or Phase 7 files. Default + CI must not run this job. +- Do not implement node2vec, MovieLens 1M, GCN, LightGBM, or batch export. ## Boundaries @@ -166,3 +181,12 @@ CLI (`sagerec_serve`), and the ADR-009 HTTP serving layer importable by `unittest discover` without FastAPI installed at import time. Do not download MovieLens or regenerate `results/serving_latency.*` from tests. Do not bind a public port. +- Phase 10 re-rank: `test_rerank.py` covers MAP@10, train-only + features (held-out rows do not change popularity), genre parsing, MLP + learning and seed determinism, artifact round-trip, and a tiny + two-stage eval. API tests cover `rerank=false` preserving order, + `rerank=true` reordering inside `n`, 422 when the artifact is missing + or `n < k`, 404 for an unknown user, and startup failure when + `stage1_model` does not match. Do not download MovieLens or run the + 100K re-rank job in default CI. The committed `results/rerank_eval.*` + check only validates the stored file. diff --git a/python/sagerec_api.py b/python/sagerec_api.py index ce9904f..4a81c30 100644 --- a/python/sagerec_api.py +++ b/python/sagerec_api.py @@ -1,18 +1,21 @@ -"""FastAPI serving layer for a precomputed SageRec artifact (ADR-009). +"""FastAPI serving layer for a precomputed SageRec artifact (ADR-009, ADR-010). Routes: - ``GET /healthz`` -- ``GET /recommend?user_id=&k=`` +- ``GET /recommend?user_id=&k=&rerank=&n=`` ``item_id`` in each row is the zero-based movie id. Already-seen train positives are excluded. Unknown users return HTTP 404. Invalid query parameters return HTTP 422. Top-K is exact NumPy (``sagerec_serving``); FAISS is not required. -This module does not import PyTorch, PyG, or ``graph_sampler``, and it does -not download MovieLens or train a model. Load a ``.npz`` produced by -``sagerec_export``. +``rerank`` defaults to false, which is the ADR-009 retrieval path. When +``rerank=true`` and ``SAGEREC_RERANK_ARTIFACT`` (or ``rerank_path``) is +loaded, stage 1 returns ``n`` candidates and the exported MLP re-ranks them +to ``k``. The service does not train, sample, or download on the request. + +This module does not import PyTorch, PyG, or ``graph_sampler``. """ from __future__ import annotations @@ -23,15 +26,28 @@ from fastapi import FastAPI, HTTPException, Query +import sagerec_rerank as rerank import sagerec_serving as serving +_RERANK_MISSING = ( + "rerank=true requires a loaded sagerec_rerank_artifact. Set " + "SAGEREC_RERANK_ARTIFACT or pass rerank_path to create_app. The service " + "does not train a re-ranker on the request path." +) + -def create_app(artifact_path: str | Path | None = None) -> FastAPI: - """Build an app bound to one serving artifact. +def create_app( + artifact_path: str | Path | None = None, + rerank_path: str | Path | None = None, +) -> FastAPI: + """Build an app bound to one serving artifact and an optional re-ranker. Uvicorn's factory calls this with no arguments and reads - ``SAGEREC_SERVING_ARTIFACT``. Tests pass ``artifact_path`` directly. + ``SAGEREC_SERVING_ARTIFACT`` plus optional ``SAGEREC_RERANK_ARTIFACT``. + Tests pass ``artifact_path`` directly. An explicit artifact path does + not inherit an ambient re-rank env var; pass ``rerank_path`` in tests. """ + factory_mode = artifact_path is None and rerank_path is None if artifact_path is None: artifact_path = os.environ.get("SAGEREC_SERVING_ARTIFACT") if not artifact_path: @@ -41,6 +57,18 @@ def create_app(artifact_path: str | Path | None = None) -> FastAPI: "The service does not download MovieLens or train a model." ) index = serving.load_serving_artifact(artifact_path) + if rerank_path is None and factory_mode: + raw_rerank = os.environ.get("SAGEREC_RERANK_ARTIFACT", "").strip() + rerank_path = raw_rerank or None + bundle: rerank.RerankBundle | None = None + if rerank_path: + bundle = rerank.load_rerank_artifact(rerank_path) + rerank.assert_rerank_compatible( + bundle, + num_users=index.num_users, + num_items=index.num_items, + stage1_model=str(index.metadata["model"]), + ) app = FastAPI( title="SageRec", version="1", @@ -59,6 +87,7 @@ def healthz() -> dict[str, Any]: "num_users": index.num_users, "num_items": index.num_items, "embedding_dim": index.embedding_dim, + "rerank_loaded": bundle is not None, } @app.get("/recommend") @@ -72,21 +101,81 @@ def recommend( ge=1, description="Number of unseen items to return (default 10).", ), + rerank_enabled: bool = Query( + False, + alias="rerank", + description=( + "When true, re-rank the stage-1 pool with the loaded MLP. " + "Default false keeps ADR-009 retrieval order." + ), + ), + n: int | None = Query( + None, + ge=1, + description=( + "Stage-1 candidate count used only when rerank=true. " + "Defaults to 200, or the artifact's stage1_n when that is set." + ), + ), ) -> dict[str, Any]: + if not rerank_enabled: + try: + rows = serving.recommend_topk(index, user_id, k) + except serving.UnknownUserError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + return { + "user_id": user_id, + "k": k, + "rerank": False, + "recommendations": [ + { + "item_id": row.item_id, + "title": row.title, + "score": row.score, + } + for row in rows + ], + } + if bundle is None: + raise HTTPException(status_code=422, detail=_RERANK_MISSING) + if n is None: + stage1_n = int(bundle.metadata.get("stage1_n", rerank.DEFAULT_STAGE1_N)) + else: + stage1_n = int(n) + if stage1_n < k: + raise HTTPException( + status_code=422, + detail=( + f"n must be >= k when rerank=true so stage 1 can fill the " + f"re-rank pool, got n={stage1_n} k={k}" + ), + ) try: - rows = serving.recommend_topk(index, user_id, k) + stage1 = serving.recommend_topk(index, user_id, stage1_n) + ordered = rerank.order_candidates( + bundle, + user_id, + [row.item_id for row in stage1], + [row.score for row in stage1], + ) except serving.UnknownUserError as exc: raise HTTPException(status_code=404, detail=str(exc)) from exc + except rerank.RerankError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + retrieval_by_id = {row.item_id: row.score for row in stage1} return { "user_id": user_id, "k": k, + "rerank": True, + "stage1_n": stage1_n, "recommendations": [ { - "item_id": row.item_id, - "title": row.title, - "score": row.score, + "item_id": item_id, + "title": index.item_titles[item_id], + "score": score, + "retrieval_score": retrieval_by_id[item_id], } - for row in rows + for item_id, score in ordered[:k] ], } diff --git a/python/sagerec_download.py b/python/sagerec_download.py index 64e4e19..22ad130 100644 --- a/python/sagerec_download.py +++ b/python/sagerec_download.py @@ -151,8 +151,21 @@ def verify_md5(data: bytes, expected_md5: str, *, label: str) -> str: return actual +def extract_uitem(archive_bytes: bytes) -> tuple[bytes, str]: + """Return ``(u.item bytes, zip member name)`` from a 100K zip. + + Genre flags for ADR-010 live in ``u.item``. This does not parse them and + does not download the archive. + """ + return _extract_named_member(archive_bytes, "u.item") + + def extract_udata(archive_bytes: bytes) -> tuple[bytes, str]: """Return ``(u.data bytes, zip member name)`` from a 100K zip.""" + return _extract_named_member(archive_bytes, "u.data") + + +def _extract_named_member(archive_bytes: bytes, filename: str) -> tuple[bytes, str]: try: archive = zipfile.ZipFile(io.BytesIO(archive_bytes)) except zipfile.BadZipFile as exc: @@ -165,21 +178,22 @@ def extract_udata(archive_bytes: bytes) -> tuple[bytes, str]: matches = [ name for name in names - if Path(name).name == "u.data" and not name.endswith("/") + if Path(name).name == filename and not name.endswith("/") ] if not matches: preview = ", ".join(names[:12]) or "(empty zip)" raise MovieLensDownloadError( - "zip layout is not MovieLens 100K: no member named u.data. " + f"zip layout is not MovieLens 100K: no member named {filename}. " f"Archive members include: {preview}" ) + preferred_suffix = f"ml-100k/{filename}" preferred = next( - (name for name in matches if name.replace("\\", "/").endswith("ml-100k/u.data")), + (name for name in matches if name.replace("\\", "/").endswith(preferred_suffix)), matches[0], ) - if len(matches) > 1 and not preferred.replace("\\", "/").endswith("ml-100k/u.data"): + if len(matches) > 1 and not preferred.replace("\\", "/").endswith(preferred_suffix): raise MovieLensDownloadError( - "zip contains multiple u.data members and none is ml-100k/u.data: " + f"zip contains multiple {filename} members and none is {preferred_suffix}: " + ", ".join(matches) ) try: diff --git a/python/sagerec_loadtest.py b/python/sagerec_loadtest.py index 5ac2001..93ad21e 100644 --- a/python/sagerec_loadtest.py +++ b/python/sagerec_loadtest.py @@ -1,10 +1,11 @@ -"""Local HTTP load test for the ADR-009 serving API. +"""Local HTTP load test for ADR-009 retrieval and ADR-010 re-ranking. Starts a uvicorn subprocess on a synthetic catalog (or a caller-supplied artifact), issues concurrent ``GET /recommend`` requests with httpx, and -writes p50/p95/p99 latency plus throughput. Default user and item counts -match the public MovieLens 100K cardinality so the NumPy dot product is a -similar size. Embeddings are seeded noise, not a trained model. +writes p50/p95/p99 latency plus throughput for retrieval-only and for +retrieval plus re-rank. Default user and item counts match the public +MovieLens 100K cardinality. Embeddings and the re-rank MLP are seeded +noise, not a trained model. This module does not download MovieLens and does not import ``graph_sampler``. httpx and uvicorn are imported only when a load test runs. @@ -31,10 +32,12 @@ import numpy as np +import sagerec_rerank as rerank import sagerec_serving as serving -PROTOCOL_VERSION = "serving-latency-v1" -ADR_ID = "ADR-009" +PROTOCOL_VERSION = "serving-latency-v2" +ADR_ID = "ADR-010" +RETRIEVAL_ADR = "ADR-009" DEFAULT_NUM_USERS = 943 DEFAULT_NUM_ITEMS = 1682 DEFAULT_EMBEDDING_DIM = 16 @@ -44,7 +47,9 @@ DEFAULT_WARMUP = 100 DEFAULT_CONCURRENCY = 16 DEFAULT_K = serving.DEFAULT_K +DEFAULT_STAGE1_N = rerank.DEFAULT_STAGE1_N PERCENTILE_METHOD = "linear" +LATENCY_MODES = ("retrieval_only", "retrieval_plus_rerank") REQUIRED_LATENCY_KEYS = ( "benchmark", @@ -58,6 +63,7 @@ "wall_seconds", "n_requests", "n_errors", + "modes", "cpu", "platform", "python", @@ -158,23 +164,38 @@ def build_latency_payload( wall_seconds: float, n_requests: int, n_errors: int, + modes: Mapping[str, Any], root: Path | None = None, generated_at: str | None = None, ) -> dict[str, Any]: - """Machine-readable serving-latency record. Does not invent timings.""" + """Machine-readable serving-latency record. Does not invent timings. + + Top-level latency fields duplicate ``modes['retrieval_only']``. The + re-rank mode lives under ``modes['retrieval_plus_rerank']``. + """ root = repo_root() if root is None else root + missing_modes = [name for name in LATENCY_MODES if name not in modes] + if missing_modes: + raise LoadTestError(f"latency modes missing {missing_modes}") dirty = git_dirty(root) note = ( - "One-machine ADR-009 HTTP latency for GET /recommend on a local " - "uvicorn process. Top-K is exact NumPy over precomputed embeddings. " - "Not a ranking-quality result, not sampler timing, and not a " - "production SLO." + "One-machine HTTP latency for GET /recommend on a local uvicorn " + "process, measured twice: retrieval-only (ADR-009 NumPy top-K) and " + "retrieval plus re-rank (ADR-010 MLP over the stage-1 pool). " + "Top-level latency_ms duplicates retrieval_only. Not a " + "ranking-quality result, not sampler timing, and not a production SLO." ) if catalog.get("kind") == "synthetic": note += ( " The catalog is seeded synthetic noise with MovieLens 100K " "user/item counts; no ratings were read." ) + if catalog.get("rerank_kind") == "synthetic": + note += ( + " The re-ranker is a seeded synthetic MLP over synthetic " + "popularity, genre, and recency tables, not the MovieLens-trained " + "model." + ) if dirty: note += ( " git_dirty was true at generation, so git_commit is the HEAD " @@ -184,6 +205,7 @@ def build_latency_payload( "benchmark": "sagerec_http_recommend", "protocol_version": PROTOCOL_VERSION, "adr": ADR_ID, + "retrieval_adr": RETRIEVAL_ADR, "endpoint": "GET /recommend", "catalog": dict(catalog), "workload": dict(workload), @@ -192,6 +214,7 @@ def build_latency_payload( "wall_seconds": float(wall_seconds), "n_requests": int(n_requests), "n_errors": int(n_errors), + "modes": {name: dict(modes[name]) for name in LATENCY_MODES}, "cpu": cpu_info(), "platform": { "system": platform.system(), @@ -221,12 +244,23 @@ def render_serving_latency_markdown(payload: Mapping[str, Any]) -> str: def fmt(value: float) -> str: return f"{float(value):.6f}" + modes = payload["modes"] + for mode_name in LATENCY_MODES: + if mode_name not in modes: + raise LoadTestError(f"latency payload is missing modes.{mode_name}") + mode_latency = modes[mode_name]["latency_ms"] + for key in ("p50", "p95", "p99", "mean"): + if key not in mode_latency: + raise LoadTestError(f"modes.{mode_name}.latency_ms is missing {key}") + lines = [ "# SageRec HTTP recommend latency", "", - "One-machine ADR-009 measurement of `GET /recommend` against a local " - "uvicorn process. Top-K is exact NumPy. Not a ranking-quality result " - "and not a production SLO.", + "One-machine measurement of `GET /recommend` against a local uvicorn " + "process. Retrieval-only is ADR-009 NumPy top-K. Retrieval plus " + "re-rank is the ADR-010 MLP over the stage-1 pool. Not a " + "ranking-quality result and not a production SLO. Top-level numbers " + "in the JSON duplicate retrieval-only.", "", f"- Protocol: `{payload['protocol_version']}` ({payload['adr']})", ( @@ -235,9 +269,10 @@ def fmt(value: float) -> str: f"seed {catalog.get('seed')}" ), ( - f"- Workload: {workload['n_requests']} measured requests, " + f"- Workload: {workload['n_requests']} measured requests per mode, " f"{workload['warmup_requests']} warm-up, concurrency " - f"{workload['concurrency']}, k={workload['k']}" + f"{workload['concurrency']}, k={workload['k']}, " + f"stage1_n={workload.get('stage1_n')}" ), f"- Percentile method: `{latency['percentile_method']}`", ( @@ -247,19 +282,24 @@ def fmt(value: float) -> str: ), f"- Commit: `{payload['git_commit']}` (dirty={payload['git_dirty']})", "", - "| Statistic | Value |", - "| --- | ---: |", - f"| p50 ms | {fmt(latency['p50'])} |", - f"| p95 ms | {fmt(latency['p95'])} |", - f"| p99 ms | {fmt(latency['p99'])} |", - f"| mean ms | {fmt(latency['mean'])} |", - f"| throughput requests/s | {fmt(payload['throughput_requests_per_s'])} |", - f"| wall seconds | {fmt(payload['wall_seconds'])} |", - f"| errors | {payload['n_errors']} |", - "", - payload["note"], - "", + "| Mode | p50 ms | p95 ms | p99 ms | requests/s |", + "| --- | ---: | ---: | ---: | ---: |", ] + for mode_name in LATENCY_MODES: + mode = modes[mode_name] + mode_latency = mode["latency_ms"] + lines.append( + f"| {mode_name} | {fmt(mode_latency['p50'])} | " + f"{fmt(mode_latency['p95'])} | {fmt(mode_latency['p99'])} | " + f"{fmt(mode['throughput_requests_per_s'])} |" + ) + lines.extend( + [ + "", + payload["note"], + "", + ] + ) return "\n".join(lines) @@ -318,6 +358,7 @@ async def _collect_latencies( k: int, concurrency: int, warmup: int, + extra_params: Mapping[str, Any] | None = None, ) -> tuple[list[float], int, float]: import httpx @@ -334,10 +375,10 @@ async def _collect_latencies( async def one(user_id: int) -> tuple[int, float]: async with semaphore: started = time.perf_counter() - response = await client.get( - "/recommend", - params={"user_id": int(user_id), "k": int(k)}, - ) + params: dict[str, Any] = {"user_id": int(user_id), "k": int(k)} + if extra_params: + params.update(dict(extra_params)) + response = await client.get("/recommend", params=params) elapsed = time.perf_counter() - started return response.status_code, elapsed @@ -364,9 +405,41 @@ async def one(user_id: int) -> tuple[int, float]: return latencies, errors, wall_seconds +def _mode_record( + *, + rerank_enabled: bool, + stage1_n: int | None, + latencies: Sequence[float], + n_errors: int, + wall_seconds: float, + n_requests: int, +) -> dict[str, Any]: + if n_errors: + raise LoadTestError( + f"{n_errors} recommend requests failed; refusing to write latency results" + ) + if len(latencies) != n_requests: + raise LoadTestError( + f"expected {n_requests} latency samples, got {len(latencies)}" + ) + throughput = float(n_requests) / float(wall_seconds) if wall_seconds > 0 else 0.0 + record: dict[str, Any] = { + "rerank": bool(rerank_enabled), + "latency_ms": latency_percentiles_ms(latencies), + "throughput_requests_per_s": throughput, + "wall_seconds": float(wall_seconds), + "n_requests": int(n_requests), + "n_errors": int(n_errors), + } + if rerank_enabled: + record["stage1_n"] = int(stage1_n if stage1_n is not None else DEFAULT_STAGE1_N) + return record + + def run_http_load_test( *, artifact_path: str | Path | None = None, + rerank_path: str | Path | None = None, num_users: int = DEFAULT_NUM_USERS, num_items: int = DEFAULT_NUM_ITEMS, embedding_dim: int = DEFAULT_EMBEDDING_DIM, @@ -376,22 +449,31 @@ def run_http_load_test( warmup_requests: int = DEFAULT_WARMUP, concurrency: int = DEFAULT_CONCURRENCY, k: int = DEFAULT_K, + stage1_n: int = DEFAULT_STAGE1_N, json_out: str | Path | None = None, md_out: str | Path | None = None, root: Path | None = None, ) -> dict[str, Any]: - """Measure recommend latency and write JSON plus markdown. + """Measure recommend latency with and without re-ranking. When ``artifact_path`` is omitted, a synthetic catalog is written to a temporary file and deleted afterward. The committed default uses that - synthetic catalog. + synthetic catalog plus a synthetic re-rank MLP. ``stage1_n`` is the + pool width for the re-rank requests. """ root = repo_root() if root is None else Path(root) - if n_requests < 1 or warmup_requests < 0 or concurrency < 1 or k < 1: + if ( + n_requests < 1 + or warmup_requests < 0 + or concurrency < 1 + or k < 1 + or stage1_n < k + ): raise LoadTestError( - "n_requests and concurrency and k must be >= 1, and warmup must be >= 0, " + "n_requests, concurrency, and k must be >= 1, warmup must be >= 0, " + "and stage1_n must be >= k, " f"got n_requests={n_requests}, warmup={warmup_requests}, " - f"concurrency={concurrency}, k={k}" + f"concurrency={concurrency}, k={k}, stage1_n={stage1_n}" ) python_dir = root / "python" if json_out is None: @@ -419,6 +501,7 @@ def run_http_load_test( "seen_per_user": seen_per_user, "seed": seed, "model": index.metadata["model"], + "rerank_kind": "synthetic" if rerank_path is None else "provided", } else: artifact = Path(artifact_path) @@ -432,8 +515,31 @@ def run_http_load_test( "seed": index.metadata.get("seed"), "model": index.metadata["model"], "artifact_file": artifact.name, + "rerank_kind": "synthetic" if rerank_path is None else "provided", } + rerank_temp: tempfile.TemporaryDirectory[str] | None = None + if rerank_path is None: + rerank_temp = tempfile.TemporaryDirectory(prefix="sagerec-rerank-") + rerank_file = Path(rerank_temp.name) / "rerank.npz" + bundle = rerank.build_synthetic_rerank_bundle( + num_users=index.num_users, + num_items=index.num_items, + seed=seed, + stage1_model=str(index.metadata["model"]), + stage1_n=stage1_n, + ) + rerank.save_rerank_artifact(rerank_file, bundle) + else: + rerank_file = Path(rerank_path) + bundle = rerank.load_rerank_artifact(rerank_file) + rerank.assert_rerank_compatible( + bundle, + num_users=index.num_users, + num_items=index.num_items, + stage1_model=str(index.metadata["model"]), + ) + port = _free_port() base_url = f"http://127.0.0.1:{port}" stderr_fd, stderr_name = tempfile.mkstemp(prefix="sagerec-uvicorn-", suffix=".log") @@ -442,6 +548,7 @@ def run_http_load_test( env = os.environ.copy() env["PYTHONPATH"] = str(python_dir) + os.pathsep + env.get("PYTHONPATH", "") env["SAGEREC_SERVING_ARTIFACT"] = str(artifact.resolve()) + env["SAGEREC_RERANK_ARTIFACT"] = str(rerank_file.resolve()) command = [ sys.executable, "-m", @@ -471,29 +578,43 @@ def run_http_load_test( f"/healthz num_users {health['num_users']} does not match " f"the artifact ({index.num_users})" ) - rng = np.random.default_rng(seed) - user_ids = rng.integers(0, index.num_users, size=warmup_requests + n_requests) - latencies, n_errors, wall_seconds = asyncio.run( - _collect_latencies( - base_url, - [int(user_id) for user_id in user_ids], - k=k, - concurrency=concurrency, - warmup=warmup_requests, - ) - ) - if n_errors: + if health.get("rerank_loaded") is not True: raise LoadTestError( - f"{n_errors} recommend requests failed; refusing to write latency results" + "/healthz did not report rerank_loaded=true. The load test " + "measures both modes and needs SAGEREC_RERANK_ARTIFACT." ) - if len(latencies) != n_requests: - raise LoadTestError( - f"expected {n_requests} latency samples, got {len(latencies)}" + rng = np.random.default_rng(seed) + user_ids = [int(user_id) for user_id in rng.integers( + 0, index.num_users, size=warmup_requests + n_requests + )] + mode_params = { + "retrieval_only": {"rerank": "false"}, + "retrieval_plus_rerank": {"rerank": "true", "n": str(stage1_n)}, + } + modes: dict[str, Any] = {} + for mode_name in LATENCY_MODES: + latencies, n_errors, wall_seconds = asyncio.run( + _collect_latencies( + base_url, + user_ids, + k=k, + concurrency=concurrency, + warmup=warmup_requests, + extra_params=mode_params[mode_name], + ) + ) + modes[mode_name] = _mode_record( + rerank_enabled=mode_name == "retrieval_plus_rerank", + stage1_n=stage1_n, + latencies=latencies, + n_errors=n_errors, + wall_seconds=wall_seconds, + n_requests=n_requests, ) - latency_ms = latency_percentiles_ms(latencies) - throughput = float(n_requests) / float(wall_seconds) if wall_seconds > 0 else 0.0 + retrieval = modes["retrieval_only"] workload = { "k": int(k), + "stage1_n": int(stage1_n), "n_requests": int(n_requests), "warmup_requests": int(warmup_requests), "concurrency": int(concurrency), @@ -501,15 +622,17 @@ def run_http_load_test( "percentile_method": PERCENTILE_METHOD, "server": "uvicorn", "client": "httpx.AsyncClient", + "modes": list(LATENCY_MODES), } payload = build_latency_payload( catalog=catalog, workload=workload, - latency_ms=latency_ms, - throughput_requests_per_s=throughput, - wall_seconds=wall_seconds, + latency_ms=retrieval["latency_ms"], + throughput_requests_per_s=retrieval["throughput_requests_per_s"], + wall_seconds=retrieval["wall_seconds"], n_requests=n_requests, - n_errors=n_errors, + n_errors=retrieval["n_errors"], + modes=modes, root=root, ) write_latency_results(payload, json_out, md_out) @@ -525,18 +648,26 @@ def run_http_load_test( proc.wait(timeout=5) if temp_dir is not None: temp_dir.cleanup() + if rerank_temp is not None: + rerank_temp.cleanup() stderr_path.unlink(missing_ok=True) def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description=( - "Load-test GET /recommend and write results/serving_latency.json " - "and .md. Default catalog is synthetic (MovieLens 100K cardinality, " - "seeded noise). Does not download MovieLens." + "Load-test GET /recommend with and without re-ranking and write " + "results/serving_latency.json and .md. Default catalog and " + "re-ranker are synthetic. Does not download MovieLens." ) ) parser.add_argument("--artifact", type=Path, default=None) + parser.add_argument( + "--rerank", + type=Path, + default=None, + help="Optional re-rank artifact. Default is a synthetic MLP matched to the catalog.", + ) parser.add_argument("--num-users", type=int, default=DEFAULT_NUM_USERS) parser.add_argument("--num-items", type=int, default=DEFAULT_NUM_ITEMS) parser.add_argument("--embedding-dim", type=int, default=DEFAULT_EMBEDDING_DIM) @@ -546,6 +677,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--warmup", type=int, default=DEFAULT_WARMUP) parser.add_argument("--concurrency", type=int, default=DEFAULT_CONCURRENCY) parser.add_argument("--k", type=int, default=DEFAULT_K) + parser.add_argument("--stage1-n", type=int, default=DEFAULT_STAGE1_N) parser.add_argument( "--json-out", type=Path, @@ -571,6 +703,7 @@ def main(argv: Sequence[str] | None = None) -> int: try: payload = run_http_load_test( artifact_path=args.artifact, + rerank_path=args.rerank, num_users=args.num_users, num_items=args.num_items, embedding_dim=args.embedding_dim, @@ -580,20 +713,24 @@ def main(argv: Sequence[str] | None = None) -> int: warmup_requests=args.warmup, concurrency=args.concurrency, k=args.k, + stage1_n=args.stage1_n, json_out=args.json_out, md_out=args.md_out, ) - except (LoadTestError, serving.ServingError, OSError) as exc: + except (LoadTestError, serving.ServingError, rerank.RerankError, OSError) as exc: print(f"error: {exc}", file=sys.stderr) return 1 - latency = payload["latency_ms"] + retrieval = payload["modes"]["retrieval_only"]["latency_ms"] + reranked = payload["modes"]["retrieval_plus_rerank"]["latency_ms"] print( - "p50_ms={p50:.6f} p95_ms={p95:.6f} p99_ms={p99:.6f} " - "throughput_rps={rps:.6f} cpu={cpu}".format( - p50=latency["p50"], - p95=latency["p95"], - p99=latency["p99"], - rps=payload["throughput_requests_per_s"], + "retrieval p50_ms={p50:.6f} p95_ms={p95:.6f} p99_ms={p99:.6f} " + "rerank p50_ms={r50:.6f} p95_ms={r95:.6f} p99_ms={r99:.6f} cpu={cpu}".format( + p50=retrieval["p50"], + p95=retrieval["p95"], + p99=retrieval["p99"], + r50=reranked["p50"], + r95=reranked["p95"], + r99=reranked["p99"], cpu=payload["cpu"]["model"], ) ) diff --git a/python/sagerec_metrics.py b/python/sagerec_metrics.py index f8e4129..a92412a 100644 --- a/python/sagerec_metrics.py +++ b/python/sagerec_metrics.py @@ -46,6 +46,31 @@ def recall_at_k( return hits / float(len(relevant)) +def average_precision_at_k( + ranked_items: Sequence[int], + relevant: Set[int], + k: int = DEFAULT_K, +) -> float: + """Binary AP@k. One relevant item yields ``1/rank`` when ``rank <= k``, else 0. + + The denominator is ``min(k, |relevant|)``. Macro-averaging this value is + MAP@k. Rank is 1-based within ``ranked_items``. + """ + k = _require_k(k) + if not relevant: + raise ValueError("relevant is empty; average precision is undefined") + hits = 0 + precision_sum = 0.0 + for rank, item in enumerate(ranked_items[:k], start=1): + if item in relevant: + hits += 1 + precision_sum += hits / float(rank) + denom = float(min(k, len(relevant))) + if denom == 0.0: + return 0.0 + return precision_sum / denom + + def ndcg_at_k( ranked_items: Sequence[int], relevant: Set[int], diff --git a/python/sagerec_rerank.py b/python/sagerec_rerank.py new file mode 100644 index 0000000..bb6e751 --- /dev/null +++ b/python/sagerec_rerank.py @@ -0,0 +1,1215 @@ +"""Two-stage re-ranking on top of embedding retrieval (ADR-010). + +Stage 1 is the existing dot-product retrieval (ADR-009). Stage 2 is a small +NumPy MLP with a linear skip, trained listwise on the retrieved pool. Features +are the retrieval score, train-only item popularity, train-only genre affinity, +and train-only item recency. Held-out interactions never enter those tables. + +This module does not import PyTorch, PyG, or ``graph_sampler``, and it does +not download MovieLens or train on an HTTP request. MovieLens orchestration +lives in ``sagerec_rerank_eval``. +""" + +from __future__ import annotations + +import json +from collections import defaultdict +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + +import sagerec_metrics as metrics +from sagerec_prep import SplitName, SplitResult + +ADR_ID = "ADR-010" +RERANK_SCHEMA = "sagerec_rerank_artifact" +RERANK_SCHEMA_VERSION = 1 +DEFAULT_STAGE1_N = 200 +DEFAULT_K = 10 +DEFAULT_HIDDEN = 8 +DEFAULT_EPOCHS = 30 +DEFAULT_LEARNING_RATE = 0.05 +DEFAULT_L2 = 1e-4 +DEFAULT_SEED = 7 +DEFAULT_INIT_STD = 0.1 +FEATURE_NAMES = ( + "retrieval_score", + "item_popularity", + "genre_affinity", + "item_recency", +) +N_FEATURES = len(FEATURE_NAMES) +# MovieLens 100K u.item genre columns, in file order, including "unknown". +GENRE_NAMES = ( + "unknown", + "Action", + "Adventure", + "Animation", + "Children's", + "Comedy", + "Crime", + "Documentary", + "Drama", + "Fantasy", + "Film-Noir", + "Horror", + "Musical", + "Mystery", + "Romance", + "Sci-Fi", + "Thriller", + "War", + "Western", +) +TIE_BREAK_POLICY = "score descending, then movie_id ascending" +SCORE_FORMULA = ( + "relu(features @ w1 + b1) @ w2 + b2 + features @ w_linear; " + "features are standardized with the training mean and std" +) +MAP_DEFINITION = ( + "MAP@10. One relevant item per eligible user. AP@10 is 1/rank when the " + "held-out item's 1-based rank is <= 10, else 0. MAP is the unweighted " + "mean over eligible users. Both systems rank inside the same stage-1 " + "top-N pool, so retrieval-only top-10 matches full-catalog top-10 when " + "N >= 10 and the tie-break matches." +) +_STD_FLOOR = 1e-12 +_FLOAT_DTYPES = (np.float32, np.float64) + + +class RerankError(ValueError): + """Actionable failure while building features, training, or re-ranking.""" + + +@dataclass(frozen=True) +class RerankConfig: + """Explicit stage-2 hyperparameters. Stage 1 width is ``stage1_n``.""" + + stage1_n: int = DEFAULT_STAGE1_N + hidden_dim: int = DEFAULT_HIDDEN + n_epochs: int = DEFAULT_EPOCHS + learning_rate: float = DEFAULT_LEARNING_RATE + l2: float = DEFAULT_L2 + seed: int = DEFAULT_SEED + k: int = DEFAULT_K + init_std: float = DEFAULT_INIT_STD + + def __post_init__(self) -> None: + stage1_n = _require_int(self.stage1_n, "stage1_n") + hidden_dim = _require_int(self.hidden_dim, "hidden_dim") + n_epochs = _require_int(self.n_epochs, "n_epochs") + seed = _require_int(self.seed, "seed") + k = _require_int(self.k, "k") + learning_rate = _require_float(self.learning_rate, "learning_rate") + l2 = _require_float(self.l2, "l2") + init_std = _require_float(self.init_std, "init_std") + if stage1_n < 1: + raise RerankError(f"stage1_n must be >= 1, got {stage1_n}") + if k < 1: + raise RerankError(f"k must be >= 1, got {k}") + if stage1_n < k: + raise RerankError( + f"stage1_n must be >= k so stage 2 can fill top-k, " + f"got stage1_n={stage1_n} k={k}" + ) + if hidden_dim < 1: + raise RerankError(f"hidden_dim must be >= 1, got {hidden_dim}") + if n_epochs < 1: + raise RerankError(f"n_epochs must be >= 1, got {n_epochs}") + if seed < 0: + raise RerankError(f"seed must be >= 0, got {seed}") + if learning_rate <= 0.0: + raise RerankError(f"learning_rate must be > 0, got {learning_rate}") + if l2 < 0.0: + raise RerankError(f"l2 must be >= 0, got {l2}") + if init_std <= 0.0: + raise RerankError(f"init_std must be > 0, got {init_std}") + + def as_dict(self) -> dict[str, Any]: + return { + "stage1_n": self.stage1_n, + "hidden_dim": self.hidden_dim, + "n_epochs": self.n_epochs, + "learning_rate": self.learning_rate, + "l2": self.l2, + "seed": self.seed, + "k": self.k, + "init_std": self.init_std, + } + + +@dataclass(frozen=True) +class FeatureTables: + """Train-only side information used to build stage-2 features.""" + + item_popularity: np.ndarray + item_recency: np.ndarray + item_genres: np.ndarray + user_genre_profile: np.ndarray + genres_available: bool + + @property + def num_users(self) -> int: + return int(self.user_genre_profile.shape[0]) + + @property + def num_items(self) -> int: + return int(self.item_popularity.shape[0]) + + @property + def n_genres(self) -> int: + return int(self.item_genres.shape[1]) + + +@dataclass(frozen=True) +class CandidatePool: + """One user's stage-1 pool. ``positive_index`` is -1 when the label missed.""" + + user_id: int + item_ids: np.ndarray + retrieval_scores: np.ndarray + relevant_id: int + positive_index: int + + +@dataclass +class RerankBundle: + """Exported stage-2 model plus the feature tables it was trained against.""" + + item_popularity: np.ndarray + item_recency: np.ndarray + item_genres: np.ndarray + user_genre_profile: np.ndarray + w1: np.ndarray + b1: np.ndarray + w2: np.ndarray + b2: np.ndarray + w_linear: np.ndarray + feature_mean: np.ndarray + feature_std: np.ndarray + metadata: dict[str, Any] + + @property + def num_users(self) -> int: + return int(self.user_genre_profile.shape[0]) + + @property + def num_items(self) -> int: + return int(self.item_popularity.shape[0]) + + @property + def hidden_dim(self) -> int: + return int(self.w1.shape[1]) + + +def _require_int(value: Any, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, (int, np.integer)): + raise RerankError(f"{field} must be an int, got {type(value).__name__}") + return int(value) + + +def _require_float(value: Any, field: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float, np.floating)): + raise RerankError(f"{field} must be a real number, got {type(value).__name__}") + return float(value) + + +def load_item_genres( + path: str | Path, + num_items: int, + movie_source_ids: Sequence[int], +) -> np.ndarray: + """Align MovieLens ``u.item`` genre flags to local movie ids. + + Returns a float64 array of shape ``(num_items, 19)``. Source ids come from + ``mappings.json`` ``movie_source_ids`` (local rank to source id). + """ + file_path = Path(path) + num_items = _require_int(num_items, "num_items") + if num_items < 1: + raise RerankError(f"num_items must be >= 1, got {num_items}") + if len(movie_source_ids) != num_items: + raise RerankError( + f"movie_source_ids length {len(movie_source_ids)} does not match " + f"num_items={num_items}" + ) + if not file_path.is_file(): + raise RerankError( + f"u.item path {file_path} does not exist. Genre affinity needs the " + "MovieLens 100K genre flags. This loader does not download them." + ) + try: + text = file_path.read_text(encoding="latin-1") + except OSError as exc: + raise RerankError(f"could not read u.item at {file_path}: {exc}") from exc + return parse_uitem_genres(text, num_items, movie_source_ids) + + +def parse_uitem_genres( + text: str, + num_items: int, + movie_source_ids: Sequence[int], +) -> np.ndarray: + """Parse pipe-separated ``u.item`` text into local-id genre rows.""" + num_items = _require_int(num_items, "num_items") + if len(movie_source_ids) != num_items: + raise RerankError( + f"movie_source_ids length {len(movie_source_ids)} does not match " + f"num_items={num_items}" + ) + n_genres = len(GENRE_NAMES) + by_source: dict[int, np.ndarray] = {} + for line_number, line in enumerate(text.splitlines(), start=1): + if not line.strip(): + continue + parts = line.split("|") + if len(parts) < 5 + n_genres: + raise RerankError( + f"u.item line {line_number} has {len(parts)} fields; expected at " + f"least {5 + n_genres} (id, title, dates, url, {n_genres} genres)" + ) + try: + source_id = int(parts[0]) + except ValueError as exc: + raise RerankError( + f"u.item line {line_number} has a non-integer movie id" + ) from exc + if source_id in by_source: + raise RerankError( + f"u.item line {line_number} repeats source movie id {source_id}" + ) + flags = np.empty(n_genres, dtype=np.float64) + for offset, raw in enumerate(parts[5 : 5 + n_genres]): + if raw not in ("0", "1"): + raise RerankError( + f"u.item line {line_number} genre {GENRE_NAMES[offset]!r} " + f"must be 0 or 1, got {raw!r}" + ) + flags[offset] = float(raw) + by_source[source_id] = flags + rows = np.zeros((num_items, n_genres), dtype=np.float64) + missing: list[int] = [] + for local_id, source_id in enumerate(movie_source_ids): + source_int = _require_int(source_id, f"movie_source_ids[{local_id}]") + flags = by_source.get(source_int) + if flags is None: + missing.append(source_int) + continue + rows[local_id] = flags + if missing: + preview = ", ".join(str(source_id) for source_id in missing[:8]) + raise RerankError( + f"u.item is missing genre flags for source movie ids {preview}" + ) + return rows + + +def build_feature_tables( + split: SplitResult, + genres: np.ndarray | None = None, +) -> FeatureTables: + """Popularity, recency, and genre profiles from **training** interactions. + + Validation and test rows are ignored. Popularity is ``log1p`` of the train + interaction count. Recency min-max normalizes each item's latest train + timestamp; items with no train interaction stay 0. When every train item + shares one timestamp, those items score 1. Genre affinity uses the user's + mean train genre vector. ``genres is None`` stores a zero genre axis and + sets ``genres_available`` false. + """ + num_users = _require_int(split.num_users, "split.num_users") + num_items = _require_int(split.num_movies, "split.num_movies") + if genres is None: + item_genres = np.zeros((num_items, 1), dtype=np.float64) + genres_available = False + else: + item_genres = np.asarray(genres, dtype=np.float64) + if item_genres.ndim != 2 or item_genres.shape[0] != num_items: + raise RerankError( + "genres must have shape (num_movies, n_genres), " + f"got {item_genres.shape} for num_movies={num_items}" + ) + if item_genres.shape[1] < 1 or not np.isfinite(item_genres).all(): + raise RerankError("genres must be finite and have at least one column") + genres_available = True + n_genres = int(item_genres.shape[1]) + counts = np.zeros(num_items, dtype=np.float64) + max_ts = np.full(num_items, -1, dtype=np.int64) + user_sum = np.zeros((num_users, n_genres), dtype=np.float64) + user_count = np.zeros(num_users, dtype=np.float64) + for row in split.interactions_for("train"): + movie_id = int(row.movie_id) + user_id = int(row.user_id) + if movie_id < 0 or movie_id >= num_items or user_id < 0 or user_id >= num_users: + raise RerankError( + f"train interaction ({user_id}, {movie_id}) is outside " + f"users [0, {num_users}) movies [0, {num_items})" + ) + counts[movie_id] += 1.0 + timestamp = int(row.timestamp) + if timestamp > int(max_ts[movie_id]): + max_ts[movie_id] = timestamp + user_sum[user_id] += item_genres[movie_id] + user_count[user_id] += 1.0 + popularity = np.log1p(counts) + recency = np.zeros(num_items, dtype=np.float64) + observed = max_ts >= 0 + if bool(observed.any()): + observed_ts = max_ts[observed].astype(np.float64) + lo = float(observed_ts.min()) + hi = float(observed_ts.max()) + if hi > lo: + recency[observed] = (observed_ts - lo) / (hi - lo) + else: + recency[observed] = 1.0 + profile = np.zeros((num_users, n_genres), dtype=np.float64) + nonzero = user_count > 0.0 + profile[nonzero] = user_sum[nonzero] / user_count[nonzero, None] + return FeatureTables( + item_popularity=popularity, + item_recency=recency, + item_genres=np.asarray(item_genres, dtype=np.float64), + user_genre_profile=profile, + genres_available=genres_available, + ) + + +def candidate_features( + tables: FeatureTables, + user_id: int, + item_ids: Sequence[int] | np.ndarray, + retrieval_scores: Sequence[float] | np.ndarray, +) -> np.ndarray: + """Build the fixed 4-column feature matrix for one retrieved pool. + + Column order is ``FEATURE_NAMES``: retrieval score, item popularity, + genre affinity (user profile dot item genres), item recency. + """ + user_id = _require_int(user_id, "user_id") + if user_id < 0 or user_id >= tables.num_users: + raise RerankError( + f"user_id {user_id} is outside [0, {tables.num_users})" + ) + items = np.asarray(list(item_ids), dtype=np.int64) + scores = np.asarray(list(retrieval_scores), dtype=np.float64) + if items.ndim != 1 or scores.ndim != 1 or items.shape != scores.shape: + raise RerankError( + "item_ids and retrieval_scores must be 1-d and the same length, " + f"got {items.shape} and {scores.shape}" + ) + if items.size and (int(items.min()) < 0 or int(items.max()) >= tables.num_items): + raise RerankError( + f"item_ids must lie in [0, {tables.num_items})" + ) + if scores.size and not np.isfinite(scores).all(): + raise RerankError("retrieval_scores must be finite") + if items.size == 0: + return np.zeros((0, N_FEATURES), dtype=np.float64) + affinity = tables.item_genres[items] @ tables.user_genre_profile[user_id] + return np.column_stack( + ( + scores, + tables.item_popularity[items], + np.asarray(affinity, dtype=np.float64), + tables.item_recency[items], + ) + ) + + +def _items_by_user(split: SplitResult, name: SplitName) -> dict[int, set[int]]: + by_user: dict[int, set[int]] = defaultdict(set) + for row in split.interactions_for(name): + by_user[int(row.user_id)].add(int(row.movie_id)) + return dict(by_user) + + +def top_n_item_ids( + scores: np.ndarray, + exclude: set[int], + n: int, +) -> tuple[np.ndarray, np.ndarray]: + """Highest-score unseen items. Ties break on the smaller item id.""" + n = _require_int(n, "n") + if n < 1: + raise RerankError(f"n must be >= 1, got {n}") + values = np.asarray(scores, dtype=np.float64) + if values.ndim != 1: + raise RerankError(f"scores must be 1-d, got shape {values.shape}") + if not np.isfinite(values).all(): + raise RerankError("retrieval scores must be finite before masking") + masked = values.copy() + if exclude: + drop = np.fromiter(exclude, dtype=np.int64, count=len(exclude)) + if drop.size and (int(drop.min()) < 0 or int(drop.max()) >= masked.shape[0]): + raise RerankError( + f"exclude ids must lie in [0, {masked.shape[0]})" + ) + masked[drop] = -np.inf + n_unseen = int(np.isfinite(masked).sum()) + take = min(n, n_unseen) + if take <= 0: + empty = np.zeros(0, dtype=np.int64) + return empty, np.zeros(0, dtype=np.float64) + order = np.lexsort((np.arange(masked.shape[0], dtype=np.int64), -masked)) + chosen = order[:take] + return chosen.astype(np.int64, copy=False), values[chosen] + + +def build_candidate_pools( + split: SplitResult, + score_user: Callable[[int], np.ndarray], + *, + label_split: SplitName, + extra_exclude_splits: Sequence[SplitName] = (), + stage1_n: int, +) -> list[CandidatePool]: + """Stage-1 pools for eligible users. + + ``label_split`` is the single relevant item. ``extra_exclude_splits`` are + removed from the pool (training drops the test item; test eval drops the + validation item). Train positives are always removed. The relevant item is + kept even if a caller also listed its split in ``extra_exclude_splits``. + """ + if label_split not in ("validation", "test"): + raise RerankError( + f"label_split must be 'validation' or 'test', got {label_split!r}" + ) + stage1_n = _require_int(stage1_n, "stage1_n") + if stage1_n < 1: + raise RerankError(f"stage1_n must be >= 1, got {stage1_n}") + for name in extra_exclude_splits: + if name not in ("train", "validation", "test"): + raise RerankError(f"unknown extra exclude split {name!r}") + if not split.eligible_user_ids: + raise RerankError("no eligible users; ADR-003 ranking needs at least 3 interactions") + train_by_user = _items_by_user(split, "train") + extra_by_split = [_items_by_user(split, name) for name in extra_exclude_splits] + label_by_user = _items_by_user(split, label_split) + pools: list[CandidatePool] = [] + for user_id in split.eligible_user_ids: + relevant_ids = label_by_user.get(int(user_id), set()) + if len(relevant_ids) != 1: + raise RerankError( + f"eligible user {user_id} must have exactly one {label_split} " + f"item, got {sorted(relevant_ids)}" + ) + relevant_id = next(iter(relevant_ids)) + exclude = set(train_by_user.get(int(user_id), set())) + for extra in extra_by_split: + exclude |= extra.get(int(user_id), set()) + exclude.discard(relevant_id) + scores = np.asarray(score_user(int(user_id)), dtype=np.float64) + if scores.shape != (split.num_movies,): + raise RerankError( + f"score_user({user_id}) must have shape ({split.num_movies},), " + f"got {scores.shape}" + ) + item_ids, retrieval_scores = top_n_item_ids(scores, exclude, stage1_n) + matches = np.flatnonzero(item_ids == relevant_id) + positive_index = int(matches[0]) if matches.size else -1 + pools.append( + CandidatePool( + user_id=int(user_id), + item_ids=item_ids, + retrieval_scores=retrieval_scores, + relevant_id=int(relevant_id), + positive_index=positive_index, + ) + ) + return pools + + +def _softmax_nll(logits: np.ndarray, positive_index: int) -> tuple[float, np.ndarray]: + shifted = logits - float(np.max(logits)) + exp = np.exp(shifted) + total = float(exp.sum()) + if not np.isfinite(total) or total <= 0.0: + raise RerankError("listwise softmax received non-finite logits") + loss = float(-(shifted[positive_index] - np.log(total))) + grad = exp / total + grad[positive_index] -= 1.0 + return loss, grad + + +def _init_mlp(config: RerankConfig, rng: np.random.Generator) -> dict[str, Any]: + hidden = config.hidden_dim + return { + "w1": rng.normal(0.0, config.init_std, size=(N_FEATURES, hidden)), + "b1": np.full(hidden, 0.1, dtype=np.float64), + "w2": rng.normal(0.0, config.init_std, size=(hidden,)), + "b2": 0.0, + "w_linear": rng.normal(0.0, config.init_std, size=(N_FEATURES,)), + } + + +def _forward( + features: np.ndarray, + weights: Mapping[str, Any], +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + pre = features @ weights["w1"] + weights["b1"] + hidden = np.maximum(pre, 0.0) + logits = hidden @ weights["w2"] + float(weights["b2"]) + features @ weights["w_linear"] + return pre, hidden, np.asarray(logits, dtype=np.float64) + + +def train_listwise_mlp( + feature_matrices: Sequence[np.ndarray], + positive_indices: Sequence[int], + config: RerankConfig, +) -> tuple[dict[str, Any], list[float]]: + """Listwise softmax SGD. Each matrix is one user's standardized pool. + + Softmax gradients already sum to zero, so they are not divided by the + pool length. L2 is spread across the epoch (one pass applies ``l2`` + once in total). Returns weights and the mean loss per epoch. + """ + if len(feature_matrices) != len(positive_indices) or not feature_matrices: + raise RerankError( + "train_listwise_mlp needs one non-empty feature matrix and one " + f"positive index per user, got {len(feature_matrices)} matrices " + f"and {len(positive_indices)} labels" + ) + prepared: list[tuple[np.ndarray, int]] = [] + for index, (matrix, positive_index) in enumerate(zip(feature_matrices, positive_indices)): + features = np.asarray(matrix, dtype=np.float64) + if features.ndim != 2 or features.shape[1] != N_FEATURES: + raise RerankError( + f"feature matrix {index} must have shape (pool, {N_FEATURES}), " + f"got {features.shape}" + ) + if not np.isfinite(features).all(): + raise RerankError(f"feature matrix {index} must be finite") + label = _require_int(positive_index, f"positive_indices[{index}]") + if label < 0 or label >= features.shape[0]: + raise RerankError( + f"positive_indices[{index}]={label} is outside the pool of " + f"length {features.shape[0]}" + ) + prepared.append((features, label)) + rng = np.random.default_rng(config.seed) + weights = _init_mlp(config, rng) + losses: list[float] = [] + lr = config.learning_rate + l2_step = config.l2 / float(len(prepared)) + for _epoch in range(config.n_epochs): + order = rng.permutation(len(prepared)) + total = 0.0 + for example_index in order: + features, label = prepared[int(example_index)] + pre, hidden, logits = _forward(features, weights) + loss, grad_logits = _softmax_nll(logits, label) + if not np.isfinite(loss): + raise RerankError("listwise loss became non-finite; lower the learning rate") + grad_w2 = hidden.T @ grad_logits + grad_b2 = float(grad_logits.sum()) + d_hidden = grad_logits[:, None] * weights["w2"][None, :] + d_hidden *= pre > 0.0 + grad_w1 = features.T @ d_hidden + grad_b1 = d_hidden.sum(axis=0) + grad_linear = features.T @ grad_logits + weights["w1"] = weights["w1"] - lr * (grad_w1 + l2_step * weights["w1"]) + weights["b1"] = weights["b1"] - lr * grad_b1 + weights["w2"] = weights["w2"] - lr * (grad_w2 + l2_step * weights["w2"]) + weights["b2"] = float(weights["b2"]) - lr * grad_b2 + weights["w_linear"] = weights["w_linear"] - lr * ( + grad_linear + l2_step * weights["w_linear"] + ) + total += loss + losses.append(total / float(len(prepared))) + for key, value in weights.items(): + array = np.asarray(value, dtype=np.float64) + if not np.isfinite(array).all(): + raise RerankError(f"trained weight {key} is non-finite") + return weights, losses + + +def _standardize_fit(matrices: Sequence[np.ndarray]) -> tuple[np.ndarray, np.ndarray]: + flat = np.concatenate([np.asarray(matrix, dtype=np.float64) for matrix in matrices], axis=0) + if flat.ndim != 2 or flat.shape[1] != N_FEATURES or flat.shape[0] < 1: + raise RerankError(f"cannot fit feature norm on shape {flat.shape}") + mean = flat.mean(axis=0) + std = flat.std(axis=0) + std = np.where(std < _STD_FLOOR, 1.0, std) + return mean.astype(np.float64), std.astype(np.float64) + + +def _apply_norm( + matrix: np.ndarray, + mean: np.ndarray, + std: np.ndarray, +) -> np.ndarray: + return (np.asarray(matrix, dtype=np.float64) - mean) / std + + +def _bundle_from_fit( + tables: FeatureTables, + weights: Mapping[str, Any], + mean: np.ndarray, + std: np.ndarray, + config: RerankConfig, + *, + stage1_model: str, + train_stats: Mapping[str, Any], +) -> RerankBundle: + if not isinstance(stage1_model, str) or not stage1_model: + raise RerankError("stage1_model must be a non-empty string") + metadata: dict[str, Any] = { + "schema": RERANK_SCHEMA, + "schema_version": RERANK_SCHEMA_VERSION, + "adr": ADR_ID, + "stage1_model": stage1_model, + "num_users": tables.num_users, + "num_items": tables.num_items, + "n_genres": tables.n_genres, + "hidden_dim": config.hidden_dim, + "feature_names": list(FEATURE_NAMES), + "genre_names": list(GENRE_NAMES) if tables.n_genres == len(GENRE_NAMES) else [ + f"genre_{index}" for index in range(tables.n_genres) + ], + "genres_available": bool(tables.genres_available), + "stage1_n": config.stage1_n, + "score_formula": SCORE_FORMULA, + "tie_break": TIE_BREAK_POLICY, + "hyperparams": config.as_dict(), + "train": dict(train_stats), + } + bundle = RerankBundle( + item_popularity=np.asarray(tables.item_popularity, dtype=np.float64), + item_recency=np.asarray(tables.item_recency, dtype=np.float64), + item_genres=np.asarray(tables.item_genres, dtype=np.float64), + user_genre_profile=np.asarray(tables.user_genre_profile, dtype=np.float64), + w1=np.asarray(weights["w1"], dtype=np.float64), + b1=np.asarray(weights["b1"], dtype=np.float64), + w2=np.asarray(weights["w2"], dtype=np.float64), + b2=np.asarray([float(weights["b2"])], dtype=np.float64), + w_linear=np.asarray(weights["w_linear"], dtype=np.float64), + feature_mean=np.asarray(mean, dtype=np.float64), + feature_std=np.asarray(std, dtype=np.float64), + metadata=metadata, + ) + _validate_bundle(bundle) + return bundle + + +def fit_reranker( + split: SplitResult, + score_user: Callable[[int], np.ndarray], + tables: FeatureTables, + config: RerankConfig | None = None, + *, + stage1_model: str, +) -> tuple[RerankBundle, dict[str, Any]]: + """Fit stage 2 on validation labels inside the stage-1 pool. + + Training pools exclude train positives and test positives. Users whose + validation item was not retrieved are skipped and counted. Feature mean + and std are fit on the training pools only. + """ + config = RerankConfig() if config is None else config + if tables.num_users != split.num_users or tables.num_items != split.num_movies: + raise RerankError( + "feature tables do not match the split " + f"(tables {tables.num_users}x{tables.num_items}, " + f"split {split.num_users}x{split.num_movies})" + ) + pools = build_candidate_pools( + split, + score_user, + label_split="validation", + extra_exclude_splits=("test",), + stage1_n=config.stage1_n, + ) + labeled = [pool for pool in pools if pool.positive_index >= 0] + n_eligible = len(pools) + if not labeled: + raise RerankError( + "no eligible user had the validation item inside the stage-1 " + f"top-{config.stage1_n}. Increase stage1_n or check the retriever. " + "Refusing to train a re-ranker with no positives." + ) + raw = [ + candidate_features(tables, pool.user_id, pool.item_ids, pool.retrieval_scores) + for pool in labeled + ] + mean, std = _standardize_fit(raw) + standardized = [_apply_norm(matrix, mean, std) for matrix in raw] + weights, losses = train_listwise_mlp( + standardized, + [pool.positive_index for pool in labeled], + config, + ) + stats = { + "n_eligible_users": n_eligible, + "n_train_users": len(labeled), + "validation_hit_rate_at_n": len(labeled) / float(n_eligible), + "loss_first_epoch": float(losses[0]), + "loss_final_epoch": float(losses[-1]), + "n_epochs": config.n_epochs, + "label_split": "validation", + "excluded_from_training_pools": ["train", "test"], + } + bundle = _bundle_from_fit( + tables, + weights, + mean, + std, + config, + stage1_model=stage1_model, + train_stats=stats, + ) + return bundle, stats + + +def _logits(bundle: RerankBundle, features: np.ndarray) -> np.ndarray: + if features.size == 0: + return np.zeros(0, dtype=np.float64) + normed = _apply_norm(features, bundle.feature_mean, bundle.feature_std) + _pre, _hidden, logits = _forward( + normed, + { + "w1": bundle.w1, + "b1": bundle.b1, + "w2": bundle.w2, + "b2": float(bundle.b2[0]), + "w_linear": bundle.w_linear, + }, + ) + if not np.isfinite(logits).all(): + raise RerankError("re-ranker produced non-finite scores") + return logits + + +def order_candidates( + bundle: RerankBundle, + user_id: int, + item_ids: Sequence[int], + retrieval_scores: Sequence[float], +) -> list[tuple[int, float]]: + """Re-rank a stage-1 pool. Score descending, then smaller ``item_id``.""" + user_id = _require_int(user_id, "user_id") + if user_id < 0 or user_id >= bundle.num_users: + raise RerankError( + f"user_id {user_id} is outside the re-rank artifact [0, {bundle.num_users})" + ) + items = np.asarray(list(item_ids), dtype=np.int64) + if items.size == 0: + return [] + tables = FeatureTables( + item_popularity=bundle.item_popularity, + item_recency=bundle.item_recency, + item_genres=bundle.item_genres, + user_genre_profile=bundle.user_genre_profile, + genres_available=bool(bundle.metadata.get("genres_available")), + ) + features = candidate_features(tables, user_id, items, retrieval_scores) + logits = _logits(bundle, features) + order = np.lexsort((items, -logits)) + return [(int(items[index]), float(logits[index])) for index in order] + + +def _metric_block( + ranked_by_user: Sequence[Sequence[int]], + relevant_by_user: Sequence[int], + k: int, +) -> dict[str, float]: + recalls: list[float] = [] + ndcgs: list[float] = [] + aps: list[float] = [] + for ranked, relevant_id in zip(ranked_by_user, relevant_by_user): + relevant = {int(relevant_id)} + recalls.append(metrics.recall_at_k(ranked, relevant, k)) + ndcgs.append(metrics.ndcg_at_k(ranked, relevant, k)) + aps.append(metrics.average_precision_at_k(ranked, relevant, k)) + n_users = float(len(relevant_by_user)) + return { + "recall_at_k": sum(recalls) / n_users, + "ndcg_at_k": sum(ndcgs) / n_users, + "map_at_k": sum(aps) / n_users, + } + + +def _rank_rerank(bundle: RerankBundle, pool: CandidatePool) -> list[int]: + ordered = order_candidates( + bundle, + pool.user_id, + pool.item_ids.tolist(), + pool.retrieval_scores.tolist(), + ) + return [item_id for item_id, _score in ordered] + + +def evaluate_two_stage( + split: SplitResult, + score_user: Callable[[int], np.ndarray], + tables: FeatureTables, + config: RerankConfig | None = None, + *, + stage1_model: str, +) -> dict[str, Any]: + """Train on validation pools and score the test split both ways. + + Test pools exclude train and validation positives. Retrieval-only keeps + the stage-1 order. Retrieval plus re-rank reorders that same pool. + """ + config = RerankConfig() if config is None else config + bundle, train_stats = fit_reranker( + split, + score_user, + tables, + config, + stage1_model=stage1_model, + ) + test_pools = build_candidate_pools( + split, + score_user, + label_split="test", + extra_exclude_splits=("validation",), + stage1_n=config.stage1_n, + ) + retrieval_ranked = [pool.item_ids.tolist() for pool in test_pools] + rerank_ranked = [_rank_rerank(bundle, pool) for pool in test_pools] + relevant = [pool.relevant_id for pool in test_pools] + retrieval_metrics = _metric_block(retrieval_ranked, relevant, config.k) + rerank_metrics = _metric_block(rerank_ranked, relevant, config.k) + hits = sum(1 for pool in test_pools if pool.positive_index >= 0) + n_users = len(test_pools) + delta = { + key: rerank_metrics[key] - retrieval_metrics[key] + for key in ("recall_at_k", "ndcg_at_k", "map_at_k") + } + return { + "k": config.k, + "stage1_n": config.stage1_n, + "n_users": n_users, + "stage1_hit_rate_at_n": hits / float(n_users), + "retrieval_only": retrieval_metrics, + "retrieval_plus_rerank": rerank_metrics, + "delta_retrieval_plus_rerank_minus_retrieval_only": delta, + "rerank_train": train_stats, + "bundle": bundle, + "config": config.as_dict(), + "stage1_model": stage1_model, + "genres_available": bool(tables.genres_available), + "feature_names": list(FEATURE_NAMES), + } + + +def _validate_bundle(bundle: RerankBundle) -> None: + meta = bundle.metadata + if not isinstance(meta, dict): + raise RerankError(f"metadata must be a dict, got {type(meta).__name__}") + if meta.get("schema") != RERANK_SCHEMA: + raise RerankError( + f"metadata.schema is {meta.get('schema')!r}, expected {RERANK_SCHEMA!r}" + ) + if meta.get("schema_version") != RERANK_SCHEMA_VERSION: + raise RerankError( + f"metadata.schema_version is {meta.get('schema_version')!r}, " + f"expected {RERANK_SCHEMA_VERSION}" + ) + stage1_model = meta.get("stage1_model") + if not isinstance(stage1_model, str) or not stage1_model: + raise RerankError("metadata.stage1_model must be a non-empty string") + if list(meta.get("feature_names") or []) != list(FEATURE_NAMES): + raise RerankError( + f"metadata.feature_names must be {list(FEATURE_NAMES)}, " + f"got {meta.get('feature_names')!r}" + ) + popularity = np.asarray(bundle.item_popularity) + recency = np.asarray(bundle.item_recency) + genres = np.asarray(bundle.item_genres) + profile = np.asarray(bundle.user_genre_profile) + if popularity.ndim != 1 or recency.shape != popularity.shape: + raise RerankError( + "item_popularity and item_recency must be 1-d and the same length, " + f"got {popularity.shape} and {recency.shape}" + ) + num_items = int(popularity.shape[0]) + num_users = int(profile.shape[0]) + if num_users < 1 or num_items < 1: + raise RerankError( + f"re-rank tables need at least one user and item, got {num_users} x {num_items}" + ) + if genres.ndim != 2 or genres.shape[0] != num_items or profile.shape != (num_users, genres.shape[1]): + raise RerankError( + "item_genres and user_genre_profile must share a genre axis, " + f"got genres {genres.shape} profile {profile.shape}" + ) + if genres.shape[1] < 1: + raise RerankError("item_genres must have at least one column") + w1 = np.asarray(bundle.w1) + b1 = np.asarray(bundle.b1) + w2 = np.asarray(bundle.w2) + b2 = np.asarray(bundle.b2) + linear = np.asarray(bundle.w_linear) + mean = np.asarray(bundle.feature_mean) + std = np.asarray(bundle.feature_std) + if w1.shape != (N_FEATURES, w1.shape[1] if w1.ndim == 2 else 0): + raise RerankError(f"w1 must have shape ({N_FEATURES}, hidden), got {w1.shape}") + hidden = int(w1.shape[1]) + if hidden < 1 or b1.shape != (hidden,) or w2.shape != (hidden,): + raise RerankError( + f"hidden weights must match hidden={hidden}, got b1 {b1.shape} w2 {w2.shape}" + ) + if b2.shape != (1,) or linear.shape != (N_FEATURES,) or mean.shape != (N_FEATURES,): + raise RerankError( + "b2, w_linear, and feature_mean must match the feature width, " + f"got b2 {b2.shape} w_linear {linear.shape} mean {mean.shape}" + ) + if std.shape != (N_FEATURES,) or np.any(std <= 0.0): + raise RerankError( + f"feature_std must be positive with shape ({N_FEATURES},), got {std}" + ) + arrays = (popularity, recency, genres, profile, w1, b1, w2, b2, linear, mean, std) + if any(array.dtype != np.float64 or not np.isfinite(array).all() for array in arrays): + raise RerankError("re-rank arrays must be finite float64") + for field, expected in ( + ("num_users", num_users), + ("num_items", num_items), + ("n_genres", int(genres.shape[1])), + ("hidden_dim", hidden), + ): + if meta.get(field) != expected: + raise RerankError( + f"metadata.{field} is {meta.get(field)!r}, expected {expected}" + ) + + +def assert_rerank_compatible( + bundle: RerankBundle, + *, + num_users: int, + num_items: int, + stage1_model: str, +) -> None: + """Fail when a serving catalog and a re-rank artifact disagree.""" + _validate_bundle(bundle) + if bundle.num_users != num_users or bundle.num_items != num_items: + raise RerankError( + "re-rank artifact catalog is " + f"{bundle.num_users} users x {bundle.num_items} items, but the " + f"serving artifact is {num_users} x {num_items}. Export both from " + "the same checkpoint." + ) + trained_for = bundle.metadata["stage1_model"] + if trained_for != stage1_model: + raise RerankError( + f"re-rank artifact stage1_model is {trained_for!r} but the serving " + f"artifact model is {stage1_model!r}. Train the re-ranker on the " + "same embedding model, or omit SAGEREC_RERANK_ARTIFACT. " + "Re-ranking is optional." + ) + + +def save_rerank_artifact(path: str | Path, bundle: RerankBundle) -> Path: + """Write ``bundle`` to a compressed ``.npz`` artifact.""" + dest = Path(path) + if dest.suffix != ".npz": + raise RerankError( + f"re-rank artifact path {dest} must end in .npz " + f"(schema {RERANK_SCHEMA!r})" + ) + _validate_bundle(bundle) + dest.parent.mkdir(parents=True, exist_ok=True) + payload = json.dumps(bundle.metadata, sort_keys=True, separators=(",", ":")).encode("utf-8") + np.savez_compressed( + dest, + item_popularity=np.asarray(bundle.item_popularity, dtype=np.float64), + item_recency=np.asarray(bundle.item_recency, dtype=np.float64), + item_genres=np.asarray(bundle.item_genres, dtype=np.float64), + user_genre_profile=np.asarray(bundle.user_genre_profile, dtype=np.float64), + w1=np.asarray(bundle.w1, dtype=np.float64), + b1=np.asarray(bundle.b1, dtype=np.float64), + w2=np.asarray(bundle.w2, dtype=np.float64), + b2=np.asarray(bundle.b2, dtype=np.float64), + w_linear=np.asarray(bundle.w_linear, dtype=np.float64), + feature_mean=np.asarray(bundle.feature_mean, dtype=np.float64), + feature_std=np.asarray(bundle.feature_std, dtype=np.float64), + metadata_json=np.frombuffer(payload, dtype=np.uint8).copy(), + ) + return dest + + +def load_rerank_artifact(path: str | Path) -> RerankBundle: + """Load a schema-v1 re-rank artifact. Missing or corrupt files fail.""" + file_path = Path(path) + if not file_path.exists(): + raise RerankError( + f"re-rank artifact {file_path} does not exist. Train one with " + "scripts/run_rerank_eval.py. The HTTP service does not train a " + "re-ranker and does not download MovieLens." + ) + if not file_path.is_file(): + raise RerankError(f"re-rank artifact {file_path} is not a file") + required = ( + "item_popularity", + "item_recency", + "item_genres", + "user_genre_profile", + "w1", + "b1", + "w2", + "b2", + "w_linear", + "feature_mean", + "feature_std", + "metadata_json", + ) + try: + with np.load(file_path, allow_pickle=False) as data: + missing = [key for key in required if key not in data.files] + if missing: + raise RerankError( + f"re-rank artifact {file_path} is missing arrays {missing}. " + f"Expected schema {RERANK_SCHEMA!r} version {RERANK_SCHEMA_VERSION}." + ) + metadata = json.loads(data["metadata_json"].tobytes().decode("utf-8")) + bundle = RerankBundle( + item_popularity=np.array(data["item_popularity"], dtype=np.float64, copy=True), + item_recency=np.array(data["item_recency"], dtype=np.float64, copy=True), + item_genres=np.array(data["item_genres"], dtype=np.float64, copy=True), + user_genre_profile=np.array(data["user_genre_profile"], dtype=np.float64, copy=True), + w1=np.array(data["w1"], dtype=np.float64, copy=True), + b1=np.array(data["b1"], dtype=np.float64, copy=True), + w2=np.array(data["w2"], dtype=np.float64, copy=True), + b2=np.array(data["b2"], dtype=np.float64, copy=True), + w_linear=np.array(data["w_linear"], dtype=np.float64, copy=True), + feature_mean=np.array(data["feature_mean"], dtype=np.float64, copy=True), + feature_std=np.array(data["feature_std"], dtype=np.float64, copy=True), + metadata=metadata, + ) + except RerankError: + raise + except Exception as exc: + raise RerankError( + f"re-rank artifact {file_path} is corrupt or not a SageRec re-rank " + f"artifact: {exc}. Expected schema {RERANK_SCHEMA!r} version " + f"{RERANK_SCHEMA_VERSION}." + ) from exc + try: + _validate_bundle(bundle) + except RerankError as exc: + raise RerankError(f"re-rank artifact {file_path} failed validation: {exc}") from exc + return bundle + + +def build_synthetic_rerank_bundle( + *, + num_users: int, + num_items: int, + seed: int = DEFAULT_SEED, + stage1_model: str, + stage1_n: int = DEFAULT_STAGE1_N, + hidden_dim: int = DEFAULT_HIDDEN, + n_genres: int = len(GENRE_NAMES), +) -> RerankBundle: + """Seeded noise MLP and feature tables for latency measurement. + + This is not a MovieLens-trained re-ranker and not a ranking-quality + result. The forward pass still multiplies real arrays so the timed path + does the same work as a loaded model. + """ + num_users = _require_int(num_users, "num_users") + num_items = _require_int(num_items, "num_items") + seed = _require_int(seed, "seed") + stage1_n = _require_int(stage1_n, "stage1_n") + hidden_dim = _require_int(hidden_dim, "hidden_dim") + n_genres = _require_int(n_genres, "n_genres") + if not isinstance(stage1_model, str) or not stage1_model: + raise RerankError("stage1_model must be a non-empty string") + if min(num_users, num_items, hidden_dim, n_genres, stage1_n) < 1: + raise RerankError( + "synthetic re-rank sizes must be >= 1, got " + f"users={num_users} items={num_items} hidden={hidden_dim} " + f"genres={n_genres} stage1_n={stage1_n}" + ) + if seed < 0: + raise RerankError(f"seed must be >= 0, got {seed}") + rng = np.random.default_rng(seed) + tables = FeatureTables( + item_popularity=np.log1p(rng.integers(0, 50, size=num_items)).astype(np.float64), + item_recency=rng.random(num_items), + item_genres=(rng.random((num_items, n_genres)) < 0.2).astype(np.float64), + user_genre_profile=rng.random((num_users, n_genres)), + genres_available=False, + ) + config = RerankConfig( + stage1_n=stage1_n, + hidden_dim=hidden_dim, + n_epochs=1, + learning_rate=DEFAULT_LEARNING_RATE, + l2=0.0, + seed=seed, + k=min(DEFAULT_K, stage1_n), + init_std=DEFAULT_INIT_STD, + ) + weights = _init_mlp(config, rng) + mean = np.zeros(N_FEATURES, dtype=np.float64) + std = np.ones(N_FEATURES, dtype=np.float64) + bundle = _bundle_from_fit( + tables, + weights, + mean, + std, + config, + stage1_model=stage1_model, + train_stats={ + "kind": "synthetic", + "note": ( + "Seeded synthetic MLP for ADR-010 latency. Feature tables are " + "noise, not MovieLens train counts, genres, or timestamps." + ), + }, + ) + bundle.metadata["catalog_kind"] = "synthetic" + bundle.metadata["seed"] = seed + bundle.metadata["note"] = ( + "Synthetic re-rank artifact for HTTP latency. Not the MovieLens " + "trained model and not a ranking-quality result." + ) + _validate_bundle(bundle) + return bundle + + +def zero_linear_bundle( + tables: FeatureTables, + *, + stage1_model: str, + popularity_weight: float = 0.0, + stage1_n: int = DEFAULT_STAGE1_N, +) -> RerankBundle: + """Hand-built bundle whose score is ``popularity_weight * popularity``. + + Other features and the hidden layer are zero. Used by tests that need a + known re-rank order without training. + """ + hidden = 1 + weights = { + "w1": np.zeros((N_FEATURES, hidden), dtype=np.float64), + "b1": np.zeros(hidden, dtype=np.float64), + "w2": np.zeros(hidden, dtype=np.float64), + "b2": 0.0, + "w_linear": np.asarray( + [0.0, float(popularity_weight), 0.0, 0.0], + dtype=np.float64, + ), + } + config = RerankConfig( + stage1_n=max(stage1_n, DEFAULT_K), + hidden_dim=hidden, + n_epochs=1, + seed=0, + k=DEFAULT_K, + ) + return _bundle_from_fit( + tables, + weights, + np.zeros(N_FEATURES, dtype=np.float64), + np.ones(N_FEATURES, dtype=np.float64), + config, + stage1_model=stage1_model, + train_stats={"kind": "hand_built"}, + ) diff --git a/python/sagerec_rerank_eval.py b/python/sagerec_rerank_eval.py new file mode 100644 index 0000000..2fdb38a --- /dev/null +++ b/python/sagerec_rerank_eval.py @@ -0,0 +1,569 @@ +"""MovieLens 100K retrieval versus retrieval-plus-re-rank evaluation (ADR-010). + +Stage 1 is implicit MF trained on ADR-003 train positives (the same dot-product +retrieval ADR-009 serves). Stage 2 is the NumPy MLP in ``sagerec_rerank``, +fit on validation labels inside the stage-1 pool and scored on the test split. +Genre flags come from ``u.item``. This module does not start HTTP and does +not overwrite Phase 5 or Phase 7 result files. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import subprocess +import sys +from collections.abc import Mapping, Sequence +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import numpy as np + +import sagerec_baseline as baseline +import sagerec_dataset as dataset +import sagerec_download as download +import sagerec_prep as prep +import sagerec_rerank as rerank +import sagerec_serving as serving + +PROTOCOL_VERSION = "rerank-eval-v1" +ADR_ID = rerank.ADR_ID +METRIC_KEYS = ("recall_at_10", "ndcg_at_10", "map") + +REQUIRED_RERANK_EVAL_KEYS = ( + "benchmark", + "protocol_version", + "adr", + "dataset_edition", + "source_url", + "checksum", + "license", + "split_policy_id", + "split_policy_version", + "seed", + "stage1", + "stage2", + "protocol", + "n_users", + "stage1_hit_rate_at_n", + "retrieval_only", + "retrieval_plus_rerank", + "delta_retrieval_plus_rerank_minus_retrieval_only", + "rerank_train", + "eligibility", + "cpu", + "platform", + "python", + "numpy", + "git_commit", + "git_dirty", + "generated_at", + "note", +) + + +class RerankEvalError(RuntimeError): + """The MovieLens re-rank evaluation could not be measured.""" + + +def repo_root() -> Path: + return Path(__file__).resolve().parents[1] + + +def cpu_info() -> dict[str, Any]: + model: str | None = None + cpuinfo = Path("/proc/cpuinfo") + if cpuinfo.is_file(): + for line in cpuinfo.read_text(encoding="utf-8", errors="replace").splitlines(): + if line.lower().startswith("model name"): + model = line.split(":", 1)[1].strip() + break + return { + "model": model or (platform.processor() or "unknown"), + "logical_cpus": os.cpu_count(), + "machine": platform.machine(), + } + + +def git_commit(root: Path) -> str | None: + try: + value = subprocess.check_output( + ["git", "rev-parse", "HEAD"], + cwd=root, + stderr=subprocess.DEVNULL, + text=True, + ).strip() + except (OSError, subprocess.CalledProcessError): + return None + return value or None + + +def git_dirty(root: Path) -> bool: + try: + value = subprocess.check_output( + ["git", "status", "--porcelain"], + cwd=root, + stderr=subprocess.DEVNULL, + text=True, + ) + except (OSError, subprocess.CalledProcessError): + return True + return bool(value.strip()) + + +def _finite(value: float, field: str) -> float: + number = float(value) + if not np.isfinite(number): + raise RerankEvalError(f"{field} is not finite: {value!r}") + return number + + +def _at_10(block: Mapping[str, float], *, k: int) -> dict[str, float]: + if k != 10: + raise RerankEvalError( + f"stored rerank-eval-v1 requires k=10, got k={k}. " + "Do not relabel another cutoff as Recall@10." + ) + return { + "recall_at_10": _finite(block["recall_at_k"], "recall_at_k"), + "ndcg_at_10": _finite(block["ndcg_at_k"], "ndcg_at_k"), + "map": _finite(block["map_at_k"], "map_at_k"), + } + + +def resolve_uitem(raw_dir: Path, uitem_path: Path | None) -> tuple[str, str]: + """Return ``(u.item text, origin)`` without downloading a new archive. + + Looks at ``--uitem``, then ``raw_dir/ml-100k/u.item``, then extracts + ``u.item`` from an already-present ``raw_dir/ml-100k.zip``. + """ + if uitem_path is not None: + if not uitem_path.is_file(): + raise RerankEvalError( + f"--uitem {uitem_path} does not exist. Pass the MovieLens " + "100K u.item file. This script does not download it on its own." + ) + return uitem_path.read_text(encoding="latin-1"), str(uitem_path) + extracted = raw_dir / "ml-100k" / "u.item" + if extracted.is_file(): + return extracted.read_text(encoding="latin-1"), str(extracted) + archive_path = raw_dir / "ml-100k.zip" + if not archive_path.is_file(): + raise RerankEvalError( + "MovieLens u.item was not found. Pass --uitem, place u.item at " + f"{extracted}, or keep the official zip at {archive_path}. " + "Genre affinity needs those flags. Genres are not invented." + ) + payload, member = download.extract_uitem(archive_path.read_bytes()) + extracted.parent.mkdir(parents=True, exist_ok=True) + extracted.write_bytes(payload) + return payload.decode("latin-1"), f"{archive_path}#{member}" + + +def _movie_source_ids(processed_dir: Path, num_items: int) -> list[int]: + path = processed_dir / dataset.MAPPINGS_NAME + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise RerankEvalError( + f"could not read movie source ids from {path}: {exc}" + ) from exc + source_ids = payload.get("movie_source_ids") + if not isinstance(source_ids, list) or len(source_ids) != num_items: + raise RerankEvalError( + f"{path} movie_source_ids must be a list of length {num_items}" + ) + return [int(source_id) for source_id in source_ids] + + +def mf_score_matrix(model: baseline.ImplicitMF) -> np.ndarray: + """Full user-item score matrix, matching ``ImplicitMF.score_pairs``.""" + scores = model.user_factors @ model.item_factors.T + scores += model.user_bias[:, None] + scores += model.item_bias[None, :] + probe_items = [0, model.num_items // 2, model.num_items - 1] + probe_users = [0] + for user_id in probe_users: + live = np.asarray( + model.score_pairs([user_id] * len(probe_items), probe_items), + dtype=np.float64, + ) + if not np.allclose(scores[user_id, probe_items], live, rtol=1e-8, atol=1e-8): + raise RerankEvalError( + "stage-1 matrix scores do not match ImplicitMF.score_pairs" + ) + return scores + + +def build_rerank_eval_payload( + *, + manifest: Mapping[str, Any], + measured: Mapping[str, Any], + mf_config: baseline.MFConfig, + uitem_origin: str, + root: Path | None = None, + generated_at: str | None = None, +) -> dict[str, Any]: + """Machine-readable comparison. Does not invent metrics.""" + root = repo_root() if root is None else root + retrieval = _at_10(measured["retrieval_only"], k=int(measured["k"])) + reranked = _at_10(measured["retrieval_plus_rerank"], k=int(measured["k"])) + delta_raw = measured["delta_retrieval_plus_rerank_minus_retrieval_only"] + delta = { + "recall_at_10": _finite(delta_raw["recall_at_k"], "delta.recall_at_k"), + "ndcg_at_10": _finite(delta_raw["ndcg_at_k"], "delta.ndcg_at_k"), + "map": _finite(delta_raw["map_at_k"], "delta.map_at_k"), + } + for key in METRIC_KEYS: + if abs((reranked[key] - retrieval[key]) - delta[key]) > 1e-12: + raise RerankEvalError(f"delta.{key} does not match the two metric blocks") + dirty = git_dirty(root) + note = ( + "Single-seed MovieLens 100K two-stage comparison (ADR-010). " + "Stage 1 is implicit MF (ADR-004) dot-product retrieval, the same " + "score ADR-009 serves, trained on ADR-003 train positives only. " + "Stage 2 is a NumPy MLP with a linear skip, fit listwise on " + "validation items that stage 1 retrieved, and scored on the test " + "split. Both systems rank inside that stage-1 top-N pool. " + "MAP is MAP@10. Features use train interactions only. Genres come " + "from u.item. This is not a GraphSAGE result, not the Phase 5 or " + "Phase 7 leaderboard, and not a SOTA claim. Re-ranking cannot " + "recover a test item that stage 1 did not retrieve." + ) + if dirty: + note += ( + " git_dirty was true at generation, so git_commit is the HEAD " + "the process saw and may not contain this results file." + ) + counts = manifest["counts"] + return { + "benchmark": "sagerec_two_stage_rerank", + "protocol_version": PROTOCOL_VERSION, + "adr": ADR_ID, + "dataset_edition": manifest["dataset_edition"], + "source_url": manifest.get("source_url"), + "checksum": manifest.get("checksum"), + "license": manifest.get("license"), + "split_policy_id": manifest["split_policy_id"], + "split_policy_version": manifest["split_policy_version"], + "seed": int(mf_config.seed), + "stage1": { + "model": "implicit_mf", + "score_formula": serving.SCORE_FORMULA, + "tie_break": serving.TIE_BREAK_POLICY, + "hyperparams": mf_config.as_dict(), + "stage1_n": int(measured["stage1_n"]), + "candidate_filter": ( + "test eval excludes train and validation positives; " + "the test positive stays in the pool" + ), + }, + "stage2": { + "model": "numpy_mlp_listwise", + "score_formula": rerank.SCORE_FORMULA, + "tie_break": rerank.TIE_BREAK_POLICY, + "hyperparams": dict(measured["config"]), + "feature_names": list(measured["feature_names"]), + "genres_available": bool(measured["genres_available"]), + "genre_names": list(rerank.GENRE_NAMES), + "uitem_origin": uitem_origin, + "label_split": "validation", + "eval_split": "test", + "training_pool_excludes": ["train", "test"], + }, + "protocol": { + "k": 10, + "stage1_n": int(measured["stage1_n"]), + "metrics": list(METRIC_KEYS), + "map": rerank.MAP_DEFINITION, + "eligible_users": "ADR-003 users with at least 3 interactions", + "same_pool": True, + }, + "n_users": int(measured["n_users"]), + "stage1_hit_rate_at_n": _finite( + measured["stage1_hit_rate_at_n"], "stage1_hit_rate_at_n" + ), + "retrieval_only": retrieval, + "retrieval_plus_rerank": reranked, + "delta_retrieval_plus_rerank_minus_retrieval_only": delta, + "rerank_train": { + key: measured["rerank_train"][key] + for key in ( + "n_eligible_users", + "n_train_users", + "validation_hit_rate_at_n", + "loss_first_epoch", + "loss_final_epoch", + "n_epochs", + "label_split", + "excluded_from_training_pools", + ) + }, + "eligibility": { + "users": counts["users"], + "movies": counts["movies"], + "interactions": counts["interactions"], + "eligible_users": counts["eligible_users"], + "cold_start_users": counts["cold_start_users"], + "evaluated_users": int(measured["n_users"]), + }, + "cpu": cpu_info(), + "platform": { + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "processor": platform.processor(), + }, + "python": sys.version.split()[0], + "numpy": np.__version__, + "git_commit": git_commit(root), + "git_dirty": dirty, + "generated_at": generated_at or datetime.now(timezone.utc).isoformat(), + "note": note, + } + + +def render_rerank_eval_markdown(payload: Mapping[str, Any]) -> str: + """Markdown table rendered from a rerank-eval payload (no extra rounding).""" + missing = [key for key in REQUIRED_RERANK_EVAL_KEYS if key not in payload] + if missing: + raise RerankEvalError(f"rerank eval payload is missing keys {missing}") + retrieval = payload["retrieval_only"] + reranked = payload["retrieval_plus_rerank"] + delta = payload["delta_retrieval_plus_rerank_minus_retrieval_only"] + for block_name, block in ( + ("retrieval_only", retrieval), + ("retrieval_plus_rerank", reranked), + ("delta", delta), + ): + for key in METRIC_KEYS: + if key not in block: + raise RerankEvalError(f"{block_name} is missing {key}") + + def fmt(value: float) -> str: + return f"{float(value):.6f}" + + stage1 = payload["stage1"] + stage2 = payload["stage2"] + cpu = payload["cpu"] + lines = [ + "# SageRec two-stage re-rank evaluation", + "", + "MovieLens 100K, ADR-003 leave-one-out, test split. Stage 1 is implicit " + "MF retrieval. Stage 2 re-ranks that top-N pool. Not a GraphSAGE result " + "and not the Phase 5 or Phase 7 leaderboard.", + "", + f"- Protocol: `{payload['protocol_version']}` ({payload['adr']})", + ( + f"- Seed: {payload['seed']}; stage-1 N: {stage1['stage1_n']}; " + f"users: {payload['n_users']}; " + f"stage-1 hit rate @N: {fmt(payload['stage1_hit_rate_at_n'])}" + ), + ( + f"- Features: {', '.join(stage2['feature_names'])}; " + f"genres_available={stage2['genres_available']}" + ), + ( + f"- Python: {payload['python']}; NumPy: {payload['numpy']}; " + f"CPU: {cpu['model']} ({cpu['logical_cpus']} logical)" + ), + f"- Commit: `{payload['git_commit']}` (dirty={payload['git_dirty']})", + "", + "| System | Recall@10 | NDCG@10 | MAP@10 |", + "| --- | ---: | ---: | ---: |", + ( + f"| retrieval only | {fmt(retrieval['recall_at_10'])} | " + f"{fmt(retrieval['ndcg_at_10'])} | {fmt(retrieval['map'])} |" + ), + ( + f"| retrieval + re-rank | {fmt(reranked['recall_at_10'])} | " + f"{fmt(reranked['ndcg_at_10'])} | {fmt(reranked['map'])} |" + ), + ( + f"| delta (re-rank − retrieval) | {fmt(delta['recall_at_10'])} | " + f"{fmt(delta['ndcg_at_10'])} | {fmt(delta['map'])} |" + ), + "", + payload["note"], + "", + ] + return "\n".join(lines) + + +def write_rerank_eval_results( + payload: Mapping[str, Any], + json_path: str | Path, + md_path: str | Path, +) -> tuple[Path, Path]: + missing = [key for key in REQUIRED_RERANK_EVAL_KEYS if key not in payload] + if missing: + raise RerankEvalError(f"refusing to write rerank eval missing {missing}") + json_dest = Path(json_path) + md_dest = Path(md_path) + json_dest.parent.mkdir(parents=True, exist_ok=True) + md_dest.parent.mkdir(parents=True, exist_ok=True) + json_dest.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + md_dest.write_text(render_rerank_eval_markdown(payload), encoding="utf-8") + return json_dest, md_dest + + +def run_movielens_rerank_eval( + *, + processed_dir: str | Path, + raw_dir: str | Path, + uitem_path: str | Path | None = None, + seed: int = rerank.DEFAULT_SEED, + stage1_n: int = rerank.DEFAULT_STAGE1_N, + hidden_dim: int = rerank.DEFAULT_HIDDEN, + n_epochs: int = rerank.DEFAULT_EPOCHS, + learning_rate: float = rerank.DEFAULT_LEARNING_RATE, + l2: float = rerank.DEFAULT_L2, + json_out: str | Path | None = None, + md_out: str | Path | None = None, + rerank_out: str | Path | None = None, + root: Path | None = None, +) -> dict[str, Any]: + """Train MF, fit the re-ranker, and write the measured comparison.""" + root = repo_root() if root is None else Path(root) + processed = Path(processed_dir) + raw = Path(raw_dir) + if json_out is None: + json_out = root / "results" / "rerank_eval.json" + if md_out is None: + md_out = root / "results" / "rerank_eval.md" + split, manifest = dataset.ensure_movielens_100k_processed(processed, raw) + if manifest.get("dataset_edition") != prep.DATASET_EDITION_100K: + raise RerankEvalError( + f"expected dataset edition {prep.DATASET_EDITION_100K}, " + f"got {manifest.get('dataset_edition')!r}" + ) + if manifest.get("split_policy_id") != prep.SPLIT_POLICY_ID: + raise RerankEvalError( + f"expected split {prep.SPLIT_POLICY_ID}, got {manifest.get('split_policy_id')!r}" + ) + uitem_text, uitem_origin = resolve_uitem(raw, Path(uitem_path) if uitem_path else None) + genres = rerank.parse_uitem_genres( + uitem_text, + split.num_movies, + _movie_source_ids(processed, split.num_movies), + ) + mf_config = baseline.movielens_100k_config(seed=seed) + model = baseline.train_implicit_mf(split, mf_config) + score_matrix = mf_score_matrix(model) + tables = rerank.build_feature_tables(split, genres) + config = rerank.RerankConfig( + stage1_n=stage1_n, + hidden_dim=hidden_dim, + n_epochs=n_epochs, + learning_rate=learning_rate, + l2=l2, + seed=seed, + k=10, + ) + + def score_user(user_id: int) -> np.ndarray: + return score_matrix[int(user_id)] + + measured = rerank.evaluate_two_stage( + split, + score_user, + tables, + config, + stage1_model="implicit_mf", + ) + payload = build_rerank_eval_payload( + manifest=manifest, + measured=measured, + mf_config=mf_config, + uitem_origin=uitem_origin, + root=root, + ) + write_rerank_eval_results(payload, json_out, md_out) + if rerank_out is not None: + rerank.save_rerank_artifact(rerank_out, measured["bundle"]) + return payload + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Compare MovieLens 100K retrieval-only vs retrieval plus re-rank " + "and write results/rerank_eval.json. Uses ADR-003 prep and implicit " + "MF as stage 1. Does not overwrite Phase 5 or Phase 7 files." + ) + ) + parser.add_argument("--processed-dir", type=Path, default=Path("data/processed")) + parser.add_argument("--raw-dir", type=Path, default=Path("data/raw")) + parser.add_argument("--uitem", type=Path, default=None) + parser.add_argument("--seed", type=int, default=rerank.DEFAULT_SEED) + parser.add_argument("--stage1-n", type=int, default=rerank.DEFAULT_STAGE1_N) + parser.add_argument("--hidden-dim", type=int, default=rerank.DEFAULT_HIDDEN) + parser.add_argument("--epochs", type=int, default=rerank.DEFAULT_EPOCHS) + parser.add_argument("--learning-rate", type=float, default=rerank.DEFAULT_LEARNING_RATE) + parser.add_argument("--l2", type=float, default=rerank.DEFAULT_L2) + parser.add_argument("--json-out", type=Path, default=None) + parser.add_argument("--md-out", type=Path, default=None) + parser.add_argument( + "--rerank-out", + type=Path, + default=None, + help="Optional gitignored .npz re-rank artifact (for example results/serving/rerank.npz).", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + parser = build_parser() + try: + args = parser.parse_args(list(argv) if argv is not None else None) + except SystemExit as exc: + code = exc.code + return 0 if code is None else int(code) + try: + payload = run_movielens_rerank_eval( + processed_dir=args.processed_dir, + raw_dir=args.raw_dir, + uitem_path=args.uitem, + seed=args.seed, + stage1_n=args.stage1_n, + hidden_dim=args.hidden_dim, + n_epochs=args.epochs, + learning_rate=args.learning_rate, + l2=args.l2, + json_out=args.json_out, + md_out=args.md_out, + rerank_out=args.rerank_out, + ) + except ( + RerankEvalError, + rerank.RerankError, + ValueError, + dataset.MovieLensPrepError, + download.MovieLensDownloadError, + OSError, + ) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + retrieval = payload["retrieval_only"] + reranked = payload["retrieval_plus_rerank"] + print( + "retrieval_only recall@10={r_rec:.6f} ndcg@10={r_ndcg:.6f} map@10={r_map:.6f} " + "rerank recall@10={k_rec:.6f} ndcg@10={k_ndcg:.6f} map@10={k_map:.6f} " + "hit_at_n={hit:.6f}".format( + r_rec=retrieval["recall_at_10"], + r_ndcg=retrieval["ndcg_at_10"], + r_map=retrieval["map"], + k_rec=reranked["recall_at_10"], + k_ndcg=reranked["ndcg_at_10"], + k_map=reranked["map"], + hit=payload["stage1_hit_rate_at_n"], + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/tests/AGENTS.md b/python/tests/AGENTS.md index 87056e4..a0d9bbf 100644 --- a/python/tests/AGENTS.md +++ b/python/tests/AGENTS.md @@ -63,4 +63,13 @@ directory on `sys.path` so `PYTHONPATH=build` still works). Prefer `sample_neighbors`. Keep the module import-safe for unittest discovery. Do not download MovieLens, bind a public port, or rewrite `results/serving_latency.*` from the test. +- Cover Phase 10 in `test_rerank.py` (unittest discovery): MAP@10, + train-only popularity/recency/genre affinity, `u.item` parsing, MLP + learning and determinism, artifact order and tie-break, and a tiny + synthetic two-stage eval. `test_serving_api.py` covers optional + `rerank` / `n` on `/recommend` (order preserved when the flag is off, + 422 without an artifact or when `n < k`, 404 for an unknown user). + Do not download MovieLens or run `scripts/run_rerank_eval.py` in + default CI. The committed-file test reads `results/rerank_eval.json`; + it does not regenerate it. - Keep type hints on test helpers that form part of a public-looking fixture. diff --git a/python/tests/test_movielens_download_prep.py b/python/tests/test_movielens_download_prep.py index ada212d..65e89be 100644 --- a/python/tests/test_movielens_download_prep.py +++ b/python/tests/test_movielens_download_prep.py @@ -49,6 +49,13 @@ def test_extract_udata_from_official_layout(self) -> None: self.assertEqual(member, "ml-100k/u.data") self.assertEqual(payload, _TINY_UDATA.encode()) + def test_extract_uitem_from_official_layout(self) -> None: + payload = b"1|Toy|01-Jan-1995||http://example|0|1\n" + archive = _zip_bytes("ml-100k/u.item", payload) + extracted, member = download.extract_uitem(archive) + self.assertEqual(member, "ml-100k/u.item") + self.assertEqual(extracted, payload) + def test_extract_udata_rejects_missing_member(self) -> None: archive = _zip_bytes("ml-100k/README", b"hello") with self.assertRaises(download.MovieLensDownloadError) as ctx: diff --git a/python/tests/test_rerank.py b/python/tests/test_rerank.py new file mode 100644 index 0000000..1983e4d --- /dev/null +++ b/python/tests/test_rerank.py @@ -0,0 +1,329 @@ +"""Phase 10 ADR-010 feature, re-ranker, and offline-eval tests. + +Tiny in-memory fixtures only. Metrics here verify the protocol; they are not +MovieLens 100K quality results. The committed ``results/rerank_eval.json`` +test checks the measured file's schema and does not retrain 100K. +""" + +from __future__ import annotations + +import json +import sys +import tempfile +import unittest +from pathlib import Path + +import numpy as np + +_PYTHON_DIR = Path(__file__).resolve().parents[1] +_REPO_ROOT = _PYTHON_DIR.parent +if str(_PYTHON_DIR) not in sys.path: + sys.path.insert(0, str(_PYTHON_DIR)) + +import sagerec_metrics as metrics +import sagerec_prep as prep +import sagerec_rerank as rerank + +_RERANK_JSON = _REPO_ROOT / "results" / "rerank_eval.json" +_RERANK_MD = _REPO_ROOT / "results" / "rerank_eval.md" + + +def _row( + user_id: int, + movie_id: int, + timestamp: int, + rating: int = 1, +) -> prep.NormalizedInteraction: + return prep.NormalizedInteraction(user_id, movie_id, timestamp, rating) + + +def _uitem_line(source_id: int, flags: list[int]) -> str: + prefix = f"{source_id}|Movie {source_id}|01-Jan-1995||http://example" + return prefix + "|" + "|".join(str(flag) for flag in flags) + + +class MapMetricTests(unittest.TestCase): + def test_average_precision_at_10_for_one_relevant_item(self) -> None: + relevant = {4} + self.assertEqual(metrics.average_precision_at_k([4, 1, 2], relevant, 10), 1.0) + self.assertEqual(metrics.average_precision_at_k([1, 4, 2], relevant, 10), 0.5) + self.assertEqual(metrics.average_precision_at_k([1, 2, 3], relevant, 10), 0.0) + self.assertEqual(metrics.average_precision_at_k([1, 2, 4], relevant, 2), 0.0) + with self.assertRaises(ValueError): + metrics.average_precision_at_k([1], set(), 10) + + +class FeatureTests(unittest.TestCase): + def _split(self) -> prep.SplitResult: + interactions = [ + _row(0, 0, 10), + _row(0, 1, 30), + _row(0, 2, 40), + _row(0, 3, 1000), + _row(1, 0, 10), + _row(1, 4, 20), + _row(1, 2, 30), + ] + return prep.split_interactions(interactions, num_users=2, num_movies=5) + + def test_train_only_popularity_recency_and_genre_affinity(self) -> None: + split = self._split() + genres = np.asarray( + [ + [1.0, 0.0], + [0.0, 1.0], + [1.0, 1.0], + [1.0, 1.0], + [0.0, 0.0], + ], + dtype=np.float64, + ) + tables = rerank.build_feature_tables(split, genres) + self.assertTrue(tables.genres_available) + np.testing.assert_allclose(tables.item_popularity, np.log1p([2.0, 1.0, 0.0, 0.0, 0.0])) + np.testing.assert_allclose(tables.item_recency, [0.0, 1.0, 0.0, 0.0, 0.0]) + np.testing.assert_allclose(tables.user_genre_profile[0], [0.5, 0.5]) + features = rerank.candidate_features( + tables, + 0, + [0, 1, 3], + [0.2, 0.4, 0.6], + ) + self.assertEqual(features.shape, (3, 4)) + np.testing.assert_allclose(features[:, 0], [0.2, 0.4, 0.6]) + np.testing.assert_allclose(features[:, 1], tables.item_popularity[[0, 1, 3]]) + np.testing.assert_allclose(features[:, 2], [0.5, 0.5, 1.0]) + np.testing.assert_allclose(features[:, 3], tables.item_recency[[0, 1, 3]]) + self.assertEqual(list(rerank.FEATURE_NAMES), [ + "retrieval_score", + "item_popularity", + "genre_affinity", + "item_recency", + ]) + + def test_held_out_rows_do_not_change_feature_tables(self) -> None: + full = self._split() + genres = np.ones((full.num_movies, 1), dtype=np.float64) + tables = rerank.build_feature_tables(full, genres) + train_counts = np.zeros(full.num_movies, dtype=np.float64) + for row in full.interactions_for("train"): + train_counts[row.movie_id] += 1.0 + np.testing.assert_allclose(tables.item_popularity, np.log1p(train_counts)) + self.assertEqual(float(tables.item_popularity[3]), 0.0) + self.assertEqual(float(tables.item_recency[3]), 0.0) + self.assertGreater(float(train_counts[0]), 0.0) + + def test_training_pool_drops_the_test_item_and_test_pool_drops_validation(self) -> None: + split = self._split() + + def score_user(_user_id: int) -> np.ndarray: + return np.arange(split.num_movies, dtype=np.float64) + + train_pools = rerank.build_candidate_pools( + split, + score_user, + label_split="validation", + extra_exclude_splits=("test",), + stage1_n=5, + ) + user0_train = next(pool for pool in train_pools if pool.user_id == 0) + self.assertNotIn(3, user0_train.item_ids.tolist()) + self.assertIn(2, user0_train.item_ids.tolist()) + self.assertGreaterEqual(user0_train.positive_index, 0) + + test_pools = rerank.build_candidate_pools( + split, + score_user, + label_split="test", + extra_exclude_splits=("validation",), + stage1_n=5, + ) + user0_test = next(pool for pool in test_pools if pool.user_id == 0) + self.assertNotIn(2, user0_test.item_ids.tolist()) + self.assertNotIn(0, user0_test.item_ids.tolist()) + self.assertIn(3, user0_test.item_ids.tolist()) + + +class GenreParseTests(unittest.TestCase): + def test_uitem_genres_align_to_local_source_ids(self) -> None: + flags_a = [1] + [0] * 18 + flags_b = [0, 1] + [0] * 17 + text = "\n".join( + [ + _uitem_line(5, flags_a), + _uitem_line(1, flags_b), + ] + ) + genres = rerank.parse_uitem_genres(text, 2, [5, 1]) + self.assertEqual(genres.shape, (2, 19)) + self.assertEqual(float(genres[0, 0]), 1.0) + self.assertEqual(float(genres[1, 1]), 1.0) + self.assertEqual(len(rerank.GENRE_NAMES), 19) + + def test_uitem_rejects_a_short_row(self) -> None: + with self.assertRaises(rerank.RerankError) as ctx: + rerank.parse_uitem_genres("1|Movie|date||url|0\n", 1, [1]) + self.assertIn("fields", str(ctx.exception)) + + +class MlpTests(unittest.TestCase): + def test_listwise_mlp_prefers_the_popularity_feature_and_is_deterministic(self) -> None: + positive = np.asarray([0.0, 4.0, 0.0, 0.0], dtype=np.float64) + negative = np.asarray([4.0, 0.0, 0.0, 0.0], dtype=np.float64) + matrix = np.vstack([positive, negative, negative, negative]) + matrices = [matrix.copy() for _ in range(24)] + labels = [0] * len(matrices) + config = rerank.RerankConfig( + stage1_n=4, + hidden_dim=8, + n_epochs=40, + learning_rate=0.2, + l2=0.0, + seed=7, + k=2, + init_std=0.1, + ) + first, first_loss = rerank.train_listwise_mlp(matrices, labels, config) + second, second_loss = rerank.train_listwise_mlp(matrices, labels, config) + np.testing.assert_allclose(first["w_linear"], second["w_linear"]) + np.testing.assert_allclose(first["w1"], second["w1"]) + self.assertLess(first_loss[-1], first_loss[0]) + _pre, _hidden, logits = rerank._forward(matrix, first) + self.assertGreater(float(logits[0]), float(np.max(logits[1:]))) + + def test_stage1_n_below_k_is_rejected(self) -> None: + with self.assertRaises(rerank.RerankError): + rerank.RerankConfig(stage1_n=5, k=10) + + def test_artifact_roundtrip_preserves_order(self) -> None: + tables = rerank.FeatureTables( + item_popularity=np.asarray([0.0, 3.0, 1.0], dtype=np.float64), + item_recency=np.zeros(3, dtype=np.float64), + item_genres=np.zeros((3, 1), dtype=np.float64), + user_genre_profile=np.zeros((1, 1), dtype=np.float64), + genres_available=False, + ) + bundle = rerank.zero_linear_bundle( + tables, + stage1_model="fixture", + popularity_weight=1.0, + ) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "rerank.npz" + rerank.save_rerank_artifact(path, bundle) + loaded = rerank.load_rerank_artifact(path) + ordered = rerank.order_candidates(loaded, 0, [0, 1, 2], [5.0, 0.0, 4.0]) + self.assertEqual([item_id for item_id, _score in ordered], [1, 2, 0]) + tied = rerank.zero_linear_bundle( + rerank.FeatureTables( + item_popularity=np.zeros(3, dtype=np.float64), + item_recency=np.zeros(3, dtype=np.float64), + item_genres=np.zeros((3, 1), dtype=np.float64), + user_genre_profile=np.zeros((1, 1), dtype=np.float64), + genres_available=False, + ), + stage1_model="fixture", + popularity_weight=1.0, + ) + tie_order = rerank.order_candidates(tied, 0, [2, 0, 1], [1.0, 1.0, 1.0]) + self.assertEqual([item_id for item_id, _score in tie_order], [0, 1, 2]) + with tempfile.TemporaryDirectory() as tmp: + missing = Path(tmp) / "missing.npz" + with self.assertRaises(rerank.RerankError) as ctx: + rerank.load_rerank_artifact(missing) + self.assertIn("does not exist", str(ctx.exception)) + + +class TwoStageEvalTests(unittest.TestCase): + def test_tiny_eval_is_finite_and_repeatable(self) -> None: + interactions = [ + _row(0, 0, 10), + _row(0, 1, 20), + _row(0, 2, 30), + _row(0, 3, 40), + _row(0, 4, 50), + _row(1, 5, 11), + _row(1, 6, 21), + _row(1, 7, 31), + _row(1, 8, 41), + ] + split = prep.split_interactions(interactions, num_users=2, num_movies=12) + genres = np.zeros((split.num_movies, 2), dtype=np.float64) + genres[:, 0] = 1.0 + tables = rerank.build_feature_tables(split, genres) + + def score_user(user_id: int) -> np.ndarray: + scores = np.linspace(0.0, 1.0, split.num_movies) + scores[user_id] += 0.01 + return scores + + config = rerank.RerankConfig( + stage1_n=12, + k=2, + n_epochs=3, + hidden_dim=4, + learning_rate=0.05, + l2=1e-4, + seed=7, + ) + first = rerank.evaluate_two_stage( + split, score_user, tables, config, stage1_model="fixture" + ) + second = rerank.evaluate_two_stage( + split, score_user, tables, config, stage1_model="fixture" + ) + self.assertEqual(first["n_users"], second["n_users"]) + self.assertGreater(first["n_users"], 0) + self.assertEqual(first["retrieval_only"], second["retrieval_only"]) + self.assertEqual(first["retrieval_plus_rerank"], second["retrieval_plus_rerank"]) + np.testing.assert_allclose(first["bundle"].w1, second["bundle"].w1) + for block in (first["retrieval_only"], first["retrieval_plus_rerank"]): + for key in ("recall_at_k", "ndcg_at_k", "map_at_k"): + self.assertGreaterEqual(block[key], 0.0) + self.assertLessEqual(block[key], 1.0) + self.assertGreater(first["rerank_train"]["n_train_users"], 0) + self.assertIn("test", first["rerank_train"]["excluded_from_training_pools"]) + + +class CommittedRerankEvalTests(unittest.TestCase): + def test_committed_rerank_eval_matches_markdown(self) -> None: + import sagerec_rerank_eval as rerank_eval + + self.assertTrue(_RERANK_JSON.is_file(), "results/rerank_eval.json is missing") + self.assertTrue(_RERANK_MD.is_file(), "results/rerank_eval.md is missing") + payload = json.loads(_RERANK_JSON.read_text(encoding="utf-8")) + for key in rerank_eval.REQUIRED_RERANK_EVAL_KEYS: + self.assertIn(key, payload) + self.assertEqual(payload["protocol_version"], rerank_eval.PROTOCOL_VERSION) + self.assertEqual(payload["adr"], "ADR-010") + self.assertEqual(payload["dataset_edition"], "100k") + self.assertEqual(payload["split_policy_id"], prep.SPLIT_POLICY_ID) + self.assertEqual(payload["seed"], 7) + self.assertEqual(payload["stage1"]["model"], "implicit_mf") + self.assertEqual(payload["stage1"]["stage1_n"], 200) + self.assertEqual(payload["stage2"]["model"], "numpy_mlp_listwise") + self.assertEqual(payload["stage2"]["feature_names"], list(rerank.FEATURE_NAMES)) + self.assertTrue(payload["stage2"]["genres_available"]) + self.assertEqual(payload["n_users"], 943) + self.assertGreaterEqual(payload["stage1_hit_rate_at_n"], 0.0) + self.assertLessEqual(payload["stage1_hit_rate_at_n"], 1.0) + for name in ("retrieval_only", "retrieval_plus_rerank"): + block = payload[name] + for key in ("recall_at_10", "ndcg_at_10", "map"): + self.assertGreaterEqual(block[key], 0.0) + self.assertLessEqual(block[key], 1.0) + delta = payload["delta_retrieval_plus_rerank_minus_retrieval_only"] + retrieval = payload["retrieval_only"] + reranked = payload["retrieval_plus_rerank"] + for key in ("recall_at_10", "ndcg_at_10", "map"): + self.assertAlmostEqual(delta[key], reranked[key] - retrieval[key]) + note = payload["note"].lower() + self.assertIn("not a graphsage", note) + self.assertIn("map@10", note) + rendered = rerank_eval.render_rerank_eval_markdown(payload) + self.assertEqual(_RERANK_MD.read_text(encoding="utf-8"), rendered) + self.assertIn("retrieval + re-rank", rendered) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/tests/test_serving_api.py b/python/tests/test_serving_api.py index 7058072..8abd8ee 100644 --- a/python/tests/test_serving_api.py +++ b/python/tests/test_serving_api.py @@ -143,13 +143,17 @@ def test_healthz_and_recommend_contract(tmp_path: Path) -> None: assert body["num_items"] == 4 assert body["embedding_dim"] == 2 assert body["schema"] == serving.SERVING_SCHEMA + assert body["rerank_loaded"] is False recommend = client.get("/recommend", params={"user_id": 0, "k": 2}) assert recommend.status_code == 200 payload = recommend.json() assert payload["user_id"] == 0 assert payload["k"] == 2 + assert payload["rerank"] is False + assert "stage1_n" not in payload assert [row["item_id"] for row in payload["recommendations"]] == [0, 1] + assert "retrieval_score" not in payload["recommendations"][0] assert [row["title"] for row in payload["recommendations"]] == ["Alpha", "Beta"] assert payload["recommendations"][0]["score"] == 1.0 assert payload["recommendations"][1]["score"] == 1.0 @@ -196,7 +200,8 @@ def test_http_surface_is_healthz_and_recommend(tmp_path: Path) -> None: def test_api_source_does_not_sample_or_download() -> None: api_source = (_PYTHON_DIR / "sagerec_api.py").read_text(encoding="utf-8") serving_source = (_PYTHON_DIR / "sagerec_serving.py").read_text(encoding="utf-8") - for source in (api_source, serving_source): + rerank_source = (_PYTHON_DIR / "sagerec_rerank.py").read_text(encoding="utf-8") + for source in (api_source, serving_source, rerank_source): assert "NeighborLoader" not in source assert "ClusterLoader" not in source assert "torch_geometric" not in source @@ -205,6 +210,8 @@ def test_api_source_does_not_sample_or_download() -> None: assert "import torch" not in source assert "import graph_sampler" not in source assert "recommend_topk" in api_source + assert "order_candidates" in api_source + assert "SAGEREC_RERANK_ARTIFACT" in api_source assert "numpy" in serving_source assert "FAISS is not required" in serving_source @@ -417,7 +424,8 @@ def test_committed_serving_latency_matches_markdown() -> None: for key in loadtest.REQUIRED_LATENCY_KEYS: assert key in payload assert payload["protocol_version"] == loadtest.PROTOCOL_VERSION - assert payload["adr"] == "ADR-009" + assert payload["adr"] == "ADR-010" + assert payload["retrieval_adr"] == "ADR-009" assert payload["endpoint"] == "GET /recommend" assert payload["n_errors"] == 0 assert payload["n_requests"] == loadtest.DEFAULT_REQUESTS @@ -428,10 +436,35 @@ def test_committed_serving_latency_matches_markdown() -> None: assert payload["workload"]["concurrency"] == loadtest.DEFAULT_CONCURRENCY assert payload["workload"]["warmup_requests"] == loadtest.DEFAULT_WARMUP assert payload["workload"]["k"] == loadtest.DEFAULT_K + assert payload["workload"]["stage1_n"] == loadtest.DEFAULT_STAGE1_N latency = payload["latency_ms"] assert latency["percentile_method"] == "linear" assert latency["min"] <= latency["p50"] <= latency["p95"] <= latency["p99"] <= latency["max"] assert payload["throughput_requests_per_s"] > 0 + assert payload["catalog"]["rerank_kind"] == "synthetic" + modes = payload["modes"] + assert list(modes) == list(loadtest.LATENCY_MODES) + retrieval = modes["retrieval_only"] + reranked = modes["retrieval_plus_rerank"] + assert retrieval["rerank"] is False + assert reranked["rerank"] is True + assert reranked["stage1_n"] == loadtest.DEFAULT_STAGE1_N + assert retrieval["n_errors"] == 0 + assert reranked["n_errors"] == 0 + assert retrieval["n_requests"] == loadtest.DEFAULT_REQUESTS + assert reranked["n_requests"] == loadtest.DEFAULT_REQUESTS + for mode in (retrieval, reranked): + mode_latency = mode["latency_ms"] + assert mode_latency["percentile_method"] == "linear" + assert ( + mode_latency["min"] + <= mode_latency["p50"] + <= mode_latency["p95"] + <= mode_latency["p99"] + <= mode_latency["max"] + ) + assert mode["throughput_requests_per_s"] > 0 + assert retrieval["latency_ms"] == payload["latency_ms"] assert isinstance(payload["cpu"]["model"], str) and payload["cpu"]["model"] assert isinstance(payload["cpu"]["logical_cpus"], int) assert payload["platform"]["system"] @@ -441,4 +474,114 @@ def test_committed_serving_latency_matches_markdown() -> None: rendered = loadtest.render_serving_latency_markdown(payload) assert _LATENCY_MD.read_text(encoding="utf-8") == rendered assert "p50" in rendered + assert "retrieval_only" in rendered + assert "retrieval_plus_rerank" in rendered assert payload["cpu"]["model"] in rendered + + +def _popularity_bundle(): + import sagerec_rerank as rerank + + tables = rerank.FeatureTables( + item_popularity=np.asarray([0.0, 0.0, 10.0, 0.0], dtype=np.float64), + item_recency=np.zeros(4, dtype=np.float64), + item_genres=np.zeros((4, 1), dtype=np.float64), + user_genre_profile=np.zeros((2, 1), dtype=np.float64), + genres_available=False, + ) + return rerank.zero_linear_bundle( + tables, + stage1_model="fixture", + popularity_weight=1.0, + ) + + +def _client_with_rerank(tmp_path: Path): + from fastapi.testclient import TestClient + + import sagerec_api + import sagerec_rerank as rerank + + artifact = tmp_path / "embeddings.npz" + rerank_path = tmp_path / "rerank.npz" + serving.save_serving_artifact(artifact, _fixture_index()) + rerank.save_rerank_artifact(rerank_path, _popularity_bundle()) + return TestClient(sagerec_api.create_app(artifact, rerank_path)) + + +def test_rerank_false_keeps_retrieval_order_when_a_model_is_loaded(tmp_path: Path) -> None: + client = _client_with_rerank(tmp_path) + health = client.get("/healthz") + assert health.status_code == 200 + assert health.json()["rerank_loaded"] is True + response = client.get("/recommend", params={"user_id": 0, "k": 2, "rerank": "false"}) + assert response.status_code == 200 + body = response.json() + assert body["rerank"] is False + assert [row["item_id"] for row in body["recommendations"]] == [0, 1] + assert "retrieval_score" not in body["recommendations"][0] + ignored_n = client.get( + "/recommend", + params={"user_id": 0, "k": 2, "rerank": "false", "n": 1}, + ) + assert ignored_n.status_code == 200 + assert [row["item_id"] for row in ignored_n.json()["recommendations"]] == [0, 1] + + +def test_rerank_true_reorders_the_stage1_pool_and_keeps_seen_items_out(tmp_path: Path) -> None: + client = _client_with_rerank(tmp_path) + response = client.get("/recommend", params={"user_id": 0, "k": 2, "rerank": "true", "n": 3}) + assert response.status_code == 200 + body = response.json() + assert body["rerank"] is True + assert body["stage1_n"] == 3 + rows = body["recommendations"] + assert [row["item_id"] for row in rows] == [2, 0] + assert rows[0]["score"] == 10.0 + assert rows[0]["retrieval_score"] == 0.5 + assert rows[1]["retrieval_score"] == 1.0 + assert all(row["item_id"] != 3 for row in rows) + + narrow = client.get("/recommend", params={"user_id": 0, "k": 2, "rerank": "true", "n": 2}) + assert narrow.status_code == 200 + assert [row["item_id"] for row in narrow.json()["recommendations"]] == [0, 1] + + +def test_rerank_true_without_a_model_is_422_and_unknown_user_stays_404(tmp_path: Path) -> None: + client = _client_for(_fixture_index(), tmp_path) + missing = client.get("/recommend", params={"user_id": 0, "k": 1, "rerank": "true"}) + assert missing.status_code == 422 + detail = missing.json()["detail"] + assert isinstance(detail, str) + assert "sagerec_rerank_artifact" in detail + assert "SAGEREC_RERANK_ARTIFACT" in detail + + loaded = _client_with_rerank(tmp_path) + unknown = loaded.get("/recommend", params={"user_id": 99, "k": 1, "rerank": "true"}) + assert unknown.status_code == 404 + assert "unknown user_id 99" in unknown.json()["detail"] + too_small = loaded.get("/recommend", params={"user_id": 0, "k": 2, "rerank": "true", "n": 1}) + assert too_small.status_code == 422 + assert "n must be >= k" in too_small.json()["detail"] + bad_flag = loaded.get("/recommend", params={"user_id": 0, "k": 1, "rerank": "maybe"}) + assert bad_flag.status_code == 422 + + +def test_mismatched_rerank_artifact_fails_at_startup(tmp_path: Path) -> None: + import sagerec_api + import sagerec_rerank as rerank + + artifact = tmp_path / "embeddings.npz" + rerank_path = tmp_path / "rerank.npz" + serving.save_serving_artifact(artifact, _fixture_index()) + bundle = _popularity_bundle() + bundle.metadata["stage1_model"] = "other_model" + rerank.save_rerank_artifact(rerank_path, bundle) + try: + sagerec_api.create_app(artifact, rerank_path) + except rerank.RerankError as exc: + message = str(exc) + else: + raise AssertionError("expected RerankError") + assert "other_model" in message + assert "fixture" in message diff --git a/results/AGENTS.md b/results/AGENTS.md index d8284b3..8f6c979 100644 --- a/results/AGENTS.md +++ b/results/AGENTS.md @@ -24,7 +24,12 @@ dependency versions, hardware/device, timestamp, and protocol version. - Keep checkpoints and bulky per-run logs in ignored subdirectories (`results/checkpoints/`). Phase 8 demo-CLI tests write temp-dir checkpoints only; do not commit weights or retcon 100K quality JSON. -- Phase 9 serving latency (`serving_latency.json` / `.md`) is written - by `sagerec_loadtest` from a real HTTP run. Do not hand-edit those +- Phase 9/10 serving latency (`serving_latency.json` / `.md`) is written + by `sagerec_loadtest` from a real HTTP run (both modes under + `serving-latency-v2`). Do not hand-edit those numbers. They are one-machine request latency, not ranking quality. Serving `.npz` artifacts stay under ignored `results/serving/`. +- Phase 10 re-rank quality (`rerank_eval.json` / `.md`) is written by + `sagerec_rerank_eval` from a real MovieLens 100K run. Do not + hand-edit those numbers. They are single-seed implicit-MF retrieval + plus a NumPy MLP, not GraphSAGE and not the ADR-007 leaderboard. diff --git a/results/README.md b/results/README.md index 04859b2..491eaf8 100644 --- a/results/README.md +++ b/results/README.md @@ -23,11 +23,21 @@ Machine-readable run summaries belong here. Optional live MovieLens 100K timing is a script flag and is not the committed default. -- `serving_latency.json` and `.md` are the Phase 9 ADR-009 HTTP load - test (`GET /recommend` p50/p95/p99 and throughput). Values are +- `serving_latency.json` and `.md` are the ADR-010 HTTP load + test (`GET /recommend` p50/p95/p99 and throughput for retrieval-only + and retrieval-plus-re-rank; protocol `serving-latency-v2`). Values are measured on one machine. The committed catalog is synthetic noise at - MovieLens 100K user/item counts, not a trained model and not a - ranking-quality result. Do not hand-edit them. + MovieLens 100K user/item counts, and the timed re-ranker is a seeded + synthetic MLP, not a trained MovieLens model and not a + ranking-quality result. Top-level `latency_ms` duplicates + retrieval-only. Do not hand-edit them. + +- `rerank_eval.json` and `.md` are the Phase 10 ADR-010 MovieLens 100K + comparison of retrieval-only versus retrieval plus re-rank (Recall@10, + NDCG@10, MAP@10). Stage 1 is single-seed implicit MF. Stage 2 is the + NumPy MLP fit on validation labels inside top-N. This is not a + GraphSAGE result and does not replace Phase 5 or the ADR-007 + leaderboard. Do not hand-edit them. Do not invent placeholder metrics or hand-edit timings. Checkpoints and bulky logs stay ignored. Serving embedding artifacts under diff --git a/results/rerank_eval.json b/results/rerank_eval.json new file mode 100644 index 0000000..b20517a --- /dev/null +++ b/results/rerank_eval.json @@ -0,0 +1,149 @@ +{ + "benchmark": "sagerec_two_stage_rerank", + "protocol_version": "rerank-eval-v1", + "adr": "ADR-010", + "dataset_edition": "100k", + "source_url": "https://files.grouplens.org/datasets/movielens/ml-100k.zip", + "checksum": "md5:0e33842e24a9c977be4e0107933c0723", + "license": "GroupLens MovieLens research license; see ml-100k-README.txt in the archive", + "split_policy_id": "per_user_chronological_leave_one_out", + "split_policy_version": "adr-003-v1", + "seed": 7, + "stage1": { + "model": "implicit_mf", + "score_formula": "dot(user_embedding, item_embedding) + user_bias + item_bias", + "tie_break": "score descending, then movie_id ascending", + "hyperparams": { + "n_factors": 16, + "n_epochs": 1, + "learning_rate": 0.05, + "n_negatives": 2, + "l2": 0.01, + "init_std": 0.1 + }, + "stage1_n": 200, + "candidate_filter": "test eval excludes train and validation positives; the test positive stays in the pool" + }, + "stage2": { + "model": "numpy_mlp_listwise", + "score_formula": "relu(features @ w1 + b1) @ w2 + b2 + features @ w_linear; features are standardized with the training mean and std", + "tie_break": "score descending, then movie_id ascending", + "hyperparams": { + "stage1_n": 200, + "hidden_dim": 8, + "n_epochs": 30, + "learning_rate": 0.05, + "l2": 0.0001, + "seed": 7, + "k": 10, + "init_std": 0.1 + }, + "feature_names": [ + "retrieval_score", + "item_popularity", + "genre_affinity", + "item_recency" + ], + "genres_available": true, + "genre_names": [ + "unknown", + "Action", + "Adventure", + "Animation", + "Children's", + "Comedy", + "Crime", + "Documentary", + "Drama", + "Fantasy", + "Film-Noir", + "Horror", + "Musical", + "Mystery", + "Romance", + "Sci-Fi", + "Thriller", + "War", + "Western" + ], + "uitem_origin": "data/raw/ml-100k.zip#ml-100k/u.item", + "label_split": "validation", + "eval_split": "test", + "training_pool_excludes": [ + "train", + "test" + ] + }, + "protocol": { + "k": 10, + "stage1_n": 200, + "metrics": [ + "recall_at_10", + "ndcg_at_10", + "map" + ], + "map": "MAP@10. One relevant item per eligible user. AP@10 is 1/rank when the held-out item's 1-based rank is <= 10, else 0. MAP is the unweighted mean over eligible users. Both systems rank inside the same stage-1 top-N pool, so retrieval-only top-10 matches full-catalog top-10 when N >= 10 and the tie-break matches.", + "eligible_users": "ADR-003 users with at least 3 interactions", + "same_pool": true + }, + "n_users": 943, + "stage1_hit_rate_at_n": 0.35737009544008486, + "retrieval_only": { + "recall_at_10": 0.03817603393425239, + "ndcg_at_10": 0.020303175177886473, + "map": 0.01495017589927452 + }, + "retrieval_plus_rerank": { + "recall_at_10": 0.053022269353128315, + "ndcg_at_10": 0.026510184267505867, + "map": 0.018499722264303388 + }, + "delta_retrieval_plus_rerank_minus_retrieval_only": { + "recall_at_10": 0.014846235418875926, + "ndcg_at_10": 0.006207009089619395, + "map": 0.003549546365028867 + }, + "rerank_train": { + "n_eligible_users": 943, + "n_train_users": 442, + "validation_hit_rate_at_n": 0.4687168610816543, + "loss_first_epoch": 5.18127992724679, + "loss_final_epoch": 5.172139239139217, + "n_epochs": 30, + "label_split": "validation", + "excluded_from_training_pools": [ + "train", + "test" + ] + }, + "eligibility": { + "users": 943, + "movies": 1682, + "interactions": { + "total": 100000, + "train": 98114, + "validation": 943, + "test": 943 + }, + "eligible_users": 943, + "cold_start_users": 0, + "evaluated_users": 943 + }, + "cpu": { + "model": "Intel(R) Xeon(R) Processor", + "logical_cpus": 4, + "machine": "x86_64" + }, + "platform": { + "system": "Linux", + "release": "6.12.94+", + "machine": "x86_64", + "processor": "x86_64" + }, + "python": "3.12.3", + "numpy": "1.26.4", + "git_commit": "03c7891040c9972abb6621845f9803544642ef5a", + "git_dirty": true, + "generated_at": "2026-10-23T23:31:31.336612+00:00", + "note": "Single-seed MovieLens 100K two-stage comparison (ADR-010). Stage 1 is implicit MF (ADR-004) dot-product retrieval, the same score ADR-009 serves, trained on ADR-003 train positives only. Stage 2 is a NumPy MLP with a linear skip, fit listwise on validation items that stage 1 retrieved, and scored on the test split. Both systems rank inside that stage-1 top-N pool. MAP is MAP@10. Features use train interactions only. Genres come from u.item. This is not a GraphSAGE result, not the Phase 5 or Phase 7 leaderboard, and not a SOTA claim. Re-ranking cannot recover a test item that stage 1 did not retrieve. git_dirty was true at generation, so git_commit is the HEAD the process saw and may not contain this results file." +} diff --git a/results/rerank_eval.md b/results/rerank_eval.md new file mode 100644 index 0000000..b9cea40 --- /dev/null +++ b/results/rerank_eval.md @@ -0,0 +1,17 @@ +# SageRec two-stage re-rank evaluation + +MovieLens 100K, ADR-003 leave-one-out, test split. Stage 1 is implicit MF retrieval. Stage 2 re-ranks that top-N pool. Not a GraphSAGE result and not the Phase 5 or Phase 7 leaderboard. + +- Protocol: `rerank-eval-v1` (ADR-010) +- Seed: 7; stage-1 N: 200; users: 943; stage-1 hit rate @N: 0.357370 +- Features: retrieval_score, item_popularity, genre_affinity, item_recency; genres_available=True +- Python: 3.12.3; NumPy: 1.26.4; CPU: Intel(R) Xeon(R) Processor (4 logical) +- Commit: `03c7891040c9972abb6621845f9803544642ef5a` (dirty=True) + +| System | Recall@10 | NDCG@10 | MAP@10 | +| --- | ---: | ---: | ---: | +| retrieval only | 0.038176 | 0.020303 | 0.014950 | +| retrieval + re-rank | 0.053022 | 0.026510 | 0.018500 | +| delta (re-rank − retrieval) | 0.014846 | 0.006207 | 0.003550 | + +Single-seed MovieLens 100K two-stage comparison (ADR-010). Stage 1 is implicit MF (ADR-004) dot-product retrieval, the same score ADR-009 serves, trained on ADR-003 train positives only. Stage 2 is a NumPy MLP with a linear skip, fit listwise on validation items that stage 1 retrieved, and scored on the test split. Both systems rank inside that stage-1 top-N pool. MAP is MAP@10. Features use train interactions only. Genres come from u.item. This is not a GraphSAGE result, not the Phase 5 or Phase 7 leaderboard, and not a SOTA claim. Re-ranking cannot recover a test item that stage 1 did not retrieve. git_dirty was true at generation, so git_commit is the HEAD the process saw and may not contain this results file. diff --git a/results/serving_latency.json b/results/serving_latency.json index 0914dc4..fbcc7f3 100644 --- a/results/serving_latency.json +++ b/results/serving_latency.json @@ -1,7 +1,8 @@ { "benchmark": "sagerec_http_recommend", - "protocol_version": "serving-latency-v1", - "adr": "ADR-009", + "protocol_version": "serving-latency-v2", + "adr": "ADR-010", + "retrieval_adr": "ADR-009", "endpoint": "GET /recommend", "catalog": { "kind": "synthetic", @@ -10,31 +11,72 @@ "embedding_dim": 16, "seen_per_user": 20, "seed": 7, - "model": "synthetic" + "model": "synthetic", + "rerank_kind": "synthetic" }, "workload": { "k": 10, + "stage1_n": 200, "n_requests": 1000, "warmup_requests": 100, "concurrency": 16, "request_seed": 7, "percentile_method": "linear", "server": "uvicorn", - "client": "httpx.AsyncClient" + "client": "httpx.AsyncClient", + "modes": [ + "retrieval_only", + "retrieval_plus_rerank" + ] }, "latency_ms": { - "p50": 19.909990499968444, - "p95": 59.1005087500548, - "p99": 85.91370133994928, - "mean": 24.707682000997465, - "min": 6.654086000025927, - "max": 127.66264699996555, + "p50": 23.443566500077395, + "p95": 66.87042594986677, + "p99": 93.59026779989787, + "mean": 28.48181201199827, + "min": 7.338863000086349, + "max": 133.45369600006052, "percentile_method": "linear" }, - "throughput_requests_per_s": 617.9517943900427, - "wall_seconds": 1.6182492049999837, + "throughput_requests_per_s": 534.0127735919701, + "wall_seconds": 1.872614381999938, "n_requests": 1000, "n_errors": 0, + "modes": { + "retrieval_only": { + "rerank": false, + "latency_ms": { + "p50": 23.443566500077395, + "p95": 66.87042594986677, + "p99": 93.59026779989787, + "mean": 28.48181201199827, + "min": 7.338863000086349, + "max": 133.45369600006052, + "percentile_method": "linear" + }, + "throughput_requests_per_s": 534.0127735919701, + "wall_seconds": 1.872614381999938, + "n_requests": 1000, + "n_errors": 0 + }, + "retrieval_plus_rerank": { + "rerank": true, + "latency_ms": { + "p50": 21.012949500004652, + "p95": 66.63823284982296, + "p99": 94.78142839999234, + "mean": 27.05330871199658, + "min": 8.055002999981298, + "max": 230.55966900005842, + "percentile_method": "linear" + }, + "throughput_requests_per_s": 568.753940373489, + "wall_seconds": 1.7582295770000655, + "n_requests": 1000, + "n_errors": 0, + "stage1_n": 200 + } + }, "cpu": { "model": "Intel(R) Xeon(R) Processor", "logical_cpus": 4, @@ -53,8 +95,8 @@ "uvicorn": "0.54.0", "httpx": "0.28.1" }, - "git_commit": "70a565104f0aa00cfe85923487d3a5813f13fefb", + "git_commit": "03c7891040c9972abb6621845f9803544642ef5a", "git_dirty": true, - "generated_at": "2026-10-21T21:00:46.825913+00:00", - "note": "One-machine ADR-009 HTTP latency for GET /recommend on a local uvicorn process. Top-K is exact NumPy over precomputed embeddings. Not a ranking-quality result, not sampler timing, and not a production SLO. The catalog is seeded synthetic noise with MovieLens 100K user/item counts; no ratings were read. git_dirty was true at generation, so git_commit is the HEAD the process saw and may not contain this results file." + "generated_at": "2026-10-23T23:29:04.698787+00:00", + "note": "One-machine HTTP latency for GET /recommend on a local uvicorn process, measured twice: retrieval-only (ADR-009 NumPy top-K) and retrieval plus re-rank (ADR-010 MLP over the stage-1 pool). Top-level latency_ms duplicates retrieval_only. Not a ranking-quality result, not sampler timing, and not a production SLO. The catalog is seeded synthetic noise with MovieLens 100K user/item counts; no ratings were read. The re-ranker is a seeded synthetic MLP over synthetic popularity, genre, and recency tables, not the MovieLens-trained model. git_dirty was true at generation, so git_commit is the HEAD the process saw and may not contain this results file." } diff --git a/results/serving_latency.md b/results/serving_latency.md index ffb8c79..156cf7a 100644 --- a/results/serving_latency.md +++ b/results/serving_latency.md @@ -1,22 +1,17 @@ # SageRec HTTP recommend latency -One-machine ADR-009 measurement of `GET /recommend` against a local uvicorn process. Top-K is exact NumPy. Not a ranking-quality result and not a production SLO. +One-machine measurement of `GET /recommend` against a local uvicorn process. Retrieval-only is ADR-009 NumPy top-K. Retrieval plus re-rank is the ADR-010 MLP over the stage-1 pool. Not a ranking-quality result and not a production SLO. Top-level numbers in the JSON duplicate retrieval-only. -- Protocol: `serving-latency-v1` (ADR-009) +- Protocol: `serving-latency-v2` (ADR-010) - Catalog: `synthetic` with 943 users, 1682 items, dim 16, seed 7 -- Workload: 1000 measured requests, 100 warm-up, concurrency 16, k=10 +- Workload: 1000 measured requests per mode, 100 warm-up, concurrency 16, k=10, stage1_n=200 - Percentile method: `linear` - Python: 3.12.3; CPU: Intel(R) Xeon(R) Processor (4 logical); Linux 6.12.94+ -- Commit: `70a565104f0aa00cfe85923487d3a5813f13fefb` (dirty=True) +- Commit: `03c7891040c9972abb6621845f9803544642ef5a` (dirty=True) -| Statistic | Value | -| --- | ---: | -| p50 ms | 19.909990 | -| p95 ms | 59.100509 | -| p99 ms | 85.913701 | -| mean ms | 24.707682 | -| throughput requests/s | 617.951794 | -| wall seconds | 1.618249 | -| errors | 0 | +| Mode | p50 ms | p95 ms | p99 ms | requests/s | +| --- | ---: | ---: | ---: | ---: | +| retrieval_only | 23.443567 | 66.870426 | 93.590268 | 534.012774 | +| retrieval_plus_rerank | 21.012950 | 66.638233 | 94.781428 | 568.753940 | -One-machine ADR-009 HTTP latency for GET /recommend on a local uvicorn process. Top-K is exact NumPy over precomputed embeddings. Not a ranking-quality result, not sampler timing, and not a production SLO. The catalog is seeded synthetic noise with MovieLens 100K user/item counts; no ratings were read. git_dirty was true at generation, so git_commit is the HEAD the process saw and may not contain this results file. +One-machine HTTP latency for GET /recommend on a local uvicorn process, measured twice: retrieval-only (ADR-009 NumPy top-K) and retrieval plus re-rank (ADR-010 MLP over the stage-1 pool). Top-level latency_ms duplicates retrieval_only. Not a ranking-quality result, not sampler timing, and not a production SLO. The catalog is seeded synthetic noise with MovieLens 100K user/item counts; no ratings were read. The re-ranker is a seeded synthetic MLP over synthetic popularity, genre, and recency tables, not the MovieLens-trained model. git_dirty was true at generation, so git_commit is the HEAD the process saw and may not contain this results file. diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index f89211d..8ab13a5 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -63,9 +63,22 @@ Opened Phase 9 HTTP serving (ADR-009): or bind a port. - `serve_api.py` — serve `GET /recommend` from that artifact. Delegates to uvicorn and `python/sagerec_api.py`. -- `loadtest_serving.py` — measure p50/p95/p99 and throughput and write - `results/serving_latency.json` and `.md`. Default catalog is - synthetic. Does not download MovieLens. Do not hand-edit the output. +- `loadtest_serving.py` — measure p50/p95/p99 and throughput for + retrieval-only and retrieval-plus-re-rank, and write + `results/serving_latency.json` and `.md` (`serving-latency-v2`). + Default catalog and re-ranker are synthetic. Does not download + MovieLens. Do not hand-edit the output. + +Opened Phase 10 two-stage re-ranking (ADR-010): + +- `run_rerank_eval.py` — train implicit MF on MovieLens 100K, fit the + NumPy MLP on validation labels inside the stage-1 pool, and write + `results/rerank_eval.json` and `.md`. Requires `u.item` (extracted + from the official zip when it is already on disk). Does not download + on its own beyond `ensure_movielens_100k_processed`, does not start + HTTP, and does not overwrite Phase 5 or Phase 7 files. Optional + `--rerank-out` writes a gitignored artifact under `results/serving/`. + Not a default CI job. Do not add MovieLens 1M scripts. The Phase 4 mini-batch helper lives in `python/sagerec_minibatch.py`. GraphSAGE training lives in @@ -74,4 +87,6 @@ neighborhoods). Sampler timing logic lives in `python/sagerec_sampler_benchmark.py`. Demo serving lives in `python/sagerec_serve.py`. HTTP serving lives in `python/sagerec_api.py` and `python/sagerec_serving.py`. Embedding export lives in -`python/sagerec_export.py`. +`python/sagerec_export.py`. Two-stage features and the MLP live in +`python/sagerec_rerank.py`. The MovieLens comparison lives in +`python/sagerec_rerank_eval.py`. diff --git a/scripts/loadtest_serving.py b/scripts/loadtest_serving.py index 55da172..466a307 100644 --- a/scripts/loadtest_serving.py +++ b/scripts/loadtest_serving.py @@ -1,5 +1,8 @@ #!/usr/bin/env python3 -"""Thin CLI: measure GET /recommend latency and write results/ (ADR-009).""" +"""Thin CLI: measure GET /recommend latency and write results/ (ADR-010). + +Records retrieval-only and retrieval-plus-re-rank (protocol serving-latency-v2). +""" from __future__ import annotations diff --git a/scripts/run_rerank_eval.py b/scripts/run_rerank_eval.py new file mode 100644 index 0000000..af5cf75 --- /dev/null +++ b/scripts/run_rerank_eval.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Thin CLI: MovieLens 100K retrieval vs retrieval + re-rank (ADR-010).""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[1] +_PYTHON_DIR = _REPO_ROOT / "python" +if str(_PYTHON_DIR) not in sys.path: + sys.path.insert(0, str(_PYTHON_DIR)) + +import sagerec_rerank_eval as rerank_eval + + +def main() -> int: + return rerank_eval.main() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/serve_api.py b/scripts/serve_api.py index 121426e..c870ab7 100644 --- a/scripts/serve_api.py +++ b/scripts/serve_api.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Thin CLI: serve GET /recommend from a precomputed artifact (ADR-009).""" +"""Thin CLI: serve GET /recommend from a precomputed artifact (ADR-009/010).""" from __future__ import annotations @@ -21,6 +21,15 @@ def main(argv: list[str] | None = None) -> int: ) ) parser.add_argument("--artifact", required=True, type=Path) + parser.add_argument( + "--rerank", + type=Path, + default=None, + help=( + "Optional sagerec_rerank_artifact .npz. Requests stay retrieval-only " + "until the client passes rerank=true." + ), + ) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=8000) try: @@ -36,6 +45,15 @@ def main(argv: list[str] | None = None) -> int: ) return 1 os.environ["SAGEREC_SERVING_ARTIFACT"] = str(args.artifact.resolve()) + if args.rerank is not None: + if not args.rerank.is_file(): + print( + f"error: re-rank artifact {args.rerank} does not exist. " + "Train one with scripts/run_rerank_eval.py --rerank-out.", + file=sys.stderr, + ) + return 1 + os.environ["SAGEREC_RERANK_ARTIFACT"] = str(args.rerank.resolve()) try: import uvicorn except ImportError: