From 4f3b2c1c8ce03282515dc4428959b2f15be98eba Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 00:01:50 -0400 Subject: [PATCH 01/26] feat(bench): benchmark and stress harness against real MongoDB An unpublished workspace package answering performance questions with data from real MongoDB (replica set and sharded cluster). - mongospecs-bench run: workloads B1-B8 (materialise, find, include strategies, writes, pagination, plugin overhead, HTTP per plug, cost of change tracking) - mongospecs-bench stress: scenarios X1-X7 (pool pressure, transaction contention, optimistic-lock storm, cancellation, 16MB results, change stream across a step-down, threads with own loops) - mongospecs-bench trace: Sentry SDK through mongospecs-web into a local rustrak server, asserting one trace per request with Model.op database spans - results are JSON under bench/results// with server, topology and machine metadata; --quick only proves the harness runs - smoke tests keep every workload and scenario runnable Co-Authored-By: Claude Opus 5.5 --- bench/pyproject.toml | 2 +- bench/src/mongospecs_bench/cli.py | 110 ++++ bench/src/mongospecs_bench/harness.py | 217 ++++++++ bench/src/mongospecs_bench/models.py | 177 +++++++ bench/src/mongospecs_bench/stress.py | 360 +++++++++++++ bench/src/mongospecs_bench/tracing.py | 204 ++++++++ bench/src/mongospecs_bench/workloads.py | 643 ++++++++++++++++++++++++ bench/tests/test_bench_smoke.py | 44 ++ docker/compose.rustrak.yaml | 19 + pyproject.toml | 2 + uv.lock | 4 +- 11 files changed, 1779 insertions(+), 3 deletions(-) create mode 100644 bench/src/mongospecs_bench/cli.py create mode 100644 bench/src/mongospecs_bench/harness.py create mode 100644 bench/src/mongospecs_bench/models.py create mode 100644 bench/src/mongospecs_bench/stress.py create mode 100644 bench/src/mongospecs_bench/tracing.py create mode 100644 bench/src/mongospecs_bench/workloads.py create mode 100644 bench/tests/test_bench_smoke.py create mode 100644 docker/compose.rustrak.yaml diff --git a/bench/pyproject.toml b/bench/pyproject.toml index 2a7bb90..28e1295 100644 --- a/bench/pyproject.toml +++ b/bench/pyproject.toml @@ -8,7 +8,7 @@ version = "0.0.0" description = "Benchmarks and stress scenarios for mongospecs against real MongoDB. Not published." requires-python = ">=3.11" classifiers = ["Private :: Do Not Upload"] -dependencies = ["mongospecs[all]", "mongospecs-web[all]", "httpx>=0.27"] +dependencies = ["mongospecs[all,sentry]", "mongospecs-web[all]", "httpx>=0.27"] [project.scripts] mongospecs-bench = "mongospecs_bench.cli:main" diff --git a/bench/src/mongospecs_bench/cli.py b/bench/src/mongospecs_bench/cli.py new file mode 100644 index 0000000..1b47e40 --- /dev/null +++ b/bench/src/mongospecs_bench/cli.py @@ -0,0 +1,110 @@ +"""``mongospecs-bench``: run benchmarks, stress scenarios and the tracing check. + +:: + + mongospecs-bench run B1 B2 # workloads (default: all) + mongospecs-bench run B3 --topology sharded + mongospecs-bench stress # X1-X7 + mongospecs-bench trace # Sentry SDK -> rustrak, end to end + mongospecs-bench report # markdown summary of the newest results + +Add ``--quick`` to check that everything runs; quick numbers are not for conclusions. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import pathlib +import sys +import typing as t + +from mongospecs_bench.harness import RESULTS, Result, Settings, metadata, table, write_result + + +async def _run(names: list[str], registry: dict[str, t.Any], settings: Settings) -> int: + meta = await metadata(settings) + failed = 0 + for name in names: + fn = registry[name] + print(f"\n== {name} ({settings.topology}{', quick' if settings.quick else ''})", flush=True) + result: Result = await fn(settings) + path = write_result(result, meta) + print(f"{result.title}\n{result.question}\n") + print(table(result.rows)) + for note in result.notes: + print(f"note: {note}") + if result.conclusion: + print(result.conclusion) + failed += result.conclusion.startswith("FAIL") + print(f"-> {path.relative_to(RESULTS.parent)}") + return 1 if failed else 0 + + +def report(folder: pathlib.Path | None = None) -> str: + """Markdown summary of one results folder (default: the most recently written).""" + if folder is None: + folders = sorted((p for p in RESULTS.iterdir() if p.is_dir()), key=lambda p: p.stat().st_mtime) + if not folders: + return "No results yet. Run `mongospecs-bench run` first." + folder = folders[-1] + lines = [f"# Benchmark results ({folder.name})", ""] + for path in sorted(folder.glob("*.json")): + data = json.loads(path.read_text()) + meta = data["meta"] + lines += [ + f"## {data['workload']}: {data['title']} ({meta['topology']}{', quick' if meta['quick'] else ''})", + "", + data["question"], + "", + f"MongoDB {meta['mongodb']}, Python {meta['python']}, {meta['machine']}, {meta['timestamp']}", + "", + ] + rows = data["rows"] + if rows: + columns = list(dict.fromkeys(key for row in rows for key in row)) + lines.append("| " + " | ".join(columns) + " |") + lines.append("|" + "---|" * len(columns)) + lines.extend("| " + " | ".join(str(row.get(c, "")) for c in columns) + " |" for row in rows) + lines.append("") + lines.extend(f"- {note}" for note in data["notes"]) + if data["conclusion"]: + lines += ["", f"**{data['conclusion']}**"] + lines.append("") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + from mongospecs_bench.stress import SCENARIOS + from mongospecs_bench.workloads import WORKLOADS + + parser = argparse.ArgumentParser( + prog="mongospecs-bench", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + sub = parser.add_subparsers(dest="command", required=True) + for name, choices in (("run", WORKLOADS), ("stress", SCENARIOS)): + cmd = sub.add_parser(name) + cmd.add_argument("names", nargs="*", choices=[*choices, []], default=[]) + cmd.add_argument("--topology", choices=("rs", "sharded"), default="rs") + cmd.add_argument("--quick", action="store_true") + cmd.add_argument("--backend", action="append", choices=("msgspec", "pydantic", "attrs")) + sub.add_parser("trace") + rep = sub.add_parser("report") + rep.add_argument("folder", nargs="?") + args = parser.parse_args(argv) + + if args.command == "trace": + from mongospecs_bench.tracing import main as trace_main + + return trace_main() + if args.command == "report": + print(report(RESULTS / args.folder if args.folder else None)) + return 0 + registry = WORKLOADS if args.command == "run" else SCENARIOS + settings = Settings(args.topology, args.quick, tuple(args.backend or ("msgspec", "pydantic", "attrs"))) + return asyncio.run(_run(list(args.names) or list(registry), registry, settings)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/src/mongospecs_bench/harness.py b/bench/src/mongospecs_bench/harness.py new file mode 100644 index 0000000..99d9643 --- /dev/null +++ b/bench/src/mongospecs_bench/harness.py @@ -0,0 +1,217 @@ +"""Timing, environment and result-file plumbing shared by workloads and stress scenarios.""" + +from __future__ import annotations + +import asyncio +import datetime as dt +import gc +import json +import os +import pathlib +import platform +import statistics +import subprocess +import sys +import time +import typing as t +import uuid +from contextlib import asynccontextmanager +from dataclasses import dataclass, field + +from mongospecs import Registry + +RS_URI = os.environ.get("MONGOSPECS_TEST_URI", "mongodb://localhost:27117/?replicaSet=rs0") +SHARDED_URI = os.environ.get("MONGOSPECS_TEST_SHARDED_URI", "mongodb://localhost:27118") +RESULTS = pathlib.Path(__file__).resolve().parents[2] / "results" + + +def uri_for(topology: str) -> str: + return SHARDED_URI if topology == "sharded" else RS_URI + + +@dataclass +class Timing: + samples: list[float] + """Seconds per call.""" + + @property + def median_ms(self) -> float: + return statistics.median(self.samples) * 1000 + + @property + def min_ms(self) -> float: + return min(self.samples) * 1000 + + @property + def p95_ms(self) -> float: + ordered = sorted(self.samples) + return ordered[min(len(ordered) - 1, round(0.95 * (len(ordered) - 1)))] * 1000 + + def as_dict(self) -> dict[str, float]: + return { + "median_ms": round(self.median_ms, 4), + "min_ms": round(self.min_ms, 4), + "p95_ms": round(self.p95_ms, 4), + "runs": len(self.samples), + } + + +async def measure( + fn: t.Callable[[], t.Awaitable[t.Any]], + *, + repeat: int = 15, + warmup: int = 2, + setup: t.Callable[[], t.Awaitable[t.Any]] | None = None, +) -> Timing: + """Time an async callable. ``setup`` runs before each call and is not timed.""" + for _ in range(warmup): + if setup is not None: + await setup() + await fn() + samples: list[float] = [] + gc.collect() + for _ in range(repeat): + if setup is not None: + await setup() + start = time.perf_counter() + await fn() + samples.append(time.perf_counter() - start) + return Timing(samples) + + +def measure_sync(fn: t.Callable[[], t.Any], *, repeat: int = 15, warmup: int = 2) -> Timing: + for _ in range(warmup): + fn() + samples: list[float] = [] + gc.collect() + for _ in range(repeat): + start = time.perf_counter() + fn() + samples.append(time.perf_counter() - start) + return Timing(samples) + + +@dataclass +class Settings: + topology: str = "rs" + quick: bool = False + """Small sizes and few repetitions: for checking the harness, not for publishing numbers.""" + backends: tuple[str, ...] = ("msgspec", "pydantic", "attrs") + + @property + def repeat(self) -> int: + return 3 if self.quick else 15 + + def scale(self, full: int, quick: int) -> int: + return quick if self.quick else full + + +@dataclass +class Result: + workload: str + title: str + question: str + rows: list[dict[str, t.Any]] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + conclusion: str = "" + + def add(self, **row: t.Any) -> None: + self.rows.append(row) + + +@asynccontextmanager +async def environment( + settings: Settings, models: t.Iterable[type] = (), **registry_options: t.Any +) -> t.AsyncIterator[Registry]: + """An open registry on a fresh database of the chosen topology, dropped afterwards.""" + name = f"ms_bench_{uuid.uuid4().hex[:10]}" + async with Registry( + uri_for(settings.topology), database=name, models=models, **registry_options + ) as registry: + if settings.topology == "sharded": + await registry.client.admin.command("enableSharding", name) + try: + yield registry + finally: + await registry.client.drop_database(name) + + +def _git(*args: str) -> str: + try: + return subprocess.run( + ["git", *args], capture_output=True, text=True, check=True, cwd=RESULTS.parent + ).stdout.strip() + except Exception: + return "unknown" + + +async def metadata(settings: Settings) -> dict[str, t.Any]: + import msgspec + import pydantic + import pymongo + + import mongospecs + + async with Registry(uri_for(settings.topology), database="admin") as registry: + info = await registry.client.server_info() + return { + "git_sha": _git("rev-parse", "--short", "HEAD"), + "git_dirty": bool(_git("status", "--porcelain")), + "timestamp": dt.datetime.now(dt.UTC).isoformat(timespec="seconds"), + "topology": settings.topology, + "quick": settings.quick, + "mongodb": info.get("version"), + "python": sys.version.split()[0], + "free_threaded": not getattr(sys, "_is_gil_enabled", lambda: True)(), + "platform": platform.platform(), + "machine": platform.machine(), + "cpus": os.cpu_count(), + "versions": { + "mongospecs": mongospecs.__version__, + "pymongo": pymongo.version, + "msgspec": msgspec.__version__, + "pydantic": pydantic.VERSION, + }, + } + + +def write_result(result: Result, meta: dict[str, t.Any]) -> pathlib.Path: + """Write one result file under ``bench/results//``.""" + folder = RESULTS / str(meta["git_sha"]) + folder.mkdir(parents=True, exist_ok=True) + suffix = "" if meta["topology"] == "rs" else f"-{meta['topology']}" + path = folder / f"{result.workload}{suffix}{'-quick' if meta['quick'] else ''}.json" + payload = { + "meta": meta, + "workload": result.workload, + "title": result.title, + "question": result.question, + "rows": result.rows, + "notes": result.notes, + "conclusion": result.conclusion, + } + path.write_text(json.dumps(payload, indent=2, default=str) + "\n") + return path + + +def table(rows: list[dict[str, t.Any]]) -> str: + """Render rows as a fixed-width text table.""" + if not rows: + return "(no rows)" + columns = list(dict.fromkeys(key for row in rows for key in row)) + cells = [[_cell(row.get(c, "")) for c in columns] for row in rows] + widths = [max(len(c), *(len(r[i]) for r in cells)) for i, c in enumerate(columns)] + lines = [" ".join(c.ljust(w) for c, w in zip(columns, widths, strict=True))] + lines.append(" ".join("-" * w for w in widths)) + lines.extend(" ".join(v.ljust(w) for v, w in zip(row, widths, strict=True)) for row in cells) + return "\n".join(lines) + + +def _cell(value: t.Any) -> str: + if isinstance(value, float): + return f"{value:.3f}" if abs(value) < 100 else f"{value:.1f}" + return str(value) + + +def run(coro: t.Coroutine[t.Any, t.Any, t.Any]) -> t.Any: + return asyncio.run(coro) diff --git a/bench/src/mongospecs_bench/models.py b/bench/src/mongospecs_bench/models.py new file mode 100644 index 0000000..6620050 --- /dev/null +++ b/bench/src/mongospecs_bench/models.py @@ -0,0 +1,177 @@ +"""Benchmark models, declared once per backend at module level. + +No ``from __future__ import annotations``: the classes are built by a factory. +""" + +import datetime as dt +import types +import typing as t + +import attrs +import msgspec +import pydantic + +from mongospecs import CreatedAt, Index, Ref, Revision, SpecConfig, UpdatedAt +from mongospecs import attrs as at +from mongospecs import msgspec as ms +from mongospecs import pydantic as pd + +BACKENDS = ("msgspec", "pydantic", "attrs") +SHAPES = ("small", "wide", "nested") +WIDE_FIELDS = 40 + + +def _build(backend: str) -> types.SimpleNamespace: + if backend == "msgspec": + Spec: t.Any = ms.Spec + define: t.Any = lambda cls: cls # noqa: E731 + + class Address(msgspec.Struct, kw_only=True): + street: str = "" + city: str = "" + zip: str = "" + + class Line(msgspec.Struct, kw_only=True): + sku: str = "" + qty: int = 0 + price: float = 0.0 + + elif backend == "pydantic": + Spec = pd.Spec + define = lambda cls: cls # noqa: E731 + + class Address(pydantic.BaseModel): # type: ignore[no-redef] + street: str = "" + city: str = "" + zip: str = "" + + class Line(pydantic.BaseModel): # type: ignore[no-redef] + sku: str = "" + qty: int = 0 + price: float = 0.0 + + else: + Spec = at.Spec + define = attrs.define(kw_only=True) + + @attrs.define(kw_only=True) + class Address: # type: ignore[no-redef] + street: str = "" + city: str = "" + zip: str = "" + + @attrs.define(kw_only=True) + class Line: # type: ignore[no-redef] + sku: str = "" + qty: int = 0 + price: float = 0.0 + + @define + class Small(Spec): # type: ignore[misc] + __spec__ = SpecConfig(collection="bench_small", indexes=[Index("n")]) + n: int = 0 + name: str = "" + score: float = 0.0 + active: bool = True + created: dt.datetime | None = None + + # 40 scalar fields, built so the three backends get identical definitions. + wide_annotations: dict[str, t.Any] = {} + wide_defaults: dict[str, t.Any] = {} + for i in range(WIDE_FIELDS): + kind = i % 4 + wide_annotations[f"f{i:02d}"] = (int, str, float, bool)[kind] + wide_defaults[f"f{i:02d}"] = (0, "", 0.0, False)[kind] + wide_ns = { + "__annotations__": {"n": int, **wide_annotations}, + "__spec__": SpecConfig(collection="bench_wide", indexes=[Index("n")]), + "n": 0, + **wide_defaults, + } + Wide = define(types.new_class("Wide", (Spec,), {}, lambda ns: ns.update(wide_ns))) + + @define + class Nested(Spec): # type: ignore[misc] + __spec__ = SpecConfig(collection="bench_nested", indexes=[Index("n")]) + n: int = 0 + customer: str = "" + address: Address | None = None + lines: list[Line] = [] + tags: list[str] = [] + meta: dict[str, str] = {} + placed: dt.datetime | None = None + + @define + class Child(Spec): # type: ignore[misc] + __spec__ = SpecConfig(collection="bench_children") + n: int = 0 + label: str = "" + payload: str = "" + + @define + class Parent(Spec): # type: ignore[misc] + __spec__ = SpecConfig(collection="bench_parents", indexes=[Index("n")]) + n: int = 0 + name: str = "" + owner: Ref[Child] | None = None + children: list[Ref[Child]] = [] + + @define + class Managed(Spec): # type: ignore[misc] + """A model with the three built-in plugins active.""" + + __spec__ = SpecConfig(collection="bench_managed") + n: int = 0 + name: str = "" + created_at: t.Annotated[dt.datetime | None, CreatedAt] = None + updated_at: t.Annotated[dt.datetime | None, UpdatedAt] = None + rev: t.Annotated[int, Revision] = 0 + + @define + class Plain(Spec): # type: ignore[misc] + __spec__ = SpecConfig(collection="bench_plain") + n: int = 0 + name: str = "" + + return types.SimpleNamespace( + backend=backend, + Small=Small, + Wide=Wide, + Nested=Nested, + Child=Child, + Parent=Parent, + Managed=Managed, + Plain=Plain, + ALL=[Small, Wide, Nested, Child, Parent, Managed, Plain], + ) + + +MODELS: dict[str, types.SimpleNamespace] = {backend: _build(backend) for backend in BACKENDS} + + +def raw_document(shape: str, n: int) -> dict[str, t.Any]: + """A raw MongoDB document of the given shape (without ``_id``).""" + when = dt.datetime(2024, 1, 1, tzinfo=dt.UTC) + dt.timedelta(minutes=n) + if shape == "small": + return {"n": n, "name": f"name-{n}", "score": n * 1.5, "active": n % 2 == 0, "created": when} + if shape == "wide": + doc: dict[str, t.Any] = {"n": n} + for i in range(WIDE_FIELDS): + kind = i % 4 + doc[f"f{i:02d}"] = (n + i, f"value-{n}-{i}", (n + i) * 0.5, (n + i) % 2 == 0)[kind] + return doc + if shape == "nested": + return { + "n": n, + "customer": f"customer-{n}", + "address": {"street": f"{n} Main St", "city": "Springfield", "zip": f"{n % 100000:05d}"}, + "lines": [{"sku": f"sku-{n}-{i}", "qty": i + 1, "price": 9.99 + i} for i in range(8)], + "tags": [f"tag-{(n + i) % 7}" for i in range(5)], + "meta": {"source": "bench", "batch": str(n // 100)}, + "placed": when, + } + raise ValueError(shape) + + +def model_for(backend: str, shape: str) -> type: + return t.cast(type, getattr(MODELS[backend], shape.capitalize())) diff --git a/bench/src/mongospecs_bench/stress.py b/bench/src/mongospecs_bench/stress.py new file mode 100644 index 0000000..f442374 --- /dev/null +++ b/bench/src/mongospecs_bench/stress.py @@ -0,0 +1,360 @@ +"""Stress scenarios X1-X7 (plan section 6). Each returns a :class:`Result` with a pass/fail verdict.""" + +from __future__ import annotations + +import asyncio +import gc +import random +import threading +import typing as t + +from mongospecs import ( + DocumentTooLarge, + MongoSpecsError, + PoolTimeout, + Q, + Registry, + StaleDocumentError, + tracking, +) +from mongospecs_bench.harness import Result, Settings, environment, uri_for +from mongospecs_bench.models import MODELS + +Scenario = t.Callable[[Settings], t.Awaitable[Result]] + + +def _verdict(result: Result, passed: bool, summary: str) -> Result: + result.conclusion = ("PASS: " if passed else "FAIL: ") + summary + result.add(check="verdict", value="pass" if passed else "fail") + return result + + +async def x1(settings: Settings) -> Result: + result = Result( + "X1", + "Many tasks, small pool", + "Do thousands of concurrent tasks finish on a pool of 5, and do pool waits surface as typed errors?", + ) + tasks = settings.scale(3000, 200) + ns = MODELS["msgspec"] + async with environment(settings, maxPoolSize=5) as registry: + manager = registry.manager(ns.Plain) + + async def work(i: int) -> int: + doc = await manager.insert(ns.Plain(n=i, name=f"t{i}")) + found = await manager.get(doc.id) + return found.n if found else -1 + + done = await asyncio.wait_for(asyncio.gather(*(work(i) for i in range(tasks))), timeout=300) + count = await manager.count() + result.add(check="tasks completed", value=len(done)) + result.add(check="documents written", value=count) + ok = sorted(done) == list(range(tasks)) and count == tasks + async with environment(settings, maxPoolSize=1, waitQueueTimeoutMS=20) as starved: + manager = starved.manager(ns.Plain) + await manager.insert(ns.Plain(n=0)) + from mongospecs import trusted + + slow = [manager.find(trusted({"$where": "sleep(300) || true"})).all() for _ in range(6)] + outcomes = await asyncio.gather(*slow, return_exceptions=True) + typed = sum(isinstance(o, PoolTimeout) for o in outcomes) + untyped = [o for o in outcomes if isinstance(o, BaseException) and not isinstance(o, MongoSpecsError)] + result.add(check="pool waits raised as PoolTimeout", value=typed) + result.add(check="untyped errors", value=len(untyped)) + ok = ok and typed >= 1 and not untyped + return _verdict( + result, ok, f"{tasks} tasks completed on 5 connections; starved pool raised PoolTimeout {typed} times" + ) + + +async def x2(settings: Settings) -> Result: + result = Result( + "X2", + "Transaction contention", + "Do contended read-modify-write transactions lose updates, and are retries bounded?", + ) + workers = settings.scale(40, 8) + ns = MODELS["msgspec"] + async with environment(settings) as registry: + manager = registry.manager(ns.Plain) + counter = await manager.insert(ns.Plain(n=0, name="counter")) + attempts = 0 + + async def bump(tx: t.Any) -> None: + nonlocal attempts + attempts += 1 + doc = await manager.find(Q.id == counter.id, session=tx).one() + doc.n += 1 + await manager.save(doc, session=tx) + + await asyncio.gather(*(registry.transaction(bump) for _ in range(workers))) + final = await manager.get(counter.id) + assert final is not None + result.add(check="workers", value=workers) + result.add(check="final counter", value=final.n) + result.add(check="callback runs (incl. retries)", value=attempts) + ok = final.n == workers and attempts <= workers * 50 + return _verdict(result, ok, f"counter reached {final.n}/{workers} with {attempts - workers} retries") + + +async def x3(settings: Settings) -> Result: + result = Result( + "X3", + "Optimistic-lock storm", + "With many writers on one revisioned document, does exactly one win per revision?", + ) + writers = settings.scale(40, 8) + ns = MODELS["msgspec"] + async with environment(settings) as registry: + manager = registry.manager(ns.Managed) + doc = await manager.insert(ns.Managed(n=0, name="hot")) + copies = [await manager.find(Q.id == doc.id).one() for _ in range(writers)] + + async def attempt(copy: t.Any) -> bool: + copy.n += 1 + try: + await manager.save(copy) + except StaleDocumentError: + return False + return True + + first_round = await asyncio.gather(*(attempt(c) for c in copies)) + result.add(check="winners of the first round", value=sum(first_round)) + conflicts = 0 + + async def persistent(_: int) -> None: + nonlocal conflicts + while True: + fresh = await manager.find(Q.id == doc.id).one() + fresh.n += 1 + try: + await manager.save(fresh) + return + except StaleDocumentError: + conflicts += 1 + + await asyncio.gather(*(persistent(i) for i in range(writers))) + final = await manager.get(doc.id) + assert final is not None + result.add(check="final n", value=final.n) + result.add(check="final rev", value=final.rev) + result.add(check="conflicts retried", value=conflicts) + ok = sum(first_round) == 1 and final.n == writers + 1 and final.rev == writers + 1 + return _verdict( + result, + ok, + f"1 winner in the first round; after retries n={final.n}, rev={final.rev} for {writers} writers", + ) + + +async def x4(settings: Settings) -> Result: + result = Result( + "X4", + "Cancellation", + "Does cancelling tasks mid-operation and mid-transaction leak sessions, transactions or tracked state?", + ) + count = settings.scale(200, 30) + ns = MODELS["msgspec"] + async with environment(settings) as registry: + manager = registry.manager(ns.Plain) + hot = await manager.insert(ns.Plain(n=0, name="hot")) + gc.collect() + tracked_before = tracking.tracked_count() + + async def in_transaction(i: int) -> None: + async def body(tx: t.Any) -> None: + await manager.insert(ns.Plain(n=i, name="tx"), session=tx) + await asyncio.sleep(random.random() * 0.2) + await manager.update(hot.id, session=tx, name=f"tx{i}") + + await registry.transaction(body) + + async def plain(i: int) -> None: + docs = await manager.find().limit(50).all() + await asyncio.sleep(random.random() * 0.2) + await manager.insert(ns.Plain(n=i + len(docs), name="plain")) + + tasks = [asyncio.create_task(in_transaction(i) if i % 2 else plain(i)) for i in range(count)] + await asyncio.sleep(0.05) + cancelled = 0 + for task in tasks: + if random.random() < 0.7 and not task.done(): + task.cancel() + cancelled += 1 + outcomes = await asyncio.gather(*tasks, return_exceptions=True) + unexpected = [ + o + for o in outcomes + if isinstance(o, BaseException) and not isinstance(o, asyncio.CancelledError | MongoSpecsError) + ] + await asyncio.sleep(1) + admin = registry.client.admin + cursor = await admin.aggregate( + [ + {"$currentOp": {"allUsers": True, "idleSessions": True}}, + {"$match": {"transaction": {"$exists": True}}}, + ] + ) + open_transactions = len(await cursor.to_list()) + # locks are released: the hot document can be written immediately + await asyncio.wait_for(manager.update(hot.id, name="after"), timeout=5) + del tasks, outcomes + gc.collect() + result.add(check="tasks cancelled", value=cancelled) + result.add(check="unexpected error types", value=len(unexpected)) + result.add(check="transactions still open on the server", value=open_transactions) + result.add( + check="tracked documents leaked", value=max(0, tracking.tracked_count() - tracked_before - 1) + ) + ok = not unexpected and open_transactions == 0 and tracking.tracked_count() - tracked_before <= 1 + return _verdict( + result, + ok, + f"{cancelled} of {count} tasks cancelled; {open_transactions} open transactions afterwards", + ) + + +async def x5(settings: Settings) -> Result: + result = Result( + "X5", + "Result documents near 16MB", + "What happens when a $lookup would build a document over the BSON limit?", + ) + ns = MODELS["msgspec"] + async with environment(settings) as registry: + parents, children = registry.manager(ns.Parent), registry.manager(ns.Child) + payload = "x" * (1024 * 1024) + ids = [] + for i in range(18): # 18 x 1MB children referenced by a single parent + child = await children.insert(ns.Child(n=i, label=f"c{i}", payload=payload)) + ids.append(child.id) + await parents.insert(ns.Parent(n=0, name="big", children=ids)) + try: + await parents.find().include("children", strategy="lookup").all() + lookup = "returned" + except DocumentTooLarge: + lookup = "DocumentTooLarge" + except MongoSpecsError as exc: + lookup = f"{type(exc).__name__}: {exc}"[:120] + except Exception as exc: + lookup = f"UNTYPED {type(exc).__name__}" + loaded = await parents.find().include("children", strategy="batch").one() + batch_loaded = sum(ref.loaded for ref in loaded.children) + result.add(check="$lookup outcome", value=lookup) + result.add(check="batch strategy loaded children", value=batch_loaded) + ok = not lookup.startswith("UNTYPED") and lookup != "returned" and batch_loaded == 18 + return _verdict( + result, ok, f"$lookup over the limit -> {lookup}; batch strategy loaded all {batch_loaded} children" + ) + + +async def x6(settings: Settings) -> Result: + result = Result( + "X6", + "Change stream across a primary step-down", + "Does a consumer see every event exactly once across a step-down?", + ) + ns = MODELS["msgspec"] + async with environment(settings) as registry: + manager = registry.manager(ns.Plain) + await manager.insert(ns.Plain(n=-1, name="create collection")) + seen: list[int] = [] + stream = manager.watch([{"$match": {"operationType": "insert"}}], max_await_time_ms=500) + await stream._open() + + async def consume(expected: int) -> None: + async for event in stream: + seen.append(event.document.n) + if len(seen) == expected: + return + + consumer = asyncio.create_task(consume(20)) + for i in range(10): + await manager.insert(ns.Plain(n=i, name="before")) + stepped = "not attempted" + if settings.topology == "rs": + try: + await registry.client.admin.command("replSetStepDown", 2, force=True) + stepped = "stepped down" + except Exception as exc: # the command drops connections; that is the point + stepped = f"stepped down ({type(exc).__name__})" + inserted_after = 0 + deadline = asyncio.get_running_loop().time() + 90 + while inserted_after < 10 and asyncio.get_running_loop().time() < deadline: + try: + await manager.insert(ns.Plain(n=10 + inserted_after, name="after")) + inserted_after += 1 + except MongoSpecsError: + await asyncio.sleep(0.5) # no primary yet + try: + await asyncio.wait_for(consumer, timeout=60) + except TimeoutError: + consumer.cancel() + await stream.close() + result.add(check="step-down", value=stepped) + result.add(check="events seen", value=len(seen)) + result.add(check="duplicates", value=len(seen) - len(set(seen))) + ok = seen == list(range(20)) + return _verdict(result, ok, f"{len(seen)} of 20 events seen in order across the step-down ({stepped})") + + +async def x7(settings: Settings) -> Result: + import sys + + result = Result( + "X7", + "Threads with their own loops and registries", + "Is any state shared across registries when each thread runs its own event loop?", + ) + threads = settings.scale(8, 3) + per_thread = settings.scale(200, 20) + ns = MODELS["msgspec"] + uri = uri_for(settings.topology) + errors: list[str] = [] + counts: dict[int, int] = {} + + def worker(index: int) -> None: + async def main() -> None: + name = f"ms_x7_{index}_{random.randrange(10**9)}" + async with Registry(uri, database=name) as registry: + manager = registry.manager(ns.Managed) # explicit manager: no global binding involved + try: + docs = ( + await manager.insert_many( + [ns.Managed(n=i, name=f"t{index}") for i in range(per_thread)] + ) + ).inserted + for doc in docs[:20]: + doc.n += 1000 + await manager.save(doc) + loaded = await manager.find().all() + if any(d.name != f"t{index}" for d in loaded): + errors.append(f"thread {index} saw another thread's documents") + counts[index] = len(loaded) + finally: + await registry.client.drop_database(name) + + try: + asyncio.run(main()) + except Exception as exc: + errors.append(f"thread {index}: {type(exc).__name__}: {exc}") + + pool = [threading.Thread(target=worker, args=(i,)) for i in range(threads)] + for thread in pool: + thread.start() + for thread in pool: + thread.join() + free_threaded = not getattr(sys, "_is_gil_enabled", lambda: True)() + result.add(check="free-threaded build", value=free_threaded) + result.add(check="threads", value=threads) + result.add(check="errors", value=len(errors)) + result.notes.extend(errors[:5]) + ok = not errors and all(counts.get(i) == per_thread for i in range(threads)) + return _verdict( + result, + ok, + f"{threads} threads x {per_thread} documents, each with its own loop and registry; free-threaded={free_threaded}", + ) + + +SCENARIOS: dict[str, Scenario] = {"X1": x1, "X2": x2, "X3": x3, "X4": x4, "X5": x5, "X6": x6, "X7": x7} diff --git a/bench/src/mongospecs_bench/tracing.py b/bench/src/mongospecs_bench/tracing.py new file mode 100644 index 0000000..4afaa95 --- /dev/null +++ b/bench/src/mongospecs_bench/tracing.py @@ -0,0 +1,204 @@ +"""End-to-end tracing check (plan node E3). + +Runs a small mongospecs-web application with the Sentry SDK pointed at a local +rustrak server (``docker/compose.rustrak.yaml``), makes a few HTTP requests and +then asks rustrak what it received. The check passes when every request shows +up as one trace holding the HTTP transaction and the ``Model.op`` database spans +recorded by ``mongospecs.sentry_listener``. +""" + +from __future__ import annotations + +import asyncio +import datetime as dt +import os +import typing as t +import uuid +from dataclasses import dataclass, field + +import httpx + +from mongospecs import CreatedAt, Policy, Ref, Registry, Revision, SpecConfig, sentry_listener +from mongospecs.msgspec import Spec +from mongospecs_web import Api, Resource + +RUSTRAK_URL = os.environ.get("RUSTRAK_URL", "http://localhost:8787") +RUSTRAK_USER = os.environ.get("RUSTRAK_SUPERUSER", "admin@example.com:mongospecs-dev") +MONGO_URI = os.environ.get("MONGOSPECS_TEST_URI", "mongodb://localhost:27117/?replicaSet=rs0") + + +@dataclass +class TraceReport: + project: str + dsn: str + requests: list[str] = field(default_factory=list) + transactions: list[dict[str, t.Any]] = field(default_factory=list) + problems: list[str] = field(default_factory=list) + + @property + def ok(self) -> bool: + return not self.problems + + def render(self) -> str: + lines = [f"rustrak project {self.project!r}", f"dsn {self.dsn}", ""] + for tx in self.transactions: + lines.append(f" {tx['name']} trace={tx['trace_id'][:8]} {tx['duration_ms']:.1f}ms") + for span in tx["spans"]: + lines.append(f" {span['op']:<6} {span['description']:<22} {span['status']}") + lines.append("") + lines.extend(f"PROBLEM: {p}" for p in self.problems) + lines.append("end-to-end tracing: " + ("OK" if self.ok else "FAILED")) + return "\n".join(lines) + + +async def _rustrak_project(client: httpx.AsyncClient) -> tuple[int, str]: + email, _, password = RUSTRAK_USER.partition(":") + login = await client.post("/auth/login", json={"email": email, "password": password}) + login.raise_for_status() + created = await client.post("/api/projects", json={"name": f"mongospecs-e2e-{uuid.uuid4().hex[:8]}"}) + created.raise_for_status() + project = created.json() + key = str(project["sentry_key"]).replace("-", "") + host = RUSTRAK_URL.split("://", 1)[1] + scheme = RUSTRAK_URL.split("://", 1)[0] + return int(project["id"]), f"{scheme}://{key}@{host}/{project['id']}" + + +class Writer(Spec): + __spec__ = SpecConfig(collection="writers") + name: str = "" + + +class Story(Spec): + __spec__ = SpecConfig(collection="stories") + title: t.Annotated[str, Policy(filter=("eq",), sort=True)] = "" + words: int = 0 + writer: Ref[Writer] | None = None + created_at: t.Annotated[dt.datetime | None, CreatedAt] = None + rev: t.Annotated[int, Revision] = 0 + + +def _build_app(registry: t.Any) -> t.Any: + registry.bind(Writer, Story) + return Api(Resource(Writer), Resource(Story, include=["writer"]), prefix="/api").asgi() + + +async def run_trace_check(*, wait_seconds: float = 30.0) -> TraceReport: + import sentry_sdk + from sentry_sdk.integrations.asgi import SentryAsgiMiddleware + + async with httpx.AsyncClient(base_url=RUSTRAK_URL, timeout=10) as rustrak: + project_id, dsn = await _rustrak_project(rustrak) + report = TraceReport(f"project {project_id}", dsn) + sentry_sdk.init( + dsn=dsn, + traces_sample_rate=1.0, + default_integrations=False, + auto_enabling_integrations=False, + release="mongospecs-bench", + ) + database = f"ms_trace_{uuid.uuid4().hex[:8]}" + try: + async with Registry(MONGO_URI, database=database, on_command=sentry_listener()) as registry: + inner = _build_app(registry) + app = SentryAsgiMiddleware(inner, transaction_style="url") + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://app" + ) as client: + writer = (await client.post("/api/writers", json={"name": "Ann"})).json()["data"] + story = ( + await client.post("/api/stories", json={"title": "Traced", "writer": writer["id"]}) + ).json()["data"] + await client.get("/api/stories?include=writer&sort=title") + await client.patch(f"/api/stories/{story['id']}", json={"words": 120}) + report.requests = [ + "POST writers", + "POST stories", + "GET stories?include", + "PATCH stories/{id}", + ] + await registry.client.drop_database(database) + sentry_sdk.flush(timeout=10) + finally: + sentry_sdk.init(dsn=None, default_integrations=False, auto_enabling_integrations=False) + + # rustrak ingests in two phases; poll until the transactions are queryable. + deadline = asyncio.get_running_loop().time() + wait_seconds + items: list[dict[str, t.Any]] = [] + while asyncio.get_running_loop().time() < deadline: + listed = await rustrak.get(f"/api/projects/{project_id}/transactions") + listed.raise_for_status() + body = listed.json() + items = body.get("items", body if isinstance(body, list) else []) + if len(items) >= len(report.requests): + break + await asyncio.sleep(0.5) + for item in items: + detail = await rustrak.get(f"/api/projects/{project_id}/transactions/{item['id']}") + detail.raise_for_status() + report.transactions.append(_summarise(detail.json())) + + report.transactions.sort(key=lambda tx: tx["start"]) + _verify(report) + return report + + +def _summarise(detail: dict[str, t.Any]) -> dict[str, t.Any]: + data = detail.get("data") or detail.get("event") or detail + trace = (data.get("contexts") or {}).get("trace") or {} + spans = data.get("spans") or detail.get("spans") or [] + return { + "name": detail.get("transaction") or detail.get("name") or data.get("transaction") or "?", + "trace_id": detail.get("trace_id") or trace.get("trace_id") or "", + "duration_ms": float(detail.get("duration_ms") or detail.get("duration") or 0.0), + "start": str( + detail.get("start_timestamp") or data.get("start_timestamp") or detail.get("timestamp") or "" + ), + "spans": [ + { + "op": span.get("op") or "", + "description": span.get("description") or span.get("name") or "", + "status": span.get("status") or "", + "trace_id": span.get("trace_id") or "", + } + for span in spans + ], + } + + +EXPECTED: list[tuple[str, list[str]]] = [ + ("writers", ["Writer.insert"]), + ("stories", ["Story.insert"]), + ("stories", ["Story.find", "Writer.find"]), + ("stories", ["Story.update"]), +] + + +def _verify(report: TraceReport) -> None: + if len(report.transactions) != len(EXPECTED): + report.problems.append( + f"expected {len(EXPECTED)} transactions in rustrak, found {len(report.transactions)}" + ) + return + for tx, (path_part, wanted) in zip(report.transactions, EXPECTED, strict=True): + if path_part not in tx["name"]: + report.problems.append(f"transaction {tx['name']!r} does not look like a request to {path_part}") + db = [s["description"] for s in tx["spans"] if s["op"] == "db"] + if db != wanted: + report.problems.append(f"{tx['name']}: database spans {db} != expected {wanted}") + if any(s["trace_id"] and s["trace_id"] != tx["trace_id"] for s in tx["spans"]): + report.problems.append(f"{tx['name']}: a span belongs to a different trace") + if any(s["status"] not in ("ok", "") for s in tx["spans"] if s["op"] == "db"): + report.problems.append(f"{tx['name']}: a database span did not finish ok") + if len({tx["trace_id"] for tx in report.transactions}) != len(report.transactions): + report.problems.append("requests did not get one trace each") + + +def main() -> int: + report = asyncio.run(run_trace_check()) + print(report.render()) + return 0 if report.ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/src/mongospecs_bench/workloads.py b/bench/src/mongospecs_bench/workloads.py new file mode 100644 index 0000000..3b16d05 --- /dev/null +++ b/bench/src/mongospecs_bench/workloads.py @@ -0,0 +1,643 @@ +"""Benchmark workloads B1-B8 (plan section 6). Each returns a :class:`Result`. + +Everything that touches the network runs against real MongoDB. Numbers from a +``--quick`` run only prove the harness works; they are not for conclusions. +""" + +from __future__ import annotations + +import gc +import tracemalloc +import typing as t + +import bson +import httpx + +from mongospecs import Inc, Plugin, Q, meta_of, relations, tracking +from mongospecs.query import _encode_cursor +from mongospecs_bench.harness import Result, Settings, environment, measure, measure_sync +from mongospecs_bench.models import MODELS, SHAPES, model_for, raw_document +from mongospecs_web import Api, Resource + +Workload = t.Callable[[Settings], t.Awaitable[Result]] + + +def _ratio(value: float, base: float) -> float: + return round(value / base, 2) if base else 0.0 + + +# -- B1 ------------------------------------------------------------------------------- + + +async def b1(settings: Settings) -> Result: + result = Result( + "B1", + "Materialise documents", + "What does turning a raw document into a model cost, per backend, and how does it compare " + "with the driver's own BSON decoding?", + ) + n = settings.scale(10_000, 300) + for shape in SHAPES: + docs = [{"_id": bson.ObjectId(), **raw_document(shape, i)} for i in range(n)] + encoded = [bson.encode(d) for d in docs] + options: t.Any = bson.codec_options.CodecOptions(tz_aware=True) + base = measure_sync(lambda: [bson.decode(b, options) for b in encoded], repeat=settings.repeat) + result.add( + shape=shape, + backend="driver", + mode="bson->dict", + n=n, + us_per_doc=round(base.median_ms * 1000 / n, 2), + vs_bson=1.0, + ) + for backend in settings.backends: + meta = meta_of(model_for(backend, shape)) + modes: dict[str, t.Callable[[], t.Any]] = { + "validated": lambda: [meta.decode(d, trusted=False) for d in docs], + "trusted": lambda: [meta.decode(d, trusted=True) for d in docs], + "tracked": lambda: [meta.encode(meta.decode(d, trusted=False)) for d in docs], + } + for mode, fn in modes.items(): + timing = measure_sync(fn, repeat=settings.repeat) + result.add( + shape=shape, + backend=backend, + mode=mode, + n=n, + us_per_doc=round(timing.median_ms * 1000 / n, 2), + vs_bson=_ratio(timing.median_ms, base.median_ms), + ) + result.notes.append("'tracked' is decode plus the encode that builds the change-tracking snapshot.") + result.notes.append("'trusted' only differs for flat pydantic models; msgspec and attrs always convert.") + return result + + +# -- B2 ------------------------------------------------------------------------------- + + +async def b2(settings: Settings) -> Result: + result = Result( + "B2", + "find() end to end", + "How much does the ODM add on top of the raw driver for reads of 1, 100 and 10,000 documents?", + ) + total = settings.scale(10_000, 300) + sizes = [n for n in (1, 100, 10_000) if n <= total] or [total] + async with environment(settings) as registry: + for shape in ("small", "nested"): + for backend in settings.backends: + model = model_for(backend, shape) + manager: t.Any = registry.manager(model) + if backend == settings.backends[0]: + await manager.collection.insert_many([raw_document(shape, i) for i in range(total)]) + for n in sizes: + collection = manager.collection + raw = await measure( + lambda: collection.find({}, limit=n).to_list(), repeat=settings.repeat + ) + cases = { + "odm raw()": lambda: manager.find().limit(n).raw().all(), + "odm lean()": lambda: manager.find().limit(n).lean().all(), + "odm tracked": lambda: manager.find().limit(n).all(), + } + if backend == settings.backends[0]: + result.add( + shape=shape, + backend="pymongo", + case="driver dicts", + n=n, + median_ms=round(raw.median_ms, 3), + vs_driver=1.0, + ) + for case, fn in cases.items(): + timing = await measure(fn, repeat=settings.repeat) + result.add( + shape=shape, + backend=backend, + case=case, + n=n, + median_ms=round(timing.median_ms, 3), + vs_driver=_ratio(timing.median_ms, raw.median_ms), + ) + await registry.manager(model_for(settings.backends[0], shape)).collection.drop() + return result + + +# -- B3 ------------------------------------------------------------------------------- + + +async def b3(settings: Settings) -> Result: + result = Result( + "B3", + "include(): $lookup vs batched $in", + "Which strategy should include(strategy='auto') use, across fan-outs and topologies?", + ) + plans = [(1, 400), (10, 200), (100, 50), (1000, 10)] + if settings.quick: + plans = [(1, 20), (10, 10)] + ns = MODELS["msgspec"] + async with environment(settings) as registry: + parents, children = registry.manager(ns.Parent), registry.manager(ns.Child) + if settings.topology == "sharded": + db = parents.collection.database.name + await registry.client.admin.command( + "shardCollection", f"{db}.bench_children", key={"_id": "hashed"} + ) + for fan_out, parent_count in plans: + await parents.collection.delete_many({}) + await children.collection.delete_many({}) + child_docs = [ + {"_id": bson.ObjectId(), "n": i, "label": f"c{i}", "payload": "x" * 200} + for i in range(fan_out * parent_count) + ] + await children.collection.insert_many(child_docs) + await parents.collection.insert_many( + [ + { + "n": p, + "name": f"p{p}", + "children": [c["_id"] for c in child_docs[p * fan_out : (p + 1) * fan_out]], + } + for p in range(parent_count) + ] + ) + timings = {} + strategy: t.Any + for strategy in ("batch", "lookup"): + timings[strategy] = await measure( + lambda: parents.find().lean().include("children", strategy=strategy).all(), + repeat=settings.repeat, + ) + result.add( + relation="list, distinct children", + fan_out=fan_out, + parents=parent_count, + batch_ms=round(timings["batch"].median_ms, 3), + lookup_ms=round(timings["lookup"].median_ms, 3), + lookup_vs_batch=_ratio(timings["lookup"].median_ms, timings["batch"].median_ms), + ) + # To-one with heavy sharing: many parents, few targets. $lookup repeats each target per parent. + await parents.collection.delete_many({}) + await children.collection.delete_many({}) + owners = [ + {"_id": bson.ObjectId(), "n": i, "label": f"o{i}", "payload": "x" * 2000} for i in range(20) + ] + await children.collection.insert_many(owners) + count = settings.scale(2000, 100) + await parents.collection.insert_many( + [{"n": p, "name": f"p{p}", "owner": owners[p % 20]["_id"]} for p in range(count)] + ) + timings = {} + for strategy in ("batch", "lookup"): + timings[strategy] = await measure( + lambda: parents.find().lean().include("owner", strategy=strategy).all(), + repeat=settings.repeat, + ) + result.add( + relation="to-one, 20 shared targets", + fan_out=1, + parents=count, + batch_ms=round(timings["batch"].median_ms, 3), + lookup_ms=round(timings["lookup"].median_ms, 3), + lookup_vs_batch=_ratio(timings["lookup"].median_ms, timings["batch"].median_ms), + ) + ratios = [row["lookup_vs_batch"] for row in result.rows] + wins = sum(1 for r in ratios if r < 0.95) + result.conclusion = ( + f"$lookup was faster than batched $in in {wins} of {len(ratios)} cases on {settings.topology} " + f"(lookup/batch ratios: {ratios}). Current default: {relations.DEFAULT_STRATEGY}." + ) + result.notes.append( + "Strategy choice does not depend on the model backend; measured with msgspec, lean reads." + ) + return result + + +# -- B4 ------------------------------------------------------------------------------- + + +async def b4(settings: Settings) -> Result: + result = Result("B4", "Writes", "What does the ODM add to inserts, atomic patches, saves and upserts?") + n = settings.scale(1000, 50) + ops = settings.scale(200, 20) + async with environment(settings) as registry: + for backend in settings.backends: + ns = MODELS[backend] + plain, managed = registry.manager(ns.Plain), registry.manager(ns.Managed) + collection = plain.collection + + async def reset() -> None: + await collection.delete_many({}) + await managed.collection.delete_many({}) + + raw_insert = await measure( + lambda: collection.insert_many([{"n": i, "name": f"n{i}"} for i in range(n)]), + repeat=settings.repeat, + setup=reset, + ) + cases: dict[str, t.Callable[[], t.Awaitable[t.Any]]] = { + "insert_many (no plugins)": lambda: plain.insert_many( + [ns.Plain(n=i, name=f"n{i}") for i in range(n)] + ), + "insert_many (timestamps+revision)": lambda: managed.insert_many( + [ns.Managed(n=i, name=f"n{i}") for i in range(n)] + ), + } + result.add( + backend=backend, + op=f"insert {n}", + case="driver insert_many", + median_ms=round(raw_insert.median_ms, 3), + vs_driver=1.0, + ) + for case, fn in cases.items(): + timing = await measure(fn, repeat=settings.repeat, setup=reset) + result.add( + backend=backend, + op=f"insert {n}", + case=case, + median_ms=round(timing.median_ms, 3), + vs_driver=_ratio(timing.median_ms, raw_insert.median_ms), + ) + + await reset() + docs = (await plain.insert_many([ns.Plain(n=i, name=f"n{i}") for i in range(ops)])).inserted + ids = [d.id for d in docs] + + async def raw_patch() -> None: + for _id in ids: + await collection.update_one({"_id": _id}, {"$inc": {"n": 1}}) + + async def odm_patch() -> None: + for _id in ids: + await plain.update(_id, Inc(n=1)) + + async def odm_save() -> None: + for doc in docs: + doc.n += 1 + await plain.save(doc) + + async def raw_upsert() -> None: + for i in range(ops): + await collection.update_one({"name": f"u{i}"}, {"$inc": {"n": 1}}, upsert=True) + + async def odm_upsert() -> None: + for i in range(ops): + await plain.upsert(Q.name == f"u{i}", Inc(n=1)) + + base = await measure(raw_patch, repeat=settings.repeat) + result.add( + backend=backend, + op=f"patch x{ops}", + case="driver update_one", + median_ms=round(base.median_ms, 3), + vs_driver=1.0, + ) + for case, fn in (("Model.update (returns doc)", odm_patch), ("doc.save() delta", odm_save)): + timing = await measure(fn, repeat=settings.repeat) + result.add( + backend=backend, + op=f"patch x{ops}", + case=case, + median_ms=round(timing.median_ms, 3), + vs_driver=_ratio(timing.median_ms, base.median_ms), + ) + base = await measure(raw_upsert, repeat=settings.repeat) + timing = await measure(odm_upsert, repeat=settings.repeat) + result.add( + backend=backend, + op=f"upsert x{ops}", + case="driver update_one(upsert)", + median_ms=round(base.median_ms, 3), + vs_driver=1.0, + ) + result.add( + backend=backend, + op=f"upsert x{ops}", + case="Model.upsert (returns doc)", + median_ms=round(timing.median_ms, 3), + vs_driver=_ratio(timing.median_ms, base.median_ms), + ) + await reset() + result.notes.append( + "Model.update and Model.upsert use findAndModify to return the document; the driver baseline does not." + ) + return result + + +# -- B5 ------------------------------------------------------------------------------- + + +async def b5(settings: Settings) -> Result: + result = Result( + "B5", + "Pagination", + "How do offset and cursor pagination compare at depth, and should a total come from $facet or a count?", + ) + total = settings.scale(50_000, 2_000) + size = 20 + ns = MODELS["msgspec"] + async with environment(settings) as registry: + manager = registry.manager(ns.Small) + await manager.collection.insert_many([raw_document("small", i) for i in range(total)]) + await manager.collection.create_index([("n", 1)]) + for page in (1, 100, total // size // 2, total // size - 1): + if page < 1 or page * size > total: + continue + skip = (page - 1) * size + offset = await measure( + lambda: manager.find().sort("n").lean().page(size, number=page), repeat=settings.repeat + ) + if skip: + last = await manager.collection.find_one({}, sort=[("n", 1)], skip=skip - 1) + assert last is not None + token: str | None = _encode_cursor([last["n"], last["_id"]]) + else: + token = None + cursor = await measure( + lambda: manager.find().sort("n").lean().page(size, after=token), repeat=settings.repeat + ) + result.add( + page=page, + skip=skip, + offset_ms=round(offset.median_ms, 3), + cursor_ms=round(cursor.median_ms, 3), + offset_vs_cursor=_ratio(offset.median_ms, cursor.median_ms), + ) + flt = {"n": {"$gte": total // 4}} + facet_pipeline: list[dict[str, t.Any]] = [ + {"$match": flt}, + {"$facet": {"items": [{"$sort": {"n": 1}}, {"$limit": size}], "total": [{"$count": "n"}]}}, + ] + + async def facet() -> t.Any: + cursor_ = await manager.collection.aggregate(facet_pipeline) + return await cursor_.to_list() + + concurrent = await measure( + lambda: manager.find(Q.n >= total // 4).sort("n").lean().page(size, number=1, total=True), + repeat=settings.repeat, + ) + faceted = await measure(facet, repeat=settings.repeat) + result.add( + page="total: concurrent count", + skip=0, + offset_ms=round(concurrent.median_ms, 3), + cursor_ms="", + offset_vs_cursor="", + ) + result.add( + page="total: $facet (raw driver)", + skip=0, + offset_ms=round(faceted.median_ms, 3), + cursor_ms="", + offset_vs_cursor="", + ) + result.notes.append(f"Collection of {total} documents, page size {size}, sorted on an indexed field.") + result.notes.append( + "$facet returns the page and the total as one document, so the page is bound by the 16MB limit." + ) + return result + + +# -- B6 ------------------------------------------------------------------------------- + + +class _Noop(Plugin): + def __init__(self, index: int) -> None: + self.name = f"noop{index}" + + +async def b6(settings: Settings) -> Result: + result = Result("B6", "Plugin overhead", "What does each active plugin cost per operation?") + ops = settings.scale(300, 20) + ns = MODELS["msgspec"] + baseline: dict[str, float] = {} + for count in (0, 3, 10): + async with environment(settings, plugins=[_Noop(i) for i in range(count)]) as registry: + manager = registry.manager(ns.Plain) + + async def inserts() -> None: + for i in range(ops): + await manager.insert(ns.Plain(n=i)) + + docs = (await manager.insert_many([ns.Plain(n=i) for i in range(ops)])).inserted + + async def saves() -> None: + for doc in docs: + doc.n += 1 + await manager.save(doc) + + async def finds() -> None: + for doc in docs: + await manager.get(doc.id) + + for op, fn in (("insert", inserts), ("save", saves), ("get", finds)): + timing = await measure(fn, repeat=settings.repeat) + per_op = timing.median_ms * 1000 / ops + baseline.setdefault(op, per_op) + result.add( + plugins=f"{count} no-op", + op=op, + us_per_op=round(per_op, 1), + vs_no_plugins=_ratio(per_op, baseline[op]), + ) + async with environment(settings) as registry: + manager = registry.manager(ns.Managed) + + async def managed_inserts() -> None: + for i in range(ops): + await manager.insert(ns.Managed(n=i)) + + timing = await measure(managed_inserts, repeat=settings.repeat) + per_op = timing.median_ms * 1000 / ops + result.add( + plugins="timestamps+revision", + op="insert", + us_per_op=round(per_op, 1), + vs_no_plugins=_ratio(per_op, baseline["insert"]), + ) + result.notes.append( + "Each operation is one round trip to MongoDB, which dominates; plugin calls are in-process." + ) + return result + + +# -- B7 ------------------------------------------------------------------------------- + + +def _apps(api: Api) -> dict[str, t.Any]: + from fastapi import FastAPI + from litestar import Litestar + from quart import Quart + from starlette.applications import Starlette + + from mongospecs_web.plugs import fastapi as fastapi_plug + from mongospecs_web.plugs import quart as quart_plug + from mongospecs_web.plugs import starlette as starlette_plug + from mongospecs_web.plugs.litestar import MongospecsPlugin + + fast = FastAPI() + fastapi_plug.mount(fast, api) + quart = Quart(__name__) + quart_plug.init_app(quart, api) + return { + "asgi": api.asgi(), + "starlette": Starlette(routes=starlette_plug.routes(api)), + "fastapi": fast, + "litestar": Litestar(plugins=[MongospecsPlugin(api)], openapi_config=None), + "quart": quart, + } + + +async def b7(settings: Settings) -> Result: + result = Result( + "B7", + "HTTP request to response", + "What does each plug add over calling the API core directly, per model backend?", + ) + calls = settings.scale(200, 10) + async with environment(settings) as registry: + for backend in settings.backends: + ns = MODELS[backend] + manager = registry.manager(ns.Small) + await manager.collection.delete_many({}) + await manager.collection.insert_many([raw_document("small", i) for i in range(200)]) + first = await manager.collection.find_one({}) + assert first is not None + from mongospecs_web.backend import MongoBackend + + api = Api( + Resource(ns.Small, name="small", backend=MongoBackend(ns.Small, manager)), prefix="/api" + ) + one, many = f"/api/small/{first['_id']}", "/api/small?limit=20" + + async def direct(path: str) -> None: + for _ in range(calls): + await api.handle( + "GET", path.split("?")[0], query=[("limit", "20")] if "?" in path else [] + ) + + base = { + "get one": await measure(lambda: direct(one), repeat=settings.repeat), + "list 20": await measure(lambda: direct(many), repeat=settings.repeat), + } + for case, timing in base.items(): + result.add( + backend=backend, + plug="(core, no HTTP)", + case=case, + us_per_request=round(timing.median_ms * 1000 / calls, 1), + vs_core=1.0, + ) + for plug, app in _apps(api).items(): + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://bench" + ) as client: + + async def hit(path: str) -> None: + for _ in range(calls): + response = await client.get(path) + assert response.status_code == 200, response.text + + for case, path in (("get one", one), ("list 20", many)): + timing = await measure(lambda: hit(path), repeat=settings.repeat) + per = timing.median_ms * 1000 / calls + result.add( + backend=backend, + plug=plug, + case=case, + us_per_request=round(per, 1), + vs_core=_ratio(per, base[case].median_ms * 1000 / calls), + ) + result.notes.append( + "Requests go through httpx's in-process ASGI transport, so client overhead is included equally for every plug." + ) + return result + + +# -- B8 ------------------------------------------------------------------------------- + + +async def b8(settings: Settings) -> Result: + result = Result( + "B8", + "Cost of change tracking", + "What do tracked reads cost in time and memory over lean and raw reads, and what does a delta save send?", + ) + n = settings.scale(5000, 200) + async with environment(settings) as registry: + for backend in settings.backends: + manager: t.Any = registry.manager(model_for(backend, "nested")) + await manager.collection.delete_many({}) + await manager.collection.insert_many([raw_document("nested", i) for i in range(n)]) + timings = { + "raw()": await measure(lambda: manager.find().raw().all(), repeat=settings.repeat), + "lean()": await measure(lambda: manager.find().lean().all(), repeat=settings.repeat), + "tracked": await measure(lambda: manager.find().all(), repeat=settings.repeat), + } + memory: dict[str, float] = {} + for mode, query in ( + ("raw()", manager.find().raw()), + ("lean()", manager.find().lean()), + ("tracked", manager.find()), + ): + gc.collect() + tracemalloc.start() + before = tracemalloc.get_traced_memory()[0] + held = await query.all() + memory[mode] = (tracemalloc.get_traced_memory()[0] - before) / len(held) + tracemalloc.stop() + del held + for mode, timing in timings.items(): + result.add( + backend=backend, + mode=mode, + n=n, + median_ms=round(timing.median_ms, 2), + vs_lean=_ratio(timing.median_ms, timings["lean()"].median_ms), + bytes_per_doc=round(memory[mode]), + ) + doc = await manager.find().first() + assert doc is not None + full = bson.encode( + {"$set": {k: v for k, v in meta_of(type(doc)).encode(doc).items() if k != "_id"}} + ) + t.cast(t.Any, doc).customer = "changed" + delta = bson.encode(t.cast(t.Any, doc).changes()) + result.add( + backend=backend, + mode="save payload: full $set", + n=1, + median_ms="", + vs_lean="", + bytes_per_doc=len(full), + ) + result.add( + backend=backend, + mode="save payload: delta", + n=1, + median_ms="", + vs_lean="", + bytes_per_doc=len(delta), + ) + gc.collect() + result.notes.append( + f"{tracking.tracked_count()} tracked documents remained after the run (should be ~0)." + ) + result.notes.append( + "bytes_per_doc for reads is Python heap per document held; for save payloads it is the BSON update size." + ) + return result + + +WORKLOADS: dict[str, Workload] = { + "B1": b1, + "B2": b2, + "B3": b3, + "B4": b4, + "B5": b5, + "B6": b6, + "B7": b7, + "B8": b8, +} diff --git a/bench/tests/test_bench_smoke.py b/bench/tests/test_bench_smoke.py new file mode 100644 index 0000000..c63eca6 --- /dev/null +++ b/bench/tests/test_bench_smoke.py @@ -0,0 +1,44 @@ +"""The harness must keep working: every workload and scenario runs in quick mode.""" + +import json + +import pytest + +from mongospecs_bench import harness +from mongospecs_bench.cli import report +from mongospecs_bench.harness import Settings, metadata, write_result +from mongospecs_bench.stress import SCENARIOS +from mongospecs_bench.workloads import WORKLOADS + +QUICK = Settings(quick=True) + + +@pytest.mark.parametrize("name", sorted(WORKLOADS)) +async def test_workload_runs(name): + result = await WORKLOADS[name](QUICK) + assert result.workload == name and result.rows + assert all(isinstance(row, dict) for row in result.rows) + + +@pytest.mark.parametrize("name", sorted(set(SCENARIOS) - {"X6"})) +async def test_stress_scenario_passes(name): + result = await SCENARIOS[name](QUICK) + assert result.conclusion.startswith("PASS"), result.conclusion + + +@pytest.mark.sharded +async def test_include_workload_on_sharded_cluster(): + result = await WORKLOADS["B3"](Settings(topology="sharded", quick=True)) + assert all(row["batch_ms"] > 0 and row["lookup_ms"] > 0 for row in result.rows) + + +async def test_results_are_written_and_reported(tmp_path, monkeypatch): + monkeypatch.setattr(harness, "RESULTS", tmp_path) + monkeypatch.setattr("mongospecs_bench.cli.RESULTS", tmp_path) + meta = await metadata(QUICK) + result = await WORKLOADS["B6"](QUICK) + path = write_result(result, meta) + data = json.loads(path.read_text()) + assert data["workload"] == "B6" and data["meta"]["mongodb"] and data["meta"]["quick"] is True + text = report() + assert "## B6: Plugin overhead" in text and "| plugins |" in text diff --git a/docker/compose.rustrak.yaml b/docker/compose.rustrak.yaml new file mode 100644 index 0000000..72fadc0 --- /dev/null +++ b/docker/compose.rustrak.yaml @@ -0,0 +1,19 @@ +# rustrak: a self-hosted, Sentry-compatible server, used to verify tracing end to end. +# docker compose -f docker/compose.rustrak.yaml up -d --wait +# dashboard and API: http://localhost:8787 (admin@example.com / mongospecs-dev) +# Development only: the credentials below are throwaway defaults. +name: mongospecs-rustrak +services: + rustrak: + image: rustrak/rustrak-server:${RUSTRAK_VERSION:-latest} + ports: + - "8787:8080" + environment: + - SESSION_SECRET_KEY=${RUSTRAK_SESSION_SECRET:-0000000000000000000000000000000000000000000000000000000000000000} + - CREATE_SUPERUSER=${RUSTRAK_SUPERUSER:-admin@example.com:mongospecs-dev} + - RUSTRAK_TELEMETRY=off + volumes: + - rustrak_data:/data + +volumes: + rustrak_data: diff --git a/pyproject.toml b/pyproject.toml index 55cdf73..52809c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,8 @@ ignore = ["E501", "RUF012"] [tool.ruff.lint.per-file-ignores] "**/tests/**" = ["B", "RUF"] +# Benchmarks define closures inside loops and call them before the next iteration. +"bench/**" = ["B023"] [tool.mypy] python_version = "3.11" diff --git a/uv.lock b/uv.lock index 249bda8..57e90f6 100644 --- a/uv.lock +++ b/uv.lock @@ -1100,14 +1100,14 @@ version = "0.0.0" source = { editable = "bench" } dependencies = [ { name = "httpx" }, - { name = "mongospecs", extra = ["all"] }, + { name = "mongospecs", extra = ["all", "sentry"] }, { name = "mongospecs-web", extra = ["all"] }, ] [package.metadata] requires-dist = [ { name = "httpx", specifier = ">=0.27" }, - { name = "mongospecs", extras = ["all"], editable = "packages/mongospecs" }, + { name = "mongospecs", extras = ["all", "sentry"], editable = "packages/mongospecs" }, { name = "mongospecs-web", extras = ["all"], editable = "packages/mongospecs-web" }, ] From 3c415daa65026da79fe7ff279dc0f081d21056ce Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 00:25:23 -0400 Subject: [PATCH 02/26] perf(core): raw-BSON snapshots and batched cursor reads; e2e storefront Changes driven by the first full benchmark run and by booting a real application. Performance - tracked reads keep the raw BSON the driver received as the snapshot and build the comparable form only when a document is saved, instead of re-encoding every loaded document - cursors are drained with to_list() batches, not one await per document - remove the 'trusted' read flag: the pydantic fast path it enabled was slower than validating, and it did nothing on msgspec and attrs Fixes found by the end-to-end tests - aggregate, union, graph and backref-count paths sent max_time_ms in find() spelling and failed whenever a default time limit was set - web: unexpected errors are now logged server-side (the client still gets a bare 500) - ids given as strings to reference fields are coerced to the target's id type on backends that do not validate on construction - stored documents are read leniently on strict backends (an integer stored as a double), input stays strict - a document stored before a Revision field existed can be saved Reference application and tests - bench: 'storefront', a production-shaped app (auth, admin guards, row scope, transactional orders that cannot oversell, soft delete, ETags, reviews as a reverse relation, a resumable exactly-once change-stream projector, health check, graceful shutdown) - end-to-end tests boot it under uvicorn and drive it over HTTP on every plug and model backend: customer journey, concurrent buyers, all-or-nothing multi-item orders, projector resume across restarts, in-flight requests on shutdown, consistency across a primary step-down - HTTP load benchmark against the running server with a hand-written pymongo + Starlette baseline of the same endpoints - Resource(server_fields=...) for values the server sets on create - guide pages whose examples are executed by the test suite, migration guide, Mongoose parity page, Mongoose benchmark comparator Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 36 ++ bench/mongoose/.gitignore | 2 + bench/mongoose/bench.mjs | 228 ++++++++ bench/mongoose/package.json | 12 + bench/pyproject.toml | 2 +- bench/results/4f3b2c1/B1.json | 271 ++++++++++ bench/results/4f3b2c1/B2.json | 508 ++++++++++++++++++ bench/results/4f3b2c1/B3.json | 70 +++ bench/results/4f3b2c1/B4.json | 198 +++++++ bench/results/4f3b2c1/B5.json | 73 +++ bench/results/4f3b2c1/B6.json | 90 ++++ bench/src/mongospecs_bench/cli.py | 9 +- bench/src/mongospecs_bench/load.py | 214 ++++++++ .../mongospecs_bench/storefront/__init__.py | 16 + bench/src/mongospecs_bench/storefront/app.py | 405 ++++++++++++++ bench/src/mongospecs_bench/storefront/auth.py | 68 +++ .../src/mongospecs_bench/storefront/models.py | 160 ++++++ .../mongospecs_bench/storefront/projector.py | 87 +++ .../mongospecs_bench/storefront/raw_app.py | 226 ++++++++ bench/src/mongospecs_bench/storefront/seed.py | 132 +++++ .../src/mongospecs_bench/storefront/server.py | 140 +++++ bench/src/mongospecs_bench/workloads.py | 25 +- bench/tests/conftest.py | 80 +++ bench/tests/test_e2e_storefront.py | 388 +++++++++++++ docs/benchmarks.md | 1 + docs/guide/models.md | 134 +++++ docs/guide/operations.md | 92 ++++ docs/guide/plugins.md | 68 +++ docs/guide/queries.md | 129 +++++ docs/guide/relations.md | 93 ++++ docs/guide/writes.md | 159 ++++++ docs/index.md | 44 +- docs/migration.md | 66 +++ docs/parity.md | 68 +++ docs/reference/bulk.md | 3 + docs/reference/errors.md | 3 + docs/reference/expr.md | 3 + docs/reference/hooks.md | 3 + docs/reference/indexes.md | 3 + docs/reference/manager.md | 3 + docs/reference/markers.md | 3 + docs/reference/model.md | 3 + docs/reference/query.md | 3 + docs/reference/refs.md | 3 + docs/reference/registry.md | 3 + docs/reference/relations.md | 3 + docs/reference/streams.md | 3 + docs/reference/updates.md | 3 + docs/reference/web_api.md | 3 + docs/reference/web_backend.md | 3 + docs/reference/web_errors.md | 3 + docs/reference/web_filters.md | 3 + docs/reference/web_openapi.md | 3 + docs/reference/web_policy.md | 3 + docs/reference/web_resource.md | 3 + docs/reference/web_testing.md | 3 + docs/web/plugs.md | 76 +++ docs/web/resources.md | 131 +++++ mkdocs.yaml | 54 +- .../mongospecs-web/src/mongospecs_web/api.py | 6 +- .../src/mongospecs_web/backend.py | 2 +- .../src/mongospecs_web/resource.py | 20 +- .../src/mongospecs_web/testing.py | 2 +- .../tests/test_docs_examples.py | 51 ++ packages/mongospecs/src/mongospecs/attrs.py | 4 +- packages/mongospecs/src/mongospecs/bulk.py | 2 +- packages/mongospecs/src/mongospecs/codec.py | 13 +- packages/mongospecs/src/mongospecs/manager.py | 34 +- packages/mongospecs/src/mongospecs/markers.py | 3 - packages/mongospecs/src/mongospecs/meta.py | 6 +- packages/mongospecs/src/mongospecs/model.py | 11 +- packages/mongospecs/src/mongospecs/msgspec.py | 6 +- .../src/mongospecs/plugins/revision.py | 4 +- .../mongospecs/src/mongospecs/pydantic.py | 23 +- packages/mongospecs/src/mongospecs/query.py | 86 +-- .../mongospecs/src/mongospecs/relations.py | 4 +- .../mongospecs/src/mongospecs/tracking.py | 58 +- .../mongospecs/tests/test_registry_errors.py | 6 + packages/mongospecs/tests/test_tracking_tx.py | 37 ++ pyproject.toml | 1 + uv.lock | 15 + 81 files changed, 4886 insertions(+), 129 deletions(-) create mode 100644 bench/README.md create mode 100644 bench/mongoose/.gitignore create mode 100644 bench/mongoose/bench.mjs create mode 100644 bench/mongoose/package.json create mode 100644 bench/results/4f3b2c1/B1.json create mode 100644 bench/results/4f3b2c1/B2.json create mode 100644 bench/results/4f3b2c1/B3.json create mode 100644 bench/results/4f3b2c1/B4.json create mode 100644 bench/results/4f3b2c1/B5.json create mode 100644 bench/results/4f3b2c1/B6.json create mode 100644 bench/src/mongospecs_bench/load.py create mode 100644 bench/src/mongospecs_bench/storefront/__init__.py create mode 100644 bench/src/mongospecs_bench/storefront/app.py create mode 100644 bench/src/mongospecs_bench/storefront/auth.py create mode 100644 bench/src/mongospecs_bench/storefront/models.py create mode 100644 bench/src/mongospecs_bench/storefront/projector.py create mode 100644 bench/src/mongospecs_bench/storefront/raw_app.py create mode 100644 bench/src/mongospecs_bench/storefront/seed.py create mode 100644 bench/src/mongospecs_bench/storefront/server.py create mode 100644 bench/tests/conftest.py create mode 100644 bench/tests/test_e2e_storefront.py create mode 100644 docs/benchmarks.md create mode 100644 docs/guide/models.md create mode 100644 docs/guide/operations.md create mode 100644 docs/guide/plugins.md create mode 100644 docs/guide/queries.md create mode 100644 docs/guide/relations.md create mode 100644 docs/guide/writes.md create mode 100644 docs/migration.md create mode 100644 docs/parity.md create mode 100644 docs/reference/bulk.md create mode 100644 docs/reference/errors.md create mode 100644 docs/reference/expr.md create mode 100644 docs/reference/hooks.md create mode 100644 docs/reference/indexes.md create mode 100644 docs/reference/manager.md create mode 100644 docs/reference/markers.md create mode 100644 docs/reference/model.md create mode 100644 docs/reference/query.md create mode 100644 docs/reference/refs.md create mode 100644 docs/reference/registry.md create mode 100644 docs/reference/relations.md create mode 100644 docs/reference/streams.md create mode 100644 docs/reference/updates.md create mode 100644 docs/reference/web_api.md create mode 100644 docs/reference/web_backend.md create mode 100644 docs/reference/web_errors.md create mode 100644 docs/reference/web_filters.md create mode 100644 docs/reference/web_openapi.md create mode 100644 docs/reference/web_policy.md create mode 100644 docs/reference/web_resource.md create mode 100644 docs/reference/web_testing.md create mode 100644 docs/web/plugs.md create mode 100644 docs/web/resources.md create mode 100644 packages/mongospecs-web/tests/test_docs_examples.py diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 0000000..aa1eb6a --- /dev/null +++ b/bench/README.md @@ -0,0 +1,36 @@ +# mongospecs-bench + +Benchmarks, stress scenarios and an end-to-end tracing check for mongospecs, all against real +MongoDB. This package is a workspace member and is never published. + +``` +make up # replica set + sharded cluster +uv run mongospecs-bench run # workloads B1-B8 +uv run mongospecs-bench run B3 --topology sharded +uv run mongospecs-bench stress # scenarios X1-X7 +uv run mongospecs-bench report # markdown summary of the newest results + +docker compose -f docker/compose.rustrak.yaml up -d +uv run mongospecs-bench trace # Sentry SDK -> rustrak, end to end + +cd bench/mongoose && npm install && npm run bench # Mongoose comparator +``` + +Results are JSON files under `results//`, each with the MongoDB version, topology, +Python version and machine. `--quick` uses tiny sizes to prove the harness runs; do not draw +conclusions from quick numbers. + +Absolute times depend on the machine. The comparisons that travel are ratios: mongospecs over raw +`pymongo`, and Mongoose over the raw Node driver. + +| ID | Measures | +|---|---| +| B1 | Turning raw documents into models, per backend, next to the driver's BSON decode | +| B2 | `find()` end to end for 1, 100 and 10,000 documents in raw, lean and tracked modes | +| B3 | `include()` by `$lookup` vs batched `$in` across fan-outs, on both topologies | +| B4 | Inserts, atomic patches, delta saves and upserts | +| B5 | Offset vs cursor pagination at depth; total by count vs `$facet` | +| B6 | Per-operation cost of plugins | +| B7 | HTTP request to response through each plug and model backend | +| B8 | Time and memory of change tracking; size of a delta save | +| X1-X7 | Pool pressure, transaction contention, optimistic-lock storm, cancellation, 16MB results, change stream across a step-down, threads with their own loops | diff --git a/bench/mongoose/.gitignore b/bench/mongoose/.gitignore new file mode 100644 index 0000000..504afef --- /dev/null +++ b/bench/mongoose/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json diff --git a/bench/mongoose/bench.mjs b/bench/mongoose/bench.mjs new file mode 100644 index 0000000..e1748cc --- /dev/null +++ b/bench/mongoose/bench.mjs @@ -0,0 +1,228 @@ +// Mongoose comparator (plan node E2). +// +// Runs the equivalents of workloads B1, B2, B4 and B8 with Mongoose against the +// same MongoDB and the same document shapes, and reports Mongoose's overhead +// over the raw Node driver. Absolute times are not comparable across runtimes; +// the overhead ratios are what is set next to mongospecs' ratios over pymongo. +// +// cd bench/mongoose && npm install && npm run bench [-- --quick] + +import { execSync } from 'node:child_process'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import mongoose from 'mongoose'; + +const QUICK = process.argv.includes('--quick'); +const URI = process.env.MONGOSPECS_TEST_URI || 'mongodb://localhost:27117/?replicaSet=rs0'; +const REPEAT = QUICK ? 3 : 15; +const here = dirname(fileURLToPath(import.meta.url)); + +function rawDocument(shape, n) { + const when = new Date(Date.UTC(2024, 0, 1) + n * 60000); + if (shape === 'small') { + return { n, name: `name-${n}`, score: n * 1.5, active: n % 2 === 0, created: when }; + } + return { + n, + customer: `customer-${n}`, + address: { street: `${n} Main St`, city: 'Springfield', zip: String(n % 100000).padStart(5, '0') }, + lines: Array.from({ length: 8 }, (_, i) => ({ sku: `sku-${n}-${i}`, qty: i + 1, price: 9.99 + i })), + tags: Array.from({ length: 5 }, (_, i) => `tag-${(n + i) % 7}`), + meta: { source: 'bench', batch: String(Math.floor(n / 100)) }, + placed: when, + }; +} + +const Small = mongoose.model( + 'Small', + new mongoose.Schema( + { n: Number, name: String, score: Number, active: Boolean, created: Date }, + { collection: 'bench_small', versionKey: false }, + ), +); +const Nested = mongoose.model( + 'Nested', + new mongoose.Schema( + { + n: Number, + customer: String, + address: new mongoose.Schema({ street: String, city: String, zip: String }, { _id: false }), + lines: [new mongoose.Schema({ sku: String, qty: Number, price: Number }, { _id: false })], + tags: [String], + meta: { type: Map, of: String }, + placed: Date, + }, + { collection: 'bench_nested', versionKey: false }, + ), +); +const Plain = mongoose.model( + 'Plain', + new mongoose.Schema({ n: Number, name: String }, { collection: 'bench_plain', versionKey: false }), +); +const Managed = mongoose.model( + 'Managed', + new mongoose.Schema( + { n: Number, name: String }, + { collection: 'bench_managed', timestamps: true, optimisticConcurrency: true }, + ), +); +const MODELS = { small: Small, nested: Nested }; + +function median(values) { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; +} + +async function measure(fn, { setup } = {}) { + for (let i = 0; i < 2; i++) { + if (setup) await setup(); + await fn(); + } + const samples = []; + for (let i = 0; i < REPEAT; i++) { + if (setup) await setup(); + const start = process.hrtime.bigint(); + await fn(); + samples.push(Number(process.hrtime.bigint() - start) / 1e6); + } + return median(samples); +} + +const round = (value, digits = 3) => Number(value.toFixed(digits)); +const ratio = (value, base) => (base ? round(value / base, 2) : 0); +const rows = []; + +async function materialise() { + const n = QUICK ? 300 : 10000; + for (const [shape, Model] of Object.entries(MODELS)) { + const docs = Array.from({ length: n }, (_, i) => ({ _id: new mongoose.Types.ObjectId(), ...rawDocument(shape, i) })); + const encoded = docs.map((d) => mongoose.mongo.BSON.serialize(d)); + const base = await measure(() => encoded.map((b) => mongoose.mongo.BSON.deserialize(b))); + rows.push({ workload: 'B1', shape, case: 'driver bson->object', n, us_per_doc: round((base * 1000) / n, 2), vs_driver: 1 }); + const hydrated = await measure(() => docs.map((d) => Model.hydrate(d))); + rows.push({ workload: 'B1', shape, case: 'Model.hydrate', n, us_per_doc: round((hydrated * 1000) / n, 2), vs_driver: ratio(hydrated, base) }); + } +} + +async function finds() { + const total = QUICK ? 300 : 10000; + for (const [shape, Model] of Object.entries(MODELS)) { + await Model.collection.deleteMany({}); + await Model.collection.insertMany(Array.from({ length: total }, (_, i) => rawDocument(shape, i))); + for (const n of [1, 100, 10000].filter((size) => size <= total)) { + const raw = await measure(() => Model.collection.find({}, { limit: n }).toArray()); + const lean = await measure(() => Model.find().limit(n).lean()); + const full = await measure(() => Model.find().limit(n)); + rows.push({ workload: 'B2', shape, case: 'driver objects', n, median_ms: round(raw), vs_driver: 1 }); + rows.push({ workload: 'B2', shape, case: 'mongoose lean()', n, median_ms: round(lean), vs_driver: ratio(lean, raw) }); + rows.push({ workload: 'B2', shape, case: 'mongoose documents', n, median_ms: round(full), vs_driver: ratio(full, raw) }); + } + } +} + +async function writes() { + const n = QUICK ? 50 : 1000; + const ops = QUICK ? 20 : 200; + const reset = async () => { + await Plain.collection.deleteMany({}); + await Managed.collection.deleteMany({}); + }; + const make = () => Array.from({ length: n }, (_, i) => ({ n: i, name: `n${i}` })); + const rawInsert = await measure(() => Plain.collection.insertMany(make()), { setup: reset }); + const plainInsert = await measure(() => Plain.insertMany(make()), { setup: reset }); + const managedInsert = await measure(() => Managed.insertMany(make()), { setup: reset }); + rows.push({ workload: 'B4', op: `insert ${n}`, case: 'driver insertMany', median_ms: round(rawInsert), vs_driver: 1 }); + rows.push({ workload: 'B4', op: `insert ${n}`, case: 'Model.insertMany (no plugins)', median_ms: round(plainInsert), vs_driver: ratio(plainInsert, rawInsert) }); + rows.push({ workload: 'B4', op: `insert ${n}`, case: 'Model.insertMany (timestamps+versioning)', median_ms: round(managedInsert), vs_driver: ratio(managedInsert, rawInsert) }); + + await reset(); + const docs = await Plain.insertMany(Array.from({ length: ops }, (_, i) => ({ n: i, name: `n${i}` }))); + const ids = docs.map((d) => d._id); + const rawPatch = await measure(async () => { + for (const id of ids) await Plain.collection.updateOne({ _id: id }, { $inc: { n: 1 } }); + }); + const odmPatch = await measure(async () => { + for (const id of ids) await Plain.findByIdAndUpdate(id, { $inc: { n: 1 } }, { new: true }); + }); + const odmSave = await measure(async () => { + for (const doc of docs) { + doc.n += 1; + await doc.save(); + } + }); + rows.push({ workload: 'B4', op: `patch x${ops}`, case: 'driver updateOne', median_ms: round(rawPatch), vs_driver: 1 }); + rows.push({ workload: 'B4', op: `patch x${ops}`, case: 'findByIdAndUpdate (returns doc)', median_ms: round(odmPatch), vs_driver: ratio(odmPatch, rawPatch) }); + rows.push({ workload: 'B4', op: `patch x${ops}`, case: 'doc.save() delta', median_ms: round(odmSave), vs_driver: ratio(odmSave, rawPatch) }); + await reset(); +} + +async function tracking() { + const n = QUICK ? 200 : 5000; + await Nested.collection.deleteMany({}); + await Nested.collection.insertMany(Array.from({ length: n }, (_, i) => rawDocument('nested', i))); + const lean = await measure(() => Nested.find().lean()); + const full = await measure(() => Nested.find()); + const heap = async (query) => { + globalThis.gc?.(); + const before = process.memoryUsage().heapUsed; + const held = await query(); + globalThis.gc?.(); + const used = (process.memoryUsage().heapUsed - before) / held.length; + return Math.round(used); + }; + const leanBytes = await heap(() => Nested.find().lean()); + const fullBytes = await heap(() => Nested.find()); + rows.push({ workload: 'B8', mode: 'lean()', n, median_ms: round(lean, 2), vs_lean: 1, bytes_per_doc: leanBytes }); + rows.push({ workload: 'B8', mode: 'documents (tracked)', n, median_ms: round(full, 2), vs_lean: ratio(full, lean), bytes_per_doc: fullBytes }); + await Nested.collection.deleteMany({}); +} + +function git(args) { + try { + return execSync(`git ${args}`, { cwd: here, encoding: 'utf8' }).trim(); + } catch { + return 'unknown'; + } +} + +const database = `ms_bench_mongoose_${Date.now().toString(36)}`; +await mongoose.connect(URI, { dbName: database }); +try { + await materialise(); + await finds(); + await writes(); + await tracking(); + const info = await mongoose.connection.db.admin().serverInfo(); + const meta = { + git_sha: git('rev-parse --short HEAD'), + timestamp: new Date().toISOString(), + topology: 'rs', + quick: QUICK, + mongodb: info.version, + node: process.version, + versions: { mongoose: mongoose.version, driver: mongoose.mongo.BSON ? 'bundled with mongoose' : 'unknown' }, + platform: `${process.platform}-${process.arch}`, + }; + const folder = join(here, '..', 'results', meta.git_sha); + mkdirSync(folder, { recursive: true }); + const path = join(folder, `mongoose${QUICK ? '-quick' : ''}.json`); + const payload = { + meta, + workload: 'E2', + title: 'Mongoose comparator', + question: "What is Mongoose's overhead over the raw Node driver for the same workloads?", + rows, + notes: [ + 'Ratios are against the raw Node driver in the same process. Compare them with mongospecs ratios against pymongo, not absolute times.', + ], + conclusion: '', + }; + writeFileSync(path, JSON.stringify(payload, null, 2) + '\n'); + console.table(rows); + console.log(`-> ${path}`); +} finally { + await mongoose.connection.dropDatabase(); + await mongoose.disconnect(); +} diff --git a/bench/mongoose/package.json b/bench/mongoose/package.json new file mode 100644 index 0000000..e4aa52c --- /dev/null +++ b/bench/mongoose/package.json @@ -0,0 +1,12 @@ +{ + "name": "mongospecs-bench-mongoose", + "private": true, + "type": "module", + "description": "Mongoose comparator for the mongospecs benchmark track. Not published.", + "scripts": { + "bench": "node --expose-gc bench.mjs" + }, + "dependencies": { + "mongoose": "^8.9.0" + } +} diff --git a/bench/pyproject.toml b/bench/pyproject.toml index 28e1295..ec1478c 100644 --- a/bench/pyproject.toml +++ b/bench/pyproject.toml @@ -8,7 +8,7 @@ version = "0.0.0" description = "Benchmarks and stress scenarios for mongospecs against real MongoDB. Not published." requires-python = ">=3.11" classifiers = ["Private :: Do Not Upload"] -dependencies = ["mongospecs[all,sentry]", "mongospecs-web[all]", "httpx>=0.27"] +dependencies = ["mongospecs[all,sentry]", "mongospecs-web[all]", "httpx>=0.27", "uvicorn>=0.30"] [project.scripts] mongospecs-bench = "mongospecs_bench.cli:main" diff --git a/bench/results/4f3b2c1/B1.json b/bench/results/4f3b2c1/B1.json new file mode 100644 index 0000000..978b2f0 --- /dev/null +++ b/bench/results/4f3b2c1/B1.json @@ -0,0 +1,271 @@ +{ + "meta": { + "git_sha": "4f3b2c1", + "git_dirty": false, + "timestamp": "2026-10-13T04:01:56+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B1", + "title": "Materialise documents", + "question": "What does turning a raw document into a model cost, per backend, and how does it compare with the driver's own BSON decoding?", + "rows": [ + { + "shape": "small", + "backend": "driver", + "mode": "bson->dict", + "n": 10000, + "us_per_doc": 1.07, + "vs_bson": 1.0 + }, + { + "shape": "small", + "backend": "msgspec", + "mode": "validated", + "n": 10000, + "us_per_doc": 1.21, + "vs_bson": 1.13 + }, + { + "shape": "small", + "backend": "msgspec", + "mode": "trusted", + "n": 10000, + "us_per_doc": 1.17, + "vs_bson": 1.09 + }, + { + "shape": "small", + "backend": "msgspec", + "mode": "tracked", + "n": 10000, + "us_per_doc": 3.12, + "vs_bson": 2.93 + }, + { + "shape": "small", + "backend": "pydantic", + "mode": "validated", + "n": 10000, + "us_per_doc": 3.19, + "vs_bson": 2.99 + }, + { + "shape": "small", + "backend": "pydantic", + "mode": "trusted", + "n": 10000, + "us_per_doc": 2.13, + "vs_bson": 2.0 + }, + { + "shape": "small", + "backend": "pydantic", + "mode": "tracked", + "n": 10000, + "us_per_doc": 4.51, + "vs_bson": 4.22 + }, + { + "shape": "small", + "backend": "attrs", + "mode": "validated", + "n": 10000, + "us_per_doc": 1.93, + "vs_bson": 1.81 + }, + { + "shape": "small", + "backend": "attrs", + "mode": "trusted", + "n": 10000, + "us_per_doc": 1.92, + "vs_bson": 1.8 + }, + { + "shape": "small", + "backend": "attrs", + "mode": "tracked", + "n": 10000, + "us_per_doc": 5.76, + "vs_bson": 5.39 + }, + { + "shape": "wide", + "backend": "driver", + "mode": "bson->dict", + "n": 10000, + "us_per_doc": 4.19, + "vs_bson": 1.0 + }, + { + "shape": "wide", + "backend": "msgspec", + "mode": "validated", + "n": 10000, + "us_per_doc": 1.96, + "vs_bson": 0.47 + }, + { + "shape": "wide", + "backend": "msgspec", + "mode": "trusted", + "n": 10000, + "us_per_doc": 1.98, + "vs_bson": 0.47 + }, + { + "shape": "wide", + "backend": "msgspec", + "mode": "tracked", + "n": 10000, + "us_per_doc": 9.56, + "vs_bson": 2.28 + }, + { + "shape": "wide", + "backend": "pydantic", + "mode": "validated", + "n": 10000, + "us_per_doc": 5.11, + "vs_bson": 1.22 + }, + { + "shape": "wide", + "backend": "pydantic", + "mode": "trusted", + "n": 10000, + "us_per_doc": 17.36, + "vs_bson": 4.14 + }, + { + "shape": "wide", + "backend": "pydantic", + "mode": "tracked", + "n": 10000, + "us_per_doc": 11.22, + "vs_bson": 2.67 + }, + { + "shape": "wide", + "backend": "attrs", + "mode": "validated", + "n": 10000, + "us_per_doc": 6.0, + "vs_bson": 1.43 + }, + { + "shape": "wide", + "backend": "attrs", + "mode": "trusted", + "n": 10000, + "us_per_doc": 5.67, + "vs_bson": 1.35 + }, + { + "shape": "wide", + "backend": "attrs", + "mode": "tracked", + "n": 10000, + "us_per_doc": 14.2, + "vs_bson": 3.38 + }, + { + "shape": "nested", + "backend": "driver", + "mode": "bson->dict", + "n": 10000, + "us_per_doc": 7.94, + "vs_bson": 1.0 + }, + { + "shape": "nested", + "backend": "msgspec", + "mode": "validated", + "n": 10000, + "us_per_doc": 4.3, + "vs_bson": 0.54 + }, + { + "shape": "nested", + "backend": "msgspec", + "mode": "trusted", + "n": 10000, + "us_per_doc": 5.98, + "vs_bson": 0.75 + }, + { + "shape": "nested", + "backend": "msgspec", + "mode": "tracked", + "n": 10000, + "us_per_doc": 19.52, + "vs_bson": 2.46 + }, + { + "shape": "nested", + "backend": "pydantic", + "mode": "validated", + "n": 10000, + "us_per_doc": 16.06, + "vs_bson": 2.02 + }, + { + "shape": "nested", + "backend": "pydantic", + "mode": "trusted", + "n": 10000, + "us_per_doc": 14.18, + "vs_bson": 1.78 + }, + { + "shape": "nested", + "backend": "pydantic", + "mode": "tracked", + "n": 10000, + "us_per_doc": 15.61, + "vs_bson": 1.96 + }, + { + "shape": "nested", + "backend": "attrs", + "mode": "validated", + "n": 10000, + "us_per_doc": 6.71, + "vs_bson": 0.84 + }, + { + "shape": "nested", + "backend": "attrs", + "mode": "trusted", + "n": 10000, + "us_per_doc": 6.5, + "vs_bson": 0.82 + }, + { + "shape": "nested", + "backend": "attrs", + "mode": "tracked", + "n": 10000, + "us_per_doc": 21.46, + "vs_bson": 2.7 + } + ], + "notes": [ + "'tracked' is decode plus the encode that builds the change-tracking snapshot.", + "'trusted' only differs for flat pydantic models; msgspec and attrs always convert." + ], + "conclusion": "" +} diff --git a/bench/results/4f3b2c1/B2.json b/bench/results/4f3b2c1/B2.json new file mode 100644 index 0000000..6c68a53 --- /dev/null +++ b/bench/results/4f3b2c1/B2.json @@ -0,0 +1,508 @@ +{ + "meta": { + "git_sha": "4f3b2c1", + "git_dirty": false, + "timestamp": "2026-10-13T04:01:56+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B2", + "title": "find() end to end", + "question": "How much does the ODM add on top of the raw driver for reads of 1, 100 and 10,000 documents?", + "rows": [ + { + "shape": "small", + "backend": "pymongo", + "case": "driver dicts", + "n": 1, + "median_ms": 0.41, + "vs_driver": 1.0 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm raw()", + "n": 1, + "median_ms": 0.551, + "vs_driver": 1.35 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm lean()", + "n": 1, + "median_ms": 0.536, + "vs_driver": 1.31 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm tracked", + "n": 1, + "median_ms": 0.444, + "vs_driver": 1.08 + }, + { + "shape": "small", + "backend": "pymongo", + "case": "driver dicts", + "n": 100, + "median_ms": 0.734, + "vs_driver": 1.0 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm raw()", + "n": 100, + "median_ms": 0.714, + "vs_driver": 0.97 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm lean()", + "n": 100, + "median_ms": 0.782, + "vs_driver": 1.07 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm tracked", + "n": 100, + "median_ms": 1.113, + "vs_driver": 1.52 + }, + { + "shape": "small", + "backend": "pymongo", + "case": "driver dicts", + "n": 10000, + "median_ms": 24.777, + "vs_driver": 1.0 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm raw()", + "n": 10000, + "median_ms": 36.537, + "vs_driver": 1.47 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm lean()", + "n": 10000, + "median_ms": 49.699, + "vs_driver": 2.01 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm tracked", + "n": 10000, + "median_ms": 88.151, + "vs_driver": 3.56 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm raw()", + "n": 1, + "median_ms": 0.684, + "vs_driver": 1.03 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm lean()", + "n": 1, + "median_ms": 0.69, + "vs_driver": 1.04 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm tracked", + "n": 1, + "median_ms": 0.85, + "vs_driver": 1.28 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm raw()", + "n": 100, + "median_ms": 0.861, + "vs_driver": 1.41 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm lean()", + "n": 100, + "median_ms": 1.126, + "vs_driver": 1.85 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm tracked", + "n": 100, + "median_ms": 1.723, + "vs_driver": 2.83 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm raw()", + "n": 10000, + "median_ms": 33.949, + "vs_driver": 1.26 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm lean()", + "n": 10000, + "median_ms": 57.318, + "vs_driver": 2.13 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm tracked", + "n": 10000, + "median_ms": 106.646, + "vs_driver": 3.96 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm raw()", + "n": 1, + "median_ms": 0.849, + "vs_driver": 1.0 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm lean()", + "n": 1, + "median_ms": 0.756, + "vs_driver": 0.89 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm tracked", + "n": 1, + "median_ms": 0.867, + "vs_driver": 1.02 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm raw()", + "n": 100, + "median_ms": 1.287, + "vs_driver": 1.41 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm lean()", + "n": 100, + "median_ms": 1.537, + "vs_driver": 1.68 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm tracked", + "n": 100, + "median_ms": 2.097, + "vs_driver": 2.3 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm raw()", + "n": 10000, + "median_ms": 37.138, + "vs_driver": 1.38 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm lean()", + "n": 10000, + "median_ms": 59.104, + "vs_driver": 2.19 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm tracked", + "n": 10000, + "median_ms": 100.8, + "vs_driver": 3.74 + }, + { + "shape": "nested", + "backend": "pymongo", + "case": "driver dicts", + "n": 1, + "median_ms": 0.408, + "vs_driver": 1.0 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm raw()", + "n": 1, + "median_ms": 0.452, + "vs_driver": 1.11 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm lean()", + "n": 1, + "median_ms": 0.459, + "vs_driver": 1.13 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm tracked", + "n": 1, + "median_ms": 0.48, + "vs_driver": 1.18 + }, + { + "shape": "nested", + "backend": "pymongo", + "case": "driver dicts", + "n": 100, + "median_ms": 1.16, + "vs_driver": 1.0 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm raw()", + "n": 100, + "median_ms": 1.161, + "vs_driver": 1.0 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm lean()", + "n": 100, + "median_ms": 1.441, + "vs_driver": 1.24 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm tracked", + "n": 100, + "median_ms": 3.464, + "vs_driver": 2.99 + }, + { + "shape": "nested", + "backend": "pymongo", + "case": "driver dicts", + "n": 10000, + "median_ms": 100.05, + "vs_driver": 1.0 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm raw()", + "n": 10000, + "median_ms": 104.699, + "vs_driver": 1.05 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm lean()", + "n": 10000, + "median_ms": 155.254, + "vs_driver": 1.55 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm tracked", + "n": 10000, + "median_ms": 427.674, + "vs_driver": 4.27 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm raw()", + "n": 1, + "median_ms": 0.836, + "vs_driver": 1.11 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm lean()", + "n": 1, + "median_ms": 0.905, + "vs_driver": 1.21 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm tracked", + "n": 1, + "median_ms": 0.962, + "vs_driver": 1.28 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm raw()", + "n": 100, + "median_ms": 1.624, + "vs_driver": 0.84 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm lean()", + "n": 100, + "median_ms": 2.735, + "vs_driver": 1.41 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm tracked", + "n": 100, + "median_ms": 4.617, + "vs_driver": 2.38 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm raw()", + "n": 10000, + "median_ms": 106.912, + "vs_driver": 0.98 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm lean()", + "n": 10000, + "median_ms": 261.247, + "vs_driver": 2.4 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm tracked", + "n": 10000, + "median_ms": 388.983, + "vs_driver": 3.58 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm raw()", + "n": 1, + "median_ms": 0.856, + "vs_driver": 1.01 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm lean()", + "n": 1, + "median_ms": 0.843, + "vs_driver": 0.99 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm tracked", + "n": 1, + "median_ms": 0.881, + "vs_driver": 1.04 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm raw()", + "n": 100, + "median_ms": 1.795, + "vs_driver": 1.02 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm lean()", + "n": 100, + "median_ms": 2.326, + "vs_driver": 1.33 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm tracked", + "n": 100, + "median_ms": 4.143, + "vs_driver": 2.36 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm raw()", + "n": 10000, + "median_ms": 92.473, + "vs_driver": 1.02 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm lean()", + "n": 10000, + "median_ms": 158.192, + "vs_driver": 1.75 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm tracked", + "n": 10000, + "median_ms": 359.47, + "vs_driver": 3.97 + } + ], + "notes": [], + "conclusion": "" +} diff --git a/bench/results/4f3b2c1/B3.json b/bench/results/4f3b2c1/B3.json new file mode 100644 index 0000000..49ad0d6 --- /dev/null +++ b/bench/results/4f3b2c1/B3.json @@ -0,0 +1,70 @@ +{ + "meta": { + "git_sha": "4f3b2c1", + "git_dirty": false, + "timestamp": "2026-10-13T04:01:56+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B3", + "title": "include(): $lookup vs batched $in", + "question": "Which strategy should include(strategy='auto') use, across fan-outs and topologies?", + "rows": [ + { + "relation": "list, distinct children", + "fan_out": 1, + "parents": 400, + "batch_ms": 6.716, + "lookup_ms": 10.275, + "lookup_vs_batch": 1.53 + }, + { + "relation": "list, distinct children", + "fan_out": 10, + "parents": 200, + "batch_ms": 26.252, + "lookup_ms": 20.198, + "lookup_vs_batch": 0.77 + }, + { + "relation": "list, distinct children", + "fan_out": 100, + "parents": 50, + "batch_ms": 97.258, + "lookup_ms": 40.653, + "lookup_vs_batch": 0.42 + }, + { + "relation": "list, distinct children", + "fan_out": 1000, + "parents": 10, + "batch_ms": 101.378, + "lookup_ms": 82.163, + "lookup_vs_batch": 0.81 + }, + { + "relation": "to-one, 20 shared targets", + "fan_out": 1, + "parents": 2000, + "batch_ms": 20.164, + "lookup_ms": 49.826, + "lookup_vs_batch": 2.47 + } + ], + "notes": [ + "Strategy choice does not depend on the model backend; measured with msgspec, lean reads." + ], + "conclusion": "$lookup was faster than batched $in in 3 of 5 cases on rs (lookup/batch ratios: [1.53, 0.77, 0.42, 0.81, 2.47]). Current default: batch." +} diff --git a/bench/results/4f3b2c1/B4.json b/bench/results/4f3b2c1/B4.json new file mode 100644 index 0000000..b765df9 --- /dev/null +++ b/bench/results/4f3b2c1/B4.json @@ -0,0 +1,198 @@ +{ + "meta": { + "git_sha": "4f3b2c1", + "git_dirty": false, + "timestamp": "2026-10-13T04:01:56+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B4", + "title": "Writes", + "question": "What does the ODM add to inserts, atomic patches, saves and upserts?", + "rows": [ + { + "backend": "msgspec", + "op": "insert 1000", + "case": "driver insert_many", + "median_ms": 8.739, + "vs_driver": 1.0 + }, + { + "backend": "msgspec", + "op": "insert 1000", + "case": "insert_many (no plugins)", + "median_ms": 11.722, + "vs_driver": 1.34 + }, + { + "backend": "msgspec", + "op": "insert 1000", + "case": "insert_many (timestamps+revision)", + "median_ms": 14.774, + "vs_driver": 1.69 + }, + { + "backend": "msgspec", + "op": "patch x200", + "case": "driver update_one", + "median_ms": 648.9, + "vs_driver": 1.0 + }, + { + "backend": "msgspec", + "op": "patch x200", + "case": "Model.update (returns doc)", + "median_ms": 679.692, + "vs_driver": 1.05 + }, + { + "backend": "msgspec", + "op": "patch x200", + "case": "doc.save() delta", + "median_ms": 605.333, + "vs_driver": 0.93 + }, + { + "backend": "msgspec", + "op": "upsert x200", + "case": "driver update_one(upsert)", + "median_ms": 1063.307, + "vs_driver": 1.0 + }, + { + "backend": "msgspec", + "op": "upsert x200", + "case": "Model.upsert (returns doc)", + "median_ms": 992.707, + "vs_driver": 0.93 + }, + { + "backend": "pydantic", + "op": "insert 1000", + "case": "driver insert_many", + "median_ms": 10.299, + "vs_driver": 1.0 + }, + { + "backend": "pydantic", + "op": "insert 1000", + "case": "insert_many (no plugins)", + "median_ms": 13.502, + "vs_driver": 1.31 + }, + { + "backend": "pydantic", + "op": "insert 1000", + "case": "insert_many (timestamps+revision)", + "median_ms": 18.901, + "vs_driver": 1.84 + }, + { + "backend": "pydantic", + "op": "patch x200", + "case": "driver update_one", + "median_ms": 586.275, + "vs_driver": 1.0 + }, + { + "backend": "pydantic", + "op": "patch x200", + "case": "Model.update (returns doc)", + "median_ms": 690.681, + "vs_driver": 1.18 + }, + { + "backend": "pydantic", + "op": "patch x200", + "case": "doc.save() delta", + "median_ms": 598.93, + "vs_driver": 1.02 + }, + { + "backend": "pydantic", + "op": "upsert x200", + "case": "driver update_one(upsert)", + "median_ms": 689.839, + "vs_driver": 1.0 + }, + { + "backend": "pydantic", + "op": "upsert x200", + "case": "Model.upsert (returns doc)", + "median_ms": 935.368, + "vs_driver": 1.36 + }, + { + "backend": "attrs", + "op": "insert 1000", + "case": "driver insert_many", + "median_ms": 10.134, + "vs_driver": 1.0 + }, + { + "backend": "attrs", + "op": "insert 1000", + "case": "insert_many (no plugins)", + "median_ms": 13.516, + "vs_driver": 1.33 + }, + { + "backend": "attrs", + "op": "insert 1000", + "case": "insert_many (timestamps+revision)", + "median_ms": 14.908, + "vs_driver": 1.47 + }, + { + "backend": "attrs", + "op": "patch x200", + "case": "driver update_one", + "median_ms": 602.354, + "vs_driver": 1.0 + }, + { + "backend": "attrs", + "op": "patch x200", + "case": "Model.update (returns doc)", + "median_ms": 584.293, + "vs_driver": 0.97 + }, + { + "backend": "attrs", + "op": "patch x200", + "case": "doc.save() delta", + "median_ms": 644.737, + "vs_driver": 1.07 + }, + { + "backend": "attrs", + "op": "upsert x200", + "case": "driver update_one(upsert)", + "median_ms": 833.59, + "vs_driver": 1.0 + }, + { + "backend": "attrs", + "op": "upsert x200", + "case": "Model.upsert (returns doc)", + "median_ms": 818.948, + "vs_driver": 0.98 + } + ], + "notes": [ + "Model.update and Model.upsert use findAndModify to return the document; the driver baseline does not." + ], + "conclusion": "" +} diff --git a/bench/results/4f3b2c1/B5.json b/bench/results/4f3b2c1/B5.json new file mode 100644 index 0000000..c3c66dc --- /dev/null +++ b/bench/results/4f3b2c1/B5.json @@ -0,0 +1,73 @@ +{ + "meta": { + "git_sha": "4f3b2c1", + "git_dirty": false, + "timestamp": "2026-10-13T04:01:56+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B5", + "title": "Pagination", + "question": "How do offset and cursor pagination compare at depth, and should a total come from $facet or a count?", + "rows": [ + { + "page": 1, + "skip": 0, + "offset_ms": 0.57, + "cursor_ms": 19.848, + "offset_vs_cursor": 0.03 + }, + { + "page": 100, + "skip": 1980, + "offset_ms": 4.242, + "cursor_ms": 36.729, + "offset_vs_cursor": 0.12 + }, + { + "page": 1250, + "skip": 24980, + "offset_ms": 6.524, + "cursor_ms": 18.783, + "offset_vs_cursor": 0.35 + }, + { + "page": 2499, + "skip": 49960, + "offset_ms": 11.733, + "cursor_ms": 0.735, + "offset_vs_cursor": 15.96 + }, + { + "page": "total: concurrent count", + "skip": 0, + "offset_ms": 5.97, + "cursor_ms": "", + "offset_vs_cursor": "" + }, + { + "page": "total: $facet (raw driver)", + "skip": 0, + "offset_ms": 28.805, + "cursor_ms": "", + "offset_vs_cursor": "" + } + ], + "notes": [ + "Collection of 50000 documents, page size 20, sorted on an indexed field.", + "$facet returns the page and the total as one document, so the page is bound by the 16MB limit." + ], + "conclusion": "" +} diff --git a/bench/results/4f3b2c1/B6.json b/bench/results/4f3b2c1/B6.json new file mode 100644 index 0000000..c52665a --- /dev/null +++ b/bench/results/4f3b2c1/B6.json @@ -0,0 +1,90 @@ +{ + "meta": { + "git_sha": "4f3b2c1", + "git_dirty": false, + "timestamp": "2026-10-13T04:01:56+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B6", + "title": "Plugin overhead", + "question": "What does each active plugin cost per operation?", + "rows": [ + { + "plugins": "0 no-op", + "op": "insert", + "us_per_op": 2574.7, + "vs_no_plugins": 1.0 + }, + { + "plugins": "0 no-op", + "op": "save", + "us_per_op": 3124.7, + "vs_no_plugins": 1.0 + }, + { + "plugins": "0 no-op", + "op": "get", + "us_per_op": 421.8, + "vs_no_plugins": 1.0 + }, + { + "plugins": "3 no-op", + "op": "insert", + "us_per_op": 2703.9, + "vs_no_plugins": 1.05 + }, + { + "plugins": "3 no-op", + "op": "save", + "us_per_op": 3619.3, + "vs_no_plugins": 1.16 + }, + { + "plugins": "3 no-op", + "op": "get", + "us_per_op": 455.2, + "vs_no_plugins": 1.08 + }, + { + "plugins": "10 no-op", + "op": "insert", + "us_per_op": 2724.2, + "vs_no_plugins": 1.06 + }, + { + "plugins": "10 no-op", + "op": "save", + "us_per_op": 3170.5, + "vs_no_plugins": 1.01 + }, + { + "plugins": "10 no-op", + "op": "get", + "us_per_op": 423.6, + "vs_no_plugins": 1.0 + }, + { + "plugins": "timestamps+revision", + "op": "insert", + "us_per_op": 2646.1, + "vs_no_plugins": 1.03 + } + ], + "notes": [ + "Each operation is one round trip to MongoDB, which dominates; plugin calls are in-process." + ], + "conclusion": "" +} diff --git a/bench/src/mongospecs_bench/cli.py b/bench/src/mongospecs_bench/cli.py index 1b47e40..9f9016c 100644 --- a/bench/src/mongospecs_bench/cli.py +++ b/bench/src/mongospecs_bench/cli.py @@ -4,6 +4,7 @@ mongospecs-bench run B1 B2 # workloads (default: all) mongospecs-bench run B3 --topology sharded + mongospecs-bench run LOAD # the storefront as a real server under HTTP load mongospecs-bench stress # X1-X7 mongospecs-bench trace # Sentry SDK -> rustrak, end to end mongospecs-bench report # markdown summary of the newest results @@ -83,7 +84,7 @@ def main(argv: list[str] | None = None) -> int: prog="mongospecs-bench", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) sub = parser.add_subparsers(dest="command", required=True) - for name, choices in (("run", WORKLOADS), ("stress", SCENARIOS)): + for name, choices in (("run", [*WORKLOADS, "LOAD"]), ("stress", SCENARIOS)): cmd = sub.add_parser(name) cmd.add_argument("names", nargs="*", choices=[*choices, []], default=[]) cmd.add_argument("--topology", choices=("rs", "sharded"), default="rs") @@ -101,9 +102,13 @@ def main(argv: list[str] | None = None) -> int: if args.command == "report": print(report(RESULTS / args.folder if args.folder else None)) return 0 + from mongospecs_bench.load import load + + WORKLOADS = {**WORKLOADS, "LOAD": load} registry = WORKLOADS if args.command == "run" else SCENARIOS settings = Settings(args.topology, args.quick, tuple(args.backend or ("msgspec", "pydantic", "attrs"))) - return asyncio.run(_run(list(args.names) or list(registry), registry, settings)) + default = [name for name in registry if name != "LOAD"] + return asyncio.run(_run(list(args.names) or default, registry, settings)) if __name__ == "__main__": diff --git a/bench/src/mongospecs_bench/load.py b/bench/src/mongospecs_bench/load.py new file mode 100644 index 0000000..6e4a9ef --- /dev/null +++ b/bench/src/mongospecs_bench/load.py @@ -0,0 +1,214 @@ +"""HTTP load benchmark against the storefront running as a real server process. + +Virtual users mix browsing, product detail with reviews, and placing orders +over real TCP connections. The same mix is run against a hand-written +pymongo + Starlette implementation of the same endpoints, so the result shows +what mongospecs and mongospecs-web cost in a whole request, not in isolation. + +After every run the stock invariant is checked against the database: a +benchmark that loses or oversells units is a failed run, whatever its speed. +""" + +from __future__ import annotations + +import asyncio +import random +import statistics +import time +import typing as t +import uuid + +import httpx + +from mongospecs import Registry +from mongospecs_bench.harness import Result, Settings, uri_for +from mongospecs_bench.storefront import models as shop_models +from mongospecs_bench.storefront.auth import Tokens +from mongospecs_bench.storefront.seed import CATEGORIES, Seeded, seed, stock_invariant +from mongospecs_bench.storefront.server import running_server + +SECRET = "load-secret" +MIX = (("list", 60), ("detail", 30), ("order", 10)) + + +def _percentile(values: list[float], q: float) -> float: + ordered = sorted(values) + return ordered[min(len(ordered) - 1, round(q * (len(ordered) - 1)))] + + +async def _virtual_user( + index: int, + client: httpx.AsyncClient, + seeded: Seeded, + stop: asyncio.Event, + samples: dict[str, list[float]], + failures: dict[str, int], + rng: random.Random, +) -> None: + token = {"authorization": f"Bearer {seeded.user_tokens[index % len(seeded.user_tokens)]}"} + actions = [name for name, weight in MIX for _ in range(weight)] + cursor: str | None = None + category = rng.choice(CATEGORIES) + while not stop.is_set(): + action = rng.choice(actions) + started = time.perf_counter() + try: + if action == "list": + url = f"/api/products?category={category}&sort=price&limit=20" + if cursor and rng.random() < 0.5: + url += f"&after={cursor}" + else: + category = rng.choice(CATEGORIES) + url = f"/api/products?category={category}&sort=price&limit=20" + response = await client.get(url) + cursor = response.json()["page"]["next_cursor"] if response.status_code == 200 else None + elif action == "detail": + response = await client.get( + f"/api/products/{rng.choice(seeded.product_ids)}?include=reviews,review_count" + ) + else: + lines = [ + {"sku": sku, "qty": rng.randint(1, 2)} + for sku in rng.sample(seeded.skus, k=rng.randint(1, 3)) + ] + response = await client.post("/api/orders", json={"lines": lines}, headers=token) + ok = response.status_code in (200, 201) + except httpx.HTTPError: + ok = False + elapsed = time.perf_counter() - started + if ok: + samples[action].append(elapsed) + else: + failures[action] += 1 + + +async def run_load( + settings: Settings, + *, + app: str = "storefront", + plug: str = "asgi", + backend: str = "msgspec", + users: int = 32, + seconds: float = 15.0, + products: int = 5000, +) -> list[dict[str, t.Any]]: + """One load run. Returns rows per action plus an ``all`` row; raises if the stock invariant fails.""" + database = f"ms_load_{uuid.uuid4().hex[:10]}" + uri = uri_for(settings.topology) + m = shop_models.build(backend) + label = "raw pymongo + starlette" if app == "raw" else f"mongospecs ({backend}, {plug})" + async with Registry(uri, database=database) as registry: + try: + seeded = await seed(registry, m, Tokens(SECRET), products=products, users=max(users, 8)) + for model in m.ALL: # both apps get the same indexes + registry.bind(model) + await registry.apply_indexes() + async with running_server( + app=app, + database=database, + mongo_uri=uri, + secret=SECRET, + plug=plug, + backend=backend, + projector=False, + ) as server: + limits = httpx.Limits(max_connections=users, max_keepalive_connections=users) + async with httpx.AsyncClient(base_url=server.base_url, timeout=30, limits=limits) as client: + for warm in (2.0, seconds): # a short warm-up, then the measured run + samples: dict[str, list[float]] = {name: [] for name, _ in MIX} + failures: dict[str, int] = {name: 0 for name, _ in MIX} + stop = asyncio.Event() + tasks = [ + asyncio.create_task( + _virtual_user( + i, client, seeded, stop, samples, failures, random.Random(1000 + i) + ) + ) + for i in range(users) + ] + started = time.perf_counter() + await asyncio.sleep(warm) + stop.set() + await asyncio.gather(*tasks) + elapsed = time.perf_counter() - started + invariant = await stock_invariant(registry, m, seeded) + finally: + await registry.client.drop_database(database) + if not invariant["ok"]: + raise AssertionError(f"{label}: stock invariant violated under load: {invariant}") + rows: list[dict[str, t.Any]] = [] + everything: list[float] = [] + for action, _ in MIX: + values = samples[action] + everything.extend(values) + if not values: + continue + rows.append( + { + "app": label, + "action": action, + "requests": len(values), + "rps": round(len(values) / elapsed, 1), + "p50_ms": round(statistics.median(values) * 1000, 2), + "p95_ms": round(_percentile(values, 0.95) * 1000, 2), + "p99_ms": round(_percentile(values, 0.99) * 1000, 2), + "failed": failures[action], + } + ) + rows.append( + { + "app": label, + "action": "all", + "requests": len(everything), + "rps": round(len(everything) / elapsed, 1), + "p50_ms": round(statistics.median(everything) * 1000, 2), + "p95_ms": round(_percentile(everything, 0.95) * 1000, 2), + "p99_ms": round(_percentile(everything, 0.99) * 1000, 2), + "failed": sum(failures.values()), + } + ) + rows[-1]["orders_placed"] = invariant["orders_placed"] + return rows + + +async def load(settings: Settings) -> Result: + """Workload ``LOAD``: the raw baseline, then mongospecs on every plug and model backend.""" + result = Result( + "LOAD", + "Storefront under HTTP load", + "What do mongospecs and mongospecs-web cost per request in a real server, against the same " + "endpoints written by hand with pymongo?", + ) + users = settings.scale(32, 6) + seconds = float(settings.scale(15, 3)) + products = settings.scale(5000, 200) + runs: list[tuple[str, str, str]] = [("raw", "asgi", "msgspec")] + runs += [ + ("storefront", plug, "msgspec") for plug in ("asgi", "starlette", "fastapi", "litestar", "quart") + ] + runs += [("storefront", "asgi", backend) for backend in settings.backends if backend != "msgspec"] + if settings.quick: + runs = runs[:2] + baseline: dict[str, float] = {} + for app, plug, backend in runs: + rows = await run_load( + settings, app=app, plug=plug, backend=backend, users=users, seconds=seconds, products=products + ) + for row in rows: + if app == "raw": + baseline[row["action"]] = row["rps"] + row["rps_vs_raw"] = ( + round(row["rps"] / baseline[row["action"]], 2) if baseline.get(row["action"]) else "" + ) + result.add(**row) + result.notes.append( + f"{users} virtual users for {seconds:.0f}s after a 2s warm-up; mix {dict(MIX)}; {products} products, " + "3 reviews each; one uvicorn worker; load generator and server on the same machine." + ) + result.notes.append( + "The stock invariant (stock + units sold = initial stock, never negative) held after every run." + ) + result.notes.append( + "The raw baseline implements the same three endpoints by hand, including the order transaction." + ) + return result diff --git a/bench/src/mongospecs_bench/storefront/__init__.py b/bench/src/mongospecs_bench/storefront/__init__.py new file mode 100644 index 0000000..5d03ae0 --- /dev/null +++ b/bench/src/mongospecs_bench/storefront/__init__.py @@ -0,0 +1,16 @@ +"""Storefront: a production-shaped reference application built on mongospecs and mongospecs-web. + +It is what the end-to-end tests boot as a real server process and what the load +benchmark drives over HTTP. It is deliberately not a toy: + +* authentication with signed bearer tokens, admin-only operations, row scope on orders +* placing an order is a multi-document transaction that must never oversell stock +* products are soft-deleted, revisioned (ETag / If-Match) and carry reviews as a reverse relation +* a change-stream projector keeps sales statistics up to date, exactly once, resumable +* indexes and validators are applied at startup; shutdown is graceful; ``/healthz`` checks the database +* optional tracing through the Sentry SDK (``SENTRY_DSN``) + +Run it:: + + SHOP_DB=shop uv run uvicorn mongospecs_bench.storefront.app:create_app --factory +""" diff --git a/bench/src/mongospecs_bench/storefront/app.py b/bench/src/mongospecs_bench/storefront/app.py new file mode 100644 index 0000000..be5f44e --- /dev/null +++ b/bench/src/mongospecs_bench/storefront/app.py @@ -0,0 +1,405 @@ +"""The storefront application: resources, custom operations, lifecycle.""" + +from __future__ import annotations + +import json +import os +import typing as t +from dataclasses import dataclass, field + +from mongospecs import DocumentNotFound, DuplicateKeyError, Inc, Q, Registry, sentry_listener +from mongospecs_bench.storefront import models as shop_models +from mongospecs_bench.storefront.auth import Principal, Tokens, hash_password, verify_password +from mongospecs_bench.storefront.projector import SalesProjector +from mongospecs_web import ( + Api, + Conflict, + Create, + Ctx, + Cursor, + Delete, + Forbidden, + Get, + List, + NotFound, + Patch, + Resource, + Unauthorized, + Unprocessable, +) + +PLUGS = ("asgi", "starlette", "fastapi", "litestar", "quart") + + +@dataclass +class Config: + mongo_uri: str = "mongodb://localhost:27117/?replicaSet=rs0" + database: str = "storefront" + backend: str = "msgspec" + plug: str = "asgi" + secret: str = "change-me" + sentry_dsn: str | None = None + projector: bool = True + max_pool_size: int = 100 + max_time_ms: int = 10_000 + options: dict[str, t.Any] = field(default_factory=dict) + + @classmethod + def from_env(cls) -> Config: + env = os.environ + return cls( + mongo_uri=env.get("SHOP_MONGO_URI", cls.mongo_uri), + database=env.get("SHOP_DB", cls.database), + backend=env.get("SHOP_BACKEND", cls.backend), + plug=env.get("SHOP_PLUG", cls.plug), + secret=env.get("SHOP_SECRET", cls.secret), + sentry_dsn=env.get("SENTRY_DSN") or None, + projector=env.get("SHOP_PROJECTOR", "1") != "0", + max_pool_size=int(env.get("SHOP_POOL", cls.max_pool_size)), + ) + + +def require_login(ctx: Ctx[t.Any]) -> bool: + if ctx.principal is None: + raise Unauthorized("Log in first.", headers={"www-authenticate": "Bearer"}) + return True + + +def require_admin(ctx: Ctx[t.Any]) -> bool: + require_login(ctx) + return bool(ctx.principal.is_admin) + + +def build_api(m: t.Any, registry: Registry, tokens: Tokens) -> Api: + """Declare the storefront's resources. ``m`` is the model namespace from ``models.build``.""" + + # -- products: public reads, admin writes, soft delete, ETags ------------------------- + products: Resource[t.Any] = Resource( + m.Product, + ops=[List, Get, Create, Patch, Delete], + guards={Create: require_admin, Patch: require_admin, Delete: require_admin}, + include=["reviews", "review_count"], + page=Cursor(default=20, max=100), + ) + + @products.op("restock", method="POST", guards=[require_admin], summary="Add stock to a product") + async def restock(ctx: Ctx[t.Any]) -> t.Any: + amount = (ctx.body() or {}).get("amount") + if not isinstance(amount, int) or isinstance(amount, bool) or amount <= 0: + raise Unprocessable( + "amount must be a positive integer.", + errors=[{"field": "amount", "message": "must be a positive integer"}], + ) + try: + return await m.Product.update(ctx.id, Inc(stock=amount)) + except DocumentNotFound: + raise NotFound("The product does not exist.") from None + + # -- reviews: anyone reads, logged-in users write, the author is set by the server ----- + reviews: Resource[t.Any] = Resource( + m.Review, + ops=[List, Get, Create, Delete], + guards={Create: require_login, Delete: require_login}, + server_fields=lambda ctx: {"author": ctx.principal.id}, + page=Cursor(default=20, max=100), + ) + + @reviews.before(Create) + async def check_review(ctx: Ctx[t.Any]) -> None: + body = ctx.body() or {} + rating = body.get("rating") + if not isinstance(rating, int) or isinstance(rating, bool) or not 1 <= rating <= 5: + raise Unprocessable( + "rating must be between 1 and 5.", + errors=[{"field": "rating", "message": "must be between 1 and 5"}], + ) + if not await m.Product.exists(Q.id == body.get("product")): + raise Unprocessable( + "The product does not exist.", errors=[{"field": "product", "message": "does not exist"}] + ) + + @reviews.before(Delete) + async def only_own_review(ctx: Ctx[t.Any]) -> None: + review = await ctx.load() + if not ctx.principal.is_admin and ( + review.author is None or str(review.author.id) != ctx.principal.id + ): + raise Forbidden("You can only delete your own reviews.") + + # -- orders: each user sees their own; placing and cancelling are transactions ---------- + def own_orders(ctx: Ctx[t.Any]) -> t.Any: + require_login(ctx) + return None if ctx.principal.is_admin else Q.user == ctx.principal.id + + orders: Resource[t.Any] = Resource( + m.Order, ops=[List, Get], scope=own_orders, page=Cursor(default=20, max=100) + ) + + @orders.op("place", method="POST", path="", status=201, guards=[require_login], summary="Place an order") + async def place(ctx: Ctx[t.Any]) -> t.Any: + wanted = _order_lines(ctx.body()) + + async def work(tx: t.Any) -> t.Any: + lines: list[t.Any] = [] + total = 0.0 + # A fixed order of products keeps concurrent orders from deadlocking each other. + for sku in sorted(wanted): + qty = wanted[sku] + try: + # One atomic step: only decrement if enough stock is left. + product = await m.Product.update( + (Q.sku == sku) & (Q.stock >= qty), Inc(stock=-qty), session=tx + ) + except DocumentNotFound: + raise Conflict( + f"{sku} is out of stock or does not exist.", + errors=[{"field": "lines", "message": f"{sku}: insufficient stock"}], + ) from None + lines.append( + m.OrderLine( + product=product.id, sku=sku, name=product.name, qty=qty, unit_price=product.price + ) + ) + total += qty * product.price + return await m.Order(user=ctx.principal.id, lines=lines, total=round(total, 2)).insert(session=tx) + + return await registry.transaction(work) + + @orders.op( + "cancel", method="POST", guards=[require_login], summary="Cancel an order and return its stock" + ) + async def cancel(ctx: Ctx[t.Any]) -> t.Any: + scope = await ctx.scope() + + async def work(tx: t.Any) -> t.Any: + condition = (Q.id == ctx.id) & (Q.status == "placed") + try: + order = await m.Order.update( + condition if scope is None else condition & scope, status="cancelled", session=tx + ) + except DocumentNotFound: + raise Conflict("The order does not exist or was already cancelled.") from None + for line in order.lines: + await ( + m.Product.find(Q.id == line.product, session=tx) + .unscoped("soft_delete") + .update(Inc(stock=line.qty)) + ) + return order + + return await registry.transaction(work) + + # -- accounts ------------------------------------------------------------------------- + users: Resource[t.Any] = Resource(m.User, ops=[Get], scope=lambda ctx: _self_only(ctx)) + + @users.op("signup", method="POST", path="/signup", status=201) + async def signup(ctx: Ctx[t.Any]) -> t.Any: + body = ctx.body() or {} + email, password = body.get("email"), body.get("password") + if not isinstance(email, str) or "@" not in email: + raise Unprocessable( + "A valid email is required.", errors=[{"field": "email", "message": "is not a valid email"}] + ) + if not isinstance(password, str) or len(password) < 8: + raise Unprocessable( + "The password is too short.", + errors=[{"field": "password", "message": "must be at least 8 characters"}], + ) + try: + user = await m.User( + email=email.lower(), name=str(body.get("name", "")), password_hash=hash_password(password) + ).insert() + except DuplicateKeyError: + raise Conflict( + "That email is already registered.", errors=[{"field": "email", "message": "must be unique"}] + ) from None + return {"user": users.view.serialize(user), "token": tokens.issue(user.id)} + + @users.op("login", method="POST", path="/login") + async def login(ctx: Ctx[t.Any]) -> t.Any: + body = ctx.body() or {} + user = ( + await m.User.find(Q.email == str(body.get("email", "")).lower()).select("+password_hash").first() + ) + if user is None or not verify_password(str(body.get("password", "")), user.password_hash): + raise Unauthorized("Wrong email or password.") + return {"token": tokens.issue(user.id, is_admin=user.is_admin)} + + stats: Resource[t.Any] = Resource(m.SalesStat, name="sales", ops=[List], guards=[require_admin]) + + return Api( + products, + reviews, + orders, + users, + stats, + prefix="/api", + title="Storefront", + authenticate=tokens.authenticate, + ) + + +def _self_only(ctx: Ctx[t.Any]) -> t.Any: + require_login(ctx) + return None if ctx.principal.is_admin else Q.id == ctx.principal.id + + +def _order_lines(body: t.Any) -> dict[str, int]: + lines = body.get("lines") if isinstance(body, dict) else None + if not isinstance(lines, list) or not lines or len(lines) > 50: + raise Unprocessable( + "lines must be a list of 1 to 50 items.", + errors=[{"field": "lines", "message": "must have 1 to 50 items"}], + ) + wanted: dict[str, int] = {} + for item in lines: + sku = item.get("sku") if isinstance(item, dict) else None + qty = item.get("qty") if isinstance(item, dict) else None + if ( + not isinstance(sku, str) + or not isinstance(qty, int) + or isinstance(qty, bool) + or not 1 <= qty <= 1000 + ): + raise Unprocessable( + "Each line needs a sku and a qty from 1 to 1000.", + errors=[{"field": "lines", "message": "each line needs sku and qty (1-1000)"}], + ) + wanted[sku] = wanted.get(sku, 0) + qty + return wanted + + +class Storefront: + """The ASGI application: lifecycle and ``/healthz`` around the framework-specific app.""" + + def __init__(self, config: Config) -> None: + self.config = config + self.m = shop_models.build(config.backend) + listeners = [sentry_listener()] if config.sentry_dsn else None + self.registry = Registry( + config.mongo_uri, + database=config.database, + models=self.m.ALL, + max_time_ms=config.max_time_ms, + on_command=listeners, + maxPoolSize=config.max_pool_size, + **config.options, + ) + self.tokens = Tokens(config.secret) + self.api = build_api(self.m, self.registry, self.tokens) + self.projector = SalesProjector(self.registry, self.m) if config.projector else None + self.inner = self._mount(config.plug) + if config.sentry_dsn: + import sentry_sdk + from sentry_sdk.integrations.asgi import SentryAsgiMiddleware + + sentry_sdk.init( + dsn=config.sentry_dsn, + traces_sample_rate=1.0, + default_integrations=False, + auto_enabling_integrations=False, + ) + self.inner = SentryAsgiMiddleware(self.inner, transaction_style="url") + + def _mount(self, plug: str) -> t.Any: + if plug == "asgi": + return self.api.asgi() + if plug == "starlette": + from starlette.applications import Starlette + + from mongospecs_web.plugs import starlette as starlette_plug + + return Starlette(routes=starlette_plug.routes(self.api)) + if plug == "fastapi": + from fastapi import FastAPI + + from mongospecs_web.plugs import fastapi as fastapi_plug + + app = FastAPI(title="Storefront", openapi_url="/api/openapi.json") + fastapi_plug.mount(app, self.api) + return app + if plug == "litestar": + from litestar import Litestar + + from mongospecs_web.plugs.litestar import MongospecsPlugin + + return Litestar(plugins=[MongospecsPlugin(self.api)], openapi_config=None) + if plug == "quart": + from quart import Quart + + from mongospecs_web.plugs import quart as quart_plug + + quart_app = Quart(__name__) + quart_plug.init_app(quart_app, self.api) + return quart_app + raise ValueError(f"unknown plug {plug!r}; expected one of {PLUGS}") + + async def startup(self) -> None: + await self.registry.open() + await self.registry.ping() + await self.registry.apply_indexes() + if self.projector is not None: + await self.projector.start() + + async def shutdown(self) -> None: + if self.projector is not None: + await self.projector.stop() + await self.registry.close() + print("storefront: shutdown complete", flush=True) + if self.config.sentry_dsn: + import sentry_sdk + + sentry_sdk.flush(timeout=5) + + async def __call__(self, scope: dict[str, t.Any], receive: t.Any, send: t.Any) -> None: + if scope["type"] == "lifespan": + while True: + message = await receive() + if message["type"] == "lifespan.startup": + try: + await self.startup() + except Exception as exc: + await send( + {"type": "lifespan.startup.failed", "message": f"{type(exc).__name__}: {exc}"} + ) + return + await send({"type": "lifespan.startup.complete"}) + elif message["type"] == "lifespan.shutdown": + await self.shutdown() + await send({"type": "lifespan.shutdown.complete"}) + return + elif scope["type"] == "http" and scope["path"] == "/healthz": + await self._health(send) + else: + await self.inner(scope, receive, send) + + async def _health(self, send: t.Any) -> None: + try: + await self.registry.ping() + status, body = 200, {"status": "ok", "backend": self.config.backend, "plug": self.config.plug} + except Exception as exc: + status, body = 503, {"status": "unavailable", "error": type(exc).__name__} + payload = json.dumps(body).encode() + await send( + { + "type": "http.response.start", + "status": status, + "headers": [ + (b"content-type", b"application/json"), + (b"content-length", str(len(payload)).encode()), + ], + } + ) + await send({"type": "http.response.body", "body": payload}) + + +def create_app() -> Storefront: + """ASGI factory: ``uvicorn mongospecs_bench.storefront.app:create_app --factory``.""" + return Storefront(Config.from_env()) + + +def issue_admin_token(secret: str, user_id: t.Any) -> str: + return Tokens(secret).issue(user_id, is_admin=True) + + +__all__ = ["PLUGS", "Config", "Principal", "Storefront", "build_api", "create_app", "issue_admin_token"] diff --git a/bench/src/mongospecs_bench/storefront/auth.py b/bench/src/mongospecs_bench/storefront/auth.py new file mode 100644 index 0000000..66534a7 --- /dev/null +++ b/bench/src/mongospecs_bench/storefront/auth.py @@ -0,0 +1,68 @@ +"""Stateless bearer tokens and password hashing, standard library only.""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import os +import typing as t +from dataclasses import dataclass + +from mongospecs_web import Request, Unauthorized + + +@dataclass(frozen=True, slots=True) +class Principal: + id: str + is_admin: bool + + +def hash_password(password: str, *, salt: bytes | None = None) -> str: + salt = salt or os.urandom(16) + digest = hashlib.scrypt(password.encode(), salt=salt, n=2**12, r=8, p=1, dklen=32) + return base64.b64encode(salt + digest).decode() + + +def verify_password(password: str, stored: str) -> bool: + try: + raw = base64.b64decode(stored.encode()) + except ValueError: + return False + salt, digest = raw[:16], raw[16:] + candidate = hashlib.scrypt(password.encode(), salt=salt, n=2**12, r=8, p=1, dklen=32) + return hmac.compare_digest(candidate, digest) + + +class Tokens: + """``..``: nothing to look up, nothing to store.""" + + def __init__(self, secret: str) -> None: + self._secret = secret.encode() + + def _sign(self, payload: str) -> str: + return hmac.new(self._secret, payload.encode(), hashlib.sha256).hexdigest() + + def issue(self, user_id: t.Any, *, is_admin: bool = False) -> str: + payload = f"{user_id}.{'a' if is_admin else 'u'}" + return f"{payload}.{self._sign(payload)}" + + def verify(self, token: str) -> Principal: + user_id, _, rest = token.partition(".") + role, _, signature = rest.partition(".") + if ( + not user_id + or role not in ("a", "u") + or not hmac.compare_digest(signature, self._sign(f"{user_id}.{role}")) + ): + raise Unauthorized("Invalid token.", headers={"www-authenticate": "Bearer"}) + return Principal(user_id, role == "a") + + def authenticate(self, request: Request) -> Principal | None: + header = request.headers.get("authorization") + if not header: + return None + scheme, _, token = header.partition(" ") + if scheme.lower() != "bearer" or not token: + raise Unauthorized("Use a Bearer token.", headers={"www-authenticate": "Bearer"}) + return self.verify(token) diff --git a/bench/src/mongospecs_bench/storefront/models.py b/bench/src/mongospecs_bench/storefront/models.py new file mode 100644 index 0000000..c168f3c --- /dev/null +++ b/bench/src/mongospecs_bench/storefront/models.py @@ -0,0 +1,160 @@ +"""Storefront models, available on each backend (no ``from __future__ import annotations``: +the classes are built by a factory, so annotations must be real objects).""" + +import datetime as dt +import functools +import types +import typing as t + +from mongospecs import ( + OID, + Backref, + CreatedAt, + DeletedAt, + Immutable, + Index, + OnDelete, + Policy, + Ref, + Revision, + SpecConfig, + UpdatedAt, +) + +READ_ONLY = Policy(insert=False, update=False) + + +@functools.cache +def build(backend: str = "msgspec") -> types.SimpleNamespace: + if backend == "msgspec": + import msgspec + + from mongospecs.msgspec import Spec + + define: t.Any = lambda cls: cls # noqa: E731 + + class OrderLine(msgspec.Struct, kw_only=True): + product: OID + sku: str + name: str + qty: int + unit_price: float + + elif backend == "pydantic": + import pydantic + + from mongospecs.pydantic import Spec # type: ignore[assignment] + + define = lambda cls: cls # noqa: E731 + + class OrderLine(pydantic.BaseModel): # type: ignore[no-redef] + product: OID + sku: str + name: str + qty: int + unit_price: float + + elif backend == "attrs": + import attrs + + from mongospecs.attrs import Spec # type: ignore[assignment] + + define = attrs.define(kw_only=True) + + @attrs.define(kw_only=True) + class OrderLine: # type: ignore[no-redef] + product: OID + sku: str + name: str + qty: int + unit_price: float + + else: + raise ValueError(f"unknown backend {backend!r}") + + @define + class User(Spec): + __spec__ = SpecConfig(collection="users", indexes=[Index("email", unique=True)]) + email: t.Annotated[str, Policy(update=False)] + name: str = "" + password_hash: t.Annotated[str, Policy(private=True)] = "" + is_admin: t.Annotated[bool, READ_ONLY] = False + created_at: t.Annotated[dt.datetime | None, CreatedAt] = None + + @define + class Product(Spec): + __spec__ = SpecConfig( + collection="products", + indexes=[ + Index("sku", unique=True), + # Cursor pagination sorts by the requested key and then by id, so the id + # belongs at the end of every index that serves a listing. + Index("category", "price", "id"), + Index("price", "id"), + Index("name", "id"), + ], + ) + sku: t.Annotated[str, Immutable, Policy(filter=("eq", "in"))] + name: t.Annotated[str, Policy(filter=("eq", "prefix"), sort=True)] + category: t.Annotated[str, Policy(filter=("eq", "in"))] = "general" + price: t.Annotated[float, Policy(filter=("gte", "lte", "gt", "lt"), sort=True)] = 0.0 + stock: t.Annotated[int, Policy(update=False, filter=("gte", "eq"))] = 0 + """Changed only by orders and restocks, never by a plain PATCH.""" + tags: list[str] = [] + description: str = "" + reviews: t.Annotated[list[t.Any], Backref("product", model="Review")] = [] + review_count: t.Annotated[int, Backref("product", model="Review", count=True)] = 0 + created_at: t.Annotated[dt.datetime | None, CreatedAt, Policy(sort=True)] = None + updated_at: t.Annotated[dt.datetime | None, UpdatedAt] = None + deleted_at: t.Annotated[dt.datetime | None, DeletedAt, Policy(private=True)] = None + rev: t.Annotated[int, Revision] = 0 + + @define + class Review(Spec): + __spec__ = SpecConfig( + collection="reviews", indexes=[Index("product", "-created_at"), Index("author")] + ) + product: t.Annotated[Ref[Product], OnDelete("cascade"), Policy(update=False, filter=("eq",))] + rating: t.Annotated[int, Policy(filter=("eq", "gte", "lte"), sort=True)] + body: str = "" + author: t.Annotated[Ref[User] | None, Policy(insert=False, update=False, filter=("eq",))] = None + created_at: t.Annotated[dt.datetime | None, CreatedAt, Policy(sort=True)] = None + + @define + class Order(Spec): + __spec__ = SpecConfig(collection="orders", indexes=[Index("user", "-created_at"), Index("status")]) + user: t.Annotated[Ref[User], Policy(filter=("eq",))] + lines: list[OrderLine] + total: float + status: t.Annotated[str, Policy(filter=("eq",))] = "placed" + created_at: t.Annotated[dt.datetime | None, CreatedAt, Policy(sort=True)] = None + updated_at: t.Annotated[dt.datetime | None, UpdatedAt] = None + rev: t.Annotated[int, Revision] = 0 + + @define + class SalesStat(Spec): + """Units and revenue per SKU, maintained by the change-stream projector.""" + + __spec__ = SpecConfig(collection="sales_stats", indexes=[Index("sku", unique=True)]) + sku: t.Annotated[str, Policy(filter=("eq",))] + units: t.Annotated[int, Policy(sort=True)] = 0 + revenue: float = 0.0 + + @define + class ProjectorState(Spec): + __spec__ = SpecConfig(collection="projector_state", indexes=[Index("name", unique=True)]) + name: str + resume_token: dict[str, t.Any] | None = None + events: int = 0 + + return types.SimpleNamespace( + backend=backend, + OrderLine=OrderLine, + User=User, + Product=Product, + Review=Review, + Order=Order, + SalesStat=SalesStat, + ProjectorState=ProjectorState, + ALL=[User, Product, Review, Order, SalesStat, ProjectorState], + ) diff --git a/bench/src/mongospecs_bench/storefront/projector.py b/bench/src/mongospecs_bench/storefront/projector.py new file mode 100644 index 0000000..80c96ca --- /dev/null +++ b/bench/src/mongospecs_bench/storefront/projector.py @@ -0,0 +1,87 @@ +"""Sales statistics kept current from the orders change stream. + +Each event is applied in a transaction together with the stream's resume token, +so an event is counted exactly once even if the process dies between events, +and a restarted projector continues where the last one stopped. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import typing as t + +from mongospecs import Inc, MongoSpecsError, Q, Registry, Set + +log = logging.getLogger("storefront.projector") + + +class SalesProjector: + name = "sales" + + def __init__(self, registry: Registry, models: t.Any) -> None: + self.registry = registry + self.m = models + self._task: asyncio.Task[None] | None = None + self.ready = asyncio.Event() + + async def start(self) -> None: + self._task = asyncio.create_task(self._run(), name="sales-projector") + await self.ready.wait() + + async def stop(self) -> None: + if self._task is not None: + self._task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._task + self._task = None + + async def _apply(self, event: t.Any) -> None: + order = event.document + sign = 1 if event.operation == "insert" else -1 # an update here is a cancellation + + async def work(tx: t.Any) -> None: + for line in order.lines: + await self.m.SalesStat.upsert( + Q.sku == line.sku, + Inc(units=sign * line.qty, revenue=sign * line.qty * line.unit_price), + session=tx, + ) + await self.m.ProjectorState.upsert( + Q.name == self.name, + Set(resume_token=dict(event.resume_token)), + Inc(events=1), + session=tx, + ) + + await self.registry.transaction(work) + + async def _run(self) -> None: + pipeline = [ + { + "$match": { + "$or": [ + {"operationType": "insert"}, + {"operationType": "update", "updateDescription.updatedFields.status": "cancelled"}, + ] + } + } + ] + while True: + state = await self.m.ProjectorState.find(Q.name == self.name).first() + options: dict[str, t.Any] = {"max_await_time_ms": 250} + if state is not None and state.resume_token: + options["resume_after"] = state.resume_token + try: + async with self.m.Order.watch(pipeline, **options) as stream: + self.ready.set() + async for event in stream: + await self._apply(event) + except asyncio.CancelledError: + raise + except ( + MongoSpecsError + ) as exc: # lost the stream (step-down, network): resume from the stored token + log.warning("projector restarting after %s: %s", type(exc).__name__, exc) + await asyncio.sleep(0.5) diff --git a/bench/src/mongospecs_bench/storefront/raw_app.py b/bench/src/mongospecs_bench/storefront/raw_app.py new file mode 100644 index 0000000..8e1e7b5 --- /dev/null +++ b/bench/src/mongospecs_bench/storefront/raw_app.py @@ -0,0 +1,226 @@ +"""The same hot paths as the storefront, written by hand with pymongo and Starlette. + +This is the baseline for the load benchmark: what the three busiest endpoints +cost when there is no ODM and no resource layer, only the driver and a +framework. It implements the same behaviour (soft-delete filter, cursor +pagination, reviews embedded on demand, a transaction that cannot oversell) so +the comparison is like for like. +""" + +from __future__ import annotations + +import base64 +import contextlib +import datetime as dt +import json +import os +import typing as t + +import bson +from bson import ObjectId +from pymongo import AsyncMongoClient, ReturnDocument +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import JSONResponse, Response +from starlette.routing import Route + +from mongospecs_bench.storefront.auth import Tokens +from mongospecs_web import HTTPError + +PRODUCT_FIELDS = ( + "sku", + "name", + "category", + "price", + "stock", + "tags", + "description", + "created_at", + "updated_at", + "rev", +) + + +def _json(value: t.Any) -> t.Any: + if isinstance(value, ObjectId): + return str(value) + if isinstance(value, dt.datetime): + return value.isoformat().replace("+00:00", "Z") + if isinstance(value, dict): + return {k: _json(v) for k, v in value.items()} + if isinstance(value, list): + return [_json(v) for v in value] + return value + + +def _product(doc: dict[str, t.Any]) -> dict[str, t.Any]: + return {"id": str(doc["_id"]), **{k: _json(doc.get(k)) for k in PRODUCT_FIELDS}} + + +def _problem(status: int, title: str, detail: str) -> Response: + body = json.dumps({"type": "about:blank", "title": title, "status": status, "detail": detail}) + return Response(body, status, media_type="application/problem+json") + + +def create_app() -> Starlette: + uri = os.environ.get("SHOP_MONGO_URI", "mongodb://localhost:27117/?replicaSet=rs0") + database = os.environ.get("SHOP_DB", "storefront") + tokens = Tokens(os.environ.get("SHOP_SECRET", "change-me")) + state: dict[str, t.Any] = {} + + @contextlib.asynccontextmanager + async def lifespan(app: Starlette) -> t.AsyncIterator[None]: + client: AsyncMongoClient[dict[str, t.Any]] = AsyncMongoClient( + uri, tz_aware=True, maxPoolSize=int(os.environ.get("SHOP_POOL", "100")) + ) + state["client"] = client + state["db"] = client[database] + await client.admin.command("ping") + yield + await client.close() + + async def health(request: Request) -> Response: + await state["client"].admin.command("ping") + return JSONResponse({"status": "ok", "backend": "none", "plug": "raw"}) + + async def list_products(request: Request) -> Response: + params = request.query_params + flt: dict[str, t.Any] = {"deleted_at": None} + if "category" in params: + flt["category"] = params["category"] + sort_param = params.get("sort", "") + direction = -1 if sort_param.startswith("-") else 1 + key = sort_param.lstrip("-") or "_id" + if key not in ("price", "name", "created_at", "_id"): + return _problem(400, "Bad Request", f"{key!r} cannot be used for sorting") + try: + limit = min(max(int(params.get("limit", 20)), 1), 100) + except ValueError: + return _problem(400, "Bad Request", "limit must be an integer") + sort = [(key, direction)] + ([("_id", 1)] if key != "_id" else []) + if "after" in params: + try: + padded = params["after"] + "=" * (-len(params["after"]) % 4) + values = bson.decode(base64.urlsafe_b64decode(padded))["v"] + except Exception: + return _problem(400, "Bad Request", "invalid pagination cursor") + op = "$gt" if direction == 1 else "$lt" + if key == "_id": + flt["_id"] = {"$gt": values[0]} + else: + flt["$or"] = [{key: {op: values[0]}}, {key: values[0], "_id": {"$gt": values[1]}}] + docs = await state["db"].products.find(flt, sort=sort, limit=limit + 1).to_list() + has_next = len(docs) > limit + docs = docs[:limit] + cursor = None + if has_next and docs: + last = docs[-1] + values = [last["_id"]] if key == "_id" else [last[key], last["_id"]] + cursor = base64.urlsafe_b64encode(bson.encode({"v": values})).decode().rstrip("=") + return JSONResponse( + { + "data": [_product(d) for d in docs], + "page": {"size": limit, "has_next": has_next, "next_cursor": cursor}, + } + ) + + async def get_product(request: Request) -> Response: + try: + oid = ObjectId(request.path_params["id"]) + except Exception: + return _problem(404, "Not Found", "The resource does not exist.") + doc = await state["db"].products.find_one({"_id": oid, "deleted_at": None}) + if doc is None: + return _problem(404, "Not Found", "The resource does not exist.") + out = _product(doc) + include = request.query_params.get("include", "").split(",") + if "reviews" in include: + reviews = await state["db"].reviews.find({"product": oid}).to_list() + out["reviews"] = [ + { + "id": str(r["_id"]), + "product": str(r["product"]), + "rating": r["rating"], + "body": r["body"], + "author": _json(r.get("author")), + "created_at": _json(r.get("created_at")), + } + for r in reviews + ] + if "review_count" in include: + out["review_count"] = await state["db"].reviews.count_documents({"product": oid}) + return JSONResponse({"data": out}, headers={"etag": f'"{doc.get("rev", 0)}"'}) + + async def place_order(request: Request) -> Response: + header = request.headers.get("authorization", "") + try: + principal = tokens.verify(header.partition(" ")[2]) if header else None + except HTTPError: + principal = None + if principal is None: + return _problem(401, "Unauthorized", "Log in first.") + try: + body = json.loads(await request.body()) + wanted: dict[str, int] = {} + for item in body["lines"]: + wanted[item["sku"]] = wanted.get(item["sku"], 0) + int(item["qty"]) + except Exception: + return _problem(422, "Unprocessable Content", "lines must be a list of {sku, qty}.") + db = state["db"] + + class OutOfStock(Exception): + pass + + async def work(session: t.Any) -> dict[str, t.Any]: + now = dt.datetime.now(dt.UTC) + now = now.replace(microsecond=now.microsecond // 1000 * 1000) + lines, total = [], 0.0 + for sku in sorted(wanted): + qty = wanted[sku] + product = await db.products.find_one_and_update( + {"sku": sku, "stock": {"$gte": qty}, "deleted_at": None}, + {"$inc": {"stock": -qty, "rev": 1}, "$set": {"updated_at": now}}, + return_document=ReturnDocument.AFTER, + session=session, + ) + if product is None: + raise OutOfStock(sku) + lines.append( + { + "product": product["_id"], + "sku": sku, + "name": product["name"], + "qty": qty, + "unit_price": product["price"], + } + ) + total += qty * product["price"] + order = { + "user": ObjectId(principal.id), + "lines": lines, + "total": round(total, 2), + "status": "placed", + "created_at": now, + "updated_at": now, + "rev": 0, + } + await db.orders.insert_one(order, session=session) + return order + + try: + async with state["client"].start_session() as session: + order = await session.with_transaction(work) + except OutOfStock as exc: + return _problem(409, "Conflict", f"{exc} is out of stock or does not exist.") + out = {"id": str(order["_id"]), **{k: _json(v) for k, v in order.items() if k != "_id"}} + return JSONResponse({"data": out}, status_code=201, headers={"etag": '"0"'}) + + return Starlette( + routes=[ + Route("/healthz", health), + Route("/api/products", list_products), + Route("/api/products/{id}", get_product), + Route("/api/orders", place_order, methods=["POST"]), + ], + lifespan=lifespan, + ) diff --git a/bench/src/mongospecs_bench/storefront/seed.py b/bench/src/mongospecs_bench/storefront/seed.py new file mode 100644 index 0000000..0778178 --- /dev/null +++ b/bench/src/mongospecs_bench/storefront/seed.py @@ -0,0 +1,132 @@ +"""Seed data for the storefront, written with the raw driver for speed.""" + +from __future__ import annotations + +import datetime as dt +import random +import typing as t +from dataclasses import dataclass + +from bson import ObjectId + +from mongospecs import Registry +from mongospecs_bench.storefront.auth import Tokens + +CATEGORIES = ("books", "games", "garden", "kitchen", "music", "office", "outdoor", "toys") + + +@dataclass +class Seeded: + product_ids: list[str] + skus: list[str] + user_tokens: list[str] + admin_token: str + initial_stock: int + products: int + + +async def seed( + registry: Registry, + m: t.Any, + tokens: Tokens, + *, + products: int = 2000, + users: int = 50, + reviews_per_product: int = 3, + stock: int = 1_000_000, + seed_value: int = 7, +) -> Seeded: + rng = random.Random(seed_value) + now = dt.datetime.now(dt.UTC).replace(microsecond=0) + user_docs = [ + { + "_id": ObjectId(), + "email": f"user{i}@example.com", + "name": f"User {i}", + "password_hash": "", + "is_admin": False, + "created_at": now, + } + for i in range(users) + ] + admin = { + "_id": ObjectId(), + "email": "admin@example.com", + "name": "Admin", + "password_hash": "", + "is_admin": True, + "created_at": now, + } + await registry.manager(m.User).collection.insert_many([*user_docs, admin]) + + product_docs = [] + for i in range(products): + product_docs.append( + { + "_id": ObjectId(), + "sku": f"SKU-{i:06d}", + "name": f"{rng.choice(('Classic', 'Deluxe', 'Compact', 'Pro', 'Eco'))} item {i}", + "category": CATEGORIES[i % len(CATEGORIES)], + "price": round(rng.uniform(1, 500), 2), + "stock": stock, + "tags": rng.sample(("new", "sale", "gift", "limited", "popular", "eco"), k=2), + "description": "Lorem ipsum dolor sit amet. " * 6, + "created_at": now - dt.timedelta(minutes=i), + "updated_at": now - dt.timedelta(minutes=i), + "deleted_at": None, + "rev": 0, + } + ) + collection = registry.manager(m.Product).collection + for start in range(0, len(product_docs), 5000): + await collection.insert_many(product_docs[start : start + 5000]) + + review_docs = [ + { + "product": product["_id"], + "rating": rng.randint(1, 5), + "body": "Works as described. " * rng.randint(1, 4), + "author": rng.choice(user_docs)["_id"], + "created_at": now - dt.timedelta(minutes=rng.randint(0, 100_000)), + } + for product in product_docs + for _ in range(reviews_per_product) + ] + reviews = registry.manager(m.Review).collection + for start in range(0, len(review_docs), 5000): + await reviews.insert_many(review_docs[start : start + 5000]) + + return Seeded( + product_ids=[str(p["_id"]) for p in product_docs], + skus=[str(p["sku"]) for p in product_docs], + user_tokens=[tokens.issue(u["_id"]) for u in user_docs], + admin_token=tokens.issue(admin["_id"], is_admin=True), + initial_stock=stock, + products=products, + ) + + +async def stock_invariant(registry: Registry, m: t.Any, seeded: Seeded) -> dict[str, t.Any]: + """Every unit is either still in stock or on a placed order: nothing lost, nothing oversold.""" + products = registry.manager(m.Product).collection + orders = registry.manager(m.Order).collection + stock_now = {d["sku"]: d["stock"] async for d in products.find({}, {"sku": 1, "stock": 1})} + sold: dict[str, int] = {} + cursor = await orders.aggregate( + [ + {"$match": {"status": "placed"}}, + {"$unwind": "$lines"}, + {"$group": {"_id": "$lines.sku", "units": {"$sum": "$lines.qty"}}}, + ] + ) + async for row in cursor: + sold[row["_id"]] = row["units"] + negative = [sku for sku, value in stock_now.items() if value < 0] + mismatched = [sku for sku, value in stock_now.items() if value + sold.get(sku, 0) != seeded.initial_stock] + return { + "orders_placed": await orders.count_documents({"status": "placed"}), + "units_sold": sum(sold.values()), + "negative_stock": len(negative), + "stock_mismatches": len(mismatched), + "ok": not negative and not mismatched, + } diff --git a/bench/src/mongospecs_bench/storefront/server.py b/bench/src/mongospecs_bench/storefront/server.py new file mode 100644 index 0000000..f70a29f --- /dev/null +++ b/bench/src/mongospecs_bench/storefront/server.py @@ -0,0 +1,140 @@ +"""Run the storefront (or the raw baseline) as a real server process.""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +import signal +import socket +import subprocess +import sys +import typing as t +from dataclasses import dataclass + +import httpx + +APPS = { + "storefront": "mongospecs_bench.storefront.app:create_app", + "raw": "mongospecs_bench.storefront.raw_app:create_app", +} + + +def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +@dataclass +class Server: + base_url: str + process: subprocess.Popen[bytes] + env: dict[str, str] + exit_code: int | None = None + _log: str | None = None + + async def stop(self, *, grace_seconds: float = 20.0) -> int: + """Ask the server to shut down gracefully (SIGTERM) and return its exit code.""" + if self.process.poll() is None: + self.process.send_signal(signal.SIGTERM) + try: + await asyncio.to_thread(self.process.wait, grace_seconds) + except subprocess.TimeoutExpired: + self.process.kill() + await asyncio.to_thread(self.process.wait) + self.exit_code = self.process.returncode + return self.exit_code + + def stopped_gracefully(self) -> bool: + """True when the process ended on our SIGTERM after running its shutdown handlers. + + uvicorn finishes in-flight requests, runs the ASGI lifespan shutdown and + then re-raises the signal, so a clean stop ends with exit code 0 or -SIGTERM. + """ + if self._log is None: + self._log = self.output() + return self.exit_code in (0, -signal.SIGTERM) and "shutdown complete" in self._log + + def output(self) -> str: + """Everything the server printed. Only complete once the process has exited.""" + if self._log is not None: + return self._log + if self.process.stdout is None: + return "" + with contextlib.suppress(Exception): + text = self.process.stdout.read().decode(errors="replace") + if self.process.poll() is not None: + self._log = text + return text + return "" + + +async def start_server( + *, + app: str = "storefront", + database: str, + mongo_uri: str, + secret: str = "e2e-secret", + plug: str = "asgi", + backend: str = "msgspec", + projector: bool = True, + extra_env: t.Mapping[str, str] | None = None, + startup_seconds: float = 30.0, +) -> Server: + port = free_port() + env = { + **os.environ, + "SHOP_MONGO_URI": mongo_uri, + "SHOP_DB": database, + "SHOP_SECRET": secret, + "SHOP_PLUG": plug, + "SHOP_BACKEND": backend, + "SHOP_PROJECTOR": "1" if projector else "0", + **(extra_env or {}), + } + process = subprocess.Popen( # noqa: ASYNC220 - starting a server process, not doing I/O with it + [ + sys.executable, + "-m", + "uvicorn", + APPS[app], + "--factory", + "--host", + "127.0.0.1", + "--port", + str(port), + "--log-level", + "warning", + "--no-access-log", + ], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + server = Server(f"http://127.0.0.1:{port}", process, env) + deadline = asyncio.get_running_loop().time() + startup_seconds + async with httpx.AsyncClient(base_url=server.base_url, timeout=2) as client: + while True: + if process.poll() is not None: + raise RuntimeError( + f"server exited during startup with code {process.returncode}:\n{server.output()}" + ) + try: + if (await client.get("/healthz")).status_code == 200: + return server + except httpx.HTTPError: + pass + if asyncio.get_running_loop().time() > deadline: + await server.stop() + raise RuntimeError(f"server did not become healthy in {startup_seconds}s:\n{server.output()}") + await asyncio.sleep(0.1) + + +@contextlib.asynccontextmanager +async def running_server(**options: t.Any) -> t.AsyncIterator[Server]: + server = await start_server(**options) + try: + yield server + finally: + await server.stop() diff --git a/bench/src/mongospecs_bench/workloads.py b/bench/src/mongospecs_bench/workloads.py index 3b16d05..e6d4c55 100644 --- a/bench/src/mongospecs_bench/workloads.py +++ b/bench/src/mongospecs_bench/workloads.py @@ -53,9 +53,8 @@ async def b1(settings: Settings) -> Result: for backend in settings.backends: meta = meta_of(model_for(backend, shape)) modes: dict[str, t.Callable[[], t.Any]] = { - "validated": lambda: [meta.decode(d, trusted=False) for d in docs], - "trusted": lambda: [meta.decode(d, trusted=True) for d in docs], - "tracked": lambda: [meta.encode(meta.decode(d, trusted=False)) for d in docs], + "decode": lambda: [meta.decode(d) for d in docs], + "decode + encode": lambda: [meta.encode(meta.decode(d)) for d in docs], } for mode, fn in modes.items(): timing = measure_sync(fn, repeat=settings.repeat) @@ -67,8 +66,9 @@ async def b1(settings: Settings) -> Result: us_per_doc=round(timing.median_ms * 1000 / n, 2), vs_bson=_ratio(timing.median_ms, base.median_ms), ) - result.notes.append("'tracked' is decode plus the encode that builds the change-tracking snapshot.") - result.notes.append("'trusted' only differs for flat pydantic models; msgspec and attrs always convert.") + result.notes.append( + "'decode' always validates. 'decode + encode' adds turning the model back into a BSON-ready dict (what a save does)." + ) return result @@ -341,6 +341,17 @@ async def b5(settings: Settings) -> Result: manager = registry.manager(ns.Small) await manager.collection.insert_many([raw_document("small", i) for i in range(total)]) await manager.collection.create_index([("n", 1)]) + # Cursor pagination sorts by (key, _id). An index on the key alone cannot provide that + # order, so the server sorts in memory; measure that first, then add the right index. + unindexed = await measure(lambda: manager.find().sort("n").lean().page(size), repeat=settings.repeat) + result.add( + page="1, index on (n) only", + skip=0, + offset_ms="", + cursor_ms=round(unindexed.median_ms, 3), + offset_vs_cursor="", + ) + await manager.collection.create_index([("n", 1), ("_id", 1)]) for page in (1, 100, total // size // 2, total // size - 1): if page < 1 or page * size > total: continue @@ -393,7 +404,9 @@ async def facet() -> t.Any: cursor_ms="", offset_vs_cursor="", ) - result.notes.append(f"Collection of {total} documents, page size {size}, sorted on an indexed field.") + result.notes.append( + f"Collection of {total} documents, page size {size}; index on (n, _id) unless stated." + ) result.notes.append( "$facet returns the page and the total as one document, so the page is bound by the 16MB limit." ) diff --git a/bench/tests/conftest.py b/bench/tests/conftest.py new file mode 100644 index 0000000..05e2b92 --- /dev/null +++ b/bench/tests/conftest.py @@ -0,0 +1,80 @@ +import os +import uuid + +import pytest_asyncio + +from mongospecs import Registry +from mongospecs_bench.storefront import models as shop_models +from mongospecs_bench.storefront.auth import Tokens +from mongospecs_bench.storefront.seed import seed +from mongospecs_bench.storefront.server import start_server + +RS_URI = os.environ.get("MONGOSPECS_TEST_URI", "mongodb://localhost:27117/?replicaSet=rs0") +SECRET = "e2e-secret" + + +class Shop: + """A running storefront server plus direct database access for assertions.""" + + def __init__(self, server, registry, m, seeded, database, options): + self.server = server + self.registry = registry + self.m = m + self.seeded = seeded + self.database = database + self.options = options + self.tokens = Tokens(SECRET) + + @property + def url(self): + return self.server.base_url + + def user(self, index=0): + return {"authorization": f"Bearer {self.seeded.user_tokens[index]}"} + + @property + def admin(self): + return {"authorization": f"Bearer {self.seeded.admin_token}"} + + def manager(self, name): + return self.registry.manager(getattr(self.m, name)) + + async def restart(self, **overrides): + """Stop the server gracefully and start a new one on the same database.""" + await self.server.stop() + assert self.server.stopped_gracefully(), self.server.output() + self.server = await start_server( + database=self.database, mongo_uri=RS_URI, secret=SECRET, **{**self.options, **overrides} + ) + + +@pytest_asyncio.fixture +async def make_shop(): + """Factory: seed a fresh database and boot a real server process against it.""" + shops = [] + + async def factory( + *, plug="asgi", backend="msgspec", app="storefront", products=40, users=6, stock=100, projector=True + ): + database = f"ms_e2e_{uuid.uuid4().hex[:10]}" + m = shop_models.build(backend) + registry = Registry(RS_URI, database=database) + await registry.open() + seeded = await seed( + registry, m, Tokens(SECRET), products=products, users=users, stock=stock, reviews_per_product=2 + ) + options = {"plug": plug, "backend": backend, "app": app, "projector": projector} + server = await start_server(database=database, mongo_uri=RS_URI, secret=SECRET, **options) + shop = Shop(server, registry, m, seeded, database, options) + shops.append(shop) + return shop + + yield factory + for shop in shops: + code = await shop.server.stop() + await shop.registry.client.drop_database(shop.database) + await shop.registry.close() + assert shop.server.stopped_gracefully(), ( + f"server did not shut down cleanly (exit {code}):\n{shop.server.output()}" + ) + assert "Traceback" not in shop.server.output(), shop.server.output() diff --git a/bench/tests/test_e2e_storefront.py b/bench/tests/test_e2e_storefront.py new file mode 100644 index 0000000..68221b1 --- /dev/null +++ b/bench/tests/test_e2e_storefront.py @@ -0,0 +1,388 @@ +"""End-to-end tests: a real server process, real HTTP, real MongoDB. + +Every test boots ``uvicorn`` with the storefront application against a freshly +seeded database, talks to it over TCP, and checks the database directly for the +invariants that matter in production (no oversold stock, no partial orders, +nothing private in a response, a clean shutdown). +""" + +import asyncio + +import httpx +import pytest + +from mongospecs_bench.storefront.app import PLUGS +from mongospecs_bench.storefront.seed import stock_invariant + + +def client(shop, **kwargs): + return httpx.AsyncClient(base_url=shop.url, timeout=30, **kwargs) + + +async def eventually(check, *, seconds=20.0, every=0.1): + """Poll ``check`` (async, returns a truthy value when satisfied).""" + deadline = asyncio.get_running_loop().time() + seconds + while True: + value = await check() + if value: + return value + if asyncio.get_running_loop().time() > deadline: + raise AssertionError("condition was not met in time") + await asyncio.sleep(every) + + +@pytest.mark.parametrize("plug", PLUGS) +async def test_customer_journey_on_every_plug(make_shop, plug): + shop = await make_shop(plug=plug) + async with client(shop) as http: + health = (await http.get("/healthz")).json() + assert health == {"status": "ok", "backend": "msgspec", "plug": plug} + + # -- sign up and log in --------------------------------------------------------- + signup = await http.post( + "/api/users/signup", json={"email": "New@Example.com", "password": "correct horse", "name": "New"} + ) + assert signup.status_code == 201, signup.text + me = signup.json()["data"] + assert me["user"]["email"] == "new@example.com" and "password_hash" not in me["user"] + assert ( + await http.post( + "/api/users/signup", json={"email": "new@example.com", "password": "correct horse"} + ) + ).status_code == 409 + assert ( + await http.post("/api/users/signup", json={"email": "bad", "password": "x"}) + ).status_code == 422 + assert ( + await http.post("/api/users/login", json={"email": "new@example.com", "password": "wrong"}) + ).status_code == 401 + login = await http.post( + "/api/users/login", json={"email": "new@example.com", "password": "correct horse"} + ) + token = {"authorization": f"Bearer {login.json()['data']['token']}"} + assert (await http.get(f"/api/users/{me['user']['id']}", headers=token)).json()["data"][ + "name" + ] == "New" + assert (await http.get(f"/api/users/{me['user']['id']}", headers=shop.user(0))).status_code == 404 + assert (await http.get(f"/api/users/{me['user']['id']}")).status_code == 401 + + # -- browse ------------------------------------------------------------------------ + page = (await http.get("/api/products?category=books&sort=price&limit=3")).json() + prices = [p["price"] for p in page["data"]] + assert len(prices) == 3 and prices == sorted(prices) and page["page"]["has_next"] + assert all(p["category"] == "books" and "deleted_at" not in p for p in page["data"]) + following = ( + await http.get( + f"/api/products?category=books&sort=price&limit=3&after={page['page']['next_cursor']}" + ) + ).json() + assert following["data"][0]["price"] >= prices[-1] + assert {p["id"] for p in following["data"]}.isdisjoint(p["id"] for p in page["data"]) + assert (await http.get("/api/products?stock[$gt]=1")).status_code == 400 + assert (await http.get("/api/products?description=x")).status_code == 400 + product = page["data"][0] + detail = (await http.get(f"/api/products/{product['id']}?include=reviews,review_count")).json()[ + "data" + ] + assert ( + detail["review_count"] == 2 + and len(detail["reviews"]) == 2 + and detail["reviews"][0]["rating"] in range(1, 6) + ) + + # -- order: stock moves, the order is mine alone ------------------------------------- + other = page["data"][1] + placed = await http.post( + "/api/orders", + json={"lines": [{"sku": product["sku"], "qty": 2}, {"sku": other["sku"], "qty": 1}]}, + headers=token, + ) + assert placed.status_code == 201, placed.text + order = placed.json()["data"] + assert order["status"] == "placed" and order["user"] == me["user"]["id"] + assert order["total"] == round(2 * product["price"] + other["price"], 2) + assert sorted(line["sku"] for line in order["lines"]) == sorted([product["sku"], other["sku"]]) + assert (await http.get(f"/api/products/{product['id']}")).json()["data"]["stock"] == 98 + assert (await http.post("/api/orders", json={"lines": []}, headers=token)).status_code == 422 + assert ( + await http.post("/api/orders", json={"lines": [{"sku": product["sku"], "qty": 1}]}) + ).status_code == 401 + mine = (await http.get("/api/orders", headers=token)).json()["data"] + assert [o["id"] for o in mine] == [order["id"]] + assert (await http.get("/api/orders", headers=shop.user(0))).json()["data"] == [] + assert (await http.get(f"/api/orders/{order['id']}", headers=shop.user(0))).status_code == 404 + assert (await http.get("/api/orders")).status_code == 401 + assert len((await http.get("/api/orders", headers=shop.admin)).json()["data"]) == 1 + + # -- cancel: stock comes back, once ---------------------------------------------------- + assert (await http.post(f"/api/orders/{order['id']}/cancel", headers=shop.user(0))).status_code == 409 + cancelled = await http.post(f"/api/orders/{order['id']}/cancel", headers=token) + assert cancelled.status_code == 200 and cancelled.json()["data"]["status"] == "cancelled" + assert (await http.post(f"/api/orders/{order['id']}/cancel", headers=token)).status_code == 409 + assert (await http.get(f"/api/products/{product['id']}")).json()["data"]["stock"] == 100 + + # -- reviews: the server decides who the author is ----------------------------------------- + forged = await http.post( + "/api/reviews", + json={"product": product["id"], "rating": 5, "author": shop.seeded.product_ids[0]}, + headers=token, + ) + assert forged.status_code == 422 + assert ( + await http.post("/api/reviews", json={"product": product["id"], "rating": 9}, headers=token) + ).status_code == 422 + assert ( + await http.post("/api/reviews", json={"product": product["id"], "rating": 5}) + ).status_code == 401 + review = await http.post( + "/api/reviews", json={"product": product["id"], "rating": 5, "body": "great"}, headers=token + ) + assert review.status_code == 201 and review.json()["data"]["author"] == me["user"]["id"] + assert ( + await http.delete(f"/api/reviews/{review.json()['data']['id']}", headers=shop.user(1)) + ).status_code == 403 + listed = (await http.get(f"/api/reviews?product={product['id']}&sort=-created_at")).json()["data"] + assert len(listed) == 3 and listed[0]["body"] == "great" + + # -- admin: conditional update, protected fields, soft delete ------------------------------- + assert ( + await http.patch(f"/api/products/{product['id']}", json={"price": 1}, headers=token) + ).status_code == 403 + current = await http.get(f"/api/products/{product['id']}") + etag = current.headers["etag"] + changed = await http.patch( + f"/api/products/{product['id']}", json={"price": 9.5}, headers={**shop.admin, "if-match": etag} + ) + assert changed.status_code == 200 and changed.json()["data"]["price"] == 9.5 + stale = await http.patch( + f"/api/products/{product['id']}", json={"price": 1}, headers={**shop.admin, "if-match": etag} + ) + assert stale.status_code == 412 + assert ( + await http.patch(f"/api/products/{product['id']}", json={"stock": 10**6}, headers=shop.admin) + ).status_code == 422 + assert ( + await http.patch(f"/api/products/{product['id']}", json={"sku": "X"}, headers=shop.admin) + ).status_code == 422 + restocked = await http.post( + f"/api/products/{product['id']}/restock", json={"amount": 5}, headers=shop.admin + ) + assert restocked.json()["data"]["stock"] == 105 + assert (await http.delete(f"/api/products/{product['id']}", headers=shop.admin)).status_code == 204 + assert (await http.get(f"/api/products/{product['id']}")).status_code == 404 + assert ( + await http.post("/api/orders", json={"lines": [{"sku": product["sku"], "qty": 1}]}, headers=token) + ).status_code == 409 + raw = await shop.manager("Product").collection.find_one({"sku": product["sku"]}) + assert raw["deleted_at"] is not None and raw["price"] == 9.5 # soft-deleted, still there + assert await shop.manager("Review").collection.count_documents({"product": raw["_id"]}) == 3 + for problem in (stale, forged): + assert problem.headers["content-type"].startswith("application/problem+json") + + +@pytest.mark.parametrize("backend", ["pydantic", "attrs"]) +async def test_ordering_works_on_every_model_backend(make_shop, backend): + shop = await make_shop(backend=backend, plug="starlette", products=8) + async with client(shop) as http: + sku = shop.seeded.skus[0] + placed = await http.post( + "/api/orders", json={"lines": [{"sku": sku, "qty": 3}]}, headers=shop.user(0) + ) + assert placed.status_code == 201, placed.text + order = placed.json()["data"] + assert order["lines"][0]["qty"] == 3 and order["lines"][0]["product"] == shop.seeded.product_ids[0] + detail = ( + await http.get(f"/api/products/{shop.seeded.product_ids[0]}?include=reviews,review_count") + ).json()["data"] + assert detail["stock"] == 97 and detail["review_count"] == 2 + assert (await http.post(f"/api/orders/{order['id']}/cancel", headers=shop.user(0))).status_code == 200 + assert (await stock_invariant(shop.registry, shop.m, shop.seeded))["ok"] + + +async def test_concurrent_buyers_never_oversell(make_shop): + shop = await make_shop(products=3, users=40, stock=10) + sku = shop.seeded.skus[0] + async with client(shop, limits=httpx.Limits(max_connections=40)) as http: + responses = await asyncio.gather( + *( + http.post("/api/orders", json={"lines": [{"sku": sku, "qty": 1}]}, headers=shop.user(i)) + for i in range(40) + ) + ) + statuses = sorted(r.status_code for r in responses) + assert statuses == [201] * 10 + [409] * 30, statuses + assert (await http.get(f"/api/products/{shop.seeded.product_ids[0]}")).json()["data"]["stock"] == 0 + + # the change-stream projector catches up: exactly the ten sold units, counted once + async def stats_ready(): + rows = (await http.get(f"/api/sales?sku={sku}", headers=shop.admin)).json()["data"] + return rows if rows and rows[0]["units"] == 10 else None + + (stat,) = await eventually(stats_ready) + assert stat["units"] == 10 + invariant = await stock_invariant(shop.registry, shop.m, shop.seeded) + assert invariant == { + "orders_placed": 10, + "units_sold": 10, + "negative_stock": 0, + "stock_mismatches": 0, + "ok": True, + } + + +async def test_multi_item_orders_are_all_or_nothing(make_shop): + shop = await make_shop(products=4, stock=5) + plenty, scarce = shop.seeded.skus[0], shop.seeded.skus[1] + async with client(shop) as http: + refused = await http.post( + "/api/orders", + json={"lines": [{"sku": plenty, "qty": 2}, {"sku": scarce, "qty": 6}]}, + headers=shop.user(0), + ) + assert refused.status_code == 409 and scarce in refused.json()["detail"] + missing = await http.post( + "/api/orders", + json={"lines": [{"sku": plenty, "qty": 1}, {"sku": "NOPE", "qty": 1}]}, + headers=shop.user(0), + ) + assert missing.status_code == 409 + # contention across two products in opposite line order must neither deadlock nor oversell + mixed = await asyncio.gather( + *( + http.post( + "/api/orders", + json={"lines": [{"sku": a, "qty": 1}, {"sku": b, "qty": 1}]}, + headers=shop.user(i % 6), + ) + for i in range(12) + for a, b in ([plenty, scarce] if i % 2 else [scarce, plenty],) + ) + ) + assert sorted(r.status_code for r in mixed) == [201] * 5 + [409] * 7 + stocks = {d["sku"]: d["stock"] async for d in shop.manager("Product").collection.find({})} + assert stocks[plenty] == 0 and stocks[scarce] == 0 # nothing leaked from the refused orders + assert await shop.manager("Order").collection.count_documents({}) == 5 + assert (await stock_invariant(shop.registry, shop.m, shop.seeded))["ok"] + + +async def test_projector_resumes_after_a_restart_without_double_counting(make_shop): + shop = await make_shop(products=3, stock=1000) + sku = shop.seeded.skus[0] + + async def units(http): + rows = (await http.get(f"/api/sales?sku={sku}", headers=shop.admin)).json()["data"] + return rows[0]["units"] if rows else 0 + + async with client(shop) as http: + for _ in range(5): + assert ( + await http.post("/api/orders", json={"lines": [{"sku": sku, "qty": 2}]}, headers=shop.user(0)) + ).status_code == 201 + + async def ten(): + return await units(http) == 10 + + await eventually(ten) + # Orders written while no projector is running must be picked up from the stored resume token. + await shop.restart(projector=False) + async with client(shop) as http: + for _ in range(3): + assert ( + await http.post("/api/orders", json={"lines": [{"sku": sku, "qty": 1}]}, headers=shop.user(1)) + ).status_code == 201 + cancel_me = (await http.get("/api/orders?sort=-created_at&limit=1", headers=shop.user(1))).json()[ + "data" + ][0] + assert ( + await http.post(f"/api/orders/{cancel_me['id']}/cancel", headers=shop.user(1)) + ).status_code == 200 + assert await units(http) == 10 # nothing is projecting right now + await shop.restart() # back with the projector + async with client(shop) as http: + + async def caught_up(): + return await units(http) == 12 # 10 + 3 placed - 1 cancelled + + await eventually(caught_up) + await asyncio.sleep(1) + assert await units(http) == 12 # and it stays there: no event was applied twice + state = await shop.manager("ProjectorState").collection.find_one({"name": "sales"}) + assert state["events"] == 9 # 5 + 3 inserts and 1 cancellation + + +async def test_requests_in_flight_finish_on_graceful_shutdown(make_shop): + shop = await make_shop(products=200, stock=1000) + async with client(shop) as http: + slow = [ + asyncio.create_task(http.get("/api/products?limit=100&include=reviews,review_count&sort=price")) + for _ in range(20) + ] + await asyncio.sleep(0.02) + stopping = asyncio.create_task(shop.server.stop()) + responses = await asyncio.gather(*slow, return_exceptions=True) + await stopping + assert shop.server.stopped_gracefully(), shop.server.output() + finished = [r for r in responses if isinstance(r, httpx.Response)] + # requests the server had accepted were answered in full before it exited + assert finished and all(r.status_code == 200 and len(r.json()["data"]) == 100 for r in finished) + # anything else was refused at the connection level, never answered with a broken body + assert all(isinstance(r, httpx.Response | httpx.HTTPError) for r in responses) + + +@pytest.mark.chaos +async def test_orders_stay_consistent_across_a_primary_step_down(make_shop): + shop = await make_shop(products=20, users=10, stock=100_000) + statuses = [] + stop = asyncio.Event() + + async def shopper(index, http): + while not stop.is_set(): + sku = shop.seeded.skus[index % 20] + try: + response = await http.post( + "/api/orders", json={"lines": [{"sku": sku, "qty": 1}]}, headers=shop.user(index) + ) + statuses.append(response.status_code) + if response.status_code >= 400: + assert response.headers["content-type"].startswith("application/problem+json") + listing = await http.get("/api/products?limit=5") + statuses.append(listing.status_code) + except httpx.HTTPError: + statuses.append(0) + await asyncio.sleep(0.01) + + async with client(shop, limits=httpx.Limits(max_connections=20)) as http: + shoppers = [asyncio.create_task(shopper(i, http)) for i in range(10)] + await asyncio.sleep(1.5) + before = len(statuses) + try: + await shop.registry.client.admin.command("replSetStepDown", 3, force=True) + except Exception: + pass # the command closes connections; that is expected + await asyncio.sleep(1) + + async def healthy_again(): + try: + return ( + await http.post( + "/api/orders", + json={"lines": [{"sku": shop.seeded.skus[0], "qty": 1}]}, + headers=shop.user(0), + ) + ).status_code == 201 + except httpx.HTTPError: + return False + + await eventually(healthy_again, seconds=90, every=0.5) + await asyncio.sleep(1.5) + stop.set() + await asyncio.gather(*shoppers) + assert (await http.get("/healthz")).status_code == 200 + assert before > 20 and statuses.count(201) > before // 2 + # during the election requests may fail, but only as declared errors: never a 500, never a hang + assert set(statuses) <= {200, 201, 503, 504, 409}, sorted(set(statuses)) + invariant = await stock_invariant(shop.registry, shop.m, shop.seeded) + assert invariant["ok"], ( + invariant + ) # every accepted order moved stock exactly once; no rejected order moved any + assert invariant["orders_placed"] == statuses.count(201) + 1 diff --git a/docs/benchmarks.md b/docs/benchmarks.md new file mode 100644 index 0000000..649d4df --- /dev/null +++ b/docs/benchmarks.md @@ -0,0 +1 @@ +placeholder until results exist diff --git a/docs/guide/models.md b/docs/guide/models.md new file mode 100644 index 0000000..c6b9f87 --- /dev/null +++ b/docs/guide/models.md @@ -0,0 +1,134 @@ +# Models + +A model is a class from the library you already use, with the `Spec` base of that backend. +Everything else in mongospecs is imported from `mongospecs` and is the same for all three. + +```python +from datetime import datetime +from typing import Annotated + +from mongospecs import OID, CreatedAt, Deferred, Immutable, Index, Policy, Registry, SpecConfig, UpdatedAt +from mongospecs.msgspec import Spec # or: from mongospecs.pydantic import Spec / mongospecs.attrs + + +class User(Spec): + __spec__ = SpecConfig(collection="users", indexes=[Index("email", unique=True)]) + + email: Annotated[str, Immutable] + name: str = "" + password_hash: Annotated[str | None, Deferred, Policy(private=True)] = None + created_at: Annotated[datetime | None, CreatedAt] = None + updated_at: Annotated[datetime | None, UpdatedAt] = None +``` + +- Every document has an `id` attribute, stored as `_id`. Its type is `OID | None` unless you + redeclare it (`id: str | None = None` for string ids). +- `__spec__` is assigned **without an annotation**. It holds the collection name (default: the + class name), indexes, and per-model options. +- With attrs, decorate each model with `@attrs.define(kw_only=True)`. + +## Binding and lifecycle + +Models do nothing until they are bound to an open registry. Constructing a registry does no I/O; +`async with` opens it; closing is final. + +```python +registry = Registry("mongodb://localhost/app", models=[User]) +await registry.open() +await registry.apply_indexes() + +ann = await User(email="ann@example.com", name="Ann").insert() +assert isinstance(ann.id, OID.__origin__) and ann.created_at is not None +``` + +Operations on a registry that is not open fail immediately with `RegistryClosedError`; nothing is +queued. + +## Field markers + +Markers go in `typing.Annotated`, so one declaration works on every backend. + +| Marker | Effect | +|---|---| +| `CreatedAt`, `UpdatedAt` | Timestamps on every write path, including bulk and filter-based updates | +| `DeletedAt` | Soft delete: `delete()` sets it, reads exclude documents where it is set | +| `Revision` | Optimistic locking; the value doubles as an HTTP ETag | +| `Immutable` | Cannot change after insert | +| `Deferred` | Not loaded unless asked for with `select("+name")` | +| `Alias("x")` | Stored under a shorter name (top-level fields) | +| `Policy(...)` | How the field may be used from outside (read by `mongospecs-web`) | +| `Backref(...)`, `OnDelete(...)`, `To(...)` | Relations, see [Relations](relations.md) | + +```python +loaded = await User.get(ann.id) +assert loaded.password_hash is None # Deferred: not fetched + +with_hash = await User.find().select("+password_hash").one() +assert with_hash.email == "ann@example.com" +``` + +## Existing classes + +`SpecAdapter` turns a class you already have into a document class without touching it. + +```python +import msgspec + +from mongospecs.msgspec import SpecAdapter + + +class Dragon(msgspec.Struct, kw_only=True): + name: str + breed: str = "unknown" + + +DragonDoc = SpecAdapter(Dragon, collection="dragons") +registry.bind(DragonDoc) +burt = await DragonDoc(name="Burt").insert() +assert isinstance(burt, Dragon) and burt.id is not None +``` + +## Single-collection inheritance + +Set `discriminator` on the base; subclasses share the collection and are picked on read. + +```python +class Animal(Spec): + __spec__ = SpecConfig(collection="animals", discriminator="kind") + name: str = "" + + +class Cat(Animal): + lives: int = 9 + + +registry.bind(Animal, Cat) +await Cat(name="Tom").insert() +(animal,) = await Animal.find() +assert type(animal) is Cat +``` + +## Documents written by older code + +`upcast` runs on each raw document before it is decoded, so renamed or added fields can be +migrated lazily. Fields the model does not know are ignored on read and left alone on write. + +```python +def upcast(doc): + if "full_name" in doc: + doc["name"] = doc.pop("full_name") + return doc + + +class Person(Spec): + __spec__ = SpecConfig(collection="people", upcast=upcast) + name: str + + +registry.bind(Person) +await Person.collection().insert_one({"full_name": "Old Shape", "legacy_flag": True}) +person = await Person.find().one() +assert person.name == "Old Shape" + +await registry.close() +``` diff --git a/docs/guide/operations.md b/docs/guide/operations.md new file mode 100644 index 0000000..0feccdf --- /dev/null +++ b/docs/guide/operations.md @@ -0,0 +1,92 @@ +# Running in production + +```python +import logging +from typing import Annotated + +from mongospecs import Action, Index, Registry, SpecConfig, logging_listener +from mongospecs.msgspec import Spec + + +class Account(Spec): + __spec__ = SpecConfig(collection="accounts", indexes=[Index("email", unique=True), Index("-balance")]) + email: str = "" + balance: Annotated[int, "cents"] = 0 + + +events = [] +registry = await Registry( + "mongodb://localhost/app", + models=[Account], + max_time_ms=5000, # default time limit for reads + on_command=[events.append, logging_listener(level=logging.DEBUG)], +).open() +``` + +## Indexes + +Nothing is built implicitly. `plan_indexes()` compares what is declared with what exists; +`apply_indexes()` creates what is missing and only drops or recreates with `drop=True`. + +```python +plan = await registry.plan_indexes() +assert {a.name: a.action for a in plan} == {"email_1": Action.CREATE, "balance_-1": Action.CREATE} +await registry.apply_indexes(plan) +assert (await registry.plan_indexes()).in_sync +``` + +| Action | Meaning | Destructive | +|---|---|---| +| `CREATE` | declared, missing | no | +| `RETAIN` | declared and identical | no | +| `RECREATE` | same name, different definition | yes | +| `CONFLICT` | same keys under another name | yes | +| `DROP` | exists, not declared | yes | + +`plan.advisories` lists non-blocking findings, such as a reverse relation whose foreign key has no +index. + +## Observability + +Every driver command produces `started` and `succeeded`/`failed` events carrying the model and the +mongospecs operation. + +```python +await Account(email="a@example.com").insert() +started = [e for e in events if e.phase == "started" and e.model] +assert (started[-1].model, started[-1].op, started[-1].command_name) == ("Account", "insert", "insert") +``` + +`logging_listener()`, `otel_listener()` (extra `otel`) and `sentry_listener()` (extra `sentry`) are +ready-made listeners. With the Sentry SDK, database work shows up as `Account.insert`, +`Account.find` and so on inside the active trace, on Sentry or any compatible server. + +## Errors + +mongospecs raises its own errors and keeps the driver error as `__cause__`. + +| Error | When | +|---|---| +| `ValidationError` | input does not fit the model | +| `QueryError` | unknown field, uncastable value, untrusted operator | +| `DocumentNotFound` | an acknowledged read or write matched nothing | +| `StaleDocumentError` | a revision check failed | +| `DuplicateKeyError` | unique index violated; `.fields` names the values | +| `OperationTimeout`, `ConnectionFailed`, `PoolTimeout` | the database is slow or unreachable | +| `DocumentTooLarge` | a document or pipeline result exceeded 16MB | + +## More than one database + +```python +archive = Account.using(db="app_archive") +await archive.insert(Account(email="old@example.com")) +assert await archive.count() == 1 and await Account.count() == 1 + +await registry.client.drop_database("app_archive") +await registry.client.drop_database("app") +await registry.close() +``` + +Also: `SpecConfig(shard_key=[...])` adds the shard key to document-level writes so they are +targeted; `Model.watch()` opens a change stream; `SpecConfig(json_schema_validation="strict")` +installs a `$jsonSchema` validator derived from the model when indexes are applied. diff --git a/docs/guide/plugins.md b/docs/guide/plugins.md new file mode 100644 index 0000000..81428d0 --- /dev/null +++ b/docs/guide/plugins.md @@ -0,0 +1,68 @@ +# Hooks and plugins + +mongospecs has the same four kinds of middleware as Mongoose, with typed signatures. + +| Kind | Where | Runs for | +|---|---|---| +| Document hooks | `@Model.before("insert" \| "save" \| "delete")`, `@Model.after(...)` | document-level operations | +| Command and filter rewriting | `Plugin.on_insert`, `on_update`, `on_filter`, `on_delete` | every write and read, including filter-based ones | +| Pipeline rewriting | `Plugin.on_pipeline` | `Model.aggregate` | +| Bulk hooks | `Plugin.before_bulk`, `after_bulk` | `insert_many`, `bulk_write`, `bulk_save` | + +```python +from mongospecs import Plugin, Q, Registry, SpecConfig +from mongospecs.msgspec import Spec + + +class Note(Spec): + __spec__ = SpecConfig(collection="notes") + text: str = "" + slug: str = "" + + +@Note.before("insert") +async def slugify(ctx, note): + note.slug = note.text.lower().replace(" ", "-") +``` + +Hooks receive an `OpCtx` (the session, registry and model) and the document. They run in +registration order and the first exception stops the operation. + +Document hooks do not run for filter-based writes such as `Note.find(...).update(...)`: no +documents are loaded. Plugins still rewrite those commands, which is how timestamps, revisions and +scopes keep working there. + +## Writing a plugin + +A plugin overrides only what it needs. This one confines every model of a registry to one tenant. + +```python +class Tenant(Plugin): + name = "tenant" + + def __init__(self, tenant_id): + self.tenant_id = tenant_id + + def on_insert(self, ctx, doc): + doc["tenant_id"] = self.tenant_id + + def on_filter(self, ctx, flt): + return {"tenant_id": self.tenant_id} + + +registry = await Registry("mongodb://localhost/app", models=[Note], plugins=[Tenant("acme")]).open() +note = await Note(text="Hello World").insert() +assert note.slug == "hello-world" + +await Note.collection().insert_one({"text": "someone else's", "tenant_id": "other"}) +assert [n.text for n in await Note.find()] == ["Hello World"] +assert await Note.find().update(text="changed") == 1 # the other tenant's note is untouched +assert await Note.find().unscoped("tenant").count() == 2 + +await registry.client.drop_database("app") +await registry.close() +``` + +Plugins attach in three ways: by a field marker (the built-in timestamps, soft delete and +revisions), per model with `SpecConfig(plugins=[...])`, or to every model with +`Registry(plugins=[...])`. diff --git a/docs/guide/queries.md b/docs/guide/queries.md new file mode 100644 index 0000000..5519d44 --- /dev/null +++ b/docs/guide/queries.md @@ -0,0 +1,129 @@ +# Queries + +Queries are immutable and chainable. Nothing is sent until you await a terminal method or iterate. + +```python +from typing import Annotated + +from mongospecs import Deferred, Q, QueryError, Registry, SpecConfig, trusted +from mongospecs.msgspec import Spec + + +class Article(Spec): + __spec__ = SpecConfig(collection="articles") + title: str = "" + views: int = 0 + tags: list[str] = [] + body: Annotated[str, Deferred] = "" + + +registry = await Registry("mongodb://localhost/app", models=[Article]).open() +await Article.insert_many([Article(title=f"post {i}", views=i * 10, tags=["a"] if i % 2 else ["b"]) for i in range(10)]) +``` + +## Conditions + +`Q.field` builds conditions. Field names are checked against the model and values are cast to the +field's type, so a typo is an error instead of a query that silently matches nothing. + +```python +popular = await Article.find(Q.views >= 50, Q.title.prefix("post")).sort("-views").limit(3) +assert [a.views for a in popular] == [90, 80, 70] + +either = await Article.find((Q.views < 10) | (Q.views > 80)).sort("views").all() +assert [a.views for a in either] == [0, 90] + +assert await Article.count(Q.tags == "a") == 5 # a scalar against a list field means "contains" +assert await Article.exists(~Q.title.prefix("post")) is False + +try: + Article.find(Q.titel == "x").compile() +except QueryError as exc: + assert "no field 'titel'" in str(exc) +``` + +Operators: `== != > >= < <=`, `.in_()`, `.nin()`, `.all()`, `.exists()`, `.size()`, +`.prefix()`, `.contains()`, `.regex()`, `.elem_match()`, combined with `&`, `|`, `~` or +`And`/`Or`/`Nor`. + +Plain dicts work for equality. Anything containing a MongoDB operator must be wrapped in +`trusted()`, which marks it as written by you rather than built from user input. + +```python +assert await Article.count({"title": "post 3"}) == 1 +assert await Article.count(trusted({"views": {"$mod": [20, 0]}})) == 5 +``` + +## Results + +```python +first = await Article.find(Q.views > 0).sort("views").first() # a document, or None +one = await Article.find(Q.title == "post 4").one() # raises DocumentNotFound if missing +by_id = await Article.get(one.id) +assert first.views == 10 and by_id.title == "post 4" + +async for article in Article.find().sort("-views").limit(2): + assert article.views >= 80 + +titles = [a["title"] for a in await Article.find().raw().limit(2)] # plain dicts +``` + +| Mode | Returns | Tracked for `save()` | +|---|---|---| +| default | model instances | yes | +| `.lean()` | model instances | no (read-only) | +| `.raw()` | dicts as stored | no | + +## Projection + +`select("a", "b")` includes, `select("-a")` excludes, `select("+a")` adds a `Deferred` field. +A document loaded with a projection only saves the fields that were loaded. + +```python +partial = await Article.find().select("title").first() +assert partial.title and partial.views == 0 # not loaded: the default, not the stored value +``` + +## Pagination + +Cursor pagination is the default and stays stable while documents are inserted. Offset pagination +and totals are available when you need them. + +Cursor pages are sorted by your keys and then by `_id`, so that the order is total. Give the query +an index that ends in the id (`Index("views", "id")`); with an index on the key alone MongoDB has +to sort in memory. + +```python +page = await Article.find().sort("-views").page(4) +assert [a.views for a in page] == [90, 80, 70, 60] and page.has_next + +following = await Article.find().sort("-views").page(4, after=page.next_cursor) +assert [a.views for a in following] == [50, 40, 30, 20] + +third = await Article.find().sort("views").page(4, number=3, total=True) +assert [a.views for a in third] == [80, 90] and third.total == 10 +``` + +## Named scopes + +```python +@Article.scope +def trending(query, minimum=50): + return query.filter(Q.views >= minimum) + + +assert await Article.find().trending(80).count() == 2 +``` + +## Escape hatches + +`Model.aggregate(pipeline, as_=...)` runs a raw pipeline (scopes still apply) and +`Model.collection()` returns the driver collection. + +```python +rows = await Article.aggregate([{"$group": {"_id": None, "views": {"$sum": "$views"}}}]) +assert rows[0]["views"] == 450 + +await registry.client.drop_database("app") +await registry.close() +``` diff --git a/docs/guide/relations.md b/docs/guide/relations.md new file mode 100644 index 0000000..ee2677a --- /dev/null +++ b/docs/guide/relations.md @@ -0,0 +1,93 @@ +# Relations + +```python +from typing import Annotated, Any + +from mongospecs import Backref, Index, NotLoadedError, OnDelete, Q, Ref, Registry, SpecConfig +from mongospecs.msgspec import Spec + + +class Author(Spec): + __spec__ = SpecConfig(collection="authors") + name: str = "" + books: Annotated[list[Any], Backref("author", model="Book")] = [] + book_count: Annotated[int, Backref("author", model="Book", count=True)] = 0 + + +class Book(Spec): + __spec__ = SpecConfig(collection="books", indexes=[Index("author")]) + title: str = "" + author: Annotated[Ref[Author] | None, OnDelete("cascade")] = None + editors: list[Ref[Author]] = [] + + +registry = await Registry("mongodb://localhost/app", models=[Author, Book]).open() +ann = await Author(name="Ann").insert() +bob = await Author(name="Bob").insert() +await Book.insert_many([Book(title="One", author=ann, editors=[bob]), Book(title="Two", author=ann)]) +``` + +## References + +`Ref[T]` is stored as the target's id. It always has `.id`; the document is available after +`include()` or `fetch()`. + +```python +book = await Book.find(Q.title == "One").one() +assert book.author.id == ann.id and not book.author.loaded +try: + book.author.doc +except NotLoadedError: + pass +assert (await book.author.fetch()).name == "Ann" + +assert await Book.count(Q.author == ann) == 2 # documents and ids both work in conditions +``` + +## include() + +```python +books = await Book.find().sort("title").include("author").include("editors") +assert books[0].author.doc.name == "Ann" +assert [e.doc.name for e in books[0].editors] == ["Bob"] +``` + +Options per relation: `match`, `select`, `sort`, `limit` (overall), `per_parent_limit`, and nested +paths such as `include("author.books")`. + +`include()` can be served by one extra query per relation (`strategy="batch"`, the default) or by a +`$lookup` stage in a single pipeline (`strategy="lookup"`). Both return the same result; which is +faster depends on fan-out and topology, see [Benchmarks](../benchmarks.md). + +```python +same = await Book.find().sort("title").include("author", strategy="lookup") +assert same[0].author.doc.name == "Ann" +``` + +## Reverse relations + +A `Backref` field is never stored. It is filled only when included; with `count=True` it holds the +number of referencing documents. + +```python +author = await Author.find(Q.name == "Ann").include("books", sort=["title"]).include("book_count").one() +assert [b.title for b in author.books] == ["One", "Two"] and author.book_count == 2 +``` + +## Cascades + +`OnDelete` on a reference says what happens to the documents holding it when the target is +deleted: `"cascade"` deletes them, `"nullify"` clears the field, `"pull"` removes the id from a +list. The delete and its cascades run in one transaction. + +```python +await ann.delete() +assert await Book.count() == 0 + +await registry.client.drop_database("app") +await registry.close() +``` + +Also available: polymorphic references (`Ref[Post | Video]`), `dict[str, Ref[T]]`, +`Model.find(...).graph("parent")` for `$graphLookup` over a self-reference, and +`a.find(...).union(b.find(...))` for `$unionWith` across collections. diff --git a/docs/guide/writes.md b/docs/guide/writes.md new file mode 100644 index 0000000..054de90 --- /dev/null +++ b/docs/guide/writes.md @@ -0,0 +1,159 @@ +# Writes + +```python +from datetime import datetime +from typing import Annotated + +from mongospecs import ( + CreatedAt, + DeletedAt, + DocumentNotFound, + Inc, + Push, + Q, + Registry, + Revision, + SetOnInsert, + SpecConfig, + StaleDocumentError, + UpdatedAt, + UpdateOne, +) +from mongospecs.msgspec import Spec + + +class Article(Spec): + __spec__ = SpecConfig(collection="articles") + title: str = "" + views: int = 0 + tags: list[str] = [] + created_at: Annotated[datetime | None, CreatedAt] = None + updated_at: Annotated[datetime | None, UpdatedAt] = None + deleted_at: Annotated[datetime | None, DeletedAt] = None + rev: Annotated[int, Revision] = 0 + + +registry = await Registry("mongodb://localhost/app", models=[Article]).open() +``` + +## Change tracking + +A document remembers what it looked like when it was loaded. `save()` sends only the difference; +an unchanged document sends nothing. + +```python +article = await Article(title="Draft").insert() +article.title = "Final" +article.tags.append("news") +assert article.changes() == {"$set": {"title": "Final", "tags": ["news"]}} +await article.save() +assert article.changes() == {} and article.rev == 1 +``` + +Nested documents are diffed into dotted paths. A changed list is sent whole; for element-level +changes that must not overwrite concurrent writers, use the atomic operators below. + +## Atomic updates + +`Model.update` changes one document in a single round trip and returns it. Every supplied field +is validated; unknown and immutable fields raise. + +```python +updated = await Article.update(article.id, Inc(views=1), Push(tags="featured"), title="Final v2") +assert updated.views == 1 and updated.tags == ["news", "featured"] and updated.rev == 2 + +assert await Article.find(Q.views >= 1).update(Inc(views=10)) == 1 # every match, no documents loaded + +created = await Article.upsert(Q.title == "Landing", Inc(views=1), SetOnInsert(tags=["page"])) +assert created.views == 1 and created.created_at is not None +``` + +Operators: `Set`, `Unset`, `Inc`, `Mul`, `Min`, `Max`, `Push`, `AddToSet`, `Pull`, `Pop`, +`SetOnInsert`, `CurrentDate`; `Each([...])` for several values. + +## Optimistic locking + +With a `Revision` field, saving a stale copy fails instead of overwriting. + +```python +stale = article # loaded at rev 1, the database is now at rev 3 +stale.title = "Overwrite?" +try: + await stale.save() +except StaleDocumentError: + await stale.reload() +assert stale.title == "Final v2" +``` + +## Soft delete + +With a `DeletedAt` field, `delete()` hides the document instead of removing it. + +```python +await stale.delete() +assert await Article.get(stale.id) is None +assert await Article.find().unscoped("soft_delete").count() == 2 +await Article.find(Q.id == stale.id).unscoped("soft_delete").update(deleted_at=None) # restore +await (await Article.get(stale.id)).delete(hard=True) # really remove +``` + +## Bulk + +```python +result = await Article.insert_many([Article(title=f"bulk {i}") for i in range(3)]) +assert len(result.inserted) == 3 and not result.errors + +docs = await Article.find(Q.title.prefix("bulk")).all() +for doc in docs: + doc.views += 5 +assert await Article.bulk_save(docs) == 3 # one round trip, deltas only + +await Article.bulk_write([UpdateOne(Q.title == "bulk 0", Inc(views=1))]) +assert (await Article.find(Q.title == "bulk 0").one()).views == 6 +``` + +`insert_many(..., ordered=False)` attempts every document and reports failures per item instead of +raising on the first. + +## Transactions + +`registry.transaction(fn)` commits when `fn` returns. The driver re-runs `fn` on transient errors, +so it must be safe to call twice. If the transaction aborts, documents written inside it are put +back to their previous in-memory state. + +```python +landing = await Article.find(Q.title == "Landing").one() + + +async def promote(tx): + landing.tags.append("promoted") + await landing.save(session=tx) + raise RuntimeError("changed my mind") + + +try: + await registry.transaction(promote) +except RuntimeError: + pass + +assert (await Article.get(landing.id)).tags == ["page"] # rolled back in the database +assert landing.rev == 1 and landing.changes() == {"$set": {"tags": ["page", "promoted"]}} # and in memory +``` + +Sessions are always explicit: pass `session=tx` to every operation inside the function. For manual +control use `async with registry.session() as s, s.start_transaction(): ...` (no automatic retry). + +```python +try: + await Article.update(landing.id, title=123) +except Exception as exc: + assert type(exc).__name__ == "ValidationError" + +try: + await Article.update("0" * 24, title="nobody") +except DocumentNotFound: + pass + +await registry.client.drop_database("app") +await registry.close() +``` diff --git a/docs/index.md b/docs/index.md index 54206c3..90f345f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,44 @@ # mongospecs -An async MongoDB ODM for msgspec, pydantic and attrs models, and a framework-neutral REST layer. +An async MongoDB ODM for the model library you already use (msgspec, pydantic or attrs) and a +framework-neutral REST layer on top of it. -- [Design and work graph](https://github.com/jaykv/mongospecs/blob/main/claudedocs/v2-plan.md) -- [v1 test inventory](v1-test-inventory.md): what became of every 0.1.x behaviour +``` +pip install "mongospecs[msgspec]" # or [pydantic], [attrs] +pip install "mongospecs-web[fastapi]" # or [starlette], [litestar], [quart]; none needed for plain ASGI +``` + +Requires Python 3.11+ and MongoDB 7.0+. Status: alpha. + +## Where to start + +- [Models](guide/models.md): declaring documents, binding them, field markers +- [Queries](guide/queries.md) and [Writes and transactions](guide/writes.md) +- [Relations](guide/relations.md), [Hooks and plugins](guide/plugins.md) +- [Running in production](guide/operations.md): indexes, observability, errors +- [Resources](web/resources.md) and [Plugs](web/plugs.md): the REST layer +- [Coming from Mongoose](parity.md), [Migrating from 0.1.x](migration.md) +- [Benchmarks](benchmarks.md): measured against real MongoDB + +Every example in the guide is executed by the test suite. + +## What it looks like + +```py +class Article(Spec): + __spec__ = SpecConfig(collection="articles", indexes=[Index("title", unique=True)]) + + title: Annotated[str, Policy(filter=("eq", "prefix"), sort=True)] + views: int = 0 + author: Ref[User] | None = None + updated_at: Annotated[datetime | None, UpdatedAt] = None + rev: Annotated[int, Revision] = 0 + + +async with Registry("mongodb://localhost/app", models=[User, Article]) as registry: + article = await Article.find(Q.title.prefix("Hello")).include("author").one() + article.views += 1 + await article.save() # sends {"$set": {"views": ...}} and checks the revision + +app = Api(Resource(Article), prefix="/api").asgi(registry=registry) # a REST API, on any framework +``` diff --git a/docs/migration.md b/docs/migration.md new file mode 100644 index 0000000..1937b7a --- /dev/null +++ b/docs/migration.md @@ -0,0 +1,66 @@ +# Migrating from 0.1.x + +0.2 is a rewrite. It is async-only, needs Python 3.11+ and MongoDB 7.0+, and its API is not +compatible with 0.1.x. The synchronous line stays available on the +[`0.1.x` branch](https://github.com/jaykv/mongospecs/tree/0.1.x) and as `mongospecs<0.2`. + +What stayed: models on msgspec, pydantic or attrs; `from mongospecs. import Spec`; `Q` +expressions and `And`/`Or`/`Nor`/`In`/...; `SpecAdapter`. + +## The shape of the change + +```py +# 0.1.x +Spec._client = MongoClient("mongodb://localhost/app") +burt = Dragon(name="Burt") +burt.insert() +doc = Dragon.one(Q.name == "Burt") + +# 0.2 +async with Registry("mongodb://localhost/app", models=[Dragon]): + burt = await Dragon(name="Burt").insert() + doc = await Dragon.find(Q.name == "Burt").first() +``` + +## API map + +| 0.1.x | 0.2 | Notes | +|---|---|---| +| `Spec._client = client` | `Registry(uri, models=[...])` | explicit lifecycle, no class-level client | +| `_collection = "dragons"` | `__spec__ = SpecConfig(collection="dragons")` | assigned without an annotation | +| `_id` / `id` depending on backend | `id` everywhere, stored as `_id` | | +| `Model.by_id(id)` | `await Model.get(id)` | | +| `Model.one(filter)` | `await Model.find(filter).first()` or `.one()` | `.one()` raises when nothing matches | +| `Model.many(filter)` | `await Model.find(filter)` | also `async for` | +| `Model.find(...)`, `find_one(...)` (raw dicts) | `Model.find(...).raw()` | | +| `Model.ids(filter)` | `await Model.find(filter).ids()` | | +| `Model.count(filter)` | `await Model.count(filter)` | always exact; `estimated_count()` is separate | +| `doc.update()` / `update("field")` | `await doc.save()` | sends only what changed | +| `doc.upsert()` | `await Model.upsert(filter, ...)` | one round trip | +| `doc.unset("field")` | `await Model.update(id, Unset("field"))` | | +| `Model.update_many(docs)` | `await Model.bulk_save(docs)` or `Model.find(...).update(...)` | | +| `Model.delete_many(docs)` | `await Model.find(Q.id.in_(ids)).delete()` | | +| projection with `$ref` | `.include("field")` | `Ref[T]` fields | +| projection with `$sub` / `SubSpec` | a nested model as the field type | decoded by the backend | +| `Model.cascade` / `nullify` / `pull` | `OnDelete("cascade" \| "nullify" \| "pull")` on the reference | | +| `Model.listen("insert", fn)` | `@Model.before("insert")` / `@Model.after(...)` | async, typed, no global signals | +| `soft_delete()` / `find_active()` | a `DeletedAt` field | scope applied everywhere | +| `with Model.transaction() as session` | `await registry.transaction(fn)` | retried, restores document state | +| `Model.with_options(...)` | `Model.using(read_preference=..., db=...)` | returns a manager, changes nothing globally | +| `Paginator(Model, ...)` | `await Model.find(...).page(size, after=...)` | cursor by default; `number=` for offsets | +| `create_index` / `drop_index` | `SpecConfig(indexes=[Index(...)])` + `registry.apply_indexes()` | | +| `to_json_type()` | your backend's own serialisation, or `mongospecs_web.view_of(Model).serialize(doc)` | | +| `Empty` | `None`, plus `Unset(...)` for removing a field | | + +## Behaviour differences worth knowing + +- Reads are validated. A stored document that does not fit the model raises `ValidationError` + instead of producing a half-typed object. `SpecConfig(upcast=...)` migrates old shapes on read. +- Unknown field names in conditions and updates raise `QueryError`. +- Raw dict filters may not contain operators unless wrapped in `trusted(...)`. +- Reference fields hold a `Ref` object (`.id`, `.doc`), not a bare id or a document. +- msgspec is no longer installed implicitly: pick an extra (`mongospecs[msgspec]`, `[pydantic]`, + `[attrs]`). +- Documents compare with their backend's own equality, not by id. + +Every 0.1.x test is accounted for in the [v1 test inventory](v1-test-inventory.md). diff --git a/docs/parity.md b/docs/parity.md new file mode 100644 index 0000000..0ff8e17 --- /dev/null +++ b/docs/parity.md @@ -0,0 +1,68 @@ +# Coming from Mongoose + +mongospecs takes Mongoose as its reference for what a production ODM should do. This page maps +Mongoose features to their mongospecs form and to the tests that cover them, and lists where +mongospecs deliberately behaves differently. + +Test paths are relative to `packages/mongospecs/tests/`. "Source" says how the Mongoose behaviour +was established: **docs** means it was re-read in the Mongoose documentation for this project, +**memory** means it is stated from general knowledge of Mongoose 8.x and has not been re-checked. + +## Feature map + +| Mongoose | mongospecs | Covered by | Source | +|---|---|---|---| +| Change tracking, delta `save()`, `isNew`, `modifiedPaths()` | snapshot at load; `save()` sends the diff; `doc.is_new`, `doc.changes()` | `test_crud.py::test_save_new_then_delta`, `test_tracking_tx.py` | memory | +| `lean()` | `.lean()` (models, untracked), `.raw()` (dicts) | `test_crud.py::test_read_modes` | docs | +| `Model.hydrate()` | `Model.hydrate(raw)` | `test_crud.py::test_hydrate_and_collection_escape_hatch` | memory | +| Middleware: document, query, aggregate, model | document hooks; `on_insert`/`on_update`/`on_filter`/`on_delete`; `on_pipeline`; `before_bulk`/`after_bulk` | `test_plugins_hooks.py` | docs | +| `schema.plugin()`, `mongoose.plugin()` | marker plugins, `SpecConfig(plugins=)`, `Registry(plugins=)` | `test_plugins_hooks.py::test_registry_level_plugin_applies_to_every_model` | memory | +| `timestamps` | `CreatedAt`, `UpdatedAt` | `test_plugins_hooks.py::test_timestamps_on_every_write_path` | memory | +| `optimisticConcurrency`, `versionKey` | `Revision` | `test_plugins_hooks.py::test_revision_increments_everywhere` | memory | +| `populate()` with `match`, `select`, `options`, `perDocumentLimit`, nested paths | `include()` with `match`, `select`, `sort`, `limit`, `per_parent_limit`, dotted paths | `test_relations.py` | memory | +| Virtual populate (`localField`/`foreignField`, `count`) | `Backref(...)`, `Backref(..., count=True)` | `test_relations.py::test_backrefs_and_counts` | memory | +| `refPath` | `Ref[A \| B]` | `test_relations.py::test_polymorphic_reference` | memory | +| Discriminators | `SpecConfig(discriminator=...)` | `test_models_misc.py::test_discriminators_share_a_collection` | docs | +| Query casting | values cast to the field type | `test_expr.py::test_values_are_cast_to_field_types` | memory | +| `strictQuery: 'throw'` | always: unknown fields raise | `test_expr.py::test_field_names_are_translated_and_checked` | docs | +| `sanitizeFilter`, `trusted()` | raw filters untrusted unless `trusted()` | `test_expr.py::test_untrusted_mappings_and_operator_injection` | memory | +| `select: false` | `Deferred` | `test_crud.py::test_deferred_field_is_not_loaded_or_clobbered` | docs | +| `immutable` | `Immutable` | `test_updates.py::test_immutable_and_required_fields_are_protected` | memory | +| `alias` | `Alias("x")` | `test_codec.py::test_alias_and_transient_fields` | memory | +| Query helpers | `@Model.scope` | `test_plugins_hooks.py::test_named_scopes` | memory | +| `orFail()` | `.or_fail()`, `.one()` | `test_crud.py::test_find_all_first_one_and_or_fail` | memory | +| `cursor()`, `eachAsync({parallel})` | `async for`, `.each(fn, concurrency=n)` | `test_crud.py::test_each_with_concurrency` | memory | +| `bulkWrite`, `bulkSave`, `insertMany({ordered: false})` | `bulk_write`, `bulk_save`, `insert_many(ordered=False)` | `test_bulk_pagination.py` | memory | +| `Connection.transaction(fn)` resets document state on abort | `registry.transaction(fn)` does the same, also before each retry | `test_tracking_tx.py::test_transaction_abort_rolls_back_database_and_memory` | docs | +| `syncIndexes`, `diffIndexes`, `autoIndex: false` | `plan_indexes`, `apply_indexes(drop=...)`; never automatic | `test_indexes.py` | memory | +| `shardKey` | `SpecConfig(shard_key=...)` | `test_streams_sharded.py::test_shard_key_is_added_to_document_writes` | memory | +| `createConnection`, `useDb` | several registries; `Model.using(db=...)` | `test_models_misc.py::test_using_another_database` | memory | +| `maxTimeMS`, `hint`, `comment`, `collation`, read/write concern | `.options(...)`, `SpecConfig`, `Registry(max_time_ms=)` | `test_registry_errors.py::test_default_max_time_ms_from_registry_and_model` | memory | +| `Model.aggregate()`, `Model.collection` | `Model.aggregate(pipeline, as_=)`, `Model.collection()` | `test_crud.py::test_aggregate_decodes_and_explain` | memory | +| `mongoose.set('debug')` | `Registry(on_command=...)` with logging, OpenTelemetry, Sentry listeners | `test_registry_errors.py::test_command_events_carry_model_and_operation` | memory | +| `Model.watch()` | `Model.watch()` | `test_streams_sharded.py::test_change_stream_events` | memory | +| `exists`, `countDocuments`, `estimatedDocumentCount` | `exists`, `count`, `estimated_count` | `test_crud.py::test_count_exists_ids_estimated` | memory | +| Capped, time-series, TTL options | `SpecConfig(collection_options=)`, `Index(ttl=)` | `test_indexes.py::test_collection_options_are_applied` | memory | +| Subdocuments, arrays of subdocuments, Maps | nested models, `list[T]`, `dict[str, T]`; tracked through the parent | `test_crud.py::test_nested_document_diff_uses_dotted_paths` | memory | +| Getters, setters, custom validators | your backend's converters and validators | `test_codec.py::test_validate_field_runs_the_models_own_validators` | memory | +| Virtuals | Python properties; `@computed` to include one in API output | `packages/mongospecs-web/tests/test_mongo_resource.py` | memory | +| Statics and methods | classmethods and methods | (plain Python) | memory | + +## Deliberate differences + +| Mongoose | mongospecs | Why | +|---|---|---| +| Global default connection and model registry | Explicit `Registry`; names resolved per registry | No hidden state, isolated tests | +| `bufferCommands` queues operations until connected | Operations fail immediately when the registry is not open | A dead connection should not look like a slow one | +| `strictQuery` can strip unknown filter paths, so a typo matches everything | Unknown fields raise `QueryError` | Fail closed | +| `strict` silently strips unknown update paths | Unknown fields raise | Same | +| Update validators are off unless `runValidators` | Atomic updates are always validated | Unvalidated writes should be the explicit case | +| Filter casting follows schema types | Same; storage validators are not applied to filters | A search for a value that would be invalid to store is legitimate | +| `findOneAndUpdate` returns the document as it was unless `new: true` | `Model.update` returns the updated document; `returning="before"` for the old one | Least surprise | +| `Model.aggregate()` results are uncast and plugins must hook it | Scopes apply to `aggregate()`; `as_=` decodes | Soft delete and tenancy should not have a hole | +| Indexes are built on startup by default | Never automatic | Unsafe on large collections | +| Collection names are pluralised | The class name, or what you declare | No guessing | +| `transactionAsyncLocalStorage` offers an ambient session | Sessions are always passed explicitly | Tasks spawned inside a transaction would silently share one session | +| Populated paths replace the id with the document | A `Ref` keeps `.id` and gains `.doc` | One type whether or not it is loaded | +| Arrays are tracked per element | A changed list is sent whole | Per-element intent is expressed with atomic operators | +| A document's delete is not version-checked | Soft delete and `save()` are revision-checked; hard delete is not | Recorded as a known gap | diff --git a/docs/reference/bulk.md b/docs/reference/bulk.md new file mode 100644 index 0000000..e54108c --- /dev/null +++ b/docs/reference/bulk.md @@ -0,0 +1,3 @@ +# mongospecs.bulk + +::: mongospecs.bulk diff --git a/docs/reference/errors.md b/docs/reference/errors.md new file mode 100644 index 0000000..4c9c04d --- /dev/null +++ b/docs/reference/errors.md @@ -0,0 +1,3 @@ +# mongospecs.errors + +::: mongospecs.errors diff --git a/docs/reference/expr.md b/docs/reference/expr.md new file mode 100644 index 0000000..62e8f7e --- /dev/null +++ b/docs/reference/expr.md @@ -0,0 +1,3 @@ +# mongospecs.expr + +::: mongospecs.expr diff --git a/docs/reference/hooks.md b/docs/reference/hooks.md new file mode 100644 index 0000000..19139a3 --- /dev/null +++ b/docs/reference/hooks.md @@ -0,0 +1,3 @@ +# mongospecs.hooks + +::: mongospecs.hooks diff --git a/docs/reference/indexes.md b/docs/reference/indexes.md new file mode 100644 index 0000000..c6f622a --- /dev/null +++ b/docs/reference/indexes.md @@ -0,0 +1,3 @@ +# mongospecs.indexes + +::: mongospecs.indexes diff --git a/docs/reference/manager.md b/docs/reference/manager.md new file mode 100644 index 0000000..a810f89 --- /dev/null +++ b/docs/reference/manager.md @@ -0,0 +1,3 @@ +# mongospecs.manager + +::: mongospecs.manager diff --git a/docs/reference/markers.md b/docs/reference/markers.md new file mode 100644 index 0000000..052b33c --- /dev/null +++ b/docs/reference/markers.md @@ -0,0 +1,3 @@ +# mongospecs.markers + +::: mongospecs.markers diff --git a/docs/reference/model.md b/docs/reference/model.md new file mode 100644 index 0000000..b27b3db --- /dev/null +++ b/docs/reference/model.md @@ -0,0 +1,3 @@ +# mongospecs.model + +::: mongospecs.model diff --git a/docs/reference/query.md b/docs/reference/query.md new file mode 100644 index 0000000..705ce75 --- /dev/null +++ b/docs/reference/query.md @@ -0,0 +1,3 @@ +# mongospecs.query + +::: mongospecs.query diff --git a/docs/reference/refs.md b/docs/reference/refs.md new file mode 100644 index 0000000..4cddce7 --- /dev/null +++ b/docs/reference/refs.md @@ -0,0 +1,3 @@ +# mongospecs.refs + +::: mongospecs.refs diff --git a/docs/reference/registry.md b/docs/reference/registry.md new file mode 100644 index 0000000..0a4ef3b --- /dev/null +++ b/docs/reference/registry.md @@ -0,0 +1,3 @@ +# mongospecs.registry + +::: mongospecs.registry diff --git a/docs/reference/relations.md b/docs/reference/relations.md new file mode 100644 index 0000000..0197c5f --- /dev/null +++ b/docs/reference/relations.md @@ -0,0 +1,3 @@ +# mongospecs.relations + +::: mongospecs.relations diff --git a/docs/reference/streams.md b/docs/reference/streams.md new file mode 100644 index 0000000..5efd0d1 --- /dev/null +++ b/docs/reference/streams.md @@ -0,0 +1,3 @@ +# mongospecs.streams + +::: mongospecs.streams diff --git a/docs/reference/updates.md b/docs/reference/updates.md new file mode 100644 index 0000000..67328f7 --- /dev/null +++ b/docs/reference/updates.md @@ -0,0 +1,3 @@ +# mongospecs.updates + +::: mongospecs.updates diff --git a/docs/reference/web_api.md b/docs/reference/web_api.md new file mode 100644 index 0000000..fe84991 --- /dev/null +++ b/docs/reference/web_api.md @@ -0,0 +1,3 @@ +# mongospecs_web.api + +::: mongospecs_web.api diff --git a/docs/reference/web_backend.md b/docs/reference/web_backend.md new file mode 100644 index 0000000..073ed9e --- /dev/null +++ b/docs/reference/web_backend.md @@ -0,0 +1,3 @@ +# mongospecs_web.backend + +::: mongospecs_web.backend diff --git a/docs/reference/web_errors.md b/docs/reference/web_errors.md new file mode 100644 index 0000000..d43683b --- /dev/null +++ b/docs/reference/web_errors.md @@ -0,0 +1,3 @@ +# mongospecs_web.errors + +::: mongospecs_web.errors diff --git a/docs/reference/web_filters.md b/docs/reference/web_filters.md new file mode 100644 index 0000000..a381d9e --- /dev/null +++ b/docs/reference/web_filters.md @@ -0,0 +1,3 @@ +# mongospecs_web.filters + +::: mongospecs_web.filters diff --git a/docs/reference/web_openapi.md b/docs/reference/web_openapi.md new file mode 100644 index 0000000..4ef594c --- /dev/null +++ b/docs/reference/web_openapi.md @@ -0,0 +1,3 @@ +# mongospecs_web.openapi + +::: mongospecs_web.openapi diff --git a/docs/reference/web_policy.md b/docs/reference/web_policy.md new file mode 100644 index 0000000..56aef52 --- /dev/null +++ b/docs/reference/web_policy.md @@ -0,0 +1,3 @@ +# mongospecs_web.policy + +::: mongospecs_web.policy diff --git a/docs/reference/web_resource.md b/docs/reference/web_resource.md new file mode 100644 index 0000000..8a70204 --- /dev/null +++ b/docs/reference/web_resource.md @@ -0,0 +1,3 @@ +# mongospecs_web.resource + +::: mongospecs_web.resource diff --git a/docs/reference/web_testing.md b/docs/reference/web_testing.md new file mode 100644 index 0000000..03cc34a --- /dev/null +++ b/docs/reference/web_testing.md @@ -0,0 +1,3 @@ +# mongospecs_web.testing + +::: mongospecs_web.testing diff --git a/docs/web/plugs.md b/docs/web/plugs.md new file mode 100644 index 0000000..6001f92 --- /dev/null +++ b/docs/web/plugs.md @@ -0,0 +1,76 @@ +# Plugs + +A plug mounts an `Api` on a framework. It only translates requests and responses; routing targets, +validation, authorization and serialisation are the same code everywhere, and a contract suite +checks that every plug behaves identically. + +```py +api = Api(articles, users, prefix="/api", authenticate=current_user) +``` + +## Pure ASGI + +```py +app = api.asgi(registry=registry) # opens and closes the registry on lifespan events +``` + +## Starlette + +```py +from starlette.applications import Starlette +from mongospecs_web.plugs import starlette as plug + +app = Starlette(routes=plug.routes(api), lifespan=plug.lifespan(registry)) +``` + +## FastAPI + +```py +from fastapi import FastAPI +from mongospecs_web.plugs import fastapi as plug + +app = FastAPI(lifespan=plug.lifespan(registry)) +plug.mount(app, api) +``` + +The operations and their schemas are merged into the application's own `/openapi.json` and +`/docs`, for msgspec and attrs models as well as pydantic ones. + +## Litestar + +```py +from litestar import Litestar +from mongospecs_web.plugs.litestar import MongospecsPlugin + +app = Litestar(plugins=[MongospecsPlugin(api, registry=registry)]) +``` + +Handlers are left out of Litestar's own schema generation; the API's OpenAPI document is served at +`/openapi.json`. + +## Quart + +```py +from quart import Quart +from mongospecs_web.plugs import quart as plug + +app = Quart(__name__) +plug.init_app(app, api, registry=registry) +``` + +## Writing another plug + +Convert the framework's request to `mongospecs_web.Request`, call `await api.run(route, request)` +for the matching entry of `api.routes`, and convert the `Response` back. Then prove it: + +```py +from mongospecs_web.testing import contract_api, reference_transcript, run_contract + +api = contract_api("msgspec") +app = my_framework_app(api) +async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") as client: + assert await run_contract(client, api) == await reference_transcript("msgspec") +``` + +Routes the framework rejects by itself (unknown path, unsupported method) keep that framework's +native 404/405 body; the contract compares only their status. diff --git a/docs/web/resources.md b/docs/web/resources.md new file mode 100644 index 0000000..3ab162d --- /dev/null +++ b/docs/web/resources.md @@ -0,0 +1,131 @@ +# Resources + +`mongospecs-web` exposes a model as REST operations. The description is pure data; a plug mounts it +on a framework. + +```python +from typing import Annotated + +from mongospecs import Policy, Q, Registry, Revision, SpecConfig +from mongospecs.msgspec import Spec +from mongospecs_web import Api, Create, Delete, Get, List, Patch, Resource, Unauthorized + + +class Task(Spec): + __spec__ = SpecConfig(collection="tasks") + title: Annotated[str, Policy(filter=("eq", "prefix"), sort=True)] + done: Annotated[bool, Policy(filter=("eq",))] = False + owner: Annotated[str, Policy(insert=False, update=False, filter=("eq",))] = "" + secret_note: Annotated[str | None, Policy(private=True)] = None + rev: Annotated[int, Revision] = 0 + + +def authenticate(request): + return request.headers.get("x-user") # your real authentication goes here + + +def logged_in(ctx): + if ctx.principal is None: + raise Unauthorized("Log in first.") + return True + + +tasks = Resource( + Task, + ops=[List, Get, Create, Patch, Delete], + guards=[logged_in], # who may call the operations + scope=lambda ctx: Q.owner == ctx.principal, # which rows they may touch + server_fields=lambda ctx: {"owner": ctx.principal}, # set by the server on create +) +api = Api(tasks, prefix="/api", authenticate=authenticate) + +registry = await Registry("mongodb://localhost/app", models=[Task]).open() +``` + +`Policy` on each field decides what a client may do with it: + +| Policy | Meaning | +|---|---| +| `insert=False` / `update=False` | not settable on create / on patch | +| `private=True` | never returned, filtered or sorted; not writable unless `insert`/`update` say so | +| `filter=("eq", "gte", ...)` | operators allowed in the query string (`True` for all) | +| `sort=True` | usable in `?sort=` | + +## Calling it + +`api.handle(...)` runs a request without any server, which is also how you test resources. + +```python +ann = {"x-user": "ann"} +created = await api.handle("POST", "/api/tasks", body=b'{"title": "Write docs"}', headers=ann) +assert created.status == 201 and created.headers["etag"] == '"0"' + +import json + +task = json.loads(created.body)["data"] +assert task["owner"] == "ann" and "secret_note" not in task + +listed = await api.handle("GET", "/api/tasks", query=[("title[prefix]", "Write"), ("sort", "-title")], headers=ann) +assert [t["title"] for t in json.loads(listed.body)["data"]] == ["Write docs"] + +# someone else cannot see or change it: the scope is part of the query +assert (await api.handle("GET", f"/api/tasks/{task['id']}", headers={"x-user": "bob"})).status == 404 + +# conditional update with the ETag +patched = await api.handle( + "PATCH", f"/api/tasks/{task['id']}", body=b'{"done": true}', headers={**ann, "if-match": '"0"'} +) +assert patched.status == 200 +stale = await api.handle( + "PATCH", f"/api/tasks/{task['id']}", body=b'{"done": false}', headers={**ann, "if-match": '"0"'} +) +assert stale.status == 412 + +# input the policy does not allow never reaches the database +for body in (b'{"owner": "mallory"}', b'{"title": {"$ne": null}}', b'{"nope": 1}'): + assert (await api.handle("PATCH", f"/api/tasks/{task['id']}", body=body, headers=ann)).status == 422 +assert (await api.handle("GET", "/api/tasks", query=[("done[$ne]", "1")], headers=ann)).status == 400 +``` + +Errors are RFC 9457 `application/problem+json` documents. Unexpected exceptions become a bare 500 +with no detail. + +## Custom operations and hooks + +```python +@tasks.op("complete", method="POST", summary="Mark a task as done") +async def complete(ctx): + task = await ctx.load() # 404 unless it is within the caller's scope + task.done = True + return await task.save() + + +@tasks.after(Create) +async def audit(ctx, task): + ctx.state["audited"] = task.id + + +done = await api.handle("POST", f"/api/tasks/{task['id']}/complete", headers=ann) +assert json.loads(done.body)["data"]["done"] is True + +spec = api.openapi() +assert "/api/tasks/{id}/complete" in spec["paths"] and "TaskPatch" in spec["components"]["schemas"] + +await registry.client.drop_database("app") +await registry.close() +``` + +## Query string + +``` +?title=hello equality +?views[gte]=10&views[lt]=50 operators in brackets +?tag[in]=a,b comma-separated lists +?sort=-views,title +?limit=20&after= cursor pagination (default) +?limit=20&page=3 offset pagination, with page=Offset(...) +?total=true also count all matches +?include=author embed allowlisted relations +``` + +Only fields and operators the policy allows are accepted; everything else is a 400. diff --git a/mkdocs.yaml b/mkdocs.yaml index fb9472d..2f137a3 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -65,21 +65,57 @@ plugins: mkdocstrings: handlers: python: + paths: [packages/mongospecs/src, packages/mongospecs-web/src] options: docstring_section_style: list filters: ["!^_"] - heading_level: 1 - inherited_members: true - merge_init_into_class: false - preload_modules: ["msgspec"] + heading_level: 2 + merge_init_into_class: true separate_signature: true - show_if_no_docstring: true - show_root_heading: true - show_root_full_path: false + show_root_heading: false show_signature_annotations: true + show_source: false signature_crossrefs: true nav: -- Home: - - Overview: index.md +- Home: index.md +- Guide: + - Models: guide/models.md + - Queries: guide/queries.md + - Writes and transactions: guide/writes.md + - Relations: guide/relations.md + - Hooks and plugins: guide/plugins.md + - Running in production: guide/operations.md +- Web: + - Resources: web/resources.md + - Plugs: web/plugs.md +- Benchmarks: benchmarks.md +- Coming from Mongoose: parity.md +- Migrating from 0.1.x: + - Guide: migration.md - v1 test inventory: v1-test-inventory.md +- Reference: + - mongospecs: + - registry: reference/registry.md + - model: reference/model.md + - manager: reference/manager.md + - query: reference/query.md + - expr: reference/expr.md + - updates: reference/updates.md + - bulk: reference/bulk.md + - markers: reference/markers.md + - refs: reference/refs.md + - relations: reference/relations.md + - indexes: reference/indexes.md + - hooks: reference/hooks.md + - streams: reference/streams.md + - errors: reference/errors.md + - mongospecs-web: + - resource: reference/web_resource.md + - api: reference/web_api.md + - backend: reference/web_backend.md + - filters: reference/web_filters.md + - policy: reference/web_policy.md + - errors: reference/web_errors.md + - openapi: reference/web_openapi.md + - testing: reference/web_testing.md diff --git a/packages/mongospecs-web/src/mongospecs_web/api.py b/packages/mongospecs-web/src/mongospecs_web/api.py index 342789e..443ebe6 100644 --- a/packages/mongospecs-web/src/mongospecs_web/api.py +++ b/packages/mongospecs-web/src/mongospecs_web/api.py @@ -3,6 +3,7 @@ from __future__ import annotations import inspect +import logging import re import typing as t from dataclasses import dataclass @@ -115,7 +116,10 @@ async def run(self, route: Route, request: Request) -> Response: principal = self.authenticate(request) request.principal = await principal if inspect.isawaitable(principal) else principal except Exception as exc: - return Response.error(exc) + response = Response.error(exc) + if response.status >= 500: + logging.getLogger("mongospecs_web").error("authentication failed unexpectedly", exc_info=exc) + return response return await route.resource.dispatch(route.operation, request) async def handle( diff --git a/packages/mongospecs-web/src/mongospecs_web/backend.py b/packages/mongospecs-web/src/mongospecs_web/backend.py index 360ac6c..001b6dc 100644 --- a/packages/mongospecs-web/src/mongospecs_web/backend.py +++ b/packages/mongospecs-web/src/mongospecs_web/backend.py @@ -116,7 +116,7 @@ async def list(self, query: ListQuery, *, scope: t.Any) -> Page[T]: async def create(self, data: dict[str, t.Any]) -> T: stored = {self.meta.fields[k].stored: v for k, v in data.items()} - doc = self.meta.decode(stored, trusted=False) + doc = self.meta.decode(stored, stored=False) return t.cast(T, await self.manager.insert(doc)) async def patch(self, id: t.Any, data: dict[str, t.Any], *, scope: t.Any, if_rev: int | None = None) -> T: diff --git a/packages/mongospecs-web/src/mongospecs_web/resource.py b/packages/mongospecs-web/src/mongospecs_web/resource.py index e2b0fe8..050e4f2 100644 --- a/packages/mongospecs-web/src/mongospecs_web/resource.py +++ b/packages/mongospecs-web/src/mongospecs_web/resource.py @@ -10,6 +10,7 @@ import inspect import json +import logging import typing as t from dataclasses import dataclass, field @@ -32,6 +33,8 @@ T = t.TypeVar("T") F = t.TypeVar("F", bound=t.Callable[..., t.Any]) +log = logging.getLogger("mongospecs_web") + # Built-in operation kinds. Used as keys for ``ops=``, ``guards=`` and hooks. List = "list" Get = "get" @@ -191,12 +194,17 @@ def __init__( include: t.Sequence[str] = (), page: Pagination | None = None, tags: t.Sequence[str] | None = None, + server_fields: t.Callable[[Ctx[T]], t.Any] | None = None, ) -> None: self.model = model self.view: View = view_of(model) self.name = name or self.view.meta.collection self.backend: ResourceBackend[T] = backend if backend is not None else MongoBackend(model) self.scope_fn = scope + self.server_fields = server_fields + """``server_fields(ctx) -> dict`` of values the server sets on create (the author of a + review, a tenant id). They are not subject to the field policy, so the fields can stay + closed to clients with ``Policy(insert=False)``.""" self.includes: tuple[str, ...] = tuple(include) for path in self.includes: head = path.partition(".")[0] @@ -320,7 +328,11 @@ async def dispatch(self, op: Operation, request: Request) -> Response: await _maybe(hook(ctx, ctx.state.get("result", result))) return self._respond(op, result, ctx) except Exception as exc: - return Response.error(exc) + response = Response.error(exc) + if response.status >= 500: + # The client gets a bare 500; the cause belongs in the server log. + log.error("%s %s failed", self.name, op.name, exc_info=exc) + return response def _respond(self, op: Operation, result: t.Any, ctx: Ctx[T]) -> Response: if isinstance(result, Response): @@ -471,8 +483,10 @@ async def handler(ctx: Ctx[T]) -> Response: body = ctx.body() if body is None: raise Unprocessable("A JSON request body is required.") - data = self.view.check_input(body, self.view.insertable, "create") - doc = await self.backend.create(dict(data)) + data = dict(self.view.check_input(body, self.view.insertable, "create")) + if self.server_fields is not None: + data.update(await _maybe(self.server_fields(ctx))) + doc = await self.backend.create(data) ctx.state["result"] = doc headers = {"location": str(jsonable(getattr(doc, "id", "")))} etag = self.view.etag(doc) diff --git a/packages/mongospecs-web/src/mongospecs_web/testing.py b/packages/mongospecs-web/src/mongospecs_web/testing.py index 30986e7..b600611 100644 --- a/packages/mongospecs-web/src/mongospecs_web/testing.py +++ b/packages/mongospecs-web/src/mongospecs_web/testing.py @@ -158,7 +158,7 @@ async def list(self, query: ListQuery, *, scope: t.Any) -> Page[T]: async def create(self, data: dict[str, t.Any]) -> T: stored = {self.meta.fields[k].stored: v for k, v in data.items()} - doc = self.meta.decode(stored, trusted=False) + doc = self.meta.decode(stored, stored=False) self._check_unique(doc) doc.id = ObjectId(f"{self._next:024x}") self._next += 1 diff --git a/packages/mongospecs-web/tests/test_docs_examples.py b/packages/mongospecs-web/tests/test_docs_examples.py new file mode 100644 index 0000000..4ab7b29 --- /dev/null +++ b/packages/mongospecs-web/tests/test_docs_examples.py @@ -0,0 +1,51 @@ +"""The guide pages are executable: every ``python`` block on a page runs, in order, against MongoDB. + +Blocks fenced as ``py`` are illustrative and are not run. +""" + +import ast +import pathlib +import re +import uuid + +import pytest + +DOCS = pathlib.Path(__file__).resolve().parents[3] / "docs" +PAGES = sorted(p for p in (*DOCS.glob("guide/*.md"), *DOCS.glob("web/*.md")) if "```python" in p.read_text()) +BLOCK = re.compile(r"```python\n(.*?)```", re.S) +RS = "mongodb://localhost:27117/{db}?replicaSet=rs0" + + +@pytest.mark.parametrize("page", PAGES, ids=lambda p: f"{p.parent.name}/{p.stem}") +async def test_page_examples_run(page): + import os + + db = f"ms_docs_{uuid.uuid4().hex[:10]}" + uri = os.environ.get("MONGOSPECS_TEST_URI", RS.format(db=db)) + if "{db}" not in RS or db not in uri: # a custom URI without a database: add ours + base, _, query = uri.partition("?") + uri = f"{base.rstrip('/')}/{db}" + (f"?{query}" if query else "") + source = "\n".join(BLOCK.findall(page.read_text())) + assert source.strip(), f"{page} has no runnable examples" + source = ( + source.replace("mongodb://localhost/app", uri) + .replace('"app_archive"', f'"{db}_archive"') + .replace('drop_database("app")', f'drop_database("{db}")') + ) + code = compile(source, str(page), "exec", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) + namespace = {"__name__": f"docs_{page.stem}"} + result = eval(code, namespace) + if result is not None: + await result + + +def test_every_guide_page_is_covered(): + assert {p.stem for p in PAGES} >= { + "models", + "queries", + "writes", + "relations", + "plugins", + "operations", + "resources", + } diff --git a/packages/mongospecs/src/mongospecs/attrs.py b/packages/mongospecs/src/mongospecs/attrs.py index 1e7f9d1..fea5ab3 100644 --- a/packages/mongospecs/src/mongospecs/attrs.py +++ b/packages/mongospecs/src/mongospecs/attrs.py @@ -36,9 +36,9 @@ def raw_fields(self, cls: type) -> list[RawField]: for a in attrs.fields(cls) ] - def _decode(self, cls: type, doc: dict[str, t.Any], trusted: bool) -> t.Any: + def _decode(self, cls: type, doc: dict[str, t.Any], lax: bool) -> t.Any: try: - return msgspec.convert(doc, cls, dec_hook=dec_hook) + return msgspec.convert(doc, cls, dec_hook=dec_hook, strict=not lax) except (msgspec.ValidationError, ValueError) as exc: raise _error(cls, exc) from exc diff --git a/packages/mongospecs/src/mongospecs/bulk.py b/packages/mongospecs/src/mongospecs/bulk.py index a0a04f7..d27635e 100644 --- a/packages/mongospecs/src/mongospecs/bulk.py +++ b/packages/mongospecs/src/mongospecs/bulk.py @@ -63,7 +63,7 @@ def compile_bulk(manager: Manager[t.Any], ctx: OpCtx, operations: t.Iterable[t.A requests: list[t.Any] = [] for op in operations: if isinstance(op, InsertOne): - doc = manager.meta.decode(dict(op.doc)) if isinstance(op.doc, t.Mapping) else op.doc + doc = manager.meta.decode(dict(op.doc), stored=False) if isinstance(op.doc, t.Mapping) else op.doc encoded = manager.meta.encode(doc) if encoded.get("_id") is None: encoded.pop("_id", None) diff --git a/packages/mongospecs/src/mongospecs/codec.py b/packages/mongospecs/src/mongospecs/codec.py index d513c12..9705992 100644 --- a/packages/mongospecs/src/mongospecs/codec.py +++ b/packages/mongospecs/src/mongospecs/codec.py @@ -242,7 +242,7 @@ def __init__(self) -> None: def raw_fields(self, cls: type) -> list[RawField]: ... @abc.abstractmethod - def _decode(self, cls: type, doc: dict[str, t.Any], trusted: bool) -> t.Any: ... + def _decode(self, cls: type, doc: dict[str, t.Any], lax: bool) -> t.Any: ... @abc.abstractmethod def _encode(self, obj: t.Any) -> dict[str, t.Any]: @@ -293,8 +293,13 @@ def _renames(self, cls: type) -> tuple[dict[str, str], dict[str, str]]: cache[cls] = hit return hit - def decode(self, cls: type, doc: t.Any, *, trusted: bool = False) -> t.Any: - """Build a model from a MongoDB document (a mapping or raw BSON bytes).""" + def decode(self, cls: type, doc: t.Any, *, lax: bool = False) -> t.Any: + """Build a model from a MongoDB document (a mapping or raw BSON bytes). + + ``lax`` is for documents read back from the database: it tolerates the + representations other writers produce (an integer stored as a double, + for instance) where the backend would otherwise be strict. + """ if isinstance(doc, bytes | bytearray | memoryview): doc = bson.decode(bytes(doc)) elif isinstance(doc, RawBSONDocument): @@ -302,7 +307,7 @@ def decode(self, cls: type, doc: t.Any, *, trusted: bool = False) -> t.Any: to_native, _ = self._renames(cls) if to_native: doc = {to_native.get(k, k): v for k, v in doc.items()} - return self._decode(cls, doc, trusted) + return self._decode(cls, doc, lax) def encode(self, obj: t.Any) -> dict[str, t.Any]: """Encode a model to a BSON-ready dict keyed by stored names.""" diff --git a/packages/mongospecs/src/mongospecs/manager.py b/packages/mongospecs/src/mongospecs/manager.py index 9087dc9..a7db1fc 100644 --- a/packages/mongospecs/src/mongospecs/manager.py +++ b/packages/mongospecs/src/mongospecs/manager.py @@ -40,6 +40,8 @@ T = t.TypeVar("T") _MANAGED_MARKERS = (CreatedAt, UpdatedAt, DeletedAt, Revision) +# find() option name -> aggregate() option name +_AGG_NAMES = {"max_time_ms": "maxTimeMS", "batch_size": "batchSize", "allow_disk_use": "allowDiskUse"} class BulkResult(t.Generic[T]): @@ -84,6 +86,7 @@ def __init__( *(f.name for f in self.meta.fields.values() if any(f.has(m) for m in _MANAGED_MARKERS)), ) self._collection: AsyncCollection[dict[str, t.Any]] | None = None + self._raw_collection: t.Any = None self._cascades: list[t.Any] | None = None # -- plumbing -------------------------------------------------------------- @@ -105,6 +108,16 @@ def collection(self) -> AsyncCollection[dict[str, t.Any]]: ) return self._collection + @property + def raw_collection(self) -> t.Any: + """The collection, configured to hand back undecoded BSON (used for tracked reads).""" + if self._raw_collection is None or not self.registry.is_open: + from bson.raw_bson import RawBSONDocument + + options = self.registry._codec_options.with_options(document_class=RawBSONDocument) + self._raw_collection = self.collection.with_options(codec_options=options) + return self._raw_collection + def using( self, *, @@ -151,6 +164,11 @@ def _read_kw(self, session: t.Any, options: t.Mapping[str, t.Any] | None = None) kw["collation"] = self.meta.config.collation return kw + def _agg_kw(self, session: t.Any, options: t.Mapping[str, t.Any] | None = None) -> dict[str, t.Any]: + """Read options spelled the way ``aggregate`` and ``count_documents`` expect them.""" + kw = self._read_kw(session, options) + return {_AGG_NAMES.get(k, k): v for k, v in kw.items() if k != "no_cursor_timeout"} + def scoped( self, ctx: OpCtx, @@ -517,7 +535,9 @@ async def insert_many( With ``ordered=False`` the server attempts every document; failures are reported in ``BulkResult.errors`` instead of raising. """ - items: list[t.Any] = [self.meta.decode(dict(d)) if isinstance(d, t.Mapping) else d for d in docs] + items: list[t.Any] = [ + self.meta.decode(dict(d), stored=False) if isinstance(d, t.Mapping) else d for d in docs + ] if not items: return BulkResult([], []) ctx = self.ctx("insert", session, many=True) @@ -716,16 +736,16 @@ async def __aiter__(self) -> t.AsyncIterator[T]: decode = self._decoder() with manager._op("aggregate"): cursor = await manager.collection.aggregate( - self.compiled(), **manager._read_kw(self._session, self._options) + self.compiled(), **manager._agg_kw(self._session, self._options) ) async with cursor: while True: with manager._op("aggregate"): - try: - raw = await cursor.__anext__() - except StopAsyncIteration: - return - yield decode(raw) + batch = await cursor.to_list(1000) + for raw in batch: + yield decode(raw) + if len(batch) < 1000: + return async def all(self) -> list[T]: return [item async for item in self] diff --git a/packages/mongospecs/src/mongospecs/markers.py b/packages/mongospecs/src/mongospecs/markers.py index e92a468..e6c7d86 100644 --- a/packages/mongospecs/src/mongospecs/markers.py +++ b/packages/mongospecs/src/mongospecs/markers.py @@ -234,9 +234,6 @@ class SpecConfig: upcast: t.Callable[[dict[str, t.Any]], dict[str, t.Any]] | None = None """Called with each raw document before decoding, to migrate old shapes lazily.""" - trusted: bool = False - """Skip read validation where the backend allows it.""" - max_time_ms: int | None = None read_preference: t.Any = None read_concern: t.Any = None diff --git a/packages/mongospecs/src/mongospecs/meta.py b/packages/mongospecs/src/mongospecs/meta.py index a7d501d..5ade142 100644 --- a/packages/mongospecs/src/mongospecs/meta.py +++ b/packages/mongospecs/src/mongospecs/meta.py @@ -234,13 +234,15 @@ def validate_type(self, tp: t.Any, value: t.Any, *, lax: bool = False) -> t.Any: # -- encode / decode ------------------------------------------------------- - def decode(self, doc: t.Any, *, trusted: bool | None = None) -> t.Any: + def decode(self, doc: t.Any, *, stored: bool = True) -> t.Any: + """Build a model. ``stored=True`` (the default) is for documents read from the database; + pass ``stored=False`` for input, which is held to the backend's strict rules.""" if self.config.upcast is not None: doc = self.config.upcast(dict(doc)) cls = self.model if self.disc_key: cls = self.subclass_for(doc.get(self.disc_key)) - return self.codec.decode(cls, doc, trusted=self.config.trusted if trusted is None else trusted) + return self.codec.decode(cls, doc, lax=stored) def encode(self, obj: t.Any) -> dict[str, t.Any]: doc = self.codec.encode(obj) diff --git a/packages/mongospecs/src/mongospecs/model.py b/packages/mongospecs/src/mongospecs/model.py index b3aa49e..65ee4f7 100644 --- a/packages/mongospecs/src/mongospecs/model.py +++ b/packages/mongospecs/src/mongospecs/model.py @@ -247,22 +247,25 @@ def _normalize_refs(self) -> None: refs = _REF_FIELDS[cls] = _ref_fields(cls) if not refs: return - from mongospecs.refs import Ref + from mongospecs.refs import Ref, coerce_ref + # coerce_ref also turns an id given as a string into the target's id type. for name, shape, model in refs: value = getattr(self, name) if value is None: continue if shape == "one": if type(value) is not Ref: - setattr(self, name, Ref(value, model)) + setattr(self, name, coerce_ref(value, model)) elif value._model is None: value._model = model elif shape == "list": if any(type(v) is not Ref for v in value): - setattr(self, name, [v if type(v) is Ref else Ref(v, model) for v in value]) + setattr(self, name, [v if type(v) is Ref else coerce_ref(v, model) for v in value]) elif any(type(v) is not Ref for v in value.values()): - setattr(self, name, {k: v if type(v) is Ref else Ref(v, model) for k, v in value.items()}) + setattr( + self, name, {k: v if type(v) is Ref else coerce_ref(v, model) for k, v in value.items()} + ) _codec_module._DOCUMENT = Document diff --git a/packages/mongospecs/src/mongospecs/msgspec.py b/packages/mongospecs/src/mongospecs/msgspec.py index 7550b36..e37b6b8 100644 --- a/packages/mongospecs/src/mongospecs/msgspec.py +++ b/packages/mongospecs/src/mongospecs/msgspec.py @@ -62,11 +62,9 @@ class MsgspecCodec(Codec): def raw_fields(self, cls: type) -> list[RawField]: return [(f.name, f.encode_name, f.type, f.required) for f in msgspec.structs.fields(cls)] - def _decode(self, cls: type, doc: dict[str, t.Any], trusted: bool) -> t.Any: - # msgspec has no unvalidated construction path; conversion is cheap, so - # ``trusted`` changes nothing here. + def _decode(self, cls: type, doc: dict[str, t.Any], lax: bool) -> t.Any: try: - return msgspec.convert(doc, cls, dec_hook=dec_hook) + return msgspec.convert(doc, cls, dec_hook=dec_hook, strict=not lax) except (msgspec.ValidationError, ValueError) as exc: raise _error(cls, exc) from exc diff --git a/packages/mongospecs/src/mongospecs/plugins/revision.py b/packages/mongospecs/src/mongospecs/plugins/revision.py index 07eebbc..ad35c61 100644 --- a/packages/mongospecs/src/mongospecs/plugins/revision.py +++ b/packages/mongospecs/src/mongospecs/plugins/revision.py @@ -48,7 +48,9 @@ def on_update(self, ctx: OpCtx, flt: dict[str, t.Any], update: dict[str, t.Any]) update.setdefault("$inc", {})[stored] = 1 snapshot = ctx.extra.get("snapshot") if snapshot is not None: - flt[stored] = snapshot.get(stored) + loaded = snapshot.get(stored) + # A document written before the model had a revision field has none stored. + flt[stored] = {"$in": [0, None]} if not loaded else loaded elif ctx.extra.get("if_rev") is not None: flt[stored] = ctx.extra["if_rev"] diff --git a/packages/mongospecs/src/mongospecs/pydantic.py b/packages/mongospecs/src/mongospecs/pydantic.py index 7aaeafc..b2a940d 100644 --- a/packages/mongospecs/src/mongospecs/pydantic.py +++ b/packages/mongospecs/src/mongospecs/pydantic.py @@ -18,7 +18,6 @@ __all__ = ["CODEC", "PydanticCodec", "Spec", "SpecAdapter"] _HAS_EXCLUDE_COMPUTED = "exclude_computed_fields" in inspect.signature(BaseModel.model_dump).parameters -_SCALARS: tuple[type, ...] = (str, int, float, bool, bytes) def _error(cls: type, exc: pydantic.ValidationError, name: str | None = None) -> ValidationError: @@ -44,7 +43,6 @@ class PydanticCodec(Codec): def __init__(self) -> None: super().__init__() - self._flat: dict[type, bool] = {} self._adapters: dict[tuple[type, str], TypeAdapter[t.Any]] = {} def raw_fields(self, cls: type) -> list[RawField]: @@ -54,25 +52,9 @@ def raw_fields(self, cls: type) -> list[RawField]: for name, info in model.model_fields.items() ] - def _is_flat(self, cls: type) -> bool: - """True when every field is a plain scalar, so unvalidated construction is safe.""" - flat = self._flat.get(cls) - if flat is None: - from bson import ObjectId - - from mongospecs.codec import strip_optional - - flat = all(strip_optional(f.type) in (*_SCALARS, ObjectId) for f in self.fields(cls).values()) - self._flat[cls] = flat - return flat - - def _decode(self, cls: type, doc: dict[str, t.Any], trusted: bool) -> t.Any: - model = t.cast("type[BaseModel]", cls) - if trusted and self._is_flat(cls): - by_alias = {f.stored: f.name for f in self.fields(cls).values()} - return model.model_construct(**{by_alias.get(k, k): v for k, v in doc.items() if k in by_alias}) + def _decode(self, cls: type, doc: dict[str, t.Any], lax: bool) -> t.Any: try: - return model.model_validate(doc) + return t.cast("type[BaseModel]", cls).model_validate(doc) except pydantic.ValidationError as exc: raise _error(cls, exc) from exc @@ -109,7 +91,6 @@ def _json_schema(self, cls: type) -> dict[str, t.Any]: def clear_cache(self) -> None: super().clear_cache() - self._flat.clear() self._adapters.clear() diff --git a/packages/mongospecs/src/mongospecs/query.py b/packages/mongospecs/src/mongospecs/query.py index 72a4744..01ae684 100644 --- a/packages/mongospecs/src/mongospecs/query.py +++ b/packages/mongospecs/src/mongospecs/query.py @@ -30,8 +30,6 @@ Mode = t.Literal["tracked", "lean", "raw"] -# find() option name -> aggregate() option name -_AGG_OPTIONS = {"max_time_ms": "maxTimeMS", "batch_size": "batchSize", "allow_disk_use": "allowDiskUse"} _FIND_OPTIONS = frozenset( {"max_time_ms", "batch_size", "allow_disk_use", "hint", "comment", "collation", "no_cursor_timeout"} ) @@ -309,13 +307,15 @@ def compile(self) -> dict[str, t.Any]: # -- execution ------------------------------------------------------------- - async def _cursor(self, plan: dict[str, t.Any]) -> t.Any: + async def _cursor(self, plan: dict[str, t.Any], *, raw_bson: bool = False) -> t.Any: manager = self._manager kw = manager._read_kw(self._session, self._options) + collection = manager.raw_collection if raw_bson else manager.collection if "pipeline" in plan: - agg_kw = {_AGG_OPTIONS.get(k, k): v for k, v in kw.items() if k != "no_cursor_timeout"} - return await manager.collection.aggregate(plan["pipeline"], **agg_kw) - return manager.collection.find( + return await collection.aggregate( + plan["pipeline"], **manager._agg_kw(self._session, self._options) + ) + return collection.find( plan["filter"], plan["projection"], sort=plan["sort"] or None, @@ -334,40 +334,49 @@ async def _chunks(self, chunk: int | None = None) -> t.AsyncIterator[list[t.Any] lookups, batched, partial = plan["lookups"], plan["batched"], plan["partial"] size = chunk or self._options.get("batch_size") or _CHUNK mode = self._mode - - async def flush(raws: list[dict[str, t.Any]]) -> list[t.Any]: - joined = [pop_lookups(raw, lookups) for raw in raws] if lookups else None - if mode == "raw": - items: list[t.Any] = raws - else: - items = [] - for raw in raws: - doc = meta.decode(raw) - if mode == "tracked": - tracking.track(doc, meta_of(type(doc)).encode(doc), manager, partial) + # Tracked reads ask the driver for undecoded BSON: the bytes are the snapshot, so nothing + # is re-encoded at load time. With $lookup the stored form is not what came back, so the + # snapshot is rebuilt from the decoded document instead. + raw_bson = mode == "tracked" and not lookups + options = manager.registry._codec_options + + async def flush(raws: list[t.Any]) -> list[t.Any]: + if raw_bson: + items: list[t.Any] = [] + decode = meta.decode + for item in raws: + data = item.raw + doc = decode(bson.decode(data, options)) + tracking.track(doc, data, manager, partial) items.append(doc) - if joined is not None: - attach_lookups(manager, items, joined, lookups, mode) + else: + joined = [pop_lookups(raw, lookups) for raw in raws] if lookups else None + if mode == "raw": + items = raws + else: + items = [] + for raw in raws: + doc = meta.decode(raw) + if mode == "tracked": + tracking.track(doc, meta_of(type(doc)).encode(doc), manager, partial) + items.append(doc) + if joined is not None: + attach_lookups(manager, items, joined, lookups, mode) if batched: await load_batched(manager, items, batched, mode, self._session) return items with manager._op("find"): - cursor = await self._cursor(plan) + cursor = await self._cursor(plan, raw_bson=raw_bson) async with cursor: - buffer: list[dict[str, t.Any]] = [] while True: with manager._op("find"): - try: - raw = await cursor.__anext__() - except StopAsyncIteration: - break - buffer.append(raw) - if len(buffer) >= size: - yield await flush(buffer) - buffer = [] - if buffer: - yield await flush(buffer) + batch = await cursor.to_list(size) + if not batch: + break + yield await flush(batch) + if len(batch) < size: + break async def __aiter__(self) -> t.AsyncIterator[T]: async for chunk in self._chunks(): @@ -376,7 +385,9 @@ async def __aiter__(self) -> t.AsyncIterator[T]: async def all(self) -> list[T]: out: list[T] = [] - async for chunk in self._chunks(): + # Without batched relations there is nothing to do per chunk, so fetch in large steps. + step = None if self._includes else 10_000 + async for chunk in self._chunks(step): out.extend(chunk) if self._fail and not out: raise DocumentNotFound(f"{self._manager.model.__name__}: no document matched") @@ -402,11 +413,10 @@ async def one(self) -> T: async def count(self) -> int: manager = self._manager - kw = manager._read_kw(self._session, self._options) agg_kw = { - _AGG_OPTIONS.get(k, k): v - for k, v in kw.items() - if k in ("session", "max_time_ms", "hint", "collation", "comment") + k: v + for k, v in manager._agg_kw(self._session, self._options).items() + if k in ("session", "maxTimeMS", "hint", "collation", "comment") } if self._skip: agg_kw["skip"] = self._skip @@ -514,7 +524,9 @@ async def page( Without ``number`` this is cursor (keyset) pagination: pass the previous page's ``next_cursor`` as ``after``. The sort is made total by appending - ``_id``; sort fields should not be null. With ``number`` (1-based) it is + ``_id``, so the index serving it should end in ``_id`` (``Index("price", "id")``); + otherwise the server sorts in memory. Sort fields should not be null. + With ``number`` (1-based) it is offset pagination. ``total=True`` adds a count of all matches. """ if size < 1: diff --git a/packages/mongospecs/src/mongospecs/relations.py b/packages/mongospecs/src/mongospecs/relations.py index afaef23..78840dd 100644 --- a/packages/mongospecs/src/mongospecs/relations.py +++ b/packages/mongospecs/src/mongospecs/relations.py @@ -507,7 +507,7 @@ async def all(self) -> list[GraphResult[t.Any]]: mode = query._mode results: list[GraphResult[t.Any]] = [] with manager._op("graph"): - cursor = await manager.collection.aggregate(self.compile(), **manager._kw(query._session)) + cursor = await manager.collection.aggregate(self.compile(), **manager._agg_kw(query._session)) raws = [raw async for raw in cursor] for raw in raws: nodes_raw = sorted(raw.pop("__graph", []), key=lambda n: n.get("__depth", 0)) @@ -602,7 +602,7 @@ async def __aiter__(self) -> t.AsyncIterator[t.Any]: first = self._queries[0] manager = first._manager with manager._op("union"): - cursor = await manager.collection.aggregate(self.compile(), **manager._kw(first._session)) + cursor = await manager.collection.aggregate(self.compile(), **manager._agg_kw(first._session)) async with cursor: async for raw in cursor: query = self._queries[raw.pop("__src")] diff --git a/packages/mongospecs/src/mongospecs/tracking.py b/packages/mongospecs/src/mongospecs/tracking.py index ea5ae27..938f6fb 100644 --- a/packages/mongospecs/src/mongospecs/tracking.py +++ b/packages/mongospecs/src/mongospecs/tracking.py @@ -1,36 +1,66 @@ """Change tracking by snapshot. -When a document is loaded (or written), its encoded form is kept in a side table -keyed by object identity. ``save()`` diffs the current encoding against that -snapshot and sends only the difference. No attribute interception is involved, -so the mechanism is the same for msgspec, pydantic and attrs. - -The table entry is removed by a weakref finaliser when the document is -collected, which is why document classes must be weak-referenceable. +A tracked document remembers how it looked when it was loaded. ``save()`` +compares the current encoding with that snapshot and sends only the difference. +No attribute interception is involved, so the mechanism is the same for +msgspec, pydantic and attrs. + +For documents read by a query, what is kept is the raw BSON the driver received, +as bytes: nothing is re-encoded at load time and the bytes are smaller than the +equivalent Python objects. The snapshot proper (the document decoded into the +model and encoded again, so that defaults and type coercions are already +applied) is only built if the document is actually saved. + +State lives in a side table keyed by object identity and is removed by a weakref +finaliser when the document is collected, which is why document classes must be +weak-referenceable. """ from __future__ import annotations -import copy import typing as t import weakref +import bson + __all__ = ["DocState", "diff", "forget", "state_of", "track"] class DocState: """What mongospecs remembers about a tracked document.""" - __slots__ = ("manager", "partial", "snapshot") + __slots__ = ("_doc", "_raw", "manager", "partial") - def __init__(self, snapshot: dict[str, t.Any], manager: t.Any, partial: frozenset[str] | None) -> None: - self.snapshot = snapshot + def __init__( + self, + snapshot: dict[str, t.Any] | bytes, + manager: t.Any, + partial: frozenset[str] | None, + ) -> None: + self._raw: bytes | None = snapshot if isinstance(snapshot, bytes) else None + self._doc: dict[str, t.Any] | None = None if isinstance(snapshot, bytes) else snapshot self.manager = manager self.partial = partial """Stored keys that were loaded, when the document was read with a projection.""" - def copy(self) -> DocState: - return DocState(copy.deepcopy(self.snapshot), self.manager, self.partial) + @property + def snapshot(self) -> dict[str, t.Any]: + """The document as the model encoded it when it was loaded. + + Built on first use from the raw bytes by decoding into the model and + encoding again. Going through the model means a field the stored + document lacks already has its default here, so loading a document never + looks like editing it. + """ + if self._doc is None: + assert self._raw is not None + from mongospecs.meta import meta_of + + meta = self.manager.meta + loaded = meta.decode(bson.decode(self._raw, self.manager.registry._codec_options)) + self._doc = meta_of(type(loaded)).encode(loaded) + self._raw = None + return self._doc _states: dict[int, DocState] = {} @@ -42,7 +72,7 @@ def state_of(obj: t.Any) -> DocState | None: def track( obj: t.Any, - snapshot: dict[str, t.Any], + snapshot: dict[str, t.Any] | bytes, manager: t.Any, partial: frozenset[str] | None = None, ) -> DocState: diff --git a/packages/mongospecs/tests/test_registry_errors.py b/packages/mongospecs/tests/test_registry_errors.py index 50d0daa..4a5d786 100644 --- a/packages/mongospecs/tests/test_registry_errors.py +++ b/packages/mongospecs/tests/test_registry_errors.py @@ -140,6 +140,12 @@ async def test_default_max_time_ms_from_registry_and_model(m, make_registry): assert [c["maxTimeMS"] for c in finds] == [1234, 99] counts = [e.command for e in events if e.phase == "started" and e.command_name == "aggregate"] assert counts[0]["maxTimeMS"] == 1234 + # every path that runs a pipeline carries the default too (found by the storefront e2e tests) + events.clear() + await m.User.aggregate([{"$match": {}}]) + await m.User.find().union(m.User.find()) + pipelines = [e.command for e in events if e.phase == "started" and e.command_name == "aggregate"] + assert len(pipelines) >= 2 and all(c["maxTimeMS"] == 1234 for c in pipelines) async def test_command_events_carry_model_and_operation(m, make_registry, caplog): diff --git a/packages/mongospecs/tests/test_tracking_tx.py b/packages/mongospecs/tests/test_tracking_tx.py index 4f5d288..1837e85 100644 --- a/packages/mongospecs/tests/test_tracking_tx.py +++ b/packages/mongospecs/tests/test_tracking_tx.py @@ -188,3 +188,40 @@ async def slow(tx): assert await m.User.count() == 0 assert await m.User.find(Q.name == "x").exists() is False await m.User(name="after").insert() # the client is still usable + + +async def test_freshly_loaded_documents_report_no_changes(m, registry): + """The snapshot is the raw stored form; loading must not look like editing, whatever the types.""" + user = await m.User(name="ann", age=3, color=m.Color.BLUE, address=m.Address(city="Oslo")).insert() + article = await m.Article( + title="t", author=user, editors=[user], reviewers={"a": user}, tags=["x"] + ).insert() + # documents written by other code: an int where the model has a float-like default, missing optionals + await m.User.collection().insert_one({"name": "sparse"}) + await m.Video.collection().insert_one({"title": "v", "seconds": 60.0}) + for model in (m.User, m.Article, m.Video): + for doc in await model.find(): + assert doc.changes() == {}, (model.__name__, doc.changes()) + assert not doc.is_new + loaded = await m.Article.get(article.id) + loaded.tags.append("y") + assert loaded.changes() == {"$set": {"tags": ["x", "y"]}} + sparse = await m.User.find(Q.name == "sparse").one() + sparse.age = 7 + await sparse.save() + assert await m.User.collection().find_one({"name": "sparse"}, {"_id": 0}) == {"name": "sparse", "age": 7} + + +async def test_documents_written_before_a_field_existed(m, registry): + await m.User.collection().insert_one({"name": "old shape"}) + await m.Article.collection().insert_one({"title": "no revision yet"}) + doc = await m.User.find(Q.name == "old shape").one() + assert doc.changes() == {} and doc.age == 0 # the default is applied in memory, not written + doc.name = "renamed" + await doc.save() + assert await m.User.collection().find_one({"name": "renamed"}, {"_id": 0}) == {"name": "renamed"} + # a stored document without the revision field can still be saved, and gets one + article = await m.Article.find().one() + article.title = "now revisioned" + await article.save() + assert article.rev == 1 and (await m.Article.collection().find_one({}))["rev"] == 1 diff --git a/pyproject.toml b/pyproject.toml index 52809c9..09120d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ asyncio_default_fixture_loop_scope = "session" asyncio_default_test_loop_scope = "session" markers = [ "sharded: needs the sharded cluster from docker/compose.sharded.yaml", + "chaos: steps the replica set primary down while the system is under load", ] [tool.ruff] diff --git a/uv.lock b/uv.lock index 57e90f6..132b066 100644 --- a/uv.lock +++ b/uv.lock @@ -1102,6 +1102,7 @@ dependencies = [ { name = "httpx" }, { name = "mongospecs", extra = ["all", "sentry"] }, { name = "mongospecs-web", extra = ["all"] }, + { name = "uvicorn" }, ] [package.metadata] @@ -1109,6 +1110,7 @@ requires-dist = [ { name = "httpx", specifier = ">=0.27" }, { name = "mongospecs", extras = ["all", "sentry"], editable = "packages/mongospecs" }, { name = "mongospecs-web", extras = ["all"], editable = "packages/mongospecs-web" }, + { name = "uvicorn", specifier = ">=0.30" }, ] [[package]] @@ -2376,6 +2378,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717 }, ] +[[package]] +name = "uvicorn" +version = "0.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/34/30e9280707135d2cfc589dfff3cb796bd07a3aeb1a3e415ba09dd89d7bb4/uvicorn-0.54.0.tar.gz", hash = "sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620", size = 112283 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/0c/b54a4fdd7f90a3af8b02ebc9ce6712c2c208b7926a2f7bad95c33ebbe943/uvicorn-0.54.0-py3-none-any.whl", hash = "sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf", size = 87427 }, +] + [[package]] name = "watchdog" version = "6.0.0" From 6dc837fd83362204797ac468559185a601d9ed93 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 00:45:48 -0400 Subject: [PATCH 03/26] perf(web): native JSON serialisation and untracked reads; fix load generator - Codec.to_json: msgspec.to_builtins / pydantic model_dump(mode='json') produce the Read view directly instead of encoding to BSON types and converting those; responses are dumped with msgspec's JSON encoder when available - list and get requests read lean documents: nothing is saved, so nothing is tracked - upserts give the inserted document the model's defaults, so it has the same shape as one created by insert() - load benchmark: the httpx-based generator capped out below the server's capacity, so its throughput numbers measured the client. Replaced with a minimal keep-alive client; the primary metric is now server CPU time per request, with latency taken on an unloaded server and throughput at saturation - CI job for the Sentry -> rustrak tracing check; plan amendments A14-A21 and nodes E4-E6 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yaml | 13 + README.md | 16 +- bench/src/mongospecs_bench/load.py | 276 ++++++++++++------ claudedocs/v2-plan.md | 16 + docs/parity.md | 12 +- packages/mongospecs-web/README.md | 52 +++- .../src/mongospecs_web/backend.py | 14 +- .../src/mongospecs_web/policy.py | 65 ++--- .../src/mongospecs_web/resource.py | 24 +- .../src/mongospecs_web/testing.py | 4 +- packages/mongospecs/README.md | 16 +- packages/mongospecs/src/mongospecs/attrs.py | 17 ++ packages/mongospecs/src/mongospecs/codec.py | 16 + .../mongospecs/src/mongospecs/jsonable.py | 53 ++++ packages/mongospecs/src/mongospecs/manager.py | 19 ++ packages/mongospecs/src/mongospecs/meta.py | 14 + packages/mongospecs/src/mongospecs/msgspec.py | 28 ++ .../mongospecs/src/mongospecs/pydantic.py | 16 + packages/mongospecs/tests/test_updates.py | 25 ++ 19 files changed, 539 insertions(+), 157 deletions(-) create mode 100644 packages/mongospecs/src/mongospecs/jsonable.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b360269..4e32943 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -53,3 +53,16 @@ jobs: run: | docker compose -f docker/compose.rs.yaml logs --tail 100 docker compose -f docker/compose.sharded.yaml logs --tail 100 + + tracing: + # End to end: Sentry SDK -> mongospecs-web -> MongoDB, received by a Sentry-compatible server. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - run: docker compose -f docker/compose.rs.yaml up -d --wait + - run: docker compose -f docker/compose.rustrak.yaml up -d + - run: uv sync --all-packages --python 3.13 + - name: Wait for rustrak + run: timeout 60 sh -c 'until curl -sf http://localhost:8787/health >/dev/null; do sleep 1; done' + - run: uv run mongospecs-bench trace diff --git a/README.md b/README.md index 85c4f07..aa58713 100644 --- a/README.md +++ b/README.md @@ -87,12 +87,26 @@ async def main(): | Operations | explicit `Registry` lifecycle, transactions, index planner with gated destructive actions, `$jsonSchema` validators, change streams, multi-database use, shard-key aware writes | | Observability | per-command events with model and operation names; logging, OpenTelemetry and Sentry listeners | +## How it is tested + +- Unit and contract suites run against real MongoDB (a replica set and a sharded cluster), on + every backend, on Python 3.11 to 3.15. +- A reference application, the storefront in `bench/`, is booted as a real server process and + driven over HTTP: a full customer journey on every plug, concurrent buyers that must never + oversell, multi-item orders that are all-or-nothing, a change-stream projector that must count + each sale exactly once across restarts, and orders that stay consistent across a primary + step-down. +- Every example in the guide pages is executed. +- Benchmarks and stress scenarios in `bench/` run against real MongoDB; results and the + conclusions drawn from them are in `docs/benchmarks.md`. + ## Development ``` make install # uv sync for the whole workspace make up # MongoDB replica set + sharded cluster in Docker -make check # ruff, mypy --strict, tests against real MongoDB +make check # ruff, mypy --strict, tests against real MongoDB (unit, contract, end-to-end) +uv run mongospecs-bench run # benchmarks; `run LOAD` for the HTTP load test; `stress`; `trace` ``` The repository is a uv workspace: `packages/mongospecs`, `packages/mongospecs-web` and `bench/` diff --git a/bench/src/mongospecs_bench/load.py b/bench/src/mongospecs_bench/load.py index 6e4a9ef..ceabdb8 100644 --- a/bench/src/mongospecs_bench/load.py +++ b/bench/src/mongospecs_bench/load.py @@ -12,14 +12,17 @@ from __future__ import annotations import asyncio +import contextlib +import json +import os +import pathlib import random import statistics +import subprocess import time import typing as t import uuid -import httpx - from mongospecs import Registry from mongospecs_bench.harness import Result, Settings, uri_for from mongospecs_bench.storefront import models as shop_models @@ -36,50 +39,156 @@ def _percentile(values: list[float], q: float) -> float: return ordered[min(len(ordered) - 1, round(q * (len(ordered) - 1)))] +class Connection: + """A minimal HTTP/1.1 keep-alive client connection. + + The load generator must not be the bottleneck. A general-purpose client + spends more CPU per request than the server under test does; this one writes + a request and reads a length-delimited response, nothing else. + """ + + def __init__(self, host: str, port: int) -> None: + self.host, self.port = host, port + self.reader: asyncio.StreamReader | None = None + self.writer: asyncio.StreamWriter | None = None + + async def open(self) -> None: + self.reader, self.writer = await asyncio.open_connection(self.host, self.port) + + async def close(self) -> None: + if self.writer is not None: + self.writer.close() + with contextlib.suppress(Exception): + await self.writer.wait_closed() + + async def request( + self, method: str, path: str, *, token: str | None = None, body: bytes = b"" + ) -> tuple[int, bytes]: + assert self.reader is not None and self.writer is not None + head = [f"{method} {path} HTTP/1.1", f"host: {self.host}:{self.port}"] + if token: + head.append(f"authorization: Bearer {token}") + if body: + head.append("content-type: application/json") + head.append(f"content-length: {len(body)}") + self.writer.write(("\r\n".join(head) + "\r\n\r\n").encode() + body) + await self.writer.drain() + raw = await self.reader.readuntil(b"\r\n\r\n") + lines = raw.split(b"\r\n") + status = int(lines[0].split(b" ", 2)[1]) + length, chunked = 0, False + for line in lines[1:]: + name, _, value = line.partition(b":") + lowered = name.lower() + if lowered == b"content-length": + length = int(value) + elif lowered == b"transfer-encoding" and b"chunked" in value.lower(): + chunked = True + if chunked: + parts = [] + while True: + size = int((await self.reader.readline()).strip() or b"0", 16) + if size == 0: + await self.reader.readline() + break + parts.append(await self.reader.readexactly(size)) + await self.reader.readline() + return status, b"".join(parts) + return status, await self.reader.readexactly(length) if length else b"" + + +def process_cpu_seconds(pid: int) -> float: + """CPU time (user + system) a process has used so far.""" + stat = pathlib.Path(f"/proc/{pid}/stat") + if stat.exists(): + fields = stat.read_text().rsplit(")", 1)[1].split() + return (int(fields[11]) + int(fields[12])) / os.sysconf("SC_CLK_TCK") + out = subprocess.run( + ["ps", "-o", "cputime=", "-p", str(pid)], capture_output=True, text=True + ).stdout.strip() + total = 0.0 + for part in out.replace("-", ":").split(":"): + total = total * 60 + float(part) + return total + + async def _virtual_user( index: int, - client: httpx.AsyncClient, + host: str, + port: int, seeded: Seeded, stop: asyncio.Event, samples: dict[str, list[float]], failures: dict[str, int], rng: random.Random, ) -> None: - token = {"authorization": f"Bearer {seeded.user_tokens[index % len(seeded.user_tokens)]}"} + token = seeded.user_tokens[index % len(seeded.user_tokens)] actions = [name for name, weight in MIX for _ in range(weight)] cursor: str | None = None category = rng.choice(CATEGORIES) - while not stop.is_set(): - action = rng.choice(actions) - started = time.perf_counter() - try: - if action == "list": - url = f"/api/products?category={category}&sort=price&limit=20" - if cursor and rng.random() < 0.5: - url += f"&after={cursor}" + connection = Connection(host, port) + await connection.open() + try: + while not stop.is_set(): + action = rng.choice(actions) + started = time.perf_counter() + try: + if action == "list": + if cursor and rng.random() < 0.5: + path = f"/api/products?category={category}&sort=price&limit=20&after={cursor}" + else: + category = rng.choice(CATEGORIES) + path = f"/api/products?category={category}&sort=price&limit=20" + status, payload = await connection.request("GET", path) + cursor = json.loads(payload)["page"]["next_cursor"] if status == 200 else None + elif action == "detail": + product = rng.choice(seeded.product_ids) + status, payload = await connection.request( + "GET", f"/api/products/{product}?include=reviews,review_count" + ) else: - category = rng.choice(CATEGORIES) - url = f"/api/products?category={category}&sort=price&limit=20" - response = await client.get(url) - cursor = response.json()["page"]["next_cursor"] if response.status_code == 200 else None - elif action == "detail": - response = await client.get( - f"/api/products/{rng.choice(seeded.product_ids)}?include=reviews,review_count" - ) + lines = [ + {"sku": sku, "qty": rng.randint(1, 2)} + for sku in rng.sample(seeded.skus, k=rng.randint(1, 3)) + ] + status, payload = await connection.request( + "POST", "/api/orders", token=token, body=json.dumps({"lines": lines}).encode() + ) + ok = status in (200, 201) + except (OSError, asyncio.IncompleteReadError, ValueError): + ok = False + await connection.close() + await connection.open() + elapsed = time.perf_counter() - started + if ok: + samples[action].append(elapsed) else: - lines = [ - {"sku": sku, "qty": rng.randint(1, 2)} - for sku in rng.sample(seeded.skus, k=rng.randint(1, 3)) - ] - response = await client.post("/api/orders", json={"lines": lines}, headers=token) - ok = response.status_code in (200, 201) - except httpx.HTTPError: - ok = False - elapsed = time.perf_counter() - started - if ok: - samples[action].append(elapsed) - else: - failures[action] += 1 + failures[action] += 1 + finally: + await connection.close() + + +async def _phase( + server: t.Any, seeded: Seeded, users: int, seconds: float +) -> tuple[dict[str, list[float]], dict[str, int], float, float]: + """Run ``users`` virtual users for ``seconds``. Returns samples, failures, elapsed and server CPU seconds.""" + host, port = server.base_url.split("://", 1)[1].split(":") + samples: dict[str, list[float]] = {name: [] for name, _ in MIX} + failures: dict[str, int] = {name: 0 for name, _ in MIX} + stop = asyncio.Event() + cpu_before = process_cpu_seconds(server.process.pid) + started = time.perf_counter() + tasks = [ + asyncio.create_task( + _virtual_user(i, host, int(port), seeded, stop, samples, failures, random.Random(1000 + i)) + ) + for i in range(users) + ] + await asyncio.sleep(seconds) + stop.set() + await asyncio.gather(*tasks) + elapsed = time.perf_counter() - started + return samples, failures, elapsed, process_cpu_seconds(server.process.pid) - cpu_before async def run_load( @@ -88,11 +197,14 @@ async def run_load( app: str = "storefront", plug: str = "asgi", backend: str = "msgspec", - users: int = 32, + users: int = 64, seconds: float = 15.0, products: int = 5000, ) -> list[dict[str, t.Any]]: - """One load run. Returns rows per action plus an ``all`` row; raises if the stock invariant fails.""" + """One load run: a light phase for latency, then a saturating phase for capacity. + + Raises if the stock invariant does not hold afterwards. + """ database = f"ms_load_{uuid.uuid4().hex[:10]}" uri = uri_for(settings.topology) m = shop_models.build(backend) @@ -112,63 +224,33 @@ async def run_load( backend=backend, projector=False, ) as server: - limits = httpx.Limits(max_connections=users, max_keepalive_connections=users) - async with httpx.AsyncClient(base_url=server.base_url, timeout=30, limits=limits) as client: - for warm in (2.0, seconds): # a short warm-up, then the measured run - samples: dict[str, list[float]] = {name: [] for name, _ in MIX} - failures: dict[str, int] = {name: 0 for name, _ in MIX} - stop = asyncio.Event() - tasks = [ - asyncio.create_task( - _virtual_user( - i, client, seeded, stop, samples, failures, random.Random(1000 + i) - ) - ) - for i in range(users) - ] - started = time.perf_counter() - await asyncio.sleep(warm) - stop.set() - await asyncio.gather(*tasks) - elapsed = time.perf_counter() - started + await _phase(server, seeded, 4, 2.0) # warm-up + light = await _phase(server, seeded, 2, seconds / 3) + heavy = await _phase(server, seeded, users, seconds) invariant = await stock_invariant(registry, m, seeded) finally: await registry.client.drop_database(database) if not invariant["ok"]: raise AssertionError(f"{label}: stock invariant violated under load: {invariant}") - rows: list[dict[str, t.Any]] = [] - everything: list[float] = [] + + samples, failures, elapsed, cpu = heavy + total = sum(len(v) for v in samples.values()) + row: dict[str, t.Any] = { + "app": label, + "server_cpu_ms_per_request": round(cpu / total * 1000, 3) if total else 0.0, + "rps_at_saturation": round(total / elapsed, 1), + "server_cpu_utilisation": round(cpu / elapsed, 2), + } + quiet = light[0] for action, _ in MIX: - values = samples[action] - everything.extend(values) - if not values: - continue - rows.append( - { - "app": label, - "action": action, - "requests": len(values), - "rps": round(len(values) / elapsed, 1), - "p50_ms": round(statistics.median(values) * 1000, 2), - "p95_ms": round(_percentile(values, 0.95) * 1000, 2), - "p99_ms": round(_percentile(values, 0.99) * 1000, 2), - "failed": failures[action], - } - ) - rows.append( - { - "app": label, - "action": "all", - "requests": len(everything), - "rps": round(len(everything) / elapsed, 1), - "p50_ms": round(statistics.median(everything) * 1000, 2), - "p95_ms": round(_percentile(everything, 0.95) * 1000, 2), - "p99_ms": round(_percentile(everything, 0.99) * 1000, 2), - "failed": sum(failures.values()), - } - ) - rows[-1]["orders_placed"] = invariant["orders_placed"] - return rows + values = quiet[action] + row[f"{action}_p50_ms"] = round(statistics.median(values) * 1000, 2) if values else "" + row[f"{action}_p95_ms"] = round(_percentile(values, 0.95) * 1000, 2) if values else "" + busy = [v for values in samples.values() for v in values] + row["saturated_p99_ms"] = round(_percentile(busy, 0.99) * 1000, 1) if busy else "" + row["failed"] = sum(failures.values()) + sum(light[1].values()) + row["orders_placed"] = invariant["orders_placed"] + return [row] async def load(settings: Settings) -> Result: @@ -179,7 +261,7 @@ async def load(settings: Settings) -> Result: "What do mongospecs and mongospecs-web cost per request in a real server, against the same " "endpoints written by hand with pymongo?", ) - users = settings.scale(32, 6) + users = settings.scale(64, 8) seconds = float(settings.scale(15, 3)) products = settings.scale(5000, 200) runs: list[tuple[str, str, str]] = [("raw", "asgi", "msgspec")] @@ -191,19 +273,25 @@ async def load(settings: Settings) -> Result: runs = runs[:2] baseline: dict[str, float] = {} for app, plug, backend in runs: - rows = await run_load( + for row in await run_load( settings, app=app, plug=plug, backend=backend, users=users, seconds=seconds, products=products - ) - for row in rows: + ): if app == "raw": - baseline[row["action"]] = row["rps"] + baseline = {"cpu": row["server_cpu_ms_per_request"], "rps": row["rps_at_saturation"]} + row["cpu_vs_raw"] = ( + round(row["server_cpu_ms_per_request"] / baseline["cpu"], 2) if baseline.get("cpu") else "" + ) row["rps_vs_raw"] = ( - round(row["rps"] / baseline[row["action"]], 2) if baseline.get(row["action"]) else "" + round(row["rps_at_saturation"] / baseline["rps"], 2) if baseline.get("rps") else "" ) result.add(**row) result.notes.append( - f"{users} virtual users for {seconds:.0f}s after a 2s warm-up; mix {dict(MIX)}; {products} products, " - "3 reviews each; one uvicorn worker; load generator and server on the same machine." + f"Mix {dict(MIX)}; {products} products with 3 reviews each; one uvicorn worker. Latency columns: 2 virtual " + f"users for {seconds / 3:.0f}s (an unloaded server). Capacity columns: {users} virtual users for {seconds:.0f}s." + ) + result.notes.append( + "server_cpu_ms_per_request is CPU time of the server process divided by requests served: it does not " + "depend on the load generator, which runs on the same machine." ) result.notes.append( "The stock invariant (stock + units sold = initial stock, never negative) held after every run." diff --git a/claudedocs/v2-plan.md b/claudedocs/v2-plan.md index ba1c40e..4809fd0 100644 --- a/claudedocs/v2-plan.md +++ b/claudedocs/v2-plan.md @@ -865,9 +865,25 @@ Recorded per §7.1 rule 4. Each one names what changed and why; none invalidates | A11 | §4.7 | `Registry(on_command=...)` listeners: `logging_listener`, `otel_listener`, and `sentry_listener` (new, requested 2026-10-13). End-to-end trace verification against a Sentry-compatible server (rustrak) is node `E3` | Jay asked for Sentry SDK integration with rustrak for end-to-end tracing | | A12 | §5 | Versions: `mongospecs` 0.2.0, `mongospecs-web` 0.1.0 | Decided by Jay | | A13 | §7.2 | Spikes S3, S4, S6 and S8 were answered by the contract tests rather than by throwaway code: markers under deferred annotations (suite passes on 3.11, 3.13, 3.15), per-field validation on all three backends, snapshot tracking, discriminators and forward references | The tests are the durable form of the spike | +| A14 | §4.4 | The `trusted` read option is removed | Measured (B1): the pydantic fast path was slower than validating (17µs vs 5µs per wide document) and the flag did nothing on msgspec and attrs | +| A15 | §4.5 | A tracked document keeps the raw BSON bytes it was read from. The comparable snapshot (decode into the model, encode again) is built only when the document is saved | Measured (B2, B8): building the snapshot at load time made tracked reads about twice as slow as lean reads | +| A16 | §4.5 | A field the stored document lacks is not written back just because the model has a default for it; loading a document never produces changes | Follows from A15. What Mongoose does with defaults applied at load time was not verified | +| A17 | §4.4 | Documents read from the database are decoded leniently on strict backends (msgspec, attrs): an integer stored as a double is accepted. Input (creates, inserts from dicts, updates) stays strict | Data written by other clients is commonly numerically "loose"; pydantic is already lax | +| A18 | §4.2 | `Policy(private=True)` also means not writable, unless `insert`/`update` are set explicitly | Secure default found while building the web layer | +| A19 | §4.6 | Cursor pagination needs an index ending in `_id`; with an index on the sort key alone the server sorts in memory | Measured (B5): 20ms vs under 1ms for the first page of 50k documents | +| A20 | §4.8 | `Resource(server_fields=...)` sets values on create outside the field policy | Needed by the reference application (the author of a review) | +| A21 | §6 | The evidence track gained a production-shaped reference application ("storefront"), end-to-end tests that boot it as a real server process, and an HTTP load benchmark with a hand-written pymongo baseline (nodes E4-E6) | Requested by Jay 2026-10-13: real end-to-end, production-ready scenarios | New node: | ID | Node | Needs | Check | |---|---|---|---| | E3 | End-to-end tracing: Sentry SDK in an app using `mongospecs-web`, sending to rustrak in Docker; one HTTP request yields one trace containing the request span and `Model.op` database spans | P1, K17 | A script asserts the trace arrived in rustrak with the expected spans | +| E4 | Storefront reference application: auth, guards, row scope, transactional orders, soft delete, ETags, reverse relations, exactly-once change-stream projector, health check, graceful shutdown; runs on every plug and model backend | P1-P5, K6, R5 | The e2e suite below | +| E5 | End-to-end tests over real HTTP against the storefront under uvicorn: customer journey on every plug, ordering on every backend, concurrent buyers never oversell, multi-item orders are atomic, projector resumes across restarts without double counting, in-flight requests finish on shutdown, orders stay consistent across a primary step-down | E4 | `uv run pytest bench/tests/test_e2e_storefront.py` passes; the stock invariant holds after each scenario | +| E6 | HTTP load benchmark: mixed browse/detail/order traffic against the running storefront on every plug and backend, next to a hand-written pymongo + Starlette implementation of the same endpoints | E4, E1 | Results committed; the stock invariant holds after every run | + +Bugs the end-to-end tests found that the unit and contract suites had not: aggregation paths +(`aggregate`, `union`, `graph`, backref counts) broke whenever a default time limit was configured; +string ids assigned to reference fields were stored as strings on msgspec and attrs; unexpected +server errors were not logged anywhere. diff --git a/docs/parity.md b/docs/parity.md index 0ff8e17..4a38ccb 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -18,9 +18,9 @@ was established: **docs** means it was re-read in the Mongoose documentation for | Middleware: document, query, aggregate, model | document hooks; `on_insert`/`on_update`/`on_filter`/`on_delete`; `on_pipeline`; `before_bulk`/`after_bulk` | `test_plugins_hooks.py` | docs | | `schema.plugin()`, `mongoose.plugin()` | marker plugins, `SpecConfig(plugins=)`, `Registry(plugins=)` | `test_plugins_hooks.py::test_registry_level_plugin_applies_to_every_model` | memory | | `timestamps` | `CreatedAt`, `UpdatedAt` | `test_plugins_hooks.py::test_timestamps_on_every_write_path` | memory | -| `optimisticConcurrency`, `versionKey` | `Revision` | `test_plugins_hooks.py::test_revision_increments_everywhere` | memory | -| `populate()` with `match`, `select`, `options`, `perDocumentLimit`, nested paths | `include()` with `match`, `select`, `sort`, `limit`, `per_parent_limit`, dotted paths | `test_relations.py` | memory | -| Virtual populate (`localField`/`foreignField`, `count`) | `Backref(...)`, `Backref(..., count=True)` | `test_relations.py::test_backrefs_and_counts` | memory | +| `optimisticConcurrency`, `versionKey` | `Revision` | `test_plugins_hooks.py::test_revision_increments_everywhere` | docs | +| `populate()` with `match`, `select`, `options`, `perDocumentLimit`, nested paths | `include()` with `match`, `select`, `sort`, `limit`, `per_parent_limit`, dotted paths | `test_relations.py` | docs | +| Virtual populate (`localField`/`foreignField`, `count`) | `Backref(...)`, `Backref(..., count=True)` | `test_relations.py::test_backrefs_and_counts` | docs | | `refPath` | `Ref[A \| B]` | `test_relations.py::test_polymorphic_reference` | memory | | Discriminators | `SpecConfig(discriminator=...)` | `test_models_misc.py::test_discriminators_share_a_collection` | docs | | Query casting | values cast to the field type | `test_expr.py::test_values_are_cast_to_field_types` | memory | @@ -32,10 +32,10 @@ was established: **docs** means it was re-read in the Mongoose documentation for | Query helpers | `@Model.scope` | `test_plugins_hooks.py::test_named_scopes` | memory | | `orFail()` | `.or_fail()`, `.one()` | `test_crud.py::test_find_all_first_one_and_or_fail` | memory | | `cursor()`, `eachAsync({parallel})` | `async for`, `.each(fn, concurrency=n)` | `test_crud.py::test_each_with_concurrency` | memory | -| `bulkWrite`, `bulkSave`, `insertMany({ordered: false})` | `bulk_write`, `bulk_save`, `insert_many(ordered=False)` | `test_bulk_pagination.py` | memory | +| `bulkWrite`, `bulkSave`, `insertMany({ordered: false})` | `bulk_write`, `bulk_save`, `insert_many(ordered=False)` | `test_bulk_pagination.py` | docs | | `Connection.transaction(fn)` resets document state on abort | `registry.transaction(fn)` does the same, also before each retry | `test_tracking_tx.py::test_transaction_abort_rolls_back_database_and_memory` | docs | | `syncIndexes`, `diffIndexes`, `autoIndex: false` | `plan_indexes`, `apply_indexes(drop=...)`; never automatic | `test_indexes.py` | memory | -| `shardKey` | `SpecConfig(shard_key=...)` | `test_streams_sharded.py::test_shard_key_is_added_to_document_writes` | memory | +| `shardKey` | `SpecConfig(shard_key=...)` | `test_streams_sharded.py::test_shard_key_is_added_to_document_writes` | docs | | `createConnection`, `useDb` | several registries; `Model.using(db=...)` | `test_models_misc.py::test_using_another_database` | memory | | `maxTimeMS`, `hint`, `comment`, `collation`, read/write concern | `.options(...)`, `SpecConfig`, `Registry(max_time_ms=)` | `test_registry_errors.py::test_default_max_time_ms_from_registry_and_model` | memory | | `Model.aggregate()`, `Model.collection` | `Model.aggregate(pipeline, as_=)`, `Model.collection()` | `test_crud.py::test_aggregate_decodes_and_explain` | memory | @@ -66,3 +66,5 @@ was established: **docs** means it was re-read in the Mongoose documentation for | Populated paths replace the id with the document | A `Ref` keeps `.id` and gains `.doc` | One type whether or not it is loaded | | Arrays are tracked per element | A changed list is sent whole | Per-element intent is expressed with atomic operators | | A document's delete is not version-checked | Soft delete and `save()` are revision-checked; hard delete is not | Recorded as a known gap | +| `populate()` always runs a separate query per path | `include()` does the same by default and can use `$lookup` instead (`strategy="lookup"`) | Both give the same result; the faster one depends on fan-out, see the benchmarks | +| `limit` in `populate()` is applied across all parents unless `perDocumentLimit` is used | `limit` is overall, `per_parent_limit` is per parent, without one query per parent | Same two meanings, named for what they do | diff --git a/packages/mongospecs-web/README.md b/packages/mongospecs-web/README.md index 7819dfa..707c92c 100644 --- a/packages/mongospecs-web/README.md +++ b/packages/mongospecs-web/README.md @@ -1,6 +1,54 @@ # mongospecs-web -Framework-neutral REST resources for [mongospecs](https://github.com/jaykv/mongospecs) models, -with plugs for a plain ASGI app, Starlette, FastAPI, Litestar and Quart. +REST resources for [mongospecs](https://github.com/jaykv/mongospecs) models, described once and +mounted on the framework you use: plain ASGI, Starlette, FastAPI, Litestar or Quart. > **Status: 0.1.0, alpha.** + +``` +pip install "mongospecs-web[fastapi]" "mongospecs[msgspec]" +``` + +```python +from typing import Annotated + +from mongospecs import Policy, Q, Registry, Revision, SpecConfig +from mongospecs.msgspec import Spec +from mongospecs_web import Api, Create, Delete, Get, List, Patch, Resource + + +class Task(Spec): + __spec__ = SpecConfig(collection="tasks") + title: Annotated[str, Policy(filter=("eq", "prefix"), sort=True)] + done: Annotated[bool, Policy(filter=("eq",))] = False + owner: Annotated[str, Policy(insert=False, update=False)] = "" + rev: Annotated[int, Revision] = 0 + + +tasks = Resource( + Task, + ops=[List, Get, Create, Patch, Delete], + scope=lambda ctx: Q.owner == ctx.principal, # row-level: part of every query + server_fields=lambda ctx: {"owner": ctx.principal}, # set by the server on create +) +registry = Registry("mongodb://localhost/app", models=[Task]) +app = Api(tasks, prefix="/api", authenticate=lambda request: request.headers.get("x-user")).asgi( + registry=registry +) +``` + +That is a complete API: `GET /api/tasks?title[prefix]=Wri&sort=-title&limit=20`, `POST`, `GET /{id}`, +`PATCH /{id}` with `If-Match`, `DELETE /{id}`, and `GET /api/openapi.json`. + +- **The field policy is the contract.** What can be created, patched, read, filtered and sorted + comes from `Policy` on each field. Input is validated against the real model; nothing a client + sends reaches MongoDB as a raw filter or update. +- **Authorization in two parts.** Guards decide who may call an operation; `scope` returns a + condition that is ANDed into list, get, patch and delete. +- **HTTP done properly.** ETag and `If-Match` from the revision field, RFC 9457 problem documents, + cursor or offset pagination, OpenAPI 3.1. +- **Framework-neutral.** The same resource mounts on five frameworks through thin plugs, and a + contract suite (`mongospecs_web.testing`) checks that they all behave identically. + +Guides: [resources](https://github.com/jaykv/mongospecs/blob/main/docs/web/resources.md), +[plugs](https://github.com/jaykv/mongospecs/blob/main/docs/web/plugs.md). diff --git a/packages/mongospecs-web/src/mongospecs_web/backend.py b/packages/mongospecs-web/src/mongospecs_web/backend.py index 001b6dc..61fefb2 100644 --- a/packages/mongospecs-web/src/mongospecs_web/backend.py +++ b/packages/mongospecs-web/src/mongospecs_web/backend.py @@ -61,7 +61,11 @@ class ResourceBackend(t.Protocol[T]): model: type[T] - async def get(self, id: t.Any, *, scope: t.Any, include: t.Sequence[str] = ()) -> T | None: ... + async def get( + self, id: t.Any, *, scope: t.Any, include: t.Sequence[str] = (), read_only: bool = False + ) -> T | None: + """``read_only`` says the caller will only serialise the document, never save it.""" + ... async def list(self, query: ListQuery, *, scope: t.Any) -> Page[T]: ... @@ -94,16 +98,20 @@ def _by_id(self, id: t.Any, scope: t.Any) -> t.Any: return None return query.filter(scope) if scope is not None else query - async def get(self, id: t.Any, *, scope: t.Any, include: t.Sequence[str] = ()) -> T | None: + async def get( + self, id: t.Any, *, scope: t.Any, include: t.Sequence[str] = (), read_only: bool = False + ) -> T | None: query = self._by_id(id, scope) if query is None: return None + if read_only: + query = query.lean() # nothing will be saved: skip change tracking for path in include: query = query.include(path) return t.cast("T | None", await query.first()) async def list(self, query: ListQuery, *, scope: t.Any) -> Page[T]: - q = self.manager.find(*(f.to_expr() for f in query.filters)) + q = self.manager.find(*(f.to_expr() for f in query.filters)).lean() # listings are read-only if scope is not None: q = q.filter(scope) if query.sort: diff --git a/packages/mongospecs-web/src/mongospecs_web/policy.py b/packages/mongospecs-web/src/mongospecs_web/policy.py index 6c687e2..447ba5e 100644 --- a/packages/mongospecs-web/src/mongospecs_web/policy.py +++ b/packages/mongospecs-web/src/mongospecs_web/policy.py @@ -7,19 +7,12 @@ from __future__ import annotations -import base64 -import datetime as dt -import decimal -import enum import typing as t -import uuid import weakref -from bson import ObjectId -from bson.decimal128 import Decimal128 - from mongospecs import CreatedAt, Deferred, DeletedAt, Immutable, Ref, Revision, UpdatedAt, meta_of from mongospecs.codec import FieldInfo +from mongospecs.jsonable import jsonable from mongospecs.markers import is_computed from mongospecs.meta import ModelMeta @@ -28,29 +21,6 @@ _MANAGED = (CreatedAt, UpdatedAt, DeletedAt, Revision) -def jsonable(value: t.Any) -> t.Any: - """Convert a BSON-ready value to plain JSON types.""" - if value is None or isinstance(value, str | bool | int | float): - return value - if isinstance(value, ObjectId | uuid.UUID | decimal.Decimal | Decimal128): - return str(value) - if isinstance(value, dt.datetime): - return value.isoformat().replace("+00:00", "Z") - if isinstance(value, dt.date | dt.time): - return value.isoformat() - if isinstance(value, enum.Enum): - return jsonable(value.value) - if isinstance(value, dict): - return {str(k): jsonable(v) for k, v in value.items()} - if isinstance(value, list | tuple | set | frozenset): - return [jsonable(v) for v in value] - if isinstance(value, bytes | bytearray): - return base64.b64encode(bytes(value)).decode("ascii") - if isinstance(value, Ref): - return jsonable(value.id) - return str(value) - - class View: """What of a model is readable, writable, filterable and sortable from outside.""" @@ -91,6 +61,11 @@ def __init__(self, model: type) -> None: for name, attr in vars(klass).items() if is_computed(attr) ) + # Serialisation plan: plain readable fields, and the relations that may be embedded. + self._plain: tuple[str, ...] = tuple(f.name for f in self.readable if not f.transient) + self._relations: tuple[FieldInfo, ...] = tuple( + f for f in self.readable if f.transient or f.ref is not None + ) revision = next((f for f in fields.values() if f.has(Revision)), None) self.revision: str | None = revision.name if revision else None @@ -100,24 +75,22 @@ def serialize(self, doc: t.Any, include: t.Collection[str] = ()) -> dict[str, t. """The Read view of a document as JSON types. Private fields never appear.""" if type(doc) is not self.model: return view_of(type(doc)).serialize(doc, include) - meta = self.meta - encoded = meta.encode(doc) - out: dict[str, t.Any] = {} - for f in self.readable: - nested = {p.partition(".")[2] for p in include if p.partition(".")[0] == f.name and "." in p} - if f.transient: - if f.name in include: - value = getattr(doc, f.name) + data = self.meta.codec.to_json(doc) + out = {name: data.get(name) for name in self._plain} + if include: + for f in self._relations: + if f.name not in include: + continue + nested = {p.partition(".")[2] for p in include if p.partition(".")[0] == f.name and "." in p} + value = getattr(doc, f.name) + if f.transient: out[f.name] = ( [_child(v, nested) for v in value] if isinstance(value, list) else jsonable(value) ) - continue - if f.ref is not None and f.name in include: - out[f.name] = _embed(getattr(doc, f.name), encoded.get(f.stored), nested) - continue - out[f.name] = jsonable(encoded.get(f.stored)) - if meta.disc_key: - out[meta.disc_key] = encoded.get(meta.disc_key) + else: + out[f.name] = _embed(value, out.get(f.name), nested) + if self.meta.disc_key: + out[self.meta.disc_key] = self.meta.disc_value for name in self.computed: out[name] = jsonable(getattr(doc, name)) return out diff --git a/packages/mongospecs-web/src/mongospecs_web/resource.py b/packages/mongospecs-web/src/mongospecs_web/resource.py index 050e4f2..7e0809c 100644 --- a/packages/mongospecs-web/src/mongospecs_web/resource.py +++ b/packages/mongospecs-web/src/mongospecs_web/resource.py @@ -15,6 +15,7 @@ from dataclasses import dataclass, field from mongospecs.expr import Expr +from mongospecs.jsonable import jsonable from mongospecs_web.backend import MongoBackend, ResourceBackend from mongospecs_web.errors import ( PROBLEM_JSON, @@ -26,7 +27,7 @@ problem, ) from mongospecs_web.filters import Cursor, Pagination, parse_includes, parse_list_query -from mongospecs_web.policy import View, jsonable, view_of +from mongospecs_web.policy import View, view_of __all__ = ["Create", "Ctx", "Delete", "Get", "List", "Operation", "Patch", "Request", "Resource", "Response"] @@ -45,6 +46,22 @@ JSON = "application/json" +def _stdlib_dumps(data: t.Any) -> bytes: + return json.dumps(data, separators=(",", ":"), ensure_ascii=False).encode() + + +def _pick_dumps() -> t.Callable[[t.Any], bytes]: + """Use msgspec's JSON encoder when it is installed (it usually is); the standard library otherwise.""" + try: + import msgspec + except ImportError: + return _stdlib_dumps + return msgspec.json.Encoder().encode + + +dumps: t.Callable[[t.Any], bytes] = _pick_dumps() + + @dataclass(slots=True) class Request: """A framework-neutral HTTP request.""" @@ -83,8 +100,7 @@ class Response: @classmethod def json(cls, data: t.Any, status: int = 200, headers: dict[str, str] | None = None) -> Response: - body = json.dumps(data, separators=(",", ":"), ensure_ascii=False).encode() - return cls(status, body, dict(headers or {}), JSON) + return cls(status, dumps(data), dict(headers or {}), JSON) @classmethod def error(cls, exc: BaseException) -> Response: @@ -461,7 +477,7 @@ def _one(self) -> dict[str, t.Any]: def _get_op(self) -> Operation: async def handler(ctx: Ctx[T]) -> Response: include = parse_includes(ctx.request.query, self.includes) - doc = await self.backend.get(ctx.id, scope=await ctx.scope(), include=include) + doc = await self.backend.get(ctx.id, scope=await ctx.scope(), include=include, read_only=True) if doc is None: raise NotFound("The resource does not exist.") ctx.state["result"] = doc diff --git a/packages/mongospecs-web/src/mongospecs_web/testing.py b/packages/mongospecs-web/src/mongospecs_web/testing.py index b600611..7996403 100644 --- a/packages/mongospecs-web/src/mongospecs_web/testing.py +++ b/packages/mongospecs-web/src/mongospecs_web/testing.py @@ -128,7 +128,9 @@ def _check_unique(self, doc: T, skip: t.Any = None) -> None: if other_id != skip and getattr(other, name) == value: raise DuplicateKeyError(f"duplicate {name}", code=11000, fields={name: value}) - async def get(self, id: t.Any, *, scope: t.Any, include: t.Sequence[str] = ()) -> T | None: + async def get( + self, id: t.Any, *, scope: t.Any, include: t.Sequence[str] = (), read_only: bool = False + ) -> T | None: return self._find(id, scope) async def list(self, query: ListQuery, *, scope: t.Any) -> Page[T]: diff --git a/packages/mongospecs/README.md b/packages/mongospecs/README.md index c0e696c..1dc7a52 100644 --- a/packages/mongospecs/README.md +++ b/packages/mongospecs/README.md @@ -87,12 +87,26 @@ async def main(): | Operations | explicit `Registry` lifecycle, transactions, index planner with gated destructive actions, `$jsonSchema` validators, change streams, multi-database use, shard-key aware writes | | Observability | per-command events with model and operation names; logging, OpenTelemetry and Sentry listeners | +## How it is tested + +- Unit and contract suites run against real MongoDB (a replica set and a sharded cluster), on + every backend, on Python 3.11 to 3.15. +- A reference application, the storefront in `bench/`, is booted as a real server process and + driven over HTTP: a full customer journey on every plug, concurrent buyers that must never + oversell, multi-item orders that are all-or-nothing, a change-stream projector that must count + each sale exactly once across restarts, and orders that stay consistent across a primary + step-down. +- Every example in the guide pages is executed. +- Benchmarks and stress scenarios in `bench/` run against real MongoDB; results and the + conclusions drawn from them are in `docs/benchmarks.md`. + ## Development ``` make install # uv sync for the whole workspace make up # MongoDB replica set + sharded cluster in Docker -make check # ruff, mypy --strict, tests against real MongoDB +make check # ruff, mypy --strict, tests against real MongoDB (unit, contract, end-to-end) +uv run mongospecs-bench run # benchmarks; `run LOAD` for the HTTP load test; `stress`; `trace` ``` The repository is a uv workspace: `packages/mongospecs`, `packages/mongospecs-web` and `bench/` diff --git a/packages/mongospecs/src/mongospecs/attrs.py b/packages/mongospecs/src/mongospecs/attrs.py index fea5ab3..27a3e63 100644 --- a/packages/mongospecs/src/mongospecs/attrs.py +++ b/packages/mongospecs/src/mongospecs/attrs.py @@ -14,6 +14,7 @@ from mongospecs.codec import Codec, RawField, field_error, to_bson from mongospecs.errors import ValidationError +from mongospecs.jsonable import json_default from mongospecs.markers import SpecConfig from mongospecs.model import Document from mongospecs.msgspec import _error, dec_hook, schema_hook @@ -64,6 +65,22 @@ def validate_field(self, cls: type, name: str, value: t.Any) -> t.Any: def _json_schema(self, cls: type) -> dict[str, t.Any]: return msgspec.json.schema(cls, schema_hook=schema_hook) + def to_json(self, obj: t.Any) -> dict[str, t.Any]: + return t.cast("dict[str, t.Any]", msgspec.to_builtins(obj, enc_hook=json_default)) + + def defaults(self, cls: type) -> dict[str, t.Any]: + out: dict[str, t.Any] = {} + for a in attrs.fields(cls): + if a.default is attrs.NOTHING: + continue + default = a.default + if isinstance(default, attrs.Factory): # type: ignore[arg-type] + if default.takes_self: + continue # depends on the instance; nothing to write without one + default = default.factory() + out[a.name] = default + return out + CODEC = AttrsCodec() diff --git a/packages/mongospecs/src/mongospecs/codec.py b/packages/mongospecs/src/mongospecs/codec.py index 9705992..05b6d25 100644 --- a/packages/mongospecs/src/mongospecs/codec.py +++ b/packages/mongospecs/src/mongospecs/codec.py @@ -255,6 +255,10 @@ def validate_field(self, cls: type, name: str, value: t.Any) -> t.Any: @abc.abstractmethod def _json_schema(self, cls: type) -> dict[str, t.Any]: ... + @abc.abstractmethod + def defaults(self, cls: type) -> dict[str, t.Any]: + """Default values by attribute name, for fields that have one (factories are called).""" + # -- shared ------------------------------------------------------------ def fields(self, cls: type) -> dict[str, FieldInfo]: @@ -334,6 +338,18 @@ def _special(self, cls: type) -> tuple[FieldInfo, ...]: def encode_value(self, value: t.Any) -> t.Any: return to_bson(value) + def to_json(self, obj: t.Any) -> dict[str, t.Any]: + """The model as JSON types, keyed by attribute names; references appear as ids. + + Backends override this with their native serialiser, which is far faster + than encoding to BSON types and converting those. + """ + from mongospecs.jsonable import jsonable + + cls = type(obj) + names = {f.stored: f.name for f in self.fields(cls).values()} + return {names.get(k, k): jsonable(v) for k, v in self.encode(obj).items()} + def json_schema(self, cls: type) -> dict[str, t.Any]: """JSON Schema for the model, keyed by attribute names, root inlined, ``$defs`` kept.""" schema = dict(self._json_schema(cls)) diff --git a/packages/mongospecs/src/mongospecs/jsonable.py b/packages/mongospecs/src/mongospecs/jsonable.py new file mode 100644 index 0000000..6eaa31f --- /dev/null +++ b/packages/mongospecs/src/mongospecs/jsonable.py @@ -0,0 +1,53 @@ +"""Conversion of BSON-ready values to plain JSON types.""" + +from __future__ import annotations + +import base64 +import datetime as dt +import decimal +import enum +import typing as t +import uuid + +from bson import ObjectId +from bson.decimal128 import Decimal128 + +from mongospecs.refs import Ref + +__all__ = ["json_default", "jsonable"] + + +def jsonable(value: t.Any) -> t.Any: + """Convert a BSON-ready value to plain JSON types.""" + if value is None or isinstance(value, str | bool | int | float): + return value + if isinstance(value, ObjectId | uuid.UUID | decimal.Decimal | Decimal128): + return str(value) + if isinstance(value, dt.datetime): + return value.isoformat().replace("+00:00", "Z") + if isinstance(value, dt.date | dt.time): + return value.isoformat() + if isinstance(value, enum.Enum): + return jsonable(value.value) + if isinstance(value, dict): + return {str(k): jsonable(v) for k, v in value.items()} + if isinstance(value, list | tuple | set | frozenset): + return [jsonable(v) for v in value] + if isinstance(value, bytes | bytearray): + return base64.b64encode(bytes(value)).decode("ascii") + if isinstance(value, Ref): + return jsonable(value.id) + return str(value) + + +def json_default(value: t.Any) -> t.Any: + """``enc_hook`` for serialisers that handle the standard types themselves: + only the types they do not know (ObjectId, Ref, Decimal128, nested documents) arrive here.""" + if isinstance(value, ObjectId | Decimal128): + return str(value) + if isinstance(value, Ref): + return jsonable(value.id) + identifier = getattr(value, "id", None) + if identifier is not None: # a document assigned where a reference is expected + return jsonable(identifier) + raise NotImplementedError(f"cannot serialise {type(value).__name__}") diff --git a/packages/mongospecs/src/mongospecs/manager.py b/packages/mongospecs/src/mongospecs/manager.py index a7db1fc..cbb7591 100644 --- a/packages/mongospecs/src/mongospecs/manager.py +++ b/packages/mongospecs/src/mongospecs/manager.py @@ -402,6 +402,12 @@ def _update_doc( ctx.extra["upsert"] = True if self.meta.disc_key: update.setdefault("$setOnInsert", {})[self.meta.disc_key] = self.meta.disc_value + # Give the inserted document the model's defaults (Mongoose: setDefaultsOnInsert), + # except where the update or the filter already decides the value. + taken = {key.partition(".")[0] for body in update.values() for key in body} | _filter_keys(flt) + for stored, value in self.meta.insert_defaults().items(): + if stored not in taken: + update.setdefault("$setOnInsert", {})[stored] = value for plugin in self.plugins: plugin.on_update(ctx, flt, update) return update @@ -779,3 +785,16 @@ def _write_error(error: t.Mapping[str, t.Any]) -> Exception: fields=dict(fields) if isinstance(fields, dict) else None, ) return OperationFailed(message, code=error.get("code"), details=dict(error)) + + +def _filter_keys(flt: t.Any) -> set[str]: + """Top-level field names a filter mentions, looking through $and/$or/$nor.""" + keys: set[str] = set() + if isinstance(flt, dict): + for key, value in flt.items(): + if key in ("$and", "$or", "$nor") and isinstance(value, list): + for part in value: + keys |= _filter_keys(part) + elif not key.startswith("$"): + keys.add(key.partition(".")[0]) + return keys diff --git a/packages/mongospecs/src/mongospecs/meta.py b/packages/mongospecs/src/mongospecs/meta.py index 5ade142..346d747 100644 --- a/packages/mongospecs/src/mongospecs/meta.py +++ b/packages/mongospecs/src/mongospecs/meta.py @@ -244,6 +244,20 @@ def decode(self, doc: t.Any, *, stored: bool = True) -> t.Any: cls = self.subclass_for(doc.get(self.disc_key)) return self.codec.decode(cls, doc, lax=stored) + def insert_defaults(self) -> dict[str, t.Any]: + """Stored key -> BSON-ready default, for fields with a non-null default. + + Used so a document created by an upsert has the same shape as one created + by ``insert()``; otherwise a query on a defaulted value would not find it. + """ + out: dict[str, t.Any] = {} + for name, value in self.codec.defaults(self.model).items(): + info = self.fields.get(name) + if info is None or info.transient or name == "id" or value is None: + continue + out[info.stored] = to_bson(value) + return out + def encode(self, obj: t.Any) -> dict[str, t.Any]: doc = self.codec.encode(obj) if self.disc_key: diff --git a/packages/mongospecs/src/mongospecs/msgspec.py b/packages/mongospecs/src/mongospecs/msgspec.py index e37b6b8..12de11a 100644 --- a/packages/mongospecs/src/mongospecs/msgspec.py +++ b/packages/mongospecs/src/mongospecs/msgspec.py @@ -11,6 +11,7 @@ from mongospecs.codec import Codec, RawField, field_error, to_bson from mongospecs.errors import ValidationError +from mongospecs.jsonable import json_default from mongospecs.markers import SpecConfig from mongospecs.model import Document from mongospecs.refs import OID, OID_PATTERN, Ref, _to_object_id, coerce_ref @@ -81,6 +82,33 @@ def validate_field(self, cls: type, name: str, value: t.Any) -> t.Any: def _json_schema(self, cls: type) -> dict[str, t.Any]: return msgspec.json.schema(cls, schema_hook=schema_hook) + def to_json(self, obj: t.Any) -> dict[str, t.Any]: + data: dict[str, t.Any] = msgspec.to_builtins(obj, enc_hook=json_default) + renames = self._json_renames(type(obj)) + for stored, name in renames: + if stored in data: + data[name] = data.pop(stored) + return data + + def _json_renames(self, cls: type) -> tuple[tuple[str, str], ...]: + cache: dict[type, tuple[tuple[str, str], ...]] = self.__dict__.setdefault("_jr", {}) + hit = cache.get(cls) + if hit is None: + hit = tuple( + (f.encode_name, f.name) for f in msgspec.structs.fields(cls) if f.encode_name != f.name + ) + cache[cls] = hit + return hit + + def defaults(self, cls: type) -> dict[str, t.Any]: + out: dict[str, t.Any] = {} + for f in msgspec.structs.fields(cls): + if f.default is not msgspec.NODEFAULT: + out[f.name] = f.default + elif f.default_factory is not msgspec.NODEFAULT: + out[f.name] = f.default_factory() + return out + CODEC = MsgspecCodec() diff --git a/packages/mongospecs/src/mongospecs/pydantic.py b/packages/mongospecs/src/mongospecs/pydantic.py index b2a940d..4515a80 100644 --- a/packages/mongospecs/src/mongospecs/pydantic.py +++ b/packages/mongospecs/src/mongospecs/pydantic.py @@ -89,6 +89,22 @@ def validate_field(self, cls: type, name: str, value: t.Any) -> t.Any: def _json_schema(self, cls: type) -> dict[str, t.Any]: return t.cast("type[BaseModel]", cls).model_json_schema(by_alias=False) + def to_json(self, obj: t.Any) -> dict[str, t.Any]: + if _HAS_EXCLUDE_COMPUTED: + return t.cast("dict[str, t.Any]", obj.model_dump(mode="json", exclude_computed_fields=True)) + data: dict[str, t.Any] = obj.model_dump(mode="json") + for name in type(obj).model_computed_fields: + data.pop(name, None) + return data + + def defaults(self, cls: type) -> dict[str, t.Any]: + fields = t.cast("type[BaseModel]", cls).model_fields + return { + name: info.get_default(call_default_factory=True) + for name, info in fields.items() + if not info.is_required() + } + def clear_cache(self) -> None: super().clear_cache() self._adapters.clear() diff --git a/packages/mongospecs/tests/test_updates.py b/packages/mongospecs/tests/test_updates.py index d75fdaf..ea45b83 100644 --- a/packages/mongospecs/tests/test_updates.py +++ b/packages/mongospecs/tests/test_updates.py @@ -160,3 +160,28 @@ async def test_if_rev_on_atomic_update(m, registry): await m.Article.update(art.id, if_rev=0, title="c") with pytest.raises(DocumentNotFound): await m.Article.update(ObjectId(), if_rev=0, title="c") + + +async def test_upsert_gives_the_new_document_the_model_defaults(m, registry): + """A document created by an upsert must look like one created by insert().""" + created = await m.Article.upsert(Q.title == "via upsert", Inc(views=5)) + inserted = await m.Article(title="via insert", views=5).insert() + raw_upsert = await m.Article.collection().find_one({"_id": created.id}, {"_id": 0, "title": 0, "rev": 0}) + raw_insert = await m.Article.collection().find_one({"_id": inserted.id}, {"_id": 0, "title": 0, "rev": 0}) + for doc in (raw_upsert, raw_insert): + doc.pop("created_at"), doc.pop("updated_at") + + def present(doc): + return {k: v for k, v in doc.items() if v is not None} + + assert present(raw_insert) == present(raw_upsert) + assert raw_upsert["tags"] == [] and raw_upsert["editors"] == [] and raw_upsert["reviewers"] == {} + # so a query on a defaulted value finds it + assert await m.Article.count(Q.tags == []) == 2 + # the filter and the update keep precedence over defaults, and an existing document is not reset + again = await m.Article.upsert(Q.title == "via upsert", Push(tags="x")) + assert again.tags == ["x"] and again.views == 5 + with_default = await m.User.upsert((Q.name == "u") & (Q.age == 7), email="u@x.io") + assert with_default.age == 7 + assert await m.Cat.upsert(Q.name == "tom", Inc(lives=-1)) is not None + assert (await m.Cat.collection().find_one({"name": "tom"}))["lives"] == -1 # $inc owns the field From 724d20e7b7d66c22d4168c83b147d39f4592e9eb Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 01:05:30 -0400 Subject: [PATCH 04/26] docs(bench): benchmark results, conclusions and decisions - results for all workloads, the HTTP load test, the stress scenarios and the Mongoose comparator at 6dc837f (earlier B1-B6 results kept as the before picture for the tracking optimisation) - docs/benchmarks.md: what was measured, how to read it, and what follows from it - decisions recorded in the plan: include() defaults to batched queries; no Rust in 0.2, with a BSON-to-model decoder for bulk reads as the one candidate; tracked stays the default read mode - drop workload B7 in favour of LOAD: measuring plugs through an in-process client measured the client - ignore stale class copies in subclass discovery (found by the free-threaded CI job) with a regression test - CI builds the docs in strict mode Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yaml | 1 + bench/README.md | 18 +- bench/results/6dc837f/B1.json | 198 +++++++ bench/results/6dc837f/B2.json | 508 ++++++++++++++++++ bench/results/6dc837f/B3-sharded.json | 70 +++ bench/results/6dc837f/B3.json | 70 +++ bench/results/6dc837f/B4.json | 198 +++++++ bench/results/6dc837f/B5.json | 80 +++ bench/results/6dc837f/B6.json | 90 ++++ bench/results/6dc837f/B8.json | 151 ++++++ bench/results/6dc837f/LOAD.json | 169 ++++++ bench/results/6dc837f/X1.json | 48 ++ bench/results/6dc837f/X2.json | 44 ++ bench/results/6dc837f/X3.json | 48 ++ bench/results/6dc837f/X4.json | 48 ++ bench/results/6dc837f/X5.json | 40 ++ bench/results/6dc837f/X6.json | 44 ++ bench/results/6dc837f/X7.json | 44 ++ bench/results/6dc837f/mongoose.json | 258 +++++++++ bench/src/mongospecs_bench/workloads.py | 101 +--- claudedocs/v2-plan.md | 12 + docs/benchmarks.md | 254 ++++++++- packages/mongospecs/src/mongospecs/meta.py | 32 +- packages/mongospecs/tests/test_models_misc.py | 39 ++ 24 files changed, 2459 insertions(+), 106 deletions(-) create mode 100644 bench/results/6dc837f/B1.json create mode 100644 bench/results/6dc837f/B2.json create mode 100644 bench/results/6dc837f/B3-sharded.json create mode 100644 bench/results/6dc837f/B3.json create mode 100644 bench/results/6dc837f/B4.json create mode 100644 bench/results/6dc837f/B5.json create mode 100644 bench/results/6dc837f/B6.json create mode 100644 bench/results/6dc837f/B8.json create mode 100644 bench/results/6dc837f/LOAD.json create mode 100644 bench/results/6dc837f/X1.json create mode 100644 bench/results/6dc837f/X2.json create mode 100644 bench/results/6dc837f/X3.json create mode 100644 bench/results/6dc837f/X4.json create mode 100644 bench/results/6dc837f/X5.json create mode 100644 bench/results/6dc837f/X6.json create mode 100644 bench/results/6dc837f/X7.json create mode 100644 bench/results/6dc837f/mongoose.json diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4e32943..3eef0a6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,6 +22,7 @@ jobs: - run: make lint - run: make types - run: make build + - run: make docs test: runs-on: ubuntu-latest diff --git a/bench/README.md b/bench/README.md index aa1eb6a..213730a 100644 --- a/bench/README.md +++ b/bench/README.md @@ -5,7 +5,8 @@ MongoDB. This package is a workspace member and is never published. ``` make up # replica set + sharded cluster -uv run mongospecs-bench run # workloads B1-B8 +uv run mongospecs-bench run # workloads B1-B6, B8 +uv run mongospecs-bench run LOAD # the storefront as a real server under HTTP load uv run mongospecs-bench run B3 --topology sharded uv run mongospecs-bench stress # scenarios X1-X7 uv run mongospecs-bench report # markdown summary of the newest results @@ -31,6 +32,19 @@ Absolute times depend on the machine. The comparisons that travel are ratios: mo | B4 | Inserts, atomic patches, delta saves and upserts | | B5 | Offset vs cursor pagination at depth; total by count vs `$facet` | | B6 | Per-operation cost of plugins | -| B7 | HTTP request to response through each plug and model backend | +| LOAD | The storefront as a real server under HTTP load, on every plug and model backend, next to a hand-written pymongo baseline (this is B7 of the plan) | | B8 | Time and memory of change tracking; size of a delta save | | X1-X7 | Pool pressure, transaction contention, optimistic-lock storm, cancellation, 16MB results, change stream across a step-down, threads with their own loops | + +## The storefront + +`mongospecs_bench.storefront` is a production-shaped application used by the end-to-end tests +(`bench/tests/test_e2e_storefront.py`) and the load benchmark: bearer-token auth, admin guards, row +scope on orders, an order transaction that cannot oversell, soft-deleted and revisioned products, +reviews as a reverse relation, an exactly-once change-stream projector, `/healthz`, graceful +shutdown. `raw_app.py` implements the three busiest endpoints by hand with pymongo and Starlette as +the baseline. + +``` +SHOP_DB=shop uv run uvicorn mongospecs_bench.storefront.app:create_app --factory +``` diff --git a/bench/results/6dc837f/B1.json b/bench/results/6dc837f/B1.json new file mode 100644 index 0000000..12ebb3e --- /dev/null +++ b/bench/results/6dc837f/B1.json @@ -0,0 +1,198 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": false, + "timestamp": "2026-10-13T04:45:53+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B1", + "title": "Materialise documents", + "question": "What does turning a raw document into a model cost, per backend, and how does it compare with the driver's own BSON decoding?", + "rows": [ + { + "shape": "small", + "backend": "driver", + "mode": "bson->dict", + "n": 10000, + "us_per_doc": 1.12, + "vs_bson": 1.0 + }, + { + "shape": "small", + "backend": "msgspec", + "mode": "decode", + "n": 10000, + "us_per_doc": 1.25, + "vs_bson": 1.11 + }, + { + "shape": "small", + "backend": "msgspec", + "mode": "decode + encode", + "n": 10000, + "us_per_doc": 3.14, + "vs_bson": 2.81 + }, + { + "shape": "small", + "backend": "pydantic", + "mode": "decode", + "n": 10000, + "us_per_doc": 2.09, + "vs_bson": 1.87 + }, + { + "shape": "small", + "backend": "pydantic", + "mode": "decode + encode", + "n": 10000, + "us_per_doc": 4.52, + "vs_bson": 4.04 + }, + { + "shape": "small", + "backend": "attrs", + "mode": "decode", + "n": 10000, + "us_per_doc": 1.94, + "vs_bson": 1.74 + }, + { + "shape": "small", + "backend": "attrs", + "mode": "decode + encode", + "n": 10000, + "us_per_doc": 4.38, + "vs_bson": 3.92 + }, + { + "shape": "wide", + "backend": "driver", + "mode": "bson->dict", + "n": 10000, + "us_per_doc": 6.92, + "vs_bson": 1.0 + }, + { + "shape": "wide", + "backend": "msgspec", + "mode": "decode", + "n": 10000, + "us_per_doc": 2.07, + "vs_bson": 0.3 + }, + { + "shape": "wide", + "backend": "msgspec", + "mode": "decode + encode", + "n": 10000, + "us_per_doc": 8.89, + "vs_bson": 1.28 + }, + { + "shape": "wide", + "backend": "pydantic", + "mode": "decode", + "n": 10000, + "us_per_doc": 6.18, + "vs_bson": 0.89 + }, + { + "shape": "wide", + "backend": "pydantic", + "mode": "decode + encode", + "n": 10000, + "us_per_doc": 10.05, + "vs_bson": 1.45 + }, + { + "shape": "wide", + "backend": "attrs", + "mode": "decode", + "n": 10000, + "us_per_doc": 6.58, + "vs_bson": 0.95 + }, + { + "shape": "wide", + "backend": "attrs", + "mode": "decode + encode", + "n": 10000, + "us_per_doc": 21.38, + "vs_bson": 3.09 + }, + { + "shape": "nested", + "backend": "driver", + "mode": "bson->dict", + "n": 10000, + "us_per_doc": 10.09, + "vs_bson": 1.0 + }, + { + "shape": "nested", + "backend": "msgspec", + "mode": "decode", + "n": 10000, + "us_per_doc": 3.32, + "vs_bson": 0.33 + }, + { + "shape": "nested", + "backend": "msgspec", + "mode": "decode + encode", + "n": 10000, + "us_per_doc": 25.96, + "vs_bson": 2.57 + }, + { + "shape": "nested", + "backend": "pydantic", + "mode": "decode", + "n": 10000, + "us_per_doc": 15.88, + "vs_bson": 1.57 + }, + { + "shape": "nested", + "backend": "pydantic", + "mode": "decode + encode", + "n": 10000, + "us_per_doc": 17.75, + "vs_bson": 1.76 + }, + { + "shape": "nested", + "backend": "attrs", + "mode": "decode", + "n": 10000, + "us_per_doc": 8.32, + "vs_bson": 0.83 + }, + { + "shape": "nested", + "backend": "attrs", + "mode": "decode + encode", + "n": 10000, + "us_per_doc": 20.68, + "vs_bson": 2.05 + } + ], + "notes": [ + "'decode' always validates. 'decode + encode' adds turning the model back into a BSON-ready dict (what a save does)." + ], + "conclusion": "" +} diff --git a/bench/results/6dc837f/B2.json b/bench/results/6dc837f/B2.json new file mode 100644 index 0000000..5f46352 --- /dev/null +++ b/bench/results/6dc837f/B2.json @@ -0,0 +1,508 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": false, + "timestamp": "2026-10-13T04:45:53+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B2", + "title": "find() end to end", + "question": "How much does the ODM add on top of the raw driver for reads of 1, 100 and 10,000 documents?", + "rows": [ + { + "shape": "small", + "backend": "pymongo", + "case": "driver dicts", + "n": 1, + "median_ms": 0.421, + "vs_driver": 1.0 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm raw()", + "n": 1, + "median_ms": 0.503, + "vs_driver": 1.19 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm lean()", + "n": 1, + "median_ms": 0.392, + "vs_driver": 0.93 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm tracked", + "n": 1, + "median_ms": 0.434, + "vs_driver": 1.03 + }, + { + "shape": "small", + "backend": "pymongo", + "case": "driver dicts", + "n": 100, + "median_ms": 0.562, + "vs_driver": 1.0 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm raw()", + "n": 100, + "median_ms": 0.578, + "vs_driver": 1.03 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm lean()", + "n": 100, + "median_ms": 0.696, + "vs_driver": 1.24 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm tracked", + "n": 100, + "median_ms": 0.899, + "vs_driver": 1.6 + }, + { + "shape": "small", + "backend": "pymongo", + "case": "driver dicts", + "n": 10000, + "median_ms": 19.226, + "vs_driver": 1.0 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm raw()", + "n": 10000, + "median_ms": 43.938, + "vs_driver": 2.29 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm lean()", + "n": 10000, + "median_ms": 41.261, + "vs_driver": 2.15 + }, + { + "shape": "small", + "backend": "msgspec", + "case": "odm tracked", + "n": 10000, + "median_ms": 63.621, + "vs_driver": 3.31 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm raw()", + "n": 1, + "median_ms": 0.589, + "vs_driver": 1.05 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm lean()", + "n": 1, + "median_ms": 0.73, + "vs_driver": 1.3 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm tracked", + "n": 1, + "median_ms": 1.62, + "vs_driver": 2.88 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm raw()", + "n": 100, + "median_ms": 1.817, + "vs_driver": 0.61 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm lean()", + "n": 100, + "median_ms": 3.942, + "vs_driver": 1.33 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm tracked", + "n": 100, + "median_ms": 4.373, + "vs_driver": 1.47 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm raw()", + "n": 10000, + "median_ms": 33.515, + "vs_driver": 0.65 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm lean()", + "n": 10000, + "median_ms": 54.828, + "vs_driver": 1.07 + }, + { + "shape": "small", + "backend": "pydantic", + "case": "odm tracked", + "n": 10000, + "median_ms": 93.484, + "vs_driver": 1.82 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm raw()", + "n": 1, + "median_ms": 0.82, + "vs_driver": 1.06 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm lean()", + "n": 1, + "median_ms": 0.854, + "vs_driver": 1.1 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm tracked", + "n": 1, + "median_ms": 1.046, + "vs_driver": 1.35 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm raw()", + "n": 100, + "median_ms": 0.806, + "vs_driver": 0.78 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm lean()", + "n": 100, + "median_ms": 1.273, + "vs_driver": 1.23 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm tracked", + "n": 100, + "median_ms": 1.695, + "vs_driver": 1.63 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm raw()", + "n": 10000, + "median_ms": 25.294, + "vs_driver": 1.04 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm lean()", + "n": 10000, + "median_ms": 72.603, + "vs_driver": 3.0 + }, + { + "shape": "small", + "backend": "attrs", + "case": "odm tracked", + "n": 10000, + "median_ms": 70.821, + "vs_driver": 2.93 + }, + { + "shape": "nested", + "backend": "pymongo", + "case": "driver dicts", + "n": 1, + "median_ms": 0.432, + "vs_driver": 1.0 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm raw()", + "n": 1, + "median_ms": 0.455, + "vs_driver": 1.05 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm lean()", + "n": 1, + "median_ms": 0.481, + "vs_driver": 1.11 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm tracked", + "n": 1, + "median_ms": 0.507, + "vs_driver": 1.17 + }, + { + "shape": "nested", + "backend": "pymongo", + "case": "driver dicts", + "n": 100, + "median_ms": 1.193, + "vs_driver": 1.0 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm raw()", + "n": 100, + "median_ms": 1.146, + "vs_driver": 0.96 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm lean()", + "n": 100, + "median_ms": 5.1, + "vs_driver": 4.28 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm tracked", + "n": 100, + "median_ms": 4.955, + "vs_driver": 4.15 + }, + { + "shape": "nested", + "backend": "pymongo", + "case": "driver dicts", + "n": 10000, + "median_ms": 93.285, + "vs_driver": 1.0 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm raw()", + "n": 10000, + "median_ms": 97.613, + "vs_driver": 1.05 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm lean()", + "n": 10000, + "median_ms": 135.797, + "vs_driver": 1.46 + }, + { + "shape": "nested", + "backend": "msgspec", + "case": "odm tracked", + "n": 10000, + "median_ms": 150.438, + "vs_driver": 1.61 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm raw()", + "n": 1, + "median_ms": 0.49, + "vs_driver": 0.99 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm lean()", + "n": 1, + "median_ms": 0.512, + "vs_driver": 1.03 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm tracked", + "n": 1, + "median_ms": 0.521, + "vs_driver": 1.05 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm raw()", + "n": 100, + "median_ms": 1.503, + "vs_driver": 1.13 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm lean()", + "n": 100, + "median_ms": 2.651, + "vs_driver": 1.99 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm tracked", + "n": 100, + "median_ms": 2.782, + "vs_driver": 2.09 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm raw()", + "n": 10000, + "median_ms": 90.612, + "vs_driver": 0.93 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm lean()", + "n": 10000, + "median_ms": 262.019, + "vs_driver": 2.68 + }, + { + "shape": "nested", + "backend": "pydantic", + "case": "odm tracked", + "n": 10000, + "median_ms": 264.325, + "vs_driver": 2.7 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm raw()", + "n": 1, + "median_ms": 0.932, + "vs_driver": 0.9 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm lean()", + "n": 1, + "median_ms": 0.551, + "vs_driver": 0.53 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm tracked", + "n": 1, + "median_ms": 0.536, + "vs_driver": 0.52 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm raw()", + "n": 100, + "median_ms": 1.858, + "vs_driver": 1.17 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm lean()", + "n": 100, + "median_ms": 2.571, + "vs_driver": 1.62 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm tracked", + "n": 100, + "median_ms": 3.024, + "vs_driver": 1.91 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm raw()", + "n": 10000, + "median_ms": 87.638, + "vs_driver": 0.58 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm lean()", + "n": 10000, + "median_ms": 159.615, + "vs_driver": 1.06 + }, + { + "shape": "nested", + "backend": "attrs", + "case": "odm tracked", + "n": 10000, + "median_ms": 181.859, + "vs_driver": 1.21 + } + ], + "notes": [], + "conclusion": "" +} diff --git a/bench/results/6dc837f/B3-sharded.json b/bench/results/6dc837f/B3-sharded.json new file mode 100644 index 0000000..4cf5fea --- /dev/null +++ b/bench/results/6dc837f/B3-sharded.json @@ -0,0 +1,70 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": true, + "timestamp": "2026-10-13T04:55:34+00:00", + "topology": "sharded", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B3", + "title": "include(): $lookup vs batched $in", + "question": "Which strategy should include(strategy='auto') use, across fan-outs and topologies?", + "rows": [ + { + "relation": "list, distinct children", + "fan_out": 1, + "parents": 400, + "batch_ms": 9.173, + "lookup_ms": 43.112, + "lookup_vs_batch": 4.7 + }, + { + "relation": "list, distinct children", + "fan_out": 10, + "parents": 200, + "batch_ms": 26.31, + "lookup_ms": 130.27, + "lookup_vs_batch": 4.95 + }, + { + "relation": "list, distinct children", + "fan_out": 100, + "parents": 50, + "batch_ms": 52.175, + "lookup_ms": 70.531, + "lookup_vs_batch": 1.35 + }, + { + "relation": "list, distinct children", + "fan_out": 1000, + "parents": 10, + "batch_ms": 107.094, + "lookup_ms": 96.36, + "lookup_vs_batch": 0.9 + }, + { + "relation": "to-one, 20 shared targets", + "fan_out": 1, + "parents": 2000, + "batch_ms": 21.974, + "lookup_ms": 198.309, + "lookup_vs_batch": 9.02 + } + ], + "notes": [ + "Strategy choice does not depend on the model backend; measured with msgspec, lean reads." + ], + "conclusion": "$lookup was faster than batched $in in 1 of 5 cases on sharded (lookup/batch ratios: [4.7, 4.95, 1.35, 0.9, 9.02]). Current default: batch." +} diff --git a/bench/results/6dc837f/B3.json b/bench/results/6dc837f/B3.json new file mode 100644 index 0000000..d4c4a47 --- /dev/null +++ b/bench/results/6dc837f/B3.json @@ -0,0 +1,70 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": false, + "timestamp": "2026-10-13T04:45:53+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B3", + "title": "include(): $lookup vs batched $in", + "question": "Which strategy should include(strategy='auto') use, across fan-outs and topologies?", + "rows": [ + { + "relation": "list, distinct children", + "fan_out": 1, + "parents": 400, + "batch_ms": 7.674, + "lookup_ms": 8.341, + "lookup_vs_batch": 1.09 + }, + { + "relation": "list, distinct children", + "fan_out": 10, + "parents": 200, + "batch_ms": 17.435, + "lookup_ms": 19.091, + "lookup_vs_batch": 1.1 + }, + { + "relation": "list, distinct children", + "fan_out": 100, + "parents": 50, + "batch_ms": 43.373, + "lookup_ms": 54.28, + "lookup_vs_batch": 1.25 + }, + { + "relation": "list, distinct children", + "fan_out": 1000, + "parents": 10, + "batch_ms": 85.993, + "lookup_ms": 82.976, + "lookup_vs_batch": 0.96 + }, + { + "relation": "to-one, 20 shared targets", + "fan_out": 1, + "parents": 2000, + "batch_ms": 17.118, + "lookup_ms": 42.392, + "lookup_vs_batch": 2.48 + } + ], + "notes": [ + "Strategy choice does not depend on the model backend; measured with msgspec, lean reads." + ], + "conclusion": "$lookup was faster than batched $in in 0 of 5 cases on rs (lookup/batch ratios: [1.09, 1.1, 1.25, 0.96, 2.48]). Current default: batch." +} diff --git a/bench/results/6dc837f/B4.json b/bench/results/6dc837f/B4.json new file mode 100644 index 0000000..83220a7 --- /dev/null +++ b/bench/results/6dc837f/B4.json @@ -0,0 +1,198 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": false, + "timestamp": "2026-10-13T04:45:53+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B4", + "title": "Writes", + "question": "What does the ODM add to inserts, atomic patches, saves and upserts?", + "rows": [ + { + "backend": "msgspec", + "op": "insert 1000", + "case": "driver insert_many", + "median_ms": 11.347, + "vs_driver": 1.0 + }, + { + "backend": "msgspec", + "op": "insert 1000", + "case": "insert_many (no plugins)", + "median_ms": 12.212, + "vs_driver": 1.08 + }, + { + "backend": "msgspec", + "op": "insert 1000", + "case": "insert_many (timestamps+revision)", + "median_ms": 14.248, + "vs_driver": 1.26 + }, + { + "backend": "msgspec", + "op": "patch x200", + "case": "driver update_one", + "median_ms": 381.189, + "vs_driver": 1.0 + }, + { + "backend": "msgspec", + "op": "patch x200", + "case": "Model.update (returns doc)", + "median_ms": 424.609, + "vs_driver": 1.11 + }, + { + "backend": "msgspec", + "op": "patch x200", + "case": "doc.save() delta", + "median_ms": 588.127, + "vs_driver": 1.54 + }, + { + "backend": "msgspec", + "op": "upsert x200", + "case": "driver update_one(upsert)", + "median_ms": 692.723, + "vs_driver": 1.0 + }, + { + "backend": "msgspec", + "op": "upsert x200", + "case": "Model.upsert (returns doc)", + "median_ms": 913.096, + "vs_driver": 1.32 + }, + { + "backend": "pydantic", + "op": "insert 1000", + "case": "driver insert_many", + "median_ms": 11.164, + "vs_driver": 1.0 + }, + { + "backend": "pydantic", + "op": "insert 1000", + "case": "insert_many (no plugins)", + "median_ms": 12.583, + "vs_driver": 1.13 + }, + { + "backend": "pydantic", + "op": "insert 1000", + "case": "insert_many (timestamps+revision)", + "median_ms": 18.683, + "vs_driver": 1.67 + }, + { + "backend": "pydantic", + "op": "patch x200", + "case": "driver update_one", + "median_ms": 599.82, + "vs_driver": 1.0 + }, + { + "backend": "pydantic", + "op": "patch x200", + "case": "Model.update (returns doc)", + "median_ms": 775.698, + "vs_driver": 1.29 + }, + { + "backend": "pydantic", + "op": "patch x200", + "case": "doc.save() delta", + "median_ms": 638.09, + "vs_driver": 1.06 + }, + { + "backend": "pydantic", + "op": "upsert x200", + "case": "driver update_one(upsert)", + "median_ms": 834.695, + "vs_driver": 1.0 + }, + { + "backend": "pydantic", + "op": "upsert x200", + "case": "Model.upsert (returns doc)", + "median_ms": 932.299, + "vs_driver": 1.12 + }, + { + "backend": "attrs", + "op": "insert 1000", + "case": "driver insert_many", + "median_ms": 19.75, + "vs_driver": 1.0 + }, + { + "backend": "attrs", + "op": "insert 1000", + "case": "insert_many (no plugins)", + "median_ms": 12.879, + "vs_driver": 0.65 + }, + { + "backend": "attrs", + "op": "insert 1000", + "case": "insert_many (timestamps+revision)", + "median_ms": 15.12, + "vs_driver": 0.77 + }, + { + "backend": "attrs", + "op": "patch x200", + "case": "driver update_one", + "median_ms": 564.26, + "vs_driver": 1.0 + }, + { + "backend": "attrs", + "op": "patch x200", + "case": "Model.update (returns doc)", + "median_ms": 678.06, + "vs_driver": 1.2 + }, + { + "backend": "attrs", + "op": "patch x200", + "case": "doc.save() delta", + "median_ms": 511.812, + "vs_driver": 0.91 + }, + { + "backend": "attrs", + "op": "upsert x200", + "case": "driver update_one(upsert)", + "median_ms": 826.581, + "vs_driver": 1.0 + }, + { + "backend": "attrs", + "op": "upsert x200", + "case": "Model.upsert (returns doc)", + "median_ms": 918.963, + "vs_driver": 1.11 + } + ], + "notes": [ + "Model.update and Model.upsert use findAndModify to return the document; the driver baseline does not." + ], + "conclusion": "" +} diff --git a/bench/results/6dc837f/B5.json b/bench/results/6dc837f/B5.json new file mode 100644 index 0000000..6b78994 --- /dev/null +++ b/bench/results/6dc837f/B5.json @@ -0,0 +1,80 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": false, + "timestamp": "2026-10-13T04:45:53+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B5", + "title": "Pagination", + "question": "How do offset and cursor pagination compare at depth, and should a total come from $facet or a count?", + "rows": [ + { + "page": "1, index on (n) only", + "skip": 0, + "offset_ms": "", + "cursor_ms": 28.781, + "offset_vs_cursor": "" + }, + { + "page": 1, + "skip": 0, + "offset_ms": 0.575, + "cursor_ms": 0.581, + "offset_vs_cursor": 0.99 + }, + { + "page": 100, + "skip": 1980, + "offset_ms": 1.199, + "cursor_ms": 0.649, + "offset_vs_cursor": 1.85 + }, + { + "page": 1250, + "skip": 24980, + "offset_ms": 7.055, + "cursor_ms": 0.663, + "offset_vs_cursor": 10.64 + }, + { + "page": 2499, + "skip": 49960, + "offset_ms": 12.623, + "cursor_ms": 0.669, + "offset_vs_cursor": 18.86 + }, + { + "page": "total: concurrent count", + "skip": 0, + "offset_ms": 6.025, + "cursor_ms": "", + "offset_vs_cursor": "" + }, + { + "page": "total: $facet (raw driver)", + "skip": 0, + "offset_ms": 28.463, + "cursor_ms": "", + "offset_vs_cursor": "" + } + ], + "notes": [ + "Collection of 50000 documents, page size 20; index on (n, _id) unless stated.", + "$facet returns the page and the total as one document, so the page is bound by the 16MB limit." + ], + "conclusion": "" +} diff --git a/bench/results/6dc837f/B6.json b/bench/results/6dc837f/B6.json new file mode 100644 index 0000000..56d7113 --- /dev/null +++ b/bench/results/6dc837f/B6.json @@ -0,0 +1,90 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": false, + "timestamp": "2026-10-13T04:45:53+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B6", + "title": "Plugin overhead", + "question": "What does each active plugin cost per operation?", + "rows": [ + { + "plugins": "0 no-op", + "op": "insert", + "us_per_op": 2494.2, + "vs_no_plugins": 1.0 + }, + { + "plugins": "0 no-op", + "op": "save", + "us_per_op": 3044.1, + "vs_no_plugins": 1.0 + }, + { + "plugins": "0 no-op", + "op": "get", + "us_per_op": 422.3, + "vs_no_plugins": 1.0 + }, + { + "plugins": "3 no-op", + "op": "insert", + "us_per_op": 2935.7, + "vs_no_plugins": 1.18 + }, + { + "plugins": "3 no-op", + "op": "save", + "us_per_op": 3666.9, + "vs_no_plugins": 1.2 + }, + { + "plugins": "3 no-op", + "op": "get", + "us_per_op": 427.9, + "vs_no_plugins": 1.01 + }, + { + "plugins": "10 no-op", + "op": "insert", + "us_per_op": 2348.5, + "vs_no_plugins": 0.94 + }, + { + "plugins": "10 no-op", + "op": "save", + "us_per_op": 3092.5, + "vs_no_plugins": 1.02 + }, + { + "plugins": "10 no-op", + "op": "get", + "us_per_op": 425.7, + "vs_no_plugins": 1.01 + }, + { + "plugins": "timestamps+revision", + "op": "insert", + "us_per_op": 3077.7, + "vs_no_plugins": 1.23 + } + ], + "notes": [ + "Each operation is one round trip to MongoDB, which dominates; plugin calls are in-process." + ], + "conclusion": "" +} diff --git a/bench/results/6dc837f/B8.json b/bench/results/6dc837f/B8.json new file mode 100644 index 0000000..7ab4fce --- /dev/null +++ b/bench/results/6dc837f/B8.json @@ -0,0 +1,151 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": false, + "timestamp": "2026-10-13T04:45:53+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B8", + "title": "Cost of change tracking", + "question": "What do tracked reads cost in time and memory over lean and raw reads, and what does a delta save send?", + "rows": [ + { + "backend": "msgspec", + "mode": "raw()", + "n": 5000, + "median_ms": 41.42, + "vs_lean": 0.54, + "bytes_per_doc": 5263 + }, + { + "backend": "msgspec", + "mode": "lean()", + "n": 5000, + "median_ms": 76.11, + "vs_lean": 1.0, + "bytes_per_doc": 2398 + }, + { + "backend": "msgspec", + "mode": "tracked", + "n": 5000, + "median_ms": 63.1, + "vs_lean": 0.83, + "bytes_per_doc": 3625 + }, + { + "backend": "msgspec", + "mode": "save payload: full $set", + "n": 1, + "median_ms": "", + "vs_lean": "", + "bytes_per_doc": 659 + }, + { + "backend": "msgspec", + "mode": "save payload: delta", + "n": 1, + "median_ms": "", + "vs_lean": "", + "bytes_per_doc": 38 + }, + { + "backend": "pydantic", + "mode": "raw()", + "n": 5000, + "median_ms": 41.34, + "vs_lean": 0.34, + "bytes_per_doc": 5263 + }, + { + "backend": "pydantic", + "mode": "lean()", + "n": 5000, + "median_ms": 121.05, + "vs_lean": 1.0, + "bytes_per_doc": 7103 + }, + { + "backend": "pydantic", + "mode": "tracked", + "n": 5000, + "median_ms": 126.49, + "vs_lean": 1.04, + "bytes_per_doc": 8329 + }, + { + "backend": "pydantic", + "mode": "save payload: full $set", + "n": 1, + "median_ms": "", + "vs_lean": "", + "bytes_per_doc": 659 + }, + { + "backend": "pydantic", + "mode": "save payload: delta", + "n": 1, + "median_ms": "", + "vs_lean": "", + "bytes_per_doc": 38 + }, + { + "backend": "attrs", + "mode": "raw()", + "n": 5000, + "median_ms": 48.21, + "vs_lean": 0.57, + "bytes_per_doc": 5263 + }, + { + "backend": "attrs", + "mode": "lean()", + "n": 5000, + "median_ms": 84.78, + "vs_lean": 1.0, + "bytes_per_doc": 2543 + }, + { + "backend": "attrs", + "mode": "tracked", + "n": 5000, + "median_ms": 89.97, + "vs_lean": 1.06, + "bytes_per_doc": 3768 + }, + { + "backend": "attrs", + "mode": "save payload: full $set", + "n": 1, + "median_ms": "", + "vs_lean": "", + "bytes_per_doc": 659 + }, + { + "backend": "attrs", + "mode": "save payload: delta", + "n": 1, + "median_ms": "", + "vs_lean": "", + "bytes_per_doc": 38 + } + ], + "notes": [ + "1 tracked documents remained after the run (should be ~0).", + "bytes_per_doc for reads is Python heap per document held; for save payloads it is the BSON update size." + ], + "conclusion": "" +} diff --git a/bench/results/6dc837f/LOAD.json b/bench/results/6dc837f/LOAD.json new file mode 100644 index 0000000..2465342 --- /dev/null +++ b/bench/results/6dc837f/LOAD.json @@ -0,0 +1,169 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": true, + "timestamp": "2026-10-13T04:56:11+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "LOAD", + "title": "Storefront under HTTP load", + "question": "What do mongospecs and mongospecs-web cost per request in a real server, against the same endpoints written by hand with pymongo?", + "rows": [ + { + "app": "raw pymongo + starlette", + "server_cpu_ms_per_request": 0.631, + "rps_at_saturation": 1511.5, + "server_cpu_utilisation": 0.95, + "list_p50_ms": 1.2, + "list_p95_ms": 3.63, + "detail_p50_ms": 2.11, + "detail_p95_ms": 6.65, + "order_p50_ms": 3.74, + "order_p95_ms": 10.31, + "saturated_p99_ms": 150.5, + "failed": 0, + "orders_placed": 3079, + "cpu_vs_raw": 1.0, + "rps_vs_raw": 1.0 + }, + { + "app": "mongospecs (msgspec, asgi)", + "server_cpu_ms_per_request": 0.764, + "rps_at_saturation": 1257.2, + "server_cpu_utilisation": 0.96, + "list_p50_ms": 1.34, + "list_p95_ms": 4.36, + "detail_p50_ms": 2.46, + "detail_p95_ms": 9.21, + "order_p50_ms": 4.15, + "order_p95_ms": 13.87, + "saturated_p99_ms": 182.9, + "failed": 0, + "orders_placed": 2520, + "cpu_vs_raw": 1.21, + "rps_vs_raw": 0.83 + }, + { + "app": "mongospecs (msgspec, starlette)", + "server_cpu_ms_per_request": 0.796, + "rps_at_saturation": 1211.1, + "server_cpu_utilisation": 0.96, + "list_p50_ms": 1.45, + "list_p95_ms": 5.24, + "detail_p50_ms": 2.66, + "detail_p95_ms": 10.82, + "order_p50_ms": 4.79, + "order_p95_ms": 13.75, + "saturated_p99_ms": 194.6, + "failed": 0, + "orders_placed": 2421, + "cpu_vs_raw": 1.26, + "rps_vs_raw": 0.8 + }, + { + "app": "mongospecs (msgspec, fastapi)", + "server_cpu_ms_per_request": 0.877, + "rps_at_saturation": 1109.8, + "server_cpu_utilisation": 0.97, + "list_p50_ms": 1.46, + "list_p95_ms": 3.57, + "detail_p50_ms": 2.63, + "detail_p95_ms": 6.73, + "order_p50_ms": 4.66, + "order_p95_ms": 10.29, + "saturated_p99_ms": 195.4, + "failed": 0, + "orders_placed": 2341, + "cpu_vs_raw": 1.39, + "rps_vs_raw": 0.73 + }, + { + "app": "mongospecs (msgspec, litestar)", + "server_cpu_ms_per_request": 0.785, + "rps_at_saturation": 1239.0, + "server_cpu_utilisation": 0.97, + "list_p50_ms": 1.37, + "list_p95_ms": 4.87, + "detail_p50_ms": 2.48, + "detail_p95_ms": 10.36, + "order_p50_ms": 3.95, + "order_p95_ms": 15.25, + "saturated_p99_ms": 172.6, + "failed": 0, + "orders_placed": 2493, + "cpu_vs_raw": 1.24, + "rps_vs_raw": 0.82 + }, + { + "app": "mongospecs (msgspec, quart)", + "server_cpu_ms_per_request": 0.842, + "rps_at_saturation": 1171.0, + "server_cpu_utilisation": 0.99, + "list_p50_ms": 1.64, + "list_p95_ms": 4.09, + "detail_p50_ms": 2.96, + "detail_p95_ms": 9.69, + "order_p50_ms": 5.56, + "order_p95_ms": 12.82, + "saturated_p99_ms": 168.2, + "failed": 0, + "orders_placed": 2394, + "cpu_vs_raw": 1.33, + "rps_vs_raw": 0.77 + }, + { + "app": "mongospecs (pydantic, asgi)", + "server_cpu_ms_per_request": 0.763, + "rps_at_saturation": 1288.0, + "server_cpu_utilisation": 0.98, + "list_p50_ms": 1.49, + "list_p95_ms": 3.97, + "detail_p50_ms": 2.62, + "detail_p95_ms": 7.34, + "order_p50_ms": 4.4, + "order_p95_ms": 10.56, + "saturated_p99_ms": 146.9, + "failed": 0, + "orders_placed": 2611, + "cpu_vs_raw": 1.21, + "rps_vs_raw": 0.85 + }, + { + "app": "mongospecs (attrs, asgi)", + "server_cpu_ms_per_request": 0.783, + "rps_at_saturation": 1245.2, + "server_cpu_utilisation": 0.97, + "list_p50_ms": 1.43, + "list_p95_ms": 4.15, + "detail_p50_ms": 2.53, + "detail_p95_ms": 7.71, + "order_p50_ms": 3.99, + "order_p95_ms": 11.31, + "saturated_p99_ms": 166.6, + "failed": 0, + "orders_placed": 2560, + "cpu_vs_raw": 1.24, + "rps_vs_raw": 0.82 + } + ], + "notes": [ + "Mix {'list': 60, 'detail': 30, 'order': 10}; 5000 products with 3 reviews each; one uvicorn worker. Latency columns: 2 virtual users for 5s (an unloaded server). Capacity columns: 64 virtual users for 15s.", + "server_cpu_ms_per_request is CPU time of the server process divided by requests served: it does not depend on the load generator, which runs on the same machine.", + "The stock invariant (stock + units sold = initial stock, never negative) held after every run.", + "The raw baseline implements the same three endpoints by hand, including the order transaction." + ], + "conclusion": "" +} diff --git a/bench/results/6dc837f/X1.json b/bench/results/6dc837f/X1.json new file mode 100644 index 0000000..bbb6b4d --- /dev/null +++ b/bench/results/6dc837f/X1.json @@ -0,0 +1,48 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": true, + "timestamp": "2026-10-13T04:59:26+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "X1", + "title": "Many tasks, small pool", + "question": "Do thousands of concurrent tasks finish on a pool of 5, and do pool waits surface as typed errors?", + "rows": [ + { + "check": "tasks completed", + "value": 3000 + }, + { + "check": "documents written", + "value": 3000 + }, + { + "check": "pool waits raised as PoolTimeout", + "value": 5 + }, + { + "check": "untyped errors", + "value": 0 + }, + { + "check": "verdict", + "value": "pass" + } + ], + "notes": [], + "conclusion": "PASS: 3000 tasks completed on 5 connections; starved pool raised PoolTimeout 5 times" +} diff --git a/bench/results/6dc837f/X2.json b/bench/results/6dc837f/X2.json new file mode 100644 index 0000000..a9fb108 --- /dev/null +++ b/bench/results/6dc837f/X2.json @@ -0,0 +1,44 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": true, + "timestamp": "2026-10-13T04:59:26+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "X2", + "title": "Transaction contention", + "question": "Do contended read-modify-write transactions lose updates, and are retries bounded?", + "rows": [ + { + "check": "workers", + "value": 40 + }, + { + "check": "final counter", + "value": 40 + }, + { + "check": "callback runs (incl. retries)", + "value": 214 + }, + { + "check": "verdict", + "value": "pass" + } + ], + "notes": [], + "conclusion": "PASS: counter reached 40/40 with 174 retries" +} diff --git a/bench/results/6dc837f/X3.json b/bench/results/6dc837f/X3.json new file mode 100644 index 0000000..71fb522 --- /dev/null +++ b/bench/results/6dc837f/X3.json @@ -0,0 +1,48 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": true, + "timestamp": "2026-10-13T04:59:26+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "X3", + "title": "Optimistic-lock storm", + "question": "With many writers on one revisioned document, does exactly one win per revision?", + "rows": [ + { + "check": "winners of the first round", + "value": 1 + }, + { + "check": "final n", + "value": 41 + }, + { + "check": "final rev", + "value": 41 + }, + { + "check": "conflicts retried", + "value": 332 + }, + { + "check": "verdict", + "value": "pass" + } + ], + "notes": [], + "conclusion": "PASS: 1 winner in the first round; after retries n=41, rev=41 for 40 writers" +} diff --git a/bench/results/6dc837f/X4.json b/bench/results/6dc837f/X4.json new file mode 100644 index 0000000..763dc68 --- /dev/null +++ b/bench/results/6dc837f/X4.json @@ -0,0 +1,48 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": true, + "timestamp": "2026-10-13T04:59:26+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "X4", + "title": "Cancellation", + "question": "Does cancelling tasks mid-operation and mid-transaction leak sessions, transactions or tracked state?", + "rows": [ + { + "check": "tasks cancelled", + "value": 141 + }, + { + "check": "unexpected error types", + "value": 0 + }, + { + "check": "transactions still open on the server", + "value": 0 + }, + { + "check": "tracked documents leaked", + "value": 0 + }, + { + "check": "verdict", + "value": "pass" + } + ], + "notes": [], + "conclusion": "PASS: 141 of 200 tasks cancelled; 0 open transactions afterwards" +} diff --git a/bench/results/6dc837f/X5.json b/bench/results/6dc837f/X5.json new file mode 100644 index 0000000..a0176ff --- /dev/null +++ b/bench/results/6dc837f/X5.json @@ -0,0 +1,40 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": true, + "timestamp": "2026-10-13T04:59:26+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "X5", + "title": "Result documents near 16MB", + "question": "What happens when a $lookup would build a document over the BSON limit?", + "rows": [ + { + "check": "$lookup outcome", + "value": "DocumentTooLarge" + }, + { + "check": "batch strategy loaded children", + "value": 18 + }, + { + "check": "verdict", + "value": "pass" + } + ], + "notes": [], + "conclusion": "PASS: $lookup over the limit -> DocumentTooLarge; batch strategy loaded all 18 children" +} diff --git a/bench/results/6dc837f/X6.json b/bench/results/6dc837f/X6.json new file mode 100644 index 0000000..b16a592 --- /dev/null +++ b/bench/results/6dc837f/X6.json @@ -0,0 +1,44 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": true, + "timestamp": "2026-10-13T04:59:26+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "X6", + "title": "Change stream across a primary step-down", + "question": "Does a consumer see every event exactly once across a step-down?", + "rows": [ + { + "check": "step-down", + "value": "stepped down" + }, + { + "check": "events seen", + "value": 20 + }, + { + "check": "duplicates", + "value": 0 + }, + { + "check": "verdict", + "value": "pass" + } + ], + "notes": [], + "conclusion": "PASS: 20 of 20 events seen in order across the step-down (stepped down)" +} diff --git a/bench/results/6dc837f/X7.json b/bench/results/6dc837f/X7.json new file mode 100644 index 0000000..bc2b5a8 --- /dev/null +++ b/bench/results/6dc837f/X7.json @@ -0,0 +1,44 @@ +{ + "meta": { + "git_sha": "6dc837f", + "git_dirty": true, + "timestamp": "2026-10-13T04:59:26+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "X7", + "title": "Threads with their own loops and registries", + "question": "Is any state shared across registries when each thread runs its own event loop?", + "rows": [ + { + "check": "free-threaded build", + "value": false + }, + { + "check": "threads", + "value": 8 + }, + { + "check": "errors", + "value": 0 + }, + { + "check": "verdict", + "value": "pass" + } + ], + "notes": [], + "conclusion": "PASS: 8 threads x 200 documents, each with its own loop and registry; free-threaded=False" +} diff --git a/bench/results/6dc837f/mongoose.json b/bench/results/6dc837f/mongoose.json new file mode 100644 index 0000000..9c2c555 --- /dev/null +++ b/bench/results/6dc837f/mongoose.json @@ -0,0 +1,258 @@ +{ + "meta": { + "git_sha": "6dc837f", + "timestamp": "2026-10-13T05:00:39.969Z", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "node": "v22.14.0", + "versions": { + "mongoose": "8.24.5", + "driver": "bundled with mongoose" + }, + "platform": "darwin-arm64" + }, + "workload": "E2", + "title": "Mongoose comparator", + "question": "What is Mongoose's overhead over the raw Node driver for the same workloads?", + "rows": [ + { + "workload": "B1", + "shape": "small", + "case": "driver bson->object", + "n": 10000, + "us_per_doc": 0.78, + "vs_driver": 1 + }, + { + "workload": "B1", + "shape": "small", + "case": "Model.hydrate", + "n": 10000, + "us_per_doc": 2.08, + "vs_driver": 2.67 + }, + { + "workload": "B1", + "shape": "nested", + "case": "driver bson->object", + "n": 10000, + "us_per_doc": 5.07, + "vs_driver": 1 + }, + { + "workload": "B1", + "shape": "nested", + "case": "Model.hydrate", + "n": 10000, + "us_per_doc": 23.86, + "vs_driver": 4.71 + }, + { + "workload": "B2", + "shape": "small", + "case": "driver objects", + "n": 1, + "median_ms": 0.42, + "vs_driver": 1 + }, + { + "workload": "B2", + "shape": "small", + "case": "mongoose lean()", + "n": 1, + "median_ms": 0.442, + "vs_driver": 1.05 + }, + { + "workload": "B2", + "shape": "small", + "case": "mongoose documents", + "n": 1, + "median_ms": 0.439, + "vs_driver": 1.05 + }, + { + "workload": "B2", + "shape": "small", + "case": "driver objects", + "n": 100, + "median_ms": 0.897, + "vs_driver": 1 + }, + { + "workload": "B2", + "shape": "small", + "case": "mongoose lean()", + "n": 100, + "median_ms": 0.72, + "vs_driver": 0.8 + }, + { + "workload": "B2", + "shape": "small", + "case": "mongoose documents", + "n": 100, + "median_ms": 0.756, + "vs_driver": 0.84 + }, + { + "workload": "B2", + "shape": "small", + "case": "driver objects", + "n": 10000, + "median_ms": 63.424, + "vs_driver": 1 + }, + { + "workload": "B2", + "shape": "small", + "case": "mongoose lean()", + "n": 10000, + "median_ms": 22.648, + "vs_driver": 0.36 + }, + { + "workload": "B2", + "shape": "small", + "case": "mongoose documents", + "n": 10000, + "median_ms": 53.451, + "vs_driver": 0.84 + }, + { + "workload": "B2", + "shape": "nested", + "case": "driver objects", + "n": 1, + "median_ms": 0.331, + "vs_driver": 1 + }, + { + "workload": "B2", + "shape": "nested", + "case": "mongoose lean()", + "n": 1, + "median_ms": 0.401, + "vs_driver": 1.21 + }, + { + "workload": "B2", + "shape": "nested", + "case": "mongoose documents", + "n": 1, + "median_ms": 0.588, + "vs_driver": 1.78 + }, + { + "workload": "B2", + "shape": "nested", + "case": "driver objects", + "n": 100, + "median_ms": 0.948, + "vs_driver": 1 + }, + { + "workload": "B2", + "shape": "nested", + "case": "mongoose lean()", + "n": 100, + "median_ms": 0.91, + "vs_driver": 0.96 + }, + { + "workload": "B2", + "shape": "nested", + "case": "mongoose documents", + "n": 100, + "median_ms": 3.91, + "vs_driver": 4.13 + }, + { + "workload": "B2", + "shape": "nested", + "case": "driver objects", + "n": 10000, + "median_ms": 92.983, + "vs_driver": 1 + }, + { + "workload": "B2", + "shape": "nested", + "case": "mongoose lean()", + "n": 10000, + "median_ms": 81.711, + "vs_driver": 0.88 + }, + { + "workload": "B2", + "shape": "nested", + "case": "mongoose documents", + "n": 10000, + "median_ms": 352.387, + "vs_driver": 3.79 + }, + { + "workload": "B4", + "op": "insert 1000", + "case": "driver insertMany", + "median_ms": 10.228, + "vs_driver": 1 + }, + { + "workload": "B4", + "op": "insert 1000", + "case": "Model.insertMany (no plugins)", + "median_ms": 14.665, + "vs_driver": 1.43 + }, + { + "workload": "B4", + "op": "insert 1000", + "case": "Model.insertMany (timestamps+versioning)", + "median_ms": 20.153, + "vs_driver": 1.97 + }, + { + "workload": "B4", + "op": "patch x200", + "case": "driver updateOne", + "median_ms": 703.776, + "vs_driver": 1 + }, + { + "workload": "B4", + "op": "patch x200", + "case": "findByIdAndUpdate (returns doc)", + "median_ms": 568.354, + "vs_driver": 0.81 + }, + { + "workload": "B4", + "op": "patch x200", + "case": "doc.save() delta", + "median_ms": 802.97, + "vs_driver": 1.14 + }, + { + "workload": "B8", + "mode": "lean()", + "n": 5000, + "median_ms": 36.59, + "vs_lean": 1, + "bytes_per_doc": 1910 + }, + { + "workload": "B8", + "mode": "documents (tracked)", + "n": 5000, + "median_ms": 171.12, + "vs_lean": 4.68, + "bytes_per_doc": 7168 + } + ], + "notes": [ + "Ratios are against the raw Node driver in the same process. Compare them with mongospecs ratios against pymongo, not absolute times." + ], + "conclusion": "" +} diff --git a/bench/src/mongospecs_bench/workloads.py b/bench/src/mongospecs_bench/workloads.py index e6d4c55..6c6870f 100644 --- a/bench/src/mongospecs_bench/workloads.py +++ b/bench/src/mongospecs_bench/workloads.py @@ -1,4 +1,7 @@ -"""Benchmark workloads B1-B8 (plan section 6). Each returns a :class:`Result`. +"""Benchmark workloads B1-B6 and B8 (plan section 6). Each returns a :class:`Result`. + +B7 (HTTP per plug) is the ``LOAD`` workload in :mod:`mongospecs_bench.load`: measuring plugs through +an in-process test client mostly measured the client. Everything that touches the network runs against real MongoDB. Numbers from a ``--quick`` run only prove the harness works; they are not for conclusions. @@ -11,13 +14,11 @@ import typing as t import bson -import httpx from mongospecs import Inc, Plugin, Q, meta_of, relations, tracking from mongospecs.query import _encode_cursor from mongospecs_bench.harness import Result, Settings, environment, measure, measure_sync from mongospecs_bench.models import MODELS, SHAPES, model_for, raw_document -from mongospecs_web import Api, Resource Workload = t.Callable[[Settings], t.Awaitable[Result]] @@ -476,99 +477,6 @@ async def managed_inserts() -> None: return result -# -- B7 ------------------------------------------------------------------------------- - - -def _apps(api: Api) -> dict[str, t.Any]: - from fastapi import FastAPI - from litestar import Litestar - from quart import Quart - from starlette.applications import Starlette - - from mongospecs_web.plugs import fastapi as fastapi_plug - from mongospecs_web.plugs import quart as quart_plug - from mongospecs_web.plugs import starlette as starlette_plug - from mongospecs_web.plugs.litestar import MongospecsPlugin - - fast = FastAPI() - fastapi_plug.mount(fast, api) - quart = Quart(__name__) - quart_plug.init_app(quart, api) - return { - "asgi": api.asgi(), - "starlette": Starlette(routes=starlette_plug.routes(api)), - "fastapi": fast, - "litestar": Litestar(plugins=[MongospecsPlugin(api)], openapi_config=None), - "quart": quart, - } - - -async def b7(settings: Settings) -> Result: - result = Result( - "B7", - "HTTP request to response", - "What does each plug add over calling the API core directly, per model backend?", - ) - calls = settings.scale(200, 10) - async with environment(settings) as registry: - for backend in settings.backends: - ns = MODELS[backend] - manager = registry.manager(ns.Small) - await manager.collection.delete_many({}) - await manager.collection.insert_many([raw_document("small", i) for i in range(200)]) - first = await manager.collection.find_one({}) - assert first is not None - from mongospecs_web.backend import MongoBackend - - api = Api( - Resource(ns.Small, name="small", backend=MongoBackend(ns.Small, manager)), prefix="/api" - ) - one, many = f"/api/small/{first['_id']}", "/api/small?limit=20" - - async def direct(path: str) -> None: - for _ in range(calls): - await api.handle( - "GET", path.split("?")[0], query=[("limit", "20")] if "?" in path else [] - ) - - base = { - "get one": await measure(lambda: direct(one), repeat=settings.repeat), - "list 20": await measure(lambda: direct(many), repeat=settings.repeat), - } - for case, timing in base.items(): - result.add( - backend=backend, - plug="(core, no HTTP)", - case=case, - us_per_request=round(timing.median_ms * 1000 / calls, 1), - vs_core=1.0, - ) - for plug, app in _apps(api).items(): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=app), base_url="http://bench" - ) as client: - - async def hit(path: str) -> None: - for _ in range(calls): - response = await client.get(path) - assert response.status_code == 200, response.text - - for case, path in (("get one", one), ("list 20", many)): - timing = await measure(lambda: hit(path), repeat=settings.repeat) - per = timing.median_ms * 1000 / calls - result.add( - backend=backend, - plug=plug, - case=case, - us_per_request=round(per, 1), - vs_core=_ratio(per, base[case].median_ms * 1000 / calls), - ) - result.notes.append( - "Requests go through httpx's in-process ASGI transport, so client overhead is included equally for every plug." - ) - return result - - # -- B8 ------------------------------------------------------------------------------- @@ -651,6 +559,5 @@ async def b8(settings: Settings) -> Result: "B4": b4, "B5": b5, "B6": b6, - "B7": b7, "B8": b8, } diff --git a/claudedocs/v2-plan.md b/claudedocs/v2-plan.md index 4809fd0..8c69b01 100644 --- a/claudedocs/v2-plan.md +++ b/claudedocs/v2-plan.md @@ -873,6 +873,9 @@ Recorded per §7.1 rule 4. Each one names what changed and why; none invalidates | A19 | §4.6 | Cursor pagination needs an index ending in `_id`; with an index on the sort key alone the server sorts in memory | Measured (B5): 20ms vs under 1ms for the first page of 50k documents | | A20 | §4.8 | `Resource(server_fields=...)` sets values on create outside the field policy | Needed by the reference application (the author of a review) | | A21 | §6 | The evidence track gained a production-shaped reference application ("storefront"), end-to-end tests that boot it as a real server process, and an HTTP load benchmark with a hand-written pymongo baseline (nodes E4-E6) | Requested by Jay 2026-10-13: real end-to-end, production-ready scenarios | +| A22 | §6 | Workload B7 (HTTP per plug) is the `LOAD` workload: a real server process under load, with server CPU per request as the primary metric | Measuring plugs through an in-process test client, and later through httpx over TCP, measured the client | +| A23 | §4.5 | An upsert gives the inserted document the model's non-null defaults through `$setOnInsert` | Otherwise a document created by upsert differs in shape from one created by `insert()` and queries on defaulted values miss it (Mongoose: `setDefaultsOnInsert`) | +| A24 | §4.2 | Subclass discovery ignores stale class copies | `attrs.define` replaces a class with a slotted copy and the original lingers until collected; found by the free-threaded CI job | New node: @@ -887,3 +890,12 @@ Bugs the end-to-end tests found that the unit and contract suites had not: aggre (`aggregate`, `union`, `graph`, backref counts) broke whenever a default time limit was configured; string ids assigned to reference fields were stored as strings on msgspec and attrs; unexpected server errors were not logged anywhere. + +### Decisions taken from the evidence (see `docs/benchmarks.md`, results `6dc837f`) + +| Node | Decision | Basis | +|---|---|---| +| R6 | `include(strategy="auto")` uses batched queries. `$lookup` stays available explicitly | `$lookup` never won clearly on a replica set (two runs disagreed at fan-out 100), was 2.5-3x slower when many parents share few targets, and 5-9x slower through mongos for small fan-outs | +| D1 | No Rust in 0.2. The only candidate worth a native component is a BSON-to-model decoder for bulk reads, as a separate optional package behind `Codec.decode(bytes)` | BSON-to-dict is essentially all of the driver's time on bulk reads (10.1 us per nested document against 3.3 us for msgspec's dict-to-model step) but an estimated 10-20% of server CPU on request-sized reads. Needs Jay's call on whether bulk reads matter enough | +| Open decision 4 | Reads validate; the `trusted` option is removed | Measured slower than validating on pydantic, no effect elsewhere | +| Open decision 6 | Tracked stays the default read mode | 0-15% over lean and about 1.2 KB per document after the raw-BSON snapshot change | diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 649d4df..b01f216 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1 +1,253 @@ -placeholder until results exist +# Benchmarks + +Everything here was measured against real MongoDB by the unpublished `bench/` package. Raw result +files are in `bench/results//`; the numbers below are from `6dc837f`. + +**Environment.** MongoDB 8.0.32 in Docker (single-node replica set; a two-shard cluster where +stated), Python 3.13.5, pymongo 4.18.3, msgspec 0.22, pydantic 2.14, Apple Silicon (8 cores), +client and server on the same machine. + +**How to read the numbers.** + +- Absolute times depend on the machine. The comparisons that travel are ratios against the raw + driver in the same process. +- Run-to-run variation on this setup is large: the same driver baseline moved by ±30% between runs. + Treat a ratio within about 0.8–1.25 as "no difference". Conclusions below rest only on effects + that were larger than that and repeated across runs. +- Cross-runtime absolute times (Python vs Node) are not comparable; overhead ratios are. + +Reproduce with `make up` and `uv run mongospecs-bench run`, `run LOAD`, `stress`. + +## Summary + +| Question | Answer | +|---|---| +| What does a request cost in a real server compared with hand-written pymongo? | About 1.2× the server CPU per request (0.76 ms vs 0.63 ms), 0.83× the throughput at saturation | +| What do typed reads cost over the driver's dicts? | `raw()`: nothing measurable. `lean()`: 1.1–1.7× with msgspec and attrs, about 2.6× with pydantic, on 10,000 nested documents | +| What does change tracking cost? | 0–15% over a lean read and about 1.2 KB per document; a save sends 38 bytes instead of 659 | +| How does that compare with Mongoose? | Mongoose's tracked documents cost 3.8× its driver on the same workload and 3.75× the memory of lean; mongospecs' are 1.6× (msgspec) to 2.7× (pydantic) and 1.2–1.5× the memory | +| `$lookup` or batched queries for `include()`? | Batched. `$lookup` never won clearly; on a sharded cluster it was 5–9× slower for small fan-outs | +| Offset or cursor pagination? | Cursor, with an index ending in `_id`: flat 0.6 ms at any depth against 12.6 ms for offset at the last page | +| Do plugins cost anything? | Not measurably: within noise of zero for 10 plugins | +| Is Rust warranted? | Only for one thing, and only for bulk reads; see [the Rust question](#the-rust-question) | + +## Requests in a real server + +The storefront application runs under uvicorn (one worker) and is driven over TCP with a mix of +60% product listings (filter, sort, cursor page of 20), 30% product detail with reviews and a +review count, and 10% orders (a multi-document transaction). The baseline implements the same three +endpoints by hand with pymongo and Starlette, including the transaction. + +Latency is measured with 2 virtual users (an unloaded server), capacity with 64. The primary metric +is CPU time of the server process per request, which does not depend on the load generator. + +| Server | CPU ms / request | vs raw | Requests/s at saturation | list p50 | detail p50 | order p50 | +|---|---|---|---|---|---|---| +| raw pymongo + Starlette | 0.631 | 1.00 | 1512 | 1.20 ms | 2.11 ms | 3.74 ms | +| mongospecs, msgspec, pure ASGI | 0.764 | 1.21 | 1257 | 1.34 ms | 2.46 ms | 4.15 ms | +| mongospecs, msgspec, Litestar | 0.785 | 1.24 | 1239 | 1.37 ms | 2.48 ms | 3.95 ms | +| mongospecs, msgspec, Starlette | 0.796 | 1.26 | 1211 | 1.45 ms | 2.66 ms | 4.79 ms | +| mongospecs, msgspec, Quart | 0.842 | 1.33 | 1171 | 1.64 ms | 2.96 ms | 5.56 ms | +| mongospecs, msgspec, FastAPI | 0.877 | 1.39 | 1110 | 1.46 ms | 2.63 ms | 4.66 ms | +| mongospecs, pydantic, pure ASGI | 0.763 | 1.21 | 1288 | 1.49 ms | 2.62 ms | 4.40 ms | +| mongospecs, attrs, pure ASGI | 0.783 | 1.24 | 1245 | 1.43 ms | 2.53 ms | 3.99 ms | + +No request failed in any run, and after every run the stock invariant held: for every product, +stock plus units on placed orders equals the initial stock, and no stock is negative. + +What the extra 0.13 ms buys: validated typed models, the field policy, allowlisted filters, scopes, +ETags, problem documents and OpenAPI. In a profile of a listing the driver itself is the largest +part of the request; decoding and serialising the 20 documents comes next. + +A first version of this benchmark used httpx as the load generator and reported both servers at +about 330 requests/s. That was the client's ceiling, not the servers': the server sat at 10–30% +CPU. The generator was replaced with a minimal keep-alive client and CPU per request became the +primary metric. + +## Reads + +10,000 documents fetched with `find()`, median of 15 runs, ratio to the driver returning dicts in +the same loop. "Nested" documents have a sub-document, a list of 8 sub-documents, a list and a map. + +| Mode | msgspec | pydantic | attrs | +|---|---|---|---| +| `raw()` | 1.05 | 0.93 | 0.58–1.04 | +| `lean()` | 1.46 | 2.68 | 1.06–1.71 | +| tracked (default) | 1.61 | 2.70 | 1.21–1.93 | + +`raw()` is the driver. The cost of `lean()` is the backend's own validation: msgspec converts a +nested document in 3.3 µs, attrs (through msgspec) in 8.3 µs, pydantic in 15.9 µs; the driver's +BSON-to-dict step before that takes 10.1 µs. + +## Change tracking + +5,000 nested documents: + +| Backend | lean | tracked | Heap per document, lean → tracked | +|---|---|---|---| +| msgspec | 76 ms | 63 ms | 2.4 KB → 3.6 KB | +| pydantic | 121 ms | 126 ms | 7.1 KB → 8.3 KB | +| attrs | 85 ms | 90 ms | 2.5 KB → 3.8 KB | + +A tracked document keeps the raw BSON bytes it was read from (about 1.2 KB here) and nothing else; +the comparable snapshot is built only if the document is saved. Changing one field and saving sends +a 38-byte update where a full `$set` would send 659 bytes. + +This is the result of an optimisation made during this work. The first implementation re-encoded +every loaded document to build the snapshot up front: + +| 10,000 nested documents, msgspec | Before (`4f3b2c1`) | After (`6dc837f`) | +|---|---|---| +| tracked read | 428 ms (4.3× driver) | 150 ms (1.6× driver) | +| lean read | 155 ms | 136 ms | + +The same pass replaced one-await-per-document cursor iteration with batch fetches and removed a +"trusted read" option whose pydantic fast path measured slower than validating (17 µs against 5 µs +per wide document). + +## Compared with Mongoose + +Mongoose 8 on Node 22, same MongoDB, same document shapes, ratios against the raw Node driver. + +| 10,000 nested documents | Mongoose | mongospecs (msgspec) | mongospecs (pydantic) | +|---|---|---|---| +| Lean read vs driver | 0.88 | 1.46 | 2.68 | +| Tracked documents vs driver | 3.79 | 1.61 | 2.70 | +| Tracked vs lean, time | 4.68 | 0.83–1.15 | 1.04 | +| Tracked vs lean, memory | 3.75 (1.9 KB → 7.2 KB) | 1.51 | 1.17 | +| Hydrating one nested document vs BSON decode | 4.71 | 0.33 | 1.57 | +| `insertMany` of 1,000 vs driver | 1.43 | 1.08 | 1.13 | +| …with timestamps and versioning | 1.97 | 1.26 | 1.67 | + +The two are not doing identical work: Mongoose's lean objects are untyped, while a mongospecs lean +read validates into typed models, which is why Mongoose is ahead on lean reads. On tracked +documents, the default in both, mongospecs' overhead over its driver is lower. + +## `include()`: `$lookup` or batched queries + +Parents each referencing `fan_out` distinct children; ratio of `$lookup` time to batched time +(below 1 means `$lookup` is faster). Two full runs on the replica set are shown because they +disagree. + +| Relation | Replica set, run 1 | Replica set, run 2 | Sharded cluster | +|---|---|---|---| +| list, fan-out 1 | 1.27 | 1.09 | 4.70 | +| list, fan-out 10 | 1.29 | 1.10 | 4.95 | +| list, fan-out 100 | 0.61 | 1.25 | 1.35 | +| list, fan-out 1000 | 0.78 | 0.96 | 0.90 | +| to-one, 2,000 parents sharing 20 targets | 3.16 | 2.48 | 9.02 | + +**Decision: `strategy="auto"` means batched queries.** `$lookup` is never clearly ahead, is 2.5–3× +slower when many parents share few targets (the server repeats each target per parent), and is +5–9× slower through mongos for small fan-outs. It stays available as `strategy="lookup"` for the +case where a single round trip matters more than time. This is also what Mongoose does. + +A `$lookup` whose result document would exceed 16 MB fails with a typed `DocumentTooLarge`; the +batched strategy loads the same data without trouble (stress scenario X5). + +## Pagination + +50,000 documents, pages of 20, sorted on an indexed field. + +| Page | Offset | Cursor | +|---|---|---| +| 1 | 0.58 ms | 0.58 ms | +| 100 | 1.20 ms | 0.65 ms | +| 1,250 | 7.06 ms | 0.66 ms | +| 2,499 (last) | 12.62 ms | 0.67 ms | + +- Cursor pagination sorts by the requested key and then by `_id`. With an index on the key alone + the first page took 28.8 ms, because the server sorted in memory; with an index on + `(key, _id)` it took 0.58 ms. Index listings accordingly. +- A total from a concurrent `count_documents` took 6.0 ms; the same page and total through `$facet` + took 28.5 ms and is additionally bound by the 16 MB document limit. `$facet` is not used. + +## Writes and plugins + +Ratios to the driver; these are single round trips where the network dominates and noise is +highest. + +| Operation | msgspec | pydantic | attrs | +|---|---|---|---| +| `insert_many` of 1,000 | 1.08 | 1.13 | (noisy) | +| …with timestamps and revision | 1.26 | 1.67 | (noisy) | +| `Model.update` ×200 (returns the document) | 1.11 | 1.29 | 1.20 | +| `doc.save()` delta ×200 | 1.54 | 1.06 | 0.91 | +| `Model.upsert` ×200 (returns the document) | 1.32 | 1.12 | 1.11 | + +Per-operation cost with 0, 3 and 10 no-op plugins on a registry differed by less than the noise +(0.94–1.20× with no trend), as did the three built-in plugins. + +## Stress + +All scenarios pass at full size. + +| Scenario | Result | +|---|---| +| X1: 3,000 concurrent tasks on a pool of 5 | all completed; a starved pool raised `PoolTimeout` | +| X2: 40 contending read-modify-write transactions | counter reached 40; 174 automatic retries, no lost update | +| X3: 40 writers on one revisioned document | exactly one winner per revision; final revision 41 | +| X4: 141 of 200 tasks cancelled mid-operation or mid-transaction | no open transactions left on the server, no leaked tracking state | +| X5: `$lookup` result over 16 MB | typed `DocumentTooLarge`; batched strategy loaded everything | +| X6: change stream across a primary step-down | 20 of 20 events, in order, no duplicates | +| X7: 8 threads, each with its own event loop and registry | no cross-registry state | + +X7 ran on a standard (GIL) build. The unit suite runs on the free-threaded build in CI as an +experimental job. + +## End-to-end + +`bench/tests/test_e2e_storefront.py` boots the storefront under uvicorn and talks to it over HTTP: + +- a full customer journey on each of the five plugs, and ordering on each model backend +- 40 concurrent buyers for 10 units: exactly 10 orders succeed, 30 are refused, stock ends at 0 +- multi-item orders are all-or-nothing, including opposing lock orders that must not deadlock +- the change-stream projector counts every sale exactly once across a restart and a period with no + projector running +- requests in flight finish when the server is asked to stop +- under traffic, a primary step-down produces only declared errors (503/504), and afterwards every + accepted order has moved stock exactly once + +A separate check (`mongospecs-bench trace`) sends requests through the Sentry SDK to a local +[rustrak](https://github.com/rustrak/rustrak) server and asserts that each request arrives as one +trace containing the HTTP transaction and its `Model.op` database spans. + +These tests found defects the unit and contract suites had not: aggregation paths failed whenever a +default time limit was configured, ids given as strings to reference fields were stored as strings +on msgspec and attrs, and unexpected server errors were not logged. + +## The Rust question + +The plan asked whether a Rust component is justified, to be decided from these measurements. + +Where the time goes when reading 10,000 nested documents with msgspec models: + +| Step | Per document | +|---|---| +| Driver: BSON bytes to Python dict | 10.1 µs | +| msgspec: dict to model | 3.3 µs | +| Everything else in mongospecs | under 1 µs | + +- For **bulk reads**, decoding BSON into dicts is essentially all of the driver's time: measured + alone it takes 101 ms for 10,000 documents, and the driver's whole read took 93 ms in the same + run (equal within noise). It is about two thirds of a lean read. Decoding BSON + straight into models, skipping the dict, is the one change that could make mongospecs faster than + the raw driver rather than slower. That is a native-code job. +- For **request-sized reads** (20 documents) that step is an estimated 10–20% of the server CPU + per request (20 documents at 3–7 µs against 0.63 ms); the driver's asynchronous machinery + dominates. A native decoder would gain little there. +- Nothing else measured (query building, plugins, tracking, serialisation) is large enough to move + to Rust. + +**Recommendation:** no Rust in 0.2. If bulk reads matter to you, the worthwhile project is a +BSON-to-model decoder as a separate optional package behind `Codec.decode(bytes)`. The seam is +already there: tracked reads receive undecoded BSON from the driver today. A Rust driver is not +indicated by anything here. + +## Limits of these measurements + +- One machine, client and server together, MongoDB in Docker on macOS. Network cost is unrealistically + low and variance is high. +- One uvicorn worker; no TLS, no proxy. +- The free-threaded build has not been benchmarked. +- The Mongoose comparison uses default settings on both sides and no tuning of either. diff --git a/packages/mongospecs/src/mongospecs/meta.py b/packages/mongospecs/src/mongospecs/meta.py index 346d747..c52ab9a 100644 --- a/packages/mongospecs/src/mongospecs/meta.py +++ b/packages/mongospecs/src/mongospecs/meta.py @@ -69,11 +69,29 @@ def resolve_config(cls: type) -> SpecConfig: return own -def _all_subclasses(cls: type) -> t.Iterator[type]: - sub: type - for sub in cls.__subclasses__(): - yield sub - yield from _all_subclasses(sub) +def _all_subclasses(cls: type) -> list[type]: + """Every live subclass of ``cls``, one per definition. + + ``attrs.define`` replaces a class with a slotted copy; the original lingers in + ``__subclasses__()`` until it is garbage collected (noticeably longer on + free-threaded builds). Both have the same parent, module and qualified name, so + the later one, which is the real class, wins. + """ + out: list[type] = [] + + def walk(klass: type) -> None: + # Deduplicate among the direct subclasses of one parent only: unrelated classes may + # legitimately share a qualified name (built by the same factory function, say). + level: dict[tuple[str, str], type] = {} + sub: type + for sub in klass.__subclasses__(): + level[(sub.__module__, sub.__qualname__)] = sub + for sub in level.values(): + out.append(sub) + walk(sub) + + walk(cls) + return out class ModelMeta: @@ -280,7 +298,9 @@ def discriminator_filter(self) -> dict[str, t.Any]: """Restrict a query on a subclass to documents of that subclass (and its own subclasses).""" if not self.disc_key or self.model is self.root: return {} - values = [self.disc_value, *(meta_of(c).disc_value for c in _all_subclasses(self.model))] + values = list( + dict.fromkeys([self.disc_value, *(meta_of(c).disc_value for c in _all_subclasses(self.model))]) + ) return {self.disc_key: values[0] if len(values) == 1 else {"$in": values}} # -- relations ------------------------------------------------------------- diff --git a/packages/mongospecs/tests/test_models_misc.py b/packages/mongospecs/tests/test_models_misc.py index ae771e4..dff0ff6 100644 --- a/packages/mongospecs/tests/test_models_misc.py +++ b/packages/mongospecs/tests/test_models_misc.py @@ -168,3 +168,42 @@ class Orphan(MsgspecSpec): with pytest.raises(ConfigurationError, match="DoesNotExist"): meta_of(Orphan).ref_targets("other") assert ObjectId.is_valid(str(ObjectId())) + + +def test_stale_class_copies_are_not_counted_as_subclasses(): + """attrs replaces a class with a slotted copy; the original must not show up as a second subclass.""" + import gc + + import attrs + + from mongospecs.attrs import Spec as AttrsSpec + from mongospecs.meta import _all_subclasses + + @attrs.define(kw_only=True) + class Vehicle(AttrsSpec): + __spec__ = SpecConfig(collection="vehicles", discriminator="kind") + wheels: int = 4 + + gc.disable() # keep the pre-slots originals alive, as a free-threaded build may + try: + + @attrs.define(kw_only=True) + class Truck(Vehicle): + __spec__ = SpecConfig(discriminator_value="lorry") + load: int = 0 + + @attrs.define(kw_only=True) + class Pickup(Truck): + bed: int = 0 + + assert [c.__name__ for c in _all_subclasses(Vehicle)] == ["Truck", "Pickup"] + assert all( + attrs.has(c) and "load" in [a.name for a in attrs.fields(c)] for c in _all_subclasses(Vehicle) + ) + assert meta_of(Truck).discriminator_filter() == {"kind": {"$in": ["lorry", "Pickup"]}} + assert ( + meta_of(Vehicle).subclass_for("lorry") is Truck + and meta_of(Vehicle).subclass_for("Pickup") is Pickup + ) + finally: + gc.enable() From d1544af68a3bb4c93f1ec506fea9f346dc14cc91 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 01:06:16 -0400 Subject: [PATCH 05/26] test(core): consumer typing fixtures checked with mypy A fixture file written from a user's point of view; mypy --strict must reject exactly the lines marked expect-error (wrong model type, a possibly-None result used as a document, a count used as a document, a reference's target type) and accept the rest. Co-Authored-By: Claude Opus 5.5 --- packages/mongospecs/tests/test_typing.py | 29 +++++++ .../tests/typing_fixtures/consumer.py | 75 +++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 packages/mongospecs/tests/test_typing.py create mode 100644 packages/mongospecs/tests/typing_fixtures/consumer.py diff --git a/packages/mongospecs/tests/test_typing.py b/packages/mongospecs/tests/test_typing.py new file mode 100644 index 0000000..13a7adc --- /dev/null +++ b/packages/mongospecs/tests/test_typing.py @@ -0,0 +1,29 @@ +"""Consumer typing fixtures: mypy must reject exactly the lines marked ``# expect-error``.""" + +import pathlib +import re +import subprocess +import sys + +FIXTURE = pathlib.Path(__file__).parent / "typing_fixtures" / "consumer.py" +ROOT = pathlib.Path(__file__).resolve().parents[3] + + +def test_public_api_types_hold_for_a_consumer(): + expected = { + number + for number, line in enumerate(FIXTURE.read_text().splitlines(), start=1) + if line.rstrip().endswith("# expect-error") + } + assert len(expected) >= 8 + run = subprocess.run( + [sys.executable, "-m", "mypy", "--strict", "--no-error-summary", "--no-incremental", str(FIXTURE)], + capture_output=True, + text=True, + cwd=ROOT, + ) + reported = {int(m.group(1)) for m in re.finditer(r"consumer\.py:(\d+): error", run.stdout)} + assert reported == expected, ( + f"unexpected errors on lines {sorted(reported - expected)}, " + f"missing errors on lines {sorted(expected - reported)}\n{run.stdout}{run.stderr}" + ) diff --git a/packages/mongospecs/tests/typing_fixtures/consumer.py b/packages/mongospecs/tests/typing_fixtures/consumer.py new file mode 100644 index 0000000..3d59d1d --- /dev/null +++ b/packages/mongospecs/tests/typing_fixtures/consumer.py @@ -0,0 +1,75 @@ +"""What a user of the library should and should not be able to write, as seen by a type checker. + +Lines marked ``# expect-error`` must be rejected by mypy; every other line must be accepted. +This file is checked, never executed. +""" + +from mongospecs import Inc, Page, Q, Query, Ref, Registry, SpecConfig +from mongospecs.msgspec import Spec + + +class User(Spec): + name: str = "" + + +class Article(Spec): + __spec__ = SpecConfig(collection="articles") + title: str = "" + views: int = 0 + author: Ref[User] | None = None + + +async def reads(registry: Registry) -> None: + query: Query[Article] = Article.find(Q.views > 1).sort("-views").limit(5) + articles: list[Article] = await query.all() + also: list[Article] = await query + maybe: Article | None = await query.first() + one: Article = await query.one() + by_id: Article | None = await Article.get("abc") + raw: list[dict[str, object]] = [dict(d) for d in await query.raw().all()] + page: Page[Article] = await query.page(10) + count: int = await Article.count(Q.title == "x") + async for article in query: + title: str = article.title + print(articles, also, maybe, one, by_id, raw, page.items[0].views, count, title) + + wrong_model: list[User] = await query.all() # expect-error + not_optional: Article = await query.first() # expect-error + not_a_list: Article = await query.all() # expect-error + print(wrong_model, not_optional, not_a_list) + if maybe is not None: + print(maybe.title.upper()) + print(maybe.title) # expect-error + + +async def writes(registry: Registry) -> None: + article: Article = await Article(title="t").insert() + saved: Article = await article.save() + updated: Article = await Article.update(article.id, Inc(views=1), title="x") + upserted: Article = await Article.upsert(Q.title == "x", views=2) + modified: int = await Article.find(Q.views > 1).update(Inc(views=1)) + deleted: int = await Article.find(Q.views > 1).delete() + await article.delete() + print(saved, updated, upserted, modified, deleted) + + as_user: User = await Article.update(article.id, title="x") # expect-error + count_as_doc: Article = await Article.find().update(views=1) # expect-error + print(as_user, count_as_doc) + + +async def relations(article: Article) -> None: + if article.author is not None: + user: User = await article.author.fetch() + loaded: User = article.author.doc + print(user.name, loaded.name) + wrong: Article = article.author.doc # expect-error + print(wrong) + + +async def transactions(registry: Registry) -> None: + async def work(tx: object) -> int: + return 1 + + result: int = await registry.transaction(work) + text: str = await registry.transaction(work) # expect-error + print(result, text) From 279bd6c508e5b6b63d2da973b167a52f122a71c9 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 01:13:58 -0400 Subject: [PATCH 06/26] test(web): compare timestamps as instants, not strings A timestamp with a zero fractional second is printed without one by some serialisers, which made a lexical comparison fail intermittently. Co-Authored-By: Claude Opus 5.5 --- packages/mongospecs-web/tests/test_mongo_resource.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/mongospecs-web/tests/test_mongo_resource.py b/packages/mongospecs-web/tests/test_mongo_resource.py index 07c4bab..80a9a37 100644 --- a/packages/mongospecs-web/tests/test_mongo_resource.py +++ b/packages/mongospecs-web/tests/test_mongo_resource.py @@ -129,7 +129,12 @@ async def test_crud_round_trip_with_plugins(app): # conditional update through the real revision plugin ok = await client.patch(f"/v1/stories/{story['id']}", json={"words": 900}, headers={"if-match": '"0"'}) assert ok.status_code == 200 and ok.headers["etag"] == '"1"' and ok.json()["data"]["reading_minutes"] == 4 - assert ok.json()["data"]["updated_at"] >= story["updated_at"] + + # compare as instants: serialisers may or may not print a zero fractional second + def instant(text): + return dt.datetime.fromisoformat(text.replace("Z", "+00:00")) + + assert instant(ok.json()["data"]["updated_at"]) >= instant(story["updated_at"]) stale = await client.patch(f"/v1/stories/{story['id']}", json={"words": 1}, headers={"if-match": '"0"'}) assert stale.status_code == 412 # unique index -> 409 with the field named From 426c0b1afd85191e327bd40fc7294b1647e4f120 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 02:23:10 -0400 Subject: [PATCH 07/26] fix(core,web): defects found in review of the v2 pull requests Each has a regression test that failed before the fix. - cursor pagination with select() repeated documents and did not terminate: sort keys were left out of the projection, so the next cursor was built from defaults. Sort keys are now always fetched - bulk_save left memory out of sync after a partial failure. Writes in the batch are independent; the ones that succeeded are now brought in sync and the error lists the rest in .failed. Which updates applied is determined by reading the documents back, since the server only reports a total - bulk_write hard deletes skipped OnDelete cascade rules - a cycle of cascading references recursed forever; cascades now carry the set of documents already being deleted - cascades resolved ids with distinct() (16MB limit) and one unbounded $in; ids are now read with find and applied in chunks - saving an untracked instance that has an id (a copy, a deepcopy, a lean read) raised a bare duplicate-key error; it now says the instance is not tracked and what to do - reload() loaded Deferred fields - Query.update() returns an int that also carries .matched - the Location header of a create was a bare id; it is now a URL under the API prefix Co-Authored-By: Claude Opus 5.5 --- .../mongospecs-web/src/mongospecs_web/api.py | 8 +- .../mongospecs-web/tests/test_contract.py | 2 +- .../tests/test_openapi_views.py | 5 + .../mongospecs/src/mongospecs/__init__.py | 3 +- packages/mongospecs/src/mongospecs/bulk.py | 14 +- packages/mongospecs/src/mongospecs/errors.py | 4 + packages/mongospecs/src/mongospecs/manager.py | 234 +++++++++++++++--- packages/mongospecs/src/mongospecs/query.py | 12 +- .../mongospecs/src/mongospecs/relations.py | 36 +-- .../mongospecs/tests/test_review_fixes.py | 149 +++++++++++ 10 files changed, 404 insertions(+), 63 deletions(-) create mode 100644 packages/mongospecs/tests/test_review_fixes.py diff --git a/packages/mongospecs-web/src/mongospecs_web/api.py b/packages/mongospecs-web/src/mongospecs_web/api.py index 443ebe6..0e4aa5a 100644 --- a/packages/mongospecs-web/src/mongospecs_web/api.py +++ b/packages/mongospecs-web/src/mongospecs_web/api.py @@ -120,7 +120,13 @@ async def run(self, route: Route, request: Request) -> Response: if response.status >= 500: logging.getLogger("mongospecs_web").error("authentication failed unexpectedly", exc_info=exc) return response - return await route.resource.dispatch(route.operation, request) + response = await route.resource.dispatch(route.operation, request) + location = response.headers.get("location") + if location and not location.startswith(("/", "http://", "https://")): + # A resource names what it created by id; the URL of that thing depends on where + # the API is mounted, which only the Api knows. + response.headers["location"] = f"{self.prefix}/{route.resource.name}/{location}" + return response async def handle( self, diff --git a/packages/mongospecs-web/tests/test_contract.py b/packages/mongospecs-web/tests/test_contract.py index d223d1b..7c85693 100644 --- a/packages/mongospecs-web/tests/test_contract.py +++ b/packages/mongospecs-web/tests/test_contract.py @@ -148,7 +148,7 @@ def names(label): return [w["name"] for w in got[label]["body"]["data"]] created = got["create"] - assert created["headers"]["location"] == f"{1:024x}" and created["headers"]["etag"] == '"0"' + assert created["headers"]["location"] == f"/api/widgets/{1:024x}" and created["headers"]["etag"] == '"0"' assert created["body"]["data"] == { "id": f"{1:024x}", "name": "anvil", diff --git a/packages/mongospecs-web/tests/test_openapi_views.py b/packages/mongospecs-web/tests/test_openapi_views.py index 0fc2fb2..9fc997d 100644 --- a/packages/mongospecs-web/tests/test_openapi_views.py +++ b/packages/mongospecs-web/tests/test_openapi_views.py @@ -151,6 +151,11 @@ async def test_api_handle_without_a_server(): "POST", "/api/widgets", body=b'{"name":"direct"}', headers={"authorization": "Bearer ann"} ) assert created.status == 201 and b'"direct"' in created.body + # Location is a URL that resolves to the new resource, wherever the API is mounted + assert created.headers["location"].startswith("/api/widgets/") + assert ( + await api.handle("GET", created.headers["location"], headers={"authorization": "Bearer ann"}) + ).status == 200 assert (await api.handle("GET", "/api/widgets/")).status == 200 # trailing slash tolerated assert (await api.handle("GET", "/api/widgets/stats")).status == 200 # literal segment beats {id} assert (await api.handle("GET", "/api/nope")).status == 404 diff --git a/packages/mongospecs/src/mongospecs/__init__.py b/packages/mongospecs/src/mongospecs/__init__.py index 51d8030..1bb440c 100644 --- a/packages/mongospecs/src/mongospecs/__init__.py +++ b/packages/mongospecs/src/mongospecs/__init__.py @@ -47,7 +47,7 @@ ) from mongospecs.hooks import OpCtx, Plugin from mongospecs.indexes import Action, IndexAction, IndexPlan, apply_indexes, plan_indexes -from mongospecs.manager import BulkResult, Manager +from mongospecs.manager import BulkResult, Manager, UpdateCount from mongospecs.markers import ( Alias, Backref, @@ -169,6 +169,7 @@ "Tx", "Type", "Unset", + "UpdateCount", "UpdateMany", "UpdateOne", "UpdatedAt", diff --git a/packages/mongospecs/src/mongospecs/bulk.py b/packages/mongospecs/src/mongospecs/bulk.py index d27635e..6845d97 100644 --- a/packages/mongospecs/src/mongospecs/bulk.py +++ b/packages/mongospecs/src/mongospecs/bulk.py @@ -58,9 +58,16 @@ class DeleteMany(DeleteOne): __slots__ = () -def compile_bulk(manager: Manager[t.Any], ctx: OpCtx, operations: t.Iterable[t.Any]) -> list[t.Any]: - """Compile typed bulk operations to driver requests.""" +def compile_bulk( + manager: Manager[t.Any], ctx: OpCtx, operations: t.Iterable[t.Any] +) -> tuple[list[t.Any], dict[int, tuple[dict[str, t.Any], bool]]]: + """Compile typed bulk operations to driver requests. + + Also returns the hard deletes as ``{request index: (filter, many)}`` so the caller + can run cascade rules for them. + """ requests: list[t.Any] = [] + deletes: dict[int, tuple[dict[str, t.Any], bool]] = {} for op in operations: if isinstance(op, InsertOne): doc = manager.meta.decode(dict(op.doc), stored=False) if isinstance(op.doc, t.Mapping) else op.doc @@ -84,10 +91,11 @@ def compile_bulk(manager: Manager[t.Any], ctx: OpCtx, operations: t.Iterable[t.A if soft is not None: requests.append((pymongo.UpdateMany if op.many else pymongo.UpdateOne)(flt, soft)) else: + deletes[len(requests)] = (flt, op.many) requests.append((pymongo.DeleteMany if op.many else pymongo.DeleteOne)(flt)) else: raise QueryError( f"bulk_write takes mongospecs.bulk operations, got {type(op).__name__}; " "use Model.collection().bulk_write for raw driver operations" ) - return requests + return requests, deletes diff --git a/packages/mongospecs/src/mongospecs/errors.py b/packages/mongospecs/src/mongospecs/errors.py index 9716f97..76191b2 100644 --- a/packages/mongospecs/src/mongospecs/errors.py +++ b/packages/mongospecs/src/mongospecs/errors.py @@ -38,6 +38,10 @@ class MongoSpecsError(Exception): """Base class for every error raised by mongospecs.""" + failed: t.Sequence[t.Any] = () + """For batch operations that partly succeeded: the documents that were *not* written. + Every other document in the batch was written and is in sync in memory.""" + class ConfigurationError(MongoSpecsError): """A model, plugin or registry is declared incorrectly.""" diff --git a/packages/mongospecs/src/mongospecs/manager.py b/packages/mongospecs/src/mongospecs/manager.py index cbb7591..dc8c0c2 100644 --- a/packages/mongospecs/src/mongospecs/manager.py +++ b/packages/mongospecs/src/mongospecs/manager.py @@ -19,6 +19,7 @@ from mongospecs.errors import ( DocumentNotFound, DuplicateKeyError, + MongoSpecsError, OperationFailed, QueryError, StaleDocumentError, @@ -35,7 +36,7 @@ from mongospecs.query import Query -__all__ = ["BulkResult", "Manager"] +__all__ = ["BulkResult", "Manager", "UpdateCount"] T = t.TypeVar("T") @@ -64,6 +65,26 @@ def __repr__(self) -> str: return f"BulkResult(inserted={len(self.inserted)}, errors={len(self.errors)})" +class UpdateCount(int): + """The number of documents a filter-based update modified, with the match count alongside. + + It is an ``int`` (the modified count), so existing arithmetic and comparisons work; + ``.matched`` also counts documents that matched but already had the new values. + """ + + matched: int + modified: int + + def __new__(cls, matched: int, modified: int) -> UpdateCount: + self = super().__new__(cls, modified) + self.matched = matched + self.modified = modified + return self + + def __repr__(self) -> str: + return f"UpdateCount(matched={self.matched}, modified={self.modified})" + + class Manager(t.Generic[T]): def __init__( self, @@ -288,7 +309,21 @@ async def save(self, doc: T, *, session: t.Any = None) -> T: """Insert a new document, or send only what changed on a tracked one.""" state = tracking.state_of(doc) if state is None: - return await self.insert(doc, session=session) + try: + return await self.insert(doc, session=session) + except DuplicateKeyError as exc: + if "_id" not in exc.fields: + raise + # An id that already exists on a document we hold no snapshot for: it came from + # lean(), copy/deepcopy, or pickling, not from a tracked read. + raise DuplicateKeyError( + f"{self.model.__name__} {t.cast(t.Any, doc).id!r} already exists and this instance is " + "not tracked (it is a lean read, a copy, or was unpickled), so save() tried to insert " + "it. Load it with a tracked query, or adopt a raw document with Model.hydrate().", + code=exc.code, + details=exc.details, + fields=exc.fields, + ) from exc ctx = self.ctx("update", session, doc=doc, snapshot=state.snapshot) await self._hooks("before", ("save",), ctx, doc) current, update = self._delta(doc, state) @@ -338,7 +373,7 @@ async def run(s: t.Any) -> None: result = await self.collection.update_one(flt, soft, **self._kw(s)) matched = result.matched_count else: - await self._cascade([snapshot["_id"]], s) + await self._cascade([snapshot["_id"]], s, set()) with self._op("delete"): matched = (await self.collection.delete_one(flt, **self._kw(s))).deleted_count if matched == 0: @@ -370,18 +405,24 @@ def _soft_delete(self, ctx: OpCtx, flt: dict[str, t.Any]) -> dict[str, t.Any] | return None async def reload(self, doc: T, *, session: t.Any = None) -> T: - """Refresh a document in place from the database and reset its snapshot.""" + """Refresh a document in place from the database and reset its snapshot. + + ``Deferred`` fields stay unloaded, as they would be on a fresh read. + """ any_doc = t.cast(t.Any, doc) with self._op("reload"): raw = await self.collection.find_one( - {"_id": self.meta.cast("id", any_doc.id)}, **self._kw(session) + {"_id": self.meta.cast("id", any_doc.id)}, self.default_projection(), **self._kw(session) ) if raw is None: raise DocumentNotFound(f"{self.model.__name__} {any_doc.id!r} does not exist") fresh = self.meta.decode(raw) - for name in meta_of(type(doc)).fields: - setattr(doc, name, getattr(fresh, name)) - tracking.track(doc, meta_of(type(doc)).encode(doc), self) + deferred = set(self.meta.deferred) + meta = meta_of(type(doc)) + for name, info in meta.fields.items(): + if info.stored not in deferred: + setattr(doc, name, getattr(fresh, name)) + tracking.track(doc, meta.encode(doc), self, self._default_partial()) return doc # -- filter-based writes --------------------------------------------------------- @@ -472,13 +513,13 @@ async def update_many( *, session: t.Any = None, unscoped: t.Collection[str] | bool = False, - ) -> int: + ) -> UpdateCount: ctx = self.ctx("update", session, many=True) flt = self.scoped(ctx, flt, unscoped) update = self._update_doc(ctx, flt, ops, fields) with self._op("update_many"): result = await self.collection.update_many(flt, update, **self._kw(session)) - return int(result.modified_count) + return UpdateCount(int(result.matched_count), int(result.modified_count)) async def delete_many( self, @@ -487,6 +528,7 @@ async def delete_many( session: t.Any = None, hard: bool = False, unscoped: t.Collection[str] | bool = False, + _seen: set[t.Any] | None = None, ) -> int: ctx = self.ctx("delete", session, many=True) flt = self.scoped(ctx, flt, unscoped) @@ -499,18 +541,28 @@ async def delete_many( return int((await self.collection.delete_many(flt, **self._kw(session))).deleted_count) async def run(s: t.Any) -> int: - with self._op("delete_many"): - ids = await self.collection.distinct("_id", flt, **self._kw(s)) + seen = _seen if _seen is not None else set() + # Documents already being deleted further up a cascade chain are left to that + # caller; this is what makes cycles of cascading references terminate. + ids = [i for i in await self._ids(flt, s) if (self.meta.collection, _hashable(i)) not in seen] if not ids: return 0 - await self._cascade(ids, s) - with self._op("delete_many"): - return int( - (await self.collection.delete_many({"_id": {"$in": ids}}, **self._kw(s))).deleted_count - ) + await self._cascade(ids, s, seen) + deleted = 0 + for chunk in _chunks(ids): + with self._op("delete_many"): + result = await self.collection.delete_many({"_id": {"$in": chunk}}, **self._kw(s)) + deleted += int(result.deleted_count) + return deleted return await (self.registry.transaction(run) if session is None else run(session)) + async def _ids(self, flt: dict[str, t.Any], session: t.Any, limit: int = 0) -> list[t.Any]: + """Ids of the documents matching an already-scoped filter.""" + with self._op("ids"): + cursor = self.collection.find(flt, {"_id": 1}, limit=limit, **self._kw(session)) + return [raw["_id"] for raw in await cursor.to_list()] + # -- cascades ---------------------------------------------------------------------- def _cascade_rules(self) -> list[t.Any]: @@ -520,12 +572,14 @@ def _cascade_rules(self) -> list[t.Any]: self._cascades = cascade_rules(self) return self._cascades - async def _cascade(self, ids: list[t.Any], session: t.Any) -> None: + async def _cascade(self, ids: list[t.Any], session: t.Any, seen: set[t.Any] | None = None) -> None: rules = self._cascade_rules() if rules: from mongospecs.relations import apply_cascades - await apply_cascades(self, rules, ids, session) + seen = seen if seen is not None else set() + seen.update((self.meta.collection, _hashable(i)) for i in ids) + await apply_cascades(self, rules, ids, session, seen) # -- bulk ----------------------------------------------------------------------------- @@ -584,18 +638,38 @@ async def insert_many( async def bulk_write( self, operations: t.Iterable[t.Any], *, ordered: bool = True, session: t.Any = None ) -> t.Any: - """Run typed bulk operations (see :mod:`mongospecs.bulk`) in one round trip.""" + """Run typed bulk operations (see :mod:`mongospecs.bulk`) in one round trip. + + Hard deletes on a model with ``OnDelete`` rules run their cascades first, in the + same transaction, exactly as single deletes do. + """ + import pymongo + from mongospecs.bulk import compile_bulk ops = list(operations) ctx = self.ctx("bulk", session, many=True) for plugin in self.plugins: await plugin.before_bulk(ctx, ops) - requests = compile_bulk(self, ctx, ops) + requests, deletes = compile_bulk(self, ctx, ops) if not requests: return None - with self._op("bulk_write"): - result = await self.collection.bulk_write(requests, ordered=ordered, **self._kw(session)) + + async def run(s: t.Any) -> t.Any: + seen: set[t.Any] = set() + for index, (flt, many) in deletes.items(): + # Resolve the documents first so the cascade and the delete act on the same ones. + ids = await self._ids(flt, s, limit=0 if many else 1) + await self._cascade(ids, s, seen) + requests[index] = pymongo.DeleteMany({"_id": {"$in": ids}}) + with self._op("bulk_write"): + return await self.collection.bulk_write(requests, ordered=ordered, **self._kw(s)) + + if deletes and self._cascade_rules(): + result = await (self.registry.transaction(run) if session is None else run(session)) + else: + deletes = {} + result = await run(session) for plugin in self.plugins: await plugin.after_bulk(ctx, ops) return result @@ -603,15 +677,18 @@ async def bulk_write( async def bulk_save(self, docs: t.Iterable[T], *, session: t.Any = None) -> int: """Save many documents in one round trip: inserts for new ones, deltas for tracked ones. - Returns the number of documents written. Raises :class:`StaleDocumentError` - if any revision-checked update matched nothing (the batch does not say which). + Returns the number of documents written. Writes are independent: if some + fail (a duplicate key, a document that no longer exists, a revision that + moved on), the rest are still written and brought in sync in memory, and + the error raised afterwards lists the ones that were not in ``.failed``. """ from pymongo import InsertOne from pymongo import UpdateOne as DriverUpdateOne requests: list[t.Any] = [] + # (document, kind, ctx, encoding after the write, prior state, the update that was sent) pending: list[ - tuple[t.Any, str, OpCtx, dict[str, t.Any], tracking.DocState | None, frozenset[str]] + tuple[t.Any, str, OpCtx, dict[str, t.Any], tracking.DocState | None, dict[str, t.Any]] ] = [] for doc in docs: state = tracking.state_of(doc) @@ -623,7 +700,7 @@ async def bulk_save(self, docs: t.Iterable[T], *, session: t.Any = None) -> int: for plugin in self.plugins: plugin.on_insert(ctx, enc) requests.append(InsertOne(enc)) - pending.append((doc, "insert", ctx, enc, None, frozenset())) + pending.append((doc, "insert", ctx, enc, None, {})) continue ctx = self.ctx("update", session, many=True, doc=doc, snapshot=state.snapshot) current, update = self._delta(doc, state) @@ -635,22 +712,42 @@ async def bulk_save(self, docs: t.Iterable[T], *, session: t.Any = None) -> int: plugin.on_update(ctx, flt, update) requests.append(DriverUpdateOne(flt, update)) _patch(current, update, skip=user_keys) - pending.append((doc, "update", ctx, current, state, user_keys)) + pending.append((doc, "update", ctx, current, state, update)) if not requests: return 0 bulk_ctx = self.ctx("bulk", session, many=True) all_docs = [p[0] for p in pending] for plugin in self.plugins: await plugin.before_bulk(bulk_ctx, all_docs) - with self._op("bulk_save"): - result = await self.collection.bulk_write(requests, ordered=False, **self._kw(session)) - updates = sum(1 for p in pending if p[1] == "update") - if result.matched_count != updates: - raise StaleDocumentError( - f"{self.model.__name__}: {updates - result.matched_count} of {updates} documents in the batch " - "no longer exist or were changed since they were loaded" - ) - for doc, kind, ctx, enc, state, _ in pending: + + failed: set[int] = set() + error: MongoSpecsError | None = None + try: + with self._op("bulk_save"): + result = await self.collection.bulk_write(requests, ordered=False, **self._kw(session)) + matched = int(result.matched_count) + except OperationFailed as exc: + details = exc.details if isinstance(exc.details, dict) else {} + write_errors = details.get("writeErrors") + if not write_errors: + raise + failed = {int(e["index"]) for e in write_errors} + matched = int(details.get("nMatched", 0)) + error = exc + updates = [i for i, p in enumerate(pending) if p[1] == "update" and i not in failed] + if matched != len(updates): + # The server only reports a total. Read back what each update should have left behind. + unmatched = await self._unmatched(pending, updates, session) + failed |= unmatched + if error is None: + error = StaleDocumentError( + f"{self.model.__name__}: {len(unmatched)} of {len(updates)} documents in the batch " + "no longer exist or were changed since they were loaded" + ) + written = 0 + for index, (doc, kind, ctx, enc, state, _) in enumerate(pending): + if index in failed: + continue if isinstance(session, Tx): session.record(doc, self.managed_attrs) if kind == "insert": @@ -658,9 +755,40 @@ async def bulk_save(self, docs: t.Iterable[T], *, session: t.Any = None) -> int: for plugin in self.plugins: plugin.applied(ctx, doc, enc) tracking.track(doc, enc, state.manager if state else self, state.partial if state else None) + written += 1 for plugin in self.plugins: - await plugin.after_bulk(bulk_ctx, all_docs) - return len(pending) + await plugin.after_bulk(bulk_ctx, [p[0] for i, p in enumerate(pending) if i not in failed]) + if error is not None: + error.failed = [pending[i][0] for i in sorted(failed)] + raise error + return written + + async def _unmatched(self, pending: list[t.Any], updates: list[int], session: t.Any) -> set[int]: + """Which of the batch's updates did not apply. + + The server reports only a total, so the documents are read back and compared with + what each update would have left: the values it sets, the fields it removes and the + counters it increments (the revision among them). + """ + found: dict[t.Any, dict[str, t.Any]] = {} + for chunk in _chunks([pending[i][3]["_id"] for i in updates]): + with self._op("bulk_save"): + cursor = self.collection.find({"_id": {"$in": chunk}}, **self._kw(session)) + for raw in await cursor.to_list(): + found[_hashable(raw["_id"])] = raw + out: set[int] = set() + for i in updates: + enc, update = pending[i][3], pending[i][5] + stored = found.get(_hashable(enc["_id"])) + applied = ( + stored is not None + and all(_dig(stored, path) == value for path, value in update.get("$set", {}).items()) + and all(_dig(stored, path, _MISSING) is _MISSING for path in update.get("$unset", {})) + and all(stored.get(key) == enc.get(key) for key in update.get("$inc", {})) + ) + if not applied: + out.add(i) + return out # -- escape hatches --------------------------------------------------------------------- @@ -798,3 +926,31 @@ def _filter_keys(flt: t.Any) -> set[str]: elif not key.startswith("$"): keys.add(key.partition(".")[0]) return keys + + +_CHUNK_IDS = 5000 + + +def _chunks(ids: list[t.Any]) -> t.Iterator[list[t.Any]]: + """Ids in pieces small enough for one ``$in`` each.""" + for start in range(0, len(ids), _CHUNK_IDS): + yield ids[start : start + _CHUNK_IDS] + + +def _hashable(value: t.Any) -> t.Any: + try: + hash(value) + except TypeError: + return repr(value) + return value + + +_MISSING: t.Any = object() + + +def _dig(doc: t.Any, path: str, default: t.Any = None) -> t.Any: + for part in path.split("."): + if not isinstance(doc, dict) or part not in doc: + return default + doc = doc[part] + return doc diff --git a/packages/mongospecs/src/mongospecs/query.py b/packages/mongospecs/src/mongospecs/query.py index 01ae684..b3abdf4 100644 --- a/packages/mongospecs/src/mongospecs/query.py +++ b/packages/mongospecs/src/mongospecs/query.py @@ -21,7 +21,7 @@ from mongospecs.updates import UpdateOp if t.TYPE_CHECKING: - from mongospecs.manager import Manager + from mongospecs.manager import Manager, UpdateCount from mongospecs.relations import Include __all__ = ["Page", "Query"] @@ -254,7 +254,8 @@ def _projection( keys[meta.disc_key] = None projection = {k: 1 for k in keys} return projection, frozenset(k.partition(".")[0] for k in keys) | {"_id"} - excluded = [*deferred, *exclude] + required = set(needed) + excluded = [key for key in (*deferred, *exclude) if key.partition(".")[0] not in required] if not excluded: return None, None return {k: 0 for k in excluded}, meta.stored_keys - {k for k in excluded if "." not in k} @@ -268,6 +269,8 @@ def _plan(self) -> dict[str, t.Any]: manager = self._manager lookups, batched = plan_includes(manager, self._includes, self._mode) needed = [manager.meta.fields[i.name].stored for i in self._includes if i.name in manager.meta.refs] + # Sort keys are always fetched: cursor pagination builds the next cursor from them. + needed += [key.partition(".")[0] for key, _ in self._sort_spec()] projection, partial = self._projection(needed) plan: dict[str, t.Any] = { "filter": self._filter(), @@ -492,8 +495,9 @@ def _check_plain(self, what: str) -> None: if self._skip or self._limit or self._sort: raise QueryError(f"{what} applies to every match; sort, skip and limit are not supported") - async def update(self, *ops: UpdateOp, **fields: t.Any) -> int: - """Atomically update every match. Returns the number of documents modified. + async def update(self, *ops: UpdateOp, **fields: t.Any) -> UpdateCount: + """Atomically update every match. Returns the number of documents modified + (an ``int`` that also carries ``.matched`` and ``.modified``). Documents are not loaded, so document hooks do not run; plugins still rewrite the command (timestamps, revisions, scopes). diff --git a/packages/mongospecs/src/mongospecs/relations.py b/packages/mongospecs/src/mongospecs/relations.py index 78840dd..8fca230 100644 --- a/packages/mongospecs/src/mongospecs/relations.py +++ b/packages/mongospecs/src/mongospecs/relations.py @@ -420,24 +420,32 @@ def cascade_rules(manager: Manager[t.Any]) -> list[CascadeRule]: async def apply_cascades( - manager: Manager[t.Any], rules: list[CascadeRule], ids: list[t.Any], session: t.Any + manager: Manager[t.Any], rules: list[CascadeRule], ids: list[t.Any], session: t.Any, seen: set[t.Any] ) -> None: + """Apply ``OnDelete`` rules for documents of ``manager``'s model that are about to be deleted. + + ``seen`` holds every (collection, id) already being deleted in this cascade, so a cycle of + cascading references ends instead of recursing forever. + """ + from mongospecs.manager import _chunks + for rule in rules: other: Manager[t.Any] = rule.manager stored = rule.field.stored - flt = {stored: {"$in": ids}} - if rule.action == "cascade": - await other.delete_many(flt, session=session) - continue - ctx = other.ctx("update", session, many=True) - scoped = other.scoped(ctx, flt, True) - update: dict[str, t.Any] = ( - {"$set": {stored: None}} if rule.action == "nullify" else {"$pull": {stored: {"$in": ids}}} - ) - for plugin in other.plugins: - plugin.on_update(ctx, scoped, update) - with other._op("cascade"): - await other.collection.update_many(scoped, update, **other._kw(session)) + for chunk in _chunks(ids): + flt = {stored: {"$in": chunk}} + if rule.action == "cascade": + await other.delete_many(flt, session=session, _seen=seen) + continue + ctx = other.ctx("update", session, many=True) + scoped = other.scoped(ctx, flt, True) + update: dict[str, t.Any] = ( + {"$set": {stored: None}} if rule.action == "nullify" else {"$pull": {stored: {"$in": chunk}}} + ) + for plugin in other.plugins: + plugin.on_update(ctx, scoped, update) + with other._op("cascade"): + await other.collection.update_many(scoped, update, **other._kw(session)) # -- $graphLookup ---------------------------------------------------------------------------- diff --git a/packages/mongospecs/tests/test_review_fixes.py b/packages/mongospecs/tests/test_review_fixes.py new file mode 100644 index 0000000..dca56a4 --- /dev/null +++ b/packages/mongospecs/tests/test_review_fixes.py @@ -0,0 +1,149 @@ +"""Regression tests for defects reported in review of the v2 pull requests.""" + +import copy + +import pytest + +from mongospecs import ( + BulkWriteFailed, + DeleteMany, + DeleteOne, + DuplicateKeyError, + Inc, + Q, + StaleDocumentError, +) + + +async def test_cursor_pages_with_a_projection_do_not_repeat(m, registry): + """select() used to drop the sort key from the projection, so the cursor was built from defaults.""" + await m.User.insert_many([m.User(name=f"u{i:02d}", age=i % 5, email=f"{i}@x.io") for i in range(30)]) + for select in (("name",), ("-age",), ("email",)): + query = m.User.find().select(*select).sort("age", "-name") + seen, cursor, pages = [], None, 0 + while True: + page = await query.page(7, after=cursor) + seen.extend(u.name for u in page) + pages += 1 + assert pages <= 6, f"select{select}: pagination does not terminate" + if not page.has_next: + break + cursor = page.next_cursor + assert len(seen) == len(set(seen)) == 30 and pages == 5, (select, pages, len(seen)) + raw = await m.User.find().raw().select("name").sort("-age").page(4) + assert "age" in raw[0] # sort keys are always fetched: the cursor is built from them + + +async def test_bulk_save_keeps_memory_in_sync_after_a_partial_failure(m, registry): + docs = (await m.Article.insert_many([m.Article(title=f"a{i}") for i in range(3)])).inserted + fresh = m.Article(title="new") + await m.Article.collection().delete_one({"_id": docs[1].id}) # this one cannot be updated + for doc in docs: + doc.views = 5 + with pytest.raises(StaleDocumentError) as info: + await m.Article.bulk_save([*docs, fresh]) + assert info.value.failed == [docs[1]] + # the others were written, and say so + for doc in (docs[0], docs[2]): + assert doc.rev == 1 and doc.changes() == {} + assert (await m.Article.collection().find_one({"_id": doc.id}))["views"] == 5 + assert not fresh.is_new and fresh.changes() == {} + assert docs[1].rev == 0 and docs[1].changes() == {"$set": {"views": 5}} + docs[0].views = 6 + assert await m.Article.bulk_save([docs[0], docs[2]]) == 1 # saving again works + + +async def test_bulk_save_reports_revision_conflicts_per_document(m, registry): + docs = (await m.Article.insert_many([m.Article(title=f"a{i}") for i in range(3)])).inserted + await m.Article.update(docs[2].id, Inc(views=1)) # someone else got there first + for doc in docs: + doc.title += "!" + with pytest.raises(StaleDocumentError) as info: + await m.Article.bulk_save(docs) + assert info.value.failed == [docs[2]] + assert [d.rev for d in docs] == [1, 1, 0] + + +async def test_bulk_save_duplicate_key_does_not_hide_the_other_writes(m, registry): + await registry.apply_indexes() + await m.User(name="taken", email="t@x.io").insert() + existing = await m.User(name="e", email="e@x.io").insert() + existing.age = 9 + clash, fine = m.User(name="clash", email="t@x.io"), m.User(name="fine", email="f@x.io") + with pytest.raises((DuplicateKeyError, BulkWriteFailed)) as info: + await m.User.bulk_save([existing, clash, fine]) + assert info.value.failed == [clash] + assert existing.changes() == {} and not fine.is_new and clash.is_new + assert await m.User.count() == 3 + + +async def test_bulk_write_deletes_run_cascades(m, registry): + a1, a2, a3 = (await m.Article.insert_many([m.Article(title=t) for t in "abc"])).inserted + await m.Comment.insert_many([m.Comment(body=f"c{i}", article=a) for i, a in enumerate((a1, a1, a2, a3))]) + await m.Article.bulk_write([DeleteOne(a1.id, hard=True), DeleteMany(Q.title == "b", hard=True)]) + assert [c.body for c in await m.Comment.find()] == ["c3"] # only a3's comment is left + # a soft delete in a bulk write does not cascade, like a single soft delete + await m.Article.bulk_write([DeleteOne(a3.id)]) + assert await m.Comment.count() == 1 and await m.Article.count() == 0 + + +async def test_cascade_cycles_terminate(m, make_registry): + import typing as t + + from mongospecs import OnDelete, Ref, SpecConfig, To + + @m.define + class Left(m.Spec): + __spec__ = SpecConfig(collection="cyc_left") + other: t.Annotated[Ref | None, To("Right"), OnDelete("cascade")] = None + + @m.define + class Right(m.Spec): + __spec__ = SpecConfig(collection="cyc_right") + other: t.Annotated[Ref[Left] | None, OnDelete("cascade")] = None + + await make_registry(Left, Right) + left = await Left().insert() + right = await Right(other=left).insert() + left.other = right + await left.save() + await Left().insert() # unrelated, must survive + await left.delete() + assert await Left.count() == 1 and await Right.count() == 0 + + +async def test_large_cascades_are_chunked(m, registry): + articles = (await m.Article.insert_many([m.Article(title=f"a{i}") for i in range(2500)])).inserted + await m.Comment.insert_many([m.Comment(body="c", article=a) for a in articles]) + assert await m.Article.find().delete(hard=True) == 2500 + assert await m.Comment.count() == 0 + + +async def test_saving_an_untracked_copy_says_what_is_wrong(m, registry): + user = await m.User(name="ann").insert() + for clone in (copy.copy(user), copy.deepcopy(user), await m.User.find().lean().one()): + clone.age = 3 + with pytest.raises(DuplicateKeyError, match="not tracked"): + await clone.save() + assert (await m.User.get(user.id)).age == 0 + adopted = m.User.hydrate(await m.User.collection().find_one({"_id": user.id})) + adopted.age = 4 + await adopted.save() # the documented way to get a saveable copy + assert (await m.User.get(user.id)).age == 4 + + +async def test_reload_respects_deferred_fields(m, registry): + user = await m.User(name="ann", secret="s3cret").insert() + loaded = await m.User.get(user.id) + await m.User.collection().update_one({"_id": user.id}, {"$set": {"age": 7}}) + await loaded.reload() + assert loaded.age == 7 and loaded.secret is None # still deferred + loaded.name = "anna" + await loaded.save() + assert (await m.User.collection().find_one({"_id": user.id}))["secret"] == "s3cret" + + +async def test_filter_update_reports_matched_and_modified(m, registry): + await m.User.insert_many([m.User(name="a", age=1), m.User(name="b", age=1), m.User(name="c", age=2)]) + result = await m.User.find(Q.age >= 1).update(age=1) + assert result == 1 and result.modified == 1 and result.matched == 3 # still an int: the number modified From ea43cece77411231f055d2cd6f563475e31c7c72 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 02:29:09 -0400 Subject: [PATCH 08/26] feat(web)!: harden the request path From review of the web layer. - request bodies are read under the size limit on every plug (declared length checked first, then chunk by chunk); previously only the pure ASGI plug stopped at the limit and the others buffered the whole body - DELETE applies the row scope in the delete's own filter, not only in the lookup before it, and honours If-Match - deleting a loaded document checks its revision on hard deletes too - offset pagination refuses pages deeper than Pagination.max_offset (10,000 skipped documents by default) - totals are opt-in per resource: Pagination.total now defaults to False - reads made for a request carry a 10s time limit when neither the registry nor the model sets one - Api(openapi=False) keeps the API description private on every plug - HEAD and OPTIONS are answered by Api.handle and the pure ASGI plug - Resource(explicit_writes=True): a field is writable only when its Policy says so - mongospecs_web.testing.mount_on(plug, api) builds an app on any built-in plug BREAKING: ?total=true needs page=Cursor(total=True) / Offset(total=True). Co-Authored-By: Claude Opus 5.5 --- .../mongospecs-web/src/mongospecs_web/api.py | 42 ++++++- .../src/mongospecs_web/backend.py | 32 +++++- .../src/mongospecs_web/filters.py | 12 +- .../src/mongospecs_web/plugs/asgi.py | 27 +++-- .../src/mongospecs_web/plugs/fastapi.py | 8 +- .../src/mongospecs_web/plugs/litestar.py | 16 ++- .../src/mongospecs_web/plugs/quart.py | 33 ++++-- .../src/mongospecs_web/plugs/starlette.py | 21 ++-- .../src/mongospecs_web/policy.py | 26 +++-- .../src/mongospecs_web/resource.py | 9 +- .../src/mongospecs_web/testing.py | 61 +++++++++- .../mongospecs-web/tests/test_contract.py | 42 +------ .../tests/test_mongo_resource.py | 64 ++++++++++- .../tests/test_openapi_views.py | 108 +++++++++++++++++- packages/mongospecs/src/mongospecs/manager.py | 24 +++- packages/mongospecs/src/mongospecs/model.py | 10 +- 16 files changed, 428 insertions(+), 107 deletions(-) diff --git a/packages/mongospecs-web/src/mongospecs_web/api.py b/packages/mongospecs-web/src/mongospecs_web/api.py index 0e4aa5a..83dfbb0 100644 --- a/packages/mongospecs-web/src/mongospecs_web/api.py +++ b/packages/mongospecs-web/src/mongospecs_web/api.py @@ -52,6 +52,7 @@ def __init__( version: str = "1.0.0", authenticate: Authenticate | None = None, max_body: int = 1_048_576, + openapi: bool = True, ) -> None: self.resources: tuple[Resource[t.Any], ...] = resources self.prefix = "/" + prefix.strip("/") if prefix.strip("/") else "" @@ -59,6 +60,9 @@ def __init__( self.version = version self.authenticate = authenticate self.max_body = max_body + self.expose_openapi = openapi + """Serve ``/openapi.json``. Turn it off to keep the API description private; + ``api.openapi()`` still builds it for you to serve behind your own authentication.""" names = [r.name for r in resources] if len(set(names)) != len(names): raise ValueError(f"resource names must be unique, got {names}") @@ -81,6 +85,29 @@ def routes(self) -> list[Route]: def openapi_path(self) -> str: return f"{self.prefix}/openapi.json" + async def read_body(self, headers: t.Mapping[str, str], chunks: t.AsyncIterable[bytes]) -> bytes: + """Read a request body without ever holding more than ``max_body`` bytes. + + Raises :class:`PayloadTooLarge` as soon as the declared or the actual size passes the + limit, instead of buffering an oversized body first. + """ + declared = headers.get("content-length") + if declared is not None and declared.isdigit() and int(declared) > self.max_body: + raise PayloadTooLarge(f"The request body may be at most {self.max_body} bytes.") + body = bytearray() + async for chunk in chunks: + body += chunk + if len(body) > self.max_body: + raise PayloadTooLarge(f"The request body may be at most {self.max_body} bytes.") + return bytes(body) + + def allowed_methods(self, path: str) -> list[str]: + """Methods the routes matching ``path`` accept (empty when no route matches).""" + methods = {route.method for route in self._routes() if route.pattern.match(path)} + if "GET" in methods: + methods.add("HEAD") + return sorted(methods | {"OPTIONS"}) if methods else [] + def match(self, method: str, path: str) -> tuple[Route, dict[str, str]]: """Find the route for a request, or raise 404 / 405.""" allowed: list[str] = [] @@ -139,13 +166,22 @@ async def handle( raw: t.Any = None, ) -> Response: """Route and run a request given as plain values. Used by the ASGI plug and handy in tests.""" + method = method.upper() + if method == "OPTIONS": + allowed = self.allowed_methods(path) + if not allowed: + return Response.error(NotFound("No such route.")) + return Response(204, b"", {"allow": ", ".join(allowed)}) try: - route, params = self.match(method, path) + route, params = self.match("GET" if method == "HEAD" else method, path) except HTTPError as exc: return Response.error(exc) - return await self.run( - route, Request(method.upper(), params, query or [], headers or {}, body, None, raw) + response = await self.run( + route, Request(route.method, params, query or [], headers or {}, body, None, raw) ) + if method == "HEAD": + response.body = b"" # same status and headers as GET, no body + return response def openapi(self) -> dict[str, t.Any]: from mongospecs_web.openapi import build_openapi diff --git a/packages/mongospecs-web/src/mongospecs_web/backend.py b/packages/mongospecs-web/src/mongospecs_web/backend.py index 61fefb2..ce85e5e 100644 --- a/packages/mongospecs-web/src/mongospecs_web/backend.py +++ b/packages/mongospecs-web/src/mongospecs_web/backend.py @@ -9,7 +9,7 @@ import typing as t from dataclasses import dataclass, field -from mongospecs import DocumentNotFound, Page, Q, QueryError, meta_of +from mongospecs import DocumentNotFound, Page, Q, QueryError, StaleDocumentError, meta_of from mongospecs.expr import Condition, Expr, Field __all__ = ["OPERATORS", "Filter", "ListQuery", "MongoBackend", "ResourceBackend"] @@ -75,16 +75,25 @@ async def patch( self, id: t.Any, data: dict[str, t.Any], *, scope: t.Any, if_rev: int | None = None ) -> T: ... - async def delete(self, id: t.Any, *, scope: t.Any) -> None: ... + async def delete(self, id: t.Any, *, scope: t.Any, if_rev: int | None = None) -> None: ... class MongoBackend(t.Generic[T]): """``ResourceBackend`` over a mongospecs model (or a manager from ``Model.using(...)``).""" - def __init__(self, model: type[T], manager: t.Any = None) -> None: + def __init__(self, model: type[T], manager: t.Any = None, *, max_time_ms: int | None = 10_000) -> None: self.model = model self._manager = manager self.meta = meta_of(model) + self.max_time_ms = max_time_ms + """Time limit for reads made on behalf of a request, used when neither the registry + nor the model sets one. A request should not be able to run a query without bound.""" + + def _bounded(self, query: t.Any) -> t.Any: + manager = self.manager + if self.max_time_ms is None or manager.registry.max_time_ms or manager.meta.config.max_time_ms: + return query + return query.options(max_time_ms=self.max_time_ms) @property def manager(self) -> t.Any: @@ -104,6 +113,7 @@ async def get( query = self._by_id(id, scope) if query is None: return None + query = self._bounded(query) if read_only: query = query.lean() # nothing will be saved: skip change tracking for path in include: @@ -111,7 +121,7 @@ async def get( return t.cast("T | None", await query.first()) async def list(self, query: ListQuery, *, scope: t.Any) -> Page[T]: - q = self.manager.find(*(f.to_expr() for f in query.filters)).lean() # listings are read-only + q = self._bounded(self.manager.find(*(f.to_expr() for f in query.filters))).lean() # read-only if scope is not None: q = q.filter(scope) if query.sort: @@ -141,9 +151,19 @@ async def patch(self, id: t.Any, data: dict[str, t.Any], *, scope: t.Any, if_rev target = trusted(query._user_filter()) return t.cast(T, await self.manager.update(target, if_rev=if_rev, **data)) - async def delete(self, id: t.Any, *, scope: t.Any) -> None: + async def delete(self, id: t.Any, *, scope: t.Any, if_rev: int | None = None) -> None: query = self._by_id(id, scope) doc = await query.first() if query is not None else None if doc is None: raise DocumentNotFound(f"{self.model.__name__} {id!r} does not exist") - await self.manager.delete_document(doc) + # The document is loaded so that hooks, soft delete and cascades run. The scope is + # then also part of the delete's own filter, so it holds at the moment of the write. + condition = self.manager.compile(scope) if scope is not None else None + try: + await self.manager.delete_document(doc, condition=condition, if_rev=if_rev) + except StaleDocumentError: + if if_rev is not None: + raise + if scope is not None and not await self._by_id(id, scope).exists(): + raise DocumentNotFound(f"{self.model.__name__} {id!r} does not exist") from None + raise diff --git a/packages/mongospecs-web/src/mongospecs_web/filters.py b/packages/mongospecs-web/src/mongospecs_web/filters.py index 2a61322..86ede91 100644 --- a/packages/mongospecs-web/src/mongospecs_web/filters.py +++ b/packages/mongospecs-web/src/mongospecs_web/filters.py @@ -40,8 +40,12 @@ class Pagination: default: int = 20 max: int = 100 - total: bool = True - """Whether clients may ask for a total count with ``?total=true``.""" + total: bool = False + """Whether clients may ask for a total count with ``?total=true``. Off by default: a + count is a second query over every match, which a client should not get for free.""" + max_offset: int = 10_000 + """Offset pagination only: how many documents a page may skip. Deep offsets make the + server walk and discard everything before them; use cursors beyond this.""" kind: t.ClassVar[str] = "cursor" @@ -122,6 +126,10 @@ def parse_list_query( if after is not None: raise BadRequest("this resource uses page numbers; use page= instead of after=") number = _int(grouped["page"][-1], "page") if "page" in grouped else 1 + if (number - 1) * size > pagination.max_offset: + raise BadRequest( + f"page {number} is too deep: at most {pagination.max_offset} results can be skipped" + ) elif "page" in grouped: raise BadRequest("this resource uses cursors; use after= instead of page=") total = _bool(grouped["total"][-1], "total") if "total" in grouped else False diff --git a/packages/mongospecs-web/src/mongospecs_web/plugs/asgi.py b/packages/mongospecs-web/src/mongospecs_web/plugs/asgi.py index a929f75..00b8ebd 100644 --- a/packages/mongospecs-web/src/mongospecs_web/plugs/asgi.py +++ b/packages/mongospecs-web/src/mongospecs_web/plugs/asgi.py @@ -7,6 +7,7 @@ from urllib.parse import parse_qsl from mongospecs_web.api import Api +from mongospecs_web.errors import HTTPError from mongospecs_web.resource import Response __all__ = ["AsgiApp"] @@ -59,21 +60,27 @@ async def _http(self, scope: Scope, receive: Receive, send: Send) -> None: path: str = scope["path"] if root and path.startswith(root): path = path[len(root) :] or "/" - if method == "GET" and path == self.api.openapi_path: + if method == "GET" and path == self.api.openapi_path and self.api.expose_openapi: if self._openapi is None: self._openapi = json.dumps(self.api.openapi(), separators=(",", ":")).encode() await _send(send, Response(200, self._openapi, {})) return - body = b"" - limit = self.api.max_body - while True: - message = await receive() - if message["type"] == "http.disconnect": - return - body += message.get("body", b"") - if len(body) > limit or not message.get("more_body"): - break headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope.get("headers", [])} + + async def chunks() -> t.AsyncIterator[bytes]: + while True: + message = await receive() + if message["type"] == "http.disconnect": + return + yield message.get("body", b"") + if not message.get("more_body"): + return + + try: + body = await self.api.read_body(headers, chunks()) + except HTTPError as exc: + await _send(send, Response.error(exc)) + return query = parse_qsl(scope.get("query_string", b"").decode("latin-1"), keep_blank_values=True) response = await self.api.handle(method, path, query=query, headers=headers, body=body, raw=scope) await _send(send, response) diff --git a/packages/mongospecs-web/src/mongospecs_web/plugs/fastapi.py b/packages/mongospecs-web/src/mongospecs_web/plugs/fastapi.py index 917b1ef..a58057c 100644 --- a/packages/mongospecs-web/src/mongospecs_web/plugs/fastapi.py +++ b/packages/mongospecs-web/src/mongospecs_web/plugs/fastapi.py @@ -16,15 +16,21 @@ from starlette.responses import Response as StarletteResponse from mongospecs_web.api import Api, Route +from mongospecs_web.errors import HTTPError from mongospecs_web.openapi import build_openapi, operation_object from mongospecs_web.plugs.starlette import lifespan, to_request, to_response +from mongospecs_web.resource import Response __all__ = ["lifespan", "mount", "router"] def _endpoint(api: Api, route: Route) -> t.Callable[..., t.Awaitable[StarletteResponse]]: async def endpoint(request: StarletteRequest) -> StarletteResponse: - return to_response(await api.run(route, await to_request(request))) + try: + neutral = await to_request(api, request) + except HTTPError as exc: + return to_response(Response.error(exc)) + return to_response(await api.run(route, neutral)) return endpoint diff --git a/packages/mongospecs-web/src/mongospecs_web/plugs/litestar.py b/packages/mongospecs-web/src/mongospecs_web/plugs/litestar.py index 6bb3403..30376cf 100644 --- a/packages/mongospecs-web/src/mongospecs_web/plugs/litestar.py +++ b/packages/mongospecs-web/src/mongospecs_web/plugs/litestar.py @@ -17,6 +17,7 @@ from litestar.plugins import InitPluginProtocol from mongospecs_web.api import Api, Route +from mongospecs_web.errors import HTTPError from mongospecs_web.resource import Request, Response __all__ = ["MongospecsPlugin", "handlers"] @@ -40,12 +41,17 @@ def _handler(api: Api, target: Route) -> t.Any: @route(path, http_method=[method], name=target.name, include_in_schema=False, status_code=200) async def handler(request: LitestarRequest[t.Any, t.Any, t.Any]) -> LitestarResponse[bytes]: + headers = {k.lower(): v for k, v in request.headers.items()} + try: + body = await api.read_body(headers, request.stream()) + except HTTPError as exc: + return _to_response(Response.error(exc)) neutral = Request( request.method, {k: str(v) for k, v in request.path_params.items()}, list(request.query_params.multi_items()), - {k.lower(): v for k, v in request.headers.items()}, - await request.body(), + headers, + body, None, request, ) @@ -54,10 +60,10 @@ async def handler(request: LitestarRequest[t.Any, t.Any, t.Any]) -> LitestarResp return handler -def handlers(api: Api, *, openapi: bool = True) -> list[t.Any]: +def handlers(api: Api, *, openapi: bool | None = None) -> list[t.Any]: """Litestar route handlers for every operation of the API.""" out = [_handler(api, target) for target in api._routes()] - if openapi: + if api.expose_openapi if openapi is None else openapi: @get(api.openapi_path, name="mongospecs_openapi", include_in_schema=False, sync_to_thread=False) def spec() -> dict[str, t.Any]: @@ -70,7 +76,7 @@ def spec() -> dict[str, t.Any]: class MongospecsPlugin(InitPluginProtocol): """``Litestar(plugins=[MongospecsPlugin(api, registry=registry)])``""" - def __init__(self, api: Api, *, registry: t.Any = None, openapi: bool = True) -> None: + def __init__(self, api: Api, *, registry: t.Any = None, openapi: bool | None = None) -> None: self.api = api self.registry = registry self.openapi = openapi diff --git a/packages/mongospecs-web/src/mongospecs_web/plugs/quart.py b/packages/mongospecs-web/src/mongospecs_web/plugs/quart.py index 2cd7548..4d87c96 100644 --- a/packages/mongospecs-web/src/mongospecs_web/plugs/quart.py +++ b/packages/mongospecs-web/src/mongospecs_web/plugs/quart.py @@ -9,34 +9,43 @@ from quart import request as quart_request from mongospecs_web.api import Api, Route -from mongospecs_web.resource import Request +from mongospecs_web.errors import HTTPError +from mongospecs_web.resource import Request, Response __all__ = ["blueprint", "init_app"] +def _to_quart(response: Response) -> QuartResponse: + out = QuartResponse(response.body, response.status, dict(response.headers)) + if response.body or response.status not in (204, 304): + out.headers["content-type"] = response.media_type + else: + out.headers.pop("content-type", None) + return out + + def _view(api: Api, route: Route) -> t.Callable[..., t.Awaitable[QuartResponse]]: async def view(**path_params: t.Any) -> QuartResponse: + headers = {k.lower(): v for k, v in quart_request.headers.items()} + try: + body = await api.read_body(headers, quart_request.body) + except HTTPError as exc: + return _to_quart(Response.error(exc)) neutral = Request( quart_request.method, {k: str(v) for k, v in path_params.items()}, list(quart_request.args.items(multi=True)), - {k.lower(): v for k, v in quart_request.headers.items()}, - t.cast(bytes, await quart_request.get_data(as_text=False)), + headers, + body, None, quart_request, ) - response = await api.run(route, neutral) - out = QuartResponse(response.body, response.status, dict(response.headers)) - if response.body or response.status not in (204, 304): - out.headers["content-type"] = response.media_type - else: - out.headers.pop("content-type", None) - return out + return _to_quart(await api.run(route, neutral)) return view -def blueprint(api: Api, *, name: str = "mongospecs", openapi: bool = True) -> Blueprint: +def blueprint(api: Api, *, name: str = "mongospecs", openapi: bool | None = None) -> Blueprint: """A Blueprint with one rule per operation (plus ``openapi.json``).""" bp = Blueprint(name, __name__) for route in api._routes(): @@ -46,7 +55,7 @@ def blueprint(api: Api, *, name: str = "mongospecs", openapi: bool = True) -> Bl bp.add_url_rule( rule, endpoint=route.name.replace(".", "_"), view_func=_view(api, route), methods=[route.method] ) - if openapi: + if api.expose_openapi if openapi is None else openapi: async def spec() -> dict[str, t.Any]: return api.openapi() diff --git a/packages/mongospecs-web/src/mongospecs_web/plugs/starlette.py b/packages/mongospecs-web/src/mongospecs_web/plugs/starlette.py index 600a090..66bb5c1 100644 --- a/packages/mongospecs-web/src/mongospecs_web/plugs/starlette.py +++ b/packages/mongospecs-web/src/mongospecs_web/plugs/starlette.py @@ -12,18 +12,21 @@ from starlette.routing import Route as StarletteRoute from mongospecs_web.api import Api, Route +from mongospecs_web.errors import HTTPError from mongospecs_web.resource import Request, Response __all__ = ["lifespan", "mount", "routes", "to_request", "to_response"] -async def to_request(request: StarletteRequest) -> Request: +async def to_request(api: Api, request: StarletteRequest) -> Request: + """Translate a Starlette request. The body is read under the API's size limit.""" + headers = {k.lower(): v for k, v in request.headers.items()} return Request( request.method, {k: str(v) for k, v in request.path_params.items()}, list(request.query_params.multi_items()), - {k.lower(): v for k, v in request.headers.items()}, - await request.body(), + headers, + await api.read_body(headers, request.stream()), None, request, ) @@ -38,18 +41,22 @@ def to_response(response: Response) -> StarletteResponse: def _endpoint(api: Api, route: Route) -> t.Callable[[StarletteRequest], t.Awaitable[StarletteResponse]]: async def endpoint(request: StarletteRequest) -> StarletteResponse: - return to_response(await api.run(route, await to_request(request))) + try: + neutral = await to_request(api, request) + except HTTPError as exc: + return to_response(Response.error(exc)) + return to_response(await api.run(route, neutral)) return endpoint -def routes(api: Api, *, openapi: bool = True) -> list[BaseRoute]: +def routes(api: Api, *, openapi: bool | None = None) -> list[BaseRoute]: """Starlette routes for every operation of the API (plus ``openapi.json``).""" out: list[BaseRoute] = [ StarletteRoute(route.path, _endpoint(api, route), methods=[route.method], name=route.name) for route in api._routes() ] - if openapi: + if api.expose_openapi if openapi is None else openapi: async def spec(request: StarletteRequest) -> StarletteResponse: return JSONResponse(api.openapi()) @@ -58,7 +65,7 @@ async def spec(request: StarletteRequest) -> StarletteResponse: return out -def mount(app: t.Any, api: Api, *, openapi: bool = True) -> None: +def mount(app: t.Any, api: Api, *, openapi: bool | None = None) -> None: """Add the API's routes to an existing Starlette application.""" app.router.routes.extend(routes(api, openapi=openapi)) diff --git a/packages/mongospecs-web/src/mongospecs_web/policy.py b/packages/mongospecs-web/src/mongospecs_web/policy.py index 447ba5e..e2b31cb 100644 --- a/packages/mongospecs-web/src/mongospecs_web/policy.py +++ b/packages/mongospecs-web/src/mongospecs_web/policy.py @@ -24,8 +24,9 @@ class View: """What of a model is readable, writable, filterable and sortable from outside.""" - def __init__(self, model: type) -> None: + def __init__(self, model: type, *, explicit_writes: bool = False) -> None: self.model = model + self.explicit_writes = explicit_writes self.meta: ModelMeta = meta_of(model) fields = self.meta.fields managed = {f.name for f in fields.values() if any(f.has(m) for m in _MANAGED)} @@ -36,13 +37,19 @@ def __init__(self, model: type) -> None: writable = frozenset( f.name for f in fields.values() if f.name != "id" and f.name not in managed and not f.transient ) + + def may(flag: bool | None, default: bool) -> bool: + # explicit mode: only what the policy turns on; otherwise the policy's own default + return flag is True if explicit_writes else default + self.insertable: frozenset[str] = frozenset( - f.name - for f in fields.values() - if f.policy.can_insert and f.name != "id" and f.name not in managed and not f.transient + name for name in writable if may(fields[name].policy.insert, fields[name].policy.can_insert) ) self.updatable: frozenset[str] = frozenset( - name for name in writable if fields[name].policy.can_update and not fields[name].has(Immutable) + name + for name in writable + if may(fields[name].policy.update, fields[name].policy.can_update) + and not fields[name].has(Immutable) ) self.filterable: dict[str, tuple[str, ...]] = { f.name: f.policy.filter_ops @@ -179,11 +186,12 @@ def one(ref: t.Any, raw: t.Any) -> t.Any: return one(value, encoded) -_VIEWS: weakref.WeakKeyDictionary[type, View] = weakref.WeakKeyDictionary() +_VIEWS: weakref.WeakKeyDictionary[type, dict[bool, View]] = weakref.WeakKeyDictionary() -def view_of(model: type) -> View: - view = _VIEWS.get(model) +def view_of(model: type, *, explicit_writes: bool = False) -> View: + views = _VIEWS.setdefault(model, {}) + view = views.get(explicit_writes) if view is None: - view = _VIEWS[model] = View(model) + view = views[explicit_writes] = View(model, explicit_writes=explicit_writes) return view diff --git a/packages/mongospecs-web/src/mongospecs_web/resource.py b/packages/mongospecs-web/src/mongospecs_web/resource.py index 7e0809c..ca98619 100644 --- a/packages/mongospecs-web/src/mongospecs_web/resource.py +++ b/packages/mongospecs-web/src/mongospecs_web/resource.py @@ -211,9 +211,13 @@ def __init__( page: Pagination | None = None, tags: t.Sequence[str] | None = None, server_fields: t.Callable[[Ctx[T]], t.Any] | None = None, + explicit_writes: bool = False, ) -> None: self.model = model - self.view: View = view_of(model) + self.view: View = view_of(model, explicit_writes=explicit_writes) + """With ``explicit_writes=True`` a field is writable only if its ``Policy`` says so + (``Policy(insert=True)``, ``Policy(update=True)``); by default ordinary fields are + writable unless marked otherwise.""" self.name = name or self.view.meta.collection self.backend: ResourceBackend[T] = backend if backend is not None else MongoBackend(model) self.scope_fn = scope @@ -551,7 +555,8 @@ async def handler(ctx: Ctx[T]) -> Response: def _delete_op(self) -> Operation: async def handler(ctx: Ctx[T]) -> None: - await self.backend.delete(ctx.id, scope=await ctx.scope()) + if_rev = _if_match(ctx.request.headers.get("if-match"), self.view.revision is not None) + await self.backend.delete(ctx.id, scope=await ctx.scope(), if_rev=if_rev) return Operation(Delete, "DELETE", "/{id}", handler, Delete, f"Delete one of {self.name}", 204) diff --git a/packages/mongospecs-web/src/mongospecs_web/testing.py b/packages/mongospecs-web/src/mongospecs_web/testing.py index 7996403..18c3a5f 100644 --- a/packages/mongospecs-web/src/mongospecs_web/testing.py +++ b/packages/mongospecs-web/src/mongospecs_web/testing.py @@ -38,11 +38,11 @@ from mongospecs_web.api import Api from mongospecs_web.backend import ListQuery from mongospecs_web.errors import Unauthorized -from mongospecs_web.filters import Offset +from mongospecs_web.filters import Cursor, Offset from mongospecs_web.policy import view_of from mongospecs_web.resource import Create, Ctx, Delete, Get, List, Patch, Request, Resource -__all__ = ["FakeBackend", "contract_api", "reference_transcript", "run_contract"] +__all__ = ["PLUGS", "FakeBackend", "contract_api", "mount_on", "reference_transcript", "run_contract"] T = t.TypeVar("T") @@ -189,10 +189,13 @@ async def patch(self, id: t.Any, data: dict[str, t.Any], *, scope: t.Any, if_rev setattr(doc, revision, getattr(doc, revision) + 1) return doc - async def delete(self, id: t.Any, *, scope: t.Any) -> None: + async def delete(self, id: t.Any, *, scope: t.Any, if_rev: int | None = None) -> None: doc = self._find(id, scope) if doc is None: raise DocumentNotFound("not found") + revision = self.view.revision + if revision is not None and if_rev is not None and getattr(doc, revision) != if_rev: + raise StaleDocumentError("stale") del self.docs[t.cast(t.Any, doc).id] @@ -301,6 +304,7 @@ def own_rows(ctx: Ctx[t.Any]) -> t.Any: model, backend=store, ops=[List, Get, Create, Patch, Delete], + page=Cursor(total=True), scope=own_rows, guards={Create: require_login, Patch: require_login, Delete: [require_login, admin_only]}, ) @@ -326,7 +330,11 @@ async def stats(ctx: Ctx[t.Any]) -> dict[str, t.Any]: return {"count": len(page.items), "stock": sum(d.stock for d in page.items)} pages: Resource[t.Any] = Resource( - model, name="widget-pages", backend=store, ops=[List], page=Offset(default=2, max=3, total=True) + model, + name="widget-pages", + backend=store, + ops=[List], + page=Offset(default=2, max=3, total=True, max_offset=8), ) return Api(widgets, pages, prefix="/api", title="Contract", version="1.0.0", authenticate=_authenticate) @@ -470,6 +478,8 @@ async def call(method: str, path: str, *, label: str, **kwargs: t.Any) -> t.Any: await call("GET", "/widget-pages?sort=name", label="offset page 1") await call("GET", "/widget-pages?sort=name&page=2&total=true", label="offset page 2 with total") await call("GET", "/widget-pages?after=x", label="cursor on offset resource") + await call("GET", "/widget-pages?page=6", label="offset page too deep") + await call("GET", "/widget-pages?page=1&total=true", label="offset total") await call("POST", "/widget-pages", label="method not allowed", json={}) # -- patch -------------------------------------------------------------------- @@ -527,7 +537,10 @@ async def call(method: str, path: str, *, label: str, **kwargs: t.Any) -> t.Any: # -- delete ----------------------------------------------------------------------- await call("DELETE", f"/widgets/{second}", label="delete needs login") await call("DELETE", f"/widgets/{second}", label="delete forbidden for non-admin", headers=bob) - await call("DELETE", f"/widgets/{second}", label="delete", headers=root) + await call( + "DELETE", f"/widgets/{second}", label="delete stale revision", headers={**root, "if-match": '"7"'} + ) + await call("DELETE", f"/widgets/{second}", label="delete", headers={**root, "if-match": '"0"'}) await call("DELETE", f"/widgets/{second}", label="delete again is 404", headers=root) await call("GET", "/widgets?sort=name", label="list after delete", headers=root) @@ -551,3 +564,41 @@ async def reference_transcript(backend: str = "msgspec") -> Transcript: transport=httpx.ASGITransport(app=api.asgi()), base_url="http://contract" ) as client: return await run_contract(client, api) + + +PLUGS = ("asgi", "starlette", "fastapi", "litestar", "quart") + + +def mount_on(plug: str, api: Api) -> t.Any: + """An ASGI application serving ``api`` through one of the built-in plugs.""" + if plug == "asgi": + return api.asgi() + if plug == "starlette": + from starlette.applications import Starlette + + from mongospecs_web.plugs.starlette import routes + + return Starlette(routes=routes(api)) + if plug == "fastapi": + from fastapi import FastAPI + + from mongospecs_web.plugs.fastapi import mount + + fastapi_app = FastAPI(openapi_url=api.openapi_path) + mount(fastapi_app, api) + return fastapi_app + if plug == "litestar": + from litestar import Litestar + + from mongospecs_web.plugs.litestar import MongospecsPlugin + + return Litestar(plugins=[MongospecsPlugin(api)], openapi_config=None) + if plug == "quart": + from quart import Quart + + from mongospecs_web.plugs.quart import init_app + + quart_app = Quart(__name__) + init_app(quart_app, api) + return quart_app + raise ValueError(f"unknown plug {plug!r}; expected one of {PLUGS}") diff --git a/packages/mongospecs-web/tests/test_contract.py b/packages/mongospecs-web/tests/test_contract.py index 7c85693..6646154 100644 --- a/packages/mongospecs-web/tests/test_contract.py +++ b/packages/mongospecs-web/tests/test_contract.py @@ -3,49 +3,14 @@ import httpx import pytest -from mongospecs_web.testing import contract_api, reference_transcript, run_contract +from mongospecs_web.testing import PLUGS, contract_api, mount_on, reference_transcript, run_contract BACKENDS = ("msgspec", "pydantic", "attrs") -PLUGS = ("asgi", "starlette", "fastapi", "litestar", "quart") - - -def build_app(plug, api): - if plug == "asgi": - return api.asgi() - if plug == "starlette": - from starlette.applications import Starlette - - from mongospecs_web.plugs.starlette import routes - - return Starlette(routes=routes(api)) - if plug == "fastapi": - from fastapi import FastAPI - - from mongospecs_web.plugs.fastapi import mount - - app = FastAPI(openapi_url="/api/openapi.json") - mount(app, api) - return app - if plug == "litestar": - from litestar import Litestar - - from mongospecs_web.plugs.litestar import MongospecsPlugin - - return Litestar(plugins=[MongospecsPlugin(api)], openapi_config=None) - if plug == "quart": - from quart import Quart - - from mongospecs_web.plugs.quart import init_app - - app = Quart("contract") - init_app(app, api) - return app - raise AssertionError(plug) async def transcript_for(plug, backend): api = contract_api(backend) - app = build_app(plug, api) + app = mount_on(plug, api) async with httpx.AsyncClient( transport=httpx.ASGITransport(app=app), base_url="http://contract" ) as client: @@ -116,6 +81,8 @@ async def test_reference_transcript_is_what_the_api_should_do(backend, reference "offset page 1": 200, "offset page 2 with total": 200, "cursor on offset resource": 400, + "offset page too deep": 400, + "offset total": 200, "method not allowed": 405, "patch needs login": 401, "patch": 200, @@ -135,6 +102,7 @@ async def test_reference_transcript_is_what_the_api_should_do(backend, reference "custom collection op": 200, "delete needs login": 401, "delete forbidden for non-admin": 403, + "delete stale revision": 412, "delete": 204, "delete again is 404": 404, "list after delete": 200, diff --git a/packages/mongospecs-web/tests/test_mongo_resource.py b/packages/mongospecs-web/tests/test_mongo_resource.py index 80a9a37..0c797bf 100644 --- a/packages/mongospecs-web/tests/test_mongo_resource.py +++ b/packages/mongospecs-web/tests/test_mongo_resource.py @@ -19,7 +19,7 @@ UpdatedAt, computed, ) -from mongospecs_web import Api, Create, Delete, Get, List, Offset, Patch, Resource, Unauthorized +from mongospecs_web import Api, Create, Cursor, Delete, Get, List, Offset, Patch, Resource, Unauthorized from mongospecs_web.plugs.asgi import AsgiApp BACKENDS = ("msgspec", "pydantic", "attrs") @@ -98,10 +98,17 @@ def mine(ctx): api = Api( Resource(Author, ops=[List, Get, Create, Patch]), Resource( - Story, ops=[List, Get, Create, Patch, Delete], include=["author", "reactions", "reaction_count"] + Story, + ops=[List, Get, Create, Patch, Delete], + include=["author", "reactions", "reaction_count"], + page=Cursor(total=True), ), Resource( - Story, name="my-stories", ops=[List, Get, Patch, Delete], scope=mine, page=Offset(default=10) + Story, + name="my-stories", + ops=[List, Get, Patch, Delete], + scope=mine, + page=Offset(default=10, total=True, max_offset=50), ), prefix="/v1", authenticate=authenticate, @@ -310,3 +317,54 @@ async def send(message): await asgi({"type": "lifespan"}, receive, send) assert sent == ["lifespan.startup.complete", "lifespan.shutdown.complete"] assert not registry.is_open + + +async def test_delete_is_conditional_and_scoped_at_the_write(app): + client, (Author, Story, Reaction), api, registry = app + ann = await Author(name="Ann", email="a").insert() + story = await Story(title="s", author=ann).insert() + as_ann = {"x-user": str(ann.id)} + # If-Match on DELETE: a stale revision is refused, the current one is accepted + await client.patch(f"/v1/stories/{story.id}", json={"words": 5}) + stale = await client.delete(f"/v1/stories/{story.id}", headers={"if-match": '"0"'}) + assert stale.status_code == 412 + assert (await client.get(f"/v1/stories/{story.id}")).status_code == 200 + # the row scope is part of the delete's own filter, not only of the lookup before it + events = [] + observer = type(registry)("mongodb://localhost:27117/?replicaSet=rs0", on_command=events.append) + await observer.open() + try: + from mongospecs_web.backend import MongoBackend + + backend = MongoBackend(Story, observer.manager(Story, db=registry.database().name)) + await backend.delete(str(story.id), scope={"author": ann.id}, if_rev=1) + finally: + await observer.close() + (write,) = [e.command for e in events if e.phase == "started" and e.command_name == "update"] + assert write["updates"][0]["q"] == {"_id": story.id, "rev": 1, "author": ann.id} + assert (await client.get(f"/v1/my-stories/{story.id}", headers=as_ann)).status_code == 404 # soft-deleted + + +async def test_listing_limits(app): + client, (Author, Story, Reaction), api, registry = app + ann = await Author(name="Ann", email="a").insert() + await Story.insert_many([Story(title=f"s{i}", author=ann) for i in range(3)]) + as_ann = {"x-user": str(ann.id)} + assert (await client.get("/v1/my-stories?page=6&limit=10", headers=as_ann)).status_code == 200 + deep = await client.get("/v1/my-stories?page=7&limit=10", headers=as_ann) + assert deep.status_code == 400 and "too deep" in deep.json()["detail"] + assert (await client.get("/v1/authors?total=true")).status_code == 400 # totals are opt-in per resource + # reads made for a request carry a time limit even when nothing else sets one + events = [] + observer = type(registry)("mongodb://localhost:27117/?replicaSet=rs0", on_command=events.append) + await observer.open() + try: + from mongospecs_web.backend import ListQuery, MongoBackend + + manager = observer.manager(Story, db=registry.database().name) + await MongoBackend(Story, manager).list(ListQuery(), scope=None) + await MongoBackend(Story, manager, max_time_ms=None).list(ListQuery(), scope=None) + finally: + await observer.close() + finds = [e.command for e in events if e.phase == "started" and e.command_name == "find"] + assert finds[0]["maxTimeMS"] == 10_000 and "maxTimeMS" not in finds[1] diff --git a/packages/mongospecs-web/tests/test_openapi_views.py b/packages/mongospecs-web/tests/test_openapi_views.py index 9fc997d..eb465bf 100644 --- a/packages/mongospecs-web/tests/test_openapi_views.py +++ b/packages/mongospecs-web/tests/test_openapi_views.py @@ -4,7 +4,7 @@ from mongospecs_web import Api, Resource, view_of from mongospecs_web.policy import jsonable -from mongospecs_web.testing import contract_api +from mongospecs_web.testing import contract_api, mount_on BACKENDS = ("msgspec", "pydantic", "attrs") @@ -160,3 +160,109 @@ async def test_api_handle_without_a_server(): assert (await api.handle("GET", "/api/widgets/stats")).status == 200 # literal segment beats {id} assert (await api.handle("GET", "/api/nope")).status == 404 assert (await api.handle("PUT", "/api/widgets")).headers["allow"] == "GET, POST" + + +BODY_PLUGS = ("asgi", "starlette", "fastapi", "litestar", "quart") + + +@pytest.mark.parametrize("plug", BODY_PLUGS) +async def test_oversized_bodies_are_refused_without_being_buffered(plug): + """Every plug stops reading at the limit, whether or not the client declares the size.""" + api = contract_api("msgspec") + api.max_body = 256 + app = mount_on(plug, api) + headers = {"authorization": "Bearer ann", "content-type": "application/json"} + received = 0 + + async def endless(): + nonlocal received + yield b'{"name": "' + for _ in range(10_000): + received += 1 + yield b"x" * 64 + + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") as client: + declared = await client.post("/api/widgets", content=b"x" * 1000, headers=headers) + assert declared.status_code == 413 and declared.json()["status"] == 413 + streamed = await client.post("/api/widgets", content=endless(), headers=headers) + assert streamed.status_code == 413 + ok = await client.post("/api/widgets", content=b'{"name": "fits"}', headers=headers) + assert ok.status_code == 201 + + +async def test_body_is_not_read_past_the_limit(): + """Driving the ASGI app directly shows that reading stops once the limit is passed.""" + api = contract_api("msgspec") + api.max_body = 256 + pulls = 0 + + async def receive(): + nonlocal pulls + pulls += 1 + return {"type": "http.request", "body": b"x" * 100, "more_body": True} + + sent = [] + + async def send(message): + sent.append(message) + + scope = { + "type": "http", + "method": "POST", + "path": "/api/widgets", + "query_string": b"", + "headers": [(b"authorization", b"Bearer ann"), (b"content-type", b"application/json")], + } + await api.asgi()(scope, receive, send) + assert sent[0]["status"] == 413 and pulls == 3 # 300 bytes read, then stopped + + +async def test_head_options_and_private_openapi(): + api = contract_api("msgspec") + created = await api.handle( + "POST", "/api/widgets", body=b'{"name":"w"}', headers={"authorization": "Bearer ann"} + ) + path = created.headers["location"] + get = await api.handle("GET", path, headers={"authorization": "Bearer ann"}) + head = await api.handle("HEAD", path, headers={"authorization": "Bearer ann"}) + assert head.status == 200 and head.body == b"" and head.headers["etag"] == get.headers["etag"] + options = await api.handle("OPTIONS", path) + assert options.status == 204 and options.headers["allow"] == "DELETE, GET, HEAD, OPTIONS, PATCH" + assert (await api.handle("OPTIONS", "/api/nope")).status == 404 + assert (await api.handle("HEAD", "/api/widgets/stats")).status == 200 + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=api.asgi()), base_url="http://t" + ) as client: + assert (await client.get("/api/openapi.json")).status_code == 200 + assert (await client.head(path, headers={"authorization": "Bearer ann"})).status_code == 200 + model = api.resources[0].model + private = Api(Resource(model), prefix="/api", openapi=False) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=private.asgi()), base_url="http://t" + ) as client: + assert (await client.get("/api/openapi.json")).status_code == 404 + assert "paths" in private.openapi() # still available to serve behind your own authentication + + +def test_explicit_writes_mode_is_default_deny(): + import typing as t + + from mongospecs import Policy + from mongospecs.msgspec import Spec + + model = contract_api("msgspec").resources[0].model + open_view = Resource(model).view + strict = Resource(model, explicit_writes=True).view + assert {"name", "price", "tags", "sku"} <= open_view.insertable + # nothing on the contract model opts in explicitly, so nothing is writable + assert strict.insertable == frozenset() and strict.updatable == frozenset() + assert strict.schema("create")["properties"] == {} + assert view_of(model, explicit_writes=True) is strict and view_of(model) is open_view + + class Opt(Spec): + title: t.Annotated[str, Policy(insert=True, update=True)] = "" + internal: str = "" + created_only: t.Annotated[str, Policy(insert=True)] = "" + + view = view_of(Opt, explicit_writes=True) + assert view.insertable == {"title", "created_only"} and view.updatable == {"title"} diff --git a/packages/mongospecs/src/mongospecs/manager.py b/packages/mongospecs/src/mongospecs/manager.py index dc8c0c2..23b98e4 100644 --- a/packages/mongospecs/src/mongospecs/manager.py +++ b/packages/mongospecs/src/mongospecs/manager.py @@ -354,7 +354,21 @@ async def _raise_unmatched(self, flt: dict[str, t.Any], session: t.Any) -> t.NoR ) raise DocumentNotFound(f"{self.model.__name__} {flt['_id']!r} does not exist") - async def delete_document(self, doc: T, *, session: t.Any = None, hard: bool = False) -> None: + async def delete_document( + self, + doc: T, + *, + session: t.Any = None, + hard: bool = False, + condition: t.Mapping[str, t.Any] | None = None, + if_rev: int | None = None, + ) -> None: + """Delete a document (softly when the model has a ``DeletedAt`` field, unless ``hard``). + + ``condition`` is an extra filter the document must still satisfy at the moment of the + delete (a row scope, for instance). With a ``Revision`` field the delete also requires + the revision that was loaded, or ``if_rev`` when given. + """ any_doc = t.cast(t.Any, doc) if any_doc.id is None: raise QueryError("cannot delete a document that has no id") @@ -363,7 +377,15 @@ async def delete_document(self, doc: T, *, session: t.Any = None, hard: bool = F ctx = self.ctx("delete", session, doc=doc, snapshot=snapshot) await self._hooks("before", ("delete",), ctx, doc) flt: dict[str, t.Any] = {"_id": snapshot["_id"], **self._shard_filter(snapshot)} + if if_rev is not None: + ctx.extra["if_rev"] = if_rev soft = None if hard else self._soft_delete(ctx, flt) + revision = next((f for f in self.meta.fields.values() if f.has(Revision)), None) + if revision is not None and (state is not None or if_rev is not None): + expected = if_rev if if_rev is not None else snapshot.get(revision.stored) + flt[revision.stored] = {"$in": [0, None]} if not expected else expected + if condition: + flt = and_filters(flt, condition) async def run(s: t.Any) -> None: if isinstance(s, Tx): diff --git a/packages/mongospecs/src/mongospecs/model.py b/packages/mongospecs/src/mongospecs/model.py index 65ee4f7..fcaf342 100644 --- a/packages/mongospecs/src/mongospecs/model.py +++ b/packages/mongospecs/src/mongospecs/model.py @@ -231,9 +231,13 @@ async def save(self: T, *, session: t.Any = None) -> T: """Insert a new document, or send only the fields that changed since it was loaded.""" return t.cast(T, await self._own_manager().save(self, session=session)) - async def delete(self, *, session: t.Any = None, hard: bool = False) -> None: - """Delete this document (a soft delete when the model has a ``DeletedAt`` field, unless ``hard``).""" - await self._own_manager().delete_document(self, session=session, hard=hard) + async def delete(self, *, session: t.Any = None, hard: bool = False, if_rev: int | None = None) -> None: + """Delete this document (a soft delete when the model has a ``DeletedAt`` field, unless ``hard``). + + With a ``Revision`` field the delete fails with ``StaleDocumentError`` if the document + changed since it was loaded (or since revision ``if_rev``). + """ + await self._own_manager().delete_document(self, session=session, hard=hard, if_rev=if_rev) async def reload(self: T, *, session: t.Any = None) -> T: """Refresh this document in place from the database.""" From e6957632aace35ec27ce0a8ffa6e2565dc25b885 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 02:32:13 -0400 Subject: [PATCH 09/26] docs,ci: review follow-ups - parity page: 28 of 35 Mongoose behaviours now confirmed against its documentation; three more deliberate differences recorded - plan amendments A25-A29: the computed decorator, Atlas Search and vector indexes declared out of scope for 0.2 (not testable here), revision-checked hard deletes, the changed web defaults - guides: limits and defaults of the web layer, untracked documents, partial bulk_save failures, registries and explicit managers - CI: MongoDB 7.0 on Python 3.11, 3.13 and 3.15 - publish workflow runs lint, types and the test suite first Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yaml | 6 +++++ .github/workflows/python-publish.yaml | 13 +++++++++ claudedocs/v2-plan.md | 22 ++++++++++++++++ docs/guide/operations.md | 7 +++++ docs/guide/writes.md | 12 +++++++++ docs/parity.md | 38 ++++++++++++++------------- docs/web/resources.md | 20 +++++++++++++- 7 files changed, 99 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3eef0a6..15fadcf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -33,9 +33,15 @@ jobs: mongo: ["8.0"] experimental: [false] include: + - python: "3.11" + mongo: "7.0" + experimental: false - python: "3.13" mongo: "7.0" experimental: false + - python: "3.15" + mongo: "7.0" + experimental: false - python: "3.14t" mongo: "8.0" experimental: true diff --git a/.github/workflows/python-publish.yaml b/.github/workflows/python-publish.yaml index 041b955..66a28d0 100644 --- a/.github/workflows/python-publish.yaml +++ b/.github/workflows/python-publish.yaml @@ -11,7 +11,20 @@ permissions: contents: read jobs: + verify: + # Nothing is published from a commit that does not pass the checks. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - run: make up + - run: uv sync --all-packages --python 3.13 + - run: make lint + - run: make types + - run: uv run pytest + publish: + needs: verify runs-on: ubuntu-latest environment: name: pypi diff --git a/claudedocs/v2-plan.md b/claudedocs/v2-plan.md index 8c69b01..b49734a 100644 --- a/claudedocs/v2-plan.md +++ b/claudedocs/v2-plan.md @@ -876,6 +876,11 @@ Recorded per §7.1 rule 4. Each one names what changed and why; none invalidates | A22 | §6 | Workload B7 (HTTP per plug) is the `LOAD` workload: a real server process under load, with server CPU per request as the primary metric | Measuring plugs through an in-process test client, and later through httpx over TCP, measured the client | | A23 | §4.5 | An upsert gives the inserted document the model's non-null defaults through `$setOnInsert` | Otherwise a document created by upsert differs in shape from one created by `insert()` and queries on defaulted values miss it (Mongoose: `setDefaultsOnInsert`) | | A24 | §4.2 | Subclass discovery ignores stale class copies | `attrs.define` replaces a class with a slotted copy and the original lingers until collected; found by the free-threaded CI job | +| A25 | §4.2 | The `Computed` marker became the `@computed` property decorator | A marker goes on an annotation; a computed value is a property and has none | +| A26 | §4.6 | Atlas Search and vector search indexes are not in 0.2. `Index` covers ordinary, compound, text, geospatial, TTL, partial and collated indexes | They need Atlas (or a `mongot` deployment) to create and to test; nothing here could verify them. Declared out of scope rather than shipped untested | +| A27 | §4.3 | A8 is superseded: deleting a loaded document checks its revision on hard deletes as well, and `delete(if_rev=)` / HTTP `If-Match` on DELETE are supported | Raised in review | +| A28 | §4.8 | Web defaults changed after review: totals are opt-in per resource, offset pages are capped by `max_offset`, request reads get a 10s time limit when nothing else sets one, bodies are size-limited while being read on every plug, the row scope is part of the DELETE filter, `Api(openapi=False)` and `Resource(explicit_writes=True)` exist | Raised in review | +| A29 | §4.5 | `bulk_save` writes are independent: successes are brought in sync in memory and the error lists the failures. `Query.update()` returns an int carrying `.matched` and `.modified`. Cursor pagination always fetches its sort keys. Cascades carry a visited set and work in chunks | Defects reproduced in review, each with a regression test | New node: @@ -899,3 +904,20 @@ server errors were not logged anywhere. | D1 | No Rust in 0.2. The only candidate worth a native component is a BSON-to-model decoder for bulk reads, as a separate optional package behind `Codec.decode(bytes)` | BSON-to-dict is essentially all of the driver's time on bulk reads (10.1 us per nested document against 3.3 us for msgspec's dict-to-model step) but an estimated 10-20% of server CPU on request-sized reads. Needs Jay's call on whether bulk reads matter enough | | Open decision 4 | Reads validate; the `trusted` option is removed | Measured slower than validating on pydantic, no effect elsewhere | | Open decision 6 | Tracked stays the default read mode | 0-15% over lean and about 1.2 KB per document after the raw-BSON snapshot change | + +### Review of the pull requests (2026-10-13) + +A second session reviewed #6-#8 and reproduced five defects (duplicate pages with `select()`, a bare-id +`Location` header, `bulk_save` leaving memory out of sync, `bulk_write` skipping cascades, an unhelpful +error when saving a copy). All five are fixed with regression tests, along with an infinite recursion on +cyclic cascades found while reproducing them. Its read-only findings on the web layer are addressed in A28. + +Known and not changed: + +- `registry._BINDINGS` and `tracking._states` are process-global. `Model.find()` resolves the registry a + model is bound to, so one model class cannot be bound to two open registries at once; code that needs + several registries or one registry per thread uses `registry.manager(Model)`. A task-scoped "current + registry" was suggested and is not built. +- S7 (confirming every Mongoose parity row against its documentation) is done for 28 of 35 rows; + `docs/parity.md` marks the 7 that are still from memory (alias, `useDb`, per-query driver options, + `exists`/counts, subdocuments and maps, getters/setters, virtuals). diff --git a/docs/guide/operations.md b/docs/guide/operations.md index 0feccdf..536ac7f 100644 --- a/docs/guide/operations.md +++ b/docs/guide/operations.md @@ -75,6 +75,13 @@ mongospecs raises its own errors and keeps the driver error as `__cause__`. | `OperationTimeout`, `ConnectionFailed`, `PoolTimeout` | the database is slow or unreachable | | `DocumentTooLarge` | a document or pipeline result exceeded 16MB | +## One registry per model, or explicit managers + +`Model.find()` and the other class methods use the registry the model is bound to, and a model +class can be bound to only one open registry at a time. When you need the same model on several +registries, or one registry per thread, ask the registry for a manager instead: +`registry.manager(Article).find(...)`. Documents loaded through a manager are saved through it. + ## More than one database ```python diff --git a/docs/guide/writes.md b/docs/guide/writes.md index 054de90..2eee8c4 100644 --- a/docs/guide/writes.md +++ b/docs/guide/writes.md @@ -50,6 +50,11 @@ await article.save() assert article.changes() == {} and article.rev == 1 ``` +A document you did not get from a tracked read has no snapshot: a `lean()` result, a +`copy.copy()` or a deep copy of a tracked document, or one that was unpickled. Saving one tries to +insert it and fails with a `DuplicateKeyError` that says so. Use a tracked read, or +`Model.hydrate(raw)` to adopt a raw document. + Nested documents are diffed into dotted paths. A changed list is sent whole; for element-level changes that must not overwrite concurrent writers, use the atomic operators below. @@ -115,6 +120,13 @@ assert (await Article.find(Q.title == "bulk 0").one()).views == 6 `insert_many(..., ordered=False)` attempts every document and reports failures per item instead of raising on the first. +The writes in a `bulk_save` are independent. If some fail (a duplicate key, a revision that moved +on), the rest are still written and are in sync in memory; the error raised afterwards lists the +documents that were not written in `.failed`. + +`Model.find(...).update(...)` returns the number of documents modified, as an `int` that also +carries `.matched` and `.modified`. + ## Transactions `registry.transaction(fn)` commits when `fn` returns. The driver re-runs `fn` on transient errors, diff --git a/docs/parity.md b/docs/parity.md index 4a38ccb..a16300d 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -12,41 +12,41 @@ was established: **docs** means it was re-read in the Mongoose documentation for | Mongoose | mongospecs | Covered by | Source | |---|---|---|---| -| Change tracking, delta `save()`, `isNew`, `modifiedPaths()` | snapshot at load; `save()` sends the diff; `doc.is_new`, `doc.changes()` | `test_crud.py::test_save_new_then_delta`, `test_tracking_tx.py` | memory | +| Change tracking, delta `save()`, `isNew`, `modifiedPaths()` | snapshot at load; `save()` sends the diff; `doc.is_new`, `doc.changes()` | `test_crud.py::test_save_new_then_delta`, `test_tracking_tx.py` | docs | | `lean()` | `.lean()` (models, untracked), `.raw()` (dicts) | `test_crud.py::test_read_modes` | docs | -| `Model.hydrate()` | `Model.hydrate(raw)` | `test_crud.py::test_hydrate_and_collection_escape_hatch` | memory | +| `Model.hydrate()` | `Model.hydrate(raw)` | `test_crud.py::test_hydrate_and_collection_escape_hatch` | docs | | Middleware: document, query, aggregate, model | document hooks; `on_insert`/`on_update`/`on_filter`/`on_delete`; `on_pipeline`; `before_bulk`/`after_bulk` | `test_plugins_hooks.py` | docs | -| `schema.plugin()`, `mongoose.plugin()` | marker plugins, `SpecConfig(plugins=)`, `Registry(plugins=)` | `test_plugins_hooks.py::test_registry_level_plugin_applies_to_every_model` | memory | -| `timestamps` | `CreatedAt`, `UpdatedAt` | `test_plugins_hooks.py::test_timestamps_on_every_write_path` | memory | +| `schema.plugin()`, `mongoose.plugin()` | marker plugins, `SpecConfig(plugins=)`, `Registry(plugins=)` | `test_plugins_hooks.py::test_registry_level_plugin_applies_to_every_model` | docs | +| `timestamps` | `CreatedAt`, `UpdatedAt` | `test_plugins_hooks.py::test_timestamps_on_every_write_path` | docs | | `optimisticConcurrency`, `versionKey` | `Revision` | `test_plugins_hooks.py::test_revision_increments_everywhere` | docs | | `populate()` with `match`, `select`, `options`, `perDocumentLimit`, nested paths | `include()` with `match`, `select`, `sort`, `limit`, `per_parent_limit`, dotted paths | `test_relations.py` | docs | | Virtual populate (`localField`/`foreignField`, `count`) | `Backref(...)`, `Backref(..., count=True)` | `test_relations.py::test_backrefs_and_counts` | docs | -| `refPath` | `Ref[A \| B]` | `test_relations.py::test_polymorphic_reference` | memory | +| `refPath` | `Ref[A \| B]` | `test_relations.py::test_polymorphic_reference` | docs | | Discriminators | `SpecConfig(discriminator=...)` | `test_models_misc.py::test_discriminators_share_a_collection` | docs | -| Query casting | values cast to the field type | `test_expr.py::test_values_are_cast_to_field_types` | memory | +| Query casting | values cast to the field type | `test_expr.py::test_values_are_cast_to_field_types` | docs | | `strictQuery: 'throw'` | always: unknown fields raise | `test_expr.py::test_field_names_are_translated_and_checked` | docs | -| `sanitizeFilter`, `trusted()` | raw filters untrusted unless `trusted()` | `test_expr.py::test_untrusted_mappings_and_operator_injection` | memory | +| `sanitizeFilter`, `trusted()` | raw filters untrusted unless `trusted()` | `test_expr.py::test_untrusted_mappings_and_operator_injection` | docs | | `select: false` | `Deferred` | `test_crud.py::test_deferred_field_is_not_loaded_or_clobbered` | docs | -| `immutable` | `Immutable` | `test_updates.py::test_immutable_and_required_fields_are_protected` | memory | +| `immutable` | `Immutable` | `test_updates.py::test_immutable_and_required_fields_are_protected` | docs | | `alias` | `Alias("x")` | `test_codec.py::test_alias_and_transient_fields` | memory | -| Query helpers | `@Model.scope` | `test_plugins_hooks.py::test_named_scopes` | memory | -| `orFail()` | `.or_fail()`, `.one()` | `test_crud.py::test_find_all_first_one_and_or_fail` | memory | -| `cursor()`, `eachAsync({parallel})` | `async for`, `.each(fn, concurrency=n)` | `test_crud.py::test_each_with_concurrency` | memory | +| Query helpers | `@Model.scope` | `test_plugins_hooks.py::test_named_scopes` | docs | +| `orFail()` | `.or_fail()`, `.one()` | `test_crud.py::test_find_all_first_one_and_or_fail` | docs | +| `cursor()`, `eachAsync({parallel})` | `async for`, `.each(fn, concurrency=n)` | `test_crud.py::test_each_with_concurrency` | docs | | `bulkWrite`, `bulkSave`, `insertMany({ordered: false})` | `bulk_write`, `bulk_save`, `insert_many(ordered=False)` | `test_bulk_pagination.py` | docs | | `Connection.transaction(fn)` resets document state on abort | `registry.transaction(fn)` does the same, also before each retry | `test_tracking_tx.py::test_transaction_abort_rolls_back_database_and_memory` | docs | -| `syncIndexes`, `diffIndexes`, `autoIndex: false` | `plan_indexes`, `apply_indexes(drop=...)`; never automatic | `test_indexes.py` | memory | +| `syncIndexes`, `diffIndexes`, `autoIndex: false` | `plan_indexes`, `apply_indexes(drop=...)`; never automatic | `test_indexes.py` | docs | | `shardKey` | `SpecConfig(shard_key=...)` | `test_streams_sharded.py::test_shard_key_is_added_to_document_writes` | docs | | `createConnection`, `useDb` | several registries; `Model.using(db=...)` | `test_models_misc.py::test_using_another_database` | memory | | `maxTimeMS`, `hint`, `comment`, `collation`, read/write concern | `.options(...)`, `SpecConfig`, `Registry(max_time_ms=)` | `test_registry_errors.py::test_default_max_time_ms_from_registry_and_model` | memory | -| `Model.aggregate()`, `Model.collection` | `Model.aggregate(pipeline, as_=)`, `Model.collection()` | `test_crud.py::test_aggregate_decodes_and_explain` | memory | -| `mongoose.set('debug')` | `Registry(on_command=...)` with logging, OpenTelemetry, Sentry listeners | `test_registry_errors.py::test_command_events_carry_model_and_operation` | memory | -| `Model.watch()` | `Model.watch()` | `test_streams_sharded.py::test_change_stream_events` | memory | +| `Model.aggregate()`, `Model.collection` | `Model.aggregate(pipeline, as_=)`, `Model.collection()` | `test_crud.py::test_aggregate_decodes_and_explain` | docs | +| `mongoose.set('debug')` | `Registry(on_command=...)` with logging, OpenTelemetry, Sentry listeners | `test_registry_errors.py::test_command_events_carry_model_and_operation` | docs | +| `Model.watch()` | `Model.watch()` | `test_streams_sharded.py::test_change_stream_events` | docs | | `exists`, `countDocuments`, `estimatedDocumentCount` | `exists`, `count`, `estimated_count` | `test_crud.py::test_count_exists_ids_estimated` | memory | -| Capped, time-series, TTL options | `SpecConfig(collection_options=)`, `Index(ttl=)` | `test_indexes.py::test_collection_options_are_applied` | memory | +| Capped, time-series, TTL options | `SpecConfig(collection_options=)`, `Index(ttl=)` | `test_indexes.py::test_collection_options_are_applied` | docs | | Subdocuments, arrays of subdocuments, Maps | nested models, `list[T]`, `dict[str, T]`; tracked through the parent | `test_crud.py::test_nested_document_diff_uses_dotted_paths` | memory | | Getters, setters, custom validators | your backend's converters and validators | `test_codec.py::test_validate_field_runs_the_models_own_validators` | memory | | Virtuals | Python properties; `@computed` to include one in API output | `packages/mongospecs-web/tests/test_mongo_resource.py` | memory | -| Statics and methods | classmethods and methods | (plain Python) | memory | +| Statics and methods | classmethods and methods | (plain Python) | docs | ## Deliberate differences @@ -65,6 +65,8 @@ was established: **docs** means it was re-read in the Mongoose documentation for | `transactionAsyncLocalStorage` offers an ambient session | Sessions are always passed explicitly | Tasks spawned inside a transaction would silently share one session | | Populated paths replace the id with the document | A `Ref` keeps `.id` and gains `.doc` | One type whether or not it is loaded | | Arrays are tracked per element | A changed list is sent whole | Per-element intent is expressed with atomic operators | -| A document's delete is not version-checked | Soft delete and `save()` are revision-checked; hard delete is not | Recorded as a known gap | +| `Model.watch()` errors out when it loses the primary | The change stream resumes across a step-down (the driver's behaviour, verified by a stress scenario) | Consumers should not need their own reconnect loop | +| `syncIndexes()` drops every undeclared index | `apply_indexes()` only drops or recreates with `drop=True` | Dropping an index is not something a deploy should do by accident | +| `orFail()` on an update throws when nothing was modified | `Model.update` raises when nothing matched; `find(...).update(...)` returns the matched and modified counts | A no-op update that matched is not an error | | `populate()` always runs a separate query per path | `include()` does the same by default and can use `$lookup` instead (`strategy="lookup"`) | Both give the same result; the faster one depends on fan-out, see the benchmarks | | `limit` in `populate()` is applied across all parents unless `perDocumentLimit` is used | `limit` is overall, `per_parent_limit` is per parent, without one query per parent | Same two meanings, named for what they do | diff --git a/docs/web/resources.md b/docs/web/resources.md index 3ab162d..ec3890c 100644 --- a/docs/web/resources.md +++ b/docs/web/resources.md @@ -124,8 +124,26 @@ await registry.close() ?sort=-views,title ?limit=20&after= cursor pagination (default) ?limit=20&page=3 offset pagination, with page=Offset(...) -?total=true also count all matches +?total=true also count all matches, with page=Cursor(total=True) ?include=author embed allowlisted relations ``` Only fields and operators the policy allows are accepted; everything else is a 400. + +## Limits and defaults + +These are set so that a client cannot make the server do unbounded work. + +| Setting | Default | Meaning | +|---|---|---| +| `Api(max_body=...)` | 1 MiB | Larger request bodies get a 413. The body is never buffered past the limit, on any plug | +| `Cursor(max=...)`, `Offset(max=...)` | 100 | Largest page a client can ask for | +| `Offset(max_offset=...)` | 10,000 | How many documents a page may skip; deeper pages get a 400 | +| `Cursor(total=...)`, `Offset(total=...)` | `False` | Whether `?total=true` is allowed (a count is a second query over every match) | +| `MongoBackend(max_time_ms=...)` | 10 s | Time limit for reads made for a request, when the registry and the model set none | +| `Api(openapi=...)` | `True` | Serve `/openapi.json`; turn off to keep the description private | +| `Resource(explicit_writes=...)` | `False` | When `True`, a field is writable only if its `Policy` says `insert=True` / `update=True` | + +`DELETE` honours `If-Match` like `PATCH`, and the row scope is part of the delete's own filter. +`HEAD` and `OPTIONS` are answered by `api.handle` and the pure ASGI plug; the framework plugs leave +them to the framework. From a78acb2e3df7ff3544791ee5b2d8e8093039af01 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 02:44:32 -0400 Subject: [PATCH 10/26] fix(core,web): defects from the second review of v2 Each has a regression test that failed before the change (tests/test_review_round2.py in both packages). core - a field whose default is None is not stored while it is None, and setting it to None unsets it; a unique sparse index no longer collides on stored nulls - field names that shadow the Document API raise ConfigurationError - registry.transaction() inside a running transaction raises unless independent=True (it would commit on its own) - UpdateCount formats as its int value in str() and f-strings - filter values are cast by one rule on every backend - empty Or()/Nor(), non-text prefix()/contains(), sort keys that are not field paths and server type mismatches raise QueryError - include() checks its path and strategy when the query is built - find().on_invalid('raise' | 'skip' | 'raw') web - integers beyond 64 bits are a 422 on create and patch, and an EncodingError is no longer a 500 - Quart: init_app installs BodyLimit, which reads API bodies under the limit before Quart queues them - a guard runs once per request however many places list it - OpenAPI publishes only referenced definitions and no private field - HEAD on the ASGI plug reports the length of the GET body BREAKING: optional fields that are None are omitted from stored documents, so an exists filter now means 'has a value'. Co-Authored-By: Claude Opus 5.5 --- claudedocs/v2-plan.md | 4 + docs/guide/models.md | 11 + docs/guide/queries.md | 12 ++ docs/guide/writes.md | 9 + docs/web/plugs.md | 19 ++ docs/web/resources.md | 5 + .../mongospecs-web/src/mongospecs_web/api.py | 4 +- .../src/mongospecs_web/errors.py | 4 + .../src/mongospecs_web/plugs/asgi.py | 5 +- .../src/mongospecs_web/plugs/quart.py | 68 +++++- .../src/mongospecs_web/policy.py | 71 ++++++- .../src/mongospecs_web/resource.py | 6 +- .../mongospecs-web/tests/test_contract.py | 6 +- .../tests/test_review_round2.py | 159 ++++++++++++++ packages/mongospecs/src/mongospecs/codec.py | 25 +++ packages/mongospecs/src/mongospecs/errors.py | 7 + packages/mongospecs/src/mongospecs/expr.py | 22 +- packages/mongospecs/src/mongospecs/manager.py | 9 + packages/mongospecs/src/mongospecs/meta.py | 53 ++++- packages/mongospecs/src/mongospecs/query.py | 74 +++++-- .../mongospecs/src/mongospecs/registry.py | 22 ++ .../mongospecs/src/mongospecs/relations.py | 22 ++ packages/mongospecs/src/mongospecs/updates.py | 6 +- .../mongospecs/tests/test_review_round2.py | 196 ++++++++++++++++++ 24 files changed, 780 insertions(+), 39 deletions(-) create mode 100644 packages/mongospecs-web/tests/test_review_round2.py create mode 100644 packages/mongospecs/tests/test_review_round2.py diff --git a/claudedocs/v2-plan.md b/claudedocs/v2-plan.md index b49734a..9d209c9 100644 --- a/claudedocs/v2-plan.md +++ b/claudedocs/v2-plan.md @@ -881,6 +881,10 @@ Recorded per §7.1 rule 4. Each one names what changed and why; none invalidates | A27 | §4.3 | A8 is superseded: deleting a loaded document checks its revision on hard deletes as well, and `delete(if_rev=)` / HTTP `If-Match` on DELETE are supported | Raised in review | | A28 | §4.8 | Web defaults changed after review: totals are opt-in per resource, offset pages are capped by `max_offset`, request reads get a 10s time limit when nothing else sets one, bodies are size-limited while being read on every plug, the row scope is part of the DELETE filter, `Api(openapi=False)` and `Resource(explicit_writes=True)` exist | Raised in review | | A29 | §4.5 | `bulk_save` writes are independent: successes are brought in sync in memory and the error lists the failures. `Query.update()` returns an int carrying `.matched` and `.modified`. Cursor pagination always fetches its sort keys. Cascades carry a visited set and work in chunks | Defects reproduced in review, each with a regression test | +| A30 | §4.2 | A field whose default is `None` is not stored while it is `None` (the key is omitted; setting it to `None` unsets it) | Second review: a unique sparse index collided on stored nulls. Consequence: `exists` filters mean "has a value" | +| A31 | §4.2 | Field names that shadow the `Document` API raise `ConfigurationError` (`meta.RESERVED_FIELD_NAMES`) | Second review: a field named `count` silently replaced `Model.count()` | +| A32 | §4.4 | `registry.transaction()` inside a running transaction raises unless `independent=True`. A context variable is used to detect this only; sessions are still always passed explicitly | Second review: an inner transaction committed while the outer aborted | +| A33 | §4.3 | Filter values are cast by one rule on all backends (exact type, else text that reads as the type). `find().on_invalid("raise" \| "skip" \| "raw")` | Second review | New node: diff --git a/docs/guide/models.md b/docs/guide/models.md index c6b9f87..26d2af6 100644 --- a/docs/guide/models.md +++ b/docs/guide/models.md @@ -26,6 +26,17 @@ class User(Spec): - `__spec__` is assigned **without an annotation**. It holds the collection name (default: the class name), indexes, and per-model options. - With attrs, decorate each model with `@attrs.define(kw_only=True)`. +- A field whose default is `None` is **not stored while it is `None`**: the key is left out of the + document, and a document without the key reads back as `None`. Setting such a field to `None` + removes the key (`$unset`). This is what makes `Index("email", unique=True, sparse=True)` work + for documents without an email: MongoDB indexes an explicit `null`, so two stored nulls would + collide. `Q.email == None` still finds them (it matches a missing key), and `Exists(Q.email)` + means "has a value". A field with any other default stores `None` as `null`. +- A field may not be named after part of the document API (`count`, `find`, `get`, `update`, + `delete`, `save`, `insert`, `scope`, `watch`, `changes`, `is_new`, ... the full list is + `mongospecs.meta.RESERVED_FIELD_NAMES`); it would replace that method on the class. Doing so + raises `ConfigurationError` when the model is first used. Rename the attribute and keep the + stored key: `total: Annotated[int, Alias("count")] = 0`. ## Binding and lifecycle diff --git a/docs/guide/queries.md b/docs/guide/queries.md index 5519d44..fab9699 100644 --- a/docs/guide/queries.md +++ b/docs/guide/queries.md @@ -54,8 +54,20 @@ assert await Article.count({"title": "post 3"}) == 1 assert await Article.count(trusted({"views": {"$mod": [20, 0]}})) == 5 ``` +Filter values are cast to the field's type the same way on every backend: a value of the right +type is used as it is, and text that reads as the type is accepted (`Q.age == "12"` matches the +number 12). Anything else raises `QueryError`, as do an empty `Or()`/`Nor()`, a non-text argument +to `prefix()`/`contains()`, and a sort key that is not a field path. Values given to `update()` +and to a model's constructor follow the model library's own rules instead (pydantic coerces +`"7"` to `7`; msgspec and attrs do not). + ## Results +By default one stored document that does not fit the model fails the whole read with +`ValidationError`. `find().on_invalid("skip")` leaves such documents out, and +`find().on_invalid("raw")` returns them in place as plain dicts so they can be found and +repaired. + ```python first = await Article.find(Q.views > 0).sort("views").first() # a document, or None one = await Article.find(Q.title == "post 4").one() # raises DocumentNotFound if missing diff --git a/docs/guide/writes.md b/docs/guide/writes.md index 2eee8c4..23b7e9f 100644 --- a/docs/guide/writes.md +++ b/docs/guide/writes.md @@ -90,6 +90,9 @@ except StaleDocumentError: assert stale.title == "Final v2" ``` +A document created by `upsert()` starts at revision 1, one created by `insert()` at 0: an upsert +is an update, and every update increments the revision. + ## Soft delete With a `DeletedAt` field, `delete()` hides the document instead of removing it. @@ -133,6 +136,12 @@ carries `.matched` and `.modified`. so it must be safe to call twice. If the transaction aborts, documents written inside it are put back to their previous in-memory state. +**Transactions do not nest.** MongoDB has no savepoints: a transaction started inside another is +a separate one that commits on its own and stays committed if the outer one aborts. Calling +`registry.transaction()` inside a running one raises `TransactionError`; pass the outer `tx` to +the code that needs it. When a separate commit is what you want (an audit record that must +survive a rollback), say so: `registry.transaction(fn, independent=True)`. + ```python landing = await Article.find(Q.title == "Landing").one() diff --git a/docs/web/plugs.md b/docs/web/plugs.md index 6001f92..b7a2a6d 100644 --- a/docs/web/plugs.md +++ b/docs/web/plugs.md @@ -58,6 +58,25 @@ app = Quart(__name__) plug.init_app(app, api, registry=registry) ``` +Quart queues every body chunk as it arrives, before a view asks for it, so `init_app` also wraps +the app in `plug.BodyLimit`, which reads API request bodies under `api.max_body` before Quart +sees them. If you register `plug.blueprint(api)` yourself, add +`app.asgi_app = plug.BodyLimit(app.asgi_app, api)`. + +## What the frameworks decide themselves + +The plugs register one route per operation and leave the rest of routing to the framework, so +three things differ between them. The contract suite does not cover them. + +| | Pure ASGI | Starlette | FastAPI | Litestar | Quart | +|---|---|---|---|---|---| +| Trailing slash (`/widgets/`) | 200 | 307 redirect | 307 redirect | 200 | 404 | +| `HEAD` on a `GET` route | 200, with GET's headers and length | 200 | 405 | 405 | 200 | +| `OPTIONS` | 204 with `Allow` | 405 | 405 | 204 | 200 | + +Use the framework's own settings (or its CORS middleware for `OPTIONS`) where you need one +behaviour everywhere. + ## Writing another plug Convert the framework's request to `mongospecs_web.Request`, call `await api.run(route, request)` diff --git a/docs/web/resources.md b/docs/web/resources.md index ec3890c..dd83ba6 100644 --- a/docs/web/resources.md +++ b/docs/web/resources.md @@ -144,6 +144,11 @@ These are set so that a client cannot make the server do unbounded work. | `Api(openapi=...)` | `True` | Serve `/openapi.json`; turn off to keep the description private | | `Resource(explicit_writes=...)` | `False` | When `True`, a field is writable only if its `Policy` says `insert=True` / `update=True` | +An integer outside the 64-bit range MongoDB can store is a 422 with a field error, on create and +on patch (a float or Decimal field takes it as that type). A guard listed in more than one place +runs once per request. Private fields appear nowhere in `openapi.json`, and it only publishes +the nested definitions that the Read, Create and Patch schemas refer to. + `DELETE` honours `If-Match` like `PATCH`, and the row scope is part of the delete's own filter. `HEAD` and `OPTIONS` are answered by `api.handle` and the pure ASGI plug; the framework plugs leave them to the framework. diff --git a/packages/mongospecs-web/src/mongospecs_web/api.py b/packages/mongospecs-web/src/mongospecs_web/api.py index 83dfbb0..d75e905 100644 --- a/packages/mongospecs-web/src/mongospecs_web/api.py +++ b/packages/mongospecs-web/src/mongospecs_web/api.py @@ -180,7 +180,9 @@ async def handle( route, Request(route.method, params, query or [], headers or {}, body, None, raw) ) if method == "HEAD": - response.body = b"" # same status and headers as GET, no body + # Same status and headers as GET, including the length of the body GET would send. + response.headers.setdefault("content-length", str(len(response.body))) + response.body = b"" return response def openapi(self) -> dict[str, t.Any]: diff --git a/packages/mongospecs-web/src/mongospecs_web/errors.py b/packages/mongospecs-web/src/mongospecs_web/errors.py index 0724f78..9abb1cf 100644 --- a/packages/mongospecs-web/src/mongospecs_web/errors.py +++ b/packages/mongospecs-web/src/mongospecs_web/errors.py @@ -113,6 +113,10 @@ def problem(exc: BaseException) -> HTTPError: "A resource with the same unique value already exists.", errors=[{"field": f, "message": "must be unique"} for f in fields] or None, ) + if isinstance(exc, me.EncodingError): + # A value that passed validation but has no BSON form (an integer beyond 64 bits inside + # an untyped field, say). The request is at fault, so this is not a 500. + return Unprocessable("The request contains a value that cannot be stored.") if isinstance(exc, me.QueryError): return BadRequest(str(exc)) if isinstance(exc, me.OperationTimeout): diff --git a/packages/mongospecs-web/src/mongospecs_web/plugs/asgi.py b/packages/mongospecs-web/src/mongospecs_web/plugs/asgi.py index 00b8ebd..0051232 100644 --- a/packages/mongospecs-web/src/mongospecs_web/plugs/asgi.py +++ b/packages/mongospecs-web/src/mongospecs_web/plugs/asgi.py @@ -88,8 +88,9 @@ async def chunks() -> t.AsyncIterator[bytes]: async def _send(send: Send, response: Response) -> None: headers = [(k.lower().encode("latin-1"), v.encode("latin-1")) for k, v in response.headers.items()] - if response.body or response.status not in (204, 304): + if response.body or response.status not in (204, 304): # includes the answer to a HEAD headers.append((b"content-type", response.media_type.encode("latin-1"))) - headers.append((b"content-length", str(len(response.body)).encode("latin-1"))) + if "content-length" not in response.headers: # HEAD carries the length of the GET body + headers.append((b"content-length", str(len(response.body)).encode("latin-1"))) await send({"type": "http.response.start", "status": response.status, "headers": headers}) await send({"type": "http.response.body", "body": response.body}) diff --git a/packages/mongospecs-web/src/mongospecs_web/plugs/quart.py b/packages/mongospecs-web/src/mongospecs_web/plugs/quart.py index 4d87c96..5895304 100644 --- a/packages/mongospecs-web/src/mongospecs_web/plugs/quart.py +++ b/packages/mongospecs-web/src/mongospecs_web/plugs/quart.py @@ -10,9 +10,10 @@ from mongospecs_web.api import Api, Route from mongospecs_web.errors import HTTPError +from mongospecs_web.plugs.asgi import _send from mongospecs_web.resource import Request, Response -__all__ = ["blueprint", "init_app"] +__all__ = ["BodyLimit", "blueprint", "init_app"] def _to_quart(response: Response) -> QuartResponse: @@ -64,9 +65,72 @@ async def spec() -> dict[str, t.Any]: return bp +class BodyLimit: + """ASGI middleware: read request bodies for the API under its size limit, before Quart does. + + Quart moves every chunk a client sends into an unbounded queue as it arrives, whether or not + the view has asked for it, so a limit applied inside a view comes too late. This reads the + body first (never more than ``api.max_body`` bytes), answers 413 itself when it is too + large, and otherwise hands Quart the body it read. + """ + + def __init__(self, app: t.Any, api: Api) -> None: + self.app = app + self.api = api + + async def __call__(self, scope: t.Any, receive: t.Any, send: t.Any) -> None: + if scope["type"] != "http" or not self._ours(scope): + await self.app(scope, receive, send) + return + headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope.get("headers", [])} + gone = False + + async def chunks() -> t.AsyncIterator[bytes]: + nonlocal gone + while True: + message = await receive() + if message["type"] == "http.disconnect": + gone = True + return + yield message.get("body", b"") + if not message.get("more_body"): + return + + try: + body = await self.api.read_body(headers, chunks()) + except HTTPError as exc: + await _send(send, Response.error(exc)) + return + if gone: + return + replayed = False + + async def replay() -> t.Any: + nonlocal replayed + if replayed: + return await receive() + replayed = True + return {"type": "http.request", "body": body, "more_body": False} + + await self.app(scope, replay, send) + + def _ours(self, scope: t.Any) -> bool: + path: str = scope.get("path", "") + root: str = scope.get("root_path", "") + if root and path.startswith(root): + path = path[len(root) :] or "/" + return path == self.api.prefix or path.startswith(self.api.prefix + "/") + + def init_app(app: Quart, api: Api, *, registry: t.Any = None, name: str = "mongospecs") -> None: - """Register the API on a Quart app; with ``registry=`` also open and close it with the server.""" + """Register the API on a Quart app; with ``registry=`` also open and close it with the server. + + Also installs :class:`BodyLimit`, so request bodies for the API are never read past + ``api.max_body``. If you register :func:`blueprint` yourself, wrap the app the same way: + ``app.asgi_app = BodyLimit(app.asgi_app, api)``. + """ app.register_blueprint(blueprint(api, name=name)) + app.asgi_app = BodyLimit(app.asgi_app, api) # type: ignore[method-assign] if registry is not None: app.before_serving(registry.open) app.after_serving(registry.close) diff --git a/packages/mongospecs-web/src/mongospecs_web/policy.py b/packages/mongospecs-web/src/mongospecs_web/policy.py index e2b31cb..415886f 100644 --- a/packages/mongospecs-web/src/mongospecs_web/policy.py +++ b/packages/mongospecs-web/src/mongospecs_web/policy.py @@ -20,6 +20,53 @@ _MANAGED = (CreatedAt, UpdatedAt, DeletedAt, Revision) +INT64_MIN, INT64_MAX = -(2**63), 2**63 - 1 +"""MongoDB's largest integer type is 64 bits; a larger integer cannot be stored as one.""" + + +def _refs(node: t.Any, out: set[str]) -> None: + if isinstance(node, dict): + for key, value in node.items(): + if key == "$ref" and isinstance(value, str) and value.startswith("#/$defs/"): + out.add(value.rsplit("/", 1)[1]) + else: + _refs(value, out) + elif isinstance(node, list): + for value in node: + _refs(value, out) + + +def _reachable(schema: dict[str, t.Any], defs: dict[str, t.Any]) -> dict[str, t.Any]: + """The definitions ``schema`` actually refers to, directly or through one another.""" + found: set[str] = set() + _refs(schema, found) + pending = list(found) + while pending: + inner: set[str] = set() + _refs(defs.get(pending.pop(), {}), inner) + pending.extend(inner - found) + found |= inner + return {name: definition for name, definition in defs.items() if name in found} + + +def _has_oversized_int(value: t.Any) -> bool: + if isinstance(value, int): + return not INT64_MIN <= value <= INT64_MAX + if isinstance(value, list): + return any(_has_oversized_int(v) for v in value) + if isinstance(value, dict): + return any(_has_oversized_int(v) for v in value.values()) + return False + + +def _takes_any_number(tp: t.Any) -> bool: + """A float or Decimal field: a large JSON integer becomes one of those and fits.""" + import decimal + + from mongospecs.codec import strip_optional + + return strip_optional(tp) in (float, decimal.Decimal) + class View: """What of a model is readable, writable, filterable and sortable from outside.""" @@ -125,12 +172,29 @@ def check_input(self, data: t.Any, allowed: frozenset[str], action: str) -> dict ) if errors: raise Unprocessable("The request body names fields that cannot be written.", errors=errors) + for key, value in data.items(): + if _has_oversized_int(value) and not _takes_any_number(self.meta.fields[key].type): + errors.append( + {"field": key, "message": f"integers must be between {INT64_MIN} and {INT64_MAX}"} + ) + if errors: + raise Unprocessable("The request body failed validation.", errors=errors) return data # -- schemas ---------------------------------------------------------------- def _base(self) -> dict[str, t.Any]: - return self.meta.codec.json_schema(self.model) + base = self.meta.codec.json_schema(self.model) + own = base.get("$defs", {}).get(self.model.__name__) + if isinstance(own, dict) and self.private: + # The model's own definition survives only where the model refers to itself; + # it must not describe more than the Read view does. + hidden = self.private | {self.meta.fields[name].stored for name in self.private} + own = dict(own) + own["properties"] = {k: v for k, v in own.get("properties", {}).items() if k not in hidden} + own["required"] = [k for k in own.get("required", []) if k not in hidden] + base = {**base, "$defs": {**base["$defs"], self.model.__name__: own}} + return base def schema(self, kind: t.Literal["read", "create", "patch"]) -> dict[str, t.Any]: """JSON Schema of one view. ``$defs`` of nested models are kept alongside.""" @@ -158,8 +222,9 @@ def schema(self, kind: t.Literal["read", "create", "patch"]) -> dict[str, t.Any] schema["required"] = [n for n in required if n in allowed] if not schema.get("required"): schema.pop("required", None) - if "$defs" in base: - schema["$defs"] = base["$defs"] + defs = _reachable(schema, base.get("$defs", {})) + if defs: + schema["$defs"] = defs return schema diff --git a/packages/mongospecs-web/src/mongospecs_web/resource.py b/packages/mongospecs-web/src/mongospecs_web/resource.py index ca98619..e870a71 100644 --- a/packages/mongospecs-web/src/mongospecs_web/resource.py +++ b/packages/mongospecs-web/src/mongospecs_web/resource.py @@ -333,12 +333,16 @@ async def dispatch(self, op: Operation, request: Request) -> Response: """Run one operation. Never raises: every error becomes a problem+json response.""" ctx: Ctx[T] = Ctx(self, op, request) try: + ran: set[int] = set() for guard in ( *self._all_guards, *self._guards.get(op.kind, ()), - *self._guards.get(op.name, ()), + *self._guards.get(op.name, ()), # the same key as the kind for built-in operations *op.guards, ): + if id(guard) in ran: + continue # one guard is one check, however many places list it + ran.add(id(guard)) if not await _maybe(guard(ctx)): raise Forbidden("You are not allowed to do this.") for hook in self._before.get(op.name, ()): diff --git a/packages/mongospecs-web/tests/test_contract.py b/packages/mongospecs-web/tests/test_contract.py index 6646154..37e4029 100644 --- a/packages/mongospecs-web/tests/test_contract.py +++ b/packages/mongospecs-web/tests/test_contract.py @@ -138,10 +138,8 @@ def names(label): assert names("filter eq") == ["bolt"] and names("filter prefix") == ["anvil"] assert names("filter in") == ["anvil", "cog"] and names("filter range") == ["cog", "anvil"] assert names("filter int cast") == ["anvil", "cog"] and names("filter exists") == [ - "anvil", - "bolt", - "cog", - ] # null counts as existing, as in MongoDB + "anvil" + ] # an optional field without a value is not stored, so only the widget with a sku has one assert got["filter ne with total"]["body"]["page"]["total"] == 2 assert got["limit is capped"]["body"]["page"]["size"] == 100 assert names("offset page 1") == ["anvil", "bolt"] diff --git a/packages/mongospecs-web/tests/test_review_round2.py b/packages/mongospecs-web/tests/test_review_round2.py new file mode 100644 index 0000000..e38af9b --- /dev/null +++ b/packages/mongospecs-web/tests/test_review_round2.py @@ -0,0 +1,159 @@ +"""Regression tests for the second QA review of v2 (issue numbers refer to that review).""" + +import json + +import httpx +import pytest + +from mongospecs_web import Api, Delete, Get, List, Resource +from mongospecs_web.testing import PLUGS, contract_api, mount_on + +BACKENDS = ("msgspec", "pydantic", "attrs") +ANN = {"authorization": "Bearer ann", "content-type": "application/json"} + + +# -- 2: integers the database cannot hold ---------------------------------------------------------- + + +@pytest.mark.parametrize("backend", BACKENDS) +@pytest.mark.parametrize("number", [2**63, -(2**63) - 1, 10**400]) +async def test_2_out_of_range_integers_are_a_validation_error(backend, number): + api = contract_api(backend) + created = await api.handle("POST", "/api/widgets", body=b'{"name":"w","price":1}', headers=ANN) + assert created.status == 201 + for method, path, field in ( + ("POST", "/api/widgets", "stock"), + ("PATCH", created.headers["location"], "stock"), + ): + body = json.dumps({"name": "big", field: number}).encode() + response = await api.handle(method, path, body=body, headers=ANN) + assert response.status == 422, (method, response.status, response.body) + problem = json.loads(response.body) + assert any(e["field"] == field for e in problem["errors"]), problem + # inside a container too, and the largest values that do fit are accepted + nested = await api.handle( + "POST", "/api/widgets", body=json.dumps({"name": "n", "tags": [number]}).encode(), headers=ANN + ) + assert nested.status == 422 + for fits in (2**63 - 1, -(2**63)): + ok = await api.handle( + "POST", "/api/widgets", body=json.dumps({"name": "ok", "stock": fits}).encode(), headers=ANN + ) + assert ok.status == 201, ok.body + # a float field takes a large integer as a float + as_float = await api.handle( + "POST", "/api/widgets", body=json.dumps({"name": "f", "price": 2**63}).encode(), headers=ANN + ) + assert as_float.status == 201 + + +def test_2_an_encoding_error_is_not_a_server_error(): + from mongospecs import EncodingError + from mongospecs_web.errors import problem + + assert problem(EncodingError("cannot encode object")).status == 422 + + +# -- 3: the body limit holds on every plug, at the ASGI level ----------------------------------------- + + +@pytest.mark.parametrize("plug", PLUGS) +@pytest.mark.parametrize("declared", [None, "10"]) # no Content-Length, and one that lies +async def test_3_no_plug_reads_a_body_past_the_limit(plug, declared): + api = contract_api("msgspec") + api.max_body = 1000 + app = mount_on(plug, api) + offered = 0 + + async def receive(): + nonlocal offered + if offered >= 300: + return {"type": "http.disconnect"} + offered += 1 + return {"type": "http.request", "body": b"x" * 65536, "more_body": True} + + sent = [] + + async def send(message): + sent.append(message) + + headers = [(b"host", b"t"), (b"authorization", b"Bearer ann"), (b"content-type", b"application/json")] + if declared: + headers.append((b"content-length", declared.encode())) + scope = { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/api/widgets", + "raw_path": b"/api/widgets", + "root_path": "", + "query_string": b"", + "headers": headers, + "client": ("127.0.0.1", 1), + "server": ("t", 80), + "state": {}, + "extensions": {}, + } + await app(scope, receive, send) + status = next(m["status"] for m in sent if m["type"] == "http.response.start") + assert status == 413 + assert offered <= 3, f"{plug} read {offered} chunks of 64 KiB with a 1000 byte limit" + + +# -- 5: guards run once ------------------------------------------------------------------------ + + +@pytest.mark.parametrize("key", ["delete", "get", "list"]) +async def test_5_a_guard_runs_once_per_request(key): + model = contract_api("msgspec").resources[0].model + calls = [] + + def guard(ctx): + calls.append(ctx.op.name) + return True + + backend = contract_api("msgspec").resources[0].backend + api = Api(Resource(model, ops=[List, Get, Delete], guards={key: guard}, backend=backend), prefix="/api") + path = "/api/widgets" if key == "list" else "/api/widgets/000000000000000000000000" + await api.handle("DELETE" if key == "delete" else "GET", path) + assert calls == [key] + + # the same guard listed for the resource and for the operation is still one check + calls.clear() + both = Api(Resource(model, ops=[List], guards=[guard, guard], backend=backend), prefix="/api") + await both.handle("GET", "/api/widgets") + assert calls == ["list"] + + +# -- 8: private fields stay out of the API description --------------------------------------------- + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_8_openapi_names_no_private_field(backend): + api = contract_api(backend) + text = json.dumps(api.openapi()) + assert "internal_note" not in text + schemas = api.openapi()["components"]["schemas"] + # every schema that is published is referenced by something: no stray full model definition + for name in schemas: + if name in ("Problem",) or name.endswith(("Read", "Create", "Patch")): + continue + assert f"#/components/schemas/{name}" in text, f"{name} is published but never referenced" + + +# -- HEAD mirrors GET on the ASGI plug ----------------------------------------------------------------- + + +async def test_head_reports_the_length_of_the_get_body(): + api = contract_api("msgspec") + created = await api.handle("POST", "/api/widgets", body=b'{"name":"w"}', headers=ANN) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=api.asgi()), base_url="http://t" + ) as client: + get = await client.get(created.headers["location"], headers=ANN) + head = await client.head(created.headers["location"], headers=ANN) + assert head.status_code == 200 and head.content == b"" + assert head.headers["content-length"] == str(len(get.content)) != "0" + assert head.headers["content-type"] == get.headers["content-type"] diff --git a/packages/mongospecs/src/mongospecs/codec.py b/packages/mongospecs/src/mongospecs/codec.py index 05b6d25..2dd1b11 100644 --- a/packages/mongospecs/src/mongospecs/codec.py +++ b/packages/mongospecs/src/mongospecs/codec.py @@ -320,6 +320,9 @@ def encode(self, obj: t.Any) -> dict[str, t.Any]: _, to_stored = self._renames(cls) if to_stored: doc = {to_stored.get(k, k): v for k, v in doc.items()} + for key in self.absent_when_none(cls): + if key in doc and doc[key] is None: + del doc[key] for f in self._special(cls): if f.transient: doc.pop(f.stored, None) @@ -327,6 +330,27 @@ def encode(self, obj: t.Any) -> dict[str, t.Any]: doc[f.stored] = ref_ids(doc[f.stored], f.ref.shape) return doc + def absent_when_none(self, cls: type) -> frozenset[str]: + """Stored keys of the fields whose default is ``None``. + + For these, ``None`` means "no value" and the key is left out of the stored + document instead of being written as ``null``. A document without the key + decodes back to ``None``, so nothing is lost, and sparse and unique indexes + behave as expected: MongoDB indexes an explicit ``null``, so two documents + "without" a value would otherwise collide on a unique sparse index. + """ + cache: dict[type, frozenset[str]] = self.__dict__.setdefault("_abs", {}) + hit = cache.get(cls) + if hit is None: + defaults = self.defaults(cls) + hit = frozenset( + f.stored + for f in self.fields(cls).values() + if f.stored != "_id" and not f.transient and f.name in defaults and defaults[f.name] is None + ) + cache[cls] = hit + return hit + def _special(self, cls: type) -> tuple[FieldInfo, ...]: cache: dict[type, tuple[FieldInfo, ...]] = self.__dict__.setdefault("_sp", {}) hit = cache.get(cls) @@ -372,6 +396,7 @@ def clear_cache(self) -> None: self._fields.clear() self.__dict__.pop("_rn", None) self.__dict__.pop("_sp", None) + self.__dict__.pop("_abs", None) def field_error(cls: type, name: str | None, message: str) -> ValidationError: diff --git a/packages/mongospecs/src/mongospecs/errors.py b/packages/mongospecs/src/mongospecs/errors.py index 76191b2..845ec73 100644 --- a/packages/mongospecs/src/mongospecs/errors.py +++ b/packages/mongospecs/src/mongospecs/errors.py @@ -158,6 +158,9 @@ def _duplicate_fields(details: t.Any) -> dict[str, t.Any]: return {} +_TYPE_MISMATCH = 14 + + def translate(exc: BaseException) -> MongoSpecsError: """Map a pymongo/bson exception to exactly one mongospecs error.""" msg = str(exc) @@ -187,6 +190,10 @@ def translate(exc: BaseException) -> MongoSpecsError: if isinstance(exc, pe.OperationFailure): if exc.code in _TOO_LARGE_CODES: return DocumentTooLarge(msg, code=exc.code, details=exc.details) + if exc.code == _TYPE_MISMATCH: + # The update or query does not fit the stored value (incrementing a string, say): + # the caller's mistake, not a server failure. + return QueryError(msg) return OperationFailed(msg, code=exc.code, details=exc.details) if isinstance(exc, pe.InvalidOperation): return DriverError(msg) diff --git a/packages/mongospecs/src/mongospecs/expr.py b/packages/mongospecs/src/mongospecs/expr.py index b22becf..60bda1f 100644 --- a/packages/mongospecs/src/mongospecs/expr.py +++ b/packages/mongospecs/src/mongospecs/expr.py @@ -109,6 +109,16 @@ class SortKey: direction: int +def _text(value: t.Any, name: str) -> str: + if not isinstance(value, str): + raise QueryError(f"{name}() matches literal text and needs a str, got {type(value).__name__}") + return value + + +def _flags(ignore_case: bool) -> str: + return "i" if ignore_case else "" + + def _values(values: t.Any, name: str) -> list[t.Any]: if isinstance(values, str | bytes | t.Mapping) or not isinstance(values, t.Iterable): raise QueryError(f"{name}() needs a list of values, got {type(values).__name__}") @@ -175,11 +185,11 @@ def type_(self, value: int | str) -> Condition: def prefix(self, value: str, *, ignore_case: bool = False) -> Condition: """Literal starts-with match (the value is escaped).""" - return Condition(self._path, "$regex", ("^" + re.escape(value), "i" if ignore_case else "")) + return Condition(self._path, "$regex", ("^" + re.escape(_text(value, "prefix")), _flags(ignore_case))) def contains(self, value: str, *, ignore_case: bool = False) -> Condition: """Literal substring match (the value is escaped).""" - return Condition(self._path, "$regex", (re.escape(value), "i" if ignore_case else "")) + return Condition(self._path, "$regex", (re.escape(_text(value, "contains")), _flags(ignore_case))) def regex(self, pattern: str, options: str = "") -> Condition: """Regular-expression match. The pattern is used as written; do not pass user input.""" @@ -274,10 +284,14 @@ def And(*conditions: Expr | t.Mapping[str, t.Any]) -> Group: def Or(*conditions: Expr | t.Mapping[str, t.Any]) -> Group: + if not conditions: + raise QueryError("Or() needs at least one condition") return Group("$or", conditions) def Nor(*conditions: Expr | t.Mapping[str, t.Any]) -> Group: + if not conditions: + raise QueryError("Nor() needs at least one condition") return Group("$nor", conditions) @@ -441,5 +455,9 @@ def and_filters(*filters: t.Mapping[str, t.Any] | None) -> dict[str, t.Any]: def compile_sort(keys: t.Iterable[SortInput], meta: ModelMeta | None = None) -> list[tuple[str, int]]: out: list[tuple[str, int]] = [] for key in SortBy(*keys): + if not isinstance(key.path, str) or any( + not part or part.startswith("$") for part in key.path.split(".") + ): + raise QueryError(f"cannot sort by {key.path!r}: not a field path") out.append((meta.stored_path(key.path) if meta is not None else key.path, key.direction)) return out diff --git a/packages/mongospecs/src/mongospecs/manager.py b/packages/mongospecs/src/mongospecs/manager.py index 23b98e4..5ee8b5b 100644 --- a/packages/mongospecs/src/mongospecs/manager.py +++ b/packages/mongospecs/src/mongospecs/manager.py @@ -84,6 +84,15 @@ def __new__(cls, matched: int, modified: int) -> UpdateCount: def __repr__(self) -> str: return f"UpdateCount(matched={self.matched}, modified={self.modified})" + def __str__(self) -> str: + return int.__repr__(self) + + def __format__(self, spec: str) -> str: + return int(self).__format__(spec) + + def __reduce__(self) -> tuple[t.Any, ...]: + return (UpdateCount, (self.matched, self.modified)) + class Manager(t.Generic[T]): def __init__( diff --git a/packages/mongospecs/src/mongospecs/meta.py b/packages/mongospecs/src/mongospecs/meta.py index c52ab9a..19b2a19 100644 --- a/packages/mongospecs/src/mongospecs/meta.py +++ b/packages/mongospecs/src/mongospecs/meta.py @@ -94,6 +94,38 @@ def walk(klass: type) -> None: return out +RESERVED_FIELD_NAMES: frozenset[str] = frozenset( + { + "after", + "aggregate", + "before", + "bulk_save", + "bulk_write", + "changes", + "collection", + "count", + "delete", + "estimated_count", + "exists", + "find", + "get", + "hydrate", + "insert", + "insert_many", + "is_new", + "reload", + "save", + "scope", + "update", + "upsert", + "using", + "watch", + } +) +"""Attribute names a model field may not use: each is part of the ``Document`` API, and a field +of the same name would replace it on the class (``Model.count()`` would stop working).""" + + class ModelMeta: """Everything mongospecs derives from a model class.""" @@ -108,6 +140,13 @@ def __init__(self, model: type) -> None: self.fields: dict[str, FieldInfo] = self.codec.fields(model) if "id" not in self.fields: raise ConfigurationError(f"{model.__name__} has no `id` field") + shadowed = sorted(set(self.fields) & RESERVED_FIELD_NAMES) + if shadowed: + raise ConfigurationError( + f"{model.__name__}: field name {shadowed[0]!r} shadows the document API " + f"({model.__name__}.{shadowed[0]} is part of mongospecs). Rename the attribute and keep the " + f"stored key with Annotated[..., Alias({shadowed[0]!r})]." + ) self.id = self.fields["id"] self.by_stored: dict[str, FieldInfo] = {f.stored: f for f in self.fields.values()} self.deferred: tuple[str, ...] = tuple(f.stored for f in self.fields.values() if f.has(Deferred)) @@ -202,16 +241,26 @@ def cast(self, path: str, value: t.Any, *, lax: bool = False) -> t.Any: if loose is None: loose = self._loose[info.name] = loosen(info.type) try: - return to_bson(self.validate_type(loose, value, lax=lax)) + return to_bson(self._coerce(loose, value, lax)) except ValidationError as exc: elem = element_type(loose) if elem is not None and not isinstance(value, list | tuple | set | frozenset): try: - return to_bson(self.validate_type(elem, value, lax=lax)) + return to_bson(self._coerce(elem, value, lax)) except ValidationError: pass raise QueryError(f"cannot cast {value!r} to {self.model.__name__}.{info.name}: {exc}") from exc + def _coerce(self, tp: t.Any, value: t.Any, lax: bool) -> t.Any: + """Cast a filter value. A value of the right type is taken as it is; otherwise text that + reads as the field's type is accepted (``"12"`` for an ``int``), on every backend.""" + if lax: + return self.validate_type(tp, value, lax=True) + try: + return self.validate_type(tp, value) + except ValidationError: + return self.validate_type(tp, value, lax=True) + def _cast_ref(self, info: FieldInfo, value: t.Any) -> t.Any: if isinstance(value, list | tuple | set | frozenset): return [self._cast_ref(info, v) for v in value] diff --git a/packages/mongospecs/src/mongospecs/query.py b/packages/mongospecs/src/mongospecs/query.py index b3abdf4..8c99fa2 100644 --- a/packages/mongospecs/src/mongospecs/query.py +++ b/packages/mongospecs/src/mongospecs/query.py @@ -15,7 +15,7 @@ import bson from mongospecs import tracking -from mongospecs.errors import DocumentNotFound, QueryError +from mongospecs.errors import DocumentNotFound, QueryError, ValidationError from mongospecs.expr import SortInput, and_filters, compile_filter, compile_sort from mongospecs.meta import meta_of from mongospecs.updates import UpdateOp @@ -65,6 +65,7 @@ class Query(t.Generic[T]): "_conditions", "_fail", "_includes", + "_invalid", "_limit", "_manager", "_mode", @@ -91,6 +92,7 @@ def __init__( session: t.Any = None, options: t.Mapping[str, t.Any] | None = None, fail: bool = False, + invalid: str = "raise", ) -> None: self._manager = manager self._conditions = conditions @@ -104,6 +106,7 @@ def __init__( self._session = session self._options: dict[str, t.Any] = dict(options or {}) self._fail = fail + self._invalid = invalid def _clone(self, **changes: t.Any) -> Query[T]: state = {name.lstrip("_"): getattr(self, name) for name in self.__slots__ if name != "_manager"} @@ -157,8 +160,11 @@ def include( ``path`` may be nested (``"author.company"``). ``limit`` caps the related documents overall; ``per_parent_limit`` caps them per parent document. """ - from mongospecs.relations import add_include + from mongospecs.relations import add_include, check_include + if strategy not in ("auto", "lookup", "batch"): + raise QueryError(f"include strategy must be 'auto', 'lookup' or 'batch', got {strategy!r}") + check_include(self._manager.meta, path) return self._clone( includes=add_include( self._includes, @@ -176,6 +182,18 @@ def include( ) ) + def on_invalid(self, action: t.Literal["raise", "skip", "raw"]) -> Query[T]: + """What to do with a stored document that does not fit the model. + + ``"raise"`` (the default) fails the whole read with :class:`ValidationError`. + ``"skip"`` leaves such documents out. ``"raw"`` returns them as plain dicts in + place, so they can be inspected or repaired; relations are not loaded for them. + Skipped documents still count towards ``limit()`` and page sizes. + """ + if action not in ("raise", "skip", "raw"): + raise QueryError(f"on_invalid takes 'raise', 'skip' or 'raw', got {action!r}") + return self._clone(invalid=action) + def tracked(self) -> Query[T]: """Return model instances with change tracking (the default).""" return self._clone(mode="tracked") @@ -343,30 +361,44 @@ async def _chunks(self, chunk: int | None = None) -> t.AsyncIterator[list[t.Any] raw_bson = mode == "tracked" and not lookups options = manager.registry._codec_options + invalid = self._invalid + async def flush(raws: list[t.Any]) -> list[t.Any]: - if raw_bson: - items: list[t.Any] = [] - decode = meta.decode - for item in raws: - data = item.raw - doc = decode(bson.decode(data, options)) - tracking.track(doc, data, manager, partial) - items.append(doc) + misfits: list[tuple[int, t.Any]] = [] # (position, raw document) kept for on_invalid("raw") + joined = [pop_lookups(raw, lookups) for raw in raws] if lookups else None + if mode == "raw": + items: list[t.Any] = raws else: - joined = [pop_lookups(raw, lookups) for raw in raws] if lookups else None - if mode == "raw": - items = raws - else: - items = [] - for raw in raws: - doc = meta.decode(raw) - if mode == "tracked": - tracking.track(doc, meta_of(type(doc)).encode(doc), manager, partial) - items.append(doc) + items = [] + kept: list[t.Any] = [] + for position, item in enumerate(raws): + data = item.raw if raw_bson else None + plain = bson.decode(data, options) if data is not None else item + try: + doc = meta.decode(plain) + except ValidationError: + if invalid == "raise": + raise + if invalid == "raw": + misfits.append((position, plain)) + continue + if data is not None: + tracking.track(doc, data, manager, partial) + elif mode == "tracked": + tracking.track(doc, meta_of(type(doc)).encode(doc), manager, partial) + items.append(doc) + if joined is not None: + kept.append(joined[position]) if joined is not None: - attach_lookups(manager, items, joined, lookups, mode) + joined = kept + if joined is not None: + attach_lookups(manager, items, joined, lookups, mode) if batched: await load_batched(manager, items, batched, mode, self._session) + if misfits: # put the documents that did not fit back where the server returned them + raw_at = dict(misfits) + valid = iter(items) + return [raw_at[i] if i in raw_at else next(valid) for i in range(len(raws))] return items with manager._op("find"): diff --git a/packages/mongospecs/src/mongospecs/registry.py b/packages/mongospecs/src/mongospecs/registry.py index f34f93b..6b91ef7 100644 --- a/packages/mongospecs/src/mongospecs/registry.py +++ b/packages/mongospecs/src/mongospecs/registry.py @@ -30,6 +30,7 @@ MongoSpecsError, NotBoundError, RegistryClosedError, + TransactionError, translate, wrap_errors, ) @@ -55,6 +56,9 @@ R = t.TypeVar("R") _BINDINGS: weakref.WeakKeyDictionary[type, Registry] = weakref.WeakKeyDictionary() +_IN_TRANSACTION: contextvars.ContextVar[bool] = contextvars.ContextVar( + "mongospecs_in_transaction", default=False +) _CURRENT_OP: contextvars.ContextVar[tuple[str, str] | None] = contextvars.ContextVar( "mongospecs_current_op", default=None ) @@ -471,19 +475,35 @@ async def transaction( write_concern: t.Any = None, read_preference: t.Any = None, max_commit_time_ms: int | None = None, + independent: bool = False, ) -> R: """Run ``fn(tx)`` in a transaction and commit it. + Transactions do not nest. MongoDB has no savepoints, and a transaction + started while another one is running is a separate transaction: it commits + on its own and stays committed if the outer one aborts. Calling this from + inside another ``transaction()`` therefore raises + :class:`TransactionError`; pass the outer ``tx`` down and use it instead. + If a separate, independently committed transaction is what you want (an + audit record that must survive a rollback), say so with ``independent=True``. + ``fn`` may be called more than once: the driver re-runs it on transient transaction errors. Before every re-run, and if the transaction finally aborts, documents written inside it are restored to their prior in-memory state. Pass ``session=tx`` to every operation inside ``fn``. """ + if _IN_TRANSACTION.get() and not independent: + raise TransactionError( + "transaction() was called inside another transaction. Transactions do not nest: this one " + "would commit on its own even if the outer one aborts. Pass the outer tx to the code that " + "needs it, or use transaction(fn, independent=True) if a separate commit is intended." + ) async with self.client.start_session() as session: tx = Tx(session, self) async def attempt(_: t.Any) -> R: tx.rollback_memory() + token = _IN_TRANSACTION.set(True) # for detection only; sessions are always passed explicitly try: return await fn(tx) except MongoSpecsError as exc: @@ -494,6 +514,8 @@ async def attempt(_: t.Any) -> R: ): raise cause from None raise + finally: + _IN_TRANSACTION.reset(token) try: result = await session.with_transaction( diff --git a/packages/mongospecs/src/mongospecs/relations.py b/packages/mongospecs/src/mongospecs/relations.py index 8fca230..8ccd482 100644 --- a/packages/mongospecs/src/mongospecs/relations.py +++ b/packages/mongospecs/src/mongospecs/relations.py @@ -63,6 +63,28 @@ def add_include(includes: tuple[Include, ...], path: str, **options: t.Any) -> t return (*(i for i in includes if i.name != head), updated) +def check_include(meta: t.Any, path: str) -> None: + """Raise :class:`QueryError` unless every segment of ``path`` is a relation. + + Called when the query is built, so a mistyped path fails at once and not only + when a matching document happens to exist. + """ + metas: tuple[t.Any, ...] = (meta,) + for name in path.split("."): + holders = [ + m + for m in metas + if (f := m.fields.get(name)) is not None and (f.ref is not None or f.backref is not None) + ] + if not holders: + owner = " | ".join(m.model.__name__ for m in metas) + raise QueryError(f"{owner}.{name} is not a relation and cannot be included") + try: + metas = tuple(target for m in holders for target in m.ref_targets(name)) + except ConfigurationError: + return # a target that is not defined yet; it is reported when the query runs + + def _relation(manager: Manager[t.Any], include: Include) -> FieldInfo: info = manager.meta.fields.get(include.name) if info is None or (info.ref is None and info.backref is None): diff --git a/packages/mongospecs/src/mongospecs/updates.py b/packages/mongospecs/src/mongospecs/updates.py index def91ef..3e86804 100644 --- a/packages/mongospecs/src/mongospecs/updates.py +++ b/packages/mongospecs/src/mongospecs/updates.py @@ -252,6 +252,7 @@ def compile_update( """Compile operators (plus ``Set`` shorthand keywords) to a MongoDB update document.""" update: dict[str, dict[str, t.Any]] = {} seen: dict[str, str] = {} + absent = meta.codec.absent_when_none(meta.model) all_ops = [*ops, *([Set(fields)] if fields else [])] for op in all_ops: if not isinstance(op, UpdateOp): @@ -260,5 +261,8 @@ def compile_update( if path in seen: raise QueryError(f"field {path!r} is updated by both {seen[path]} and {op.op}") seen[path] = op.op - update.setdefault(op.op, {})[path] = value + if value is None and op.op == "$set" and path in absent: + update.setdefault("$unset", {})[path] = "" # None means "no value": the key is not stored + else: + update.setdefault(op.op, {})[path] = value return update diff --git a/packages/mongospecs/tests/test_review_round2.py b/packages/mongospecs/tests/test_review_round2.py new file mode 100644 index 0000000..e156461 --- /dev/null +++ b/packages/mongospecs/tests/test_review_round2.py @@ -0,0 +1,196 @@ +"""Regression tests for the second QA review of v2 (issue numbers refer to that review).""" + +import typing as t + +import pytest + +from mongospecs import ( + ConfigurationError, + DocumentNotFound, + DuplicateKeyError, + Inc, + Index, + MongoSpecsError, + Nor, + Or, + Q, + QueryError, + SpecConfig, + TransactionError, + ValidationError, + meta_of, +) +from mongospecs.manager import UpdateCount + +# -- 1: sparse unique index and optional fields ------------------------------------------------ + + +async def test_1_absent_optional_fields_do_not_collide_on_a_sparse_unique_index(m, registry): + await registry.apply_indexes() # User has Index("email", unique=True, sparse=True) + result = await m.User.insert_many([m.User(name="n1"), m.User(name="n2")]) + assert len(result.inserted) == 2 + await m.User(name="n3").insert() + raw = await m.User.collection().find_one({"name": "n1"}) + assert "email" not in raw # None means absent: the field is not stored + # uniqueness still holds where a value is present + await m.User(name="a", email="a@x.io").insert() + with pytest.raises(DuplicateKeyError): + await m.User(name="b", email="a@x.io").insert() + # clearing a value un-stores it, through save() and through an atomic update alike + a = await m.User.find(Q.name == "a").one() + a.email = None + assert a.changes() == {"$unset": {"email": ""}} + await a.save() + await m.User(name="c", email="c@x.io").insert() + await m.User.update(Q.name == "c", email=None) + assert await m.User.collection().count_documents({"email": {"$exists": True}}) == 0 + assert await m.User.count(Q.email == None) == 5 # noqa: E711 - None still matches "no value" + loaded = await m.User.find(Q.name == "c").one() + assert loaded.email is None and loaded.changes() == {} + + +# -- 4: field names that shadow the document API ------------------------------------------------- + + +@pytest.mark.parametrize( + "name", + ["count", "update", "delete", "find", "save", "get", "insert", "changes", "is_new", "scope", "watch"], +) +def test_4_fields_may_not_shadow_the_document_api(m, name): + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") # the backends themselves only warn, if at all + try: + if m.backend == "attrs": + import attrs + + Shadow = attrs.make_class( + "Shadow", {name: attrs.field(default=0, type=int)}, bases=(m.Spec,), kw_only=True + ) + elif m.backend == "pydantic": + import pydantic + + Shadow = pydantic.create_model("Shadow", __base__=m.Spec, **{name: (int, 0)}) + else: + import msgspec + + Shadow = msgspec.defstruct("Shadow", [(name, int, 0)], bases=(m.Spec,), kw_only=True) + except Exception as exc: # a backend may refuse the class outright, which is fine too + assert name in str(exc) + return + with pytest.raises(ConfigurationError, match=f"'{name}'.*document API"): + meta_of(Shadow) + + +def test_4_ordinary_and_id_fields_are_fine(m): + assert "id" in meta_of(m.User).fields and "name" in meta_of(m.User).fields + + +# -- 6: nested transactions ---------------------------------------------------------------------- + + +async def test_6_a_nested_transaction_is_refused_unless_declared_independent(m, registry): + async def inner(tx): + await m.User(name="inner").insert(session=tx) + + async def outer(tx): + await m.User(name="outer").insert(session=tx) + await registry.transaction(inner) # would commit on its own, whatever happens to the outer one + raise RuntimeError("outer aborts") + + with pytest.raises(TransactionError, match="inside another transaction"): + await registry.transaction(outer) + assert await m.User.count() == 0 # nothing slipped through + + async def outer_explicit(tx): + await m.User(name="outer").insert(session=tx) + await registry.transaction(inner, independent=True) # stated on purpose + raise RuntimeError("outer aborts") + + with pytest.raises(RuntimeError): + await registry.transaction(outer_explicit) + assert [u.name for u in await m.User.find()] == ["inner"] + # sequential transactions and a transaction after a failed one are unaffected + await registry.transaction(inner) + assert await m.User.count() == 2 + + +# -- 7: UpdateCount formats like an int ----------------------------------------------------------- + + +def test_7_update_count_prints_like_an_int(): + import json + import pickle + + n = UpdateCount(5, 3) + assert str(n) == "3" and f"{n}" == "3" and "%s" % n == "3" and "%d" % n == "3" # noqa: UP031 and f"{n:>4}" == " 3" + assert repr(n) == "UpdateCount(matched=5, modified=3)" + assert n == 3 and n + 1 == 4 and json.dumps({"n": n}) == '{"n": 3}' + restored = pickle.loads(pickle.dumps(n)) + assert restored == 3 and restored.matched == 5 + + +# -- smaller findings ------------------------------------------------------------------------------ + + +async def test_casting_follows_one_rule_on_every_backend(m, registry): + """Coercion is lax everywhere: text that reads as the field's type is accepted.""" + user = await m.User(name="ann", age=12).insert() + assert await m.User.count(Q.age == "12") == 1 + assert await m.User.count(Q.age.in_(["12", 13])) == 1 + with pytest.raises(QueryError): + await m.User.count(Q.age == "twelve") + assert await m.User.count(Q.age == 12) == 1 and await m.User.count(Q.name == "ann") == 1 + assert user.age == 12 + + +async def test_malformed_queries_are_query_errors_not_server_errors(m, registry): + await m.User(name="ann").insert() + for bad in (lambda: Or(), lambda: Nor(), lambda: Q.name.prefix({"$gt": ""}), lambda: Q.name.contains(5)): + with pytest.raises(QueryError): + await m.User.find(bad()).all() + with pytest.raises(QueryError): + await m.User.find().sort("address.$").all() + with pytest.raises(QueryError): + await m.User.find().sort("$natural").all() + # the server refusing an update that does not fit the stored value is the caller's error + await m.User.collection().update_one({"name": "ann"}, {"$set": {"address": {"zip": "abc"}}}) + with pytest.raises(QueryError): + await m.User.update(Q.name == "ann", Inc({"address.zip": 1})) + + +async def test_include_is_validated_when_the_query_is_built(m, registry): + with pytest.raises(QueryError, match="not a relation"): + m.Article.find().include("nope") # an empty collection used to hide this + with pytest.raises(QueryError, match="not a relation"): + m.Article.find().include("author.nope") + with pytest.raises(QueryError, match="strategy"): + m.Article.find().include("author", strategy="weird") + assert await m.Article.find().include("author").all() == [] + + +async def test_invalid_stored_documents_can_be_skipped_or_returned_raw(m, registry): + await m.User.insert_many([m.User(name="ok1", age=1), m.User(name="ok2", age=2)]) + await m.User.collection().insert_one({"name": "corrupt", "age": "not a number"}) + with pytest.raises(ValidationError): + await m.User.find().all() # the default: a corrupt document is an error + skipped = await m.User.find().sort("name").on_invalid("skip").all() + assert [u.name for u in skipped] == ["ok1", "ok2"] + mixed = await m.User.find().sort("name").on_invalid("raw").all() + assert [u["name"] if isinstance(u, dict) else u.name for u in mixed] == ["corrupt", "ok1", "ok2"] + assert isinstance(mixed[0], dict) and mixed[0]["age"] == "not a number" + lean = await m.User.find().lean().on_invalid("skip").all() + assert len(lean) == 2 + with pytest.raises(QueryError): + m.User.find().on_invalid("ignore") + + +async def test_upsert_revision_is_documented_behaviour(m, registry): + """An upsert is a write, so the document it creates starts at revision 1; insert() starts at 0.""" + assert (await m.Article(title="i").insert()).rev == 0 + assert (await m.Article.upsert(Q.title == "u", Inc(views=1))).rev == 1 + + +def test_unused_names_keep_imports_honest(): + assert Index and SpecConfig and DocumentNotFound and MongoSpecsError and t From 976d41e635376deae1eb679061b6b84f7426a792 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 02:56:05 -0400 Subject: [PATCH 11/26] fix(web): enforce Policy(private=True) on fields of embedded models Private fields of embedded (non-document) models were serialised, writable and described in OpenAPI. They are now handled like private fields of the document itself, at any depth and inside lists, dicts, optionals and unions: never in a response (including included documents), a 422 'unknown field' when a client supplies one on create or patch, and absent from every published schema. Co-Authored-By: Claude Opus 5.5 --- .../src/mongospecs_web/policy.py | 190 +++++++++++++++++- .../tests/test_embedded_private.py | 171 ++++++++++++++++ 2 files changed, 354 insertions(+), 7 deletions(-) create mode 100644 packages/mongospecs-web/tests/test_embedded_private.py diff --git a/packages/mongospecs-web/src/mongospecs_web/policy.py b/packages/mongospecs-web/src/mongospecs_web/policy.py index 415886f..7b2ae8e 100644 --- a/packages/mongospecs-web/src/mongospecs_web/policy.py +++ b/packages/mongospecs-web/src/mongospecs_web/policy.py @@ -24,6 +24,148 @@ """MongoDB's largest integer type is 64 bits; a larger integer cannot be stored as one.""" +class Nested: + """Where the private fields are inside the value of one field. + + ``private`` and ``children`` describe an embedded model (by every key it may appear + under: attribute name, stored name, the backend's own name); ``elem`` describes the + items of a list or the values of a dict. + """ + + __slots__ = ("children", "elem", "model", "private") + + def __init__(self) -> None: + self.model = False + self.private: frozenset[str] = frozenset() + self.children: dict[str, Nested] = {} + self.elem: Nested | None = None + + def hides_something(self, seen: set[int] | None = None) -> bool: + seen = set() if seen is None else seen + if id(self) in seen: + return False + seen.add(id(self)) + return ( + bool(self.private) + or any(child.hides_something(seen) for child in self.children.values()) + or (self.elem is not None and self.elem.hides_something(seen)) + ) + + +def embedded_models(tp: t.Any, found: dict[type, None] | None = None) -> dict[type, None]: + """Every embedded (non-document) model class reachable from an annotation, at any depth.""" + from mongospecs import Document + from mongospecs.codec import _codec_or_none + + found = {} if found is None else found + while t.get_origin(tp) is t.Annotated: + tp = t.get_args(tp)[0] + if isinstance(tp, type) and not issubclass(tp, Document) and t.get_origin(tp) is None: + codec = _codec_or_none(tp) + if codec is not None and tp not in found: + found[tp] = None + for info in codec.fields(tp).values(): + embedded_models(info.type, found) + return found + if t.get_origin(tp) is Ref: + return found # another document: its own view applies + for arg in t.get_args(tp): + embedded_models(arg, found) + return found + + +def _model_keys(cls: type) -> dict[str, tuple[FieldInfo, frozenset[str]]]: + from mongospecs.codec import codec_for + + codec = codec_for(cls) + natives = {name: native for name, native, _, _ in codec.raw_fields(cls)} + return { + name: (info, frozenset({name, info.stored, natives.get(name, name)})) + for name, info in codec.fields(cls).items() + } + + +def nested_plan(tp: t.Any, cache: dict[type, Nested]) -> Nested | None: + """The :class:`Nested` plan for an annotation, or ``None`` when it holds no embedded model.""" + from mongospecs import Document + from mongospecs.codec import _UNION_ORIGINS, _codec_or_none + + while t.get_origin(tp) is t.Annotated: + tp = t.get_args(tp)[0] + origin = t.get_origin(tp) + if origin is Ref: + return None + if origin is None: + if not isinstance(tp, type) or issubclass(tp, Document) or _codec_or_none(tp) is None: + return None + node = cache.get(tp) + if node is None: + node = cache[tp] = Nested() + node.model = True + private: set[str] = set() + for info, keys in _model_keys(tp).values(): + if info.policy.private: + private |= keys + continue + child = nested_plan(info.type, cache) + if child is not None: + for key in keys: + node.children[key] = child + node.private = frozenset(private) + return node + parts = [plan for arg in t.get_args(tp) if (plan := nested_plan(arg, cache)) is not None] + if not parts: + return None + if origin in _UNION_ORIGINS: + if len(parts) == 1: + return parts[0] + merged = Nested() # a union: hide whatever any member hides, so nothing can leak + for part in parts: + merged.model = merged.model or part.model + merged.private |= part.private + for key, child in part.children.items(): + merged.children.setdefault(key, child) + merged.elem = merged.elem or part.elem + return merged + container = Nested() + container.elem = parts[-1] # list[X] -> X, dict[str, X] -> X + return container + + +def strip_nested(value: t.Any, node: Nested) -> t.Any: + """Remove private keys from a JSON-shaped value, in place.""" + if isinstance(value, dict): + if node.model: + for key in node.private: + value.pop(key, None) + for key, child in node.children.items(): + if key in value: + strip_nested(value[key], child) + elif node.elem is not None: + for item in value.values(): + strip_nested(item, node.elem) + elif isinstance(value, list) and node.elem is not None: + for item in value: + strip_nested(item, node.elem) + return value + + +def nested_violations(value: t.Any, node: Nested, path: str, out: list[str]) -> None: + """Collect the dotted paths of private keys a client supplied.""" + if isinstance(value, dict): + if node.model: + out.extend(f"{path}.{key}" for key in value if key in node.private) + for key, child in node.children.items(): + if key in value: + nested_violations(value[key], child, f"{path}.{key}", out) + elif node.elem is not None: + for key, item in value.items(): + nested_violations(item, node.elem, f"{path}.{key}", out) + elif isinstance(value, list) and node.elem is not None: + for index, item in enumerate(value): + nested_violations(item, node.elem, f"{path}.{index}", out) + + def _refs(node: t.Any, out: set[str]) -> None: if isinstance(node, dict): for key, value in node.items(): @@ -120,6 +262,16 @@ def may(flag: bool | None, default: bool) -> bool: self._relations: tuple[FieldInfo, ...] = tuple( f for f in self.readable if f.transient or f.ref is not None ) + # Private fields of embedded models, at any depth below a field of this model. + cache: dict[type, Nested] = {} + self._nested: dict[str, Nested] = { + f.name: plan + for f in fields.values() + if not f.transient and (plan := nested_plan(f.type, cache)) is not None and plan.hides_something() + } + self._nested_read: tuple[tuple[str, Nested], ...] = tuple( + (name, plan) for name, plan in self._nested.items() if name in self._plain + ) revision = next((f for f in fields.values() if f.has(Revision)), None) self.revision: str | None = revision.name if revision else None @@ -131,6 +283,8 @@ def serialize(self, doc: t.Any, include: t.Collection[str] = ()) -> dict[str, t. return view_of(type(doc)).serialize(doc, include) data = self.meta.codec.to_json(doc) out = {name: data.get(name) for name in self._plain} + for name, plan in self._nested_read: + strip_nested(out[name], plan) if include: for f in self._relations: if f.name not in include: @@ -173,6 +327,11 @@ def check_input(self, data: t.Any, allowed: frozenset[str], action: str) -> dict if errors: raise Unprocessable("The request body names fields that cannot be written.", errors=errors) for key, value in data.items(): + plan = self._nested.get(key) + if plan is not None: + hidden: list[str] = [] + nested_violations(value, plan, key, hidden) + errors.extend({"field": path, "message": "unknown field"} for path in hidden) if _has_oversized_int(value) and not _takes_any_number(self.meta.fields[key].type): errors.append( {"field": key, "message": f"integers must be between {INT64_MIN} and {INT64_MAX}"} @@ -185,15 +344,32 @@ def check_input(self, data: t.Any, allowed: frozenset[str], action: str) -> dict def _base(self) -> dict[str, t.Any]: base = self.meta.codec.json_schema(self.model) - own = base.get("$defs", {}).get(self.model.__name__) - if isinstance(own, dict) and self.private: + defs = dict(base.get("$defs", {})) + hidden_by_name: dict[str, frozenset[str]] = {} + if self.private: # The model's own definition survives only where the model refers to itself; # it must not describe more than the Read view does. - hidden = self.private | {self.meta.fields[name].stored for name in self.private} - own = dict(own) - own["properties"] = {k: v for k, v in own.get("properties", {}).items() if k not in hidden} - own["required"] = [k for k in own.get("required", []) if k not in hidden] - base = {**base, "$defs": {**base["$defs"], self.model.__name__: own}} + hidden_by_name[self.model.__name__] = self.private | { + self.meta.fields[name].stored for name in self.private + } + for info in self.meta.fields.values(): + for cls in embedded_models(info.type): + keys = frozenset().union(*(k for f, k in _model_keys(cls).values() if f.policy.private)) + if keys: + hidden_by_name[cls.__name__] = hidden_by_name.get(cls.__name__, frozenset()) | keys + changed = False + for name, hidden in hidden_by_name.items(): + own = defs.get(name) + if isinstance(own, dict): + own = dict(own) + own["properties"] = {k: v for k, v in own.get("properties", {}).items() if k not in hidden} + own["required"] = [k for k in own.get("required", []) if k not in hidden] + if not own["required"]: + del own["required"] + defs[name] = own + changed = True + if changed: + base = {**base, "$defs": defs} return base def schema(self, kind: t.Literal["read", "create", "patch"]) -> dict[str, t.Any]: diff --git a/packages/mongospecs-web/tests/test_embedded_private.py b/packages/mongospecs-web/tests/test_embedded_private.py new file mode 100644 index 0000000..2c237df --- /dev/null +++ b/packages/mongospecs-web/tests/test_embedded_private.py @@ -0,0 +1,171 @@ +"""Policy(private=True) on the fields of embedded (non-document) models, at any depth.""" + +import json +import typing as t + +import httpx +import pytest + +from mongospecs import Policy, Ref, SpecConfig +from mongospecs_web import Api, Create, Get, List, Patch, Resource +from mongospecs_web.policy import view_of + +BACKENDS = ("msgspec", "pydantic", "attrs") +SECRETS = ("vault_code", "surveyor") + + +def models(backend): + if backend == "msgspec": + import msgspec + + from mongospecs.msgspec import Spec + + class Base(msgspec.Struct, kw_only=True): + pass + + define = lambda c: c # noqa: E731 + elif backend == "pydantic": + import pydantic + + from mongospecs.pydantic import Spec + + Base = pydantic.BaseModel + define = lambda c: c # noqa: E731 + else: + import attrs + + from mongospecs.attrs import Spec + + Base = object + define = attrs.define(kw_only=True) + + @define + class Geo(Base): + lat: float = 0.0 + surveyor: t.Annotated[str | None, Policy(private=True)] = None + + @define + class Addr(Base): + city: str = "" + vault_code: t.Annotated[str, Policy(private=True)] = "" + geo: Geo | None = None + + @define + class Company(Spec): + __spec__ = SpecConfig(collection="companies") + name: str = "" + hq: Addr | None = None + + @define + class Person(Spec): + __spec__ = SpecConfig(collection="persons") + name: t.Annotated[str, Policy(filter=("eq",), sort=True)] = "" + address: Addr | None = None + history: list[Addr] = [] # noqa: RUF012 + by_label: dict[str, Addr] = {} # noqa: RUF012 + employer: Ref[Company] | None = None + + return Geo, Addr, Company, Person + + +@pytest.fixture(params=BACKENDS) +async def world(request, make_registry): + Geo, Addr, Company, Person = models(request.param) + await make_registry(Company, Person) + api = Api( + Resource(Person, name="persons", ops=[List, Get, Create, Patch], include=["employer"]), + Resource(Company, name="companies", ops=[Get]), + prefix="/v1", + ) + client = httpx.AsyncClient(transport=httpx.ASGITransport(app=api.asgi()), base_url="http://t") + try: + yield client, api, (Geo, Addr, Company, Person) + finally: + await client.aclose() + + +async def seeded(Geo, Addr, Company, Person): + def addr(city): + return Addr(city=city, vault_code=f"vault-{city}", geo=Geo(lat=1.5, surveyor=f"surveyor-{city}")) + + acme = await Company(name="acme", hq=addr("hq")).insert() + return await Person( + name="ann", + address=addr("home"), + history=[addr("old1"), addr("old2")], + by_label={"work": addr("work")}, + employer=acme, + ).insert() + + +async def test_nested_private_fields_are_never_read(world): + client, api, classes = world + ann = await seeded(*classes) + stored = await classes[3].collection().find_one({"_id": ann.id}) + assert stored["address"]["vault_code"] == "vault-home" # it is stored; it is only never shown + for url in ( + f"/v1/persons/{ann.id}", + "/v1/persons", + f"/v1/persons/{ann.id}?include=employer", + "/v1/persons?include=employer", + f"/v1/companies/{ann.employer.id}", + ): + response = await client.get(url) + assert response.status_code == 200, response.text + for secret in (*SECRETS, "vault-", "surveyor-"): + assert secret not in response.text, (url, secret) + body = (await client.get(f"/v1/persons/{ann.id}?include=employer")).json()["data"] + assert body["address"] == {"city": "home", "geo": {"lat": 1.5}} + assert [a["city"] for a in body["history"]] == ["old1", "old2"] and body["by_label"]["work"][ + "city" + ] == "work" + assert body["employer"]["hq"] == {"city": "hq", "geo": {"lat": 1.5}} + + +@pytest.mark.parametrize( + ("payload", "field"), + [ + ({"address": {"city": "x", "vault_code": "client-set"}}, "address.vault_code"), + ({"address": {"city": "x", "geo": {"lat": 1, "surveyor": "me"}}}, "address.geo.surveyor"), + ({"history": [{"city": "a"}, {"city": "b", "vault_code": "v"}]}, "history.1.vault_code"), + ({"by_label": {"work": {"geo": {"surveyor": "me"}}}}, "by_label.work.geo.surveyor"), + ], +) +async def test_nested_private_fields_are_never_written(world, payload, field): + client, api, (Geo, Addr, Company, Person) = world + existing = await Person(name="bo", address=Addr(city="here", vault_code="server-set")).insert() + for method, url in (("POST", "/v1/persons"), ("PATCH", f"/v1/persons/{existing.id}")): + response = await client.request(method, url, json={"name": "eve", **payload}) + assert response.status_code == 422, (method, response.text) + assert {"field": field, "message": "unknown field"} in response.json()["errors"] + assert await Person.count() == 1 + assert (await Person.get(existing.id)).address.vault_code == "server-set" + # without the private keys the same shapes are accepted + ok = await client.post("/v1/persons", json={"name": "ok", "address": {"city": "x", "geo": {"lat": 2}}}) + assert ok.status_code == 201 and ok.json()["data"]["address"] == {"city": "x", "geo": {"lat": 2.0}} + patched = await client.patch(f"/v1/persons/{existing.id}", json={"address": {"city": "moved"}}) + assert patched.status_code == 200 and patched.json()["data"]["address"]["city"] == "moved" + + +async def test_nested_private_fields_are_not_described_or_addressable(world): + client, api, classes = world + text = json.dumps(api.openapi()) + for secret in SECRETS: + assert secret not in text + assert "city" in text and "lat" in text # the public nested fields are described + await seeded(*classes) + for query in ( + "address.vault_code=x", + "address.vault_code[eq]=x", + "sort=address.vault_code", + "include=address.vault_code", + ): + response = await client.get(f"/v1/persons?{query}") + assert response.status_code == 400, (query, response.text) + assert "vault-" not in response.text + + +def test_views_without_embedded_private_fields_do_no_extra_work(): + from mongospecs_web.testing import contract_api + + assert view_of(contract_api("msgspec").resources[0].model)._nested == {} From 0dde820d64226ad22366be1b4d0c8b7e67530a4e Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 02:59:34 -0400 Subject: [PATCH 12/26] feat(core): StoreNone, OmitNone and SpecConfig(none=...) Leaving None-valued optional fields out of stored documents stays the default. StoreNone on a field, or none='store' on a model, stores an explicit null instead (OmitNone opts a field back out). Encoding, change tracking, update(), upsert defaults and the bulk paths follow the field's mode; loading never produces a spurious change in either mode. A unique sparse index over a field that stores null is refused with a pointer to a partial index. Also applies the omit rule to embedded pydantic models, which were encoded with their None values. Co-Authored-By: Claude Opus 5.5 --- claudedocs/v2-plan.md | 2 + docs/guide/models.md | 35 +++ .../tests/test_embedded_private.py | 43 ++++ .../mongospecs/src/mongospecs/__init__.py | 4 + packages/mongospecs/src/mongospecs/codec.py | 36 +++- packages/mongospecs/src/mongospecs/markers.py | 19 ++ packages/mongospecs/src/mongospecs/meta.py | 28 ++- .../mongospecs/src/mongospecs/pydantic.py | 93 +++++++- packages/mongospecs/tests/test_store_none.py | 203 ++++++++++++++++++ 9 files changed, 455 insertions(+), 8 deletions(-) create mode 100644 packages/mongospecs/tests/test_store_none.py diff --git a/claudedocs/v2-plan.md b/claudedocs/v2-plan.md index 9d209c9..0a3207f 100644 --- a/claudedocs/v2-plan.md +++ b/claudedocs/v2-plan.md @@ -885,6 +885,8 @@ Recorded per §7.1 rule 4. Each one names what changed and why; none invalidates | A31 | §4.2 | Field names that shadow the `Document` API raise `ConfigurationError` (`meta.RESERVED_FIELD_NAMES`) | Second review: a field named `count` silently replaced `Model.count()` | | A32 | §4.4 | `registry.transaction()` inside a running transaction raises unless `independent=True`. A context variable is used to detect this only; sessions are still always passed explicitly | Second review: an inner transaction committed while the outer aborted | | A33 | §4.3 | Filter values are cast by one rule on all backends (exact type, else text that reads as the type). `find().on_invalid("raise" \| "skip" \| "raw")` | Second review | +| A34 | §4.8 | `Policy(private=True)` is enforced on fields of embedded models at any depth: not serialised, a 422 when supplied, absent from schemas | Third review item A. PATCH replaces an embedded object whole, so a client that patches the parent object clears server-set private values inside it; documented, not changed | +| A35 | §4.2 | `StoreNone` / `OmitNone` markers and `SpecConfig(none="omit" \| "store")`. Embedded models resolve the setting from their own markers, or from a `__spec__` if they declare one; they do not inherit the parent document's setting. A unique sparse index over a field that stores null is a `ConfigurationError`. No warning is raised for a partial filter on `$type: null` over an omitted field | Requested as the opt-out from A30. The embedded rule keeps one model's stored shape the same wherever it is embedded | New node: diff --git a/docs/guide/models.md b/docs/guide/models.md index 26d2af6..bf5b2f5 100644 --- a/docs/guide/models.md +++ b/docs/guide/models.md @@ -38,6 +38,41 @@ class User(Spec): raises `ConfigurationError` when the model is first used. Rename the attribute and keep the stored key: `total: Annotated[int, Alias("count")] = 0`. +## None: left out, or stored as null + +A field whose default is `None` is left out of the document while it is `None`. Where other +readers of the collection need the key to be present (BI tools, a `$jsonSchema` validator, +`{field: {$type: "null"}}` queries), store an explicit `null` instead: + +```py +class Contact(Spec): + __spec__ = SpecConfig(none="omit") # the default; "store" flips it for the model + nick: str | None = None # left out while None + email: Annotated[str | None, StoreNone] = None # stored as null + # on a model with none="store": Annotated[str | None, OmitNone] opts one field out +``` + +| | Left out (default, `OmitNone`) | Stored (`StoreNone`, `none="store"`) | +|---|---|---| +| A new document with `None` | no key | `key: null` | +| `doc.field = None; save()` and `update(field=None)` | `$unset` | `$set: null` | +| A document created by `upsert()` | no key | `key: null` | +| `Q.field == None` | matches | matches | +| `Exists(Q.field)`, `?field[exists]=true` | "has a value" | "the key is present", null included | +| `Unset("field")` | removes the key | removes the key | + +- The setting changes what is written, never what can be read. A stored `null` under the default + and a missing key under `StoreNone` both load as `None` with `changes() == {}`, and saving such a + document sends nothing for that field. To convert a collection, run one `update_many`. +- Embedded models follow their own field markers. They omit by default; an embedded model may + carry `__spec__ = SpecConfig(none="store")` to change that for its fields. +- A field that stores `null` cannot be a key of a unique **sparse** index: a sparse index still + indexes `null`, so only one document could be without a value. This raises + `ConfigurationError` when the model is first used. Use a partial index: + `Index("email", unique=True, partial={"email": {"$type": "string"}})`. +- HTTP responses are the same in both modes (`None` is `null` in JSON). A `PATCH` with `null` + follows the field's mode. + ## Binding and lifecycle Models do nothing until they are bound to an open registry. Constructing a registry does no I/O; diff --git a/packages/mongospecs-web/tests/test_embedded_private.py b/packages/mongospecs-web/tests/test_embedded_private.py index 2c237df..e16c2e1 100644 --- a/packages/mongospecs-web/tests/test_embedded_private.py +++ b/packages/mongospecs-web/tests/test_embedded_private.py @@ -169,3 +169,46 @@ def test_views_without_embedded_private_fields_do_no_extra_work(): from mongospecs_web.testing import contract_api assert view_of(contract_api("msgspec").resources[0].model)._nested == {} + + +async def test_null_in_a_request_follows_the_fields_mode(request, make_registry): + """PATCH/POST with an explicit null: unset where None is omitted, stored where StoreNone says so.""" + from mongospecs import StoreNone + + results = {} + for backend in BACKENDS: + if backend == "attrs": + import attrs + + from mongospecs.attrs import Spec + + define = attrs.define(kw_only=True) + else: + Spec = __import__(f"mongospecs.{backend}", fromlist=["Spec"]).Spec + define = lambda c: c # noqa: E731 + + @define + class Contact(Spec): + __spec__ = SpecConfig(collection=f"contacts_{backend}") + name: str = "" + nick: str | None = None + email: t.Annotated[str | None, StoreNone] = None + + await make_registry(Contact) + api = Api(Resource(Contact, name="contacts", ops=[Get, Create, Patch]), prefix="/v1") + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=api.asgi()), base_url="http://t" + ) as client: + created = await client.post("/v1/contacts", json={"name": "a", "nick": None, "email": None}) + assert created.status_code == 201 and created.json()["data"]["nick"] is None + oid = Contact.collection() + stored = await oid.find_one({"name": "a"}) + assert "nick" not in stored and "email" in stored and stored["email"] is None + await oid.update_one({"_id": stored["_id"]}, {"$set": {"nick": "n", "email": "e"}}) + patched = await client.patch(f"/v1/contacts/{stored['_id']}", json={"nick": None, "email": None}) + assert patched.status_code == 200 + assert patched.json()["data"]["nick"] is None and patched.json()["data"]["email"] is None + stored = await oid.find_one({"_id": stored["_id"]}) + assert "nick" not in stored and "email" in stored and stored["email"] is None + results[backend] = patched.json()["data"].keys() + assert len(results) == 3 diff --git a/packages/mongospecs/src/mongospecs/__init__.py b/packages/mongospecs/src/mongospecs/__init__.py index 1bb440c..f3f631a 100644 --- a/packages/mongospecs/src/mongospecs/__init__.py +++ b/packages/mongospecs/src/mongospecs/__init__.py @@ -56,10 +56,12 @@ DeletedAt, Immutable, Index, + OmitNone, OnDelete, Policy, Revision, SpecConfig, + StoreNone, To, UpdatedAt, computed, @@ -139,6 +141,7 @@ "NotBoundError", "NotIn", "NotLoadedError", + "OmitNone", "OnDelete", "OpCtx", "OperationFailed", @@ -164,6 +167,7 @@ "SortBy", "SpecConfig", "StaleDocumentError", + "StoreNone", "To", "TransactionError", "Tx", diff --git a/packages/mongospecs/src/mongospecs/codec.py b/packages/mongospecs/src/mongospecs/codec.py index 2dd1b11..a82ccd8 100644 --- a/packages/mongospecs/src/mongospecs/codec.py +++ b/packages/mongospecs/src/mongospecs/codec.py @@ -18,7 +18,17 @@ from bson.raw_bson import RawBSONDocument from mongospecs.errors import ConfigurationError, ValidationError -from mongospecs.markers import DEFAULT_POLICY, Alias, Backref, Marker, Policy, To +from mongospecs.markers import ( + DEFAULT_POLICY, + Alias, + Backref, + Marker, + OmitNone, + Policy, + SpecConfig, + StoreNone, + To, +) from mongospecs.refs import Ref, id_of __all__ = ["Codec", "FieldInfo", "RefInfo", "codec_for", "to_bson"] @@ -333,6 +343,9 @@ def encode(self, obj: t.Any) -> dict[str, t.Any]: def absent_when_none(self, cls: type) -> frozenset[str]: """Stored keys of the fields whose default is ``None``. + ``StoreNone`` on a field, or ``SpecConfig(none="store")`` on its model, takes a + field out of this set; ``OmitNone`` puts it back. + For these, ``None`` means "no value" and the key is left out of the stored document instead of being written as ``null``. A document without the key decodes back to ``None``, so nothing is lost, and sparse and unique indexes @@ -343,10 +356,16 @@ def absent_when_none(self, cls: type) -> frozenset[str]: hit = cache.get(cls) if hit is None: defaults = self.defaults(cls) + store = _none_mode(cls) == "store" hit = frozenset( f.stored for f in self.fields(cls).values() - if f.stored != "_id" and not f.transient and f.name in defaults and defaults[f.name] is None + if f.stored != "_id" + and not f.transient + and f.name in defaults + and defaults[f.name] is None + and not f.has(StoreNone) + and (not store or f.has(OmitNone)) ) cache[cls] = hit return hit @@ -397,6 +416,19 @@ def clear_cache(self) -> None: self.__dict__.pop("_rn", None) self.__dict__.pop("_sp", None) self.__dict__.pop("_abs", None) + self.__dict__.pop("_eo", None) + + +def _none_mode(cls: type) -> str: + """``"omit"`` or ``"store"`` for a model class: from its SpecConfig, own or inherited.""" + from mongospecs.model import Document + + if isinstance(cls, type) and issubclass(cls, Document): + from mongospecs.meta import resolve_config + + return resolve_config(cls).none + config = getattr(cls, "__spec__", None) # an embedded model may declare one + return config.none if isinstance(config, SpecConfig) else "omit" def field_error(cls: type, name: str | None, message: str) -> ValidationError: diff --git a/packages/mongospecs/src/mongospecs/markers.py b/packages/mongospecs/src/mongospecs/markers.py index e6c7d86..6f832d1 100644 --- a/packages/mongospecs/src/mongospecs/markers.py +++ b/packages/mongospecs/src/mongospecs/markers.py @@ -22,10 +22,12 @@ "Immutable", "Index", "Marker", + "OmitNone", "OnDelete", "Policy", "Revision", "SpecConfig", + "StoreNone", "To", "UpdatedAt", "computed", @@ -59,6 +61,18 @@ class Immutable(Marker): """The field cannot change after insert.""" +class StoreNone(Marker): + """Store ``None`` as an explicit ``null`` instead of leaving the key out. + + By default a field whose default is ``None`` is not stored while it is ``None``. + Use this where other readers of the collection expect the key to be present. + """ + + +class OmitNone(Marker): + """Leave the key out while the value is ``None``, on a model declared with ``none="store"``.""" + + class Deferred(Marker): """The field is left out of the default projection; ask for it with ``select("+name")``.""" @@ -244,4 +258,9 @@ class SpecConfig: """Options for ``create_collection`` (``timeseries``, ``capped``, ``size``, ...).""" json_schema_validation: t.Literal["off", "moderate", "strict"] = "off" + none: t.Literal["omit", "store"] = "omit" + """What to write for a field whose default is ``None`` while its value is ``None``: + leave the key out (``"omit"``) or store ``null`` (``"store"``). ``StoreNone`` and + ``OmitNone`` on a field override it. Embedded models may carry a ``__spec__`` with + this setting too; without one they omit.""" """Install a derived ``$jsonSchema`` validator when indexes and options are applied.""" diff --git a/packages/mongospecs/src/mongospecs/meta.py b/packages/mongospecs/src/mongospecs/meta.py index 19b2a19..dd0bfb8 100644 --- a/packages/mongospecs/src/mongospecs/meta.py +++ b/packages/mongospecs/src/mongospecs/meta.py @@ -159,6 +159,29 @@ def __init__(self, model: type) -> None: f.stored for f in self.fields.values() if not f.transient ) + # A sparse index still indexes an explicit null, so a unique sparse index over a field + # that stores None as null lets only one document be without a value. + absent = self.codec.absent_when_none(model) + defaults = self.codec.defaults(model) + for index in self.config.indexes: + if not (index.unique and index.sparse): + continue + for key, _ in index.keys: + f = self.fields.get(key.partition(".")[0]) + if ( + f is not None + and f.name in defaults + and defaults[f.name] is None + and f.stored not in absent + ): + raise ConfigurationError( + f"{model.__name__}.{f.name} stores None as null (StoreNone or none='store') and is a key " + "of a unique sparse index. A sparse index still indexes null, so a second document " + "without a value would be a duplicate. Use a partial index instead, for example " + f"Index({f.name!r}, unique=True, partial={{{f.name!r}: {{'$type': 'string'}}}}), " + "or mark the field OmitNone." + ) + for f in self.fields.values(): if f.has(Deferred) and f.required: raise ConfigurationError(f"{model.__name__}.{f.name}: a Deferred field needs a default") @@ -318,10 +341,13 @@ def insert_defaults(self) -> dict[str, t.Any]: by ``insert()``; otherwise a query on a defaulted value would not find it. """ out: dict[str, t.Any] = {} + absent = self.codec.absent_when_none(self.model) for name, value in self.codec.defaults(self.model).items(): info = self.fields.get(name) - if info is None or info.transient or name == "id" or value is None: + if info is None or info.transient or name == "id": continue + if value is None and info.stored in absent: + continue # not stored while None; a StoreNone field is written as null out[info.stored] = to_bson(value) return out diff --git a/packages/mongospecs/src/mongospecs/pydantic.py b/packages/mongospecs/src/mongospecs/pydantic.py index 4515a80..8cbcf32 100644 --- a/packages/mongospecs/src/mongospecs/pydantic.py +++ b/packages/mongospecs/src/mongospecs/pydantic.py @@ -9,7 +9,7 @@ import pydantic from pydantic import BaseModel, ConfigDict, Field, TypeAdapter -from mongospecs.codec import Codec, RawField, field_error +from mongospecs.codec import _UNION_ORIGINS, Codec, RawField, field_error from mongospecs.errors import ValidationError from mongospecs.markers import SpecConfig from mongospecs.model import Document @@ -38,6 +38,45 @@ def _annotation(info: t.Any) -> t.Any: return info.annotation +class _Omit: + """Which keys of an embedded model are left out while None, and where its own embedded models are.""" + + __slots__ = ("absent", "children", "elem") + + def __init__(self) -> None: + self.absent: frozenset[str] = frozenset() + self.children: dict[str, _Omit] = {} + self.elem: _Omit | None = None + + def active(self, seen: set[int] | None = None) -> bool: + seen = set() if seen is None else seen + if id(self) in seen: + return False + seen.add(id(self)) + return ( + bool(self.absent) + or any(c.active(seen) for c in self.children.values()) + or (self.elem is not None and self.elem.active(seen)) + ) + + +def _drop_nones(value: t.Any, plan: _Omit) -> None: + if isinstance(value, dict): + if plan.elem is not None: + for item in value.values(): + _drop_nones(item, plan.elem) + return + for key in plan.absent: + if key in value and value[key] is None: + del value[key] + for key, child in plan.children.items(): + if key in value: + _drop_nones(value[key], child) + elif isinstance(value, list) and plan.elem is not None: + for item in value: + _drop_nones(item, plan.elem) + + class PydanticCodec(Codec): name = "pydantic" @@ -60,12 +99,56 @@ def _decode(self, cls: type, doc: dict[str, t.Any], lax: bool) -> t.Any: def _encode(self, obj: t.Any) -> dict[str, t.Any]: if _HAS_EXCLUDE_COMPUTED: - return t.cast("dict[str, t.Any]", obj.model_dump(by_alias=True, exclude_computed_fields=True)) - doc: dict[str, t.Any] = obj.model_dump(by_alias=True) - for name in type(obj).model_computed_fields: - doc.pop(name, None) + doc: dict[str, t.Any] = obj.model_dump(by_alias=True, exclude_computed_fields=True) + else: + doc = obj.model_dump(by_alias=True) + for name in type(obj).model_computed_fields: + doc.pop(name, None) + # model_dump also dumps embedded models, so the "None is not stored" rule of each + # embedded model has to be applied here; the other backends encode them one by one. + for key, plan in self._embedded_omits(type(obj)): + if key in doc: + _drop_nones(doc[key], plan) return doc + def _embedded_omits(self, cls: type) -> tuple[tuple[str, _Omit], ...]: + cache: dict[type, tuple[tuple[str, _Omit], ...]] = self.__dict__.setdefault("_eo", {}) + hit = cache.get(cls) + if hit is None: + plans: dict[type, _Omit] = {} + natives = {name: native for name, native, _, _ in self.raw_fields(cls)} + hit = tuple( + (natives[name], plan) + for name, info in self.fields(cls).items() + if (plan := self._omit_plan(info.type, plans)) is not None and plan.active() + ) + cache[cls] = hit + return hit + + def _omit_plan(self, tp: t.Any, plans: dict[type, _Omit]) -> _Omit | None: + while t.get_origin(tp) is t.Annotated: + tp = t.get_args(tp)[0] + if t.get_origin(tp) is None: + if not (isinstance(tp, type) and issubclass(tp, BaseModel)) or issubclass(tp, Document): + return None + plan = plans.get(tp) + if plan is None: + plan = plans[tp] = _Omit() + plan.absent = self.absent_when_none(tp) + for name, _native, annotation, _ in self.raw_fields(tp): + child = self._omit_plan(annotation, plans) + if child is not None: + plan.children[self.fields(tp)[name].stored] = child + return plan + parts = [p for arg in t.get_args(tp) if (p := self._omit_plan(arg, plans)) is not None] + if not parts: + return None + if t.get_origin(tp) in _UNION_ORIGINS: + return parts[0] + container = _Omit() + container.elem = parts[-1] + return container + def validate_field(self, cls: type, name: str, value: t.Any) -> t.Any: model = t.cast("type[BaseModel]", cls) try: diff --git a/packages/mongospecs/tests/test_store_none.py b/packages/mongospecs/tests/test_store_none.py new file mode 100644 index 0000000..939e1ab --- /dev/null +++ b/packages/mongospecs/tests/test_store_none.py @@ -0,0 +1,203 @@ +"""StoreNone / OmitNone / SpecConfig(none=...): whether a None value is stored as null or left out.""" + +import typing as t + +import pytest + +from mongospecs import ( + ConfigurationError, + Exists, + Index, + InsertOne, + OmitNone, + Q, + SpecConfig, + StoreNone, + Unset, + UpdateOne, + meta_of, +) + + +def embedded_base(m): + if m.backend == "msgspec": + import msgspec + + class Base(msgspec.Struct, kw_only=True): + pass + + return Base + if m.backend == "pydantic": + import pydantic + + return pydantic.BaseModel + return object + + +@pytest.fixture +async def models(m, make_registry): + Base = embedded_base(m) + + @m.define + class Part(Base): + label: str = "" + note: str | None = None # omitted while None + code: t.Annotated[str | None, StoreNone] = None # stored as null + + @m.define + class Profile(m.Spec): + __spec__ = SpecConfig(collection="profiles") + name: str = "" + nick: str | None = None # the default: omitted + email: t.Annotated[str | None, StoreNone] = None + part: Part | None = None + parts: list[Part] = [] # noqa: RUF012 + + @m.define + class Legacy(m.Spec): + __spec__ = SpecConfig(collection="legacy", none="store") + name: str = "" + phone: str | None = None # stored as null: the model says so + fax: t.Annotated[str | None, OmitNone] = None # the field opts out + + await make_registry(Profile, Legacy) + return Profile, Legacy, Part + + +async def raw(model, doc): + return await model.collection().find_one({"_id": doc.id}) + + +async def test_round_trip_per_mode(models): + Profile, Legacy, Part = models + doc = await Profile(name="a").insert() + stored = await raw(Profile, doc) + assert stored["email"] is None and "email" in stored # StoreNone: an explicit null + assert "nick" not in stored # omit mode: no key + loaded = await Profile.get(doc.id) + assert loaded.email is None and loaded.nick is None and loaded.changes() == {} + + +async def test_delta_save_per_mode(models): + Profile, Legacy, Part = models + doc = await Profile(name="a").insert() + doc.email, doc.nick = "a@x.io", "ann" + assert doc.changes() == {"$set": {"email": "a@x.io", "nick": "ann"}} + await doc.save() + doc.email, doc.nick = None, None + assert doc.changes() == {"$set": {"email": None}, "$unset": {"nick": ""}} + await doc.save() + stored = await raw(Profile, doc) + assert stored["email"] is None and "email" in stored and "nick" not in stored + assert (await Profile.get(doc.id)).changes() == {} + + +async def test_update_and_upsert_per_mode(models): + Profile, Legacy, Part = models + doc = await Profile(name="a", email="a@x.io", nick="ann").insert() + await Profile.update(doc.id, email=None, nick=None) + stored = await raw(Profile, doc) + assert "email" in stored and stored["email"] is None and "nick" not in stored + created = await Profile.upsert(Q.name == "fresh", name="fresh") + stored = await raw(Profile, created) + assert "email" in stored and stored["email"] is None and "nick" not in stored + assert created.changes() == {} + # Unset always removes the key, whatever the mode + await Profile.update(doc.id, Unset("email")) + assert "email" not in await raw(Profile, doc) + + +async def test_bulk_paths_per_mode(models): + Profile, Legacy, Part = models + many = (await Profile.insert_many([Profile(name="m1"), Profile(name="m2")])).inserted + fresh, existing = Profile(name="b1"), many[0] + existing.name = "m1!" + await Profile.bulk_save([fresh, existing]) + await Profile.bulk_write( + [InsertOne(Profile(name="w1")), UpdateOne(Q.name == "m2", email=None, nick=None)] + ) + docs = [d async for d in Profile.collection().find({})] + assert len(docs) == 4 + for stored in docs: + assert "email" in stored and stored["email"] is None, stored + assert "nick" not in stored, stored + + +async def test_model_wide_setting_and_field_opt_out(models): + Profile, Legacy, Part = models + doc = await Legacy(name="l").insert() + stored = await raw(Legacy, doc) + assert "phone" in stored and stored["phone"] is None # none="store" + assert "fax" not in stored # OmitNone wins over the model + doc.phone, doc.fax = "1", "2" + await doc.save() + doc.phone, doc.fax = None, None + assert doc.changes() == {"$set": {"phone": None}, "$unset": {"fax": ""}} + await Legacy.update(doc.id, phone=None, fax=None) + stored = await raw(Legacy, doc) + assert "phone" in stored and "fax" not in stored + + +async def test_embedded_models_follow_their_own_field_markers(models): + Profile, Legacy, Part = models + doc = await Profile( + name="e", part=Part(label="one"), parts=[Part(label="a"), Part(label="b", code="c")] + ).insert() + stored = await raw(Profile, doc) + assert stored["part"] == {"label": "one", "code": None} # note omitted, code stored as null + assert stored["parts"] == [{"label": "a", "code": None}, {"label": "b", "code": "c"}] + loaded = await Profile.get(doc.id) + assert loaded.changes() == {} and loaded.part.note is None and loaded.parts[0].code is None + + +async def test_exists_means_different_things_per_mode(models): + Profile, Legacy, Part = models + await Profile.insert_many([Profile(name="none"), Profile(name="both", email="e", nick="n")]) + assert await Profile.count(Exists(Q.nick)) == 1 # omit: "has a value" + assert await Profile.count(Exists(Q.email)) == 2 # StoreNone: "the key is present", null included + # comparing with None matches "no value" in both modes + assert await Profile.count(Q.nick == None) == 1 and await Profile.count(Q.email == None) == 1 # noqa: E711 + + +async def test_existing_data_loads_clean_in_both_modes(models): + Profile, Legacy, Part = models + # a null where the model omits, and a missing key where the model stores null + result = await Profile.collection().insert_one({"name": "old", "nick": None}) + loaded = await Profile.get(result.inserted_id) + assert loaded.nick is None and loaded.email is None and loaded.changes() == {} + await loaded.save() # nothing to send + stored = await Profile.collection().find_one({"_id": result.inserted_id}) + assert stored == {"_id": result.inserted_id, "name": "old", "nick": None} # untouched + loaded.name = "renamed" + assert loaded.changes() == {"$set": {"name": "renamed"}} # and only real changes are sent + + +def test_sparse_unique_index_on_a_stored_null_is_refused(m): + def build(config, annotation): + if m.backend == "attrs": + import attrs + + cls = attrs.make_class( + "Clash", {"email": attrs.field(default=None, type=annotation)}, bases=(m.Spec,), kw_only=True + ) + elif m.backend == "pydantic": + import pydantic + + cls = pydantic.create_model("Clash", __base__=m.Spec, email=(annotation, None)) + else: + import msgspec + + cls = msgspec.defstruct("Clash", [("email", annotation, None)], bases=(m.Spec,), kw_only=True) + cls.__spec__ = config + return cls + + sparse = [Index("email", unique=True, sparse=True)] + with pytest.raises(ConfigurationError, match="sparse.*partial"): + meta_of(build(SpecConfig(indexes=sparse), t.Annotated[str | None, StoreNone])) + with pytest.raises(ConfigurationError, match="sparse.*partial"): + meta_of(build(SpecConfig(indexes=sparse, none="store"), str | None)) + # fine: omitted while None, or a partial index instead of a sparse one + meta_of(build(SpecConfig(indexes=sparse), str | None)) + meta_of(build(SpecConfig(indexes=sparse, none="store"), t.Annotated[str | None, OmitNone])) + partial = [Index("email", unique=True, partial={"email": {"$type": "string"}})] + meta_of(build(SpecConfig(indexes=partial), t.Annotated[str | None, StoreNone])) From 49a32a93d0546165e8358a6b2072c14f8c4720a1 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 03:00:31 -0400 Subject: [PATCH 13/26] bench: B9, CPU per call and commands per operation on the hot paths Co-Authored-By: Claude Opus 5.5 --- bench/src/mongospecs_bench/hotpaths.py | 175 ++++++++++++++++++++++++ bench/src/mongospecs_bench/workloads.py | 7 + 2 files changed, 182 insertions(+) create mode 100644 bench/src/mongospecs_bench/hotpaths.py diff --git a/bench/src/mongospecs_bench/hotpaths.py b/bench/src/mongospecs_bench/hotpaths.py new file mode 100644 index 0000000..095e3d3 --- /dev/null +++ b/bench/src/mongospecs_bench/hotpaths.py @@ -0,0 +1,175 @@ +"""Workload B9: CPU cost of the hot paths, measured through public entry points only. + +Wall-clock time on a laptop is noisy; CPU time of this process and the number of +commands sent are not. Every row here is one of those two, so a change to the +library can be judged by running this workload before and after it. +""" + +from __future__ import annotations + +import datetime as dt +import time +import typing as t + +from mongospecs import Q, Registry +from mongospecs_bench.harness import Result, Settings, environment +from mongospecs_bench.storefront import models as shop_models +from mongospecs_web import Api, Cursor, Delete, Get, List, Resource + + +async def cpu_us(fn: t.Callable[[], t.Awaitable[t.Any]], calls: int, repeats: int) -> float: + """CPU microseconds per call: the best of ``repeats`` batches of ``calls``.""" + for _ in range(min(calls, 20)): + await fn() + best = float("inf") + for _ in range(repeats): + start = time.process_time() + for _ in range(calls): + await fn() + best = min(best, (time.process_time() - start) / calls) + return round(best * 1e6, 1) + + +class Commands: + """Counts the commands a block sends.""" + + def __init__(self, registry: Registry) -> None: + self.names: list[str] = [] + self._registry = registry + + def __call__(self, event: t.Any) -> None: + if event.phase == "started": + self.names.append(event.command_name) + + +def _api(m: t.Any, routes: int = 0) -> Api: + resources: list[Resource[t.Any]] = [ + Resource(m.Product, name="products", ops=[List, Get, Delete], page=Cursor(default=20, max=100)), + Resource(m.Review, name="reviews", ops=[List, Get], include=["product", "author"]), + ] + # extra mount points of the same model, to measure routing as an API grows + resources += [Resource(m.Review, name=f"r{i}", ops=[List, Get]) for i in range(routes)] + return Api(*resources, prefix="/api") + + +async def b9(settings: Settings) -> Result: + result = Result( + "B9", + "Hot paths: CPU per call and commands per operation", + "Where does the library spend CPU on the paths every request takes, and how many commands " + "does each operation send?", + ) + calls = settings.scale(300, 20) + repeats = settings.scale(5, 2) + products = settings.scale(400, 120) + for backend in settings.backends: + m = shop_models.build(backend) + events = Commands.__new__(Commands) + events.names = [] + async with environment(settings, m.ALL, on_command=events) as registry: + await registry.apply_indexes() + now = dt.datetime.now(dt.UTC) + await m.Product.insert_many( + [ + m.Product( + sku=f"S{i:05d}", + name=f"Product {i}", + category="tools", + price=float(i), + stock=i, + tags=["a", "b", "c"], + description="x" * 200, + created_at=now, + ) + for i in range(products) + ] + ) + users = ( + await m.User.insert_many([m.User(email=f"u{i}@x.io", name=f"u{i}") for i in range(20)]) + ).inserted + items = await m.Product.find().sort("sku").all() + await m.Review.insert_many( + [ + m.Review(product=items[i % 20], rating=1 + i % 5, body="fine", author=users[i % 20]) + for i in range(2000) + ] + ) + first = items[0].id + api = _api(m) + + async def listing() -> None: + response = await api.handle("GET", "/api/products", query=[("limit", "100")]) + assert response.status == 200, response.body + + async def detail() -> None: + assert (await api.handle("GET", f"/api/products/{first}")).status == 200 + + async def model_get() -> None: + assert await m.Product.get(first) is not None + + async def tracked_page() -> None: + assert len(await m.Product.find().limit(100).all()) == 100 + + async def lean_page() -> None: + assert len(await m.Product.find().lean().limit(100).all()) == 100 + + async def two_includes() -> None: + assert len(await m.Review.find().limit(100).include("product").include("author").all()) == 100 + + async def per_parent() -> None: + query = m.Product.find(Q.sku < "S00020").include("reviews", per_parent_limit=2) + assert len(await query.all()) == 20 + + def add(path: str, unit: str, value: float, per: int = 1) -> None: + result.add(backend=backend, path=path, unit=unit, value=round(value / per, 2)) + + add("web: GET list of 100", "cpu us/doc", await cpu_us(listing, calls, repeats), 100) + add("web: GET one", "cpu us/call", await cpu_us(detail, calls, repeats)) + add("Model.get(id)", "cpu us/call", await cpu_us(model_get, calls, repeats)) + add("find().all(), 100 tracked", "cpu us/doc", await cpu_us(tracked_page, calls, repeats), 100) + add("find().lean().all(), 100", "cpu us/doc", await cpu_us(lean_page, calls, repeats), 100) + add("include x2, 100 parents", "cpu us/call", await cpu_us(two_includes, calls // 4, repeats)) + add( + "include reviews, per_parent_limit=2 of 100", + "cpu us/call", + await cpu_us(per_parent, calls // 4, repeats), + ) + + def count(name: str, *paths: str) -> None: + add(name, "commands", float(len(events.names))) + result.rows[-1]["detail"] = " ".join(events.names) + + for name, run in ( + ("web: GET one", detail), + ("web: GET list of 100", listing), + ("include x2, 100 parents", two_includes), + ("include reviews, per_parent_limit=2 of 100", per_parent), + ): + events.names.clear() + await run() + count(name) + victim = items[-1].id + events.names.clear() + assert (await api.handle("DELETE", f"/api/products/{victim}")).status == 204 + count("web: DELETE (soft)") + + if backend == settings.backends[0]: + for routes in (25, 250, 1000): + big = _api(m, routes // 2) + path = f"/api/r{routes // 2 - 1}/000000000000000000000000" + for _ in range(200): + big.match("GET", path) + start = time.process_time() + for _ in range(2000): + big.match("GET", path) + result.add( + backend="-", + path=f"route match, last of {len(big.routes)} routes", + unit="cpu us/call", + value=round((time.process_time() - start) / 2000 * 1e6, 2), + ) + result.notes.append( + "CPU time of this process (best of several batches), so the database's own work and network waits " + "are excluded. Rows with unit 'commands' are driver commands sent, listed in 'detail'." + ) + return result diff --git a/bench/src/mongospecs_bench/workloads.py b/bench/src/mongospecs_bench/workloads.py index 6c6870f..1af6047 100644 --- a/bench/src/mongospecs_bench/workloads.py +++ b/bench/src/mongospecs_bench/workloads.py @@ -552,6 +552,12 @@ async def b8(settings: Settings) -> Result: return result +async def _b9(settings: Settings) -> Result: + from mongospecs_bench.hotpaths import b9 + + return await b9(settings) + + WORKLOADS: dict[str, Workload] = { "B1": b1, "B2": b2, @@ -560,4 +566,5 @@ async def b8(settings: Settings) -> Result: "B5": b5, "B6": b6, "B8": b8, + "B9": _b9, } From ce9173182e4c209cb3c8742fe30d3a377eb62432 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 06:59:06 -0400 Subject: [PATCH 14/26] perf(core,web): cut work on the read, include, delete and routing paths - web GET list/get read stored documents with a projection of the readable fields and serialise them directly when that is certain to equal the model path; anything else falls back to the model - DELETE is one conditional write unless the model needs the document (delete hooks, cascades) - independent includes are loaded concurrently (not inside a session) - per_parent_limit on a reverse relation is applied on the server - tracked documents use one weak reference instead of weakref.finalize - Query._clone copies slots instead of re-running __init__ - routes are looked up by resource before patterns are tried fix(core): cursor pagination no longer skips or stops at documents without a value for a sort key Co-Authored-By: Claude Opus 5.5 --- bench/results/49a32a9/B9.json | 279 +++++++++++++ .../mongospecs-web/src/mongospecs_web/api.py | 50 ++- .../src/mongospecs_web/backend.py | 39 +- .../src/mongospecs_web/fastread.py | 284 +++++++++++++ .../src/mongospecs_web/policy.py | 29 +- .../src/mongospecs_web/resource.py | 29 +- .../mongospecs-web/tests/test_fast_reads.py | 382 ++++++++++++++++++ .../tests/test_mongo_resource.py | 5 +- packages/mongospecs/src/mongospecs/manager.py | 53 +++ packages/mongospecs/src/mongospecs/query.py | 59 ++- .../mongospecs/src/mongospecs/relations.py | 102 +++-- .../mongospecs/src/mongospecs/tracking.py | 54 ++- .../mongospecs/tests/test_review_round2.py | 27 ++ 13 files changed, 1318 insertions(+), 74 deletions(-) create mode 100644 bench/results/49a32a9/B9.json create mode 100644 packages/mongospecs-web/src/mongospecs_web/fastread.py create mode 100644 packages/mongospecs-web/tests/test_fast_reads.py diff --git a/bench/results/49a32a9/B9.json b/bench/results/49a32a9/B9.json new file mode 100644 index 0000000..84946fd --- /dev/null +++ b/bench/results/49a32a9/B9.json @@ -0,0 +1,279 @@ +{ + "meta": { + "git_sha": "49a32a9", + "git_dirty": false, + "timestamp": "2026-10-13T07:00:32+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B9", + "title": "Hot paths: CPU per call and commands per operation", + "question": "Where does the library spend CPU on the paths every request takes, and how many commands does each operation send?", + "rows": [ + { + "backend": "msgspec", + "path": "web: GET list of 100", + "unit": "cpu us/doc", + "value": 10.92 + }, + { + "backend": "msgspec", + "path": "web: GET one", + "unit": "cpu us/call", + "value": 272.2 + }, + { + "backend": "msgspec", + "path": "Model.get(id)", + "unit": "cpu us/call", + "value": 248.4 + }, + { + "backend": "msgspec", + "path": "find().all(), 100 tracked", + "unit": "cpu us/doc", + "value": 8.7 + }, + { + "backend": "msgspec", + "path": "find().lean().all(), 100", + "unit": "cpu us/doc", + "value": 6.66 + }, + { + "backend": "msgspec", + "path": "include x2, 100 parents", + "unit": "cpu us/call", + "value": 2239.9 + }, + { + "backend": "msgspec", + "path": "include reviews, per_parent_limit=2 of 100", + "unit": "cpu us/call", + "value": 23049.0 + }, + { + "backend": "msgspec", + "path": "web: GET one", + "unit": "commands", + "value": 1.0, + "detail": "find" + }, + { + "backend": "msgspec", + "path": "web: GET list of 100", + "unit": "commands", + "value": 1.0, + "detail": "find" + }, + { + "backend": "msgspec", + "path": "include x2, 100 parents", + "unit": "commands", + "value": 3.0, + "detail": "find find find" + }, + { + "backend": "msgspec", + "path": "include reviews, per_parent_limit=2 of 100", + "unit": "commands", + "value": 3.0, + "detail": "find find getMore" + }, + { + "backend": "msgspec", + "path": "web: DELETE (soft)", + "unit": "commands", + "value": 2.0, + "detail": "find update" + }, + { + "backend": "-", + "path": "route match, last of 29 routes", + "unit": "cpu us/call", + "value": 4.41 + }, + { + "backend": "-", + "path": "route match, last of 255 routes", + "unit": "cpu us/call", + "value": 19.12 + }, + { + "backend": "-", + "path": "route match, last of 1005 routes", + "unit": "cpu us/call", + "value": 73.25 + }, + { + "backend": "pydantic", + "path": "web: GET list of 100", + "unit": "cpu us/doc", + "value": 14.36 + }, + { + "backend": "pydantic", + "path": "web: GET one", + "unit": "cpu us/call", + "value": 251.2 + }, + { + "backend": "pydantic", + "path": "Model.get(id)", + "unit": "cpu us/call", + "value": 219.6 + }, + { + "backend": "pydantic", + "path": "find().all(), 100 tracked", + "unit": "cpu us/doc", + "value": 10.93 + }, + { + "backend": "pydantic", + "path": "find().lean().all(), 100", + "unit": "cpu us/doc", + "value": 8.18 + }, + { + "backend": "pydantic", + "path": "include x2, 100 parents", + "unit": "cpu us/call", + "value": 2281.2 + }, + { + "backend": "pydantic", + "path": "include reviews, per_parent_limit=2 of 100", + "unit": "cpu us/call", + "value": 23069.5 + }, + { + "backend": "pydantic", + "path": "web: GET one", + "unit": "commands", + "value": 1.0, + "detail": "find" + }, + { + "backend": "pydantic", + "path": "web: GET list of 100", + "unit": "commands", + "value": 1.0, + "detail": "find" + }, + { + "backend": "pydantic", + "path": "include x2, 100 parents", + "unit": "commands", + "value": 3.0, + "detail": "find find find" + }, + { + "backend": "pydantic", + "path": "include reviews, per_parent_limit=2 of 100", + "unit": "commands", + "value": 3.0, + "detail": "find find getMore" + }, + { + "backend": "pydantic", + "path": "web: DELETE (soft)", + "unit": "commands", + "value": 2.0, + "detail": "find update" + }, + { + "backend": "attrs", + "path": "web: GET list of 100", + "unit": "cpu us/doc", + "value": 17.25 + }, + { + "backend": "attrs", + "path": "web: GET one", + "unit": "cpu us/call", + "value": 252.9 + }, + { + "backend": "attrs", + "path": "Model.get(id)", + "unit": "cpu us/call", + "value": 393.9 + }, + { + "backend": "attrs", + "path": "find().all(), 100 tracked", + "unit": "cpu us/doc", + "value": 14.34 + }, + { + "backend": "attrs", + "path": "find().lean().all(), 100", + "unit": "cpu us/doc", + "value": 11.69 + }, + { + "backend": "attrs", + "path": "include x2, 100 parents", + "unit": "cpu us/call", + "value": 2787.5 + }, + { + "backend": "attrs", + "path": "include reviews, per_parent_limit=2 of 100", + "unit": "cpu us/call", + "value": 25883.3 + }, + { + "backend": "attrs", + "path": "web: GET one", + "unit": "commands", + "value": 1.0, + "detail": "find" + }, + { + "backend": "attrs", + "path": "web: GET list of 100", + "unit": "commands", + "value": 1.0, + "detail": "find" + }, + { + "backend": "attrs", + "path": "include x2, 100 parents", + "unit": "commands", + "value": 3.0, + "detail": "find find find" + }, + { + "backend": "attrs", + "path": "include reviews, per_parent_limit=2 of 100", + "unit": "commands", + "value": 3.0, + "detail": "find find getMore" + }, + { + "backend": "attrs", + "path": "web: DELETE (soft)", + "unit": "commands", + "value": 2.0, + "detail": "find update" + } + ], + "notes": [ + "CPU time of this process (best of several batches), so the database's own work and network waits are excluded. Rows with unit 'commands' are driver commands sent, listed in 'detail'." + ], + "conclusion": "" +} diff --git a/packages/mongospecs-web/src/mongospecs_web/api.py b/packages/mongospecs-web/src/mongospecs_web/api.py index d75e905..69aece3 100644 --- a/packages/mongospecs-web/src/mongospecs_web/api.py +++ b/packages/mongospecs-web/src/mongospecs_web/api.py @@ -103,7 +103,7 @@ async def read_body(self, headers: t.Mapping[str, str], chunks: t.AsyncIterable[ def allowed_methods(self, path: str) -> list[str]: """Methods the routes matching ``path`` accept (empty when no route matches).""" - methods = {route.method for route in self._routes() if route.pattern.match(path)} + methods = {route.method for route in self._candidates(path) if route.pattern.match(path)} if "GET" in methods: methods.add("HEAD") return sorted(methods | {"OPTIONS"}) if methods else [] @@ -111,27 +111,63 @@ def allowed_methods(self, path: str) -> list[str]: def match(self, method: str, path: str) -> tuple[Route, dict[str, str]]: """Find the route for a request, or raise 404 / 405.""" allowed: list[str] = [] - for route in self._routes(): + method = method.upper() + for route in self._candidates(path): found = route.pattern.match(path) if found is None: continue - if route.method == method.upper(): + if route.method == method: return route, found.groupdict() allowed.append(route.method) if allowed: raise MethodNotAllowed(headers={"allow": ", ".join(sorted(set(allowed)))}) raise NotFound("No such route.") + def _candidates(self, path: str) -> t.Sequence[Route]: + """The routes of the resource ``path`` points into: a dict lookup, then a few patterns. + + Matching used to try every route of the API in turn, which grows with the API. + """ + index: dict[str, tuple[Resource[t.Any], int, list[Route]]] | None = self.__dict__.get("_route_index") + if index is None: + index = self._build_index() + prefix = self.prefix + if prefix and not path.startswith(prefix + "/"): + return () + rest = path[len(prefix) + 1 :] + # resource names may contain slashes, so try the longest known name first + segments = rest.rstrip("/").split("/") + for depth in range(min(len(segments), self.__dict__["_route_depth"]), 0, -1): + entry = index.get("/".join(segments[:depth])) + if entry is None: + continue + resource, count, routes = entry + if len(resource.operations) != count: # an operation was added after the index was built + self.__dict__.pop("_route_index", None) + return self._candidates(path) + return routes + return () + + def _build_index(self) -> dict[str, tuple[Resource[t.Any], int, list[Route]]]: + index: dict[str, tuple[Resource[t.Any], int, list[Route]]] = {} + for resource in self.resources: + index[resource.name.strip("/")] = (resource, len(resource.operations), []) + for route in self._routes(): + index[route.resource.name.strip("/")][2].append(route) # keeps the literal-first order + self.__dict__["_route_index"] = index + self.__dict__["_route_depth"] = max((name.count("/") + 1 for name in index), default=1) + return index + def _routes(self) -> list[Route]: cached: list[Route] | None = self.__dict__.get("_route_cache") - if cached is None or self.__dict__.get("_route_count") != sum( - len(r.operations) for r in self.resources - ): + count = sum(len(r.operations) for r in self.resources) + if cached is None or self.__dict__.get("_route_count") != count: cached = self.routes # Literal segments win over parameters: /{id}/publish before /{id}, /search before /{id}. cached.sort(key=lambda r: (-r.path.count("/"), r.path.count("{"))) self.__dict__["_route_cache"] = cached - self.__dict__["_route_count"] = sum(len(r.operations) for r in self.resources) + self.__dict__["_route_count"] = count + self.__dict__.pop("_route_index", None) return cached async def run(self, route: Route, request: Request) -> Response: diff --git a/packages/mongospecs-web/src/mongospecs_web/backend.py b/packages/mongospecs-web/src/mongospecs_web/backend.py index ce85e5e..2b2e4de 100644 --- a/packages/mongospecs-web/src/mongospecs_web/backend.py +++ b/packages/mongospecs-web/src/mongospecs_web/backend.py @@ -107,21 +107,35 @@ def _by_id(self, id: t.Any, scope: t.Any) -> t.Any: return None return query.filter(scope) if scope is not None else query + supports_raw = True + """``get`` and ``list`` accept ``fields=``: return stored documents (dicts) holding only + those fields instead of models. A Resource asks for this when it will only serialise the + result and its view can do that without a model.""" + async def get( - self, id: t.Any, *, scope: t.Any, include: t.Sequence[str] = (), read_only: bool = False + self, + id: t.Any, + *, + scope: t.Any, + include: t.Sequence[str] = (), + read_only: bool = False, + fields: t.Sequence[str] | None = None, ) -> T | None: query = self._by_id(id, scope) if query is None: return None query = self._bounded(query) - if read_only: + if fields is not None: + query = query.raw().select(*fields) # only what will be shown leaves the database + elif read_only: query = query.lean() # nothing will be saved: skip change tracking for path in include: query = query.include(path) return t.cast("T | None", await query.first()) - async def list(self, query: ListQuery, *, scope: t.Any) -> Page[T]: - q = self._bounded(self.manager.find(*(f.to_expr() for f in query.filters))).lean() # read-only + async def list(self, query: ListQuery, *, scope: t.Any, fields: t.Sequence[str] | None = None) -> Page[T]: + q = self._bounded(self.manager.find(*(f.to_expr() for f in query.filters))) + q = q.raw().select(*fields) if fields is not None else q.lean() # read-only either way if scope is not None: q = q.filter(scope) if query.sort: @@ -152,15 +166,24 @@ async def patch(self, id: t.Any, data: dict[str, t.Any], *, scope: t.Any, if_rev return t.cast(T, await self.manager.update(target, if_rev=if_rev, **data)) async def delete(self, id: t.Any, *, scope: t.Any, if_rev: int | None = None) -> None: + manager = self.manager + condition = manager.compile(scope) if scope is not None else None + try: + key = manager.meta.cast("id", id) + except QueryError: + raise DocumentNotFound(f"{self.model.__name__} {id!r} does not exist") from None + # One conditional write when nothing needs the document itself (no delete hooks, no + # cascades): the scope and the revision are part of the write's own filter. + if await manager.delete_by_id(key, condition=condition, if_rev=if_rev): + return query = self._by_id(id, scope) doc = await query.first() if query is not None else None if doc is None: raise DocumentNotFound(f"{self.model.__name__} {id!r} does not exist") - # The document is loaded so that hooks, soft delete and cascades run. The scope is - # then also part of the delete's own filter, so it holds at the moment of the write. - condition = self.manager.compile(scope) if scope is not None else None + # The document is loaded so that hooks and cascades run. The scope is then also + # part of the delete's own filter, so it holds at the moment of the write. try: - await self.manager.delete_document(doc, condition=condition, if_rev=if_rev) + await manager.delete_document(doc, condition=condition, if_rev=if_rev) except StaleDocumentError: if if_rev is not None: raise diff --git a/packages/mongospecs-web/src/mongospecs_web/fastread.py b/packages/mongospecs-web/src/mongospecs_web/fastread.py new file mode 100644 index 0000000..81e6adc --- /dev/null +++ b/packages/mongospecs-web/src/mongospecs_web/fastread.py @@ -0,0 +1,284 @@ +"""Serialise the Read view straight from a stored document, without building a model. + +The ordinary read path decodes BSON into a model, dumps the whole model to JSON +types and then picks the readable fields. For most models that work can be +skipped: the readable fields are simple values whose JSON form is known from the +declared type. :func:`compile_plan` builds such a plan when, and only when, the +result is certain to equal what the model path produces: + +* every readable field has a type this module knows how to convert, +* the model has nothing that could change values on the way in or out + (validators, converters, serialisers, post-init hooks, ``upcast``, computed + fields, a discriminator), +* at run time each stored value has exactly the declared type. Anything else + (an integer stored as a double, a missing required field) raises + :class:`Slow`, and that one document goes through the model path instead. + +Private fields are never read: not at the top level, where they are left out of +the projection, and not inside embedded models. +""" + +from __future__ import annotations + +import datetime as dt +import enum +import typing as t + +from bson import ObjectId + +from mongospecs import Document, Ref, meta_of +from mongospecs.codec import _UNION_ORIGINS, Codec, _codec_or_none, to_bson +from mongospecs.markers import Deferred, Marker +from mongospecs.refs import _OIDAnnotation + +__all__ = ["Plan", "Slow", "compile_plan"] + +Conv = t.Callable[[t.Any], t.Any] + + +class Slow(Exception): + """This document needs the model path.""" + + +_SLOW = Slow() +_MISSING: t.Any = object() +_HARMLESS_METADATA = ("msgspec", "annotated_types", "pydantic.fields", "pydantic.types") +_VALUE_CHANGING_CONFIG = ("str_strip_whitespace", "str_to_lower", "str_to_upper", "coerce_numbers_to_str") + + +def _exact(kind: type) -> Conv: + def conv(value: t.Any) -> t.Any: + if type(value) is not kind: + raise _SLOW + return value + + return conv + + +def _object_id(value: t.Any) -> t.Any: + if type(value) is not ObjectId: + raise _SLOW + return str(value) + + +def _anything(value: t.Any) -> t.Any: + """An untyped value: only plain JSON types are certain to come out unchanged.""" + kind = type(value) + if kind is str or kind is int or kind is float or kind is bool or value is None: + return value + if kind is list: + return [_anything(v) for v in value] + if kind is dict: + for key in value: + if type(key) is not str: + raise _SLOW + return {k: _anything(v) for k, v in value.items()} + raise _SLOW + + +def _plain_class(cls: type, codec: Codec) -> bool: + """True when the class has nothing that could change a value between storage and output.""" + for hook in ("__post_init__", "__attrs_post_init__", "model_post_init"): + fn = getattr(cls, hook, None) + module = getattr(fn, "__module__", "") or "" + if fn is not None and not module.startswith(("mongospecs.", "pydantic.")): + return False + if codec.name == "pydantic": + decorators = getattr(cls, "__pydantic_decorators__", None) + for group in ( + "validators", + "field_validators", + "root_validators", + "model_validators", + "field_serializers", + "model_serializers", + "computed_fields", + ): + if getattr(decorators, group, None): + return False + config = getattr(cls, "model_config", {}) + if any(config.get(key) for key in _VALUE_CHANGING_CONFIG): + return False + elif codec.name == "attrs": + import attrs + + if any(a.converter is not None for a in attrs.fields(cls)): + return False + return True + + +class _Builder: + def __init__(self, codec_name: str) -> None: + self.models: dict[type, Conv | None] = {} + if codec_name == "pydantic": + from pydantic_core import to_jsonable_python + + self.temporal: Conv = to_jsonable_python + else: + import msgspec + + self.temporal = msgspec.to_builtins + + def conv(self, tp: t.Any) -> Conv | None: + """A converter for values of type ``tp``, or ``None`` when the type is not supported.""" + while t.get_origin(tp) is t.Annotated: + tp, *metadata = t.get_args(tp) + for item in metadata: + if item is _OIDAnnotation or isinstance(item, Marker): + continue + if isinstance(item, type) and issubclass(item, Marker): + continue + if not type(item).__module__.startswith(_HARMLESS_METADATA): + return None # a validator or serialiser we cannot reason about + origin = t.get_origin(tp) + if origin in _UNION_ORIGINS: + members = [a for a in t.get_args(tp) if a is not type(None)] + if len(members) != 1 or len(members) == len(t.get_args(tp)): + return None + inner = self.conv(members[0]) + if inner is None: + return None + return lambda value: None if value is None else inner(value) + if tp is t.Any: + return _anything + if tp is Ref or origin is Ref: + return _object_id + if origin is list: + (arg,) = t.get_args(tp) or (t.Any,) + item = self.conv(arg) + if item is None: + return None + + def conv_list(value: t.Any) -> t.Any: + if type(value) is not list: + raise _SLOW + return [item(v) for v in value] + + return conv_list + if origin is dict: + key, arg = t.get_args(tp) or (str, t.Any) + each = self.conv(arg) + if key is not str or each is None: + return None + + def conv_dict(value: t.Any) -> t.Any: + if type(value) is not dict: + raise _SLOW + return {k: each(v) for k, v in value.items()} + + return conv_dict + if origin is not None or not isinstance(tp, type): + return None + if tp in (str, int, float, bool): + return _exact(tp) + if tp is ObjectId: + return _object_id + if tp is dt.datetime: + temporal = self.temporal + + def conv_datetime(value: t.Any) -> t.Any: + if type(value) is not dt.datetime: + raise _SLOW + return temporal(value) + + return conv_datetime + if issubclass(tp, enum.Enum): + values = {member.value for member in tp} + if not all(type(v) in (str, int) for v in values): + return None + + def conv_enum(value: t.Any) -> t.Any: + if type(value) not in (str, int) or value not in values: + raise _SLOW + return value + + return conv_enum + if issubclass(tp, Document): + return None + codec = _codec_or_none(tp) + if codec is None: + return None + if tp not in self.models: + self.models[tp] = None # a model that contains itself is not supported + self.models[tp] = self.embedded(tp, codec) + return self.models[tp] + + def fields(self, cls: type, codec: Codec, *, top: bool) -> list[tuple[str, str, Conv, t.Any]] | None: + """``(output key, stored key, converter, default)`` per readable field, or ``None``.""" + if not _plain_class(cls, codec): + return None + natives = {name: native for name, native, _, _ in codec.raw_fields(cls)} + defaults = codec.defaults(cls) + out: list[tuple[str, str, Conv, t.Any]] = [] + for name, info in codec.fields(cls).items(): + if info.transient: + continue + if not top and info.stored != natives[name]: + return None # an alias inside an embedded model: leave it to the model path + if info.policy.private or (top and info.has(Deferred)): + if name not in defaults: + return None # the model path needs the field, and we will not fetch it + continue + conv = self.conv(info.type) + if conv is None: + return None + default = _MISSING + if name in defaults: + default = defaults[name] + if default is not None: + try: + conv(to_bson(default)) + except Slow: + return None + out.append((name if top else natives[name], info.stored, conv, default)) + return out + + def embedded(self, cls: type, codec: Codec) -> Conv | None: + fields = self.fields(cls, codec, top=False) + if fields is None: + return None + + def conv_model(value: t.Any) -> t.Any: + if type(value) is not dict: + raise _SLOW + return _apply(fields, value) + + return conv_model + + +def _apply(fields: list[tuple[str, str, Conv, t.Any]], doc: dict[str, t.Any]) -> dict[str, t.Any]: + out: dict[str, t.Any] = {} + get = doc.get + for key, stored, conv, default in fields: + value = get(stored, _MISSING) + if value is _MISSING: + if default is _MISSING: + raise _SLOW # a required field is missing: let the model report it + out[key] = None if default is None else conv(to_bson(default)) + else: + out[key] = conv(value) + return out + + +class Plan: + """How to produce the Read view of one model from a stored document.""" + + __slots__ = ("_fields", "select") + + def __init__(self, fields: list[tuple[str, str, Conv, t.Any]]) -> None: + self._fields = fields + self.select: tuple[str, ...] = tuple(key for key, _, _, _ in fields) + """Attribute names to project: the readable fields and nothing else.""" + + def serialize(self, doc: dict[str, t.Any]) -> dict[str, t.Any]: + """The Read view as JSON types. Raises :class:`Slow` when this document needs the model path.""" + return _apply(self._fields, doc) + + +def compile_plan(model: type, computed: t.Sequence[str]) -> Plan | None: + """A :class:`Plan` for ``model``, or ``None`` when only the model path is certain to be right.""" + meta = meta_of(model) + if computed or meta.disc_key or meta.config.upcast is not None: + return None + fields = _Builder(meta.codec.name).fields(model, meta.codec, top=True) + return Plan(fields) if fields is not None else None diff --git a/packages/mongospecs-web/src/mongospecs_web/policy.py b/packages/mongospecs-web/src/mongospecs_web/policy.py index 7b2ae8e..1d1da57 100644 --- a/packages/mongospecs-web/src/mongospecs_web/policy.py +++ b/packages/mongospecs-web/src/mongospecs_web/policy.py @@ -274,11 +274,22 @@ def may(flag: bool | None, default: bool) -> bool: ) revision = next((f for f in fields.values() if f.has(Revision)), None) self.revision: str | None = revision.name if revision else None + self._revision_stored: str | None = revision.stored if revision else None + from mongospecs_web.fastread import compile_plan + + self.fast = compile_plan(model, self.computed) + """Plan for serialising straight from a stored document, or ``None`` when this model + needs the model path (see :mod:`mongospecs_web.fastread`).""" # -- output ---------------------------------------------------------------- def serialize(self, doc: t.Any, include: t.Collection[str] = ()) -> dict[str, t.Any]: - """The Read view of a document as JSON types. Private fields never appear.""" + """The Read view of a document as JSON types. Private fields never appear. + + ``doc`` is a model instance, or a stored document (a ``dict``) read for this purpose. + """ + if type(doc) is dict: + return self.serialize_raw(doc, include) if type(doc) is not self.model: return view_of(type(doc)).serialize(doc, include) data = self.meta.codec.to_json(doc) @@ -303,9 +314,25 @@ def serialize(self, doc: t.Any, include: t.Collection[str] = ()) -> dict[str, t. out[name] = jsonable(getattr(doc, name)) return out + def serialize_raw(self, raw: dict[str, t.Any], include: t.Collection[str] = ()) -> dict[str, t.Any]: + """The Read view of a stored document; identical to decoding it and calling ``serialize``.""" + from mongospecs_web.fastread import Slow + + if self.fast is not None and not include: + try: + return self.fast.serialize(raw) + except Slow: + pass # a value that does not have its declared type: take the careful path + return self.serialize(self.meta.decode(raw), include) + def etag(self, doc: t.Any) -> str | None: if self.revision is None: return None + if type(doc) is dict: + revision = doc.get(self._revision_stored, 0) + if type(revision) is not int: # stored as a double by another writer: let the model read it + revision = getattr(self.meta.decode(doc), self.revision) + return f'"{revision}"' return f'"{getattr(doc, self.revision)}"' # -- input --------------------------------------------------------------- diff --git a/packages/mongospecs-web/src/mongospecs_web/resource.py b/packages/mongospecs-web/src/mongospecs_web/resource.py index e870a71..92ff515 100644 --- a/packages/mongospecs-web/src/mongospecs_web/resource.py +++ b/packages/mongospecs-web/src/mongospecs_web/resource.py @@ -232,6 +232,9 @@ def __init__( raise ValueError(f"{model.__name__}.{head} is not a readable relation and cannot be included") self.page: Pagination = page or Cursor() self.tags: tuple[str, ...] = tuple(tags) if tags is not None else (self.name,) + self.raw_reads = True + """Serve ``GET`` straight from stored documents where that is certain to give the same + response as going through the model. Set to ``False`` to always build models.""" self._all_guards: tuple[Guard, ...] = () self._guards: dict[str, tuple[Guard, ...]] = {} if isinstance(guards, t.Mapping): @@ -379,10 +382,28 @@ def _ref(self, kind: str) -> dict[str, t.Any]: def schema_name(self, kind: str) -> str: return f"{self.model.__name__}{kind.capitalize()}" + def _raw_fields(self, kind: str, include: t.Collection[str]) -> tuple[str, ...] | None: + """The fields to read as a plain stored document, or ``None`` to read a model. + + Reading without a model is possible when the response is all the document is used + for: the view can serialise it directly, no relation is embedded, and no ``after`` + hook is waiting for a model. + """ + plan = self.view.fast + if plan is None or include or self._after.get(kind) or not self.raw_reads: + return None + if not getattr(self.backend, "supports_raw", False): + return None + return plan.select + def _list_op(self) -> Operation: async def handler(ctx: Ctx[T]) -> Response: query = parse_list_query(ctx.request.query, self.view, self.page, self.includes) - page = await self.backend.list(query, scope=await ctx.scope()) + fields = self._raw_fields(List, query.include) + if fields is not None: + page = await t.cast(t.Any, self.backend).list(query, scope=await ctx.scope(), fields=fields) + else: + page = await self.backend.list(query, scope=await ctx.scope()) ctx.state["result"] = page meta: dict[str, t.Any] = {"size": page.size, "has_next": page.has_next} if page.number is not None: @@ -485,7 +506,11 @@ def _one(self) -> dict[str, t.Any]: def _get_op(self) -> Operation: async def handler(ctx: Ctx[T]) -> Response: include = parse_includes(ctx.request.query, self.includes) - doc = await self.backend.get(ctx.id, scope=await ctx.scope(), include=include, read_only=True) + fields = self._raw_fields(Get, include) + if fields is not None: + doc = await t.cast(t.Any, self.backend).get(ctx.id, scope=await ctx.scope(), fields=fields) + else: + doc = await self.backend.get(ctx.id, scope=await ctx.scope(), include=include, read_only=True) if doc is None: raise NotFound("The resource does not exist.") ctx.state["result"] = doc diff --git a/packages/mongospecs-web/tests/test_fast_reads.py b/packages/mongospecs-web/tests/test_fast_reads.py new file mode 100644 index 0000000..91739ac --- /dev/null +++ b/packages/mongospecs-web/tests/test_fast_reads.py @@ -0,0 +1,382 @@ +"""Reads served straight from stored documents must equal reads served through the model.""" + +import datetime as dt +import enum +import json +import typing as t + +import pytest + +from mongospecs import ( + Alias, + CreatedAt, + Deferred, + DeletedAt, + Policy, + Q, + Ref, + Revision, + SpecConfig, + computed, +) +from mongospecs_web import Api, Delete, Get, List, Resource +from mongospecs_web.policy import view_of + +BACKENDS = ("msgspec", "pydantic", "attrs") + + +class Colour(enum.Enum): + RED = "red" + BLUE = "blue" + + +def models(backend): + if backend == "msgspec": + import msgspec + + from mongospecs.msgspec import Spec + + class Base(msgspec.Struct, kw_only=True): + pass + + define = lambda c: c # noqa: E731 + elif backend == "pydantic": + import pydantic + + from mongospecs.pydantic import Spec + + Base = pydantic.BaseModel + define = lambda c: c # noqa: E731 + else: + import attrs + + from mongospecs.attrs import Spec + + Base = object + define = attrs.define(kw_only=True) + + @define + class Geo(Base): + lat: float = 0.0 + surveyor: t.Annotated[str | None, Policy(private=True)] = None + + @define + class Addr(Base): + city: str = "" + vault_code: t.Annotated[str, Policy(private=True)] = "" + geo: Geo | None = None + + @define + class Maker(Spec): + __spec__ = SpecConfig(collection="makers") + name: str = "" + + @define + class Item(Spec): + __spec__ = SpecConfig(collection="items") + name: t.Annotated[str, Policy(filter=("eq", "prefix"), sort=True)] = "" + label: t.Annotated[str, Alias("l")] = "" + price: t.Annotated[float, Policy(sort=True)] = 0.0 + stock: int = 0 + active: bool = True + colour: Colour | None = None + tags: list[str] = [] # noqa: RUF012 + scores: dict[str, int] = {} # noqa: RUF012 + extra: dict[str, t.Any] = {} # noqa: RUF012 + address: Addr | None = None + history: list[Addr] = [] # noqa: RUF012 + maker: Ref[Maker] | None = None + makers: list[Ref[Maker]] = [] # noqa: RUF012 + secret: t.Annotated[str | None, Policy(private=True)] = None + blob: t.Annotated[str | None, Deferred] = None + created_at: t.Annotated[dt.datetime | None, CreatedAt] = None + seen_at: dt.datetime | None = None + deleted_at: t.Annotated[dt.datetime | None, DeletedAt, Policy(private=True)] = None + rev: t.Annotated[int, Revision] = 0 + + @define + class Shape(Spec): + __spec__ = SpecConfig(collection="shapes", discriminator="kind") + name: str = "" + + @define + class Circle(Shape): + radius: float = 1.0 + + @define + class Scored(Spec): + __spec__ = SpecConfig(collection="scored") + points: int = 0 + + @computed + def double(self) -> int: + return self.points * 2 + + return Geo, Addr, Maker, Item, Shape, Circle, Scored + + +@pytest.fixture(params=BACKENDS) +async def world(request, make_registry): + classes = models(request.param) + Geo, Addr, Maker, Item, Shape, Circle, Scored = classes + events = [] + await make_registry(Maker, Item, Shape, Circle, Scored, on_command=events.append) + + def api(raw): + resources = [ + Resource(Item, name="items", ops=[List, Get, Delete], include=["maker"]), + Resource(Shape, name="shapes", ops=[List, Get]), + Resource(Scored, name="scored", ops=[List, Get]), + ] + for resource in resources: + resource.raw_reads = raw + return Api(*resources, prefix="/v1") + + return classes, api(True), api(False), events + + +async def seed(Geo, Addr, Maker, Item, *_): + acme = await Maker(name="acme").insert() + other = await Maker(name="other").insert() + await Item( + name="full", + label="L", + price=9.5, + stock=3, + active=False, + colour=Colour.BLUE, + tags=["a", "b"], + scores={"x": 1}, + extra={"k": [1, "two", {"three": 3.5, "none": None}], "flag": True}, + address=Addr(city="here", vault_code="v", geo=Geo(lat=1.25, surveyor="s")), + history=[Addr(city="old"), Addr(city="older", geo=Geo(lat=2.0))], + maker=acme, + makers=[acme, other], + secret="hidden", + blob="big", + seen_at=dt.datetime(2024, 5, 6, 7, 8, 9, 123000, tzinfo=dt.UTC), + ).insert() + await Item(name="defaults").insert() + coll = Item.collection() + # shapes other writers produce: missing keys, an int stored as a double, a double as an int, + # extra keys, a non-JSON value inside an untyped field, a soft-deleted document + await coll.insert_one({"name": "sparse"}) + await coll.insert_one({"name": "odd numbers", "stock": 4.0, "price": 7, "rev": 2.0}) + await coll.insert_one( + { + "name": "extras", + "unknown": 1, + "address": {"city": "c", "zz": 1}, + "extra": {"when": dt.datetime(2021, 1, 1)}, + } + ) + await coll.insert_one({"name": "gone", "deleted_at": dt.datetime(2020, 1, 1)}) + return acme + + +async def call(api, method, url, headers=None): + from urllib.parse import parse_qsl, urlsplit + + parts = urlsplit(url) + return await api.handle(method, parts.path, query=parse_qsl(parts.query), headers=headers or {}) + + +async def both(fast, slow, method, path): + a, b = await call(fast, method, path), await call(slow, method, path) + assert a.status == b.status, (path, a.body, b.body) + if a.body != b.body: + # The only permitted difference is the opaque cursor: it is built from the last sort key as + # stored (an int 7) on one path and as the model holds it (a float 7.0) on the other. + # Both resume at the same place, which the pagination walk below checks. + one, two = json.loads(a.body), json.loads(b.body) + assert one["page"].pop("next_cursor") and two["page"].pop("next_cursor") + assert one == two, f"{path}\nraw: {a.body!r}\nmodel: {b.body!r}" + assert a.headers == b.headers + return a + + +async def test_raw_reads_equal_model_reads(world): + classes, fast, slow, events = world + Item = classes[3] + assert view_of(Item).fast is not None # this model is served without building models + await seed(*classes) + listing = await both(fast, slow, "GET", "/v1/items?sort=name&limit=50") + rows = json.loads(listing.body)["data"] + assert [r["name"] for r in rows] == ["defaults", "extras", "full", "odd numbers", "sparse"] # not "gone" + for row in rows: + await both(fast, slow, "GET", f"/v1/items/{row['id']}") + assert "secret" not in row and "blob" not in row and "deleted_at" not in row + assert "vault_code" not in json.dumps(row) and "surveyor" not in json.dumps(row) + full = next(r for r in rows if r["name"] == "full") + assert ( + full["label"] == "L" + and full["colour"] == "blue" + and full["address"] == {"city": "here", "geo": {"lat": 1.25}} + ) + for path in ( + "/v1/items?sort=-price&limit=2", + "/v1/items?name[prefix]=o", + "/v1/items?include=maker&sort=name", + f"/v1/items/{full['id']}?include=maker", + "/v1/items/000000000000000000000000", + "/v1/items/not-an-id", + ): + await both(fast, slow, "GET", path) + # cursor pagination walks the same pages on both paths, also across a sort key stored as another number type + for sort in ("name", "-price", "price"): + walks = [] + for api in (fast, slow): + cursor, seen = None, [] + while True: + url = f"/v1/items?sort={sort}&limit=2" + (f"&after={cursor}" if cursor else "") + page = json.loads((await call(api, "GET", url)).body) + seen += [r["name"] for r in page["data"]] + cursor = page["page"]["next_cursor"] + if not page["page"]["has_next"]: + break + walks.append(seen) + # two documents have no stored price: they are paged as well, first ascending, last descending + assert walks[0] == walks[1] and len(walks[0]) == len(set(walks[0])) == 5, (sort, walks) + + +async def test_raw_reads_fetch_only_what_is_shown(world): + classes, fast, slow, events = world + Item = classes[3] + await seed(*classes) + events.clear() + assert (await call(fast, "GET", "/v1/items?sort=name")).status == 200 + (find,) = [e.command for e in events if e.phase == "started" and e.command_name == "find"] + projection = find["projection"] + assert set(projection.values()) == {1} + for hidden in ("secret", "blob", "deleted_at"): + assert hidden not in projection # private and deferred fields never leave the database + assert {"name", "l", "price", "address", "rev"} <= set(projection) + assert find["filter"].get("deleted_at", "absent") is None # the soft-delete scope still applies + # an after-hook wants a model, so the resource reads models again + items = fast.resources[0] + seen = [] + items.after("list")(lambda ctx, result: seen.append(type(result.items[0]))) + events.clear() + assert (await call(fast, "GET", "/v1/items")).status == 200 + assert seen == [Item] + (find,) = [e.command for e in events if e.phase == "started" and e.command_name == "find"] + assert "name" not in (find.get("projection") or {}) + + +async def test_models_that_need_the_model_path_take_it(world): + classes, fast, slow, events = world + Geo, Addr, Maker, Item, Shape, Circle, Scored = classes + assert view_of(Shape).fast is None and view_of(Scored).fast is None # discriminator, computed field + await Circle(name="c", radius=2.0).insert() + await Shape(name="s").insert() + await Scored(points=4).insert() + shapes = json.loads((await both(fast, slow, "GET", "/v1/shapes")).body)["data"] + assert {s["kind"] for s in shapes} == {"Circle", "Shape"} and any("radius" in s for s in shapes) + scored = json.loads((await both(fast, slow, "GET", "/v1/scored")).body)["data"] + assert scored[0]["double"] == 8 + + +def test_value_changing_models_are_not_served_raw(): + import attrs + import msgspec + import pydantic + + from mongospecs.attrs import Spec as AttrsSpec + from mongospecs.msgspec import Spec as MsgspecSpec + from mongospecs.pydantic import Spec as PydanticSpec + + class Plain(PydanticSpec): + name: str = "" + + class Validated(PydanticSpec): + name: str = "" + + @pydantic.field_validator("name") + @classmethod + def tidy(cls, value): + return value.strip() + + class Stripped(PydanticSpec): + model_config = pydantic.ConfigDict(str_strip_whitespace=True) + name: str = "" + + class Serialised(PydanticSpec): + name: str = "" + + @pydantic.field_serializer("name") + def shout(self, value): + return value.upper() + + class Annotated(PydanticSpec): + name: t.Annotated[str, pydantic.BeforeValidator(str.strip)] = "" + + @attrs.define(kw_only=True) + class Converted(AttrsSpec): + name: str = attrs.field(default="", converter=str.strip) + + class Hooked(MsgspecSpec): + name: str = "" + + def __post_init__(self): + super().__post_init__() + self.name = self.name.strip() + + class Constrained(MsgspecSpec): + name: t.Annotated[str, msgspec.Meta(max_length=50)] = "" + when: dt.date | None = None # a type the plan does not know + + class Known(MsgspecSpec): + name: t.Annotated[str, msgspec.Meta(max_length=50)] = "" + + assert view_of(Plain).fast is not None and view_of(Known).fast is not None + for model in (Validated, Stripped, Serialised, Annotated, Converted, Hooked, Constrained): + assert view_of(model).fast is None, model.__name__ + + +async def test_etag_and_conditional_get_on_raw_reads(world): + classes, fast, slow, events = world + Item = classes[3] + item = await Item(name="e").insert() + await Item.update(item.id, stock=1) + got = await both(fast, slow, "GET", f"/v1/items/{item.id}") + assert got.headers["etag"] == '"1"' + for api in (fast, slow): + cached = await call(api, "GET", f"/v1/items/{item.id}", headers={"if-none-match": '"1"'}) + assert cached.status == 304 + assert await Item.count(Q.name == "e") == 1 + + +async def test_delete_is_one_write_unless_the_document_is_needed(world): + classes, fast, slow, events = world + Geo, Addr, Maker, Item, Shape, Circle, Scored = classes + item = await Item(name="d").insert() + events.clear() + assert (await call(fast, "DELETE", f"/v1/items/{item.id}")).status == 204 + assert [e.command_name for e in events if e.phase == "started"] == [ + "update" + ] # soft delete, no read first + assert await Item.count() == 0 and await Item.collection().count_documents({}) == 1 + assert (await call(fast, "DELETE", f"/v1/items/{item.id}")).status == 404 # already deleted + assert (await call(fast, "DELETE", "/v1/items/nope")).status == 404 + # with If-Match: a stale revision is 412 and the current one deletes + other = await Item(name="d2").insert() + await Item.update(other.id, stock=2) + assert (await call(fast, "DELETE", f"/v1/items/{other.id}", headers={"if-match": '"0"'})).status == 412 + assert (await call(fast, "DELETE", f"/v1/items/{other.id}", headers={"if-match": '"1"'})).status == 204 + # a model with a delete hook still gets its document + third = await Item(name="d3").insert() + seen = [] + + @Item.before("delete") + async def note(ctx, doc): + seen.append(doc.name) + + try: + events.clear() + assert (await call(fast, "DELETE", f"/v1/items/{third.id}")).status == 204 + assert seen == ["d3"] + assert [e.command_name for e in events if e.phase == "started"] == ["find", "update"] + finally: + from mongospecs.meta import _HOOKS + + _HOOKS.pop(Item, None) diff --git a/packages/mongospecs-web/tests/test_mongo_resource.py b/packages/mongospecs-web/tests/test_mongo_resource.py index 0c797bf..c61bb92 100644 --- a/packages/mongospecs-web/tests/test_mongo_resource.py +++ b/packages/mongospecs-web/tests/test_mongo_resource.py @@ -341,7 +341,10 @@ async def test_delete_is_conditional_and_scoped_at_the_write(app): finally: await observer.close() (write,) = [e.command for e in events if e.phase == "started" and e.command_name == "update"] - assert write["updates"][0]["q"] == {"_id": story.id, "rev": 1, "author": ann.id} + # the id, the revision, the row scope and the soft-delete scope, all in the one write + assert write["updates"][0]["q"] == {"_id": story.id, "rev": 1, "author": ann.id, "deleted_at": None} + sent = [e.command_name for e in events if e.phase == "started" and e.command_name != "endSessions"] + assert sent == ["update"] # one conditional write, no read before it assert (await client.get(f"/v1/my-stories/{story.id}", headers=as_ann)).status_code == 404 # soft-deleted diff --git a/packages/mongospecs/src/mongospecs/manager.py b/packages/mongospecs/src/mongospecs/manager.py index 5ee8b5b..7ddd366 100644 --- a/packages/mongospecs/src/mongospecs/manager.py +++ b/packages/mongospecs/src/mongospecs/manager.py @@ -426,6 +426,59 @@ async def run(s: t.Any) -> None: tracking.forget(doc) await self._hooks("after", ("delete",), ctx, doc) + def _needs_document_to_delete(self, hard: bool) -> bool: + if self.meta.hooks("before", "delete") or self.meta.hooks("after", "delete"): + return True + for plugin in self.plugins: + if type(plugin).before is not Plugin.before or type(plugin).after is not Plugin.after: + return True + return False + + async def delete_by_id( + self, + id: t.Any, + *, + condition: t.Mapping[str, t.Any] | None = None, + if_rev: int | None = None, + hard: bool = False, + session: t.Any = None, + ) -> bool: + """Delete one document by id in a single conditional write, without loading it. + + Returns ``False`` without doing anything when the delete needs the document: the + model has delete hooks, or it is a hard delete with cascades to run. Otherwise + deletes (softly when the model has a ``DeletedAt`` field, unless ``hard``) and + returns ``True``. Raises :class:`DocumentNotFound` when no document with this id + satisfies ``condition``, and :class:`StaleDocumentError` when ``if_rev`` is stale. + """ + if self._needs_document_to_delete(hard): + return False + ctx = self.ctx("delete", session) + base = and_filters(self.scoped(ctx, {"_id": id}, False), condition) + flt = dict(base) + if if_rev is not None: + ctx.extra["if_rev"] = if_rev + soft = None if hard else self._soft_delete(ctx, flt) + if soft is None and self._cascade_rules(): + return False + revision = next((f for f in self.meta.fields.values() if f.has(Revision)), None) + if revision is not None and if_rev is not None and revision.stored not in flt: + # a soft delete is an update, and the revision plugin has already added the condition + flt[revision.stored] = {"$in": [0, None]} if not if_rev else if_rev + with self._op("delete"): + if soft is not None: + matched = (await self.collection.update_one(flt, soft, **self._kw(session))).matched_count + else: + matched = (await self.collection.delete_one(flt, **self._kw(session))).deleted_count + if matched: + return True + if if_rev is not None: + with self._op("delete"): + exists = await self.collection.count_documents(base, limit=1, **self._kw(session)) + if exists: + raise StaleDocumentError(f"{self.model.__name__}: revision {if_rev} is no longer current") + raise DocumentNotFound(f"{self.model.__name__} {id!r} does not exist") + def _soft_delete(self, ctx: OpCtx, flt: dict[str, t.Any]) -> dict[str, t.Any] | None: for plugin in self.plugins: update = plugin.on_delete(ctx, flt) diff --git a/packages/mongospecs/src/mongospecs/query.py b/packages/mongospecs/src/mongospecs/query.py index 8c99fa2..90abc37 100644 --- a/packages/mongospecs/src/mongospecs/query.py +++ b/packages/mongospecs/src/mongospecs/query.py @@ -66,6 +66,7 @@ class Query(t.Generic[T]): "_fail", "_includes", "_invalid", + "_keys", "_limit", "_manager", "_mode", @@ -107,11 +108,29 @@ def __init__( self._options: dict[str, t.Any] = dict(options or {}) self._fail = fail self._invalid = invalid + self._keys: dict[int, t.Any] | None = None def _clone(self, **changes: t.Any) -> Query[T]: - state = {name.lstrip("_"): getattr(self, name) for name in self.__slots__ if name != "_manager"} - state.update(changes) - return type(self)(self._manager, **state) + # A query is cloned several times on every call path; copying the slots is much + # cheaper than going through __init__ with a dozen keyword arguments. + new: Query[T] = object.__new__(type(self)) + new._manager = self._manager + new._conditions = self._conditions + new._sort = self._sort + new._skip = self._skip + new._limit = self._limit + new._select = self._select + new._includes = self._includes + new._mode = self._mode + new._unscoped = self._unscoped + new._session = self._session + new._options = self._options + new._fail = self._fail + new._invalid = self._invalid + new._keys = None + for name, value in changes.items(): + setattr(new, "_" + name, dict(value) if name == "options" else value) + return new # -- builders -------------------------------------------------------------- @@ -362,6 +381,7 @@ async def _chunks(self, chunk: int | None = None) -> t.AsyncIterator[list[t.Any] options = manager.registry._codec_options invalid = self._invalid + stored_keys = self._keys async def flush(raws: list[t.Any]) -> list[t.Any]: misfits: list[tuple[int, t.Any]] = [] # (position, raw document) kept for on_invalid("raw") @@ -382,6 +402,8 @@ async def flush(raws: list[t.Any]) -> list[t.Any]: if invalid == "raw": misfits.append((position, plain)) continue + if stored_keys is not None: + stored_keys[id(doc)] = plain if data is not None: tracking.track(doc, data, manager, partial) elif mode == "tracked": @@ -561,7 +583,8 @@ async def page( Without ``number`` this is cursor (keyset) pagination: pass the previous page's ``next_cursor`` as ``after``. The sort is made total by appending ``_id``, so the index serving it should end in ``_id`` (``Index("price", "id")``); - otherwise the server sorts in memory. Sort fields should not be null. + otherwise the server sorts in memory. Documents without a value for a sort + field are paged too: they come first ascending and last descending. With ``number`` (1-based) it is offset pagination. ``total=True`` adds a count of all matches. """ @@ -587,6 +610,10 @@ async def page( query = query.filter(Trusted(_keyset_filter(sort, _decode_cursor(after, len(sort))))) + if number is None and query._mode != "raw": + # The cursor must hold the sort keys as they are stored. A model has its + # default where the stored document has no key, which sorts elsewhere. + query._keys = {} total_count: int | None if count_task is not None and self._session is None: items, total_count = await asyncio.gather(query.all(), count_task) @@ -597,7 +624,11 @@ async def page( items = items[:size] next_cursor = None if number is None and has_next and items: - next_cursor = _encode_cursor(self._sort_values(items[-1], query._sort_spec())) + last = items[-1] + stored = query._keys.get(id(last)) if query._keys is not None else None + next_cursor = _encode_cursor( + self._sort_values(last if stored is None else stored, query._sort_spec()) + ) return Page(items, size, has_next, next_cursor, number, total_count) def _sort_values(self, item: t.Any, sort: list[tuple[str, int]]) -> list[t.Any]: @@ -665,10 +696,20 @@ def _decode_cursor(token: str, expected: int) -> list[t.Any]: def _keyset_filter(sort: list[tuple[str, int]], values: list[t.Any]) -> dict[str, t.Any]: - """Documents strictly after ``values`` in ``sort`` order.""" + """Documents strictly after ``values`` in ``sort`` order. + + A missing or null sort key sorts before every value, so it comes first ascending and + last descending; ``{key: None}`` matches both the missing key and an explicit null. + """ branches: list[dict[str, t.Any]] = [] for i, (key, direction) in enumerate(sort): - branch: dict[str, t.Any] = {sort[j][0]: values[j] for j in range(i)} - branch[key] = {"$gt" if direction == 1 else "$lt": values[i]} - branches.append(branch) + prefix: dict[str, t.Any] = {sort[j][0]: values[j] for j in range(i)} + value = values[i] + if direction == 1: + # after "no value" comes every value; after a value come the larger ones + branches.append({**prefix, key: {"$ne": None} if value is None else {"$gt": value}}) + elif value is not None: + branches.append({**prefix, key: {"$lt": value}}) + branches.append({**prefix, key: None}) # documents without a value come last + # descending and already among the documents without a value: only ties remain return branches[0] if len(branches) == 1 else {"$or": branches} diff --git a/packages/mongospecs/src/mongospecs/relations.py b/packages/mongospecs/src/mongospecs/relations.py index 8ccd482..88a4ef9 100644 --- a/packages/mongospecs/src/mongospecs/relations.py +++ b/packages/mongospecs/src/mongospecs/relations.py @@ -14,6 +14,7 @@ from __future__ import annotations +import asyncio import dataclasses import typing as t from dataclasses import dataclass @@ -326,40 +327,47 @@ async def load_batched( """Resolve includes with one query per relation (per target model) for the whole batch.""" if not items: return + if session is None and len(includes) > 1: + # Independent relations: one round trip each, all in flight together. With a session + # they stay sequential, because a session cannot run two operations at once. + await asyncio.gather(*(_load_one(manager, items, include, mode, session) for include in includes)) + return for include in includes: - info = _relation(manager, include) - targets = manager.meta.ref_targets(include.name) - if info.backref is not None: - await _load_backref(manager, items, include, info, targets[0].model, mode, session) + await _load_one(manager, items, include, mode, session) + + +async def _load_one( + manager: Manager[t.Any], items: list[t.Any], include: Include, mode: str, session: t.Any +) -> None: + info = _relation(manager, include) + targets = manager.meta.ref_targets(include.name) + if info.backref is not None: + await _load_backref(manager, items, include, info, targets[0].model, mode, session) + return + assert info.ref is not None + ids: dict[t.Any, t.Any] = {} + for item in items: + value = item.get(info.stored) if isinstance(item, dict) else getattr(item, info.name) + if value is None: continue - assert info.ref is not None - ids: dict[t.Any, t.Any] = {} - for item in items: - value = item.get(info.stored) if isinstance(item, dict) else getattr(item, info.name) - if value is None: - continue - values = ( - value - if info.ref.shape == "list" - else value.values() - if info.ref.shape == "dict" - else (value,) - ) - for v in values: - if v is not None: - ids[_key(v)] = id_of(v) - by_id: dict[t.Any, t.Any] = {} - if ids: - for target_meta in targets: - target = _target(manager, target_meta.model) - children = await _child_query( - target, include, {"_id": {"$in": list(ids.values())}}, mode, session - ).all() - for child in children: - by_id[_key(_item_id(child))] = child - model = targets[0].model if len(targets) == 1 else None - for item in items: - _attach_forward(item, info, by_id, model, include.per_parent_limit) + values = ( + value if info.ref.shape == "list" else value.values() if info.ref.shape == "dict" else (value,) + ) + for v in values: + if v is not None: + ids[_key(v)] = id_of(v) + by_id: dict[t.Any, t.Any] = {} + if ids: + for target_meta in targets: + target = _target(manager, target_meta.model) + children = await _child_query( + target, include, {"_id": {"$in": list(ids.values())}}, mode, session + ).all() + for child in children: + by_id[_key(_item_id(child))] = child + model = targets[0].model if len(targets) == 1 else None + for item in items: + _attach_forward(item, info, by_id, model, include.per_parent_limit) async def _load_backref( @@ -389,9 +397,33 @@ async def _load_backref( for item in items: _attach_backref(item, info, counts.get(_key(_item_id(item)), 0)) return - children = await _child_query( - target, include, {fk: {"$in": parent_ids}}, mode, session, extra_select=(fk_attr,) - ).all() + wanted: dict[str, t.Any] = {fk: {"$in": parent_ids}} + fk_info = target.meta.fields.get(fk_attr) + single = fk_info is not None and fk_info.ref is not None and fk_info.ref.shape == "one" + if include.per_parent_limit and include.limit is None and single: + # Choose the children on the server: at most N ids per parent come back, and only + # those documents are then fetched. Fetching every child to keep N per parent would + # transfer the whole relation for a parent with many children. + pipeline = [{"$match": wanted}] + if include.match is not None: + pipeline.append({"$match": compile_filter(include.match, target.meta)}) + order = compile_sort(include.sort, target.meta) if include.sort else [("_id", 1)] + pipeline.append({"$sort": dict(order)}) + pipeline.append( + { + "$group": { + "_id": f"${fk}", + "ids": {"$firstN": {"input": "$_id", "n": include.per_parent_limit}}, + } + } + ) + chosen = [i async for row in target.aggregate(pipeline, session=session) for i in row["ids"]] + wanted = {"_id": {"$in": chosen}} + if not chosen: + for item in items: + _attach_backref(item, info, []) + return + children = await _child_query(target, include, wanted, mode, session, extra_select=(fk_attr,)).all() grouped: dict[t.Any, list[t.Any]] = {} for child in children: value = child.get(fk) if isinstance(child, dict) else getattr(child, fk_attr) diff --git a/packages/mongospecs/src/mongospecs/tracking.py b/packages/mongospecs/src/mongospecs/tracking.py index 938f6fb..05746f8 100644 --- a/packages/mongospecs/src/mongospecs/tracking.py +++ b/packages/mongospecs/src/mongospecs/tracking.py @@ -26,10 +26,14 @@ __all__ = ["DocState", "diff", "forget", "state_of", "track"] +def _nothing() -> None: + return None + + class DocState: """What mongospecs remembers about a tracked document.""" - __slots__ = ("_doc", "_raw", "manager", "partial") + __slots__ = ("_doc", "_raw", "_ref", "manager", "partial") def __init__( self, @@ -41,6 +45,7 @@ def __init__( self._doc: dict[str, t.Any] | None = None if isinstance(snapshot, bytes) else snapshot self.manager = manager self.partial = partial + self._ref: t.Any = _nothing # the weak reference to the tracked object, set by track() """Stored keys that were loaded, when the document was read with a projection.""" @property @@ -67,7 +72,40 @@ def snapshot(self) -> dict[str, t.Any]: def state_of(obj: t.Any) -> DocState | None: - return _states.get(id(obj)) + state = _states.get(id(obj)) + if state is not None and state._ref() is not obj: + return None # the entry belongs to an object that is being collected; its id was reused + return state + + +class _Watch(weakref.ref): # type: ignore[type-arg] + """A weak reference that remembers which entry of the side table it guards. + + One plain weak reference per tracked document is several times cheaper than a + ``weakref.finalize`` (which also registers itself in a global table). + """ + + __slots__ = ("key",) + key: int + + +def _gone(watch: _Watch) -> None: + # ids are reused: only drop the entry if it still belongs to the object that died + state = _states.get(watch.key) + if state is not None and state._ref is watch: + del _states[watch.key] + + +def _install(obj: t.Any, state: DocState) -> None: + key = id(obj) + current = _states.get(key) + if current is not None and current._ref() is obj: + state._ref = current._ref + else: + watch = _Watch(obj, _gone) + watch.key = key + state._ref = watch + _states[key] = state def track( @@ -76,23 +114,17 @@ def track( manager: t.Any, partial: frozenset[str] | None = None, ) -> DocState: - key = id(obj) state = DocState(snapshot, manager, partial) - if key not in _states: - weakref.finalize(obj, _states.pop, key, None) - _states[key] = state + _install(obj, state) return state def restore(obj: t.Any, state: DocState | None) -> None: """Put back a previous state (used when a transaction aborts).""" - key = id(obj) if state is None: - _states.pop(key, None) + _states.pop(id(obj), None) else: - if key not in _states: - weakref.finalize(obj, _states.pop, key, None) - _states[key] = state + _install(obj, state) def forget(obj: t.Any) -> None: diff --git a/packages/mongospecs/tests/test_review_round2.py b/packages/mongospecs/tests/test_review_round2.py index e156461..4f08558 100644 --- a/packages/mongospecs/tests/test_review_round2.py +++ b/packages/mongospecs/tests/test_review_round2.py @@ -194,3 +194,30 @@ async def test_upsert_revision_is_documented_behaviour(m, registry): def test_unused_names_keep_imports_honest(): assert Index and SpecConfig and DocumentNotFound and MongoSpecsError and t + + +@pytest.mark.parametrize("mode", ["tracked", "lean", "raw"]) +async def test_cursor_pages_include_documents_without_the_sort_key(m, registry, mode): + """A missing or null sort key used to end the walk early or skip documents.""" + await m.User.insert_many( + [m.User(name=f"u{i}", email=f"{i}@x.io" if i % 2 else None, age=i) for i in range(6)] + ) + # no stored age at all: the model reads its default (0), the server sorts it as "no value" + await m.User.collection().insert_many([{"name": "no-age-1"}, {"name": "no-age-2"}]) + for sort, first, last in ( + ("email", "no-age-1", "u5"), + ("-email", "u5", "u4"), + ("age", "no-age-1", "u5"), + ("-age", "u5", "no-age-2"), + ): + query = getattr(m.User.find().sort(sort, "name"), mode)() + seen, cursor = [], None + while True: + page = await query.page(3, after=cursor) + seen += [u["name"] if isinstance(u, dict) else u.name for u in page] + assert len(seen) <= 12, f"{sort}: pagination does not terminate: {seen}" + if not page.has_next: + break + cursor = page.next_cursor + assert len(seen) == len(set(seen)) == 8, (sort, seen) + assert (seen[0], seen[-1]) == (first, last), (sort, seen) From 3ee709b1cc3a9dc6f77294d6053f88ac74d66db8 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 07:00:32 -0400 Subject: [PATCH 15/26] feat(web): Api(max_concurrency=...) and a 503 for pool timeouts Requests beyond the limit are answered at once with 503 and Retry-After instead of queueing for database connections. test: correctness of the optimised paths (tracking, clone, includes, per-parent limits, routing) Co-Authored-By: Claude Opus 5.5 --- .../mongospecs-web/src/mongospecs_web/api.py | 21 +- .../src/mongospecs_web/errors.py | 7 + .../tests/test_review_round2.py | 92 ++++++++ packages/mongospecs/tests/test_hot_paths.py | 208 ++++++++++++++++++ 4 files changed, 327 insertions(+), 1 deletion(-) create mode 100644 packages/mongospecs/tests/test_hot_paths.py diff --git a/packages/mongospecs-web/src/mongospecs_web/api.py b/packages/mongospecs-web/src/mongospecs_web/api.py index 69aece3..3aaf23a 100644 --- a/packages/mongospecs-web/src/mongospecs_web/api.py +++ b/packages/mongospecs-web/src/mongospecs_web/api.py @@ -53,6 +53,7 @@ def __init__( authenticate: Authenticate | None = None, max_body: int = 1_048_576, openapi: bool = True, + max_concurrency: int | None = None, ) -> None: self.resources: tuple[Resource[t.Any], ...] = resources self.prefix = "/" + prefix.strip("/") if prefix.strip("/") else "" @@ -60,6 +61,11 @@ def __init__( self.version = version self.authenticate = authenticate self.max_body = max_body + self.max_concurrency = max_concurrency + """At most this many requests are worked on at once; one more is answered at once with + 503 and ``Retry-After`` instead of waiting. Without a limit, overload turns into a + growing queue for database connections, and every request gets slower together.""" + self._in_flight = 0 self.expose_openapi = openapi """Serve ``/openapi.json``. Turn it off to keep the API description private; ``api.openapi()`` still builds it for you to serve behind your own authentication.""" @@ -183,7 +189,20 @@ async def run(self, route: Route, request: Request) -> Response: if response.status >= 500: logging.getLogger("mongospecs_web").error("authentication failed unexpectedly", exc_info=exc) return response - response = await route.resource.dispatch(route.operation, request) + if self.max_concurrency is not None and self._in_flight >= self.max_concurrency: + return Response.error( + HTTPError( + "The server is handling as many requests as it accepts. Try again shortly.", + status=503, + title="Service Unavailable", + headers={"retry-after": "1"}, + ) + ) + self._in_flight += 1 + try: + response = await route.resource.dispatch(route.operation, request) + finally: + self._in_flight -= 1 location = response.headers.get("location") if location and not location.startswith(("/", "http://", "https://")): # A resource names what it created by id; the URL of that thing depends on where diff --git a/packages/mongospecs-web/src/mongospecs_web/errors.py b/packages/mongospecs-web/src/mongospecs_web/errors.py index 9abb1cf..25c99da 100644 --- a/packages/mongospecs-web/src/mongospecs_web/errors.py +++ b/packages/mongospecs-web/src/mongospecs_web/errors.py @@ -121,6 +121,13 @@ def problem(exc: BaseException) -> HTTPError: return BadRequest(str(exc)) if isinstance(exc, me.OperationTimeout): return HTTPError("The database did not answer in time.", status=504, title="Gateway Timeout") + if isinstance(exc, me.PoolTimeout): + return HTTPError( + "The server is busy: no database connection became free in time.", + status=503, + title="Service Unavailable", + headers={"retry-after": "1"}, + ) if isinstance(exc, me.ConnectionFailed): return HTTPError("The database is unavailable.", status=503, title="Service Unavailable") return HTTPError() diff --git a/packages/mongospecs-web/tests/test_review_round2.py b/packages/mongospecs-web/tests/test_review_round2.py index e38af9b..1d8d54f 100644 --- a/packages/mongospecs-web/tests/test_review_round2.py +++ b/packages/mongospecs-web/tests/test_review_round2.py @@ -157,3 +157,95 @@ async def test_head_reports_the_length_of_the_get_body(): assert head.status_code == 200 and head.content == b"" assert head.headers["content-length"] == str(len(get.content)) != "0" assert head.headers["content-type"] == get.headers["content-type"] + + +# -- routing by resource, and refusing work beyond a limit ---------------------------------------- + + +async def test_routes_are_found_by_resource_and_keep_their_precedence(): + from mongospecs_web import Create, Patch + + base = contract_api("msgspec") + model, backend = base.resources[0].model, base.resources[0].backend + plain = Resource(model, name="things", ops=[List, Get, Create, Patch, Delete], backend=backend) + nested = Resource(model, name="shop/things", ops=[List, Get], backend=backend) + api = Api( + plain, + nested, + *[Resource(model, name=f"r{i}", ops=[List, Get], backend=backend) for i in range(40)], + prefix="/api", + ) + + @plain.op("search", method="GET", path="/search") + async def search(ctx): + return {"found": "search"} + + def target(method, path): + route, params = api.match(method, path) + return route.resource.name, route.operation.name, params + + assert target("GET", "/api/things") == ("things", "list", {}) + assert target("GET", "/api/things/") == ("things", "list", {}) + assert target("GET", "/api/things/search") == ("things", "search", {}) # a literal wins over /{id} + assert target("GET", "/api/things/abc") == ("things", "get", {"id": "abc"}) + assert target("GET", "/api/shop/things/abc") == ("shop/things", "get", {"id": "abc"}) + assert target("GET", "/api/r39/x") == ("r39", "get", {"id": "x"}) + for method, path, status in ( + ("PUT", "/api/things/abc", 405), + ("GET", "/api/nothing", 404), + ("GET", "/other/things", 404), + ("GET", "/api", 404), + ("GET", "/api/things/a/b/c", 404), + ): + assert (await api.handle(method, path)).status == status, (method, path) + assert (await api.handle("PUT", "/api/things/abc")).headers["allow"] == "DELETE, GET, PATCH" + + # an operation added after requests have been served is routable + @plain.op("archive", method="POST", path="/{id}/archive") + async def archive(ctx): + return {"archived": ctx.id} + + assert target("POST", "/api/things/abc/archive") == ("things", "archive", {"id": "abc"}) + assert (await api.handle("OPTIONS", "/api/things/abc/archive")).headers["allow"] == "OPTIONS, POST" + + +async def test_requests_beyond_max_concurrency_are_refused_at_once(): + import asyncio + + base = contract_api("msgspec") + model, backend = base.resources[0].model, base.resources[0].backend + resource = Resource(model, name="things", ops=[List], backend=backend) + release, entered = asyncio.Event(), asyncio.Event() + + @resource.op("slow", method="GET", path="/slow") + async def slow(ctx): + entered.set() + await release.wait() + return {"ok": True} + + api = Api(resource, prefix="/api", max_concurrency=2) + held = [asyncio.create_task(api.handle("GET", "/api/things/slow")) for _ in range(2)] + await entered.wait() + await asyncio.sleep(0) + refused = await api.handle("GET", "/api/things") + assert refused.status == 503 and refused.headers["retry-after"] == "1" + release.set() + assert [r.status for r in await asyncio.gather(*held)] == [200, 200] + assert (await api.handle("GET", "/api/things")).status == 200 # capacity is back + # a failing handler gives its slot back too + crash = Resource(model, name="boom", ops=[], backend=backend) + + @crash.op("fail", method="GET", path="") + async def fail(ctx): + raise RuntimeError("boom") + + small = Api(crash, prefix="/api", max_concurrency=1) + assert [(await small.handle("GET", "/api/boom")).status for _ in range(3)] == [500, 500, 500] + + +def test_a_pool_timeout_is_a_503_with_retry_after(): + from mongospecs import PoolTimeout + from mongospecs_web.errors import problem + + error = problem(PoolTimeout("no connection")) + assert error.status == 503 and error.headers["retry-after"] == "1" diff --git a/packages/mongospecs/tests/test_hot_paths.py b/packages/mongospecs/tests/test_hot_paths.py new file mode 100644 index 0000000..987975a --- /dev/null +++ b/packages/mongospecs/tests/test_hot_paths.py @@ -0,0 +1,208 @@ +"""Correctness of the paths that were made cheaper: the result must not have changed.""" + +import asyncio +import gc +import typing as t + +import pytest + +from mongospecs import Backref, Q, Ref, SpecConfig, tracking + + +def started(events, name=None): + return [e for e in events if e.phase == "started" and (name is None or e.command_name == name)] + + +# -- tracking: one weak reference per document ------------------------------------------------- + + +async def test_tracking_state_goes_away_with_the_document(m, registry): + await m.User.insert_many([m.User(name=f"u{i}") for i in range(50)]) + gc.collect() + before = tracking.tracked_count() + docs = await m.User.find().all() + assert tracking.tracked_count() == before + 50 + first = docs[0] + del docs + gc.collect() + assert tracking.tracked_count() == before + 1 and tracking.state_of(first) is not None + # re-tracking the same object (save, reload) keeps exactly one entry and stays collectable + first.age = 3 + await first.save() + await first.reload() + assert tracking.tracked_count() == before + 1 and first.changes() == {} + del first + gc.collect() + assert tracking.tracked_count() == before + + +async def test_a_reused_id_never_inherits_another_documents_state(m, registry): + await m.User(name="a").insert() + for _ in range(200): # ids of freed objects are reused quickly; a fresh object must be untracked + loaded = await m.User.find().one() + del loaded + fresh = m.User(name="fresh") + assert tracking.state_of(fresh) is None and fresh.is_new + + +async def test_transaction_rollback_restores_tracking(m, registry): + user = await m.User(name="a").insert() + + async def change(tx): + user.name = "b" + await user.save(session=tx) + raise RuntimeError("abort") + + with pytest.raises(RuntimeError): + await registry.transaction(change) + assert user.changes() in ({}, {"$set": {"name": "b"}}) + assert (await m.User.get(user.id)).name == "a" + gc.collect() + assert tracking.state_of(user) is not None + + +# -- Query._clone ------------------------------------------------------------------------------ + + +def test_clone_keeps_every_setting_and_shares_nothing_mutable(m, registry): + base = ( + m.User.find(Q.age > 1) + .sort("-age") + .skip(2) + .limit(5) + .select("name") + .lean() + .options(max_time_ms=50) + .or_fail() + .on_invalid("skip") + ) + clone = base.limit(9) + for slot in type(base).__slots__: + if slot in ("_limit", "_keys"): + continue + assert getattr(clone, slot) == getattr(base, slot), slot + assert (base._limit, clone._limit) == (5, 9) + changed = base.options(max_time_ms=99) + assert base._options["max_time_ms"] == 50 and changed._options["max_time_ms"] == 99 + assert changed._options is not base._options + assert type(clone) is type(base) + + +# -- includes ---------------------------------------------------------------------------------- + + +@pytest.fixture +async def library(m, make_registry): + @m.define + class Writer(m.Spec): + __spec__ = SpecConfig(collection="hp_writers") + name: str = "" + books: t.Annotated[list[t.Any], Backref("writer", model="Book")] = [] # noqa: RUF012 + + @m.define + class Shelf(m.Spec): + __spec__ = SpecConfig(collection="hp_shelves") + name: str = "" + + @m.define + class Book(m.Spec): + __spec__ = SpecConfig(collection="hp_books") + title: str = "" + pages: int = 0 + writer: Ref[Writer] | None = None + shelf: Ref[Shelf] | None = None + co_writers: list[Ref[Writer]] = [] # noqa: RUF012 + + events = [] + registry = await make_registry(Writer, Shelf, Book, on_command=events.append) + writers = (await Writer.insert_many([Writer(name=f"w{i}") for i in range(4)])).inserted + shelves = (await Shelf.insert_many([Shelf(name=f"s{i}") for i in range(2)])).inserted + await Book.insert_many( + [ + Book(title=f"b{w}-{i:02d}", pages=(i * 7) % 11, writer=writers[w], shelf=shelves[i % 2]) + for w in range(3) # w3 has no books + for i in range(12) + ] + ) + return registry, events, Writer, Shelf, Book + + +async def test_independent_includes_are_in_flight_together(library): + registry, events, Writer, Shelf, Book = library + # open a few pooled connections first: a second connection being set up would hide the overlap + await asyncio.gather(*(Book.count() for _ in range(6))) + events.clear() + books = await Book.find().sort("title").include("writer").include("shelf").all() + assert len(books) == 36 and all(b.writer.loaded and b.shelf.loaded for b in books) + assert books[0].writer.doc.name == "w0" and {b.shelf.doc.name for b in books} == {"s0", "s1"} + finds = [e for e in events if e.command_name == "find"] + order = [(e.phase, e.command["find"] if e.phase == "started" else None) for e in finds] + relation_starts = [i for i, (phase, coll) in enumerate(order) if coll in ("hp_writers", "hp_shelves")] + first_done_after = next( + i for i, (phase, _) in enumerate(order) if phase != "started" and i > relation_starts[0] + ) + assert ( + len(relation_starts) == 2 and relation_starts[1] < first_done_after + ) # both sent before either returned + assert len(started(events)) == 3 # still one command per relation + + # inside a transaction they run one after another: a session cannot do two things at once + async def in_tx(tx): + return await Book.find(session=tx).include("writer").include("shelf").all() + + assert len(await registry.transaction(in_tx)) == 36 + + +@pytest.mark.parametrize("mode", ["tracked", "lean", "raw"]) +async def test_per_parent_limit_is_applied_by_the_server(library, mode): + registry, events, Writer, Shelf, Book = library + expected = {} + for writer in await Writer.find().all(): + mine = await Book.find(Q.writer == writer.id, Q.pages >= 2).sort("-pages", "title").all() + expected[writer.name] = [b.title for b in mine[:3]] + events.clear() + query = getattr(Writer.find().sort("name"), mode)() + writers = await query.include( + "books", per_parent_limit=3, match=Q.pages >= 2, sort=["-pages", "title"] + ).all() + + def books_of(w): + return w["books"] if isinstance(w, dict) else w.books + + def name_of(w): + return w["name"] if isinstance(w, dict) else w.name + + def title_of(b): + return b["title"] if isinstance(b, dict) else b.title + + got = {name_of(w): [title_of(b) for b in books_of(w)] for w in writers} + assert got == expected and got["w3"] == [] and all(len(v) == 3 for k, v in got.items() if k != "w3") + # the children are chosen on the server: only the kept ones are fetched + (pick,) = started(events, "aggregate") + assert any("$firstN" in str(stage) for stage in pick.command["pipeline"]) + fetch = [e for e in started(events, "find") if e.command["find"] == "hp_books"] + assert len(fetch) == 1 and len(fetch[0].command["filter"]["_id"]["$in"]) == 9 + + # without a sort the per-parent cap still holds + capped = await Writer.find().include("books", per_parent_limit=2).all() + assert sorted(len(w.books) for w in capped) == [0, 2, 2, 2] + # an overall limit keeps the previous behaviour (limit first, then the cap per parent) + overall = await Writer.find().include("books", per_parent_limit=2, limit=5, sort=["title"]).all() + assert sum(len(w.books) for w in overall) <= 5 + + +async def test_per_parent_limit_on_a_list_reference_still_works(library): + registry, events, Writer, Shelf, Book = library + + @Writer.scope + def named(query): # an unrelated named scope must not disturb relation loading + return query.filter(Q.name != "") + + del named + books = await Book.find().limit(2).all() + writers = await Writer.find().sort("name").all() + books[0].co_writers = [writers[0], writers[1]] + await books[0].save() + loaded = await Book.find(Q.id == books[0].id).include("co_writers", per_parent_limit=1).one() + assert len(loaded.co_writers) == 2 and sum(r.loaded for r in loaded.co_writers) >= 1 + assert asyncio.get_running_loop() is not None From fb543de1b1a0b8af560ce5c77578ea435d5d5cd4 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 07:01:58 -0400 Subject: [PATCH 16/26] bench: B9 also measures serialisation alone, through the model and from the stored document Co-Authored-By: Claude Opus 5.5 --- bench/results/3ee709b/B9.json | 279 +++++++++++++++++++++++++ bench/src/mongospecs_bench/hotpaths.py | 24 +++ 2 files changed, 303 insertions(+) create mode 100644 bench/results/3ee709b/B9.json diff --git a/bench/results/3ee709b/B9.json b/bench/results/3ee709b/B9.json new file mode 100644 index 0000000..8c606b5 --- /dev/null +++ b/bench/results/3ee709b/B9.json @@ -0,0 +1,279 @@ +{ + "meta": { + "git_sha": "3ee709b", + "git_dirty": false, + "timestamp": "2026-10-13T11:00:33+00:00", + "topology": "rs", + "quick": false, + "mongodb": "8.0.32", + "python": "3.13.5", + "free_threaded": false, + "platform": "macOS-15.6.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpus": 8, + "versions": { + "mongospecs": "0.2.0", + "pymongo": "4.18.3", + "msgspec": "0.22.0", + "pydantic": "2.14.1" + } + }, + "workload": "B9", + "title": "Hot paths: CPU per call and commands per operation", + "question": "Where does the library spend CPU on the paths every request takes, and how many commands does each operation send?", + "rows": [ + { + "backend": "msgspec", + "path": "web: GET list of 100", + "unit": "cpu us/doc", + "value": 9.68 + }, + { + "backend": "msgspec", + "path": "web: GET one", + "unit": "cpu us/call", + "value": 289.4 + }, + { + "backend": "msgspec", + "path": "Model.get(id)", + "unit": "cpu us/call", + "value": 211.0 + }, + { + "backend": "msgspec", + "path": "find().all(), 100 tracked", + "unit": "cpu us/doc", + "value": 7.91 + }, + { + "backend": "msgspec", + "path": "find().lean().all(), 100", + "unit": "cpu us/doc", + "value": 6.87 + }, + { + "backend": "msgspec", + "path": "include x2, 100 parents", + "unit": "cpu us/call", + "value": 1946.7 + }, + { + "backend": "msgspec", + "path": "include reviews, per_parent_limit=2 of 100", + "unit": "cpu us/call", + "value": 1872.0 + }, + { + "backend": "msgspec", + "path": "web: GET one", + "unit": "commands", + "value": 1.0, + "detail": "find" + }, + { + "backend": "msgspec", + "path": "web: GET list of 100", + "unit": "commands", + "value": 1.0, + "detail": "find" + }, + { + "backend": "msgspec", + "path": "include x2, 100 parents", + "unit": "commands", + "value": 3.0, + "detail": "find find find" + }, + { + "backend": "msgspec", + "path": "include reviews, per_parent_limit=2 of 100", + "unit": "commands", + "value": 3.0, + "detail": "find aggregate find" + }, + { + "backend": "msgspec", + "path": "web: DELETE (soft)", + "unit": "commands", + "value": 1.0, + "detail": "update" + }, + { + "backend": "-", + "path": "route match, last of 29 routes", + "unit": "cpu us/call", + "value": 2.76 + }, + { + "backend": "-", + "path": "route match, last of 255 routes", + "unit": "cpu us/call", + "value": 2.34 + }, + { + "backend": "-", + "path": "route match, last of 1005 routes", + "unit": "cpu us/call", + "value": 13.13 + }, + { + "backend": "pydantic", + "path": "web: GET list of 100", + "unit": "cpu us/doc", + "value": 12.24 + }, + { + "backend": "pydantic", + "path": "web: GET one", + "unit": "cpu us/call", + "value": 276.2 + }, + { + "backend": "pydantic", + "path": "Model.get(id)", + "unit": "cpu us/call", + "value": 211.6 + }, + { + "backend": "pydantic", + "path": "find().all(), 100 tracked", + "unit": "cpu us/doc", + "value": 9.68 + }, + { + "backend": "pydantic", + "path": "find().lean().all(), 100", + "unit": "cpu us/doc", + "value": 8.17 + }, + { + "backend": "pydantic", + "path": "include x2, 100 parents", + "unit": "cpu us/call", + "value": 1968.4 + }, + { + "backend": "pydantic", + "path": "include reviews, per_parent_limit=2 of 100", + "unit": "cpu us/call", + "value": 1600.4 + }, + { + "backend": "pydantic", + "path": "web: GET one", + "unit": "commands", + "value": 1.0, + "detail": "find" + }, + { + "backend": "pydantic", + "path": "web: GET list of 100", + "unit": "commands", + "value": 1.0, + "detail": "find" + }, + { + "backend": "pydantic", + "path": "include x2, 100 parents", + "unit": "commands", + "value": 3.0, + "detail": "find find find" + }, + { + "backend": "pydantic", + "path": "include reviews, per_parent_limit=2 of 100", + "unit": "commands", + "value": 3.0, + "detail": "find aggregate find" + }, + { + "backend": "pydantic", + "path": "web: DELETE (soft)", + "unit": "commands", + "value": 1.0, + "detail": "update" + }, + { + "backend": "attrs", + "path": "web: GET list of 100", + "unit": "cpu us/doc", + "value": 9.1 + }, + { + "backend": "attrs", + "path": "web: GET one", + "unit": "cpu us/call", + "value": 258.3 + }, + { + "backend": "attrs", + "path": "Model.get(id)", + "unit": "cpu us/call", + "value": 225.8 + }, + { + "backend": "attrs", + "path": "find().all(), 100 tracked", + "unit": "cpu us/doc", + "value": 10.05 + }, + { + "backend": "attrs", + "path": "find().lean().all(), 100", + "unit": "cpu us/doc", + "value": 10.36 + }, + { + "backend": "attrs", + "path": "include x2, 100 parents", + "unit": "cpu us/call", + "value": 2281.6 + }, + { + "backend": "attrs", + "path": "include reviews, per_parent_limit=2 of 100", + "unit": "cpu us/call", + "value": 2019.9 + }, + { + "backend": "attrs", + "path": "web: GET one", + "unit": "commands", + "value": 1.0, + "detail": "find" + }, + { + "backend": "attrs", + "path": "web: GET list of 100", + "unit": "commands", + "value": 1.0, + "detail": "find" + }, + { + "backend": "attrs", + "path": "include x2, 100 parents", + "unit": "commands", + "value": 3.0, + "detail": "find find find" + }, + { + "backend": "attrs", + "path": "include reviews, per_parent_limit=2 of 100", + "unit": "commands", + "value": 3.0, + "detail": "find aggregate find" + }, + { + "backend": "attrs", + "path": "web: DELETE (soft)", + "unit": "commands", + "value": 1.0, + "detail": "update" + } + ], + "notes": [ + "CPU time of this process (best of several batches), so the database's own work and network waits are excluded. Rows with unit 'commands' are driver commands sent, listed in 'detail'." + ], + "conclusion": "" +} diff --git a/bench/src/mongospecs_bench/hotpaths.py b/bench/src/mongospecs_bench/hotpaths.py index 095e3d3..9f98f7f 100644 --- a/bench/src/mongospecs_bench/hotpaths.py +++ b/bench/src/mongospecs_bench/hotpaths.py @@ -135,6 +135,30 @@ def add(path: str, unit: str, value: float, per: int = 1) -> None: await cpu_us(per_parent, calls // 4, repeats), ) + # The same 100 stored documents turned into the response rows, with the database out + # of the picture: through the model, and straight from the stored document. + view = api.resources[0].view + meta = view.meta + stored = [d async for d in m.Product.collection().find({}, limit=100)] + stored = [dict(d) for d in stored] + plan = getattr(view, "fast", None) + for label, fn in ( + ("serialise 100: decode + model", lambda: [view.serialize(meta.decode(d)) for d in stored]), + ( + "serialise 100: stored document", + (lambda: [plan.serialize(d) for d in stored]) if plan else None, + ), + ): + if fn is None: + continue + best = float("inf") + for _ in range(repeats * 4): + start = time.process_time() + for _ in range(20): + fn() + best = min(best, (time.process_time() - start) / 20) + add(label, "cpu us/doc", best * 1e6, 100) + def count(name: str, *paths: str) -> None: add(name, "commands", float(len(events.names))) result.rows[-1]["detail"] = " ".join(events.names) From 9775738ad0231cc414648e85e5430b4a4f80319e Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 07:14:24 -0400 Subject: [PATCH 17/26] docs,bench: production tuning guide, B9 before/after, review decisions - docs/guide/production.md: pool and timeout settings, backpressure, read and pagination choices, relations, writes, what needs a replica set; each item marked measured, tested or guidance - docs/benchmarks.md: hot paths before and after the review - bench/results: B9 before (49a32a9) and after (fb543de) - plan: decision per review finding, amendment A36 Co-Authored-By: Claude Opus 5.5 --- bench/results/{3ee709b => fb543de}/B9.json | 90 +++++++++---- claudedocs/v2-plan.md | 26 ++++ docs/benchmarks.md | 34 +++++ docs/guide/production.md | 136 ++++++++++++++++++++ docs/web/resources.md | 8 ++ mkdocs.yaml | 1 + packages/mongospecs/tests/test_hot_paths.py | 17 +++ 7 files changed, 285 insertions(+), 27 deletions(-) rename bench/results/{3ee709b => fb543de}/B9.json (80%) create mode 100644 docs/guide/production.md diff --git a/bench/results/3ee709b/B9.json b/bench/results/fb543de/B9.json similarity index 80% rename from bench/results/3ee709b/B9.json rename to bench/results/fb543de/B9.json index 8c606b5..e35678b 100644 --- a/bench/results/3ee709b/B9.json +++ b/bench/results/fb543de/B9.json @@ -1,8 +1,8 @@ { "meta": { - "git_sha": "3ee709b", - "git_dirty": false, - "timestamp": "2026-10-13T11:00:33+00:00", + "git_sha": "fb543de", + "git_dirty": true, + "timestamp": "2026-10-13T11:01:58+00:00", "topology": "rs", "quick": false, "mongodb": "8.0.32", @@ -26,43 +26,55 @@ "backend": "msgspec", "path": "web: GET list of 100", "unit": "cpu us/doc", - "value": 9.68 + "value": 10.14 }, { "backend": "msgspec", "path": "web: GET one", "unit": "cpu us/call", - "value": 289.4 + "value": 258.8 }, { "backend": "msgspec", "path": "Model.get(id)", "unit": "cpu us/call", - "value": 211.0 + "value": 217.6 }, { "backend": "msgspec", "path": "find().all(), 100 tracked", "unit": "cpu us/doc", - "value": 7.91 + "value": 9.77 }, { "backend": "msgspec", "path": "find().lean().all(), 100", "unit": "cpu us/doc", - "value": 6.87 + "value": 7.78 }, { "backend": "msgspec", "path": "include x2, 100 parents", "unit": "cpu us/call", - "value": 1946.7 + "value": 2780.9 }, { "backend": "msgspec", "path": "include reviews, per_parent_limit=2 of 100", "unit": "cpu us/call", - "value": 1872.0 + "value": 1468.7 + }, + { + "backend": "msgspec", + "path": "serialise 100: decode + model", + "unit": "cpu us/doc", + "value": 4.34 + }, + { + "backend": "msgspec", + "path": "serialise 100: stored document", + "unit": "cpu us/doc", + "value": 2.5 }, { "backend": "msgspec", @@ -103,61 +115,73 @@ "backend": "-", "path": "route match, last of 29 routes", "unit": "cpu us/call", - "value": 2.76 + "value": 1.01 }, { "backend": "-", "path": "route match, last of 255 routes", "unit": "cpu us/call", - "value": 2.34 + "value": 1.3 }, { "backend": "-", "path": "route match, last of 1005 routes", "unit": "cpu us/call", - "value": 13.13 + "value": 15.36 }, { "backend": "pydantic", "path": "web: GET list of 100", "unit": "cpu us/doc", - "value": 12.24 + "value": 10.57 }, { "backend": "pydantic", "path": "web: GET one", "unit": "cpu us/call", - "value": 276.2 + "value": 265.7 }, { "backend": "pydantic", "path": "Model.get(id)", "unit": "cpu us/call", - "value": 211.6 + "value": 226.5 }, { "backend": "pydantic", "path": "find().all(), 100 tracked", "unit": "cpu us/doc", - "value": 9.68 + "value": 12.17 }, { "backend": "pydantic", "path": "find().lean().all(), 100", "unit": "cpu us/doc", - "value": 8.17 + "value": 8.54 }, { "backend": "pydantic", "path": "include x2, 100 parents", "unit": "cpu us/call", - "value": 1968.4 + "value": 3018.1 }, { "backend": "pydantic", "path": "include reviews, per_parent_limit=2 of 100", "unit": "cpu us/call", - "value": 1600.4 + "value": 1341.4 + }, + { + "backend": "pydantic", + "path": "serialise 100: decode + model", + "unit": "cpu us/doc", + "value": 6.74 + }, + { + "backend": "pydantic", + "path": "serialise 100: stored document", + "unit": "cpu us/doc", + "value": 2.89 }, { "backend": "pydantic", @@ -198,43 +222,55 @@ "backend": "attrs", "path": "web: GET list of 100", "unit": "cpu us/doc", - "value": 9.1 + "value": 11.16 }, { "backend": "attrs", "path": "web: GET one", "unit": "cpu us/call", - "value": 258.3 + "value": 256.4 }, { "backend": "attrs", "path": "Model.get(id)", "unit": "cpu us/call", - "value": 225.8 + "value": 301.0 }, { "backend": "attrs", "path": "find().all(), 100 tracked", "unit": "cpu us/doc", - "value": 10.05 + "value": 12.48 }, { "backend": "attrs", "path": "find().lean().all(), 100", "unit": "cpu us/doc", - "value": 10.36 + "value": 9.74 }, { "backend": "attrs", "path": "include x2, 100 parents", "unit": "cpu us/call", - "value": 2281.6 + "value": 2331.4 }, { "backend": "attrs", "path": "include reviews, per_parent_limit=2 of 100", "unit": "cpu us/call", - "value": 2019.9 + "value": 1667.0 + }, + { + "backend": "attrs", + "path": "serialise 100: decode + model", + "unit": "cpu us/doc", + "value": 6.43 + }, + { + "backend": "attrs", + "path": "serialise 100: stored document", + "unit": "cpu us/doc", + "value": 2.46 }, { "backend": "attrs", diff --git a/claudedocs/v2-plan.md b/claudedocs/v2-plan.md index 0a3207f..f966e02 100644 --- a/claudedocs/v2-plan.md +++ b/claudedocs/v2-plan.md @@ -887,6 +887,7 @@ Recorded per §7.1 rule 4. Each one names what changed and why; none invalidates | A33 | §4.3 | Filter values are cast by one rule on all backends (exact type, else text that reads as the type). `find().on_invalid("raise" \| "skip" \| "raw")` | Second review | | A34 | §4.8 | `Policy(private=True)` is enforced on fields of embedded models at any depth: not serialised, a 422 when supplied, absent from schemas | Third review item A. PATCH replaces an embedded object whole, so a client that patches the parent object clears server-set private values inside it; documented, not changed | | A35 | §4.2 | `StoreNone` / `OmitNone` markers and `SpecConfig(none="omit" \| "store")`. Embedded models resolve the setting from their own markers, or from a `__spec__` if they declare one; they do not inherit the parent document's setting. A unique sparse index over a field that stores null is a `ConfigurationError`. No warning is raised for a partial filter on `$type: null` over an omitted field | Requested as the opt-out from A30. The embedded rule keeps one model's stored shape the same wherever it is embedded | +| A36 | §4.3 | Cursor pagination handles documents without a value for a sort key (first ascending, last descending) and builds the cursor from stored values, not model defaults | Found while testing raw reads: such documents were skipped or ended the walk. Became common once None was no longer stored (A30) | New node: @@ -927,3 +928,28 @@ Known and not changed: - S7 (confirming every Mongoose parity row against its documentation) is done for 28 of 35 rows; `docs/parity.md` marks the 7 that are still from memory (alias, `useDb`, per-query driver options, `exists`/counts, subdocuments and maps, getters/setters, virtuals). + +### Performance review (2026-10-13): decisions + +Thirteen findings from a profiling review. Numbers are from workload B9, `bench/results/49a32a9` +(before) and `bench/results/fb543de` (after). + +| # | Finding | Decision | Reason | +|---|---|---|---| +| 1 | Web reads build a model only to serialise it; private fields travel over the wire | Implemented: projection of the readable fields, serialise from the stored document when certain to be identical, model path otherwise (`mongospecs_web.fastread`) | Serialise alone 4.3 → 2.5 µs/doc; whole request under 10% because the driver dominates. Kept for the projection and because it is verified equal | +| 2 | Refs handled twice on decode (dec_hook, then normalisation) | Not done | Changes the decode core of three backends for a gain of about 1 µs/doc; revisit if per-document CPU becomes the limit | +| 3 | Encode is a Python walker; save rebuilds the snapshot through the model | Not done | Same reason; the model round trip is also what keeps a load from looking like an edit (A15/A16) | +| 4 | `weakref.finalize` per tracked document | Implemented: one `weakref.ref` held on the state, guarded against id reuse | Cheap and contained; effect not visible in B9 | +| 5 | Batched includes run one after another | Implemented: gathered outside a session | A session cannot run two operations at once, so inside a transaction they stay serial | +| 6 | `per_parent_limit` applied after fetching every child | Implemented: `$sort` + `$group` with `$firstN` picks ids on the server | 23.0 → 1.5 ms CPU for 20 parents × 100 children | +| 7 | Web DELETE is two round trips | Implemented: one conditional write unless the model has delete hooks or cascades | 2 → 1 commands | +| 8 | `Query._clone` cost; `get` through `find().first()` | Clone implemented (slot copy). `find_one` fast path not done | `get` 248 → 218 µs is within noise; a second read path is not worth it | +| 9 | Hard delete of a cascade target needs a transaction, so a replica set | Documented | A cascade that half-completes is worse than a clear requirement | +| 10 | `total=True` is a full count per request | Documented; totals are already opt-in per resource | An estimate is wrong for any filtered listing; a cache belongs to the application | +| 11 | Routing scans every route | Implemented: lookup by resource, then that resource's patterns | 19.1 → 1.3 µs at 255 routes | +| 12 | No backpressure | Implemented `Api(max_concurrency=)` and 503 for `PoolTimeout`; pool defaults left to the driver and documented | Changing the driver's defaults silently would surprise; the guide says what to set | +| 13 | Memory shape of pydantic documents; `all()` materialises | Documented | Inherent to the model library | + +Design-level options not taken: per-model compiled plans shared by decode and encode, lazy `Ref` +materialisation, an explain-sampling listener, state on the instance for free-threading, a native +decoder. None is needed by a measured workload yet. diff --git a/docs/benchmarks.md b/docs/benchmarks.md index b01f216..25e38c8 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -244,6 +244,40 @@ BSON-to-model decoder as a separate optional package behind `Codec.decode(bytes) already there: tracked reads receive undecoded BSON from the driver today. A Rust driver is not indicated by anything here. +## Hot paths after the performance review + +A review profiled the library and listed thirteen places where work could be saved. Workload B9 +measures them through public entry points: CPU time of the client process, and commands sent. +Before is `bench/results/49a32a9/B9.json`, after is `bench/results/fb543de/B9.json` (msgspec rows). + +| Path | Before | After | +|---|---|---| +| Reverse relation with `per_parent_limit=2`, 20 parents × 100 children | 23.0 ms CPU | 1.5 ms CPU | +| Web `DELETE` (soft) | 2 commands | 1 command | +| Route match, last of 255 routes | 19.1 µs | 1.3 µs | +| Route match, last of 1005 routes | 73.3 µs | 15.4 µs | +| Serialise a document for a response | 4.3 µs through the model | 2.5 µs from the stored document | +| Web `GET` list of 100, per document | 10.9 µs | 10.1 µs | +| `Model.get(id)` | 248 µs | 218 µs | +| `find().all()`, 100 tracked, per document | 8.7 µs | 9.8 µs | + +What this shows: + +- The large wins are where work was avoided altogether: children that were fetched to be thrown + away, a read before a delete, a scan over every route. +- Serialising from the stored document is 1.7 to 2.6 times cheaper than going through the model + (pydantic 6.7 → 2.9 µs, attrs 6.4 → 2.5 µs), but a whole list request improves by under 10% here: + the driver and the event loop are most of a request, as the Rust section above already found. +- The last three rows are within run-to-run variation on this machine. The cheaper weak reference + per tracked document, the cheaper query clone and concurrent includes are correct and tested, + but this benchmark cannot show their effect; do not read the tracked row as a regression or the + `get` row as a gain. + +Not done, and why: compiling encode plans and skipping reference normalisation on decode (review +items 2 and 3) change the decode and encode core for a gain B1 puts at a few microseconds per +document; they are left until a workload shows per-document CPU, rather than the driver, is the +limit. + ## Limits of these measurements - One machine, client and server together, MongoDB in Docker on macOS. Network cost is unrealistically diff --git a/docs/guide/production.md b/docs/guide/production.md new file mode 100644 index 0000000..28d6d12 --- /dev/null +++ b/docs/guide/production.md @@ -0,0 +1,136 @@ +# Tuning for production and high concurrency + +Each item says what backs it: **measured** (a committed result under `bench/results/`), **tested** +(a test in this repository asserts it), or **guidance** (sound practice that nothing here verifies). + +## Connections and timeouts + +Any keyword `Registry` does not know is passed to the driver, so the pool is configured there: + +```py +registry = Registry( + uri, + models=[...], + max_time_ms=5_000, # default time limit for reads + maxPoolSize=50, # connections per process (driver default: 100) + minPoolSize=5, # kept open, so a burst does not pay for handshakes + waitQueueTimeoutMS=500, # fail instead of queueing for a connection without bound + serverSelectionTimeoutMS=5_000, + connectTimeoutMS=2_000, +) +``` + +- **Set `waitQueueTimeoutMS`.** Without it a request that finds the pool empty waits indefinitely, + so overload shows up as every request getting slower. With it the operation raises `PoolTimeout`, + which the web layer answers with 503 and `Retry-After`. *Tested* + (`test_a_full_pool_fails_fast_when_a_wait_limit_is_set`, `test_a_pool_timeout_is_a_503_with_retry_after`). +- **Size the pool for the database, not for the process.** `maxPoolSize` × processes is the number of + connections the server must hold. One event loop rarely makes use of more than a few dozen. *Guidance.* +- **Give reads a time limit.** `Registry(max_time_ms=...)` or `SpecConfig(max_time_ms=...)`; an + operation that exceeds it raises `OperationTimeout` (HTTP 504). The web layer applies 10 s to reads + made for a request when neither is set. *Tested* (`test_listing_limits`). +- **One registry per process and event loop.** Open it when the server starts and close it when it + stops (`plug.lifespan(registry)`, `init_app(..., registry=registry)`). A model class is bound to + one open registry; use `registry.manager(Model)` where you need several. *Tested* (lifecycle and + binding tests); do not share a registry across event loops. *Guidance.* + +## Refusing work you cannot do + +```py +api = Api(*resources, max_concurrency=200, max_body=1_048_576) +``` + +- `max_concurrency` answers the request beyond the limit at once with 503 and `Retry-After: 1` + instead of letting it queue. Set it a little above the concurrency your pool and database + sustain. *Tested* (`test_requests_beyond_max_concurrency_are_refused_at_once`); the right number + for your deployment is *guidance*: find it with the load workload below. +- `max_body` is enforced while the body is read, on every plug. *Tested* + (`test_3_no_plug_reads_a_body_past_the_limit`). +- `Offset(max_offset=...)` caps how far a client can page with skip/limit, and `Cursor(max=...)` the + page size. *Tested.* + +## Reads + +- **Use `lean()` for anything you will not save.** A tracked document also keeps its stored bytes: + 3.6 KB against 2.4 KB of heap per document for the benchmark's shape (msgspec). The time + difference between tracked and lean reads is small. `raw()` skips the model altogether and + roughly halves the time, but a dict takes more memory than a model (5.3 KB here). + (`bench/results/6dc837f/B8.json`). *Measured.* +- **Project.** `select("name", "price")` fetches only those fields; mark large, rarely needed + fields `Deferred` so they are left out by default. *Tested.* +- **Stream large results** with `async for doc in Model.find(...)` instead of `.all()`, which holds + the whole result. Pydantic instances take roughly 2.6 times the memory of msgspec or attrs ones + (1829 against 709 bytes per lean document of the review's sample shape). *Measured by the + reviewer; there is no committed result for it.* +- **The web layer reads without building models where it can.** `GET` on a resource fetches only + the readable fields and serialises the stored document directly when the model has nothing that + could change a value on the way (validators, converters, computed fields, a discriminator); any + other model, and any single document whose stored values do not have their declared types, goes + through the model. Serialising alone costs 2.5 µs per document this way against 4.3 (msgspec), + 6.4 (attrs) and 6.7 (pydantic) through the model; for a whole request the gain is smaller because + the driver dominates (`bench/results/fb543de/B9.json`). *Measured*, and *tested* to give the same + response (`test_raw_reads_equal_model_reads`). `Resource.raw_reads = False` turns it off. + +## Pagination + +- **Prefer cursors.** `Cursor()` (keyset) costs the same on page 1 and page 10,000; `Offset()` + makes the server walk and discard everything before the page: at page 1,250 of a 50,000-document + collection, 7.1 ms against 0.66 ms. *Measured* (`bench/results/6dc837f/B5.json`). +- **The index must end in `_id`.** Cursor pagination sorts by your key and then by `_id`: + declare `Index("price", "id")`. With an index on the key alone the first page took 28.8 ms + instead of 0.58 ms. *Measured* (B5). +- Documents without a value for the sort key are paged too (first ascending, last descending). + *Tested* (`test_cursor_pages_include_documents_without_the_sort_key`). +- **`?total=true` is a second query over every match.** It is off unless the resource opts in + with `Cursor(total=True)`. There is no cached or estimated total; if you need one, keep a counter. + *Tested* (two commands; `test_listing_limits`). + +## Relations + +- `include()` uses one query per relation for the whole page by default. Independent relations are + sent together, except inside a transaction. *Tested* + (`test_independent_includes_are_in_flight_together`). +- **Give reverse relations a `per_parent_limit`.** `include("reviews", per_parent_limit=5)` chooses + the children on the server, so a parent with 100,000 children costs the same as one with five. + With 20 parents of 100 children each this took 1.5 ms of client CPU against 23 ms when every + child was fetched (`bench/results/49a32a9/B9.json` before, `fb543de` after). *Measured and tested.* +- Index the foreign key of every reverse relation (`plan.advisories` lists the ones that lack one). + *Tested.* + +## Writes + +- `insert_many`, `bulk_save` and `bulk_write` send one command per batch; inserting 1,000 documents + costs 1.08 times the driver's own `insert_many` (`bench/results/6dc837f/B4.json`). *Measured.* +- `save()` sends only the changed fields; `Model.update(id, Inc(...))` is one atomic round trip and + needs no read. *Tested.* +- A web `DELETE` is one conditional write. It needs a read first only when the model has delete + hooks, or cascades on a hard delete. *Tested* (`test_delete_is_one_write_unless_the_document_is_needed`). + +## What needs a replica set + +Transactions and change streams need a replica set (a single-node one is enough). That includes +**hard-deleting a document that other documents refer to with `OnDelete`**: the cascade and the +delete run in one transaction, which a standalone `mongod` cannot do. Soft deletes do not cascade +and need no transaction. *Tested on a replica set; not supported on a standalone server.* + +Change streams resume across a primary step-down (`bench/results/6dc837f/X6.json`). To resume +across a restart of your own process, store `event.resume_token` and pass it back as +`resume_after=`. *Measured for the step-down; the restart pattern is guidance.* + +## Observability + +`on_command` listeners run for every command, inline. The logging, OpenTelemetry and Sentry +listeners do little per command, but anything slow in a listener is added to every operation. +*Guidance:* their cost has not been measured. + +## Measuring your own workload + +```sh +make up # MongoDB replica set and sharded cluster +uv run python -m mongospecs_bench.cli run B9 # CPU per call and commands per operation +uv run python -m mongospecs_bench.cli run LOAD # the storefront under HTTP load, against raw pymongo +``` + +Results are written to `bench/results//`. CPU time and command counts are stable from +run to run; wall-clock numbers on a shared machine vary by 20 to 30 percent, so compare those only +when the difference is large. diff --git a/docs/web/resources.md b/docs/web/resources.md index dd83ba6..2689701 100644 --- a/docs/web/resources.md +++ b/docs/web/resources.md @@ -149,6 +149,14 @@ on patch (a float or Decimal field takes it as that type). A guard listed in mor runs once per request. Private fields appear nowhere in `openapi.json`, and it only publishes the nested definitions that the Read, Create and Patch schemas refer to. +`Policy(private=True)` also applies to the fields of embedded models, at any depth: they are never +in a response, a request that supplies one gets a 422, and they are in no schema. A `PATCH` that +sends an embedded object replaces it whole, so private values inside it are cleared; patch the +other fields of the parent, or set such values in a server-side hook. + +`Api(max_concurrency=N)` answers the request beyond N in flight with 503 and `Retry-After`. See +[Tuning for production](../guide/production.md). + `DELETE` honours `If-Match` like `PATCH`, and the row scope is part of the delete's own filter. `HEAD` and `OPTIONS` are answered by `api.handle` and the pure ASGI plug; the framework plugs leave them to the framework. diff --git a/mkdocs.yaml b/mkdocs.yaml index 2f137a3..89a9134 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -86,6 +86,7 @@ nav: - Relations: guide/relations.md - Hooks and plugins: guide/plugins.md - Running in production: guide/operations.md + - Tuning for production: guide/production.md - Web: - Resources: web/resources.md - Plugs: web/plugs.md diff --git a/packages/mongospecs/tests/test_hot_paths.py b/packages/mongospecs/tests/test_hot_paths.py index 987975a..56c82a9 100644 --- a/packages/mongospecs/tests/test_hot_paths.py +++ b/packages/mongospecs/tests/test_hot_paths.py @@ -206,3 +206,20 @@ def named(query): # an unrelated named scope must not disturb relation loading loaded = await Book.find(Q.id == books[0].id).include("co_writers", per_parent_limit=1).one() assert len(loaded.co_writers) == 2 and sum(r.loaded for r in loaded.co_writers) >= 1 assert asyncio.get_running_loop() is not None + + +async def test_a_full_pool_fails_fast_when_a_wait_limit_is_set(m, make_registry, db_name): + """Backs the production guide: with waitQueueTimeoutMS an exhausted pool raises instead of queueing.""" + from mongospecs import PoolTimeout + + registry = await make_registry(m.User, maxPoolSize=1, waitQueueTimeoutMS=100) + users = registry.manager(m.User) + await users.insert(m.User(name="a")) + slow = asyncio.create_task( + users.collection.find_one({"$where": "function() { sleep(600); return true; }"}) + ) + await asyncio.sleep(0.15) # the only connection is now busy + with pytest.raises(PoolTimeout): + await users.count() + await slow + assert await users.count() == 1 # and it recovers once the connection is free From 0748110b113953e76147e6fa08fcaea3b3271807 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 07:34:35 -0400 Subject: [PATCH 18/26] fix(core): serialise BSON-only values held in untyped fields An ObjectId, Binary, Timestamp, Regex or Int64 inside a field typed Any (or dict[str, Any], list[Any]) made the JSON conversion fail, and the web layer answered 500: on pydantic for any ObjectId, on msgspec and attrs for the BSON types their serialiser does not know. All three backends now give the same JSON for such values, at any depth. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_untyped_values.py | 155 ++++++++++++++++++ packages/mongospecs/src/mongospecs/codec.py | 1 + .../mongospecs/src/mongospecs/jsonable.py | 25 ++- .../mongospecs/src/mongospecs/pydantic.py | 51 +++++- 4 files changed, 227 insertions(+), 5 deletions(-) create mode 100644 packages/mongospecs-web/tests/test_untyped_values.py diff --git a/packages/mongospecs-web/tests/test_untyped_values.py b/packages/mongospecs-web/tests/test_untyped_values.py new file mode 100644 index 0000000..00a7235 --- /dev/null +++ b/packages/mongospecs-web/tests/test_untyped_values.py @@ -0,0 +1,155 @@ +"""Values only BSON has (ObjectId, dates, decimals, binary, UUID ...) inside untyped fields.""" + +import datetime as dt +import json +import typing as t +import uuid + +import bson +import pytest + +from mongospecs import SpecConfig +from mongospecs_web import Api, Create, Get, List, Patch, Resource +from mongospecs_web.policy import view_of + +BACKENDS = ("msgspec", "pydantic", "attrs") +OID = bson.ObjectId("65a1b2c3d4e5f60718293a4b") +VALUES = { + "oid": OID, + "when": dt.datetime(2021, 1, 2, 3, 4, 5, 123000, tzinfo=dt.UTC), + "amount": bson.Decimal128("12.50"), + "blob": b"\xff\x00hi", + "text_bytes": b"plain", + "uid": uuid.UUID("12345678-1234-5678-1234-567812345678"), + "big": bson.Int64(2**40), + "binary": bson.Binary(b"\x01\x02", 128), + "stamp": bson.Timestamp(1700000000, 1), + "pattern": bson.Regex("^a", "i"), + "deep": {"list": [OID, {"again": [dt.datetime(2020, 5, 6, tzinfo=dt.UTC), None, 1.5, "s", True]}]}, +} +EXPECTED = { + "oid": "65a1b2c3d4e5f60718293a4b", + "when": "2021-01-02T03:04:05.123000Z", + "amount": "12.50", + "blob": "/wBoaQ==", + "text_bytes": "cGxhaW4=", + "uid": "12345678-1234-5678-1234-567812345678", + "big": 2**40, + "binary": "AQI=", + "stamp": "2023-11-14T22:13:20Z", + "pattern": "^a", + "deep": {"list": ["65a1b2c3d4e5f60718293a4b", {"again": ["2020-05-06T00:00:00Z", None, 1.5, "s", True]}]}, +} + + +def models(backend): + if backend == "msgspec": + import msgspec + + from mongospecs.msgspec import Spec + + class Base(msgspec.Struct, kw_only=True): + pass + + define = lambda c: c # noqa: E731 + elif backend == "pydantic": + import pydantic + + from mongospecs.pydantic import Spec + + Base = pydantic.BaseModel + define = lambda c: c # noqa: E731 + else: + import attrs + + from mongospecs.attrs import Spec + + Base = object + define = attrs.define(kw_only=True) + + @define + class Part(Base): + label: str = "" + meta: dict[str, t.Any] = {} # noqa: RUF012 + + @define + class Loose(Spec): + __spec__ = SpecConfig(collection="loose") + name: str = "" + extra: dict[str, t.Any] = {} # noqa: RUF012 + things: list[t.Any] = [] # noqa: RUF012 + anything: t.Any = None + part: Part | None = None + parts: list[Part] = [] # noqa: RUF012 + + return Part, Loose + + +@pytest.fixture(params=BACKENDS) +async def world(request, make_registry): + Part, Loose = models(request.param) + await make_registry(Loose) + + def api(raw): + resource = Resource(Loose, name="loose", ops=[List, Get, Create, Patch]) + resource.raw_reads = raw + return Api(resource, prefix="/v1") + + return Loose, api(True), api(False) + + +async def test_bson_only_values_in_untyped_fields_are_served(world): + Loose, fast, slow = world + stored = await Loose.collection().insert_one( + { + "name": "x", + "extra": dict(VALUES), + "things": list(VALUES.values()), + "anything": VALUES["oid"], + "part": {"label": "p", "meta": dict(VALUES)}, + "parts": [{"label": "q", "meta": {"k": VALUES["when"], "o": [VALUES["oid"]]}}], + } + ) + for path in (f"/v1/loose/{stored.inserted_id}", "/v1/loose"): + bodies = [] + for api in (fast, slow): + response = await api.handle("GET", path) + assert response.status == 200, (path, response.body) + bodies.append(response.body) + assert bodies[0] == bodies[1] # served from the stored document or through the model: the same + data = json.loads(bodies[0])["data"] + row = data[0] if isinstance(data, list) else data + assert row["extra"] == EXPECTED + assert row["things"] == list(EXPECTED.values()) + assert row["anything"] == EXPECTED["oid"] + assert row["part"] == {"label": "p", "meta": EXPECTED} + assert row["parts"] == [{"label": "q", "meta": {"k": EXPECTED["when"], "o": [EXPECTED["oid"]]}}] + # and the same through a tracked model outside the web layer + doc = await Loose.get(stored.inserted_id) + assert view_of(Loose).serialize(doc)["extra"] == EXPECTED + + +async def test_json_written_into_untyped_fields_round_trips(world): + """A client can only send JSON types, and they come back as sent.""" + Loose, fast, slow = world + payload = { + "name": "j", + "extra": {"a": [1, 2.5, "s", None, True, {"b": {}}]}, + "things": [[], {}, "x"], + "anything": {"k": 1}, + } + headers = {"content-type": "application/json"} + created = await fast.handle("POST", "/v1/loose", body=json.dumps(payload).encode(), headers=headers) + assert created.status == 201, created.body + row = json.loads(created.body)["data"] + assert {k: row[k] for k in payload} == payload + for api in (fast, slow): + got = json.loads((await api.handle("GET", created.headers["location"])).body)["data"] + assert {k: got[k] for k in payload} == payload + patched = await fast.handle( + "PATCH", + created.headers["location"], + body=json.dumps({"extra": {"z": [None]}}).encode(), + headers=headers, + ) + assert patched.status == 200 and json.loads(patched.body)["data"]["extra"] == {"z": [None]} diff --git a/packages/mongospecs/src/mongospecs/codec.py b/packages/mongospecs/src/mongospecs/codec.py index a82ccd8..75a3d9c 100644 --- a/packages/mongospecs/src/mongospecs/codec.py +++ b/packages/mongospecs/src/mongospecs/codec.py @@ -417,6 +417,7 @@ def clear_cache(self) -> None: self.__dict__.pop("_sp", None) self.__dict__.pop("_abs", None) self.__dict__.pop("_eo", None) + self.__dict__.pop("_ut", None) def _none_mode(cls: type) -> str: diff --git a/packages/mongospecs/src/mongospecs/jsonable.py b/packages/mongospecs/src/mongospecs/jsonable.py index 6eaa31f..24d08fb 100644 --- a/packages/mongospecs/src/mongospecs/jsonable.py +++ b/packages/mongospecs/src/mongospecs/jsonable.py @@ -10,17 +10,37 @@ import uuid from bson import ObjectId +from bson.code import Code +from bson.dbref import DBRef from bson.decimal128 import Decimal128 +from bson.max_key import MaxKey +from bson.min_key import MinKey +from bson.regex import Regex +from bson.timestamp import Timestamp from mongospecs.refs import Ref +_BSON_ONLY = (bytes, bytearray, int, Timestamp, Regex, Code, DBRef, MaxKey, MinKey) +"""Types a serialiser may hand to :func:`json_default` that only BSON has, or that are +subclasses of standard types the serialiser does not recognise (``Binary``, ``Int64``).""" + + __all__ = ["json_default", "jsonable"] def jsonable(value: t.Any) -> t.Any: """Convert a BSON-ready value to plain JSON types.""" - if value is None or isinstance(value, str | bool | int | float): + kind = type(value) + if value is None or kind is str or kind is bool or kind is int or kind is float: + return value + if isinstance(value, bool | str | float): return value + if isinstance(value, int): + return int(value) # bson.Int64 and other int subclasses + if isinstance(value, Timestamp): + return jsonable(value.as_datetime()) + if isinstance(value, Regex): + return value.pattern if isinstance(value, ObjectId | uuid.UUID | decimal.Decimal | Decimal128): return str(value) if isinstance(value, dt.datetime): @@ -47,6 +67,9 @@ def json_default(value: t.Any) -> t.Any: return str(value) if isinstance(value, Ref): return jsonable(value.id) + if isinstance(value, _BSON_ONLY): + # Found in untyped fields: whatever another writer stored there has to be readable. + return jsonable(value) identifier = getattr(value, "id", None) if identifier is not None: # a document assigned where a reference is expected return jsonable(identifier) diff --git a/packages/mongospecs/src/mongospecs/pydantic.py b/packages/mongospecs/src/mongospecs/pydantic.py index 8cbcf32..7674325 100644 --- a/packages/mongospecs/src/mongospecs/pydantic.py +++ b/packages/mongospecs/src/mongospecs/pydantic.py @@ -11,6 +11,7 @@ from mongospecs.codec import _UNION_ORIGINS, Codec, RawField, field_error from mongospecs.errors import ValidationError +from mongospecs.jsonable import jsonable from mongospecs.markers import SpecConfig from mongospecs.model import Document from mongospecs.refs import OID, Ref @@ -38,6 +39,21 @@ def _annotation(info: t.Any) -> t.Any: return info.annotation +def _has_untyped(tp: t.Any, seen: set[type]) -> bool: + while t.get_origin(tp) is t.Annotated: + tp = t.get_args(tp)[0] + if tp is t.Any or tp is object or tp in (dict, list, set, tuple, frozenset): + return True + if isinstance(tp, type) and issubclass(tp, BaseModel) and not issubclass(tp, Document): + if tp in seen: + return False + seen.add(tp) + return any(_has_untyped(f.annotation, seen) for f in tp.model_fields.values()) + if t.get_origin(tp) is Ref: + return False + return any(_has_untyped(arg, seen) for arg in t.get_args(tp)) + + class _Omit: """Which keys of an embedded model are left out while None, and where its own embedded models are.""" @@ -173,13 +189,40 @@ def _json_schema(self, cls: type) -> dict[str, t.Any]: return t.cast("type[BaseModel]", cls).model_json_schema(by_alias=False) def to_json(self, obj: t.Any) -> dict[str, t.Any]: + cls = type(obj) + loose = self._untyped(cls) + options: dict[str, t.Any] = {"exclude": loose} if loose else {} if _HAS_EXCLUDE_COMPUTED: - return t.cast("dict[str, t.Any]", obj.model_dump(mode="json", exclude_computed_fields=True)) - data: dict[str, t.Any] = obj.model_dump(mode="json") - for name in type(obj).model_computed_fields: - data.pop(name, None) + options["exclude_computed_fields"] = True + data: dict[str, t.Any] = obj.model_dump(mode="json", **options) + if not _HAS_EXCLUDE_COMPUTED: + for name in cls.model_computed_fields: + data.pop(name, None) + if loose: + # pydantic cannot serialise the BSON-only values an untyped field may hold (an + # ObjectId raises, bytes are read as UTF-8). Those fields are converted by the same + # rules the other backends use, so all three give the same JSON. + plain = obj.model_dump(mode="python", include=loose) + data = { + name: jsonable(plain[name]) if name in loose else data[name] + for name in cls.model_fields + if name in loose or name in data + } return data + def _untyped(self, cls: type) -> set[str]: + """Fields whose type leaves some value untyped (``Any``, a bare ``dict`` or ``list``), + directly or inside an embedded model.""" + cache: dict[type, set[str]] = self.__dict__.setdefault("_ut", {}) + hit = cache.get(cls) + if hit is None: + hit = cache[cls] = { + name + for name, _native, annotation, _ in self.raw_fields(cls) + if _has_untyped(annotation, set()) + } + return hit + def defaults(self, cls: type) -> dict[str, t.Any]: fields = t.cast("type[BaseModel]", cls).model_fields return { From 933df43eb8698d8f09dbefff31106eb6f89c3e32 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 08:50:15 -0400 Subject: [PATCH 19/26] feat(web)!: PATCH merges into embedded objects A PATCH that gave an object for an embedded model replaced the stored object, writing defaults over everything the client did not send, including private, read-only and immutable values it is not allowed to set. Objects are now merged key by key (RFC 7396) in one atomic update; lists and dicts are still replaced whole. - nested keys are held to the embedded model's own Policy - an embedded object holding protected values cannot be set to null, and a list or dict of such objects cannot be replaced through PATCH - Policy(insert=False) is enforced on nested keys at create - the OpenAPI Patch schema describes embedded objects as partial BREAKING: clients that relied on PATCH replacing an embedded object must now send null for the keys they want cleared. Co-Authored-By: Claude Opus 5.5 --- claudedocs/v2-plan.md | 2 + docs/parity.md | 1 + docs/web/resources.md | 35 ++- .../src/mongospecs_web/backend.py | 82 +++++- .../src/mongospecs_web/merge.py | 225 +++++++++++++++ .../src/mongospecs_web/policy.py | 74 ++++- .../src/mongospecs_web/resource.py | 11 +- .../src/mongospecs_web/testing.py | 53 +++- .../tests/test_patch_embedded.py | 261 ++++++++++++++++++ 9 files changed, 731 insertions(+), 13 deletions(-) create mode 100644 packages/mongospecs-web/src/mongospecs_web/merge.py create mode 100644 packages/mongospecs-web/tests/test_patch_embedded.py diff --git a/claudedocs/v2-plan.md b/claudedocs/v2-plan.md index f966e02..5286399 100644 --- a/claudedocs/v2-plan.md +++ b/claudedocs/v2-plan.md @@ -888,6 +888,8 @@ Recorded per §7.1 rule 4. Each one names what changed and why; none invalidates | A34 | §4.8 | `Policy(private=True)` is enforced on fields of embedded models at any depth: not serialised, a 422 when supplied, absent from schemas | Third review item A. PATCH replaces an embedded object whole, so a client that patches the parent object clears server-set private values inside it; documented, not changed | | A35 | §4.2 | `StoreNone` / `OmitNone` markers and `SpecConfig(none="omit" \| "store")`. Embedded models resolve the setting from their own markers, or from a `__spec__` if they declare one; they do not inherit the parent document's setting. A unique sparse index over a field that stores null is a `ConfigurationError`. No warning is raised for a partial filter on `$type: null` over an omitted field | Requested as the opt-out from A30. The embedded rule keeps one model's stored shape the same wherever it is embedded | | A36 | §4.3 | Cursor pagination handles documents without a value for a sort key (first ascending, last descending) and builds the cursor from stored values, not model defaults | Found while testing raw reads: such documents were skipped or ended the walk. Became common once None was no longer stored (A30) | +| A37 | §4.8 | HTTP `PATCH` uses JSON Merge Patch semantics for embedded objects (per-key dotted `$set`/`$unset` in one atomic update); lists and dicts still replace whole. Nested keys are held to the embedded model's own `Policy`. An embedded object holding protected values cannot be nulled, and a list or dict of such objects cannot be replaced through `PATCH` (422). `Policy(insert=False)` is enforced on nested keys at create. Supersedes the note in A34 | Decision delegated by Jay. Options weighed: (a) document only, rejected because a client could clear private and read-only values by omitting them, which no documentation fixes; (c) per-field opt-in, rejected because the safe behaviour would be off by default; (d) reject any patch of an object with protected values, rejected because such objects would be unpatchable. (b) merge is what RFC 7396 specifies for objects, is atomic, and removes the hazard instead of fencing it; (d) is kept only where merge has no meaning (list items, null on the whole object). Cost: PATCH on an embedded object is now partial, a behaviour change for clients that relied on replacement; a merge into a missing object needs its required fields; model-level cross-field validators of the embedded model do not run on a merge, as they already do not for top-level PATCH | +| A38 | §4.2 | BSON-only values in untyped fields (`Any`) serialise to JSON on all backends: ObjectId and UUID as strings, dates as ISO 8601, Decimal128 as a string, binary as base64, Int64 as a number, Timestamp as a date, Regex as its pattern | Reported: a 500 on pydantic for any ObjectId, and on msgspec/attrs for Binary, Timestamp, Regex and Int64 | New node: diff --git a/docs/parity.md b/docs/parity.md index a16300d..bc750d2 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -66,6 +66,7 @@ was established: **docs** means it was re-read in the Mongoose documentation for | Populated paths replace the id with the document | A `Ref` keeps `.id` and gains `.doc` | One type whether or not it is loaded | | Arrays are tracked per element | A changed list is sent whole | Per-element intent is expressed with atomic operators | | `Model.watch()` errors out when it loses the primary | The change stream resumes across a step-down (the driver's behaviour, verified by a stress scenario) | Consumers should not need their own reconnect loop | +| Setting a nested object overwrites it (`doc.set('child', {...})`; `{merge: true}` merges) | Model code is the same: assigning an embedded object replaces it. An HTTP `PATCH` merges into embedded objects | A client must not clear values it is not allowed to set (private or read-only fields inside the object) | | `syncIndexes()` drops every undeclared index | `apply_indexes()` only drops or recreates with `drop=True` | Dropping an index is not something a deploy should do by accident | | `orFail()` on an update throws when nothing was modified | `Model.update` raises when nothing matched; `find(...).update(...)` returns the matched and modified counts | A no-op update that matched is not an error | | `populate()` always runs a separate query per path | `include()` does the same by default and can use `$lookup` instead (`strategy="lookup"`) | Both give the same result; the faster one depends on fan-out, see the benchmarks | diff --git a/docs/web/resources.md b/docs/web/resources.md index 2689701..4e57bf8 100644 --- a/docs/web/resources.md +++ b/docs/web/resources.md @@ -149,10 +149,37 @@ on patch (a float or Decimal field takes it as that type). A guard listed in mor runs once per request. Private fields appear nowhere in `openapi.json`, and it only publishes the nested definitions that the Read, Create and Patch schemas refer to. -`Policy(private=True)` also applies to the fields of embedded models, at any depth: they are never -in a response, a request that supplies one gets a 422, and they are in no schema. A `PATCH` that -sends an embedded object replaces it whole, so private values inside it are cleared; patch the -other fields of the parent, or set such values in a server-side hook. +`Policy` also applies to the fields of embedded models, at any depth. Private fields are never in +a response and are in no schema; a request that supplies a field it may not write (private, +`insert=False` on create, `update=False` or `Immutable` on patch) gets a 422 naming the dotted path. + +## PATCH and embedded objects + +`PATCH` merges into embedded objects instead of replacing them (JSON Merge Patch, RFC 7396): + +``` +PATCH /people/1 {"address": {"zip": "0150", "note": null, "geo": {"lat": 59.9}}} +``` + +- Each key is written on its own (`address.zip`, `address.geo.lat`), all in one atomic update, with + one revision bump and the usual `If-Match` check. Keys the request does not name are untouched. +- `null` clears a key: it is removed, or stored as `null` where the field is `StoreNone`. A key that + cannot be `None` rejects `null`. +- Lists and dicts are replaced whole, as before. +- If the object does not exist yet, the request must carry all of its required fields; otherwise it + is a 422 that lists them. + +What a client cannot set, it cannot clear either: + +- a nested key it may not write is rejected (422); +- an embedded object that holds such values cannot be set to `null`; +- a list or dict of embedded objects that hold such values cannot be replaced through `PATCH` + (there is no way to merge into a list item). Give such lists their own operation with + `@resource.op`, where your code decides what happens to the server's values. + +The Patch schema in OpenAPI describes embedded objects as partial (`AddrPatch`: no required keys, +only the writable ones). Values stored in untyped fields (`Any`, `dict[str, Any]`) are served as +JSON whatever they are: an ObjectId as its string, dates as ISO 8601, binary as base64. `Api(max_concurrency=N)` answers the request beyond N in flight with 503 and `Retry-After`. See [Tuning for production](../guide/production.md). diff --git a/packages/mongospecs-web/src/mongospecs_web/backend.py b/packages/mongospecs-web/src/mongospecs_web/backend.py index 2b2e4de..dc323de 100644 --- a/packages/mongospecs-web/src/mongospecs_web/backend.py +++ b/packages/mongospecs-web/src/mongospecs_web/backend.py @@ -9,13 +9,25 @@ import typing as t from dataclasses import dataclass, field -from mongospecs import DocumentNotFound, Page, Q, QueryError, StaleDocumentError, meta_of +from mongospecs import DocumentNotFound, Page, Q, QueryError, Revision, StaleDocumentError, meta_of from mongospecs.expr import Condition, Expr, Field __all__ = ["OPERATORS", "Filter", "ListQuery", "MongoBackend", "ResourceBackend"] T = t.TypeVar("T") +_PATH_NOT_VIABLE = 28 +"""Server error for a dotted path that runs through a null or a non-object.""" + + +def _dig(doc: t.Any, path: str) -> t.Any: + for part in path.split("."): + if not isinstance(doc, dict): + return None + doc = doc.get(part) + return doc + + OPERATORS: dict[str, str] = { "eq": "$eq", "ne": "$ne", @@ -151,19 +163,77 @@ async def create(self, data: dict[str, t.Any]) -> T: doc = self.meta.decode(stored, stored=False) return t.cast(T, await self.manager.insert(doc)) - async def patch(self, id: t.Any, data: dict[str, t.Any], *, scope: t.Any, if_rev: int | None = None) -> T: + supports_merge = True + """``patch`` accepts ``merge=``: a :class:`~mongospecs_web.merge.PatchPlan` of writes inside + embedded objects, applied together with ``data`` in one atomic update.""" + + async def patch( + self, + id: t.Any, + data: dict[str, t.Any], + *, + scope: t.Any, + if_rev: int | None = None, + merge: t.Any = None, + ) -> T: query = self._by_id(id, scope) if query is None: raise DocumentNotFound(f"{self.model.__name__} {id!r} does not exist") - if not data: + nested = merge is not None and merge.nested + if not data and not nested: doc = await query.first() if doc is None: raise DocumentNotFound(f"{self.model.__name__} {id!r} does not exist") return t.cast(T, doc) - from mongospecs import trusted + from mongospecs import OperationFailed, Set, Unset, trusted + from mongospecs.expr import and_filters + + base = query._user_filter() + if not nested: + return t.cast(T, await self.manager.update(trusted(base), if_rev=if_rev, **data)) + meta = self.manager.meta + ops: list[t.Any] = [] + if merge.set: + ops.append(Set(merge.set)) + if merge.unset: + ops.append(Unset(*merge.unset)) + # A partial object can only be merged into an object that is already there. + present = {meta.stored_path(path): {"$type": "object"} for path in merge.parents} + try: + return t.cast( + T, await self.manager.update(trusted(and_filters(base, present)), *ops, if_rev=if_rev, **data) + ) + except DocumentNotFound: + if not merge.parents: + raise + raw = await self.manager.find(trusted(base)).raw().first() + if raw is None: + raise + revision = next((f for f in meta.fields.values() if f.has(Revision)), None) + if if_rev is not None and revision is not None and (raw.get(revision.stored) or 0) != if_rev: + raise StaleDocumentError( + f"{self.model.__name__}: revision {if_rev} is no longer current" + ) from None + from mongospecs_web.errors import Unprocessable + + errors = [ + { + "field": client_path, + "message": "does not exist yet; send all of its required fields: " + ", ".join(required), + } + for path, (client_path, required) in merge.parents.items() + if not isinstance(_dig(raw, meta.stored_path(path)), dict) + ] + raise Unprocessable("The request body failed validation.", errors=errors or None) from None + except OperationFailed as exc: + if exc.code != _PATH_NOT_VIABLE: + raise + from mongospecs_web.errors import Unprocessable - target = trusted(query._user_filter()) - return t.cast(T, await self.manager.update(target, if_rev=if_rev, **data)) + raise Unprocessable( + "Part of the request addresses an embedded object that is stored as null. " + "Send that object whole, with all of its required fields." + ) from None async def delete(self, id: t.Any, *, scope: t.Any, if_rev: int | None = None) -> None: manager = self.manager diff --git a/packages/mongospecs-web/src/mongospecs_web/merge.py b/packages/mongospecs-web/src/mongospecs_web/merge.py new file mode 100644 index 0000000..03f6281 --- /dev/null +++ b/packages/mongospecs-web/src/mongospecs_web/merge.py @@ -0,0 +1,225 @@ +"""PATCH on embedded objects: JSON Merge Patch semantics, bounded by field policy. + +A ``PATCH`` body that gives an object for an embedded model does not replace the +stored object. Each key it names is written on its own (``$set`` on a dotted +path, all in one atomic update), ``null`` clears a key, and keys it does not name +are left alone. That is RFC 7396 for objects; lists and dicts are still replaced +whole, as the RFC says. + +The reason is not convenience. Replacing the object would write the model's +defaults over every value the client did not send, including the ones it is not +allowed to send: private fields, fields with ``Policy(update=False)``, immutable +ones. A client must not be able to clear what it cannot set, so: + +* a nested key the client may not write is rejected, with its dotted path; +* an embedded object that holds such values cannot be set to ``null``; +* a list or dict of embedded objects that hold such values cannot be replaced + through ``PATCH`` at all (there is no way to merge into a list item). +""" + +from __future__ import annotations + +import typing as t +from dataclasses import dataclass, field + +from mongospecs import Document, Immutable, Ref, ValidationError +from mongospecs.codec import _UNION_ORIGINS, FieldInfo, _codec_or_none, codec_for, to_bson + +__all__ = ["PatchPlan", "holds_protected", "nested_create_errors", "plan_patch", "single_model"] + + +@dataclass +class PatchPlan: + """A validated PATCH, split into whole-field values and writes inside embedded objects.""" + + fields: dict[str, t.Any] = field(default_factory=dict) + """Top-level fields given a whole new value, by attribute name.""" + set: dict[str, t.Any] = field(default_factory=dict) + """Dotted path (attribute name, then stored keys) -> BSON-ready value.""" + unset: list[str] = field(default_factory=list) + parents: dict[str, tuple[str, tuple[str, ...]]] = field(default_factory=dict) + """Dotted paths that must already hold an object, because the patch does not supply every + required field of it: path -> (path as the client wrote it, required field names).""" + + @property + def nested(self) -> bool: + return bool(self.set or self.unset or self.parents) + + +def single_model(tp: t.Any) -> type | None: + """The embedded model class when ``tp`` is exactly one (optionally ``| None``), else ``None``.""" + while t.get_origin(tp) is t.Annotated: + tp = t.get_args(tp)[0] + if t.get_origin(tp) in _UNION_ORIGINS: + members = [a for a in t.get_args(tp) if a is not type(None)] + return single_model(members[0]) if len(members) == 1 else None + if isinstance(tp, type) and t.get_origin(tp) is None and not issubclass(tp, Document): + return tp if _codec_or_none(tp) is not None else None + return None + + +def _may(info: FieldInfo, action: str, explicit: bool) -> bool: + if action == "create": + return info.policy.insert is True if explicit else info.policy.can_insert + if info.has(Immutable): + return False + return info.policy.update is True if explicit else info.policy.can_update + + +def _fields(cls: type) -> dict[str, tuple[str, FieldInfo]]: + """Every key a client may use for a field of ``cls`` -> (attribute name, field).""" + codec = codec_for(cls) + out: dict[str, tuple[str, FieldInfo]] = {} + for name, native, _, _ in codec.raw_fields(cls): + info = codec.fields(cls)[name] + for key in (name, native, info.stored): + out.setdefault(key, (name, info)) + return out + + +def _protected(cls: type, explicit: bool, seen: set[type]) -> bool: + """True when ``cls`` (or a model embedded in it) has a field a client cannot update.""" + if cls in seen: + return False + seen.add(cls) + for info in codec_for(cls).fields(cls).values(): + if info.policy.private or not _may(info, "update", explicit): + return True + if holds_protected(info.type, explicit, seen): + return True + return False + + +def holds_protected(tp: t.Any, explicit: bool, seen: set[type] | None = None) -> bool: + """True when a value of type ``tp`` can contain something a client cannot update.""" + seen = set() if seen is None else seen + while t.get_origin(tp) is t.Annotated: + tp = t.get_args(tp)[0] + if t.get_origin(tp) is Ref: + return False + if isinstance(tp, type) and t.get_origin(tp) is None: + if issubclass(tp, Document) or _codec_or_none(tp) is None: + return False + return _protected(tp, explicit, seen) + return any(holds_protected(arg, explicit, seen) for arg in t.get_args(tp)) + + +def nested_create_errors( + tp: t.Any, value: t.Any, path: str, explicit: bool, out: list[dict[str, t.Any]] +) -> None: + """On create: keys inside embedded objects that the client may not set.""" + while t.get_origin(tp) is t.Annotated: + tp = t.get_args(tp)[0] + origin = t.get_origin(tp) + if origin is Ref: + return + if origin is None: + cls = single_model(tp) + if cls is None or not isinstance(value, dict): + return + fields = _fields(cls) + for key, item in value.items(): + hit = fields.get(key) + if hit is None or hit[1].policy.private: + continue # unknown keys are the model's to reject; private ones are reported already + if not _may(hit[1], "create", explicit): + out.append({"field": f"{path}.{key}", "message": "cannot be set on create"}) + else: + nested_create_errors(hit[1].type, item, f"{path}.{key}", explicit, out) + return + args = [a for a in t.get_args(tp) if a is not type(None)] + if origin in _UNION_ORIGINS: + for arg in args: + nested_create_errors(arg, value, path, explicit, out) + elif isinstance(value, list) and args: + for index, item in enumerate(value): + nested_create_errors(args[0], item, f"{path}.{index}", explicit, out) + elif isinstance(value, dict) and len(args) == 2: + for key, item in value.items(): + nested_create_errors(args[1], item, f"{path}.{key}", explicit, out) + + +def plan_patch( + fields: t.Mapping[str, FieldInfo], data: t.Mapping[str, t.Any], explicit: bool +) -> tuple[PatchPlan, list[dict[str, t.Any]]]: + """Split a PATCH body (already checked at the top level) into a :class:`PatchPlan`. + + Returns the plan and the list of field errors; the plan is only meaningful when + there are none. + """ + plan = PatchPlan() + errors: list[dict[str, t.Any]] = [] + for key, value in data.items(): + info = fields[key] + cls = single_model(info.type) + if cls is not None and isinstance(value, dict): + _merge(cls, value, key, key, plan, errors, explicit) + elif cls is not None and value is None and _protected(cls, explicit, set()): + errors.append({"field": key, "message": _CANNOT_REMOVE}) + elif cls is None and holds_protected(info.type, explicit): + errors.append({"field": key, "message": _CANNOT_REPLACE}) + else: + plan.fields[key] = value + return plan, errors + + +_CANNOT_REMOVE = "cannot be removed: it holds values only the server sets" +_CANNOT_REPLACE = ( + "cannot be replaced through PATCH: its items hold values only the server sets, " + "and replacing them would clear those" +) + + +def _merge( + cls: type, + value: dict[str, t.Any], + client_path: str, + path: str, + plan: PatchPlan, + errors: list[dict[str, t.Any]], + explicit: bool, +) -> None: + codec = codec_for(cls) + fields = _fields(cls) + defaults = codec.defaults(cls) + absent = codec.absent_when_none(cls) + supplied: set[str] = set() + for key, item in value.items(): + where = f"{client_path}.{key}" + hit = fields.get(key) + if hit is None or hit[1].policy.private: + errors.append({"field": where, "message": "unknown field"}) + continue + name, info = hit + if not _may(info, "update", explicit): + errors.append({"field": where, "message": "cannot be set on update"}) + continue + supplied.add(name) + target = f"{path}.{info.stored}" + inner = single_model(info.type) + if inner is not None and isinstance(item, dict): + _merge(inner, item, where, target, plan, errors, explicit) + elif item is None: + if inner is not None and _protected(inner, explicit, set()): + errors.append({"field": where, "message": _CANNOT_REMOVE}) + elif name in defaults and defaults[name] is None: + if info.stored in absent: + plan.unset.append(target) # not stored while None + else: + plan.set[target] = None # StoreNone: an explicit null + else: + errors.append({"field": where, "message": "may not be null"}) + elif inner is None and holds_protected(info.type, explicit): + errors.append({"field": where, "message": _CANNOT_REPLACE}) + else: + try: + plan.set[target] = to_bson(codec.validate_field(cls, name, item)) + except ValidationError as exc: + message = exc.errors[0]["message"] if exc.errors else str(exc) + errors.append({"field": where, "message": message}) + required = tuple( + name for name, info in codec.fields(cls).items() if info.required and name not in supplied + ) + if required: + # Without these the object cannot be created by this patch, so it has to exist already. + plan.parents[path] = (client_path, required) diff --git a/packages/mongospecs-web/src/mongospecs_web/policy.py b/packages/mongospecs-web/src/mongospecs_web/policy.py index 1d1da57..e17ac99 100644 --- a/packages/mongospecs-web/src/mongospecs_web/policy.py +++ b/packages/mongospecs-web/src/mongospecs_web/policy.py @@ -359,6 +359,10 @@ def check_input(self, data: t.Any, allowed: frozenset[str], action: str) -> dict hidden: list[str] = [] nested_violations(value, plan, key, hidden) errors.extend({"field": path, "message": "unknown field"} for path in hidden) + if action == "create": + from mongospecs_web.merge import nested_create_errors + + nested_create_errors(self.meta.fields[key].type, value, key, self.explicit_writes, errors) if _has_oversized_int(value) and not _takes_any_number(self.meta.fields[key].type): errors.append( {"field": key, "message": f"integers must be between {INT64_MIN} and {INT64_MAX}"} @@ -367,6 +371,66 @@ def check_input(self, data: t.Any, allowed: frozenset[str], action: str) -> dict raise Unprocessable("The request body failed validation.", errors=errors) return data + def plan_patch(self, data: t.Mapping[str, t.Any]) -> t.Any: + """Turn a checked PATCH body into a :class:`~mongospecs_web.merge.PatchPlan`. + + An object given for an embedded model is merged into the stored one, key by key; + see :mod:`mongospecs_web.merge` for the rules and the reason. + """ + from mongospecs_web.errors import Unprocessable + from mongospecs_web.merge import plan_patch + + plan, errors = plan_patch(self.meta.fields, data, self.explicit_writes) + if errors: + raise Unprocessable("The request body failed validation.", errors=errors) + return plan + + def _merge_schema(self, node: t.Any, defs: dict[str, t.Any]) -> t.Any: + """The PATCH form of a property schema: an embedded object becomes a partial object. + + References reached directly or through ``anyOf`` (an optional object) point at + ``Patch``; those inside ``items`` or ``additionalProperties`` are left alone, + because lists and dicts are replaced whole. + """ + if not isinstance(node, dict): + return node + ref = node.get("$ref") + if isinstance(ref, str) and ref.startswith("#/$defs/"): + name = ref.rsplit("/", 1)[1] + return {**node, "$ref": f"#/$defs/{self._patch_def(name, defs)}"} + out = dict(node) + for key in _MERGE_KEYS: + if isinstance(node.get(key), list): + out[key] = [self._merge_schema(item, defs) for item in node[key]] + return out + + def _patch_def(self, name: str, defs: dict[str, t.Any]) -> str: + from mongospecs_web.merge import _fields, _may + + patch_name = f"{name}Patch" + if patch_name in defs or name not in defs: + return patch_name if patch_name in defs else name + classes = { + cls.__name__: cls for info in self.meta.fields.values() for cls in embedded_models(info.type) + } + cls = classes.get(name) + if cls is None: + return name + writable = { + key + for key, (_, info) in _fields(cls).items() + if not info.policy.private and _may(info, "update", self.explicit_writes) + } + full = defs[name] + defs[patch_name] = partial = {k: v for k, v in full.items() if k not in ("required", "title")} + partial["properties"] = { + key: self._merge_schema(value, defs) + for key, value in full.get("properties", {}).items() + if key in writable + } + partial["additionalProperties"] = False + return patch_name + # -- schemas ---------------------------------------------------------------- def _base(self) -> dict[str, t.Any]: @@ -425,12 +489,20 @@ def schema(self, kind: t.Literal["read", "create", "patch"]) -> dict[str, t.Any] schema["required"] = [n for n in required if n in allowed] if not schema.get("required"): schema.pop("required", None) - defs = _reachable(schema, base.get("$defs", {})) + all_defs: dict[str, t.Any] = dict(base.get("$defs", {})) + if kind == "patch": + schema["properties"] = { + name: self._merge_schema(node, all_defs) for name, node in schema["properties"].items() + } + defs = _reachable(schema, all_defs) if defs: schema["$defs"] = defs return schema +_MERGE_KEYS = ("anyOf", "oneOf", "allOf") + + def _child(value: t.Any, include: t.Collection[str]) -> t.Any: if value is None: return None diff --git a/packages/mongospecs-web/src/mongospecs_web/resource.py b/packages/mongospecs-web/src/mongospecs_web/resource.py index 92ff515..6d3dc90 100644 --- a/packages/mongospecs-web/src/mongospecs_web/resource.py +++ b/packages/mongospecs-web/src/mongospecs_web/resource.py @@ -561,8 +561,17 @@ async def handler(ctx: Ctx[T]) -> Response: if body is None: raise Unprocessable("A JSON request body is required.") data = self.view.check_input(body, self.view.updatable, "update") + plan = self.view.plan_patch(data) if_rev = _if_match(ctx.request.headers.get("if-match"), self.view.revision is not None) - doc = await self.backend.patch(ctx.id, dict(data), scope=await ctx.scope(), if_rev=if_rev) + if plan.nested: + # Objects given for embedded models are merged, not replaced. + if not getattr(self.backend, "supports_merge", False): + raise Unprocessable("This resource cannot patch part of an embedded object.") + doc = await t.cast(t.Any, self.backend).patch( + ctx.id, plan.fields, scope=await ctx.scope(), if_rev=if_rev, merge=plan + ) + else: + doc = await self.backend.patch(ctx.id, plan.fields, scope=await ctx.scope(), if_rev=if_rev) ctx.state["result"] = doc headers: dict[str, str] = {} etag = self.view.etag(doc) diff --git a/packages/mongospecs-web/src/mongospecs_web/testing.py b/packages/mongospecs-web/src/mongospecs_web/testing.py index 18c3a5f..17b027a 100644 --- a/packages/mongospecs-web/src/mongospecs_web/testing.py +++ b/packages/mongospecs-web/src/mongospecs_web/testing.py @@ -167,7 +167,17 @@ async def create(self, data: dict[str, t.Any]) -> T: self.docs[doc.id] = doc return t.cast(T, doc) - async def patch(self, id: t.Any, data: dict[str, t.Any], *, scope: t.Any, if_rev: int | None = None) -> T: + supports_merge = True + + async def patch( + self, + id: t.Any, + data: dict[str, t.Any], + *, + scope: t.Any, + if_rev: int | None = None, + merge: t.Any = None, + ) -> T: doc = self._find(id, scope) if doc is None: raise DocumentNotFound("not found") @@ -175,6 +185,26 @@ async def patch(self, id: t.Any, data: dict[str, t.Any], *, scope: t.Any, if_rev if revision is not None and if_rev is not None and getattr(doc, revision) != if_rev: raise StaleDocumentError("stale") values = {k: self.meta.codec.validate_field(self.model, k, v) for k, v in data.items()} + if merge is not None and merge.nested: + # Apply the nested writes to the encoded document and read the touched fields back. + encoded = self.meta.encode(doc) + for path, (client_path, required) in merge.parents.items(): + if not isinstance(_walk(encoded, self.meta.stored_path(path)), dict): + from mongospecs_web.errors import Unprocessable + + message = "does not exist yet; send all of its required fields: " + ", ".join(required) + raise Unprocessable( + "The request body failed validation.", + errors=[{"field": client_path, "message": message}], + ) + for path, value in merge.set.items(): + _put(encoded, self.meta.stored_path(path), value) + for path in merge.unset: + _put(encoded, self.meta.stored_path(path), _REMOVE) + merged = self.meta.decode(encoded) + for path in (*merge.set, *merge.unset): + head = path.partition(".")[0] + values[head] = getattr(merged, head) if values: before = {k: getattr(doc, k) for k in values} for key, value in values.items(): @@ -199,6 +229,27 @@ async def delete(self, id: t.Any, *, scope: t.Any, if_rev: int | None = None) -> del self.docs[t.cast(t.Any, doc).id] +_REMOVE: t.Any = object() + + +def _walk(doc: t.Any, path: str) -> t.Any: + for part in path.split("."): + if not isinstance(doc, dict): + return None + doc = doc.get(part) + return doc + + +def _put(doc: dict[str, t.Any], path: str, value: t.Any) -> None: + *parents, last = path.split(".") + for part in parents: + doc = doc.setdefault(part, {}) + if value is _REMOVE: + doc.pop(last, None) + else: + doc[last] = value + + # -- the contract API ---------------------------------------------------------------- diff --git a/packages/mongospecs-web/tests/test_patch_embedded.py b/packages/mongospecs-web/tests/test_patch_embedded.py new file mode 100644 index 0000000..2678bb7 --- /dev/null +++ b/packages/mongospecs-web/tests/test_patch_embedded.py @@ -0,0 +1,261 @@ +"""PATCH on embedded objects merges; values only the server sets can never be cleared or changed.""" + +import json +import typing as t + +import pytest + +from mongospecs import Immutable, Policy, Revision, SpecConfig, StoreNone +from mongospecs_web import Api, Create, Get, Patch, Resource + +BACKENDS = ("msgspec", "pydantic", "attrs") +JSON = {"content-type": "application/json"} + + +def models(backend): + if backend == "msgspec": + import msgspec + + from mongospecs.msgspec import Spec + + class Base(msgspec.Struct, kw_only=True): + pass + + define = lambda c: c # noqa: E731 + elif backend == "pydantic": + import pydantic + + from mongospecs.pydantic import Spec + + Base = pydantic.BaseModel + define = lambda c: c # noqa: E731 + else: + import attrs + + from mongospecs.attrs import Spec + + Base = object + define = attrs.define(kw_only=True) + + @define + class Geo(Base): + lat: float = 0.0 + surveyor: t.Annotated[str | None, Policy(private=True)] = None + + @define + class Addr(Base): + city: str # required + zip: str | None = None + note: t.Annotated[str | None, StoreNone] = None + vault_code: t.Annotated[str, Policy(private=True)] = "" + verified: t.Annotated[bool, Policy(insert=False, update=False)] = False + code: t.Annotated[str, Immutable] = "" + geo: Geo | None = None + + @define + class Tag(Base): + label: str = "" + weight: int = 0 + + @define + class Person(Spec): + __spec__ = SpecConfig(collection="patch_persons") + name: str = "" + address: Addr | None = None + history: list[Addr] = [] # noqa: RUF012 + tag: Tag | None = None + tags: list[Tag] = [] # noqa: RUF012 + rev: t.Annotated[int, Revision] = 0 + + return Geo, Addr, Tag, Person + + +@pytest.fixture(params=BACKENDS) +async def world(request, make_registry): + Geo, Addr, Tag, Person = models(request.param) + events = [] + await make_registry(Person, on_command=events.append) + api = Api(Resource(Person, name="persons", ops=[Get, Create, Patch]), prefix="/v1") + + async def call(method, path, body=None, headers=None): + response = await api.handle( + method, + path, + body=json.dumps(body).encode() if body is not None else b"", + headers={**JSON, **(headers or {})}, + ) + return response.status, json.loads(response.body) if response.body else None + + async def person(**extra): + doc = await Person( + name="ann", + address=Addr( + city="Oslo", + zip="0150", + note="n", + vault_code="vault", + verified=True, + code="C1", + geo=Geo(lat=1.0, surveyor="sv"), + ), + history=[Addr(city="Old", vault_code="old-vault")], + tag=Tag(label="t", weight=1), + tags=[Tag(label="a"), Tag(label="b")], + **extra, + ).insert() + return doc, f"/v1/persons/{doc.id}" + + async def stored(doc): + return await Person.collection().find_one({"_id": doc.id}) + + return Person, api, call, person, stored, events + + +async def test_patching_an_embedded_object_merges_and_keeps_server_values(world): + Person, api, call, person, stored, events = world + doc, url = await person() + events.clear() + status, body = await call("PATCH", url, {"address": {"zip": "9999"}}) + assert status == 200, body + writes = [e for e in events if e.phase == "started" and e.command_name != "endSessions"] + raw = (await stored(doc))["address"] + assert raw["zip"] == "9999" and raw["city"] == "Oslo" # the rest of the object is untouched + assert raw["vault_code"] == "vault" and raw["verified"] is True and raw["code"] == "C1" + assert raw["geo"] == {"lat": 1.0, "surveyor": "sv"} + assert body["data"]["address"]["zip"] == "9999" and body["data"]["address"]["city"] == "Oslo" + assert "vault_code" not in body["data"]["address"] + # one atomic write with dotted paths, and one revision + assert [e.command_name for e in writes] == ["findAndModify"] + assert writes[0].command["update"]["$set"]["address.zip"] == "9999" + assert not any(key == "address" for key in writes[0].command["update"]["$set"]) # never the whole object + assert body["data"]["rev"] == 1 + # deeper: a nested object inside the embedded one merges too + status, body = await call("PATCH", url, {"name": "anna", "address": {"geo": {"lat": 5}}}) + assert status == 200 and body["data"]["name"] == "anna" + assert (await stored(doc))["address"]["geo"] == {"lat": 5.0, "surveyor": "sv"} + + +async def test_null_inside_a_merge_follows_the_fields_mode(world): + Person, api, call, person, stored, events = world + doc, url = await person() + status, body = await call("PATCH", url, {"address": {"zip": None, "note": None}}) + assert status == 200, body + raw = (await stored(doc))["address"] + assert "zip" not in raw # left out while None + assert "note" in raw and raw["note"] is None # StoreNone: an explicit null + assert body["data"]["address"]["zip"] is None and body["data"]["address"]["note"] is None + status, body = await call("PATCH", url, {"address": {"city": None}}) + assert status == 422 and body["errors"][0]["field"] == "address.city" + assert (await stored(doc))["address"]["city"] == "Oslo" + + +@pytest.mark.parametrize( + ("patch", "field", "message"), + [ + ({"address": {"nope": 1}}, "address.nope", "unknown field"), + ({"address": {"vault_code": "x"}}, "address.vault_code", "unknown field"), + ({"address": {"geo": {"surveyor": "me"}}}, "address.geo.surveyor", "unknown field"), + ({"address": {"verified": False}}, "address.verified", "cannot be set on update"), + ({"address": {"code": "C2"}}, "address.code", "cannot be set on update"), + ({"address": {"geo": {"lat": "north"}}}, "address.geo.lat", None), + ({"address": None}, "address", "cannot be removed"), + ({"address": {"geo": None}}, "address.geo", "cannot be removed"), + ({"history": [{"city": "New"}]}, "history", "cannot be replaced"), + ({"history": []}, "history", "cannot be replaced"), + ], +) +async def test_a_client_can_never_clear_or_change_a_server_value(world, patch, field, message): + Person, api, call, person, stored, events = world + doc, url = await person() + before = await stored(doc) + status, body = await call("PATCH", url, {"name": "changed", **patch}) + assert status == 422, body + errors = {e["field"]: e["message"] for e in body["errors"]} + assert field in errors, errors + if message: + assert message in errors[field] + assert await stored(doc) == before # nothing was written, not even the valid part + + +async def test_objects_without_server_values_can_be_replaced_or_removed(world): + Person, api, call, person, stored, events = world + doc, url = await person() + status, body = await call("PATCH", url, {"tags": [{"label": "only", "weight": 3}], "tag": {"weight": 9}}) + assert status == 200, body + raw = await stored(doc) + assert raw["tags"] == [{"label": "only", "weight": 3}] # a list is replaced whole + assert raw["tag"] == {"label": "t", "weight": 9} # an object is merged + status, body = await call("PATCH", url, {"tag": None, "tags": []}) + assert status == 200 and body["data"]["tag"] is None and body["data"]["tags"] == [] + assert "tag" not in await stored(doc) + + +async def test_merging_into_an_object_that_does_not_exist_yet(world): + Person, api, call, person, stored, events = world + bare = await Person(name="bare").insert() + url = f"/v1/persons/{bare.id}" + status, body = await call("PATCH", url, {"address": {"zip": "1"}}) + assert status == 422, body # city is required and there is nothing to merge into + assert body["errors"][0]["field"] == "address" and "city" in body["errors"][0]["message"] + assert "address" not in await stored(bare) + status, body = await call("PATCH", url, {"address": {"city": "Bergen", "zip": "1"}}) + assert status == 200 and body["data"]["address"]["city"] == "Bergen" + assert (await Person.get(bare.id)).address.zip == "1" + assert (await call("PATCH", "/v1/persons/000000000000000000000000", {"address": {"zip": "1"}}))[0] == 404 + + +async def test_merge_respects_if_match(world): + Person, api, call, person, stored, events = world + doc, url = await person() + assert (await call("PATCH", url, {"address": {"zip": "1"}}, {"if-match": '"7"'}))[0] == 412 + status, body = await call("PATCH", url, {"address": {"zip": "1"}}, {"if-match": '"0"'}) + assert status == 200 and body["data"]["rev"] == 1 + bare = await Person(name="bare").insert() + assert (await call("PATCH", f"/v1/persons/{bare.id}", {"address": {"zip": "1"}}, {"if-match": '"3"'}))[ + 0 + ] == 412 + + +async def test_create_rejects_nested_fields_the_client_may_not_set(world): + Person, api, call, person, stored, events = world + status, body = await call( + "POST", "/v1/persons", {"name": "n", "address": {"city": "x", "verified": True}} + ) + assert ( + status == 422 + and {"field": "address.verified", "message": "cannot be set on create"} in body["errors"] + ) + status, body = await call( + "POST", "/v1/persons", {"name": "n", "history": [{"city": "x", "verified": True}]} + ) + assert status == 422 and body["errors"][0]["field"] == "history.0.verified" + status, body = await call("POST", "/v1/persons", {"name": "n", "address": {"city": "x", "code": "C9"}}) + assert status == 201 and body["data"]["address"]["code"] == "C9" # immutable fields are set on create + assert await Person.count() == 1 + + +async def test_the_patch_schema_describes_a_partial_object(world): + Person, api, call, person, stored, events = world + schemas = api.openapi()["components"]["schemas"] + patch = ( + schemas["PersonPatch"] + if "PersonPatch" in schemas + else next( + v for k, v in schemas.items() if k.endswith("Patch") and "address" in v.get("properties", {}) + ) + ) + text = json.dumps(patch["properties"]["address"]) + assert "AddrPatch" in text + partial = next(v for k, v in schemas.items() if k.endswith("AddrPatch")) + assert "required" not in partial # every key is optional in a merge + assert set(partial["properties"]) == { + "city", + "zip", + "note", + "geo", + } # no private, read-only or immutable keys + assert "GeoPatch" in json.dumps(partial["properties"]["geo"]) + # a list of embedded objects is replaced whole, so its items keep the full schema + assert "AddrPatch" not in json.dumps(patch["properties"].get("history", {})) + full = next(v for k, v in schemas.items() if k.endswith("_Addr")) + assert "city" in full.get("required", []) From ecb42e652774132ee903174c77c887220b23b727 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 10:04:36 -0400 Subject: [PATCH 20/26] feat(web): Put operation and Resource(merge_patch=False) PATCH merging into embedded objects (A37) had no way out for resources that need the embedded model validated whole, need to write over a stored null, or use a backend without supports_merge. - Resource(merge_patch=False): PATCH replaces an embedded object with the one sent. Values the client cannot send go back to their defaults; nested keys the policy closes are still rejected. - Put: PUT /{id} replaces every field the client may update (the sent value, or the default when left out) and leaves the rest as stored, in one atomic update with the If-Match check. An embedded object holding protected values is replaced key by key; one without is replaced whole. It never creates, and is not in the default ops. - OpenAPI gains Put schemas where a resource has the operation. Recorded as A39 in the plan. Co-Authored-By: Claude Opus 5.5 --- claudedocs/v2-plan.md | 1 + docs/parity.md | 2 +- docs/web/resources.md | 46 ++- .../src/mongospecs_web/__init__.py | 2 + .../src/mongospecs_web/merge.py | 115 ++++++- .../src/mongospecs_web/openapi.py | 12 +- .../src/mongospecs_web/policy.py | 90 +++-- .../src/mongospecs_web/resource.py | 80 +++-- .../tests/test_put_and_replace.py | 308 ++++++++++++++++++ 9 files changed, 595 insertions(+), 61 deletions(-) create mode 100644 packages/mongospecs-web/tests/test_put_and_replace.py diff --git a/claudedocs/v2-plan.md b/claudedocs/v2-plan.md index 5286399..c2f9e73 100644 --- a/claudedocs/v2-plan.md +++ b/claudedocs/v2-plan.md @@ -890,6 +890,7 @@ Recorded per §7.1 rule 4. Each one names what changed and why; none invalidates | A36 | §4.3 | Cursor pagination handles documents without a value for a sort key (first ascending, last descending) and builds the cursor from stored values, not model defaults | Found while testing raw reads: such documents were skipped or ended the walk. Became common once None was no longer stored (A30) | | A37 | §4.8 | HTTP `PATCH` uses JSON Merge Patch semantics for embedded objects (per-key dotted `$set`/`$unset` in one atomic update); lists and dicts still replace whole. Nested keys are held to the embedded model's own `Policy`. An embedded object holding protected values cannot be nulled, and a list or dict of such objects cannot be replaced through `PATCH` (422). `Policy(insert=False)` is enforced on nested keys at create. Supersedes the note in A34 | Decision delegated by Jay. Options weighed: (a) document only, rejected because a client could clear private and read-only values by omitting them, which no documentation fixes; (c) per-field opt-in, rejected because the safe behaviour would be off by default; (d) reject any patch of an object with protected values, rejected because such objects would be unpatchable. (b) merge is what RFC 7396 specifies for objects, is atomic, and removes the hazard instead of fencing it; (d) is kept only where merge has no meaning (list items, null on the whole object). Cost: PATCH on an embedded object is now partial, a behaviour change for clients that relied on replacement; a merge into a missing object needs its required fields; model-level cross-field validators of the embedded model do not run on a merge, as they already do not for top-level PATCH | | A38 | §4.2 | BSON-only values in untyped fields (`Any`) serialise to JSON on all backends: ObjectId and UUID as strings, dates as ISO 8601, Decimal128 as a string, binary as base64, Int64 as a number, Timestamp as a date, Regex as its pattern | Reported: a 500 on pydantic for any ObjectId, and on msgspec/attrs for Binary, Timestamp, Regex and Int64 | +| A39 | §4.8 | `Put` operation and `Resource(merge_patch=False)`. `PUT /{id}` replaces every field the client may update (sent value, or the default when left out; a required field must be sent) and leaves the rest as stored, in one atomic update; an embedded object holding protected values is replaced key by key the same way, one without them is replaced whole. It never creates. `Put` is not in the default `ops`. `merge_patch=False` makes `PATCH` replace embedded objects as before A37; nested keys the policy closes are still rejected | Requested by Jay as the opt-out from A37 and its limits (validators of the embedded model see one key at a time on a merge, a stored null needs the whole object, a backend without `supports_merge` gets a 422). The flag is on the resource, not the model: it describes an HTTP verb, model code is unaffected, and embedded models do not inherit settings (A35). Cost of opting out, accepted by whoever sets it: values the client cannot send are reset to defaults on replacement. `Put` is opt-in because guards are keyed by operation, so adding it to the defaults would expose an unguarded write on resources that guard `Patch` | New node: diff --git a/docs/parity.md b/docs/parity.md index bc750d2..c246da6 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -66,7 +66,7 @@ was established: **docs** means it was re-read in the Mongoose documentation for | Populated paths replace the id with the document | A `Ref` keeps `.id` and gains `.doc` | One type whether or not it is loaded | | Arrays are tracked per element | A changed list is sent whole | Per-element intent is expressed with atomic operators | | `Model.watch()` errors out when it loses the primary | The change stream resumes across a step-down (the driver's behaviour, verified by a stress scenario) | Consumers should not need their own reconnect loop | -| Setting a nested object overwrites it (`doc.set('child', {...})`; `{merge: true}` merges) | Model code is the same: assigning an embedded object replaces it. An HTTP `PATCH` merges into embedded objects | A client must not clear values it is not allowed to set (private or read-only fields inside the object) | +| Setting a nested object overwrites it (`doc.set('child', {...})`; `{merge: true}` merges) | Model code is the same: assigning an embedded object replaces it. An HTTP `PATCH` merges into embedded objects; `Resource(merge_patch=False)` makes it overwrite, and `PUT` replaces the document | A client must not clear values it is not allowed to set (private or read-only fields inside the object) | | `syncIndexes()` drops every undeclared index | `apply_indexes()` only drops or recreates with `drop=True` | Dropping an index is not something a deploy should do by accident | | `orFail()` on an update throws when nothing was modified | `Model.update` raises when nothing matched; `find(...).update(...)` returns the matched and modified counts | A no-op update that matched is not an error | | `populate()` always runs a separate query per path | `include()` does the same by default and can use `$lookup` instead (`strategy="lookup"`) | Both give the same result; the faster one depends on fan-out, see the benchmarks | diff --git a/docs/web/resources.md b/docs/web/resources.md index 4e57bf8..d15b72e 100644 --- a/docs/web/resources.md +++ b/docs/web/resources.md @@ -46,7 +46,7 @@ registry = await Registry("mongodb://localhost/app", models=[Task]).open() | Policy | Meaning | |---|---| -| `insert=False` / `update=False` | not settable on create / on patch | +| `insert=False` / `update=False` | not settable on create / on patch and put | | `private=True` | never returned, filtered or sorted; not writable unless `insert`/`update` say so | | `filter=("eq", "gte", ...)` | operators allowed in the query string (`True` for all) | | `sort=True` | usable in `?sort=` | @@ -143,6 +143,7 @@ These are set so that a client cannot make the server do unbounded work. | `MongoBackend(max_time_ms=...)` | 10 s | Time limit for reads made for a request, when the registry and the model set none | | `Api(openapi=...)` | `True` | Serve `/openapi.json`; turn off to keep the description private | | `Resource(explicit_writes=...)` | `False` | When `True`, a field is writable only if its `Policy` says `insert=True` / `update=True` | +| `Resource(merge_patch=...)` | `True` | When `False`, `PATCH` replaces embedded objects instead of merging into them (see below) | An integer outside the 64-bit range MongoDB can store is a 422 with a field error, on create and on patch (a float or Decimal field takes it as that type). A guard listed in more than one place @@ -177,6 +178,49 @@ What a client cannot set, it cannot clear either: (there is no way to merge into a list item). Give such lists their own operation with `@resource.op`, where your code decides what happens to the server's values. +### Turning the merge off + +`Resource(Person, merge_patch=False)` makes `PATCH` replace an embedded object with the one sent, +as it did before the merge was introduced. Top-level fields are still partial. Choose it when: + +- the embedded model has validators that look at more than one field. A merge validates each key + on its own; a replacement validates the object as sent; +- clients should be able to write over an object stored as `null` without knowing that it is; +- the resource has its own backend that does not implement `supports_merge`. + +The cost is the reason the merge exists: every value in the object that the client did not send +goes back to its default, and that includes the values it is not allowed to send (private fields, +`update=False`, `Immutable`). Keep the merge for objects that carry such values. A request still +cannot *set* them: a nested key the policy closes is a 422 either way. The setting belongs to the +resource and not to the model, because it describes what an HTTP verb does; model code is not +affected by it, and two resources over one model can differ. + +## PUT + +`PUT` is not among the default operations; list it to have it: `ops=[List, Get, Create, Patch, Put, +Delete]`. Guards and hooks use the `Put` key, so it is not covered by what you registered for +`Patch`. + +``` +PUT /people/1 {"name": "Ann", "address": {"city": "Oslo"}} +``` + +`PUT /{id}` replaces everything the client may write, in one atomic update, with the same +`If-Match` check and revision bump as `PATCH`: + +- a field in the body takes that value; a writable field left out goes back to its default, and a + required one must be sent (422); +- fields the client may not update keep their stored values, and naming one is a 422, as with + `PATCH`. A body read with `GET` is therefore not accepted back unchanged: leave out `id`, the + revision and the other read-only fields; +- an embedded object with no such values inside is replaced whole and validated whole; +- an embedded object that does hold them is replaced key by key: its writable keys follow the two + rules above and the rest stay. Left out of the body, it is not touched. It cannot be set to + `null`, and a list or dict of such objects cannot be sent; +- the id comes from the URL, and a `PUT` to an id that does not exist is a 404. It never creates. + +OpenAPI gets a `PersonPut` schema (and `AddrPut` for an embedded object with protected values). + The Patch schema in OpenAPI describes embedded objects as partial (`AddrPatch`: no required keys, only the writable ones). Values stored in untyped fields (`Any`, `dict[str, Any]`) are served as JSON whatever they are: an ObjectId as its string, dates as ISO 8601, binary as base64. diff --git a/packages/mongospecs-web/src/mongospecs_web/__init__.py b/packages/mongospecs-web/src/mongospecs_web/__init__.py index ac54db2..d318f92 100644 --- a/packages/mongospecs-web/src/mongospecs_web/__init__.py +++ b/packages/mongospecs-web/src/mongospecs_web/__init__.py @@ -35,6 +35,7 @@ List, Operation, Patch, + Put, Request, Resource, Response, @@ -64,6 +65,7 @@ "Patch", "PayloadTooLarge", "PreconditionFailed", + "Put", "Request", "Resource", "ResourceBackend", diff --git a/packages/mongospecs-web/src/mongospecs_web/merge.py b/packages/mongospecs-web/src/mongospecs_web/merge.py index 03f6281..09f6be0 100644 --- a/packages/mongospecs-web/src/mongospecs_web/merge.py +++ b/packages/mongospecs-web/src/mongospecs_web/merge.py @@ -15,6 +15,12 @@ * an embedded object that holds such values cannot be set to ``null``; * a list or dict of embedded objects that hold such values cannot be replaced through ``PATCH`` at all (there is no way to merge into a list item). + +``PUT`` (:func:`plan_put`) is the same machinery asked a different question: every +field the client may write takes the value sent or goes back to its default, and +what the client cannot write stays as it is stored. ``Resource(merge_patch=False)`` +turns the merge off: ``PATCH`` then replaces embedded objects whole, and the values +a client cannot send are reset to their defaults. """ from __future__ import annotations @@ -25,7 +31,14 @@ from mongospecs import Document, Immutable, Ref, ValidationError from mongospecs.codec import _UNION_ORIGINS, FieldInfo, _codec_or_none, codec_for, to_bson -__all__ = ["PatchPlan", "holds_protected", "nested_create_errors", "plan_patch", "single_model"] +__all__ = [ + "PatchPlan", + "holds_protected", + "nested_write_errors", + "plan_patch", + "plan_put", + "single_model", +] @dataclass @@ -104,10 +117,10 @@ def holds_protected(tp: t.Any, explicit: bool, seen: set[type] | None = None) -> return any(holds_protected(arg, explicit, seen) for arg in t.get_args(tp)) -def nested_create_errors( - tp: t.Any, value: t.Any, path: str, explicit: bool, out: list[dict[str, t.Any]] +def nested_write_errors( + tp: t.Any, value: t.Any, path: str, explicit: bool, out: list[dict[str, t.Any]], action: str = "create" ) -> None: - """On create: keys inside embedded objects that the client may not set.""" + """Keys inside embedded objects, given whole, that the client may not set on ``action``.""" while t.get_origin(tp) is t.Annotated: tp = t.get_args(tp)[0] origin = t.get_origin(tp) @@ -122,33 +135,39 @@ def nested_create_errors( hit = fields.get(key) if hit is None or hit[1].policy.private: continue # unknown keys are the model's to reject; private ones are reported already - if not _may(hit[1], "create", explicit): - out.append({"field": f"{path}.{key}", "message": "cannot be set on create"}) + if not _may(hit[1], action, explicit): + out.append({"field": f"{path}.{key}", "message": f"cannot be set on {action}"}) else: - nested_create_errors(hit[1].type, item, f"{path}.{key}", explicit, out) + nested_write_errors(hit[1].type, item, f"{path}.{key}", explicit, out, action) return args = [a for a in t.get_args(tp) if a is not type(None)] if origin in _UNION_ORIGINS: for arg in args: - nested_create_errors(arg, value, path, explicit, out) + nested_write_errors(arg, value, path, explicit, out, action) elif isinstance(value, list) and args: for index, item in enumerate(value): - nested_create_errors(args[0], item, f"{path}.{index}", explicit, out) + nested_write_errors(args[0], item, f"{path}.{index}", explicit, out, action) elif isinstance(value, dict) and len(args) == 2: for key, item in value.items(): - nested_create_errors(args[1], item, f"{path}.{key}", explicit, out) + nested_write_errors(args[1], item, f"{path}.{key}", explicit, out, action) def plan_patch( - fields: t.Mapping[str, FieldInfo], data: t.Mapping[str, t.Any], explicit: bool + fields: t.Mapping[str, FieldInfo], data: t.Mapping[str, t.Any], explicit: bool, *, merge: bool = True ) -> tuple[PatchPlan, list[dict[str, t.Any]]]: """Split a PATCH body (already checked at the top level) into a :class:`PatchPlan`. Returns the plan and the list of field errors; the plan is only meaningful when - there are none. + there are none. With ``merge=False`` every value replaces the stored one whole; the + only check left is that the body does not name a nested key the client may not write. """ plan = PatchPlan() errors: list[dict[str, t.Any]] = [] + if not merge: + for key, value in data.items(): + nested_write_errors(fields[key].type, value, key, explicit, errors, "update") + plan.fields[key] = value + return plan, errors for key, value in data.items(): info = fields[key] cls = single_model(info.type) @@ -163,10 +182,51 @@ def plan_patch( return plan, errors +def plan_put( + model: type, + fields: t.Mapping[str, FieldInfo], + updatable: t.Collection[str], + data: t.Mapping[str, t.Any], + explicit: bool, +) -> tuple[PatchPlan, list[dict[str, t.Any]]]: + """Turn a PUT body (already checked at the top level) into a :class:`PatchPlan`. + + Every field the client may update is written: with the value sent, or with its default + when the body leaves it out (a required field has to be sent). Fields the client may not + update are not part of the plan, so they keep their stored values. The same holds inside + an embedded object that has such fields; one that has none is replaced whole. + """ + plan = PatchPlan() + errors: list[dict[str, t.Any]] = [] + defaults = codec_for(model).defaults(model) + for key, info in fields.items(): + if key not in updatable: + continue + cls = single_model(info.type) + guarded = holds_protected(info.type, explicit) + if key not in data: + if guarded: + continue # resetting it would clear what only the server sets + if info.required: + errors.append({"field": key, "message": "is required"}) + else: + plan.fields[key] = defaults.get(key) + elif not guarded: + plan.fields[key] = data[key] + elif cls is not None and isinstance(data[key], dict): + _merge(cls, data[key], key, key, plan, errors, explicit, replace=True) + elif cls is not None and data[key] is None: + errors.append({"field": key, "message": _CANNOT_REMOVE}) + elif cls is None: + errors.append({"field": key, "message": _CANNOT_REPLACE}) + else: + plan.fields[key] = data[key] # not an object: the model's own validation rejects it + return plan, errors + + _CANNOT_REMOVE = "cannot be removed: it holds values only the server sets" _CANNOT_REPLACE = ( - "cannot be replaced through PATCH: its items hold values only the server sets, " - "and replacing them would clear those" + "cannot be replaced: its items hold values only the server sets, and replacing them would clear those" ) @@ -178,7 +238,13 @@ def _merge( plan: PatchPlan, errors: list[dict[str, t.Any]], explicit: bool, + replace: bool = False, ) -> None: + """Plan the writes for one embedded object. + + With ``replace`` (PUT) the keys the client may write and did not send go back to their + defaults, and an object nested in this one is only walked into when it has values to keep. + """ codec = codec_for(cls) fields = _fields(cls) defaults = codec.defaults(cls) @@ -197,8 +263,10 @@ def _merge( supplied.add(name) target = f"{path}.{info.stored}" inner = single_model(info.type) + if replace and not holds_protected(info.type, explicit): + inner = None # nothing in it to keep: validated and written whole below if inner is not None and isinstance(item, dict): - _merge(inner, item, where, target, plan, errors, explicit) + _merge(inner, item, where, target, plan, errors, explicit, replace) elif item is None: if inner is not None and _protected(inner, explicit, set()): errors.append({"field": where, "message": _CANNOT_REMOVE}) @@ -217,6 +285,23 @@ def _merge( except ValidationError as exc: message = exc.errors[0]["message"] if exc.errors else str(exc) errors.append({"field": where, "message": message}) + if replace: + for name, info in codec.fields(cls).items(): + if name in supplied or info.policy.private or not _may(info, "update", explicit): + continue + if holds_protected(info.type, explicit): + continue + target = f"{path}.{info.stored}" + if info.required: + errors.append({"field": f"{client_path}.{name}", "message": "is required"}) + supplied.add(name) # reported here, not again as a missing parent + elif defaults.get(name) is None: + if info.stored in absent: + plan.unset.append(target) + else: + plan.set[target] = None + else: + plan.set[target] = to_bson(defaults[name]) required = tuple( name for name, info in codec.fields(cls).items() if info.required and name not in supplied ) diff --git a/packages/mongospecs-web/src/mongospecs_web/openapi.py b/packages/mongospecs-web/src/mongospecs_web/openapi.py index 6f11d78..5756d78 100644 --- a/packages/mongospecs-web/src/mongospecs_web/openapi.py +++ b/packages/mongospecs-web/src/mongospecs_web/openapi.py @@ -35,6 +35,7 @@ "get": ("404",), "create": ("409", "422"), "patch": ("404", "409", "412", "422"), + "put": ("404", "409", "412", "422"), "delete": ("404",), "custom": ("404", "422"), } @@ -65,12 +66,15 @@ def _rewrite_refs(node: t.Any, prefix: str) -> t.Any: def component_schemas(resource: Resource[t.Any]) -> dict[str, t.Any]: - """Read/Create/Patch schemas of a resource, plus the nested definitions they refer to.""" + """Read/Create/Patch schemas of a resource (and Put, when it has that operation), plus + the nested definitions they refer to.""" schemas: dict[str, t.Any] = {} prefix = resource.model.__name__ + "_" - kinds: tuple[t.Literal["read", "create", "patch"], ...] = ("read", "create", "patch") + kinds: list[t.Literal["read", "create", "patch", "put"]] = ["read", "create", "patch"] + if any(op.kind == "put" for op in resource.operations): + kinds.append("put") for kind in kinds: - schema = copy.deepcopy(resource.view.schema(kind)) + schema = copy.deepcopy(resource.view.schema(kind, merge=resource.merge_patch)) for name, definition in schema.pop("$defs", {}).items(): schemas[prefix + name] = _rewrite_refs(definition, prefix) schemas[resource.schema_name(kind)] = _rewrite_refs(schema, prefix) @@ -89,7 +93,7 @@ def operation_object(resource: Resource[t.Any], op: Operation, *, secured: bool) for name in op.path_parameters ] parameters.extend(op.query_parameters) - if op.kind == "patch" and resource.view.revision: + if op.kind in ("patch", "put") and resource.view.revision: parameters.append( { "name": "If-Match", diff --git a/packages/mongospecs-web/src/mongospecs_web/policy.py b/packages/mongospecs-web/src/mongospecs_web/policy.py index e17ac99..8338833 100644 --- a/packages/mongospecs-web/src/mongospecs_web/policy.py +++ b/packages/mongospecs-web/src/mongospecs_web/policy.py @@ -360,9 +360,9 @@ def check_input(self, data: t.Any, allowed: frozenset[str], action: str) -> dict nested_violations(value, plan, key, hidden) errors.extend({"field": path, "message": "unknown field"} for path in hidden) if action == "create": - from mongospecs_web.merge import nested_create_errors + from mongospecs_web.merge import nested_write_errors - nested_create_errors(self.meta.fields[key].type, value, key, self.explicit_writes, errors) + nested_write_errors(self.meta.fields[key].type, value, key, self.explicit_writes, errors) if _has_oversized_int(value) and not _takes_any_number(self.meta.fields[key].type): errors.append( {"field": key, "message": f"integers must be between {INT64_MIN} and {INT64_MAX}"} @@ -371,43 +371,58 @@ def check_input(self, data: t.Any, allowed: frozenset[str], action: str) -> dict raise Unprocessable("The request body failed validation.", errors=errors) return data - def plan_patch(self, data: t.Mapping[str, t.Any]) -> t.Any: + def plan_patch(self, data: t.Mapping[str, t.Any], *, merge: bool = True) -> t.Any: """Turn a checked PATCH body into a :class:`~mongospecs_web.merge.PatchPlan`. An object given for an embedded model is merged into the stored one, key by key; - see :mod:`mongospecs_web.merge` for the rules and the reason. + see :mod:`mongospecs_web.merge` for the rules and the reason. With ``merge=False`` + it replaces the stored object instead. """ from mongospecs_web.errors import Unprocessable from mongospecs_web.merge import plan_patch - plan, errors = plan_patch(self.meta.fields, data, self.explicit_writes) + plan, errors = plan_patch(self.meta.fields, data, self.explicit_writes, merge=merge) if errors: raise Unprocessable("The request body failed validation.", errors=errors) return plan - def _merge_schema(self, node: t.Any, defs: dict[str, t.Any]) -> t.Any: + def plan_put(self, data: t.Mapping[str, t.Any]) -> t.Any: + """Turn a checked PUT body into a :class:`~mongospecs_web.merge.PatchPlan` that + replaces everything the client may write and nothing else.""" + from mongospecs_web.errors import Unprocessable + from mongospecs_web.merge import plan_put + + plan, errors = plan_put(self.model, self.meta.fields, self.updatable, data, self.explicit_writes) + if errors: + raise Unprocessable("The request body failed validation.", errors=errors) + return plan + + def _merge_schema(self, node: t.Any, defs: dict[str, t.Any], suffix: str = "Patch") -> t.Any: """The PATCH form of a property schema: an embedded object becomes a partial object. References reached directly or through ``anyOf`` (an optional object) point at ``Patch``; those inside ``items`` or ``additionalProperties`` are left alone, - because lists and dicts are replaced whole. + because lists and dicts are replaced whole. With ``suffix="Put"`` it is the PUT form. """ if not isinstance(node, dict): return node ref = node.get("$ref") if isinstance(ref, str) and ref.startswith("#/$defs/"): name = ref.rsplit("/", 1)[1] - return {**node, "$ref": f"#/$defs/{self._patch_def(name, defs)}"} + return {**node, "$ref": f"#/$defs/{self._patch_def(name, defs, suffix)}"} out = dict(node) for key in _MERGE_KEYS: if isinstance(node.get(key), list): - out[key] = [self._merge_schema(item, defs) for item in node[key]] + out[key] = [self._merge_schema(item, defs, suffix) for item in node[key]] return out - def _patch_def(self, name: str, defs: dict[str, t.Any]) -> str: - from mongospecs_web.merge import _fields, _may + def _patch_def(self, name: str, defs: dict[str, t.Any], suffix: str = "Patch") -> str: + """Define ``Patch`` (every writable key, none required) or ``Put`` (the + writable keys of an object that also holds values only the server sets; the required + ones stay required). An object without such values has no Put form: it is sent whole.""" + from mongospecs_web.merge import _fields, _may, _protected, holds_protected, single_model - patch_name = f"{name}Patch" + patch_name = f"{name}{suffix}" if patch_name in defs or name not in defs: return patch_name if patch_name in defs else name classes = { @@ -416,18 +431,30 @@ def _patch_def(self, name: str, defs: dict[str, t.Any]) -> str: cls = classes.get(name) if cls is None: return name - writable = { - key - for key, (_, info) in _fields(cls).items() - if not info.policy.private and _may(info, "update", self.explicit_writes) - } + put = suffix == "Put" + if put and not _protected(cls, self.explicit_writes, set()): + return name + writable: set[str] = set() + kept: set[str] = set() # PUT: left as stored when the request does not send them + for key, (_, info) in _fields(cls).items(): + if info.policy.private or not _may(info, "update", self.explicit_writes): + continue + guarded = put and holds_protected(info.type, self.explicit_writes) + if guarded and single_model(info.type) is None: + continue # a list or dict of such objects cannot be replaced at all + writable.add(key) + if guarded: + kept.add(key) full = defs[name] defs[patch_name] = partial = {k: v for k, v in full.items() if k not in ("required", "title")} partial["properties"] = { - key: self._merge_schema(value, defs) + key: self._merge_schema(value, defs, suffix) for key, value in full.get("properties", {}).items() if key in writable } + required = [k for k in full.get("required", []) if k in writable and k not in kept] + if put and required: + partial["required"] = required partial["additionalProperties"] = False return patch_name @@ -463,8 +490,14 @@ def _base(self) -> dict[str, t.Any]: base = {**base, "$defs": defs} return base - def schema(self, kind: t.Literal["read", "create", "patch"]) -> dict[str, t.Any]: - """JSON Schema of one view. ``$defs`` of nested models are kept alongside.""" + def schema( + self, kind: t.Literal["read", "create", "patch", "put"], *, merge: bool = True + ) -> dict[str, t.Any]: + """JSON Schema of one view. ``$defs`` of nested models are kept alongside. + + ``merge=False`` gives the Patch view of a resource whose ``PATCH`` replaces embedded + objects instead of merging into them. + """ base = self._base() props: dict[str, t.Any] = base.get("properties", {}) required: list[str] = list(base.get("required", [])) @@ -490,10 +523,25 @@ def schema(self, kind: t.Literal["read", "create", "patch"]) -> dict[str, t.Any] if not schema.get("required"): schema.pop("required", None) all_defs: dict[str, t.Any] = dict(base.get("$defs", {})) - if kind == "patch": + if kind == "patch" and merge: schema["properties"] = { name: self._merge_schema(node, all_defs) for name, node in schema["properties"].items() } + elif kind == "put": + from mongospecs_web.merge import holds_protected, single_model + + # What holds values only the server sets is optional (left as stored when not + # sent); a list or dict of such objects cannot be sent at all. + types = {name: self.meta.fields[name].type for name in schema["properties"]} + kept = {name for name, tp in types.items() if holds_protected(tp, self.explicit_writes)} + schema["properties"] = { + name: self._merge_schema(node, all_defs, "Put") if name in kept else node + for name, node in schema["properties"].items() + if name not in kept or single_model(types[name]) is not None + } + schema["required"] = [n for n in required if n in allowed and n not in kept] + if not schema["required"]: + del schema["required"] defs = _reachable(schema, all_defs) if defs: schema["$defs"] = defs diff --git a/packages/mongospecs-web/src/mongospecs_web/resource.py b/packages/mongospecs-web/src/mongospecs_web/resource.py index 6d3dc90..da8438c 100644 --- a/packages/mongospecs-web/src/mongospecs_web/resource.py +++ b/packages/mongospecs-web/src/mongospecs_web/resource.py @@ -29,7 +29,19 @@ from mongospecs_web.filters import Cursor, Pagination, parse_includes, parse_list_query from mongospecs_web.policy import View, view_of -__all__ = ["Create", "Ctx", "Delete", "Get", "List", "Operation", "Patch", "Request", "Resource", "Response"] +__all__ = [ + "Create", + "Ctx", + "Delete", + "Get", + "List", + "Operation", + "Patch", + "Put", + "Request", + "Resource", + "Response", +] T = t.TypeVar("T") F = t.TypeVar("F", bound=t.Callable[..., t.Any]) @@ -41,6 +53,7 @@ Get = "get" Create = "create" Patch = "patch" +Put = "put" Delete = "delete" JSON = "application/json" @@ -212,12 +225,17 @@ def __init__( tags: t.Sequence[str] | None = None, server_fields: t.Callable[[Ctx[T]], t.Any] | None = None, explicit_writes: bool = False, + merge_patch: bool = True, ) -> None: self.model = model self.view: View = view_of(model, explicit_writes=explicit_writes) """With ``explicit_writes=True`` a field is writable only if its ``Policy`` says so (``Policy(insert=True)``, ``Policy(update=True)``); by default ordinary fields are writable unless marked otherwise.""" + self.merge_patch = merge_patch + """``PATCH`` merges an object given for an embedded model into the stored one. With + ``merge_patch=False`` it replaces the stored object: the embedded model is validated + whole, and every value in it that the client cannot send goes back to its default.""" self.name = name or self.view.meta.collection self.backend: ResourceBackend[T] = backend if backend is not None else MongoBackend(model) self.scope_fn = scope @@ -252,12 +270,13 @@ def __init__( Get: self._get_op, Create: self._create_op, Patch: self._patch_op, + Put: self._put_op, Delete: self._delete_op, } for kind in ops: if kind not in builders: raise ValueError( - f"unknown operation {kind!r}; use List, Get, Create, Patch, Delete or @resource.op" + f"unknown operation {kind!r}; use List, Get, Create, Patch, Put, Delete or @resource.op" ) self.operations.append(builders[kind]()) @@ -561,23 +580,7 @@ async def handler(ctx: Ctx[T]) -> Response: if body is None: raise Unprocessable("A JSON request body is required.") data = self.view.check_input(body, self.view.updatable, "update") - plan = self.view.plan_patch(data) - if_rev = _if_match(ctx.request.headers.get("if-match"), self.view.revision is not None) - if plan.nested: - # Objects given for embedded models are merged, not replaced. - if not getattr(self.backend, "supports_merge", False): - raise Unprocessable("This resource cannot patch part of an embedded object.") - doc = await t.cast(t.Any, self.backend).patch( - ctx.id, plan.fields, scope=await ctx.scope(), if_rev=if_rev, merge=plan - ) - else: - doc = await self.backend.patch(ctx.id, plan.fields, scope=await ctx.scope(), if_rev=if_rev) - ctx.state["result"] = doc - headers: dict[str, str] = {} - etag = self.view.etag(doc) - if etag: - headers["etag"] = etag - return Response.json({"data": self.view.serialize(doc)}, 200, headers) + return await self._write(ctx, self.view.plan_patch(data, merge=self.merge_patch)) return Operation( Patch, @@ -591,6 +594,45 @@ async def handler(ctx: Ctx[T]) -> Response: self._one(), ) + def _put_op(self) -> Operation: + async def handler(ctx: Ctx[T]) -> Response: + body = ctx.body() + if body is None: + raise Unprocessable("A JSON request body is required.") + data = self.view.check_input(body, self.view.updatable, "update") + return await self._write(ctx, self.view.plan_put(data)) + + return Operation( + Put, + "PUT", + "/{id}", + handler, + Put, + f"Replace one of {self.name}", + 200, + self._ref("put"), + self._one(), + ) + + async def _write(self, ctx: Ctx[T], plan: t.Any) -> Response: + """Apply a planned PATCH or PUT in one atomic update and return the document.""" + if_rev = _if_match(ctx.request.headers.get("if-match"), self.view.revision is not None) + if plan.nested: + # Writes inside an embedded object, leaving the rest of it as stored. + if not getattr(self.backend, "supports_merge", False): + raise Unprocessable("This resource cannot write part of an embedded object.") + doc = await t.cast(t.Any, self.backend).patch( + ctx.id, plan.fields, scope=await ctx.scope(), if_rev=if_rev, merge=plan + ) + else: + doc = await self.backend.patch(ctx.id, plan.fields, scope=await ctx.scope(), if_rev=if_rev) + ctx.state["result"] = doc + headers: dict[str, str] = {} + etag = self.view.etag(doc) + if etag: + headers["etag"] = etag + return Response.json({"data": self.view.serialize(doc)}, 200, headers) + def _delete_op(self) -> Operation: async def handler(ctx: Ctx[T]) -> None: if_rev = _if_match(ctx.request.headers.get("if-match"), self.view.revision is not None) diff --git a/packages/mongospecs-web/tests/test_put_and_replace.py b/packages/mongospecs-web/tests/test_put_and_replace.py new file mode 100644 index 0000000..0905b31 --- /dev/null +++ b/packages/mongospecs-web/tests/test_put_and_replace.py @@ -0,0 +1,308 @@ +"""PUT replaces what the client may write; ``merge_patch=False`` makes PATCH replace embedded objects.""" + +import json +import typing as t + +import pytest + +from mongospecs import Immutable, Policy, Revision, SpecConfig, StoreNone +from mongospecs_web import Api, Create, Get, Patch, Put, Resource + +BACKENDS = ("msgspec", "pydantic", "attrs") +JSON = {"content-type": "application/json"} + + +def models(backend): + if backend == "msgspec": + import msgspec + + from mongospecs.msgspec import Spec + + class Base(msgspec.Struct, kw_only=True): + pass + + define = lambda c: c # noqa: E731 + elif backend == "pydantic": + import pydantic + + from mongospecs.pydantic import Spec + + Base = pydantic.BaseModel + define = lambda c: c # noqa: E731 + else: + import attrs + + from mongospecs.attrs import Spec + + Base = object + define = attrs.define(kw_only=True) + + @define + class Geo(Base): + lat: float = 0.0 + surveyor: t.Annotated[str | None, Policy(private=True)] = None + + @define + class Addr(Base): + city: str # required + zip: str | None = None + note: t.Annotated[str | None, StoreNone] = None + vault_code: t.Annotated[str, Policy(private=True)] = "" + verified: t.Annotated[bool, Policy(insert=False, update=False)] = False + code: t.Annotated[str, Immutable] = "" + geo: Geo | None = None + + @define + class Tag(Base): + label: str = "" + weight: int = 0 + + @define + class Person(Spec): + __spec__ = SpecConfig(collection="put_persons") + name: str # required + nick: str | None = None + score: int = 0 + owner: t.Annotated[str, Policy(update=False)] = "" + secret: t.Annotated[str | None, Policy(private=True)] = None + address: Addr | None = None + history: list[Addr] = [] # noqa: RUF012 + tag: Tag | None = None + tags: list[Tag] = [] # noqa: RUF012 + rev: t.Annotated[int, Revision] = 0 + + return Geo, Addr, Tag, Person + + +@pytest.fixture(params=BACKENDS) +async def world(request, make_registry): + Geo, Addr, Tag, Person = models(request.param) + events = [] + await make_registry(Person, on_command=events.append) + api = Api( + Resource(Person, name="persons", ops=[Get, Create, Patch, Put]), + Resource(Person, name="replaced", ops=[Get, Patch], merge_patch=False), + prefix="/v1", + ) + + async def call(method, path, body=None, headers=None): + response = await api.handle( + method, + path, + body=json.dumps(body).encode() if body is not None else b"", + headers={**JSON, **(headers or {})}, + ) + return response.status, json.loads(response.body) if response.body else None + + async def person(): + doc = await Person( + name="ann", + nick="annie", + score=7, + owner="ann-id", + secret="s3", + address=Addr( + city="Oslo", + zip="0150", + note="n", + vault_code="vault", + verified=True, + code="C1", + geo=Geo(lat=1.0, surveyor="sv"), + ), + history=[Addr(city="Old", vault_code="old-vault")], + tag=Tag(label="t", weight=1), + tags=[Tag(label="a"), Tag(label="b")], + ).insert() + return doc, str(doc.id) + + async def stored(doc): + return await Person.collection().find_one({"_id": doc.id}) + + return Person, api, call, person, stored, events + + +async def test_put_replaces_what_the_client_may_write_and_keeps_the_rest(world): + Person, api, call, person, stored, events = world + doc, id = await person() + before = await stored(doc) + events.clear() + status, body = await call( + "PUT", f"/v1/persons/{id}", {"name": "bob", "address": {"city": "Bergen"}, "tag": {"label": "x"}} + ) + assert status == 200, body + writes = [e for e in events if e.phase == "started" and e.command_name != "endSessions"] + assert [e.command_name for e in writes] == ["findAndModify"] # one atomic write + raw = await stored(doc) + # sent: replaced. Left out: back to the default. + assert raw["name"] == "bob" and raw["score"] == 0 and raw.get("nick") is None + assert raw["tag"] == {"label": "x", "weight": 0} # nothing to keep in it: replaced whole + assert raw.get("tags", []) == [] + # what the client cannot write is untouched + assert raw["owner"] == "ann-id" and raw["secret"] == "s3" + assert raw["history"] == before["history"] + address = raw["address"] + assert address["city"] == "Bergen" and "zip" not in address and address["note"] is None + assert address["vault_code"] == "vault" and address["verified"] is True and address["code"] == "C1" + assert address["geo"] == {"lat": 1.0, "surveyor": "sv"} # not sent, holds a server value: kept + assert body["data"]["rev"] == 1 and body["data"]["name"] == "bob" + assert "secret" not in body["data"] and "vault_code" not in body["data"]["address"] + # deeper: an object sent inside the embedded one is replaced the same way + status, body = await call( + "PUT", f"/v1/persons/{id}", {"name": "bob", "address": {"city": "Bergen", "zip": "5", "geo": {}}} + ) + assert status == 200, body + address = (await stored(doc))["address"] + assert address["zip"] == "5" and address["geo"] == {"lat": 0.0, "surveyor": "sv"} + assert (await stored(doc)).get("tag") is None # left out this time + assert (await Person.get(doc.id)).address.verified is True + + +@pytest.mark.parametrize( + ("body", "field", "message"), + [ + ({}, "name", "is required"), + ({"name": "b", "address": {"zip": "1"}}, "address.city", "is required"), + ({"name": "b", "owner": "mallory"}, "owner", "cannot be set on update"), + ({"name": "b", "secret": "x"}, "secret", "unknown field"), + ({"name": "b", "address": {"city": "c", "verified": False}}, "address.verified", "cannot be set"), + ({"name": "b", "address": {"city": "c", "vault_code": "x"}}, "address.vault_code", "unknown field"), + ({"name": "b", "address": None}, "address", "cannot be removed"), + ({"name": "b", "address": {"city": "c", "geo": None}}, "address.geo", "cannot be removed"), + ({"name": "b", "history": []}, "history", "cannot be replaced"), + ({"name": "b", "score": "many"}, "score", None), + ], +) +async def test_put_rejects_what_it_cannot_do_and_writes_nothing(world, body, field, message): + Person, api, call, person, stored, events = world + doc, id = await person() + before = await stored(doc) + status, problem = await call("PUT", f"/v1/persons/{id}", body) + assert status == 422, problem + errors = {e["field"]: e["message"] for e in problem["errors"]} + assert field in errors, errors + if message: + assert message in errors[field] + assert await stored(doc) == before + + +async def test_put_conditions_and_routing(world): + Person, api, call, person, stored, events = world + doc, id = await person() + assert (await call("PUT", f"/v1/persons/{id}", {"name": "b"}, {"if-match": '"7"'}))[0] == 412 + status, body = await call("PUT", f"/v1/persons/{id}", {"name": "b"}, {"if-match": '"0"'}) + assert status == 200 and body["data"]["rev"] == 1 + assert (await stored(doc))["address"]["city"] == "Oslo" # not sent, holds server values: kept + assert (await call("PUT", "/v1/persons/000000000000000000000000", {"name": "b"}))[0] == 404 + assert await Person.count() == 1 # PUT never creates + assert (await call("PUT", f"/v1/replaced/{id}", {"name": "b"}))[0] == 405 # only where ops= lists it + + +async def test_put_into_an_embedded_object_that_does_not_exist_yet(world): + Person, api, call, person, stored, events = world + bare = await Person(name="bare").insert() + status, body = await call("PUT", f"/v1/persons/{bare.id}", {"name": "bare", "address": {"city": "Rome"}}) + assert status == 200, body + assert body["data"]["address"]["city"] == "Rome" and body["data"]["address"]["verified"] is False + + +async def test_patch_replaces_embedded_objects_when_merging_is_turned_off(world): + Person, api, call, person, stored, events = world + doc, id = await person() + url = f"/v1/replaced/{id}" + # a partial object is not enough: the embedded model is validated whole + status, body = await call("PATCH", url, {"address": {"zip": "1"}}) + assert status == 422, body + assert (await stored(doc))["address"]["zip"] == "0150" + status, body = await call("PATCH", url, {"address": {"city": "Bergen"}}) + assert status == 200, body + raw = await stored(doc) + # replaced whole: what the client did not (and could not) send is back at its default + assert raw["address"]["city"] == "Bergen" and "zip" not in raw["address"] + assert raw["address"].get("vault_code", "") == "" and raw["address"].get("verified", False) is False + assert raw["name"] == "ann" and raw["score"] == 7 # other fields are still left alone + # it still may not set what the policy closes + for patch, field in ( + ({"address": {"city": "c", "verified": True}}, "address.verified"), + ({"address": {"city": "c", "code": "C2"}}, "address.code"), + ({"address": {"city": "c", "vault_code": "x"}}, "address.vault_code"), + ({"history": [{"city": "c", "verified": True}]}, "history.0.verified"), + ): + status, body = await call("PATCH", url, patch) + assert status == 422 and field in [e["field"] for e in body["errors"]], body + # lists of such objects and null are accepted again + status, body = await call("PATCH", url, {"history": [{"city": "New"}], "address": None}) + assert status == 200, body + raw = await stored(doc) + assert raw.get("address") is None and [h["city"] for h in raw["history"]] == ["New"] + # a stored null is no obstacle to a whole object + await Person.collection().update_one({"_id": doc.id}, {"$set": {"address": None}}) + status, body = await call("PATCH", url, {"address": {"city": "Rome"}}) + assert status == 200 and body["data"]["address"]["city"] == "Rome" + # the same request on the merging resource merges + status, body = await call("PATCH", f"/v1/persons/{id}", {"address": {"zip": "9"}}) + assert status == 200 and body["data"]["address"] == { + **body["data"]["address"], + "city": "Rome", + "zip": "9", + } + + +async def test_replacing_runs_the_embedded_models_own_validators(make_registry): + import pydantic + + from mongospecs.pydantic import Spec + + class Span(pydantic.BaseModel): + low: int = 0 + high: int = 10 + + @pydantic.model_validator(mode="after") + def ordered(self): + if self.low > self.high: + raise ValueError("low must not exceed high") + return self + + class Gauge(Spec): + __spec__ = SpecConfig(collection="put_gauges") + span: Span = Span() + + await make_registry(Gauge) + api = Api(Resource(Gauge, name="replaced", ops=[Patch, Put], merge_patch=False)) + doc = await Gauge(span=Span(low=0, high=100)).insert() + + async def send(method, name, body): + return (await api.handle(method, f"/{name}/{doc.id}", body=json.dumps(body).encode())).status + + # the object as sent is what gets validated: high is left out, so it is the default 10 + assert await send("PATCH", "replaced", {"span": {"low": 50}}) == 422 + assert await send("PUT", "replaced", {"span": {"low": 50}}) == 422 + assert (await Gauge.get(doc.id)).span == Span(low=0, high=100) + assert await send("PATCH", "replaced", {"span": {"low": 50, "high": 60}}) == 200 + assert (await Gauge.get(doc.id)).span == Span(low=50, high=60) + + +async def test_openapi_describes_put_and_the_replacing_patch(world): + Person, api, call, person, stored, events = world + spec = api.openapi() + schemas = spec["components"]["schemas"] + put = spec["paths"]["/v1/persons/{id}"]["put"] + assert put["requestBody"]["content"]["application/json"]["schema"] == { + "$ref": "#/components/schemas/PersonPut" + } + assert "If-Match" in [p["name"] for p in put["parameters"]] + assert {"404", "412", "422"} <= set(put["responses"]) + body = schemas["PersonPut"] + assert body["required"] == ["name"] and body["additionalProperties"] is False + assert set(body["properties"]) == {"name", "nick", "score", "address", "tag", "tags"} + assert "AddrPut" in json.dumps(body["properties"]["address"]) + addr = next(v for k, v in schemas.items() if k.endswith("AddrPut")) + assert addr["required"] == ["city"] and set(addr["properties"]) == {"city", "zip", "note", "geo"} + assert "GeoPut" in json.dumps(addr["properties"]["geo"]) + assert "TagPut" not in json.dumps(schemas) # nothing to keep in a Tag: it is sent whole + assert "put" not in spec["paths"]["/v1/replaced/{id}"] + # one schema name per model, so two resources over one model share it; ask the view directly + replacing = api.resources[1].view.schema("patch", merge=False) + assert "AddrPatch" not in json.dumps(replacing) and "Addr" in json.dumps( + replacing["properties"]["address"] + ) From 7a4012b4228ce96365f8458794528b0baecddf4f Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 10:25:40 -0400 Subject: [PATCH 21/26] fix: nine defects found in the third review of v2 Each was reproduced against the replica set before it was fixed; the tests are in test_review_round3.py of both packages. Core - A single-field update on a pydantic model failed a model-level validator that only saw the other fields' defaults. The field's own validators still run; the cross-field verdict is no longer taken. - insert_many and bulk_save run document hooks, as insert and save do. - page() starts its count after its arguments are validated, so a bad cursor no longer leaves an un-awaited coroutine. Web - Request bodies are strict JSON: NaN, Infinity and over-deep nesting are a 400 (NaN was stored; deep nesting was a logged 500). - If-Match on a model without a Revision is a 412, not an unconditional write. If-None-Match accepts lists, weak tags and *. - The ASGI plug drops a request whose client disconnected mid-body. - Api(openapi=False) is honoured by the FastAPI plug. - Guards and hooks must name an operation the resource has. - The concurrency limit is applied before authentication. - A filter or sort key given twice is a 400. Recorded as A40 in the plan. Co-Authored-By: Claude Opus 5.5 --- claudedocs/v2-plan.md | 1 + docs/guide/plugins.md | 8 +- docs/web/plugs.md | 3 +- docs/web/resources.md | 10 +- .../mongospecs-web/src/mongospecs_web/api.py | 28 ++- .../src/mongospecs_web/filters.py | 5 + .../src/mongospecs_web/plugs/asgi.py | 5 + .../src/mongospecs_web/plugs/fastapi.py | 31 ++- .../src/mongospecs_web/resource.py | 44 +++- .../tests/test_review_round3.py | 195 ++++++++++++++++++ packages/mongospecs/src/mongospecs/manager.py | 32 ++- .../mongospecs/src/mongospecs/pydantic.py | 19 +- packages/mongospecs/src/mongospecs/query.py | 3 +- .../mongospecs/tests/test_review_round3.py | 137 ++++++++++++ 14 files changed, 484 insertions(+), 37 deletions(-) create mode 100644 packages/mongospecs-web/tests/test_review_round3.py create mode 100644 packages/mongospecs/tests/test_review_round3.py diff --git a/claudedocs/v2-plan.md b/claudedocs/v2-plan.md index c2f9e73..ca2b593 100644 --- a/claudedocs/v2-plan.md +++ b/claudedocs/v2-plan.md @@ -891,6 +891,7 @@ Recorded per §7.1 rule 4. Each one names what changed and why; none invalidates | A37 | §4.8 | HTTP `PATCH` uses JSON Merge Patch semantics for embedded objects (per-key dotted `$set`/`$unset` in one atomic update); lists and dicts still replace whole. Nested keys are held to the embedded model's own `Policy`. An embedded object holding protected values cannot be nulled, and a list or dict of such objects cannot be replaced through `PATCH` (422). `Policy(insert=False)` is enforced on nested keys at create. Supersedes the note in A34 | Decision delegated by Jay. Options weighed: (a) document only, rejected because a client could clear private and read-only values by omitting them, which no documentation fixes; (c) per-field opt-in, rejected because the safe behaviour would be off by default; (d) reject any patch of an object with protected values, rejected because such objects would be unpatchable. (b) merge is what RFC 7396 specifies for objects, is atomic, and removes the hazard instead of fencing it; (d) is kept only where merge has no meaning (list items, null on the whole object). Cost: PATCH on an embedded object is now partial, a behaviour change for clients that relied on replacement; a merge into a missing object needs its required fields; model-level cross-field validators of the embedded model do not run on a merge, as they already do not for top-level PATCH | | A38 | §4.2 | BSON-only values in untyped fields (`Any`) serialise to JSON on all backends: ObjectId and UUID as strings, dates as ISO 8601, Decimal128 as a string, binary as base64, Int64 as a number, Timestamp as a date, Regex as its pattern | Reported: a 500 on pydantic for any ObjectId, and on msgspec/attrs for Binary, Timestamp, Regex and Int64 | | A39 | §4.8 | `Put` operation and `Resource(merge_patch=False)`. `PUT /{id}` replaces every field the client may update (sent value, or the default when left out; a required field must be sent) and leaves the rest as stored, in one atomic update; an embedded object holding protected values is replaced key by key the same way, one without them is replaced whole. It never creates. `Put` is not in the default `ops`. `merge_patch=False` makes `PATCH` replace embedded objects as before A37; nested keys the policy closes are still rejected | Requested by Jay as the opt-out from A37 and its limits (validators of the embedded model see one key at a time on a merge, a stored null needs the whole object, a backend without `supports_merge` gets a 422). The flag is on the resource, not the model: it describes an HTTP verb, model code is unaffected, and embedded models do not inherit settings (A35). Cost of opting out, accepted by whoever sets it: values the client cannot send are reset to defaults on replacement. `Put` is opt-in because guards are keyed by operation, so adding it to the defaults would expose an unguarded write on resources that guard `Patch` | +| A40 | §4.4, §4.8 | Third review. Core: a single-field update on a pydantic model no longer fails a model-level validator that saw the other fields' defaults (the field's own validators still run); `insert_many` and `bulk_save` run document hooks; `page()` starts its count only after its arguments are validated. Web: request bodies are strict JSON (`NaN`, `Infinity` and over-deep nesting are a 400); `If-Match` on a model without a `Revision` is a 412; `If-None-Match` accepts lists, weak tags and `*`; the ASGI plug drops a request whose client disconnected mid-body; `Api(openapi=False)` is honoured by the FastAPI plug; guard and hook keys must name an operation; the concurrency limit is applied before authentication; a filter or sort key given twice is a 400 | Each was reproduced against the replica set first (`test_review_round3.py` in both packages). `insert_many` hooks deviate from Mongoose, where `insertMany` does not run `save` middleware: here the documents are in the caller's hands and a `before("insert")` hook that silently does not run loses data (a slug, a derived field). The hook calls are skipped when no hook is registered, so the bulk paths cost the same as before for models without hooks. Left as documented differences: trailing slash, `HEAD` and `OPTIONS` on framework plugs | New node: diff --git a/docs/guide/plugins.md b/docs/guide/plugins.md index 81428d0..6f70c82 100644 --- a/docs/guide/plugins.md +++ b/docs/guide/plugins.md @@ -4,7 +4,7 @@ mongospecs has the same four kinds of middleware as Mongoose, with typed signatu | Kind | Where | Runs for | |---|---|---| -| Document hooks | `@Model.before("insert" \| "save" \| "delete")`, `@Model.after(...)` | document-level operations | +| Document hooks | `@Model.before("insert" \| "save" \| "delete")`, `@Model.after(...)` | operations on documents you hold: `insert`, `save`, `delete`, `insert_many`, `bulk_save` | | Command and filter rewriting | `Plugin.on_insert`, `on_update`, `on_filter`, `on_delete` | every write and read, including filter-based ones | | Pipeline rewriting | `Plugin.on_pipeline` | `Model.aggregate` | | Bulk hooks | `Plugin.before_bulk`, `after_bulk` | `insert_many`, `bulk_write`, `bulk_save` | @@ -28,8 +28,10 @@ async def slugify(ctx, note): Hooks receive an `OpCtx` (the session, registry and model) and the document. They run in registration order and the first exception stops the operation. -Document hooks do not run for filter-based writes such as `Note.find(...).update(...)`: no -documents are loaded. Plugins still rewrite those commands, which is how timestamps, revisions and +Document hooks run wherever the documents are in your hands, which includes `insert_many` and +`bulk_save` (the `before` hooks for every document ahead of the one round trip, the `after` hooks +for the documents that were written). They do not run for filter-based writes such as +`Note.find(...).update(...)`, `Model.update(id, ...)` or `bulk_write`: no documents are loaded. Plugins still rewrite those commands, which is how timestamps, revisions and scopes keep working there. ## Writing a plugin diff --git a/docs/web/plugs.md b/docs/web/plugs.md index b7a2a6d..fb33fe6 100644 --- a/docs/web/plugs.md +++ b/docs/web/plugs.md @@ -34,7 +34,8 @@ plug.mount(app, api) ``` The operations and their schemas are merged into the application's own `/openapi.json` and -`/docs`, for msgspec and attrs models as well as pydantic ones. +`/docs`, for msgspec and attrs models as well as pydantic ones. With `Api(openapi=False)` (or +`plug.mount(app, api, openapi=False)`) the routes are served but stay out of both. ## Litestar diff --git a/docs/web/resources.md b/docs/web/resources.md index d15b72e..4990bd8 100644 --- a/docs/web/resources.md +++ b/docs/web/resources.md @@ -128,7 +128,8 @@ await registry.close() ?include=author embed allowlisted relations ``` -Only fields and operators the policy allows are accepted; everything else is a 400. +Only fields and operators the policy allows are accepted; everything else is a 400. So is a +filter or sort key given twice (`?name=a&name=b`); `[in]` and `[nin]` may be repeated. ## Limits and defaults @@ -229,5 +230,12 @@ JSON whatever they are: an ObjectId as its string, dates as ISO 8601, binary as [Tuning for production](../guide/production.md). `DELETE` honours `If-Match` like `PATCH`, and the row scope is part of the delete's own filter. +On a model without a `Revision` field there is nothing to compare with, so a request carrying +`If-Match` gets a 412 instead of an unconditional write (`If-Match: *` passes). `If-None-Match` on +`GET` accepts a list, weak tags and `*`. + +Request bodies are strict JSON: `NaN` and `Infinity` are a 400, and so is a body nested too +deeply to parse. Guards and hooks must name an operation the resource has; a misspelt key or +`Put` on a resource without it raises `ValueError` when the `Api` builds its routes. `HEAD` and `OPTIONS` are answered by `api.handle` and the pure ASGI plug; the framework plugs leave them to the framework. diff --git a/packages/mongospecs-web/src/mongospecs_web/api.py b/packages/mongospecs-web/src/mongospecs_web/api.py index 3aaf23a..8aaaf7e 100644 --- a/packages/mongospecs-web/src/mongospecs_web/api.py +++ b/packages/mongospecs-web/src/mongospecs_web/api.py @@ -168,6 +168,8 @@ def _routes(self) -> list[Route]: cached: list[Route] | None = self.__dict__.get("_route_cache") count = sum(len(r.operations) for r in self.resources) if cached is None or self.__dict__.get("_route_count") != count: + for resource in self.resources: + resource.check() cached = self.routes # Literal segments win over parameters: /{id}/publish before /{id}, /search before /{id}. cached.sort(key=lambda r: (-r.path.count("/"), r.path.count("{"))) @@ -178,17 +180,8 @@ def _routes(self) -> list[Route]: async def run(self, route: Route, request: Request) -> Response: """Authenticate and dispatch. Never raises.""" - try: - if len(request.body) > self.max_body: - raise PayloadTooLarge(f"The request body may be at most {self.max_body} bytes.") - if self.authenticate is not None: - principal = self.authenticate(request) - request.principal = await principal if inspect.isawaitable(principal) else principal - except Exception as exc: - response = Response.error(exc) - if response.status >= 500: - logging.getLogger("mongospecs_web").error("authentication failed unexpectedly", exc_info=exc) - return response + # The limit comes first: authentication is work too (often a database read), and an + # overloaded server should not do it for requests it is about to turn away. if self.max_concurrency is not None and self._in_flight >= self.max_concurrency: return Response.error( HTTPError( @@ -200,6 +193,19 @@ async def run(self, route: Route, request: Request) -> Response: ) self._in_flight += 1 try: + try: + if len(request.body) > self.max_body: + raise PayloadTooLarge(f"The request body may be at most {self.max_body} bytes.") + if self.authenticate is not None: + principal = self.authenticate(request) + request.principal = await principal if inspect.isawaitable(principal) else principal + except Exception as exc: + response = Response.error(exc) + if response.status >= 500: + logging.getLogger("mongospecs_web").error( + "authentication failed unexpectedly", exc_info=exc + ) + return response response = await route.resource.dispatch(route.operation, request) finally: self._in_flight -= 1 diff --git a/packages/mongospecs-web/src/mongospecs_web/filters.py b/packages/mongospecs-web/src/mongospecs_web/filters.py index 86ede91..9ce49f7 100644 --- a/packages/mongospecs-web/src/mongospecs_web/filters.py +++ b/packages/mongospecs-web/src/mongospecs_web/filters.py @@ -104,6 +104,9 @@ def parse_list_query( raise BadRequest(f"{name!r} cannot be used as a filter") if op not in allowed: raise BadRequest(f"operator {op!r} is not allowed on {name!r}; allowed: {', '.join(allowed)}") + if len(values) > 1 and op not in ("in", "nin"): + # Only one of them could apply; picking it silently hides the client's mistake. + raise BadRequest(f"{key!r} is given more than once") filters.append(Filter(name, op, _value(view, name, op, values))) sort: list[tuple[str, int]] = [] @@ -115,6 +118,8 @@ def parse_list_query( name = item.lstrip("+-") if name not in view.sortable: raise BadRequest(f"{name!r} cannot be used for sorting") + if any(name == taken for taken, _ in sort): + raise BadRequest(f"{name!r} is given more than once in sort") sort.append((name, direction)) size = pagination.default diff --git a/packages/mongospecs-web/src/mongospecs_web/plugs/asgi.py b/packages/mongospecs-web/src/mongospecs_web/plugs/asgi.py index 0051232..13cab09 100644 --- a/packages/mongospecs-web/src/mongospecs_web/plugs/asgi.py +++ b/packages/mongospecs-web/src/mongospecs_web/plugs/asgi.py @@ -66,11 +66,14 @@ async def _http(self, scope: Scope, receive: Receive, send: Send) -> None: await _send(send, Response(200, self._openapi, {})) return headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope.get("headers", [])} + gone = False async def chunks() -> t.AsyncIterator[bytes]: + nonlocal gone while True: message = await receive() if message["type"] == "http.disconnect": + gone = True return yield message.get("body", b"") if not message.get("more_body"): @@ -81,6 +84,8 @@ async def chunks() -> t.AsyncIterator[bytes]: except HTTPError as exc: await _send(send, Response.error(exc)) return + if gone: + return # the client left before the request was complete: there is nothing to run query = parse_qsl(scope.get("query_string", b"").decode("latin-1"), keep_blank_values=True) response = await self.api.handle(method, path, query=query, headers=headers, body=body, raw=scope) await _send(send, response) diff --git a/packages/mongospecs-web/src/mongospecs_web/plugs/fastapi.py b/packages/mongospecs-web/src/mongospecs_web/plugs/fastapi.py index a58057c..90e122f 100644 --- a/packages/mongospecs-web/src/mongospecs_web/plugs/fastapi.py +++ b/packages/mongospecs-web/src/mongospecs_web/plugs/fastapi.py @@ -35,9 +35,14 @@ async def endpoint(request: StarletteRequest) -> StarletteResponse: return endpoint -def router(api: Api) -> APIRouter: - """An ``APIRouter`` with one route per operation.""" +def router(api: Api, *, openapi: bool | None = None) -> APIRouter: + """An ``APIRouter`` with one route per operation. + + ``openapi`` says whether the operations appear in the application's OpenAPI document; + by default that follows ``Api(openapi=...)``. + """ out = APIRouter() + documented = api.expose_openapi if openapi is None else openapi secured = api.authenticate is not None for route in api._routes(): extra = operation_object(route.resource, route.operation, secured=secured) @@ -52,16 +57,24 @@ def router(api: Api) -> APIRouter: status_code=route.operation.status, response_class=StarletteResponse, openapi_extra=extra, + include_in_schema=documented, ) return out -def mount(app: FastAPI, api: Api) -> None: - """Add the API to a FastAPI application and merge its schemas into ``app.openapi()``.""" - app.include_router(router(api)) +def mount(app: FastAPI, api: Api, *, openapi: bool | None = None) -> None: + """Add the API to a FastAPI application and merge its schemas into ``app.openapi()``. + + With ``Api(openapi=False)`` (or ``openapi=False`` here) the routes are served but left out + of the application's ``/openapi.json`` and ``/docs``. + """ + documented = api.expose_openapi if openapi is None else openapi + app.include_router(router(api, openapi=documented)) + if not documented: + return original = app.openapi - def openapi() -> dict[str, t.Any]: + def merged() -> dict[str, t.Any]: if app.openapi_schema is not None: return app.openapi_schema schema = original() @@ -71,12 +84,12 @@ def openapi() -> dict[str, t.Any]: components.setdefault("responses", {}).update(ours["responses"]) for route in api._routes(): operation = schema["paths"][route.path][route.method.lower()] - documented = operation_object( + described = operation_object( route.resource, route.operation, secured=api.authenticate is not None ) # FastAPI adds its own default responses; ours are the truth. - operation["responses"] = documented["responses"] + operation["responses"] = described["responses"] app.openapi_schema = schema return schema - app.openapi = openapi # type: ignore[method-assign] + app.openapi = merged # type: ignore[method-assign] diff --git a/packages/mongospecs-web/src/mongospecs_web/resource.py b/packages/mongospecs-web/src/mongospecs_web/resource.py index da8438c..9e602f9 100644 --- a/packages/mongospecs-web/src/mongospecs_web/resource.py +++ b/packages/mongospecs-web/src/mongospecs_web/resource.py @@ -22,6 +22,7 @@ BadRequest, Forbidden, NotFound, + PreconditionFailed, Unprocessable, UnsupportedMediaType, problem, @@ -75,6 +76,12 @@ def _pick_dumps() -> t.Callable[[t.Any], bytes]: dumps: t.Callable[[t.Any], bytes] = _pick_dumps() +def _not_json(literal: str) -> t.NoReturn: + # Python's parser reads NaN and Infinity; JSON has neither, and a NaN that got stored + # could not be sent back as JSON. + raise ValueError(f"{literal} is not a JSON value") + + @dataclass(slots=True) class Request: """A framework-neutral HTTP request.""" @@ -97,9 +104,11 @@ def json(self) -> t.Any: if content_type != JSON and not content_type.endswith("+json"): raise UnsupportedMediaType("The request body must be application/json.") try: - return json.loads(self.body) + return json.loads(self.body, parse_constant=_not_json) except ValueError: raise BadRequest("The request body is not valid JSON.") from None + except RecursionError: + raise BadRequest("The request body is nested too deeply.") from None @dataclass(slots=True) @@ -335,6 +344,20 @@ def register(fn: F) -> F: return register + def check(self) -> None: + """Raise ``ValueError`` if a guard or hook is registered for an operation this resource + does not have: a misspelt key, or ``Put`` on a resource whose ``ops`` leave it out, would + otherwise guard nothing without a word. Called when an ``Api`` builds its routes, so + custom operations may be declared after the guards that name them.""" + known = {op.name for op in self.operations} | {op.kind for op in self.operations} + for what, keys in (("guards", self._guards), ("before", self._before), ("after", self._after)): + unknown = sorted(set(keys) - known) + if unknown: + raise ValueError( + f"{self.name}: {what} registered for {unknown}, which this resource does not have; " + f"its operations are {sorted(known - {'custom'})}" + ) + def operation(self, name: str) -> Operation: for op in self.operations: if op.name == name: @@ -537,7 +560,7 @@ async def handler(ctx: Ctx[T]) -> Response: etag = self.view.etag(doc) if etag: headers["etag"] = etag - if ctx.request.headers.get("if-none-match") == etag: + if _etag_listed(ctx.request.headers.get("if-none-match"), etag): return Response(304, b"", headers) return Response.json({"data": self.view.serialize(doc, set(include))}, 200, headers) @@ -644,13 +667,28 @@ def __repr__(self) -> str: return f"" +def _etag_listed(header: str | None, etag: str) -> bool: + """Whether ``If-None-Match`` names ``etag``: one of a list, weak or strong, or ``*``.""" + if header is None: + return False + for item in header.split(","): + item = item.strip() + if item == "*" or item.removeprefix("W/") == etag: + return True + return False + + def _if_match(header: str | None, has_revision: bool) -> int | None: """Parse ``If-Match`` into a revision number. ``*`` and absence mean "no check".""" - if header is None or not has_revision: + if header is None: return None value = header.strip() if value == "*": return None + if not has_revision: + # There is no ETag to compare with, so the condition cannot hold. Going ahead anyway + # would be an unconditional write the client explicitly did not ask for. + raise PreconditionFailed("This resource has no revisions, so If-Match cannot be satisfied.") if value.startswith("W/"): value = value[2:] try: diff --git a/packages/mongospecs-web/tests/test_review_round3.py b/packages/mongospecs-web/tests/test_review_round3.py new file mode 100644 index 0000000..c7aef7f --- /dev/null +++ b/packages/mongospecs-web/tests/test_review_round3.py @@ -0,0 +1,195 @@ +"""Regression tests for the third review of v2: one test per reproduced defect.""" + +import json +import typing as t + +import httpx +import pytest + +from mongospecs import Policy, Revision, SpecConfig +from mongospecs_web import Api, Create, Cursor, Delete, Get, List, Put, Resource +from mongospecs_web.testing import PLUGS, FakeBackend, mount_on + +JSON = {"content-type": "application/json"} + + +def api_for(*, revision=False, **options): + from mongospecs.msgspec import Spec + + if revision: + + class Item(Spec, kw_only=True): + __spec__ = SpecConfig(collection="r3_items") + name: t.Annotated[str, Policy(filter=("eq", "in"), sort=True)] = "" + price: float = 0.0 + rev: t.Annotated[int, Revision] = 0 + + else: + + class Item(Spec, kw_only=True): + __spec__ = SpecConfig(collection="r3_items") + name: t.Annotated[str, Policy(filter=("eq", "in"), sort=True)] = "" + price: float = 0.0 + + store = FakeBackend(Item) + resource = Resource(Item, name="items", backend=store, page=Cursor(total=True)) + return Api(resource, prefix="/api", **options), store + + +async def create(api, **fields): + response = await api.handle("POST", "/api/items", body=json.dumps(fields).encode(), headers=JSON) + assert response.status == 201, response.body + return response.headers["location"] + + +@pytest.mark.parametrize("literal", [b"NaN", b"Infinity", b"-Infinity"]) +async def test_2_numbers_json_does_not_have_are_rejected(literal): + api, store = api_for() + url = await create(api, name="a", price=1.0) + for method, path in (("POST", "/api/items"), ("PATCH", url)): + response = await api.handle(method, path, body=b'{"price":' + literal + b"}", headers=JSON) + assert response.status == 400, (method, response.status, response.body) + assert [d.price for d in store.docs.values()] == [1.0] + + +async def test_3_json_nested_too_deeply_is_a_bad_request(caplog): + api, _ = api_for() + response = await api.handle("POST", "/api/items", body=b"[" * 200_000, headers=JSON) + assert response.status == 400, response.body + assert not caplog.records # the client's mistake is not a server error + + +async def test_4_if_match_cannot_be_satisfied_without_revisions(): + api, store = api_for() + url = await create(api, name="a") + for method, body in (("PATCH", b'{"name":"b"}'), ("DELETE", b"")): + response = await api.handle(method, url, body=body, headers={**JSON, "if-match": '"7"'}) + assert response.status == 412, (method, response.status) + assert [d.name for d in store.docs.values()] == ["a"] + # "*" only asks that the resource exists, which needs no revision + assert ( + await api.handle("PATCH", url, body=b'{"name":"b"}', headers={**JSON, "if-match": "*"}) + ).status == 200 + + +async def test_if_none_match_reads_etags_the_way_if_match_does(): + api, _ = api_for(revision=True) + url = await create(api, name="a") + for header, status in (('"0"', 304), ('W/"0"', 304), ('"9", "0"', 304), ("*", 304), ('"9"', 200)): + response = await api.handle("GET", url, headers={"if-none-match": header}) + assert response.status == status, header + + +async def test_5_a_request_the_client_abandoned_is_not_run(): + api, store = api_for() + url = await create(api, name="victim") + messages = iter([{"type": "http.request", "body": b"", "more_body": True}, {"type": "http.disconnect"}]) + sent = [] + + async def receive(): + return next(messages) + + async def send(message): + sent.append(message) + + scope = {"type": "http", "method": "DELETE", "path": url, "headers": [], "query_string": b""} + await api.asgi()(scope, receive, send) + assert len(store.docs) == 1 and not sent + + +async def test_6_a_private_openapi_stays_private_on_fastapi(): + from fastapi import FastAPI + + from mongospecs_web.plugs.fastapi import mount + + api, _ = api_for(openapi=False) + app = FastAPI() + mount(app, api) + spec = app.openapi() + assert not [path for path in spec["paths"] if path.startswith("/api")] + assert "ItemRead" not in spec.get("components", {}).get("schemas", {}) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") as client: + assert (await client.get("/api/items")).status_code == 200 # still served + # and it can be asked for explicitly, as on the other plugs + shown = FastAPI() + mount(shown, api, openapi=True) + assert "/api/items" in shown.openapi()["paths"] + + +def test_9_guards_and_hooks_must_name_an_operation(): + from mongospecs.msgspec import Spec + + class Thing(Spec, kw_only=True): + __spec__ = SpecConfig(collection="r3_things") + name: str = "" + + def deny(ctx): + return False + + for guards in ({"patchh": deny}, {Put: deny}): # a typo; an operation this resource does not have + resource = Resource(Thing, backend=FakeBackend(Thing), guards=guards) + with pytest.raises(ValueError, match="guards"): + Api(resource).match("GET", "/r3_things") + hooked = Resource(Thing, backend=FakeBackend(Thing), ops=[List, Get]) + hooked.before(Create)(lambda ctx: None) + with pytest.raises(ValueError, match="before"): + Api(hooked).match("GET", "/r3_things") + # a custom operation may be guarded before it is declared + late = Resource(Thing, backend=FakeBackend(Thing), guards={"publish": deny, Delete: deny}) + + @late.op("publish") + async def publish(ctx): + return None + + assert Api(late).match("GET", "/r3_things")[0].operation.name == "list" + + +async def test_overload_is_refused_before_authentication_runs(): + import asyncio + + authenticated = [] + release = asyncio.Event() + + async def authenticate(request): + authenticated.append(1) + await release.wait() + return "ann" + + api, _ = api_for(authenticate=authenticate, max_concurrency=1) + first = asyncio.ensure_future(api.handle("GET", "/api/items")) + await asyncio.sleep(0) + await asyncio.sleep(0) + try: + second = await asyncio.wait_for(api.handle("GET", "/api/items"), 2) + finally: + release.set() + assert second.status == 503 and len(authenticated) == 1 + assert (await first).status == 200 + assert (await api.handle("GET", "/api/items")).status == 200 # the slot was given back + + +async def test_a_filter_or_sort_key_given_twice_is_an_error(): + api, _ = api_for() + await create(api, name="a") + for query in ( + [("name", "a"), ("name", "b")], + [("sort", "name,-name")], + [("sort", "name"), ("sort", "name")], + ): + assert (await api.handle("GET", "/api/items", query=query)).status == 400, query + listed = await api.handle("GET", "/api/items", query=[("name[in]", "a"), ("name[in]", "b")]) + assert listed.status == 200 and len(json.loads(listed.body)["data"]) == 1 + + +@pytest.mark.parametrize("plug", PLUGS) +async def test_every_plug_refuses_what_the_core_refuses(plug): + api, store = api_for() + url = await create(api, name="a", price=1.0) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=mount_on(plug, api)), base_url="http://t" + ) as client: + assert (await client.patch(url, content=b'{"price":NaN}', headers=JSON)).status_code == 400 + assert ( + await client.patch(url, content=b'{"name":"b"}', headers={**JSON, "if-match": '"3"'}) + ).status_code == 412 + assert [(d.name, d.price) for d in store.docs.values()] == [("a", 1.0)] diff --git a/packages/mongospecs/src/mongospecs/manager.py b/packages/mongospecs/src/mongospecs/manager.py index 7ddd366..7ad61f3 100644 --- a/packages/mongospecs/src/mongospecs/manager.py +++ b/packages/mongospecs/src/mongospecs/manager.py @@ -277,6 +277,13 @@ async def _hooks(self, when: str, events: tuple[str, ...], ctx: OpCtx, doc: t.An for hook in meta.hooks(when, event): await hook(ctx, doc) + def _has_hooks(self, *events: str) -> bool: + """Whether a document hook could run for any of ``events`` (bulk paths skip the calls otherwise).""" + for plugin in self.plugins: + if type(plugin).before is not Plugin.before or type(plugin).after is not Plugin.after: + return True + return any(self.meta.hooks(when, event) for when in ("before", "after") for event in events) + # -- document-level writes ----------------------------------------------------- async def insert(self, doc: T, *, session: t.Any = None) -> T: @@ -677,7 +684,9 @@ async def insert_many( """Insert documents in one round trip. With ``ordered=False`` the server attempts every document; failures are - reported in ``BulkResult.errors`` instead of raising. + reported in ``BulkResult.errors`` instead of raising. Document hooks for + ``insert`` and ``save`` run for every document, as they do for ``insert()``: + the ``before`` hooks ahead of the write, the ``after`` hooks for what was written. """ items: list[t.Any] = [ self.meta.decode(dict(d), stored=False) if isinstance(d, t.Mapping) else d for d in docs @@ -685,6 +694,10 @@ async def insert_many( if not items: return BulkResult([], []) ctx = self.ctx("insert", session, many=True) + hooks = self._has_hooks("insert", "save") + if hooks: + for doc in items: + await self._hooks("before", ("insert", "save"), ctx, doc) for plugin in self.plugins: await plugin.before_bulk(ctx, items) encoded: list[dict[str, t.Any]] = [] @@ -717,6 +730,9 @@ async def insert_many( inserted.append(doc) for plugin in self.plugins: await plugin.after_bulk(ctx, inserted) + if hooks: + for doc in inserted: + await self._hooks("after", ("insert", "save"), ctx, doc) return BulkResult(inserted, sorted(failed.items())) async def bulk_write( @@ -765,6 +781,9 @@ async def bulk_save(self, docs: t.Iterable[T], *, session: t.Any = None) -> int: fail (a duplicate key, a document that no longer exists, a revision that moved on), the rest are still written and brought in sync in memory, and the error raised afterwards lists the ones that were not in ``.failed``. + + Document hooks run as they do for ``save()``: ``insert`` and ``save`` for new + documents, ``save`` for tracked ones; the ``after`` hooks only for what was written. """ from pymongo import InsertOne from pymongo import UpdateOne as DriverUpdateOne @@ -774,10 +793,13 @@ async def bulk_save(self, docs: t.Iterable[T], *, session: t.Any = None) -> int: pending: list[ tuple[t.Any, str, OpCtx, dict[str, t.Any], tracking.DocState | None, dict[str, t.Any]] ] = [] + hooks = self._has_hooks("insert", "save") for doc in docs: state = tracking.state_of(doc) if state is None: ctx = self.ctx("insert", session, many=True, doc=doc) + if hooks: + await self._hooks("before", ("insert", "save"), ctx, doc) enc = self.meta.encode(doc) if enc.get("_id") is None: enc.pop("_id", None) @@ -787,6 +809,8 @@ async def bulk_save(self, docs: t.Iterable[T], *, session: t.Any = None) -> int: pending.append((doc, "insert", ctx, enc, None, {})) continue ctx = self.ctx("update", session, many=True, doc=doc, snapshot=state.snapshot) + if hooks: + await self._hooks("before", ("save",), ctx, doc) current, update = self._delta(doc, state) if not update: continue @@ -842,6 +866,12 @@ async def bulk_save(self, docs: t.Iterable[T], *, session: t.Any = None) -> int: written += 1 for plugin in self.plugins: await plugin.after_bulk(bulk_ctx, [p[0] for i, p in enumerate(pending) if i not in failed]) + if hooks: + for index, (doc, kind, ctx, *_rest) in enumerate(pending): + if index not in failed: + await self._hooks( + "after", ("insert", "save") if kind == "insert" else ("save",), ctx, doc + ) if error is not None: error.failed = [pending[i][0] for i in sorted(failed)] raise error diff --git a/packages/mongospecs/src/mongospecs/pydantic.py b/packages/mongospecs/src/mongospecs/pydantic.py index 7674325..4ee9129 100644 --- a/packages/mongospecs/src/mongospecs/pydantic.py +++ b/packages/mongospecs/src/mongospecs/pydantic.py @@ -167,15 +167,20 @@ def _omit_plan(self, tp: t.Any, plans: dict[type, _Omit]) -> _Omit | None: def validate_field(self, cls: type, name: str, value: t.Any) -> t.Any: model = t.cast("type[BaseModel]", cls) + # Runs the model's own field validators for this one field. The instance it is assigned + # to is blank: every other field holds its default, not the stored value. + blank = model.model_construct() + blank.__dict__.pop(name, None) try: - # Runs the model's own field validators for this one field. - blank = model.model_construct() return getattr(model.__pydantic_validator__.validate_assignment(blank, name, value), name) - except pydantic.ValidationError as exc: - raise _error(cls, exc, name) from exc - except Exception: - # A model-level validator could not run on a partial instance; fall - # back to the field's type and constraints alone. + except Exception as exc: + if isinstance(exc, pydantic.ValidationError) and any(e.get("loc") for e in exc.errors()): + raise _error(cls, exc, name) from exc # the field itself is wrong + # A model-level validator rejected, or could not run on, the blank instance. It + # compares fields, and the other fields here are defaults, so its verdict says + # nothing about the document being updated. The field was validated before it ran. + if name in blank.__dict__: + return blank.__dict__[name] adapter = self._adapters.get((cls, name)) if adapter is None: adapter = TypeAdapter(self.fields(cls)[name].type, config=_adapter_config(model)) diff --git a/packages/mongospecs/src/mongospecs/query.py b/packages/mongospecs/src/mongospecs/query.py index 90abc37..fef990c 100644 --- a/packages/mongospecs/src/mongospecs/query.py +++ b/packages/mongospecs/src/mongospecs/query.py @@ -592,7 +592,6 @@ async def page( raise QueryError("page size must be >= 1") if number is not None and after is not None: raise QueryError("use either number (offset pagination) or after (cursor pagination)") - count_task: t.Awaitable[int] | None = self._clone(skip=0, limit=0, sort=()).count() if total else None if number is not None: if number < 1: @@ -615,6 +614,8 @@ async def page( # default where the stored document has no key, which sorts elsewhere. query._keys = {} total_count: int | None + # Created only now: everything above can raise, and a coroutine nobody awaits is a warning. + count_task: t.Awaitable[int] | None = self._clone(skip=0, limit=0, sort=()).count() if total else None if count_task is not None and self._session is None: items, total_count = await asyncio.gather(query.all(), count_task) else: diff --git a/packages/mongospecs/tests/test_review_round3.py b/packages/mongospecs/tests/test_review_round3.py new file mode 100644 index 0000000..260d4af --- /dev/null +++ b/packages/mongospecs/tests/test_review_round3.py @@ -0,0 +1,137 @@ +"""Regression tests for the third review of v2: one test per reproduced defect.""" + +import gc +import warnings + +import pytest + +from mongospecs import SpecConfig, ValidationError + + +def span_model(): + import pydantic + + from mongospecs.pydantic import Spec + + class Span(Spec): + __spec__ = SpecConfig(collection="r3_spans") + low: int = 0 + high: int = 10 + label: str = "" + + @pydantic.field_validator("label") + @classmethod + def tidy(cls, value): + return value.strip() + + @pydantic.model_validator(mode="after") + def ordered(self): + if self.low > self.high: + raise ValueError("low must not exceed high") + return self + + return Span + + +async def test_1_updating_one_field_is_not_judged_against_the_other_fields_defaults(make_registry): + Span = span_model() + await make_registry(Span) + doc = await Span(low=50, high=60).insert() + # low=55 is fine next to the stored high=60; the model validator only knows the default high=10 + updated = await Span.update(doc.id, low=55) + assert (updated.low, updated.high) == (55, 60) + # the field's own rules still apply: its type, and its field validator + with pytest.raises(ValidationError): + await Span.update(doc.id, low="many") + assert (await Span.update(doc.id, label=" x ")).label == "x" + # a whole document is still validated whole + import pydantic + + with pytest.raises(pydantic.ValidationError): + Span(low=5, high=1) + + +def hooked(backend): + if backend == "msgspec": + from mongospecs.msgspec import Spec + + class Note(Spec, kw_only=True): + __spec__ = SpecConfig(collection="r3_notes") + text: str = "" + slug: str = "" + + elif backend == "pydantic": + from mongospecs.pydantic import Spec + + class Note(Spec): + __spec__ = SpecConfig(collection="r3_notes") + text: str = "" + slug: str = "" + + else: + import attrs + + from mongospecs.attrs import Spec + + @attrs.define(kw_only=True) + class Note(Spec): + __spec__ = SpecConfig(collection="r3_notes") + text: str = "" + slug: str = "" + + seen = [] + + @Note.before("insert") + async def slugify(ctx, note): + note.slug = note.text.lower().replace(" ", "-") + + @Note.before("save") + async def before_save(ctx, note): + seen.append(("before", note.text)) + + @Note.after("save") + async def after_save(ctx, note): + seen.append(("after", note.text, note.id is not None)) + + return Note, seen + + +async def test_7_document_hooks_run_for_insert_many_and_bulk_save(backend, make_registry): + Note, seen = hooked(backend) + await make_registry(Note) + result = await Note.insert_many([Note(text="Hello World"), Note(text="Two")]) + assert [n.slug for n in result] == ["hello-world", "two"] + assert [n["slug"] async for n in Note.find().raw().sort("slug")] == ["hello-world", "two"] + assert seen == [ + ("before", "Hello World"), + ("before", "Two"), + ("after", "Hello World", True), + ("after", "Two", True), + ] + seen.clear() + first = result.inserted[0] + first.text = "Changed" + assert await Note.bulk_save([first, Note(text="New One")]) == 2 + assert await Note.count({"slug": "new-one"}) == 1 + assert seen == [ + ("before", "Changed"), + ("before", "New One"), + ("after", "Changed", True), + ("after", "New One", True), + ] + assert (await Note.get(first.id)).slug == "hello-world" # "insert" hooks are for new documents only + + +async def test_8_a_bad_cursor_leaves_no_coroutine_behind(make_registry): + from mongospecs import QueryError + + Note, _ = hooked("msgspec") + await make_registry(Note) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with pytest.raises(QueryError): + await Note.find().page(5, after="not-a-cursor", total=True) + with pytest.raises(QueryError): + await Note.find().page(5, number=0, total=True) + gc.collect() + assert not [w for w in caught if "never awaited" in str(w.message)] From 0c5388ac5e03f92d926aabb76e30e9ffc76bed22 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 10:25:40 -0400 Subject: [PATCH 22/26] ci,build: run each CI job once per push; complete the extras Branch pushes and their pull requests both triggered CI, so every job ran twice. mongospecs[all] now includes the otel and sentry extras, and mongospecs-web declares msgspec as the optional encoder it already uses when present. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yaml | 3 ++- packages/mongospecs-web/pyproject.toml | 2 ++ packages/mongospecs/pyproject.toml | 2 +- uv.lock | 10 +++++++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 15fadcf..4b1ba53 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,8 +1,9 @@ name: CI on: + # Branches are tested through their pull request; a push trigger for them too ran every job twice. push: - branches: [main, "v2/**"] + branches: [main] pull_request: permissions: diff --git a/packages/mongospecs-web/pyproject.toml b/packages/mongospecs-web/pyproject.toml index 2aa70d9..867aeb3 100644 --- a/packages/mongospecs-web/pyproject.toml +++ b/packages/mongospecs-web/pyproject.toml @@ -20,6 +20,8 @@ classifiers = [ dependencies = ["mongospecs>=0.2.0,<0.3"] [project.optional-dependencies] +# Optional: responses are encoded with msgspec when it is installed, with the standard library otherwise. +msgspec = ["msgspec>=0.19"] starlette = ["starlette>=0.40"] fastapi = ["fastapi>=0.115"] litestar = ["litestar>=2.12"] diff --git a/packages/mongospecs/pyproject.toml b/packages/mongospecs/pyproject.toml index 8cad394..87b5771 100644 --- a/packages/mongospecs/pyproject.toml +++ b/packages/mongospecs/pyproject.toml @@ -30,7 +30,7 @@ pydantic = ["pydantic>=2.9"] attrs = ["attrs>=24.2", "msgspec>=0.19"] otel = ["opentelemetry-api>=1.20"] sentry = ["sentry-sdk>=2.0"] -all = ["msgspec>=0.19", "pydantic>=2.9", "attrs>=24.2"] +all = ["msgspec>=0.19", "pydantic>=2.9", "attrs>=24.2", "opentelemetry-api>=1.20", "sentry-sdk>=2.0"] [project.urls] Repository = "https://github.com/jaykv/mongospecs" diff --git a/uv.lock b/uv.lock index 132b066..f9291a1 100644 --- a/uv.lock +++ b/uv.lock @@ -1060,7 +1060,9 @@ dependencies = [ all = [ { name = "attrs" }, { name = "msgspec" }, + { name = "opentelemetry-api" }, { name = "pydantic" }, + { name = "sentry-sdk" }, ] attrs = [ { name = "attrs" }, @@ -1086,10 +1088,12 @@ requires-dist = [ { name = "msgspec", marker = "extra == 'all'", specifier = ">=0.19" }, { name = "msgspec", marker = "extra == 'attrs'", specifier = ">=0.19" }, { name = "msgspec", marker = "extra == 'msgspec'", specifier = ">=0.19" }, + { name = "opentelemetry-api", marker = "extra == 'all'", specifier = ">=1.20" }, { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.20" }, { name = "pydantic", marker = "extra == 'all'", specifier = ">=2.9" }, { name = "pydantic", marker = "extra == 'pydantic'", specifier = ">=2.9" }, { name = "pymongo", specifier = ">=4.13,<5" }, + { name = "sentry-sdk", marker = "extra == 'all'", specifier = ">=2.0" }, { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.0" }, ] provides-extras = ["msgspec", "pydantic", "attrs", "otel", "sentry", "all"] @@ -1135,6 +1139,9 @@ fastapi = [ litestar = [ { name = "litestar" }, ] +msgspec = [ + { name = "msgspec" }, +] quart = [ { name = "quart", version = "0.22.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "quart", version = "0.23.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, @@ -1150,12 +1157,13 @@ requires-dist = [ { name = "litestar", marker = "extra == 'all'", specifier = ">=2.12" }, { name = "litestar", marker = "extra == 'litestar'", specifier = ">=2.12" }, { name = "mongospecs", editable = "packages/mongospecs" }, + { name = "msgspec", marker = "extra == 'msgspec'", specifier = ">=0.19" }, { name = "quart", marker = "extra == 'all'", specifier = ">=0.19" }, { name = "quart", marker = "extra == 'quart'", specifier = ">=0.19" }, { name = "starlette", marker = "extra == 'all'", specifier = ">=0.40" }, { name = "starlette", marker = "extra == 'starlette'", specifier = ">=0.40" }, ] -provides-extras = ["starlette", "fastapi", "litestar", "quart", "all"] +provides-extras = ["msgspec", "starlette", "fastapi", "litestar", "quart", "all"] [[package]] name = "mongospecs-workspace" From c5eba4d407bfff7deecb2e8b7ff12704771edc1b Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 10:33:35 -0400 Subject: [PATCH 23/26] docs: getting-started tutorial, runnable examples and four topic guides A new user had reference-quality guides and no way into them. - examples/: a quickstart script, a to-do REST API with its tests, and a compose file for a single-node replica set. They are run by the suite, and the tutorial page and the README show them as they are. - docs/tutorial.md: from MongoDB to a model, an API and its tests. - New guides: choosing a backend, errors (what each means and the HTTP status it becomes), authentication and authorization, testing an API. The last two are executed like the other guide pages. - The replica-set requirement is stated in the README, the landing page and the tutorial instead of only in the production notes. - The package README is checked to be the root README. - The reference section is removed: it was generated from docstrings, and the source is the better reference. mkdocstrings goes with it. Co-Authored-By: Claude Opus 5.5 --- Makefile | 8 +- README.md | 55 +++++++--- docs/guide/backends.md | 67 ++++++++++++ docs/guide/errors.md | 69 ++++++++++++ docs/guide/operations.md | 13 +-- docs/index.md | 16 ++- docs/reference/bulk.md | 3 - docs/reference/errors.md | 3 - docs/reference/expr.md | 3 - docs/reference/hooks.md | 3 - docs/reference/indexes.md | 3 - docs/reference/manager.md | 3 - docs/reference/markers.md | 3 - docs/reference/model.md | 3 - docs/reference/query.md | 3 - docs/reference/refs.md | 3 - docs/reference/registry.md | 3 - docs/reference/relations.md | 3 - docs/reference/streams.md | 3 - docs/reference/updates.md | 3 - docs/reference/web_api.md | 3 - docs/reference/web_backend.md | 3 - docs/reference/web_errors.md | 3 - docs/reference/web_filters.md | 3 - docs/reference/web_openapi.md | 3 - docs/reference/web_policy.md | 3 - docs/reference/web_resource.md | 3 - docs/reference/web_testing.md | 3 - docs/tutorial.md | 122 ++++++++++++++++++++++ docs/web/auth.md | 168 ++++++++++++++++++++++++++++++ docs/web/testing.md | 135 ++++++++++++++++++++++++ examples/compose.yaml | 25 +++++ examples/quickstart.py | 65 ++++++++++++ examples/test_quickstart.py | 55 ++++++++++ examples/todo_api/app.py | 72 +++++++++++++ examples/todo_api/conftest.py | 9 ++ examples/todo_api/test_app.py | 59 +++++++++++ mkdocs.yaml | 49 ++------- packages/mongospecs-web/README.md | 10 +- packages/mongospecs/README.md | 51 +++++++-- pyproject.toml | 6 +- uv.lock | 65 +----------- 42 files changed, 969 insertions(+), 216 deletions(-) create mode 100644 docs/guide/backends.md create mode 100644 docs/guide/errors.md delete mode 100644 docs/reference/bulk.md delete mode 100644 docs/reference/errors.md delete mode 100644 docs/reference/expr.md delete mode 100644 docs/reference/hooks.md delete mode 100644 docs/reference/indexes.md delete mode 100644 docs/reference/manager.md delete mode 100644 docs/reference/markers.md delete mode 100644 docs/reference/model.md delete mode 100644 docs/reference/query.md delete mode 100644 docs/reference/refs.md delete mode 100644 docs/reference/registry.md delete mode 100644 docs/reference/relations.md delete mode 100644 docs/reference/streams.md delete mode 100644 docs/reference/updates.md delete mode 100644 docs/reference/web_api.md delete mode 100644 docs/reference/web_backend.md delete mode 100644 docs/reference/web_errors.md delete mode 100644 docs/reference/web_filters.md delete mode 100644 docs/reference/web_openapi.md delete mode 100644 docs/reference/web_policy.md delete mode 100644 docs/reference/web_resource.md delete mode 100644 docs/reference/web_testing.md create mode 100644 docs/tutorial.md create mode 100644 docs/web/auth.md create mode 100644 docs/web/testing.md create mode 100644 examples/compose.yaml create mode 100644 examples/quickstart.py create mode 100644 examples/test_quickstart.py create mode 100644 examples/todo_api/app.py create mode 100644 examples/todo_api/conftest.py create mode 100644 examples/todo_api/test_app.py diff --git a/Makefile b/Makefile index e2b3e99..292e3c9 100644 --- a/Makefile +++ b/Makefile @@ -18,12 +18,12 @@ test-fast: uv run pytest -m "not sharded" format: - uv run ruff format packages bench - uv run ruff check packages bench --fix + uv run ruff format packages bench examples + uv run ruff check packages bench examples --fix lint: - uv run ruff format --check packages bench - uv run ruff check packages bench + uv run ruff format --check packages bench examples + uv run ruff check packages bench examples types: uv run mypy packages/mongospecs/src/mongospecs packages/mongospecs-web/src/mongospecs_web bench/src/mongospecs_bench diff --git a/README.md b/README.md index aa58713..16212ea 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,13 @@ The aim is Mongoose's production feature set (change tracking, middleware, popul discriminators, transactions that restore document state) with typed Python models, plus [`mongospecs-web`](packages/mongospecs-web), a framework-neutral REST layer. +**Documentation: .** Start with +[Getting started](https://jaykv.github.io/mongospecs/tutorial/). + > **Status: 0.2.0, alpha.** This is a rewrite of 0.1.x: async-only, Python 3.11+, MongoDB 7.0+, > and not backwards compatible. The synchronous 0.1.x line lives on the -> [`0.1.x` branch](https://github.com/jaykv/mongospecs/tree/0.1.x). The design and the work graph -> are in [`claudedocs/v2-plan.md`](claudedocs/v2-plan.md). +> [`0.1.x` branch](https://github.com/jaykv/mongospecs/tree/0.1.x). Changes are listed in the +> [changelog](CHANGELOG.md). ## Install @@ -17,14 +20,30 @@ discriminators, transactions that restore document state) with typed Python mode pip install "mongospecs[msgspec]" # or [pydantic], [attrs], [all] ``` +You also need MongoDB 7.0 or newer. Transactions, cascading deletes and change streams need it to +run as a replica set; a single node is enough. [`examples/compose.yaml`](examples/compose.yaml) +starts one on `localhost:27017`: + +``` +docker compose -f examples/compose.yaml up -d --wait +``` + ## A taste +This is [`examples/quickstart.py`](examples/quickstart.py), as it is run by the test suite. + ```python +"""mongospecs in one file. Start MongoDB (see compose.yaml), then: python examples/quickstart.py""" + +import asyncio +import os from datetime import datetime from typing import Annotated from mongospecs import CreatedAt, Inc, Index, Q, Ref, Registry, Revision, SpecConfig, UpdatedAt -from mongospecs.msgspec import Spec # or mongospecs.pydantic / mongospecs.attrs +from mongospecs.msgspec import Spec # or mongospecs.pydantic / mongospecs.attrs + +URI = os.environ.get("MONGO_URI", "mongodb://localhost:27017/quickstart?replicaSet=rs0") class User(Spec): @@ -40,11 +59,11 @@ class Article(Spec): author: Ref[User] | None = None created_at: Annotated[datetime | None, CreatedAt] = None updated_at: Annotated[datetime | None, UpdatedAt] = None - rev: Annotated[int, Revision] = 0 # optimistic locking + rev: Annotated[int, Revision] = 0 # optimistic locking -async def main(): - async with Registry("mongodb://localhost/app", models=[User, Article]) as registry: +async def main(uri: str = URI) -> None: + async with Registry(uri, models=[User, Article]) as registry: await registry.apply_indexes() ann = await User(name="ann").insert() @@ -56,24 +75,34 @@ async def main(): await article.save() # typed queries: names are checked, values are cast - popular = await Article.find(Q.views >= 100, Q.title.prefix("Hello")).sort("-views").limit(10) + popular = await Article.find(Q.views >= 0, Q.title.prefix("Hello")).sort("-views").limit(10) + assert [a.title for a in popular] == ["Hello, world"] # relations: one extra query, or a $lookup with_author = await Article.find(Q.author == ann).include("author").one() - print(with_author.author.doc.name) + print(with_author.title, "by", with_author.author.doc.name) # atomic updates, always validated; the updated document comes back article = await Article.update(article.id, Inc(views=1), tags=["intro"]) + assert article.views == 1 - # transactions: the callback is retried on transient errors, and in-memory - # documents are restored if it aborts + # transactions (these need a replica set): the callback is retried on transient + # errors, and in-memory documents are restored if it aborts async def publish(tx): article.tags.append("published") await article.save(session=tx) await registry.transaction(publish) + print("tags:", (await Article.get(article.id)).tags) + + +if __name__ == "__main__": + asyncio.run(main()) ``` +A REST API over a model is a few more lines; see [`examples/todo_api`](examples/todo_api) and the +[web guide](https://jaykv.github.io/mongospecs/web/resources/). + ## What is in the box | | | @@ -96,7 +125,7 @@ async def main(): oversell, multi-item orders that are all-or-nothing, a change-stream projector that must count each sale exactly once across restarts, and orders that stay consistent across a primary step-down. -- Every example in the guide pages is executed. +- Every example in the guide pages and in `examples/` is executed. - Benchmarks and stress scenarios in `bench/` run against real MongoDB; results and the conclusions drawn from them are in `docs/benchmarks.md`. @@ -110,4 +139,6 @@ uv run mongospecs-bench run # benchmarks; `run LOAD` for the HTTP load ``` The repository is a uv workspace: `packages/mongospecs`, `packages/mongospecs-web` and `bench/` -(benchmarks and stress scenarios against real MongoDB, never published). +(benchmarks and stress scenarios against real MongoDB, never published). See +[CONTRIBUTING.md](CONTRIBUTING.md); the design decisions behind 0.2 are recorded in +[`claudedocs/v2-plan.md`](claudedocs/v2-plan.md). diff --git a/docs/guide/backends.md b/docs/guide/backends.md new file mode 100644 index 0000000..71d8162 --- /dev/null +++ b/docs/guide/backends.md @@ -0,0 +1,67 @@ +# Choosing a backend + +A model is a msgspec `Struct`, a pydantic `BaseModel` or an attrs class. Queries, writes, +relations, plugins and the web layer are the same on all three; what differs is what the model +library itself does. Pick the one your application already uses. With no preference, msgspec is +the lightest and the one the examples use. + +``` +pip install "mongospecs[msgspec]" # or [pydantic], or [attrs] (which also installs msgspec) +``` + +| | msgspec | pydantic | attrs | +|---|---|---|---| +| Import | `from mongospecs.msgspec import Spec` | `from mongospecs.pydantic import Spec` | `from mongospecs.attrs import Spec` | +| Declaring a model | `class A(Spec):` | `class A(Spec):` | `@attrs.define(kw_only=True)` on **every** model, then `class A(Spec):` | +| Wrong type passed to the constructor | accepted | `ValidationError` | accepted | +| Stored documents on read | validated | validated | validated | +| `Model.update(...)`, `PATCH`, filters | validated and cast | validated and cast | validated and cast | +| Your own validators | none in msgspec; use `Annotated[..., msgspec.Meta(...)]` constraints | `field_validator`, `model_validator` | `attrs.field(validator=..., converter=...)` | +| An empty list as a default | `tags: list[str] = []` | `tags: list[str] = []` | `tags: list[str] = attrs.field(factory=list)` | +| Adopting an existing class | `SpecAdapter` from each backend module | same | same | + +Speed and memory are measured in [Benchmarks](../benchmarks.md). + +## What to know about each + +### msgspec and attrs: the constructor does not check types + +That is how both libraries work, and mongospecs does not add a check on `insert()` or `save()`: + +```py +doc = Counter(n="many") # no error: msgspec and attrs do not validate here +await doc.insert() # stored as a string +await Counter.get(doc.id) # ValidationError: reads are validated +``` + +Values that come from outside (a request body, a file, another service) should go through +something that validates: `Model.update(id, n=value)`, `Model.insert_many([{"n": value}])` (a +mapping is validated; an instance is taken as it is), a `mongospecs-web` resource, or +`msgspec.convert(data, Counter)`. Values your own typed code builds are covered by your type +checker. + +### pydantic: validators and single-field updates + +- Models are validated when constructed. `Spec` sets `validate_assignment=False`, as pydantic does + by default, so a value assigned to an attribute afterwards is not validated, and is saved as it is. +- `Model.update(id, low=5)` and an HTTP `PATCH` validate one field at a time. The field's type, + constraints and `field_validator`s run. A `model_validator` that compares fields does not: it + would be comparing the new value with the other fields' defaults, not with what is stored. If a + rule spans fields, enforce it where you have the whole document: load it, change it, `save()`. +- `id` is declared with the alias `_id`; both names are accepted when constructing. +- The `pydantic.mypy` plugin works as usual. + +### attrs: decorate every model + +A subclass of `Spec` that is not decorated with `@attrs.define(kw_only=True)` gets no fields of +its own. `kw_only=True` is needed because `id` has a default and your required fields come after +it. + +## What is the same everywhere + +- The `id` attribute, stored as `_id`; `__spec__ = SpecConfig(...)` assigned without an annotation. +- Field markers in `typing.Annotated`: `CreatedAt`, `Revision`, `Immutable`, `Policy(...)` and the + rest (see [Models](models.md)). +- Embedded models are plain classes of the same library, without the `Spec` base. +- Two documents are equal when their backend says so (field by field), not when their ids match. +- Models from different backends can be bound to the same registry. diff --git a/docs/guide/errors.md b/docs/guide/errors.md new file mode 100644 index 0000000..a36ce18 --- /dev/null +++ b/docs/guide/errors.md @@ -0,0 +1,69 @@ +# Errors + +mongospecs raises its own exceptions, all subclasses of `MongoSpecsError` and all importable from +`mongospecs`. When one wraps a driver error, that error is its `__cause__`. + +```py +from mongospecs import DuplicateKeyError + +try: + await User(email="ann@example.com").insert() +except DuplicateKeyError as exc: + print(exc.fields) # {"email": "ann@example.com"} +``` + +## What each one means + +The last column is what a `mongospecs-web` API answers when the error reaches it. Anything not +listed becomes a 500 with no detail, and the cause is logged. + +| Error | Raised when | What to do | HTTP | +|---|---|---|---| +| `ValidationError` | a value does not fit the model: on read, in `update()`, in a request body. `.errors` lists `{"field", "message"}` | fix the input; for stored documents see `SpecConfig(upcast=...)` and `find().on_invalid(...)` | 422 | +| `QueryError` | a filter, sort or update names an unknown field, has a value that cannot be cast, or a raw dict contains an operator | fix the query; wrap filters you wrote yourself in `trusted()` | 400 | +| `DocumentNotFound` | `one()`, `or_fail()`, `update()`, `save()` or `delete()` matched nothing | treat as "gone" | 404 | +| `StaleDocumentError` | a `Revision` check failed: the document changed since you loaded it | reload and retry, or tell the user | 412 | +| `DuplicateKeyError` | a unique index was violated. `.fields` maps index keys to the offending values | report the conflict | 409 | +| `EncodingError` | a value has no BSON form (an integer beyond 64 bits, an unsupported type) | store a type MongoDB has | 422 | +| `NotLoadedError` | `ref.doc` was read before the reference was loaded | add `.include("field")` or `await ref.fetch()` | 500 | +| `OperationTimeout` | the operation passed its time limit (`max_time_ms`) | add an index, raise the limit, or retry | 504 | +| `PoolTimeout` | no pooled connection became free in time | the server is overloaded; see [Tuning for production](production.md) | 503 | +| `ConnectionFailed` | the server could not be reached | retry later | 503 | +| `TransactionError` | a transaction could not be committed, or `transaction()` was called inside another | pass the outer `tx` down | 500 | +| `DocumentTooLarge` | a document or an aggregation result passed 16 MB | split the document | 500 | +| `BulkWriteFailed` | part of a bulk write failed. `.write_errors` lists the items by input index | handle per item | 500 | +| `OperationFailed` | the server rejected the operation for another reason. `.code` is the server's error code | depends on the code | 500 | +| `ConfigurationError` | a model, plugin or registry is declared incorrectly; raised when the model is first used | fix the declaration | at startup | +| `NotBoundError` | a model was used before it was given to a registry | add it to `Registry(models=[...])` | at startup | +| `RegistryClosedError` | an operation ran on a registry that is not open | open it first; a closed registry cannot be reopened | 500 | + +`QueryError`, `ValidationError` and `EncodingError` are also `ValueError`s, and `DocumentNotFound` +is a `LookupError`, so code that already catches those keeps working. + +Batch operations that partly succeed (`bulk_save`) raise after writing what they could; the +documents that were **not** written are in the error's `.failed`. + +## Errors of your own, over HTTP + +In a guard, hook or custom operation, raise one of the errors from `mongospecs_web` to choose the +status yourself: `BadRequest`, `Unauthorized`, `Forbidden`, `NotFound`, `Conflict`, +`PreconditionFailed`, `Unprocessable`, or `HTTPError(detail, status=..., title=...)`. + +```py +from mongospecs_web import Conflict + +@orders.op("ship", method="POST") +async def ship(ctx): + order = await ctx.load() + if order.shipped: + raise Conflict("This order has already shipped.") + ... +``` + +Every error response is an RFC 9457 `application/problem+json` document: + +```json +{"type": "about:blank", "title": "Unprocessable Content", "status": 422, + "detail": "The request body failed validation.", + "errors": [{"field": "title", "message": "Expected `str`, got `int`"}]} +``` diff --git a/docs/guide/operations.md b/docs/guide/operations.md index 536ac7f..fa01624 100644 --- a/docs/guide/operations.md +++ b/docs/guide/operations.md @@ -63,17 +63,8 @@ ready-made listeners. With the Sentry SDK, database work shows up as `Account.in ## Errors -mongospecs raises its own errors and keeps the driver error as `__cause__`. - -| Error | When | -|---|---| -| `ValidationError` | input does not fit the model | -| `QueryError` | unknown field, uncastable value, untrusted operator | -| `DocumentNotFound` | an acknowledged read or write matched nothing | -| `StaleDocumentError` | a revision check failed | -| `DuplicateKeyError` | unique index violated; `.fields` names the values | -| `OperationTimeout`, `ConnectionFailed`, `PoolTimeout` | the database is slow or unreachable | -| `DocumentTooLarge` | a document or pipeline result exceeded 16MB | +mongospecs raises its own errors and keeps the driver error as `__cause__`. See [Errors](errors.md) +for each one, what to do about it and what an API client sees. ## One registry per model, or explicit managers diff --git a/docs/index.md b/docs/index.md index 90f345f..35c6a47 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,19 +8,25 @@ pip install "mongospecs[msgspec]" # or [pydantic], [attrs] pip install "mongospecs-web[fastapi]" # or [starlette], [litestar], [quart]; none needed for plain ASGI ``` -Requires Python 3.11+ and MongoDB 7.0+. Status: alpha. +Requires Python 3.11+ and MongoDB 7.0+ running as a replica set (one node is enough; transactions, +cascading deletes and change streams need it). Status: alpha. ## Where to start -- [Models](guide/models.md): declaring documents, binding them, field markers +- **[Getting started](tutorial.md)**: MongoDB, a first model, a REST API and its tests, in one page +- [Models](guide/models.md): declaring documents, binding them, field markers; + [Choosing a backend](guide/backends.md) - [Queries](guide/queries.md) and [Writes and transactions](guide/writes.md) - [Relations](guide/relations.md), [Hooks and plugins](guide/plugins.md) -- [Running in production](guide/operations.md): indexes, observability, errors -- [Resources](web/resources.md) and [Plugs](web/plugs.md): the REST layer +- [Errors](guide/errors.md): what each one means and what a client sees +- [Running in production](guide/operations.md): indexes, observability +- [Resources](web/resources.md), [Authentication and authorization](web/auth.md), + [Testing your API](web/testing.md) and [Plugs](web/plugs.md): the REST layer - [Coming from Mongoose](parity.md), [Migrating from 0.1.x](migration.md) - [Benchmarks](benchmarks.md): measured against real MongoDB -Every example in the guide is executed by the test suite. +Every example in the guide is executed by the test suite, and so is everything in +[`examples/`](https://github.com/jaykv/mongospecs/tree/main/examples). ## What it looks like diff --git a/docs/reference/bulk.md b/docs/reference/bulk.md deleted file mode 100644 index e54108c..0000000 --- a/docs/reference/bulk.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs.bulk - -::: mongospecs.bulk diff --git a/docs/reference/errors.md b/docs/reference/errors.md deleted file mode 100644 index 4c9c04d..0000000 --- a/docs/reference/errors.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs.errors - -::: mongospecs.errors diff --git a/docs/reference/expr.md b/docs/reference/expr.md deleted file mode 100644 index 62e8f7e..0000000 --- a/docs/reference/expr.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs.expr - -::: mongospecs.expr diff --git a/docs/reference/hooks.md b/docs/reference/hooks.md deleted file mode 100644 index 19139a3..0000000 --- a/docs/reference/hooks.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs.hooks - -::: mongospecs.hooks diff --git a/docs/reference/indexes.md b/docs/reference/indexes.md deleted file mode 100644 index c6f622a..0000000 --- a/docs/reference/indexes.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs.indexes - -::: mongospecs.indexes diff --git a/docs/reference/manager.md b/docs/reference/manager.md deleted file mode 100644 index a810f89..0000000 --- a/docs/reference/manager.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs.manager - -::: mongospecs.manager diff --git a/docs/reference/markers.md b/docs/reference/markers.md deleted file mode 100644 index 052b33c..0000000 --- a/docs/reference/markers.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs.markers - -::: mongospecs.markers diff --git a/docs/reference/model.md b/docs/reference/model.md deleted file mode 100644 index b27b3db..0000000 --- a/docs/reference/model.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs.model - -::: mongospecs.model diff --git a/docs/reference/query.md b/docs/reference/query.md deleted file mode 100644 index 705ce75..0000000 --- a/docs/reference/query.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs.query - -::: mongospecs.query diff --git a/docs/reference/refs.md b/docs/reference/refs.md deleted file mode 100644 index 4cddce7..0000000 --- a/docs/reference/refs.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs.refs - -::: mongospecs.refs diff --git a/docs/reference/registry.md b/docs/reference/registry.md deleted file mode 100644 index 0a4ef3b..0000000 --- a/docs/reference/registry.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs.registry - -::: mongospecs.registry diff --git a/docs/reference/relations.md b/docs/reference/relations.md deleted file mode 100644 index 0197c5f..0000000 --- a/docs/reference/relations.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs.relations - -::: mongospecs.relations diff --git a/docs/reference/streams.md b/docs/reference/streams.md deleted file mode 100644 index 5efd0d1..0000000 --- a/docs/reference/streams.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs.streams - -::: mongospecs.streams diff --git a/docs/reference/updates.md b/docs/reference/updates.md deleted file mode 100644 index 67328f7..0000000 --- a/docs/reference/updates.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs.updates - -::: mongospecs.updates diff --git a/docs/reference/web_api.md b/docs/reference/web_api.md deleted file mode 100644 index fe84991..0000000 --- a/docs/reference/web_api.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs_web.api - -::: mongospecs_web.api diff --git a/docs/reference/web_backend.md b/docs/reference/web_backend.md deleted file mode 100644 index 073ed9e..0000000 --- a/docs/reference/web_backend.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs_web.backend - -::: mongospecs_web.backend diff --git a/docs/reference/web_errors.md b/docs/reference/web_errors.md deleted file mode 100644 index d43683b..0000000 --- a/docs/reference/web_errors.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs_web.errors - -::: mongospecs_web.errors diff --git a/docs/reference/web_filters.md b/docs/reference/web_filters.md deleted file mode 100644 index a381d9e..0000000 --- a/docs/reference/web_filters.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs_web.filters - -::: mongospecs_web.filters diff --git a/docs/reference/web_openapi.md b/docs/reference/web_openapi.md deleted file mode 100644 index 4ef594c..0000000 --- a/docs/reference/web_openapi.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs_web.openapi - -::: mongospecs_web.openapi diff --git a/docs/reference/web_policy.md b/docs/reference/web_policy.md deleted file mode 100644 index 56aef52..0000000 --- a/docs/reference/web_policy.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs_web.policy - -::: mongospecs_web.policy diff --git a/docs/reference/web_resource.md b/docs/reference/web_resource.md deleted file mode 100644 index 8a70204..0000000 --- a/docs/reference/web_resource.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs_web.resource - -::: mongospecs_web.resource diff --git a/docs/reference/web_testing.md b/docs/reference/web_testing.md deleted file mode 100644 index 03cc34a..0000000 --- a/docs/reference/web_testing.md +++ /dev/null @@ -1,3 +0,0 @@ -# mongospecs_web.testing - -::: mongospecs_web.testing diff --git a/docs/tutorial.md b/docs/tutorial.md new file mode 100644 index 0000000..55525fe --- /dev/null +++ b/docs/tutorial.md @@ -0,0 +1,122 @@ +# Getting started + +This page takes you from nothing to a working model, and then to a REST API over it. Every file +shown here is in [`examples/`](https://github.com/jaykv/mongospecs/tree/main/examples) and is run +by the test suite. + +## 1. What you need + +- Python 3.11 or newer. +- MongoDB 7.0 or newer, running as a **replica set**. A single node is enough. Transactions, + cascading deletes and change streams do not work on a standalone `mongod`; everything else does. + +If you have Docker, this starts a one-node replica set on `localhost:27017`: + +```yaml title="compose.yaml" +--8<-- "examples/compose.yaml" +``` + +``` +docker compose -f compose.yaml up -d --wait +``` + +MongoDB Atlas works too (every Atlas cluster is a replica set): use its connection string +wherever you see `mongodb://localhost:27017/...` below. + +## 2. Install + +Pick the model library you want to write your models in. If you have no preference, start with +msgspec; [Choosing a backend](guide/backends.md) compares the three. + +``` +pip install "mongospecs[msgspec]" # or [pydantic], or [attrs] +``` + +## 3. A model, and what you can do with it + +```python title="quickstart.py" +--8<-- "examples/quickstart.py" +``` + +``` +$ python quickstart.py +Hello, world by ann +tags: ['intro', 'published'] +``` + +What happened there: + +- **A model is a class.** `Spec` is the base for the backend you chose; everything else comes from + `mongospecs` and is the same on all three. Every document has an `id`, stored as `_id`. +- **A `Registry` owns the connection.** Models do nothing until they are in an open registry. + `async with` opens it and closes it; in a web application you open it once at startup. +- **Nothing is built implicitly.** `apply_indexes()` creates the indexes you declared. +- **`save()` sends only what changed**, and `changes()` shows you what that is. +- **Queries are typed.** `Q.views >= 0` checks that `views` is a field and casts the value; a + misspelt name raises `QueryError` instead of matching nothing. +- **A `Ref` is not loaded until you ask**: `.include("author")` loads it with the query. +- **`Model.update(id, ...)` is one atomic command** and returns the updated document. +- **A transaction is a function.** It may run more than once, and your in-memory documents are + put back if it aborts. + +Each of these has a page: [Models](guide/models.md), [Queries](guide/queries.md), +[Writes and transactions](guide/writes.md), [Relations](guide/relations.md). + +!!! note "Top-level await" + The guide pages use `await` at the top level to stay short. In your own code that goes inside + an `async def`, as in the file above, or straight into `python -m asyncio`. + +## 4. A REST API over the model + +`mongospecs-web` describes an API as data and mounts it on plain ASGI, Starlette, FastAPI, +Litestar or Quart. This example uses plain ASGI, so the only other thing to install is a server. + +``` +pip install mongospecs-web uvicorn +``` + +```python title="app.py" +--8<-- "examples/todo_api/app.py" +``` + +``` +$ uvicorn app:app +$ curl -s -X POST localhost:8000/api/tasks -H 'authorization: Bearer ann-token' \ + -H 'content-type: application/json' -d '{"title": "Write docs"}' +{"data":{"id":"...","title":"Write docs","done":false,"owner":"ann","created_at":"...","rev":0}} +$ curl -s 'localhost:8000/api/tasks?done=false&sort=-title' -H 'authorization: Bearer ann-token' +$ curl -s localhost:8000/api/openapi.json +``` + +The parts: + +- **`Policy` on each field is the contract.** It says whether a client may set the field, change + it, see it, filter by it and sort by it. `owner` cannot be written by clients; `note_to_self` + never leaves the server. +- **`authenticate`** turns a request into a principal (anything you like), or `None`. +- **Guards** decide who may call an operation; **`scope`** decides which rows they may touch, and + is part of every query, so Bob gets a 404 for Ann's task rather than a 403. +- **`@tasks.op`** adds an operation of your own next to the built-in ones. +- **`rev`** is a `Revision` field, so responses carry an `ETag` and writes honour `If-Match`. + +More in [Resources](web/resources.md) and +[Authentication and authorization](web/auth.md). + +## 5. Test it + +`api.handle()` runs a request through routing, authentication, guards, validation and the +database, without a server: + +```python title="test_app.py" +--8<-- "examples/todo_api/test_app.py" +``` + +[Testing your API](web/testing.md) covers the other ways, including an in-memory backend. + +## Where next + +- Coming from Mongoose? [The parity table](parity.md) maps what you know. +- Coming from mongospecs 0.1? [Migrating from 0.1.x](migration.md). +- Going to production? [Running in production](guide/operations.md) and + [Tuning for production](guide/production.md). +- Something raised an error you do not recognise? [Errors](guide/errors.md). diff --git a/docs/web/auth.md b/docs/web/auth.md new file mode 100644 index 0000000..3a67f19 --- /dev/null +++ b/docs/web/auth.md @@ -0,0 +1,168 @@ +# Authentication and authorization + +Four things decide what a request may do. They are separate on purpose: each answers one question. + +| | Question | Where | +|---|---|---| +| `authenticate` | Who is calling? | `Api(authenticate=...)` | +| Guards | May they call this operation at all? | `Resource(guards=...)`, `@resource.op(guards=...)` | +| `scope` | Which documents may they see and change? | `Resource(scope=...)` | +| `Policy` | Which fields may they set, change and see? | on each field of the model | + +```python +from typing import Annotated, NamedTuple + +from mongospecs import Policy, Q, Registry, SpecConfig +from mongospecs.msgspec import Spec +from mongospecs_web import Api, Create, Delete, Forbidden, Get, List, Patch, Resource, Unauthorized + + +class Report(Spec): + __spec__ = SpecConfig(collection="reports") + title: str = "" + team: Annotated[str, Policy(insert=False, update=False)] = "" # set by the server + approved: Annotated[bool, Policy(insert=False, update=False)] = False # changed only by an operation + review_notes: Annotated[str | None, Policy(private=True)] = None # never sent to clients +``` + +## Who is calling: `authenticate` + +`authenticate(request)` returns the principal, which can be anything: a user id, a dataclass, a +model. It may be `async`. Return `None` for an anonymous request and let guards decide whether that +is allowed; raise `Unauthorized` when credentials are present but wrong. + +```python +class Principal(NamedTuple): + name: str + team: str + admin: bool = False + + +SESSIONS = {"t-ann": Principal("ann", "red"), "t-bob": Principal("bob", "blue"), "t-root": Principal("root", "red", True)} + + +async def authenticate(request): + token = request.headers.get("authorization", "").removeprefix("Bearer ") + if not token: + return None + if token not in SESSIONS: # look the token up in your session store, or verify a JWT here + raise Unauthorized("Unknown token.", headers={"www-authenticate": "Bearer"}) + return SESSIONS[token] +``` + +`request` is the framework-neutral `mongospecs_web.Request`; `request.raw` is your framework's own +request object when you need it (a session cookie, `request.state`). + +## May they call this: guards + +A guard is `guard(ctx) -> bool`, sync or async. Returning false is a 403; raise `Unauthorized` for +a 401. `ctx.principal` is what `authenticate` returned. + +```python +def logged_in(ctx): + if ctx.principal is None: + raise Unauthorized("Log in first.", headers={"www-authenticate": "Bearer"}) + return True + + +def is_admin(ctx): + return ctx.principal is not None and ctx.principal.admin +``` + +`guards=[...]` applies to every operation; `guards={Delete: is_admin}` to the ones named. A key +must name an operation the resource has, so a misspelt one is an error and not an open door. + +## Which documents: `scope` + +`scope(ctx)` returns a condition that is ANDed into list, get, patch, put and delete, in the +database query itself. A document outside the scope does not exist for that caller: they get a +404, not a 403, and it never appears in a list or a count. Return `None` for "no restriction". + +```python +def own_team(ctx): + return None if ctx.principal.admin else Q.team == ctx.principal.team + + +reports = Resource( + Report, + guards={List: logged_in, Get: logged_in, Create: logged_in, Patch: logged_in, Delete: [logged_in, is_admin]}, + scope=own_team, + server_fields=lambda ctx: {"team": ctx.principal.team}, # the server, not the client, says whose it is +) +``` + +`server_fields` is the other half: on create it sets the fields the scope filters on, so a client +cannot create a document in someone else's scope. Those fields stay closed to clients with +`Policy(insert=False, update=False)`. + +## Which fields: `Policy` + +| On a field | A client can | +|---|---| +| nothing | set it on create, change it, read it | +| `Policy(update=False)` | set it once, on create | +| `Policy(insert=False, update=False)` | only read it | +| `Policy(private=True)` | not see it, set it, filter or sort by it | +| `Policy(private=True, insert=True)` | send it on create and never read it back (a password) | +| `Policy(filter=("eq", "gte"), sort=True)` | use it in the query string | + +The same applies to the fields of embedded models, at any depth. A request that names a field it +may not write is rejected as a whole with a 422. If you would rather list what is writable than +what is not, `Resource(Model, explicit_writes=True)` makes every field read-only unless its policy +says `insert=True` or `update=True`. + +## A change only some people may make: a custom operation + +A field clients cannot write is changed by an operation with its own guard. `ctx.load()` applies +the scope, so the guard and the scope still both hold. + +```python +@reports.op("approve", method="POST", guards=[logged_in, is_admin]) +async def approve(ctx): + report = await ctx.load() + report.approved = True + return await report.save() + + +api = Api(reports, prefix="/api", authenticate=authenticate) +registry = await Registry("mongodb://localhost/app", models=[Report]).open() +``` + +## All of it at work + +```python +import json + + +async def call(method, path, token=None, body=None): + headers = {"content-type": "application/json", **({"authorization": f"Bearer {token}"} if token else {})} + response = await api.handle(method, path, headers=headers, body=json.dumps(body).encode() if body else b"") + return response.status, json.loads(response.body) if response.body else None + + +assert (await call("GET", "/api/reports"))[0] == 401 # anonymous: the guard says no +assert (await call("GET", "/api/reports", "t-nobody"))[0] == 401 # bad credentials: authenticate says no + +status, created = await call("POST", "/api/reports", "t-ann", {"title": "Q3"}) +url = f"/api/reports/{created['data']['id']}" +assert status == 201 and created["data"]["team"] == "red" # server_fields +assert "review_notes" not in created["data"] # private + +assert (await call("GET", url, "t-bob"))[0] == 404 # another team: outside the scope +assert (await call("PATCH", url, "t-ann", {"approved": True}))[0] == 422 # the policy closes the field +assert (await call("POST", f"{url}/approve", "t-ann"))[0] == 403 # logged in, but not an admin +assert (await call("POST", f"{url}/approve", "t-root"))[1]["data"]["approved"] is True +assert (await call("DELETE", url, "t-ann"))[0] == 403 and (await call("DELETE", url, "t-root"))[0] == 204 + +await registry.client.drop_database("app") +await registry.close() +``` + +## Checklist + +- Every write operation has a guard, including `Put` if you list it and every custom operation. +- Every field the scope filters on is set by `server_fields` and closed with + `Policy(insert=False, update=False)`. +- Secrets are `Policy(private=True)`, and `Deferred` too if they should not even be loaded. +- `Api(openapi=False)` if the API description itself should not be public. +- `Forbidden` and `Unauthorized` can be raised from anywhere inside an operation. diff --git a/docs/web/testing.md b/docs/web/testing.md new file mode 100644 index 0000000..ad1bd04 --- /dev/null +++ b/docs/web/testing.md @@ -0,0 +1,135 @@ +# Testing your API + +There are three ways to test a resource, from fastest to most complete. All of them run your real +guards, scope, field policy and validation; they differ in what stands behind them. + +| | Runs | Needs | +|---|---|---| +| `api.handle(...)` with `FakeBackend` | routing, auth, policy, validation, your operations | nothing | +| `api.handle(...)` with MongoDB | all of that, plus real queries, indexes and transactions | MongoDB | +| An HTTP client on the mounted app | all of that, plus the framework plug | MongoDB | + +```python +import json +from typing import Annotated + +from mongospecs import Index, Policy, Q, Registry, Revision, SpecConfig +from mongospecs.msgspec import Spec +from mongospecs_web import Api, Resource +from mongospecs_web.testing import FakeBackend + + +class Task(Spec): + __spec__ = SpecConfig(collection="tasks", indexes=[Index("owner", "title", unique=True)]) + title: Annotated[str, Policy(filter=("eq",), sort=True)] + done: bool = False + owner: Annotated[str, Policy(insert=False, update=False)] = "" + rev: Annotated[int, Revision] = 0 + + +def build_api(backend=None): + tasks = Resource( + Task, + backend=backend, # None: MongoDB + scope=lambda ctx: Q.owner == ctx.principal, + server_fields=lambda ctx: {"owner": ctx.principal}, + ) + return Api(tasks, prefix="/api", authenticate=lambda request: request.headers.get("x-user")) + + +async def call(api, method, path, body=None, user="ann", **headers): + response = await api.handle( + method, + path, + headers={"x-user": user, "content-type": "application/json", **headers}, + body=json.dumps(body).encode() if body is not None else b"", + ) + return response.status, json.loads(response.body) if response.body else None, response.headers +``` + +Build the `Api` in a function, as above, so tests can make one of their own. + +## Without a database: `FakeBackend` + +`FakeBackend(Model)` keeps documents in memory. No registry is opened and nothing is connected, so +these tests run anywhere and in microseconds. Ids are sequential, which makes them predictable. + +```python +store = FakeBackend(Task, unique=["title"]) +api = build_api(store) + +status, created, headers = await call(api, "POST", "/api/tasks", {"title": "Write docs"}) +assert status == 201 and created["data"]["owner"] == "ann" +assert (await call(api, "GET", headers["location"], user="bob"))[0] == 404 # the scope applies +assert (await call(api, "PATCH", headers["location"], {"owner": "bob"}))[0] == 422 # so does the policy +assert (await call(api, "POST", "/api/tasks", {"title": "Write docs"}))[0] == 409 # unique=["title"] +assert [task.title for task in store.docs.values()] == ["Write docs"] # look inside directly +``` + +What it does not do: it evaluates the filters a resource can produce, not arbitrary queries; it +has no indexes beyond `unique=`, no transactions and no relations to `include`. A custom operation +that calls `Task.find(...)` itself talks to MongoDB and needs the next kind of test. + +## With MongoDB: `api.handle` + +The same calls against a real database. Open a registry for the test and drop its database +afterwards; a registry cannot be reopened once closed, so each test (or session) makes its own. + +```python +registry = await Registry("mongodb://localhost/app", models=[Task]).open() +await registry.apply_indexes() +api = build_api() + +status, created, headers = await call(api, "POST", "/api/tasks", {"title": "Write docs"}) +assert status == 201 and headers["etag"] == '"0"' + +# things only the database decides: the unique index, the revision check +assert (await call(api, "POST", "/api/tasks", {"title": "Write docs"}))[0] == 409 +url = headers["location"] +assert (await call(api, "PATCH", url, {"done": True}, **{"if-match": '"0"'}))[0] == 200 +assert (await call(api, "PATCH", url, {"done": False}, **{"if-match": '"0"'}))[0] == 412 + +# and the documents are there to be checked with the model +assert (await Task.find(Q.owner == "ann").one()).done is True +``` + +With pytest and `pytest-asyncio` that becomes a fixture: + +```py +@pytest_asyncio.fixture +async def api(): + async with Registry(TEST_URI, database=f"test_{uuid.uuid4().hex[:8]}", models=[Task]) as registry: + await registry.apply_indexes() + yield build_api() + await registry.client.drop_database(registry.database().name) +``` + +## Through the framework: an HTTP client + +To test what a real client sees, including your framework's own routing and middleware, mount the +API and drive it with `httpx`. No server process is needed. + +```python +import httpx + +async with httpx.AsyncClient(transport=httpx.ASGITransport(app=api.asgi()), base_url="http://test") as client: + listed = await client.get("/api/tasks", params={"sort": "-title"}, headers={"x-user": "ann"}) + assert listed.status_code == 200 and [t["title"] for t in listed.json()["data"]] == ["Write docs"] + assert (await client.get("/api/openapi.json")).json()["info"]["title"] == "API" + +await registry.client.drop_database("app") +await registry.close() +``` + +Replace `api.asgi()` with your Starlette, FastAPI, Litestar or Quart application. + +## What to assert + +- **Authorization, from the outside in**: anonymous, the wrong user, the right user, for each + operation. A document outside the caller's scope is a 404. +- **The field policy**: a write that names a closed field is a 422 and changes nothing. +- **Errors**: responses are `application/problem+json`; field errors are in `errors`. +- **Concurrency**, where it matters: two writers with the same `If-Match`, one of which gets 412. + +If you write a plug for another framework, `mongospecs_web.testing.run_contract` checks it against +the reference behaviour; see [Plugs](plugs.md). diff --git a/examples/compose.yaml b/examples/compose.yaml new file mode 100644 index 0000000..cd88962 --- /dev/null +++ b/examples/compose.yaml @@ -0,0 +1,25 @@ +# MongoDB for the examples: a single-node replica set on localhost:27017. +# Transactions and change streams need a replica set; one node is enough. +# docker compose -f examples/compose.yaml up -d --wait +name: mongospecs-examples +services: + mongo: + image: mongo:8.0 + command: ["mongod", "--replSet", "rs0", "--bind_ip_all"] + ports: + - "27017:27017" + volumes: + - data:/data/db + healthcheck: + test: + - CMD-SHELL + - > + mongosh --quiet --eval + "try { if (!db.hello().isWritablePrimary) throw 1 } catch (e) { + try { rs.initiate({_id: 'rs0', members: [{_id: 0, host: 'localhost:27017'}]}) } catch (e2) {} + quit(1) }" + interval: 2s + timeout: 10s + retries: 60 +volumes: + data: diff --git a/examples/quickstart.py b/examples/quickstart.py new file mode 100644 index 0000000..9fcbdf8 --- /dev/null +++ b/examples/quickstart.py @@ -0,0 +1,65 @@ +"""mongospecs in one file. Start MongoDB (see compose.yaml), then: python examples/quickstart.py""" + +import asyncio +import os +from datetime import datetime +from typing import Annotated + +from mongospecs import CreatedAt, Inc, Index, Q, Ref, Registry, Revision, SpecConfig, UpdatedAt +from mongospecs.msgspec import Spec # or mongospecs.pydantic / mongospecs.attrs + +URI = os.environ.get("MONGO_URI", "mongodb://localhost:27017/quickstart?replicaSet=rs0") + + +class User(Spec): + name: str = "" + + +class Article(Spec): + __spec__ = SpecConfig(collection="articles", indexes=[Index("title", unique=True)]) + + title: str + views: int = 0 + tags: list[str] = [] + author: Ref[User] | None = None + created_at: Annotated[datetime | None, CreatedAt] = None + updated_at: Annotated[datetime | None, UpdatedAt] = None + rev: Annotated[int, Revision] = 0 # optimistic locking + + +async def main(uri: str = URI) -> None: + async with Registry(uri, models=[User, Article]) as registry: + await registry.apply_indexes() + + ann = await User(name="ann").insert() + article = await Article(title="Hello", author=ann).insert() + + # change tracking: save() sends only what changed + article.title = "Hello, world" + assert article.changes() == {"$set": {"title": "Hello, world"}} + await article.save() + + # typed queries: names are checked, values are cast + popular = await Article.find(Q.views >= 0, Q.title.prefix("Hello")).sort("-views").limit(10) + assert [a.title for a in popular] == ["Hello, world"] + + # relations: one extra query, or a $lookup + with_author = await Article.find(Q.author == ann).include("author").one() + print(with_author.title, "by", with_author.author.doc.name) + + # atomic updates, always validated; the updated document comes back + article = await Article.update(article.id, Inc(views=1), tags=["intro"]) + assert article.views == 1 + + # transactions (these need a replica set): the callback is retried on transient + # errors, and in-memory documents are restored if it aborts + async def publish(tx): + article.tags.append("published") + await article.save(session=tx) + + await registry.transaction(publish) + print("tags:", (await Article.get(article.id)).tags) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/test_quickstart.py b/examples/test_quickstart.py new file mode 100644 index 0000000..27cfe14 --- /dev/null +++ b/examples/test_quickstart.py @@ -0,0 +1,55 @@ +"""The quickstart is run as written, and the README shows exactly that file.""" + +import importlib.util +import os +import pathlib +import re +import uuid + +HERE = pathlib.Path(__file__).resolve().parent +ROOT = HERE.parent + + +def load(): + spec = importlib.util.spec_from_file_location("quickstart_example", HERE / "quickstart.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +async def test_the_quickstart_runs(capsys): + from mongospecs import Registry + + base = os.environ.get("MONGOSPECS_TEST_URI", "mongodb://localhost:27117/?replicaSet=rs0") + host, _, query = base.partition("?") + db = f"quickstart_{uuid.uuid4().hex[:8]}" + uri = f"{host.rstrip('/')}/{db}?{query}" + try: + await load().main(uri) + finally: + async with Registry(uri) as registry: + await registry.client.drop_database(db) + assert capsys.readouterr().out == "Hello, world by ann\ntags: ['intro', 'published']\n" + + +def test_the_readmes_show_the_quickstart(): + source = (HERE / "quickstart.py").read_text() + for readme in (ROOT / "README.md", ROOT / "packages" / "mongospecs" / "README.md"): + blocks = re.findall(r"```python\n(.*?)```", readme.read_text(), re.S) + assert source in blocks, f"{readme} does not show examples/quickstart.py as it is" + + +def test_the_package_readme_is_the_root_readme(): + # PyPI shows the package README; it differs from the root one only in where links point. + root = (ROOT / "README.md").read_text() + package = (ROOT / "packages" / "mongospecs" / "README.md").read_text() + blob, tree = ( + "https://github.com/jaykv/mongospecs/blob/main/", + "https://github.com/jaykv/mongospecs/tree/main/", + ) + absolute = re.sub( + r"\]\((?!https?://|#)([^)]+)\)", + lambda m: f"]({tree if '.' not in m[1].rsplit('/', 1)[-1] else blob}{m[1]})", + root, + ) + assert package == absolute diff --git a/examples/todo_api/app.py b/examples/todo_api/app.py new file mode 100644 index 0000000..a3be7a7 --- /dev/null +++ b/examples/todo_api/app.py @@ -0,0 +1,72 @@ +"""A small REST API with mongospecs-web: per-user to-do lists. + + pip install "mongospecs[msgspec]" mongospecs-web uvicorn + docker compose -f examples/compose.yaml up -d --wait + uvicorn app:app --app-dir examples/todo_api + + curl -s -X POST localhost:8000/api/tasks -H 'authorization: Bearer ann-token' \ + -H 'content-type: application/json' -d '{"title": "Write docs"}' + curl -s 'localhost:8000/api/tasks?done=false&sort=-title' -H 'authorization: Bearer ann-token' +""" + +import os +from datetime import datetime +from typing import Annotated + +from mongospecs import CreatedAt, Index, Policy, Q, Registry, Revision, SpecConfig +from mongospecs.msgspec import Spec +from mongospecs_web import Api, Create, Delete, Get, List, Patch, Resource, Unauthorized + +URI = os.environ.get("MONGO_URI", "mongodb://localhost:27017/todo?replicaSet=rs0") + +# Stands in for your real user store or token verification. +TOKENS = {"ann-token": "ann", "bob-token": "bob"} + + +class Task(Spec, kw_only=True): + __spec__ = SpecConfig(collection="tasks", indexes=[Index("owner", "title")]) + + # Policy says what a client may do with each field. + title: Annotated[str, Policy(filter=("eq", "prefix"), sort=True)] + done: Annotated[bool, Policy(filter=("eq",))] = False + owner: Annotated[str, Policy(insert=False, update=False)] = "" # set by the server + note_to_self: Annotated[str | None, Policy(private=True)] = None # never leaves the server + created_at: Annotated[datetime | None, CreatedAt] = None + rev: Annotated[int, Revision] = 0 # the ETag, checked by If-Match + + +def authenticate(request): + """Who is calling: the principal, or None for an anonymous request.""" + scheme, _, token = request.headers.get("authorization", "").partition(" ") + if not token: + return None + if scheme != "Bearer" or token not in TOKENS: + raise Unauthorized("Unknown token.", headers={"www-authenticate": "Bearer"}) + return TOKENS[token] + + +def logged_in(ctx): + if ctx.principal is None: + raise Unauthorized("Log in first.", headers={"www-authenticate": "Bearer"}) + return True + + +tasks = Resource( + Task, + ops=[List, Get, Create, Patch, Delete], + guards=[logged_in], # who may call the operations + scope=lambda ctx: Q.owner == ctx.principal, # which rows they may see and change + server_fields=lambda ctx: {"owner": ctx.principal}, # values the server sets on create +) + + +@tasks.op("complete", method="POST", summary="Mark a task as done") +async def complete(ctx): + task = await ctx.load() # 404 unless the task is within the caller's scope + task.done = True + return await task.save() + + +registry = Registry(URI, models=[Task]) +api = Api(tasks, prefix="/api", title="To-do", authenticate=authenticate) +app = api.asgi(registry=registry) # opens the registry on startup and closes it on shutdown diff --git a/examples/todo_api/conftest.py b/examples/todo_api/conftest.py new file mode 100644 index 0000000..0a912e8 --- /dev/null +++ b/examples/todo_api/conftest.py @@ -0,0 +1,9 @@ +import os +import sys +import uuid + +# The tests import the application as a module, and point it at a database of their own. +sys.path.insert(0, os.path.dirname(__file__)) +_base = os.environ.get("MONGOSPECS_TEST_URI", "mongodb://localhost:27117/?replicaSet=rs0") +_host, _, _query = _base.partition("?") +os.environ.setdefault("MONGO_URI", f"{_host.rstrip('/')}/todo_test_{uuid.uuid4().hex[:8]}?{_query}") diff --git a/examples/todo_api/test_app.py b/examples/todo_api/test_app.py new file mode 100644 index 0000000..4a3b891 --- /dev/null +++ b/examples/todo_api/test_app.py @@ -0,0 +1,59 @@ +"""Testing the API without a server: api.handle() runs a request through the whole stack.""" + +import json + +import pytest_asyncio + +from app import URI, Task, api +from mongospecs import Registry + +ANN = {"authorization": "Bearer ann-token", "content-type": "application/json"} +BOB = {"authorization": "Bearer bob-token", "content-type": "application/json"} + + +@pytest_asyncio.fixture(autouse=True) +async def database(): + # A registry cannot be reopened once closed, so each test opens its own. + async with Registry(URI, models=[Task]) as registry: + yield + await registry.client.drop_database(registry.database().name) + + +async def call(method, path, body=None, headers=ANN): + response = await api.handle( + method, path, body=json.dumps(body).encode() if body is not None else b"", headers=headers + ) + return response.status, json.loads(response.body) if response.body else None, response.headers + + +async def test_create_list_and_complete(): + status, created, headers = await call("POST", "/api/tasks", {"title": "Write docs"}) + assert status == 201 and created["data"]["owner"] == "ann" and headers["etag"] == '"0"' + assert "note_to_self" not in created["data"] # private fields are never returned + + status, listed, _ = await call("GET", "/api/tasks") + assert [task["title"] for task in listed["data"]] == ["Write docs"] + + status, done, _ = await call("POST", f"{headers['location']}/complete") + assert status == 200 and done["data"]["done"] is True + + +async def test_each_user_sees_only_their_own_tasks(): + _, _, headers = await call("POST", "/api/tasks", {"title": "Ann's"}) + assert (await call("GET", headers["location"], headers=BOB))[0] == 404 + assert (await call("GET", "/api/tasks", headers=BOB))[1]["data"] == [] + assert (await call("GET", "/api/tasks", headers={}))[0] == 401 + + +async def test_the_field_policy_is_enforced(): + _, _, headers = await call("POST", "/api/tasks", {"title": "Mine"}) + status, problem, _ = await call("PATCH", headers["location"], {"owner": "bob"}) + assert status == 422 and problem["errors"] == [{"field": "owner", "message": "cannot be set on update"}] + assert (await call("PATCH", headers["location"], {"note_to_self": "x"}))[0] == 422 + + +async def test_a_stale_etag_is_refused(): + _, _, headers = await call("POST", "/api/tasks", {"title": "Mine"}) + fresh = {**ANN, "if-match": headers["etag"]} + assert (await call("PATCH", headers["location"], {"done": True}, fresh))[0] == 200 + assert (await call("PATCH", headers["location"], {"done": False}, fresh))[0] == 412 diff --git a/mkdocs.yaml b/mkdocs.yaml index 89a9134..a01efc2 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -1,4 +1,7 @@ site_name: "mongospecs" +site_url: "https://jaykv.github.io/mongospecs/" +repo_url: "https://github.com/jaykv/mongospecs" +edit_uri: "edit/main/docs/" theme: name: "material" @@ -51,6 +54,8 @@ markdown_extensions: emoji_generator: !!python/name:material.extensions.emoji.to_svg - pymdownx.magiclink - pymdownx.snippets: + base_path: ["."] + check_paths: true - pymdownx.superfences - pymdownx.tabbed: alternate_style: true @@ -62,61 +67,27 @@ markdown_extensions: plugins: search: - mkdocstrings: - handlers: - python: - paths: [packages/mongospecs/src, packages/mongospecs-web/src] - options: - docstring_section_style: list - filters: ["!^_"] - heading_level: 2 - merge_init_into_class: true - separate_signature: true - show_root_heading: false - show_signature_annotations: true - show_source: false - signature_crossrefs: true nav: - Home: index.md +- Getting started: tutorial.md - Guide: - Models: guide/models.md + - Choosing a backend: guide/backends.md - Queries: guide/queries.md - Writes and transactions: guide/writes.md - Relations: guide/relations.md - Hooks and plugins: guide/plugins.md + - Errors: guide/errors.md - Running in production: guide/operations.md - Tuning for production: guide/production.md - Web: - Resources: web/resources.md + - Authentication and authorization: web/auth.md + - Testing your API: web/testing.md - Plugs: web/plugs.md - Benchmarks: benchmarks.md - Coming from Mongoose: parity.md - Migrating from 0.1.x: - Guide: migration.md - v1 test inventory: v1-test-inventory.md -- Reference: - - mongospecs: - - registry: reference/registry.md - - model: reference/model.md - - manager: reference/manager.md - - query: reference/query.md - - expr: reference/expr.md - - updates: reference/updates.md - - bulk: reference/bulk.md - - markers: reference/markers.md - - refs: reference/refs.md - - relations: reference/relations.md - - indexes: reference/indexes.md - - hooks: reference/hooks.md - - streams: reference/streams.md - - errors: reference/errors.md - - mongospecs-web: - - resource: reference/web_resource.md - - api: reference/web_api.md - - backend: reference/web_backend.md - - filters: reference/web_filters.md - - policy: reference/web_policy.md - - errors: reference/web_errors.md - - openapi: reference/web_openapi.md - - testing: reference/web_testing.md diff --git a/packages/mongospecs-web/README.md b/packages/mongospecs-web/README.md index 707c92c..f3b80e9 100644 --- a/packages/mongospecs-web/README.md +++ b/packages/mongospecs-web/README.md @@ -50,5 +50,11 @@ That is a complete API: `GET /api/tasks?title[prefix]=Wri&sort=-title&limit=20`, - **Framework-neutral.** The same resource mounts on five frameworks through thin plugs, and a contract suite (`mongospecs_web.testing`) checks that they all behave identically. -Guides: [resources](https://github.com/jaykv/mongospecs/blob/main/docs/web/resources.md), -[plugs](https://github.com/jaykv/mongospecs/blob/main/docs/web/plugs.md). +Start with [Getting started](https://jaykv.github.io/mongospecs/tutorial/), which builds an API +like this one step by step. Guides: [resources](https://jaykv.github.io/mongospecs/web/resources/), +[authentication and authorization](https://jaykv.github.io/mongospecs/web/auth/), +[testing](https://jaykv.github.io/mongospecs/web/testing/), +[plugs](https://jaykv.github.io/mongospecs/web/plugs/). + +Responses are encoded with msgspec when it is installed (`mongospecs-web[msgspec]`), and with the +standard library otherwise. diff --git a/packages/mongospecs/README.md b/packages/mongospecs/README.md index 1dc7a52..53f67fa 100644 --- a/packages/mongospecs/README.md +++ b/packages/mongospecs/README.md @@ -6,10 +6,13 @@ The aim is Mongoose's production feature set (change tracking, middleware, popul discriminators, transactions that restore document state) with typed Python models, plus [`mongospecs-web`](https://github.com/jaykv/mongospecs/tree/main/packages/mongospecs-web), a framework-neutral REST layer. +**Documentation: .** Start with +[Getting started](https://jaykv.github.io/mongospecs/tutorial/). + > **Status: 0.2.0, alpha.** This is a rewrite of 0.1.x: async-only, Python 3.11+, MongoDB 7.0+, > and not backwards compatible. The synchronous 0.1.x line lives on the -> [`0.1.x` branch](https://github.com/jaykv/mongospecs/tree/0.1.x). The design and the work graph -> are in [`claudedocs/v2-plan.md`](https://github.com/jaykv/mongospecs/blob/main/claudedocs/v2-plan.md). +> [`0.1.x` branch](https://github.com/jaykv/mongospecs/tree/0.1.x). Changes are listed in the +> [changelog](https://github.com/jaykv/mongospecs/blob/main/CHANGELOG.md). ## Install @@ -17,15 +20,31 @@ discriminators, transactions that restore document state) with typed Python mode pip install "mongospecs[msgspec]" # or [pydantic], [attrs], [all] ``` +You also need MongoDB 7.0 or newer. Transactions, cascading deletes and change streams need it to +run as a replica set; a single node is enough. [`examples/compose.yaml`](https://github.com/jaykv/mongospecs/blob/main/examples/compose.yaml) +starts one on `localhost:27017`: + +``` +docker compose -f examples/compose.yaml up -d --wait +``` + ## A taste +This is [`examples/quickstart.py`](https://github.com/jaykv/mongospecs/blob/main/examples/quickstart.py), as it is run by the test suite. + ```python +"""mongospecs in one file. Start MongoDB (see compose.yaml), then: python examples/quickstart.py""" + +import asyncio +import os from datetime import datetime from typing import Annotated from mongospecs import CreatedAt, Inc, Index, Q, Ref, Registry, Revision, SpecConfig, UpdatedAt from mongospecs.msgspec import Spec # or mongospecs.pydantic / mongospecs.attrs +URI = os.environ.get("MONGO_URI", "mongodb://localhost:27017/quickstart?replicaSet=rs0") + class User(Spec): name: str = "" @@ -43,8 +62,8 @@ class Article(Spec): rev: Annotated[int, Revision] = 0 # optimistic locking -async def main(): - async with Registry("mongodb://localhost/app", models=[User, Article]) as registry: +async def main(uri: str = URI) -> None: + async with Registry(uri, models=[User, Article]) as registry: await registry.apply_indexes() ann = await User(name="ann").insert() @@ -56,24 +75,34 @@ async def main(): await article.save() # typed queries: names are checked, values are cast - popular = await Article.find(Q.views >= 100, Q.title.prefix("Hello")).sort("-views").limit(10) + popular = await Article.find(Q.views >= 0, Q.title.prefix("Hello")).sort("-views").limit(10) + assert [a.title for a in popular] == ["Hello, world"] # relations: one extra query, or a $lookup with_author = await Article.find(Q.author == ann).include("author").one() - print(with_author.author.doc.name) + print(with_author.title, "by", with_author.author.doc.name) # atomic updates, always validated; the updated document comes back article = await Article.update(article.id, Inc(views=1), tags=["intro"]) + assert article.views == 1 - # transactions: the callback is retried on transient errors, and in-memory - # documents are restored if it aborts + # transactions (these need a replica set): the callback is retried on transient + # errors, and in-memory documents are restored if it aborts async def publish(tx): article.tags.append("published") await article.save(session=tx) await registry.transaction(publish) + print("tags:", (await Article.get(article.id)).tags) + + +if __name__ == "__main__": + asyncio.run(main()) ``` +A REST API over a model is a few more lines; see [`examples/todo_api`](https://github.com/jaykv/mongospecs/tree/main/examples/todo_api) and the +[web guide](https://jaykv.github.io/mongospecs/web/resources/). + ## What is in the box | | | @@ -96,7 +125,7 @@ async def main(): oversell, multi-item orders that are all-or-nothing, a change-stream projector that must count each sale exactly once across restarts, and orders that stay consistent across a primary step-down. -- Every example in the guide pages is executed. +- Every example in the guide pages and in `examples/` is executed. - Benchmarks and stress scenarios in `bench/` run against real MongoDB; results and the conclusions drawn from them are in `docs/benchmarks.md`. @@ -110,4 +139,6 @@ uv run mongospecs-bench run # benchmarks; `run LOAD` for the HTTP load ``` The repository is a uv workspace: `packages/mongospecs`, `packages/mongospecs-web` and `bench/` -(benchmarks and stress scenarios against real MongoDB, never published). +(benchmarks and stress scenarios against real MongoDB, never published). See +[CONTRIBUTING.md](https://github.com/jaykv/mongospecs/blob/main/CONTRIBUTING.md); the design decisions behind 0.2 are recorded in +[`claudedocs/v2-plan.md`](https://github.com/jaykv/mongospecs/blob/main/claudedocs/v2-plan.md). diff --git a/pyproject.toml b/pyproject.toml index 09120d4..840907e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,12 +32,11 @@ dev = [ ] docs = [ "mkdocs-material>=9.5", - "mkdocstrings[python]>=0.26", ] [tool.pytest.ini_options] addopts = "--import-mode=importlib -ra" -testpaths = ["packages/mongospecs/tests", "packages/mongospecs-web/tests", "bench/tests"] +testpaths = ["packages/mongospecs/tests", "packages/mongospecs-web/tests", "bench/tests", "examples"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "session" asyncio_default_test_loop_scope = "session" @@ -49,7 +48,7 @@ markers = [ [tool.ruff] line-length = 110 target-version = "py311" -src = ["packages/mongospecs/src", "packages/mongospecs-web/src", "bench/src"] +src = ["packages/mongospecs/src", "packages/mongospecs-web/src", "bench/src", "examples/todo_api"] [tool.ruff.lint] select = ["E", "F", "I", "B", "UP", "ASYNC", "RUF"] @@ -57,6 +56,7 @@ ignore = ["E501", "RUF012"] [tool.ruff.lint.per-file-ignores] "**/tests/**" = ["B", "RUF"] +"examples/**" = ["B", "RUF"] # Benchmarks define closures inside loops and call them before the next iteration. "bench/**" = ["B023"] diff --git a/uv.lock b/uv.lock index f9291a1..3850a74 100644 --- a/uv.lock +++ b/uv.lock @@ -401,15 +401,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034 }, ] -[[package]] -name = "griffelib" -version = "2.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2b/27/b55f1a5278918be765fb2fd8b20966bc72bbdd3f789f031937cceea7834a/griffelib-2.3.2.tar.gz", hash = "sha256:df00c7a0dee3d86268d76788997a1859272cb1fb7b865658e043d2c0c3d52e60", size = 234729 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/e5/0ae74c83c1cab2c14daadf28bd0143eb14bb81503834af987b66badc7b61/griffelib-2.3.2-py3-none-any.whl", hash = "sha256:8e710afededd5607f95bf3c8ccc175ee306e7084459faf27b9f9f44ef2a7a274", size = 169420 }, -] - [[package]] name = "h11" version = "0.16.0" @@ -953,20 +944,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/22/5b/dbc6a8cddc9cfa9c4971d59fb12bb8d42e161b7e7f8cc89e49137c5b279c/mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e", size = 3864451 }, ] -[[package]] -name = "mkdocs-autorefs" -version = "1.4.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown" }, - { name = "markupsafe" }, - { name = "mkdocs" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/52/c0/f641843de3f612a6b48253f39244165acff36657a91cc903633d456ae1ac/mkdocs_autorefs-1.4.4.tar.gz", hash = "sha256:d54a284f27a7346b9c38f1f852177940c222da508e66edc816a0fa55fc6da197", size = 56588 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/28/de/a3e710469772c6a89595fc52816da05c1e164b4c866a89e3cb82fb1b67c5/mkdocs_autorefs-1.4.4-py3-none-any.whl", hash = "sha256:834ef5408d827071ad1bc69e0f39704fa34c7fc05bc8e1c72b227dfdc5c76089", size = 25530 }, -] - [[package]] name = "mkdocs-get-deps" version = "0.2.2" @@ -1012,42 +989,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/54/662a4743aa81d9582ee9339d4ffa3c8fd40a4965e033d77b9da9774d3960/mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31", size = 8728 }, ] -[[package]] -name = "mkdocstrings" -version = "1.0.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jinja2" }, - { name = "markdown" }, - { name = "markupsafe" }, - { name = "mkdocs" }, - { name = "mkdocs-autorefs" }, - { name = "pymdown-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f5/e8/445b19834bf9559ebbbaa76897e965665b83d2861461eed8d8de4d122057/mkdocstrings-1.0.7.tar.gz", hash = "sha256:0951b147f352118a45a3db85f0f3b660833d34cb5120a2751315b8d08dafd159", size = 103739 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/c3/9062531735ff1fd27f3a79c353c4cf4d822eeffd062fd90b2c553a96f4bb/mkdocstrings-1.0.7-py3-none-any.whl", hash = "sha256:9563828667eeb35256f87a6b61fe857676ac628ec1abbce9aea9839a502c9cb1", size = 41597 }, -] - -[package.optional-dependencies] -python = [ - { name = "mkdocstrings-python" }, -] - -[[package]] -name = "mkdocstrings-python" -version = "2.0.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "griffelib" }, - { name = "mkdocs-autorefs" }, - { name = "mkdocstrings" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/96/11/640067d1ad713ea99c233e53e57a4c300d9885f904a61371d7ef90dea549/mkdocstrings_python-2.0.9.tar.gz", hash = "sha256:ae945637dc0618c6beedbee169f10c0234b77f322c53db3fc9fb4c29b3013038", size = 203006 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/0f/f7f926cfa06317bbdbd3cf0ff82f9e3a91767352eb01519c81499fddf575/mkdocstrings_python-2.0.9-py3-none-any.whl", hash = "sha256:c0233eff3f84d78110df50541918e7ec2bcff8e1614faddea34290a90321a416", size = 105698 }, -] - [[package]] name = "mongospecs" version = "0.2.0" @@ -1186,7 +1127,6 @@ dev = [ ] docs = [ { name = "mkdocs-material" }, - { name = "mkdocstrings", extra = ["python"] }, ] [package.metadata] @@ -1205,10 +1145,7 @@ dev = [ { name = "ruff", specifier = ">=0.8" }, { name = "sentry-sdk", specifier = ">=2.0" }, ] -docs = [ - { name = "mkdocs-material", specifier = ">=9.5" }, - { name = "mkdocstrings", extras = ["python"], specifier = ">=0.26" }, -] +docs = [{ name = "mkdocs-material", specifier = ">=9.5" }] [[package]] name = "msgspec" From c23ffbc92d610a5a13944e6a2910e25199caef81 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 10:33:35 -0400 Subject: [PATCH 24/26] chore: changelog, contributing and security files; publish the docs - CHANGELOG.md, CONTRIBUTING.md and SECURITY.md. - A workflow deploys the documentation to GitHub Pages from main. - The publish workflow refuses a release tag that does not match the version in the package it names. - Package metadata links to the documentation and the changelog. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/docs.yaml | 24 ++++++++++++ .github/workflows/python-publish.yaml | 9 +++++ CHANGELOG.md | 49 +++++++++++++++++++++++++ CONTRIBUTING.md | 51 ++++++++++++++++++++++++++ SECURITY.md | 24 ++++++++++++ packages/mongospecs-web/pyproject.toml | 2 + packages/mongospecs/pyproject.toml | 2 + 7 files changed, 161 insertions(+) create mode 100644 .github/workflows/docs.yaml create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 SECURITY.md diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 0000000..64748f4 --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,24 @@ +# Publishes the documentation site to GitHub Pages (the gh-pages branch) from main. +name: Docs + +on: + push: + branches: [main] + paths: ["docs/**", "examples/**", "mkdocs.yaml", ".github/workflows/docs.yaml"] + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: docs + cancel-in-progress: true + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - run: uv sync --group docs --python 3.13 + - run: uv run --group docs mkdocs gh-deploy --strict --force diff --git a/.github/workflows/python-publish.yaml b/.github/workflows/python-publish.yaml index 66a28d0..63221b0 100644 --- a/.github/workflows/python-publish.yaml +++ b/.github/workflows/python-publish.yaml @@ -42,5 +42,14 @@ jobs: mongospecs-v*) echo "package=mongospecs" >> "$GITHUB_OUTPUT" ;; *) echo "unrecognised release tag: $tag" >&2; exit 1 ;; esac + - name: The tag must be the version being published + run: | + package="${{ steps.pick.outputs.package }}" + declared="$(uv run --no-project python -c "import tomllib, sys; print(tomllib.load(open(sys.argv[1], 'rb'))['project']['version'])" "packages/$package/pyproject.toml")" + tagged="${GITHUB_REF_NAME#"$package"-v}" + if [ "$declared" != "$tagged" ]; then + echo "tag $GITHUB_REF_NAME says $tagged, but packages/$package/pyproject.toml says $declared" >&2 + exit 1 + fi - run: uv build --package ${{ steps.pick.outputs.package }} - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f7971e2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,49 @@ +# Changelog + +Two packages are released from this repository; each entry says which one it is about. Versions +follow semantic versioning, and while a package is below 1.0 a minor release may break +compatibility. Breaking changes are marked **Breaking**. + +## Unreleased + +### mongospecs 0.2.0 + +A rewrite. See [Migrating from 0.1.x](docs/migration.md) for the API map. + +- **Breaking:** async only, Python 3.11+, MongoDB 7.0+. The synchronous 0.1.x line stays on the + `0.1.x` branch and as `mongospecs<0.2`. +- **Breaking:** models are bound to an explicit `Registry`; there is no class-level client. +- **Breaking:** msgspec is no longer installed implicitly; choose an extra (`[msgspec]`, + `[pydantic]`, `[attrs]`). +- Change tracking by snapshot: `save()` sends only what changed. +- Typed, immutable queries with `Q`; `tracked`, `lean()` and `raw()` read modes; cursor and offset + pagination. +- Atomic update operators, real upsert, `bulk_write`, `bulk_save`, partial-failure `insert_many`. +- Relations: `Ref`, `Backref`, `include()`, cascade rules, `$graphLookup`, `$unionWith`. +- Document hooks and plugins; timestamps, soft delete and revisions by field marker. +- Transactions as a retried callback that restores in-memory documents when it aborts. +- Index planning with gated destructive actions; change streams; per-command observability events + with logging, OpenTelemetry and Sentry listeners. +- Document hooks run for `insert_many` and `bulk_save`. +- A single-field update on a pydantic model runs the field's validators and not the model-level + ones, which could only see the other fields' defaults. + +### mongospecs-web 0.1.0 + +First release. + +- Resources described as data and mounted on plain ASGI, Starlette, FastAPI, Litestar or Quart. +- Field `Policy` drives the create, patch, put, read, filter and sort surface; guards and row + scope for authorization; RFC 9457 errors; ETag and `If-Match`; OpenAPI 3.1. +- `PATCH` merges into embedded objects (RFC 7396); `Resource(merge_patch=False)` makes it replace + them. `PUT` is available as the `Put` operation. + +Changes made after the first review builds, for anyone who tried those: + +- **Breaking:** `?total=true` is opt-in per resource (`page=Cursor(total=True)`). +- **Breaking:** `PATCH` on an embedded object is partial. Send `null` for keys to clear, or set + `merge_patch=False`. +- **Breaking:** `If-Match` on a model without a `Revision` field is answered with 412. +- **Breaking:** a filter or sort key given twice is a 400, as are `NaN` and `Infinity` in a body. +- **Breaking:** a guard or hook registered for an operation the resource does not have raises + `ValueError`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1ea1ddc --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,51 @@ +# Contributing + +## Setting up + +You need [uv](https://docs.astral.sh/uv/) and Docker. + +``` +make install # uv sync for the whole workspace +make up # MongoDB in Docker: a replica set on :27117 and a sharded cluster on :27118 +make check # ruff, mypy --strict, and the whole test suite +make down # stop MongoDB and delete its data +``` + +Tests run against real MongoDB; nothing is mocked. `make test-fast` leaves out the tests that +need the sharded cluster. To use another server, set `MONGOSPECS_TEST_URI` (a replica set) and +`MONGOSPECS_TEST_SHARDED_URI`. + +## Where things are + +| Path | What | +|---|---| +| `packages/mongospecs` | the ODM | +| `packages/mongospecs-web` | the REST layer and its framework plugs | +| `bench` | benchmarks, stress scenarios and the storefront reference application; never published | +| `examples` | the files shown in the tutorial; they are tests too | +| `docs` | the documentation site (`make docs` builds it) | +| `claudedocs/v2-plan.md` | the design of 0.2 and the record of every decision that departed from it | + +## What a change needs + +- **A test that fails without it.** For a bug, reproduce it first. Tests for behaviour that + differs by model library are parametrised over `msgspec`, `pydantic` and `attrs`. +- **Docs for anything a user can see.** `python` blocks in `docs/guide` and `docs/web` are + executed in order, per page, against MongoDB; `py` blocks are shown but not run. Prefer `python`. +- **A changelog entry** in `CHANGELOG.md`, marked **Breaking** when it is. +- **`make check` passing.** + +Keep changes to one concern. If a change departs from the design in `claudedocs/v2-plan.md`, add +an amendment row there saying what and why. + +## Plugs for other frameworks + +A plug only translates requests and responses. Prove it with the contract suite: +`mongospecs_web.testing.run_contract` must produce the same transcript as the reference plug. See +[Plugs](docs/web/plugs.md). + +## Releasing + +Each package is released by publishing a GitHub release whose tag names it: +`mongospecs-v0.2.0` or `mongospecs-web-v0.1.0`. The tag must match the version in that package's +`pyproject.toml`; the publish workflow checks this and runs the full suite before uploading. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..143532a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,24 @@ +# Security + +## Reporting a vulnerability + +Please do not open a public issue. Use GitHub's private reporting instead: +. + +Say what you found, how to reproduce it, and which package and version it affects. + +## Supported versions + +Fixes are made on the latest release of each package. `mongospecs` 0.1.x is no longer developed. + +## What the library is meant to guarantee + +These are the properties to hold a report against: + +- A plain mapping used as a filter cannot carry MongoDB operators unless the application wraps it + in `trusted()`. +- In `mongospecs-web`, nothing a client sends reaches MongoDB as a raw filter or update: fields + and operators are allowlisted by `Policy`, and values are cast to the field's type. +- A field marked `Policy(private=True)` is never part of a response or of the OpenAPI document. +- A resource's `scope` is part of the database query of every list, get, patch, put and delete. +- Request bodies are never read past `Api(max_body=...)`. diff --git a/packages/mongospecs-web/pyproject.toml b/packages/mongospecs-web/pyproject.toml index 867aeb3..edd4f2a 100644 --- a/packages/mongospecs-web/pyproject.toml +++ b/packages/mongospecs-web/pyproject.toml @@ -29,7 +29,9 @@ quart = ["quart>=0.19"] all = ["starlette>=0.40", "fastapi>=0.115", "litestar>=2.12", "quart>=0.19"] [project.urls] +Documentation = "https://jaykv.github.io/mongospecs/" Repository = "https://github.com/jaykv/mongospecs" +Changelog = "https://github.com/jaykv/mongospecs/blob/main/CHANGELOG.md" [tool.hatch.build.targets.wheel] packages = ["src/mongospecs_web"] diff --git a/packages/mongospecs/pyproject.toml b/packages/mongospecs/pyproject.toml index 87b5771..408b392 100644 --- a/packages/mongospecs/pyproject.toml +++ b/packages/mongospecs/pyproject.toml @@ -33,7 +33,9 @@ sentry = ["sentry-sdk>=2.0"] all = ["msgspec>=0.19", "pydantic>=2.9", "attrs>=24.2", "opentelemetry-api>=1.20", "sentry-sdk>=2.0"] [project.urls] +Documentation = "https://jaykv.github.io/mongospecs/" Repository = "https://github.com/jaykv/mongospecs" +Changelog = "https://github.com/jaykv/mongospecs/blob/main/CHANGELOG.md" [tool.hatch.build.targets.wheel] packages = ["src/mongospecs"] From 1dac7f61b31022219506e958aad68e56704746f0 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 10:44:32 -0400 Subject: [PATCH 25/26] feat(core): validate documents before they are written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit msgspec and attrs do not check types when a model is constructed or assigned to, and pydantic does not check assignments. A wrongly typed value was therefore stored as it was, and the document could not be loaded afterwards. insert, save, insert_many and bulk_save now decode the document's own encoding with the backend's strict rules before sending anything, and raise ValidationError if it does not fit. save checks only when there is a change, and only the changed fields of a document loaded with a projection. The check can be skipped with validate=False on a call or SpecConfig(validate_writes=False) on a model. It cost about 3 µs (msgspec) and 5 µs (pydantic) per seven-field document when measured. Recorded as A41 in the plan. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 + claudedocs/v2-plan.md | 1 + docs/guide/backends.md | 44 +++++-- docs/guide/errors.md | 2 +- docs/guide/production.md | 6 + docs/guide/writes.md | 16 +++ docs/migration.md | 2 + .../src/mongospecs_web/backend.py | 2 +- packages/mongospecs/src/mongospecs/manager.py | 60 +++++++-- packages/mongospecs/src/mongospecs/markers.py | 8 ++ packages/mongospecs/src/mongospecs/model.py | 23 ++-- .../mongospecs/tests/test_review_round3.py | 121 ++++++++++++++++++ 12 files changed, 260 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7971e2..d010490 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,9 @@ A rewrite. See [Migrating from 0.1.x](docs/migration.md) for the API map. - Index planning with gated destructive actions; change streams; per-command observability events with logging, OpenTelemetry and Sentry listeners. - Document hooks run for `insert_many` and `bulk_save`. +- `insert`, `save`, `insert_many` and `bulk_save` validate the document before writing it, on + every backend. `validate=False` on a call or `SpecConfig(validate_writes=False)` on a model + turns the check off. - A single-field update on a pydantic model runs the field's validators and not the model-level ones, which could only see the other fields' defaults. diff --git a/claudedocs/v2-plan.md b/claudedocs/v2-plan.md index ca2b593..14ec78e 100644 --- a/claudedocs/v2-plan.md +++ b/claudedocs/v2-plan.md @@ -892,6 +892,7 @@ Recorded per §7.1 rule 4. Each one names what changed and why; none invalidates | A38 | §4.2 | BSON-only values in untyped fields (`Any`) serialise to JSON on all backends: ObjectId and UUID as strings, dates as ISO 8601, Decimal128 as a string, binary as base64, Int64 as a number, Timestamp as a date, Regex as its pattern | Reported: a 500 on pydantic for any ObjectId, and on msgspec/attrs for Binary, Timestamp, Regex and Int64 | | A39 | §4.8 | `Put` operation and `Resource(merge_patch=False)`. `PUT /{id}` replaces every field the client may update (sent value, or the default when left out; a required field must be sent) and leaves the rest as stored, in one atomic update; an embedded object holding protected values is replaced key by key the same way, one without them is replaced whole. It never creates. `Put` is not in the default `ops`. `merge_patch=False` makes `PATCH` replace embedded objects as before A37; nested keys the policy closes are still rejected | Requested by Jay as the opt-out from A37 and its limits (validators of the embedded model see one key at a time on a merge, a stored null needs the whole object, a backend without `supports_merge` gets a 422). The flag is on the resource, not the model: it describes an HTTP verb, model code is unaffected, and embedded models do not inherit settings (A35). Cost of opting out, accepted by whoever sets it: values the client cannot send are reset to defaults on replacement. `Put` is opt-in because guards are keyed by operation, so adding it to the defaults would expose an unguarded write on resources that guard `Patch` | | A40 | §4.4, §4.8 | Third review. Core: a single-field update on a pydantic model no longer fails a model-level validator that saw the other fields' defaults (the field's own validators still run); `insert_many` and `bulk_save` run document hooks; `page()` starts its count only after its arguments are validated. Web: request bodies are strict JSON (`NaN`, `Infinity` and over-deep nesting are a 400); `If-Match` on a model without a `Revision` is a 412; `If-None-Match` accepts lists, weak tags and `*`; the ASGI plug drops a request whose client disconnected mid-body; `Api(openapi=False)` is honoured by the FastAPI plug; guard and hook keys must name an operation; the concurrency limit is applied before authentication; a filter or sort key given twice is a 400 | Each was reproduced against the replica set first (`test_review_round3.py` in both packages). `insert_many` hooks deviate from Mongoose, where `insertMany` does not run `save` middleware: here the documents are in the caller's hands and a `before("insert")` hook that silently does not run loses data (a slug, a derived field). The hook calls are skipped when no hook is registered, so the bulk paths cost the same as before for models without hooks. Left as documented differences: trailing slash, `HEAD` and `OPTIONS` on framework plugs | +| A41 | §4.2, §4.4 | Document writes are validated by default. `insert`, `save`, `insert_many` and `bulk_save` decode the document's own encoding with the backend's strict rules before sending it and raise `ValidationError` if it does not fit. `save` checks only when there is a change, and only the changed fields of a document loaded with a projection. Escape hatches: `validate=False` per call, `SpecConfig(validate_writes=False)` per model | Requested by Jay after the third review found that msgspec and attrs (no constructor or assignment checks) and pydantic (no assignment checks) let a wrongly typed value be stored, after which the document could not be loaded. Checking the encoded form means the test is exactly "can this be read back", is one code path for all three backends, and reuses the encoding the write needs anyway. Cost measured ad hoc at about 3 µs (msgspec) and 5 µs (pydantic) per seven-field document; the recorded B4 run predates it. The full suite (887 tests) passed with the check on, so no existing write was falsely refused | New node: diff --git a/docs/guide/backends.md b/docs/guide/backends.md index 71d8162..2b530e6 100644 --- a/docs/guide/backends.md +++ b/docs/guide/backends.md @@ -14,6 +14,8 @@ pip install "mongospecs[msgspec]" # or [pydantic], or [attrs] (which also ins | Import | `from mongospecs.msgspec import Spec` | `from mongospecs.pydantic import Spec` | `from mongospecs.attrs import Spec` | | Declaring a model | `class A(Spec):` | `class A(Spec):` | `@attrs.define(kw_only=True)` on **every** model, then `class A(Spec):` | | Wrong type passed to the constructor | accepted | `ValidationError` | accepted | +| Wrong type assigned to an attribute | accepted | accepted | accepted | +| `insert()`, `save()`, `insert_many()`, `bulk_save()` | validated | validated | validated | | Stored documents on read | validated | validated | validated | | `Model.update(...)`, `PATCH`, filters | validated and cast | validated and cast | validated and cast | | Your own validators | none in msgspec; use `Annotated[..., msgspec.Meta(...)]` constraints | `field_validator`, `model_validator` | `attrs.field(validator=..., converter=...)` | @@ -26,28 +28,27 @@ Speed and memory are measured in [Benchmarks](../benchmarks.md). ### msgspec and attrs: the constructor does not check types -That is how both libraries work, and mongospecs does not add a check on `insert()` or `save()`: +That is how both libraries work. mongospecs checks the document when it is written instead, so a +wrong value never reaches the database: ```py doc = Counter(n="many") # no error: msgspec and attrs do not validate here -await doc.insert() # stored as a string -await Counter.get(doc.id) # ValidationError: reads are validated +await doc.insert() # ValidationError: Counter.n: Expected `int`, got `str` ``` -Values that come from outside (a request body, a file, another service) should go through -something that validates: `Model.update(id, n=value)`, `Model.insert_many([{"n": value}])` (a -mapping is validated; an instance is taken as it is), a `mongospecs-web` resource, or -`msgspec.convert(data, Counter)`. Values your own typed code builds are covered by your type -checker. +The error comes at the write, not at the line that made the mistake. To be told earlier, build +documents from outside data with something that validates on the spot, such as +`msgspec.convert(data, Counter)` or a `mongospecs-web` resource. ### pydantic: validators and single-field updates - Models are validated when constructed. `Spec` sets `validate_assignment=False`, as pydantic does - by default, so a value assigned to an attribute afterwards is not validated, and is saved as it is. + by default, so a value assigned to an attribute afterwards is not checked by pydantic; it is + checked when the document is written, like on the other backends. - `Model.update(id, low=5)` and an HTTP `PATCH` validate one field at a time. The field's type, constraints and `field_validator`s run. A `model_validator` that compares fields does not: it would be comparing the new value with the other fields' defaults, not with what is stored. If a - rule spans fields, enforce it where you have the whole document: load it, change it, `save()`. + rule spans fields, enforce it where you have the whole document: load it, change it, `save()`, which validates the whole document, model validators included. - `id` is declared with the alias `_id`; both names are accepted when constructing. - The `pydantic.mypy` plugin works as usual. @@ -57,6 +58,29 @@ A subclass of `Spec` that is not decorated with `@attrs.define(kw_only=True)` ge its own. `kw_only=True` is needed because `id` has a default and your required fields come after it. +## Writes are validated, unless you say otherwise + +Before `insert()`, `save()`, `insert_many()` and `bulk_save()` send anything, the document is +checked against its model, in the form it is about to be stored in. What is refused is exactly +what could not be read back. `save()` only checks when something changed, and for a document +loaded with a projection it checks the fields that changed rather than the whole document. + +The check costs a few microseconds per document (about 3 on msgspec and 5 on pydantic for a +document with seven fields and two embedded objects, measured on one machine; a round trip to +MongoDB is hundreds). Where even that matters, and the values are known to be sound, turn it off: + +```py +await Event.insert_many(events, validate=False) # for one call +await event.save(validate=False) + +class Event(Spec): + __spec__ = SpecConfig(validate_writes=False) # for every write of a model +``` + +With the check off, a wrongly typed value is stored as it is and the document fails to load +afterwards. `Model.update(...)` and the other filter-based writes always validate what they are +given; the switch does not apply to them. + ## What is the same everywhere - The `id` attribute, stored as `_id`; `__spec__ = SpecConfig(...)` assigned without an annotation. diff --git a/docs/guide/errors.md b/docs/guide/errors.md index a36ce18..201bd9b 100644 --- a/docs/guide/errors.md +++ b/docs/guide/errors.md @@ -19,7 +19,7 @@ listed becomes a 500 with no detail, and the cause is logged. | Error | Raised when | What to do | HTTP | |---|---|---|---| -| `ValidationError` | a value does not fit the model: on read, in `update()`, in a request body. `.errors` lists `{"field", "message"}` | fix the input; for stored documents see `SpecConfig(upcast=...)` and `find().on_invalid(...)` | 422 | +| `ValidationError` | a value does not fit the model: on read, on any write, in a request body. `.errors` lists `{"field", "message"}` | fix the input; for stored documents see `SpecConfig(upcast=...)` and `find().on_invalid(...)` | 422 | | `QueryError` | a filter, sort or update names an unknown field, has a value that cannot be cast, or a raw dict contains an operator | fix the query; wrap filters you wrote yourself in `trusted()` | 400 | | `DocumentNotFound` | `one()`, `or_fail()`, `update()`, `save()` or `delete()` matched nothing | treat as "gone" | 404 | | `StaleDocumentError` | a `Revision` check failed: the document changed since you loaded it | reload and retry, or tell the user | 412 | diff --git a/docs/guide/production.md b/docs/guide/production.md index 28d6d12..6cb13e9 100644 --- a/docs/guide/production.md +++ b/docs/guide/production.md @@ -103,6 +103,12 @@ api = Api(*resources, max_concurrency=200, max_body=1_048_576) costs 1.08 times the driver's own `insert_many` (`bench/results/6dc837f/B4.json`). *Measured.* - `save()` sends only the changed fields; `Model.update(id, Inc(...))` is one atomic round trip and needs no read. *Tested.* +- Documents are validated before `insert`, `save`, `insert_many` and `bulk_save` write them. The + check took about 3 microseconds per document on msgspec and 5 on pydantic for a seven-field + document with two embedded objects, against about 6 and 4 for encoding it. For bulk loads of + data that is already known to be sound, `validate=False` on the call or + `SpecConfig(validate_writes=False)` on the model skips it. *Measured once, on one machine; not + part of the recorded benchmark runs, which predate the check.* - A web `DELETE` is one conditional write. It needs a read first only when the model has delete hooks, or cascades on a hard delete. *Tested* (`test_delete_is_one_write_unless_the_document_is_needed`). diff --git a/docs/guide/writes.md b/docs/guide/writes.md index 23b7e9f..76fa29f 100644 --- a/docs/guide/writes.md +++ b/docs/guide/writes.md @@ -164,12 +164,28 @@ assert landing.rev == 1 and landing.changes() == {"$set": {"tags": ["page", "pro Sessions are always explicit: pass `session=tx` to every operation inside the function. For manual control use `async with registry.session() as s, s.start_transaction(): ...` (no automatic retry). +## Validation + +Every write is validated. `Model.update(...)` checks the values it is given; `insert()`, `save()`, +`insert_many()` and `bulk_save()` check the document as it is about to be stored, so a wrongly +typed attribute is an error at the write and not when the document is next read. `validate=False` +on a call, or `SpecConfig(validate_writes=False)` on a model, skips that check where its few +microseconds matter (see [Choosing a backend](backends.md)). + ```python try: await Article.update(landing.id, title=123) except Exception as exc: assert type(exc).__name__ == "ValidationError" +draft = Article(title="Draft") +draft.views = "many" # msgspec does not check this +try: + await draft.insert() +except Exception as exc: + assert type(exc).__name__ == "ValidationError" and exc.errors[0]["field"] == "views" +assert draft.is_new and await Article.count(Q.title == "Draft") == 0 + try: await Article.update("0" * 24, title="nobody") except DocumentNotFound: diff --git a/docs/migration.md b/docs/migration.md index 1937b7a..bc1d3b3 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -56,6 +56,8 @@ async with Registry("mongodb://localhost/app", models=[Dragon]): - Reads are validated. A stored document that does not fit the model raises `ValidationError` instead of producing a half-typed object. `SpecConfig(upcast=...)` migrates old shapes on read. +- Writes are validated too: `insert()` and `save()` refuse a document that does not fit its + model. `validate=False` or `SpecConfig(validate_writes=False)` skips the check. - Unknown field names in conditions and updates raise `QueryError`. - Raw dict filters may not contain operators unless wrapped in `trusted(...)`. - Reference fields hold a `Ref` object (`.id`, `.doc`), not a bare id or a document. diff --git a/packages/mongospecs-web/src/mongospecs_web/backend.py b/packages/mongospecs-web/src/mongospecs_web/backend.py index dc323de..8f7d3f0 100644 --- a/packages/mongospecs-web/src/mongospecs_web/backend.py +++ b/packages/mongospecs-web/src/mongospecs_web/backend.py @@ -161,7 +161,7 @@ async def list(self, query: ListQuery, *, scope: t.Any, fields: t.Sequence[str] async def create(self, data: dict[str, t.Any]) -> T: stored = {self.meta.fields[k].stored: v for k, v in data.items()} doc = self.meta.decode(stored, stored=False) - return t.cast(T, await self.manager.insert(doc)) + return t.cast(T, await self.manager.insert(doc)) # checked again there: hooks may change it supports_merge = True """``patch`` accepts ``merge=``: a :class:`~mongospecs_web.merge.PatchPlan` of writes inside diff --git a/packages/mongospecs/src/mongospecs/manager.py b/packages/mongospecs/src/mongospecs/manager.py index 7ad61f3..f1fc8ee 100644 --- a/packages/mongospecs/src/mongospecs/manager.py +++ b/packages/mongospecs/src/mongospecs/manager.py @@ -284,14 +284,39 @@ def _has_hooks(self, *events: str) -> bool: return True return any(self.meta.hooks(when, event) for when in ("before", "after") for event in events) + def _validated(self, doc: t.Any, validate: bool | None) -> bool: + return meta_of(type(doc)).config.validate_writes if validate is None else validate + + def _check( + self, doc: t.Any, encoded: t.Mapping[str, t.Any], changed: t.Iterable[str] | None = None + ) -> None: + """Raise :class:`ValidationError` unless ``encoded``, the document as it is about to be + stored, fits the model. The model libraries do not check a value assigned to an + attribute (msgspec and attrs do not check the constructor either), so this is where a + wrongly typed value is caught: before it is written, not when it is next read. + + ``changed`` limits the check to those stored keys, for a document that was loaded with + a projection and so cannot be judged as a whole. + """ + meta = meta_of(type(doc)) + if changed is None: + meta.codec.decode(type(doc), encoded) + return + for stored in changed: + info = meta.by_stored.get(stored) + if info is not None and not info.transient: + meta.codec.validate_field(type(doc), info.name, encoded.get(stored)) + # -- document-level writes ----------------------------------------------------- - async def insert(self, doc: T, *, session: t.Any = None) -> T: + async def insert(self, doc: T, *, session: t.Any = None, validate: bool | None = None) -> T: ctx = self.ctx("insert", session, doc=doc) await self._hooks("before", ("insert", "save"), ctx, doc) + encoded = self.meta.encode(doc) + if self._validated(doc, validate): + self._check(doc, encoded) if isinstance(session, Tx): session.record(doc, self.managed_attrs) - encoded = self.meta.encode(doc) if encoded.get("_id") is None: encoded.pop("_id", None) for plugin in self.plugins: @@ -321,12 +346,12 @@ def _delta(self, doc: t.Any, state: tracking.DocState) -> tuple[dict[str, t.Any] update["$unset"] = {key: "" for key in to_unset} return current, update - async def save(self, doc: T, *, session: t.Any = None) -> T: + async def save(self, doc: T, *, session: t.Any = None, validate: bool | None = None) -> T: """Insert a new document, or send only what changed on a tracked one.""" state = tracking.state_of(doc) if state is None: try: - return await self.insert(doc, session=session) + return await self.insert(doc, session=session, validate=validate) except DuplicateKeyError as exc: if "_id" not in exc.fields: raise @@ -346,6 +371,8 @@ async def save(self, doc: T, *, session: t.Any = None) -> T: if update: flt: dict[str, t.Any] = {"_id": state.snapshot["_id"], **self._shard_filter(state.snapshot)} user_keys = _touched(update) + if self._validated(doc, validate): + self._check(doc, current, _heads(user_keys) if state.partial is not None else None) for plugin in self.plugins: plugin.on_update(ctx, flt, update) with self._op("save"): @@ -680,6 +707,7 @@ async def insert_many( *, ordered: bool = True, session: t.Any = None, + validate: bool | None = None, ) -> BulkResult[T]: """Insert documents in one round trip. @@ -688,11 +716,14 @@ async def insert_many( ``insert`` and ``save`` run for every document, as they do for ``insert()``: the ``before`` hooks ahead of the write, the ``after`` hooks for what was written. """ + given = list(docs) items: list[t.Any] = [ - self.meta.decode(dict(d), stored=False) if isinstance(d, t.Mapping) else d for d in docs + self.meta.decode(dict(d), stored=False) if isinstance(d, t.Mapping) else d for d in given ] if not items: return BulkResult([], []) + # a mapping has just been validated by being decoded; an instance has not been + unchecked = [not isinstance(d, t.Mapping) for d in given] ctx = self.ctx("insert", session, many=True) hooks = self._has_hooks("insert", "save") if hooks: @@ -701,10 +732,12 @@ async def insert_many( for plugin in self.plugins: await plugin.before_bulk(ctx, items) encoded: list[dict[str, t.Any]] = [] - for doc in items: + for doc, instance in zip(items, unchecked, strict=True): + enc = self.meta.encode(doc) + if (instance or hooks) and self._validated(doc, validate): + self._check(doc, enc) if isinstance(session, Tx): session.record(doc, self.managed_attrs) - enc = self.meta.encode(doc) if enc.get("_id") is None: enc.pop("_id", None) for plugin in self.plugins: @@ -774,7 +807,9 @@ async def run(s: t.Any) -> t.Any: await plugin.after_bulk(ctx, ops) return result - async def bulk_save(self, docs: t.Iterable[T], *, session: t.Any = None) -> int: + async def bulk_save( + self, docs: t.Iterable[T], *, session: t.Any = None, validate: bool | None = None + ) -> int: """Save many documents in one round trip: inserts for new ones, deltas for tracked ones. Returns the number of documents written. Writes are independent: if some @@ -801,6 +836,8 @@ async def bulk_save(self, docs: t.Iterable[T], *, session: t.Any = None) -> int: if hooks: await self._hooks("before", ("insert", "save"), ctx, doc) enc = self.meta.encode(doc) + if self._validated(doc, validate): + self._check(doc, enc) if enc.get("_id") is None: enc.pop("_id", None) for plugin in self.plugins: @@ -816,6 +853,8 @@ async def bulk_save(self, docs: t.Iterable[T], *, session: t.Any = None) -> int: continue flt: dict[str, t.Any] = {"_id": state.snapshot["_id"], **self._shard_filter(state.snapshot)} user_keys = _touched(update) + if self._validated(doc, validate): + self._check(doc, current, _heads(user_keys) if state.partial is not None else None) for plugin in self.plugins: plugin.on_update(ctx, flt, update) requests.append(DriverUpdateOne(flt, update)) @@ -1006,6 +1045,11 @@ def _touched(update: t.Mapping[str, t.Any]) -> frozenset[str]: return frozenset(k for body in update.values() for k in body) +def _heads(paths: t.Iterable[str]) -> set[str]: + """The top-level stored keys that dotted update paths write under.""" + return {path.partition(".")[0] for path in paths} + + def _patch(doc: dict[str, t.Any], update: t.Mapping[str, t.Any], *, skip: frozenset[str]) -> None: """Apply plugin-written top-level ``$set``/``$inc`` values to an encoded document.""" for key, value in update.get("$set", {}).items(): diff --git a/packages/mongospecs/src/mongospecs/markers.py b/packages/mongospecs/src/mongospecs/markers.py index 6f832d1..56c6aca 100644 --- a/packages/mongospecs/src/mongospecs/markers.py +++ b/packages/mongospecs/src/mongospecs/markers.py @@ -257,6 +257,14 @@ class SpecConfig: collection_options: t.Mapping[str, t.Any] = field(default_factory=dict) """Options for ``create_collection`` (``timeseries``, ``capped``, ``size``, ...).""" + validate_writes: bool = True + """Check a document against its model before ``insert``, ``save``, ``insert_many`` and + ``bulk_save`` write it, so that nothing is stored that could not be read back. msgspec and + attrs do not check types when a model is constructed or assigned to, and pydantic does not + check assignments, so without this a wrongly typed value is stored as it is. Turn it off for + a model whose values are known to be sound and whose writes must be as cheap as possible; + ``validate=False`` on a single call does the same for that call.""" + json_schema_validation: t.Literal["off", "moderate", "strict"] = "off" none: t.Literal["omit", "store"] = "omit" """What to write for a field whose default is ``None`` while its value is ``None``: diff --git a/packages/mongospecs/src/mongospecs/model.py b/packages/mongospecs/src/mongospecs/model.py index fcaf342..2c8f6e3 100644 --- a/packages/mongospecs/src/mongospecs/model.py +++ b/packages/mongospecs/src/mongospecs/model.py @@ -135,8 +135,9 @@ async def insert_many( *, ordered: bool = True, session: t.Any = None, + validate: bool | None = None, ) -> BulkResult[T]: - return await cls._manager().insert_many(docs, ordered=ordered, session=session) + return await cls._manager().insert_many(docs, ordered=ordered, session=session, validate=validate) @classmethod async def bulk_write( @@ -145,8 +146,10 @@ async def bulk_write( return await cls._manager().bulk_write(operations, ordered=ordered, session=session) @classmethod - async def bulk_save(cls: type[T], docs: t.Iterable[T], *, session: t.Any = None) -> int: - return await cls._manager().bulk_save(docs, session=session) + async def bulk_save( + cls: type[T], docs: t.Iterable[T], *, session: t.Any = None, validate: bool | None = None + ) -> int: + return await cls._manager().bulk_save(docs, session=session, validate=validate) @classmethod def aggregate( @@ -224,12 +227,16 @@ def changes(self) -> dict[str, t.Any]: return {"$set": self._own_manager().meta.encode(self)} return self._own_manager()._delta(self, state)[1] - async def insert(self: T, *, session: t.Any = None) -> T: - return t.cast(T, await self._own_manager().insert(self, session=session)) + async def insert(self: T, *, session: t.Any = None, validate: bool | None = None) -> T: + """Insert this document. It is checked against the model first, so a wrongly typed value + is a ``ValidationError`` here and not when the document is next read; ``validate=False`` + skips the check (as does ``SpecConfig(validate_writes=False)`` for the whole model).""" + return t.cast(T, await self._own_manager().insert(self, session=session, validate=validate)) - async def save(self: T, *, session: t.Any = None) -> T: - """Insert a new document, or send only the fields that changed since it was loaded.""" - return t.cast(T, await self._own_manager().save(self, session=session)) + async def save(self: T, *, session: t.Any = None, validate: bool | None = None) -> T: + """Insert a new document, or send only the fields that changed since it was loaded. + Validated like :meth:`insert`, and only when there is something to send.""" + return t.cast(T, await self._own_manager().save(self, session=session, validate=validate)) async def delete(self, *, session: t.Any = None, hard: bool = False, if_rev: int | None = None) -> None: """Delete this document (a soft delete when the model has a ``DeletedAt`` field, unless ``hard``). diff --git a/packages/mongospecs/tests/test_review_round3.py b/packages/mongospecs/tests/test_review_round3.py index 260d4af..53580ef 100644 --- a/packages/mongospecs/tests/test_review_round3.py +++ b/packages/mongospecs/tests/test_review_round3.py @@ -135,3 +135,124 @@ async def test_8_a_bad_cursor_leaves_no_coroutine_behind(make_registry): await Note.find().page(5, number=0, total=True) gc.collect() assert not [w for w in caught if "never awaited" in str(w.message)] + + +# -- writes are validated --------------------------------------------------------------------- + + +def counter(backend, **config): + spec = SpecConfig(collection="r3_counters", **config) + if backend == "msgspec": + import msgspec + + from mongospecs.msgspec import Spec + + class Part(msgspec.Struct, kw_only=True): + size: int = 0 + + class Counter(Spec, kw_only=True): + __spec__ = spec + name: str = "" + n: int = 0 + part: Part | None = None + parts: list[Part] = [] + + elif backend == "pydantic": + import pydantic + + from mongospecs.pydantic import Spec + + class Part(pydantic.BaseModel): + size: int = 0 + + class Counter(Spec): + __spec__ = spec + name: str = "" + n: int = 0 + part: Part | None = None + parts: list[Part] = [] + + else: + import attrs + + from mongospecs.attrs import Spec + + @attrs.define(kw_only=True) + class Part: + size: int = 0 + + @attrs.define(kw_only=True) + class Counter(Spec): + __spec__ = spec + name: str = "" + n: int = 0 + part: Part | None = None + parts: list[Part] = attrs.field(factory=list) + + return Counter, Part + + +def wrong(Counter, Part, **fields): + """A document holding a wrongly typed value, however the backend lets one come about.""" + doc = Counter(name="c") + for name, value in fields.items(): + setattr(doc, name, value) + return doc + + +async def test_10_a_wrongly_typed_value_is_refused_before_it_is_written(backend, make_registry): + Counter, Part = counter(backend) + await make_registry(Counter) + bad_part = Part() + bad_part.size = "big" + for fields in ({"n": "many"}, {"part": bad_part}, {"parts": [bad_part]}, {"name": None}): + with pytest.raises(ValidationError): + await wrong(Counter, Part, **fields).insert() + with pytest.raises(ValidationError): + await wrong(Counter, Part, **fields).save() + with pytest.raises(ValidationError): + await Counter.insert_many([wrong(Counter, Part, **fields)]) + with pytest.raises(ValidationError): + await Counter.bulk_save([wrong(Counter, Part, **fields)]) + assert await Counter.count() == 0 + + doc = await Counter(name="ok", n=1, part=Part(size=2), parts=[Part(size=3)]).insert() + doc.n = "many" + with pytest.raises(ValidationError) as caught: + await doc.save() + assert caught.value.errors[0]["field"] == "n" + with pytest.raises(ValidationError): + await Counter.bulk_save([doc]) + assert (await Counter.get(doc.id)).n == 1 # nothing was sent + doc.n = 2 + assert (await doc.save()).n == 2 # and the document is still usable once it is put right + + # a document loaded with a projection is judged by what was changed, not as a whole + partial = await Counter.find().select("n").one() + partial.n = "many" + with pytest.raises(ValidationError): + await partial.save() + partial.n = 5 + await partial.save() + assert [(c.name, c.n) for c in await Counter.find()] == [("ok", 5)] + + +async def test_10_validation_of_writes_can_be_turned_off(backend, make_registry): + Counter, Part = counter(backend) + await make_registry(Counter) + doc = await wrong(Counter, Part, n="many").insert(validate=False) + assert (await Counter.collection().find_one({"_id": doc.id}))["n"] == "many" + doc.name = 7 + await doc.save(validate=False) + await Counter.insert_many([wrong(Counter, Part, n="x")], validate=False) + await Counter.bulk_save([wrong(Counter, Part, n="y")], validate=False) + assert await Counter.find().raw().count() == 3 + + +async def test_10_or_for_a_whole_model(backend, make_registry, db_name): + Counter, Part = counter(backend, validate_writes=False) + await make_registry(Counter) + doc = await wrong(Counter, Part, n="many").insert() + assert (await Counter.collection().find_one({"_id": doc.id}))["n"] == "many" + with pytest.raises(ValidationError): + await wrong(Counter, Part, n="many").insert(validate=True) # a call can still ask for it From 92df62580308025bc3c44533f195cf0f39b5a021 Mon Sep 17 00:00:00 2001 From: jaykv Date: Tue, 13 Oct 2026 10:52:30 -0400 Subject: [PATCH 26/26] build(docs): switch from mkdocs-material to zensical Move the site config to zensical.toml on the new default theme, swap the docs dependency, and deploy through the GitHub Pages actions since zensical has no gh-deploy. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/docs.yaml | 22 +- .gitignore | 2 +- Makefile | 2 +- mkdocs.yaml | 93 -------- pyproject.toml | 2 +- uv.lock | 457 +++++++++--------------------------- zensical.toml | 94 ++++++++ 7 files changed, 225 insertions(+), 447 deletions(-) delete mode 100644 mkdocs.yaml create mode 100644 zensical.toml diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 64748f4..7f96787 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -1,24 +1,36 @@ -# Publishes the documentation site to GitHub Pages (the gh-pages branch) from main. +# Publishes the documentation site to GitHub Pages from main. +# Requires Settings > Pages > Source: "GitHub Actions". name: Docs on: push: branches: [main] - paths: ["docs/**", "examples/**", "mkdocs.yaml", ".github/workflows/docs.yaml"] + paths: ["docs/**", "examples/**", "zensical.toml", ".github/workflows/docs.yaml"] workflow_dispatch: permissions: - contents: write + contents: read + pages: write + id-token: write concurrency: - group: docs + group: pages cancel-in-progress: true jobs: deploy: runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 - run: uv sync --group docs --python 3.13 - - run: uv run --group docs mkdocs gh-deploy --strict --force + - run: uv run --group docs zensical build --clean + - uses: actions/configure-pages@v5 + - uses: actions/upload-pages-artifact@v3 + with: + path: site + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index f10504c..8a28c70 100644 --- a/.gitignore +++ b/.gitignore @@ -117,7 +117,7 @@ venv.bak/ # Rope project settings .ropeproject -# mkdocs documentation +# zensical documentation /site # mypy diff --git a/Makefile b/Makefile index 292e3c9..fb2572e 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ build: uv build --all-packages docs: - uv run --group docs mkdocs build --strict + uv run --group docs zensical build --clean clean: rm -rf dist build site .*_cache .venv-* diff --git a/mkdocs.yaml b/mkdocs.yaml deleted file mode 100644 index a01efc2..0000000 --- a/mkdocs.yaml +++ /dev/null @@ -1,93 +0,0 @@ -site_name: "mongospecs" -site_url: "https://jaykv.github.io/mongospecs/" -repo_url: "https://github.com/jaykv/mongospecs" -edit_uri: "edit/main/docs/" - -theme: - name: "material" - features: - - announce.dismiss - - content.action.edit - - content.action.view - - content.code.annotate - - content.code.copy - - content.tooltips - - navigation.footer - - navigation.indexes - - navigation.sections - - navigation.tabs - - navigation.tabs.sticky - - navigation.top - - search.highlight - - search.suggest - - toc.follow - palette: - - media: "(prefers-color-scheme: light)" - scheme: default - accent: purple - toggle: - icon: material/weather-sunny - name: Switch to dark mode - - media: "(prefers-color-scheme: dark)" - scheme: slate - primary: black - accent: lime - toggle: - icon: material/weather-night - name: Switch to light mode - -markdown_extensions: -- abbr -- attr_list -- admonition -- footnotes -- md_in_html -- pymdownx.blocks.admonition -- pymdownx.blocks.details -- pymdownx.blocks.tab: - alternate_style: true - slugify: !!python/object/apply:pymdownx.slugs.slugify - kwds: - case: lower -- pymdownx.emoji: - emoji_index: !!python/name:material.extensions.emoji.twemoji - emoji_generator: !!python/name:material.extensions.emoji.to_svg -- pymdownx.magiclink -- pymdownx.snippets: - base_path: ["."] - check_paths: true -- pymdownx.superfences -- pymdownx.tabbed: - alternate_style: true - slugify: !!python/object/apply:pymdownx.slugs.slugify - kwds: - case: lower -- pymdownx.tasklist: - custom_checkbox: true - -plugins: - search: - -nav: -- Home: index.md -- Getting started: tutorial.md -- Guide: - - Models: guide/models.md - - Choosing a backend: guide/backends.md - - Queries: guide/queries.md - - Writes and transactions: guide/writes.md - - Relations: guide/relations.md - - Hooks and plugins: guide/plugins.md - - Errors: guide/errors.md - - Running in production: guide/operations.md - - Tuning for production: guide/production.md -- Web: - - Resources: web/resources.md - - Authentication and authorization: web/auth.md - - Testing your API: web/testing.md - - Plugs: web/plugs.md -- Benchmarks: benchmarks.md -- Coming from Mongoose: parity.md -- Migrating from 0.1.x: - - Guide: migration.md - - v1 test inventory: v1-test-inventory.md diff --git a/pyproject.toml b/pyproject.toml index 840907e..fe9a260 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dev = [ "openapi-spec-validator>=0.7", ] docs = [ - "mkdocs-material>=9.5", + "zensical>=0.0.70", ] [tool.pytest.ini_options] diff --git a/uv.lock b/uv.lock index 3850a74..4c1ae74 100644 --- a/uv.lock +++ b/uv.lock @@ -128,29 +128,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548 }, ] -[[package]] -name = "babel" -version = "2.18.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845 }, -] - -[[package]] -name = "backrefs" -version = "8.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/68/bc/c4b75b0279b2cf9e208c7dea5755a452d158a32abe1b7330824ff1127e86/backrefs-8.1.tar.gz", hash = "sha256:f86d9312df3ebae1241758af80809a319cafa1d8f32189edfb242c9aa55573b6", size = 6999089 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/17/7be730f397b9239be5044a147f0cdfbd9605111b07af053c737b21d79d16/backrefs-8.1-py310-none-any.whl", hash = "sha256:438bd38dc7fa0cded52786a1351a2dad9efc2cda95d047284c41d916a22312a8", size = 380382 }, - { url = "https://files.pythonhosted.org/packages/fd/d1/d25387b14119b7062fead6c9b2e87c2ab286beb346df79c6a649c095e979/backrefs-8.1-py311-none-any.whl", hash = "sha256:11f61846e446fa8637af2f5c073f1083aa58dbb7460b019fd51689f564e44f7e", size = 392180 }, - { url = "https://files.pythonhosted.org/packages/7d/63/4c3078fcbb3956c55b24472d8375a1b7ad67203d4a1917512eee34d4fbee/backrefs-8.1-py312-none-any.whl", hash = "sha256:b472c76082769f86a97ffccfd1f2599b2b706610c6154955664fffe67e909035", size = 398092 }, - { url = "https://files.pythonhosted.org/packages/df/22/60f30c9b8e497ca4bbac23991a726ebc267e23771a1b9d9c04b3a3c16dd6/backrefs-8.1-py313-none-any.whl", hash = "sha256:db577dbae1c61187534d35fc9bfbd0c01158fff6239a66fecf7590e5b17879a9", size = 400050 }, - { url = "https://files.pythonhosted.org/packages/45/c7/80fa46ecb47717845ab9c07bce8651ef8f80aac5ec698e212d02ed2a9963/backrefs-8.1-py314-none-any.whl", hash = "sha256:1e857c32e91b8fe46ddb359ac21021a37b090fa4239566a61e9856123e334041", size = 411908 }, - { url = "https://files.pythonhosted.org/packages/05/32/a57c5a5b7461b8c88266244014aa38052d76f64b064451b1d5342eb09bf4/backrefs-8.1-py315-none-any.whl", hash = "sha256:9c8fe3e2eaae4301a9ce787341d986e5ba25a887d89da6de0de5d58eca26a949", size = 417589 }, -] - [[package]] name = "blinker" version = "1.9.0" @@ -169,153 +146,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983 }, ] -[[package]] -name = "charset-normalizer" -version = "3.5.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz", hash = "sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef", size = 172659 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/67/6a0b94a7960d5e1b5eacd2fb529f3fccc47db4644f7f0a7cfdcfc3be578a/charset_normalizer-3.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6", size = 370642 }, - { url = "https://files.pythonhosted.org/packages/fb/94/01009e13b94041599004edf32e56e382c24e570f60f79bab8efe45cfe1eb/charset_normalizer-3.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5", size = 259280 }, - { url = "https://files.pythonhosted.org/packages/66/85/3b5358f60a13210f0b67d3755c168ef758701b021e655d88d4da28554467/charset_normalizer-3.5.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74", size = 245335 }, - { url = "https://files.pythonhosted.org/packages/74/75/77c1c479b09ecd751d1e767b251ea5c14d4d50ff757bf404afab2692f600/charset_normalizer-3.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab", size = 291707 }, - { url = "https://files.pythonhosted.org/packages/0b/0d/363f78cacb70f58f15f4b083961bbd9d292f335d3f5c66fc4f1cfe69cb90/charset_normalizer-3.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c", size = 286747 }, - { url = "https://files.pythonhosted.org/packages/e4/ed/cf505d3011ffceb12c2067a7a5d3cfe92b875d4d44bb0ff0d69375e2c184/charset_normalizer-3.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f", size = 269972 }, - { url = "https://files.pythonhosted.org/packages/15/d8/f0a93a431d170e7ca681d4f6650fee3de934d18560e474e7267eb4b0f987/charset_normalizer-3.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288", size = 269339 }, - { url = "https://files.pythonhosted.org/packages/86/bd/9b2bd1c5b7af02462c9752d33994834ff972a96b4c483eefde9e594488e2/charset_normalizer-3.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400", size = 261683 }, - { url = "https://files.pythonhosted.org/packages/76/a5/cac540ab0fd61f3fec88ad3dbb64509e71424593d73cfdfff5ab3e4db279/charset_normalizer-3.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd", size = 248143 }, - { url = "https://files.pythonhosted.org/packages/71/7a/ff467301deef2089fad87f72df9e000a26a78fec7acbb18e1999371b8369/charset_normalizer-3.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37", size = 292358 }, - { url = "https://files.pythonhosted.org/packages/ad/77/22d7e785d1e210afc2e2f58600dd1799d17a35665faf84383f002826c5f8/charset_normalizer-3.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac", size = 268766 }, - { url = "https://files.pythonhosted.org/packages/ae/91/e8e946267f1c2d9e2bd651726e2fbd2addf02c4d36cea5069e32ca9d7bb5/charset_normalizer-3.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a", size = 287432 }, - { url = "https://files.pythonhosted.org/packages/4e/88/7561d8a88d555e7df6623abe7c0070b4baf47549b9408783a2ae0a1a6cf7/charset_normalizer-3.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640", size = 272470 }, - { url = "https://files.pythonhosted.org/packages/35/7e/578c702301ec036f01455f30744a08d2b42f6ab35b9b2d4bf8cae0ef2a80/charset_normalizer-3.5.2-cp311-cp311-win32.whl", hash = "sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d", size = 186688 }, - { url = "https://files.pythonhosted.org/packages/e8/fc/fdf8cf52ff21cd5bf158f20978991cf985325842f74283eb6df26c8a39d8/charset_normalizer-3.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96", size = 214932 }, - { url = "https://files.pythonhosted.org/packages/97/66/3e45a506d8110b632541faf9a9470185aa9878f1ed44020f31346c1c5e5b/charset_normalizer-3.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1", size = 202828 }, - { url = "https://files.pythonhosted.org/packages/e7/c8/693809898870237d82785a03f3b2b58fe4c9f14669f84a7d4e623c92a59e/charset_normalizer-3.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491", size = 367780 }, - { url = "https://files.pythonhosted.org/packages/c9/87/2fea8c13dc24b3ca9c6f803a5b2dfdeae73eb4f9e12c7885ed908ff0433c/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c", size = 246730 }, - { url = "https://files.pythonhosted.org/packages/a8/9e/09efac30b937722f46d3110ba30b875b24b2e3a266ed746cc4e376a94d80/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0", size = 237707 }, - { url = "https://files.pythonhosted.org/packages/9e/18/70d76670b13686237863a379928d60bd10e021f17d243ab3d7014c4a5f4e/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51", size = 273050 }, - { url = "https://files.pythonhosted.org/packages/54/e2/77a8b09d5adc013ed07b95b01b8b8fa5441c4e810e83ee7e4aae2fa4d91a/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5", size = 270345 }, - { url = "https://files.pythonhosted.org/packages/7f/c5/38806a25ab5e65fc178f39affeda20858efafede2fce1ffc2556cfc9fe73/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649", size = 257601 }, - { url = "https://files.pythonhosted.org/packages/ae/8d/213565184708fdb263ae55e2c04ee1ff748129dd65d48ed0e3502da9c85a/charset_normalizer-3.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e", size = 252222 }, - { url = "https://files.pythonhosted.org/packages/7e/24/76d2cefc25472531e4c5c7dfff68865eb1c39b78482f0fdc15b46f047830/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346", size = 248482 }, - { url = "https://files.pythonhosted.org/packages/7d/dc/65a801b66ab4c197e22c433ab25e7ac24324ac6f45a2269aca42cce309bf/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1", size = 241206 }, - { url = "https://files.pythonhosted.org/packages/a7/95/ca9b5eabde673002c6f1e7ada1b223916fe18f6d661da7aabd4d643718f1/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875", size = 273190 }, - { url = "https://files.pythonhosted.org/packages/2d/8b/803b4d2a3f6e1740f63f1e87b04d14b42f3d4fdfe6ed7d4db2d34102b14f/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1", size = 253527 }, - { url = "https://files.pythonhosted.org/packages/a9/55/93c0e5dbd085ae0471346026abbe7e0db9ea2d6fea74e51f0b5a46f233a7/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413", size = 271285 }, - { url = "https://files.pythonhosted.org/packages/95/69/0dbd0e0b9b16cfa816cdfcb3e2e3854a1f680dc07fb1245ea125e7448060/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869", size = 260010 }, - { url = "https://files.pythonhosted.org/packages/58/9d/e7b88e7b1bf403590c3b573277b5e1e488c68c7a6fbacca310a2c324e90c/charset_normalizer-3.5.2-cp312-cp312-win32.whl", hash = "sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e", size = 184125 }, - { url = "https://files.pythonhosted.org/packages/eb/e6/e6e083884cbcfd49c64865af05027fe7011be7b2d9179524f099a1b611f3/charset_normalizer-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc", size = 207486 }, - { url = "https://files.pythonhosted.org/packages/c4/e3/017aea0911ada7405a825c7d937eb3a13009664e2f5b38e8c4bbf2abf894/charset_normalizer-3.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3", size = 196734 }, - { url = "https://files.pythonhosted.org/packages/c5/34/68292d68512768591aaff07c59bb53ee31341c87759433a859c4641a50c2/charset_normalizer-3.5.2-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5", size = 225946 }, - { url = "https://files.pythonhosted.org/packages/e3/80/bee0b01b90ccd5322ae1d0abb33fab1bd95b7c2eadaf02aeccf22e04ee83/charset_normalizer-3.5.2-cp313-cp313-android_24_x86_64.whl", hash = "sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e", size = 238619 }, - { url = "https://files.pythonhosted.org/packages/78/6e/60ce52a85a7fd631ae8482ae6d74521014ca2f255892679484dc04d7ef56/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a", size = 206701 }, - { url = "https://files.pythonhosted.org/packages/36/8c/71aafad23f971afc84c2b295bc0c560739ce1dac558aad9fec22e39f3639/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d", size = 210506 }, - { url = "https://files.pythonhosted.org/packages/91/da/3c5a7798c046df7d2d68ad653cf5b6c5a8bfee225055a843c6f2f42aac1a/charset_normalizer-3.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055", size = 366626 }, - { url = "https://files.pythonhosted.org/packages/e1/16/710ac3de2ee354e2bd1a9c94efe45a2d27b5c6ad39b2d6a905be2c094b6c/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858", size = 244624 }, - { url = "https://files.pythonhosted.org/packages/d6/39/45c7439f5b63d24f7d5b2a1d760f34af7628782d7144b4cc8ded45c2d4bc/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234", size = 235731 }, - { url = "https://files.pythonhosted.org/packages/4d/34/38f3154785ce92e9f56eb226f4d35bdfae6b008480dd055f58837a89c810/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21", size = 270352 }, - { url = "https://files.pythonhosted.org/packages/04/f3/859f74e7babc977705026b30593b3be04049632a522fb7000f83c033d747/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718", size = 267393 }, - { url = "https://files.pythonhosted.org/packages/4b/85/41d27f234b82e47c167a5f6c0f62501dc0c640585ff4aba79e08a390336a/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4", size = 254660 }, - { url = "https://files.pythonhosted.org/packages/58/ca/5d1a997587febe5b26d8daffe363b5c1a091cece19828eec6502fd09c5ef/charset_normalizer-3.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3", size = 249996 }, - { url = "https://files.pythonhosted.org/packages/b3/1f/d1e78246f7ed60c8c8d606b4ac27f66ce49cc3e95f24893ccbeba9f77302/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c", size = 247028 }, - { url = "https://files.pythonhosted.org/packages/8e/37/eba316edd4f0c4d3a5d945924c4eeeae59abac4056aa815d8a4268f863a2/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429", size = 240562 }, - { url = "https://files.pythonhosted.org/packages/c8/8e/aaa037d40ca9ef045977f1a661048b1aa33f223adfce3452fe9be9f79d14/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f", size = 271699 }, - { url = "https://files.pythonhosted.org/packages/26/19/1c1c9f75974adf523b87f34b8a2adc5a435cd65916812bcbd0dfa45f9a29/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a", size = 252030 }, - { url = "https://files.pythonhosted.org/packages/bc/90/0660ef18e18df0a4d2a1a0edff7dfbba42d4e50ef2425557a5bb7051f77b/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00", size = 268565 }, - { url = "https://files.pythonhosted.org/packages/79/ba/57adc269824e8658f1a0f97a9e514c247445a9632b3419b97e0ba37f16dc/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d", size = 257627 }, - { url = "https://files.pythonhosted.org/packages/9a/85/33abd4315c052d3d4f54c92b1ee49bfbc0dc7115a981e462a793b6d2ab87/charset_normalizer-3.5.2-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3", size = 143407 }, - { url = "https://files.pythonhosted.org/packages/4f/de/6435e18d1aaa5d910b896d551411c96af1f42a0c56c29afc2016c61ccc2e/charset_normalizer-3.5.2-cp313-cp313-win32.whl", hash = "sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd", size = 183207 }, - { url = "https://files.pythonhosted.org/packages/9c/76/b8ec57f4e9ee3253541abf95e4a462c0175fe8032dcd070f1f2421240942/charset_normalizer-3.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639", size = 206489 }, - { url = "https://files.pythonhosted.org/packages/3e/60/c647c6ae47480221e875ea5d743ff94946f7416e3c69415ab772928e8d32/charset_normalizer-3.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3", size = 195501 }, - { url = "https://files.pythonhosted.org/packages/58/ca/7aa91362a2f77ac8e9e28a9b902a74f7d0e11a851ef0d27a74308da8cd90/charset_normalizer-3.5.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187", size = 226010 }, - { url = "https://files.pythonhosted.org/packages/a8/cf/ac8878d0322cf88a1aad4c7b147db32ca0bd806eb0060957b2e31486dbe6/charset_normalizer-3.5.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad", size = 239266 }, - { url = "https://files.pythonhosted.org/packages/c9/6d/9a08d7e0b29b7208e2c6c01dc56c8e0520e7c7beadbbfb024b58fd69c8a5/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf", size = 206783 }, - { url = "https://files.pythonhosted.org/packages/82/44/b0aa350280e6ff5a5492d17cf10460dd39d5ee848f872f7ba2df10607f60/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995", size = 210449 }, - { url = "https://files.pythonhosted.org/packages/7c/8a/40db9aa9f5907bb0e6f8b6d64064bf8852fb33d4b813ff9414911df7647c/charset_normalizer-3.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424", size = 367815 }, - { url = "https://files.pythonhosted.org/packages/7f/72/9c5e7707b57c8ddfa9ddf7b0b1d009d7fbab9e9e887d5b721060f37e307d/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13", size = 247095 }, - { url = "https://files.pythonhosted.org/packages/83/09/71e453691e927de4ddf792770cfaab3f49d494e222f66ea5e404bbd5e39c/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d", size = 233942 }, - { url = "https://files.pythonhosted.org/packages/9f/86/85c84e4da8b27dd409577d9437926ff581c5f9d3c66038dc68c1a526de51/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4", size = 273762 }, - { url = "https://files.pythonhosted.org/packages/92/08/564955a4b5f2ccb410ab480bbe8c6a18063ff27f2d35458731c4a5335df9/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438", size = 269514 }, - { url = "https://files.pythonhosted.org/packages/18/24/bad3ac4271589df29cf5ce2f5ae490518a5739358052bd0d61209e6fea54/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a", size = 255056 }, - { url = "https://files.pythonhosted.org/packages/d6/3e/350d89ad49916b86554d6f5f2d03ec1152148f87e5ff735106c6a03b1a36/charset_normalizer-3.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56", size = 251311 }, - { url = "https://files.pythonhosted.org/packages/56/5b/4970a2d154df502e133402906dd04e3ae7cada7b3011283c88d0479a2585/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd", size = 250190 }, - { url = "https://files.pythonhosted.org/packages/88/8c/f1a91bddc8fb47c2889e29ea7ea49a194eb0d9868675d786806519c00d76/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204", size = 237098 }, - { url = "https://files.pythonhosted.org/packages/24/0e/bb5dace3cc7e79068425386a6589c19b5a2ab5fefc2a46abea6919683332/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7", size = 274228 }, - { url = "https://files.pythonhosted.org/packages/9d/79/b849ad523017ea9f5a45581bbebed91439e0cf42fd2860a6f64e358eb5a6/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd", size = 252401 }, - { url = "https://files.pythonhosted.org/packages/89/8c/75469d690cf47200bce8f6cad7655724fc23148e147abfc5ce78b5f65863/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc", size = 271846 }, - { url = "https://files.pythonhosted.org/packages/26/cd/6d52d3c7437cdcf2e310ce9f28f282e733d4ef60ed19105d1819c356255f/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874", size = 258151 }, - { url = "https://files.pythonhosted.org/packages/f7/4c/070b38bdb5f49a70199fce923ec0726a49536a63ab262abbfcaaf351110b/charset_normalizer-3.5.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655", size = 143570 }, - { url = "https://files.pythonhosted.org/packages/81/84/9ebfc8ed6c8c4fcd8e726ff6bf220cc8deb3966e31dce9be8dd8aa017e64/charset_normalizer-3.5.2-cp314-cp314-win32.whl", hash = "sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0", size = 186412 }, - { url = "https://files.pythonhosted.org/packages/d1/78/5ed86f743d4bc350db307e7636419a0a5ee1d91806d30c7f667bd5c80dae/charset_normalizer-3.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c", size = 210965 }, - { url = "https://files.pythonhosted.org/packages/53/94/a3a7698e9b1a395e1eb99ccd9a324be9347973bff4e72db2a06496d7cd27/charset_normalizer-3.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253", size = 200406 }, - { url = "https://files.pythonhosted.org/packages/c1/48/c5dd00d5ef7791f02666de250a5bb6071e29b7e133cf4b835800b6d3bc27/charset_normalizer-3.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709", size = 396871 }, - { url = "https://files.pythonhosted.org/packages/12/c8/8379554b42e8368161d898476686947a0fdbd3e8865170d7909dcabfdee8/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084", size = 249711 }, - { url = "https://files.pythonhosted.org/packages/4a/eb/2ddb1035d17320caa9f41682935123a9a250277b261c3efc86b2d2a21343/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb", size = 231146 }, - { url = "https://files.pythonhosted.org/packages/4a/24/2ecb4bde104322cd7859d6594fcfa74649f8d90b3221c9feecbef149875b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f", size = 271552 }, - { url = "https://files.pythonhosted.org/packages/3f/98/9d5f6ebc3aee9fef5d30b4aff11fb2ab7a1222b4064f8ef2c7c87cde217a/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09", size = 269810 }, - { url = "https://files.pythonhosted.org/packages/09/e1/a3b06a10461b1b7628853c934c644e03bc28e42767116afb52f19a56519b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80", size = 259845 }, - { url = "https://files.pythonhosted.org/packages/fd/d3/6f561f74a296cf27d61775a1dc665ad13f3bff6a798810ca05907f37a7c4/charset_normalizer-3.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c", size = 252825 }, - { url = "https://files.pythonhosted.org/packages/26/9f/69e13ca3b18f43e0eafcd34c04a45b732ae22a43b54a5fc9e119103356eb/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f", size = 252659 }, - { url = "https://files.pythonhosted.org/packages/73/a9/ace29806a0dae18939919c76ba526472d83214afa101105fabff2cf30625/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03", size = 238819 }, - { url = "https://files.pythonhosted.org/packages/f8/c1/6116d52a2e3311ec80f21f5fb5e17b27405f10b9608af8f6e69516841a1b/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604", size = 272973 }, - { url = "https://files.pythonhosted.org/packages/19/aa/9955c7e93bba10a9c7e8f7a5031b7ced66f3a1883a55c00712b8d5850ff3/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8", size = 253692 }, - { url = "https://files.pythonhosted.org/packages/bb/33/2a6ae7fdc1b10cb581cef91addd8cdfc5f40d50abb5702309369d5834579/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93", size = 268757 }, - { url = "https://files.pythonhosted.org/packages/a2/22/80992720a0282cd39bba1db35868e6b9c22f41281160143a836544bc1d8a/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915", size = 262220 }, - { url = "https://files.pythonhosted.org/packages/92/9f/181fd07e1bffea1d95cd80c84ac537354f50699c22cfc4d3c02b6fc16208/charset_normalizer-3.5.2-cp314-cp314t-win32.whl", hash = "sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5", size = 197797 }, - { url = "https://files.pythonhosted.org/packages/49/1c/25d8415ec1c4f2f41f1680435e4c87cfb378ff2f677d950946f2a45d0632/charset_normalizer-3.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc", size = 222765 }, - { url = "https://files.pythonhosted.org/packages/3e/b4/46b48f013dadfc0d0d33b375438e31bdf5a989dc68389c6bf627054d4df9/charset_normalizer-3.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105", size = 210985 }, - { url = "https://files.pythonhosted.org/packages/ca/e9/34e597dee616d0b8ee4b34d29399e85c2204ade174157a48505d42baa4ff/charset_normalizer-3.5.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26", size = 367676 }, - { url = "https://files.pythonhosted.org/packages/60/9f/a5d1c91c0263745e2cd344c5a4415d787c575501ab1d449f1148ac6b495d/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364", size = 248414 }, - { url = "https://files.pythonhosted.org/packages/26/79/e697f77464748a3ee3cf490c83d592459400d4898380d66c38366b03080c/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253", size = 244775 }, - { url = "https://files.pythonhosted.org/packages/ca/87/3d42a42e18ea066e2513936fd678a00696e77878b5ae04528976abdbcb83/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0", size = 273877 }, - { url = "https://files.pythonhosted.org/packages/c3/76/8a28136f3938ba9836f84280ce0c4d61ed1cf15a036b2034900c62634162/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc", size = 269455 }, - { url = "https://files.pythonhosted.org/packages/a0/a1/4fbf5d0f0f1b2a080474c1cf9a2f12c4c6531bb0e8ba591055e846d2b4e9/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229", size = 257336 }, - { url = "https://files.pythonhosted.org/packages/ba/a2/8b50aa320adb880ad579518e6f718f24944804b42a88b83d267d5d444125/charset_normalizer-3.5.2-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5", size = 257584 }, - { url = "https://files.pythonhosted.org/packages/a5/57/50e3fed84e175f40349bd0da7a4fce94c87f0378f52d74f511d89e0bdc20/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98", size = 251854 }, - { url = "https://files.pythonhosted.org/packages/d6/54/f7fbb3493c9f49091213b9c2d6dd65800696f1ce1a3f196a4205f50417b1/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3", size = 248767 }, - { url = "https://files.pythonhosted.org/packages/d9/37/b3a6385acc5a1e45b39ae9c90bfb9cf838a09b9dd37ef2740ab4c6b4a2eb/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2", size = 274479 }, - { url = "https://files.pythonhosted.org/packages/89/44/809913e2cfd279e635a9294fdbbfb1b1dc62a8189d473d561f649fce98d8/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf", size = 258310 }, - { url = "https://files.pythonhosted.org/packages/af/a2/f28400ab13359d91bd39179df8e149376b9bf36588e739a3a4f9de2b84b2/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95", size = 271100 }, - { url = "https://files.pythonhosted.org/packages/e9/89/9bab37955edf0adb3b66f8a3a6617d9f2f487e0d56f295a6a286cb640aa6/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d", size = 259936 }, - { url = "https://files.pythonhosted.org/packages/23/b5/4459e08d45a679f903d50fea08bc52cfa728cca4d7bd02c757b5e5abda2e/charset_normalizer-3.5.2-cp315-cp315-win32.whl", hash = "sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847", size = 186301 }, - { url = "https://files.pythonhosted.org/packages/98/e8/55d5fd3935b4bce6da4fe0df61898e8c82653e317e677bd58aceb9c60f13/charset_normalizer-3.5.2-cp315-cp315-win_amd64.whl", hash = "sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8", size = 210858 }, - { url = "https://files.pythonhosted.org/packages/a9/5b/974423c2fd8e524c7a7f64318c1e02240ef954912fa2b4d70344107b9c68/charset_normalizer-3.5.2-cp315-cp315-win_arm64.whl", hash = "sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a", size = 200309 }, - { url = "https://files.pythonhosted.org/packages/ee/f9/00ee0195db1013d8f7c416fd770fbeb560bb46eb2e36b054d05cb56f6cfa/charset_normalizer-3.5.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1", size = 395343 }, - { url = "https://files.pythonhosted.org/packages/04/3a/c00b50e94c964cf934c7899cd47c97952fc11dad71cc5884b3c61795b09b/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b", size = 250114 }, - { url = "https://files.pythonhosted.org/packages/50/27/d102dc880bbcffd0479ab64dfc1fb96777a854355a55e2bda72a71efadcb/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f", size = 239997 }, - { url = "https://files.pythonhosted.org/packages/a5/4a/bf7ef45794dd293fab5f98a9309817977fbb845b9998f171b8cc5d8437a3/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3", size = 272313 }, - { url = "https://files.pythonhosted.org/packages/e8/ee/008a2837737991474c5754bb3191010007663860979701990982a502cbaf/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e", size = 269986 }, - { url = "https://files.pythonhosted.org/packages/93/ad/bd74a283940dc910c5b14f8e4f80a248082bc9c0fcbe1f54530cb6d9cc5e/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9", size = 260202 }, - { url = "https://files.pythonhosted.org/packages/8a/7b/ed341c66f69f688723501fac752be3d63c7159ca0d0d4174fc611e5710bb/charset_normalizer-3.5.2-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a", size = 259083 }, - { url = "https://files.pythonhosted.org/packages/cc/9d/e41588b777965e5031a43128a1e96173ebb35ac75fc53ec3b517e7c21cd4/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115", size = 253081 }, - { url = "https://files.pythonhosted.org/packages/81/35/b761eb6d8c1eb218b9b42b9b4d5ac902afdc399fb6dac6f9a9aac7bda589/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c", size = 248712 }, - { url = "https://files.pythonhosted.org/packages/4d/2c/147169a041b747759f37405c0a97157e8e92de967968373101ff14915cba/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d", size = 274176 }, - { url = "https://files.pythonhosted.org/packages/f0/2d/0ff8db0d373ba8538db686db11cd7e8912031490b9e4f383b41912e8d594/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d", size = 259413 }, - { url = "https://files.pythonhosted.org/packages/8a/8e/b4a085fb47c9d3a7e43576a4784fdd8fe23f907514a972de8086edaf7a48/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4", size = 269434 }, - { url = "https://files.pythonhosted.org/packages/83/1c/d8d8d7322a7c3eecdf3237a4a419cf41d2eaad8e006ce7dfdd9d4c8fa2eb/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b", size = 262716 }, - { url = "https://files.pythonhosted.org/packages/a0/16/0e4c6ba9b44e97a2da150e52d331e8f9c968b21b358fbffa6c856cebcd89/charset_normalizer-3.5.2-cp315-cp315t-win32.whl", hash = "sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800", size = 197159 }, - { url = "https://files.pythonhosted.org/packages/be/33/e90bc2b1374f7f36ef106f56620de5a783907e19ca857efe2277e31cac3e/charset_normalizer-3.5.2-cp315-cp315t-win_amd64.whl", hash = "sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21", size = 221322 }, - { url = "https://files.pythonhosted.org/packages/66/89/dfa6dcb08c200b7830ab56439e8c1890f2971d51aafbb3937894a2e7fcfc/charset_normalizer-3.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58", size = 209192 }, - { url = "https://files.pythonhosted.org/packages/8c/ab/176fbfd5b64939c55d652366aa5b9ef1d767af207a3aa6ebeb0d226c484d/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd", size = 331815 }, - { url = "https://files.pythonhosted.org/packages/7e/84/371eac6b30bdbcbf2d632a1a01809103459216fcaae61b8b8d922c1bfb8a/charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7", size = 253276 }, - { url = "https://files.pythonhosted.org/packages/43/6f/c4fbae58febff71709c51bc7e18fdfa55341dc382704740f9f0cbf03817b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f", size = 241239 }, - { url = "https://files.pythonhosted.org/packages/61/71/458c3f42164a07d0c5210798e9e704b39e540a6793b05aba67f3a35243a9/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93", size = 231121 }, - { url = "https://files.pythonhosted.org/packages/09/54/ab9e89367076f6331bb6c65c4bf14a5361fa5191cb6561bf534f18504e1b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade", size = 260350 }, - { url = "https://files.pythonhosted.org/packages/7c/c1/061431ecc688d9d76602502cb57cc01e691e682c18f1beb45f9673b5bbd2/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0", size = 255430 }, - { url = "https://files.pythonhosted.org/packages/8d/1f/20c8949f0676f7ab811abdeb7f4d7f1cbc6e61ff20bef08b44edeb092bc8/charset_normalizer-3.5.2-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26", size = 250612 }, - { url = "https://files.pythonhosted.org/packages/2b/9e/46f2fa4c431fc98c4ae76a8cb5bdca54e0341e3cfc3fcfd8e82740250818/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011", size = 242083 }, - { url = "https://files.pythonhosted.org/packages/bd/39/559be29a0c0f086e0bba6922babd38916cc5e0b58ced4de13ee01ea05508/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621", size = 232738 }, - { url = "https://files.pythonhosted.org/packages/ff/6c/387b0e4f756a282831c1d9fc6aeb6c51ca4507ca202767c8de15ce9b12e2/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4", size = 260703 }, - { url = "https://files.pythonhosted.org/packages/96/92/1fdf015f09ef449f50d3ac4b67c90887c9c318b727daa95cc4f866e6521d/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e", size = 247622 }, - { url = "https://files.pythonhosted.org/packages/dc/3c/8e7b8a5671ad5d433669fb2a76f1a0164df2d9b1718b0206bc2a16d840cc/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c", size = 257500 }, - { url = "https://files.pythonhosted.org/packages/b4/f0/45b579df5cabc1d5d53ea1cc35e8437d3ca768c0acccc7041517cb6fbb32/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0", size = 255100 }, - { url = "https://files.pythonhosted.org/packages/31/68/fdec18a343f5fb3f310588dd478b09ac4799e0b187dbade3a8cd776f03ef/charset_normalizer-3.5.2-cp37-abi3-win32.whl", hash = "sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf", size = 174499 }, - { url = "https://files.pythonhosted.org/packages/9d/8a/b618149cc5207943a0242068d7a27897f56a62947b5a039085f2a22029f8/charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl", hash = "sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036", size = 200092 }, - { url = "https://files.pythonhosted.org/packages/03/cf/4c66866fa9e2b1c78e3c911516d1de497a677b7ac60f1eceda74ce777ca3/charset_normalizer-3.5.2-cp37-abi3-win_arm64.whl", hash = "sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e", size = 294363 }, - { url = "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl", hash = "sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685", size = 68872 }, -] - [[package]] name = "click" version = "8.5.0" @@ -334,6 +164,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, ] +[[package]] +name = "deepmerge" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/6e/5cb3548b4d3112fea529375e55e6f3cdc52b8054e3a66f203b1f888ba885/deepmerge-3.0.1.tar.gz", hash = "sha256:35b39a4cb92cf328d6eca61cbbf65f68a37c2ceb3085f0f853cbb2e52a59fc23", size = 22328 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/91/600003aaad107e27553fbc9cbfc57e96fa37e0223d2ef9d7d3a0e8d8d070/deepmerge-3.0.1-py3-none-any.whl", hash = "sha256:35c96f6a68fcf90719a5b31d9f8042ecef6c00fb56836660d33455d0f5cfda65", size = 14909 }, +] + [[package]] name = "dnspython" version = "2.9.0" @@ -389,18 +228,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl", hash = "sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c", size = 103424 }, ] -[[package]] -name = "ghp-import" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "python-dateutil" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d9/29/d40217cbe2f6b1359e00c6c307bb3fc876ba74068cbab3dde77f03ca0dc4/ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343", size = 10943 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034 }, -] - [[package]] name = "h11" version = "0.16.0" @@ -911,84 +738,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979 }, ] -[[package]] -name = "mergedeep" -version = "1.3.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3a/41/580bb4006e3ed0361b8151a01d324fb03f420815446c7def45d02f74c270/mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8", size = 4661 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307", size = 6354 }, -] - -[[package]] -name = "mkdocs" -version = "1.6.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "ghp-import" }, - { name = "jinja2" }, - { name = "markdown" }, - { name = "markupsafe" }, - { name = "mergedeep" }, - { name = "mkdocs-get-deps" }, - { name = "packaging" }, - { name = "pathspec" }, - { name = "pyyaml" }, - { name = "pyyaml-env-tag" }, - { name = "watchdog" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bc/c6/bbd4f061bd16b378247f12953ffcb04786a618ce5e904b8c5a01a0309061/mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2", size = 3889159 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/5b/dbc6a8cddc9cfa9c4971d59fb12bb8d42e161b7e7f8cc89e49137c5b279c/mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e", size = 3864451 }, -] - -[[package]] -name = "mkdocs-get-deps" -version = "0.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mergedeep" }, - { name = "platformdirs" }, - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ce/25/b3cccb187655b9393572bde9b09261d267c3bf2f2cdabe347673be5976a6/mkdocs_get_deps-0.2.2.tar.gz", hash = "sha256:8ee8d5f316cdbbb2834bc1df6e69c08fe769a83e040060de26d3c19fad3599a1", size = 11047 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/29/744136411e785c4b0b744d5413e56555265939ab3a104c6a4b719dad33fd/mkdocs_get_deps-0.2.2-py3-none-any.whl", hash = "sha256:e7878cbeac04860b8b5e0ca31d3abad3df9411a75a32cde82f8e44b6c16ff650", size = 9555 }, -] - -[[package]] -name = "mkdocs-material" -version = "9.7.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "babel" }, - { name = "backrefs" }, - { name = "colorama" }, - { name = "jinja2" }, - { name = "markdown" }, - { name = "mkdocs" }, - { name = "mkdocs-material-extensions" }, - { name = "paginate" }, - { name = "pygments" }, - { name = "pymdown-extensions" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f1/cd/c05d3a530ba7934f144fb45f7203cd236adc25c7bdcc34673d202f4b0278/mkdocs_material-9.7.7.tar.gz", hash = "sha256:c0649c065b1b0512d60aad8c10f947f8e455284475239b364b610f2deb4d0855", size = 4097923 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ad/21/17c1bc9e6f47c972ad66fb2ac2568f99f90f1207eeb6fc3b34d094dba7b5/mkdocs_material-9.7.7-py3-none-any.whl", hash = "sha256:8ea9bb1737a5b524a5f9dcf2e1b4ebda8274ae3008aa7845720a97083bef708f", size = 9305438 }, -] - -[[package]] -name = "mkdocs-material-extensions" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/79/9b/9b4c96d6593b2a541e1cb8b34899a6d021d208bb357042823d4d2cabdbe7/mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443", size = 11847 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/54/662a4743aa81d9582ee9339d4ffa3c8fd40a4965e033d77b9da9774d3960/mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31", size = 8728 }, -] - [[package]] name = "mongospecs" version = "0.2.0" @@ -1126,7 +875,7 @@ dev = [ { name = "sentry-sdk" }, ] docs = [ - { name = "mkdocs-material" }, + { name = "zensical" }, ] [package.metadata] @@ -1145,7 +894,7 @@ dev = [ { name = "ruff", specifier = ">=0.8" }, { name = "sentry-sdk", specifier = ">=2.0" }, ] -docs = [{ name = "mkdocs-material", specifier = ">=9.5" }] +docs = [{ name = "zensical", specifier = ">=0.0.70" }] [[package]] name = "msgspec" @@ -1544,15 +1293,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956 }, ] -[[package]] -name = "paginate" -version = "0.5.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ec/46/68dde5b6bc00c1296ec6466ab27dddede6aec9af1b99090e1107091b3b84/paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945", size = 19252 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/96/04b8e52da071d28f5e21a805b19cb9390aa17a47462ac87f5e2696b9566d/paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591", size = 13746 }, -] - [[package]] name = "pathable" version = "0.6.0" @@ -1571,15 +1311,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328 }, ] -[[package]] -name = "platformdirs" -version = "4.13.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/80/a8/66d45abadff219e36e2a824181b8f6a67e7ed4572934d6252c71c29d5731/platformdirs-4.13.0.tar.gz", hash = "sha256:1aa0b0d3f224c1f07c295121e312a5a24a180d6ae5a8425ea1784b3e3863e9c0", size = 61094 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/15/1633010b26e88e872c93b67c0b6c5e174fb74cb6fb5c1472b4d51d4a8f22/platformdirs-4.13.0-py3-none-any.whl", hash = "sha256:3dbcf4cd708f21cf876c4eaa90e58412bc4f033d87143f41b1493ff77c25b7e1", size = 32724 }, -] - [[package]] name = "pluggy" version = "1.6.0" @@ -1877,18 +1608,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930 }, ] -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892 }, -] - [[package]] name = "python-dotenv" version = "1.2.4" @@ -1953,18 +1672,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341 }, ] -[[package]] -name = "pyyaml-env-tag" -version = "1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/2e/79c822141bfd05a853236b504869ebc6b70159afc570e1d5a20641782eaa/pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff", size = 5737 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/11/432f32f8097b03e3cd5fe57e88efb685d964e2e5178a48ed61e841f7fdce/pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04", size = 4722 }, -] - [[package]] name = "quart" version = "0.22.0" @@ -2026,21 +1733,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766 }, ] -[[package]] -name = "requests" -version = "2.34.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "charset-normalizer" }, - { name = "idna" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075 }, -] - [[package]] name = "rfc3339-validator" version = "0.1.4" @@ -2284,6 +1976,78 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4e/d6/1ec1b290f9e0fb067899b61e1d37a30c923068bad260b216dbe37a7d2967/starlette-1.7.0-py3-none-any.whl", hash = "sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e", size = 78980 }, ] +[[package]] +name = "tomli" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/78/9ad63712633ed3ab5cc1a648d863d7e7da371e9425e209555a0fe711b695/tomli-2.5.0.tar.gz", hash = "sha256:264507556cd8b8c8e7c6ee037cdf443a463f03f4c958e57195e3d369711b8ff6", size = 17662 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/a6/ab99b60ee52acd949684febabc3005d0045d0f66bebd9cdebd67372d26dd/tomli-2.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c4dc1c1781f2f716de763d1e9a7b34c6a894e167e291c7c5d16c72f7a9538545", size = 163901 }, + { url = "https://files.pythonhosted.org/packages/bc/00/ee01b7ed4579180fff07142d290257f25ba786f23f3ec6005f620933c2f5/tomli-2.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:eff8babca5a7999bc137acbc7482a8b7e17ffca5075ab41f5d770ab408c7bfef", size = 163756 }, + { url = "https://files.pythonhosted.org/packages/72/c2/4efebf65372f6583185f79799312109dddb61102d47e5c33dcfd1a297aca/tomli-2.5.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:86665cee9c4835b7a7f1e8ec2c719b5258d4dc782887aded5a8ae7352a96843b", size = 268038 }, + { url = "https://files.pythonhosted.org/packages/53/07/5850468e925d898abb36038666f9c333a94d2a223e802a8ba5b6d319d23f/tomli-2.5.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7e369fd63331746182360977b1892bfc215476a30d61612d732425311639f56", size = 276422 }, + { url = "https://files.pythonhosted.org/packages/b4/87/f293984cdcf83c054196d4fd3dad44fc68ae55b4b8c44bc76cef360c3150/tomli-2.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7ad1ea345759240d6463efa0ed1c704402752e49aa21476620738d74d72d8aa1", size = 272616 }, + { url = "https://files.pythonhosted.org/packages/ce/ce/db582886b3c1219d3fec93ebd669332482e5aee7a91e0f7838d84f2d1759/tomli-2.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:96243987194634bd411066ce40c952e108f86af04db533ecd8ac3ff2a85b1885", size = 276593 }, + { url = "https://files.pythonhosted.org/packages/bf/72/7619b87dea4261fc27dd7b54c4461c129c1f7d9bb7ba3aec89c797a431b8/tomli-2.5.0-cp311-cp311-win32.whl", hash = "sha256:610b27d99f28ec5f191c7064a48f3ddb179a1fe6ca73d571483ae859f57b605e", size = 101830 }, + { url = "https://files.pythonhosted.org/packages/1e/74/220106da34502304b6751a2a9b8a9fbca6c3fd47e737a2e2e3da7c61c9db/tomli-2.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:c804ae44fe7b4bab5da295e4f980a1ff04670bca9d23fe0a4e887e08ebd741a8", size = 112742 }, + { url = "https://files.pythonhosted.org/packages/27/99/7d9c8b41837a7773613e169504147375c157a290167aa59ad74a085f521f/tomli-2.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:cfac177ebd6236003846ea339981f71457cb6eb748f23381eb257e45092e3980", size = 109332 }, + { url = "https://files.pythonhosted.org/packages/52/ed/7baa86f87493646a594de388c7c1c40a39dd0461f7e9c0359cbeefc91fe8/tomli-2.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1f4a40d03fb9f63424f0979855bdeaf44dd7696b8d59501822c10ed30ba532df", size = 164854 }, + { url = "https://files.pythonhosted.org/packages/a5/b1/44c0341f2224397855723c7a8a39f718ea6fcbcc3dacc66e5aeca0f334e3/tomli-2.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9ebf8d19b17bd0daeb7b7dec81a946a439b753942fd0210d6e96c532249eea6b", size = 164074 }, + { url = "https://files.pythonhosted.org/packages/23/04/e2d5b7d3fba47adedb23de616c16d428ea076c79a3d8e1d95d649ffe197e/tomli-2.5.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf0b5e8e0f68ebb494356e577c06c139161efd8d3b9050f93b39b7c26cc54ff0", size = 274274 }, + { url = "https://files.pythonhosted.org/packages/43/90/6090e706ff27a6f89f4a40578e3324b95c3cd8c4150868aabf33a8f414c3/tomli-2.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cf74416bdc94ae458b14e37286c1073081850ac8459a00d0c5efef5d44294c6", size = 286435 }, + { url = "https://files.pythonhosted.org/packages/0a/9e/a2c40768df16c408f22430afb0a73e9d7e5f79c950884954649d1146b74d/tomli-2.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:61ea1ebe1e55a34ea8199cc8dbff398d35027b82271c8ac4802fd3a1fd5b1bcc", size = 278119 }, + { url = "https://files.pythonhosted.org/packages/12/25/3c0cb485b98e9cfac495629b1c93c87ccf0b72fbe9d2689fd8fe62c6d5a3/tomli-2.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ed53f7e89bb04f6d9e8e7799112360b0c4d5cbff067de0814c98c37c39b920f7", size = 286177 }, + { url = "https://files.pythonhosted.org/packages/77/8b/0144c65f0e37e51c18d04ae15c21b19431c165002d0131fe9aa8b0b8b1e8/tomli-2.5.0-cp312-cp312-win32.whl", hash = "sha256:e7ad033e27a516a233bea839cdb77b80146facb3b4f40bf02cd0cac165cdd5c2", size = 102760 }, + { url = "https://files.pythonhosted.org/packages/de/32/5d6d8f42fc9a05fce69354e00ff256484192f5f2fc9a2165718fa0de61ec/tomli-2.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:bd05de8c1698f8413dd7d869492693a0bf2211543b787ac78cd5e7536af1a6d7", size = 112722 }, + { url = "https://files.pythonhosted.org/packages/30/65/df18032218db0fb9b769fb23c8039a051f15c811993995ea04c350273a32/tomli-2.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:069435bd5480429b98c5e5afb02ab21c219b6f0064680671c6dc0d46817346ea", size = 109534 }, + { url = "https://files.pythonhosted.org/packages/42/e5/51736d70da209350969e15aca5c5ab6e2ce1ea87a0a892a6c13aec172a86/tomli-2.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:943276cf269e0071948d9ff697159c1735e623c1151d88abb09b74659ef0cbea", size = 163328 }, + { url = "https://files.pythonhosted.org/packages/ec/55/086f80dab4ab497602644274e6dea7ec5dd0b4e262e443a8ad3bb7edee2d/tomli-2.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:463b16086865b97facd8d0b3fb4cb7c544e3f58d2a69dc3113d6db9653fdb043", size = 162246 }, + { url = "https://files.pythonhosted.org/packages/aa/eb/3ecc94459f3635c92321f4e7bde571323fdb2267c50e19e3188a281eae3b/tomli-2.5.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1245a6638fc4bb0a60af38a7d45413db34a13842027c77597c712c998c62fdf0", size = 272655 }, + { url = "https://files.pythonhosted.org/packages/c0/d7/494fd1f0c37a621f1ad9975c2efadb523e8101f144ed6edb2e7fe64738f2/tomli-2.5.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5d8bac3d603c97e6854424e5b2b5b741bdbde387e09f162fb0446812b4a8362b", size = 283595 }, + { url = "https://files.pythonhosted.org/packages/70/51/bb8d62b1317e6640866f6949b2d5855e5300f2c99d46de1cd245570bba65/tomli-2.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:21e4cae4114aba25aa0d4f85cdf486d290fb35c0954d7bba536248da64d43066", size = 276253 }, + { url = "https://files.pythonhosted.org/packages/66/f4/f46bd7f0763cd47de2db697dca9257c6a4adfd1a93b018cc75c8190ed5a8/tomli-2.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bbaefc84548d754be821bba7c4141c4787dda182f9e77f2f87b71213529efa7b", size = 283582 }, + { url = "https://files.pythonhosted.org/packages/ac/03/70f2bcb2923a6db37818d917e124270a7f4cfd38ea576f5aa753a91c0ef5/tomli-2.5.0-cp313-cp313-win32.whl", hash = "sha256:abdbf6313b8d9efe157edeb7ab6eae4de064b1300ad31abf73755154b30abe68", size = 102628 }, + { url = "https://files.pythonhosted.org/packages/dc/98/d52024bb5b0ff68b4f0d276d867f634c84a67319a7e9f6b7708a37742333/tomli-2.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:fd4dc129784e0c5335bd4e61dfcc4487499a013419e655cf2da1d091b7e0efdc", size = 113301 }, + { url = "https://files.pythonhosted.org/packages/6f/f2/540db3a70572a8c23a28aba3e9c358ce0ffffbafc990905c1343aa265b31/tomli-2.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:69491c143d2fe063046e0301e62a810bed338fa4d1ce0fd870c27dc1e09b0d84", size = 109744 }, + { url = "https://files.pythonhosted.org/packages/e4/49/caf6b307766eb9567664a8707e9d6be5fcc0e8903f18781c6677a60d80c7/tomli-2.5.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d3182ee2d887e507bd67319a0a61105d1dd33facc111329559a233b772c1a105", size = 162899 }, + { url = "https://files.pythonhosted.org/packages/d3/c8/68cfce773a2733a49c74f99d627fb461bd990756860099eac25617889585/tomli-2.5.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:521345fd1f19d45b8df87657aaa38b6f2ca3800059fadf428e7ebf479a383646", size = 162080 }, + { url = "https://files.pythonhosted.org/packages/7e/b2/e5bb8651fdad593f670501a7d718b1a7f73f064d44dea15e04c04dfef45d/tomli-2.5.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e95c7614e705bfe2b04b27aa124adec59752d15813df37e2156747cab3a006b", size = 273380 }, + { url = "https://files.pythonhosted.org/packages/8d/d2/9e2d7f8b1dfe0e2b34c245986ebd55c4c553ea4ce6c47c443b332673253f/tomli-2.5.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7ac2027d37c3afbdf4bdd377f2676f6f1d2122a5be1f1137b49dced590b37e75", size = 283228 }, + { url = "https://files.pythonhosted.org/packages/ba/df/ec7b876b7b1a2718bd74a3743c076fff565b04029ba33e8f61fac262739f/tomli-2.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c414be4ed9d3cac80c42e348fa5a956117d1a48227f48026e31f59cb4a7671eb", size = 277189 }, + { url = "https://files.pythonhosted.org/packages/7d/7b/e192d9eed0b9cb80da799f4d77052297fb9a2c3cc9b19f571f56ea88add6/tomli-2.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9b03d7dc168353b4132965bde20feceabaa470e570c6f59660dfae59b1f9eeb3", size = 283632 }, + { url = "https://files.pythonhosted.org/packages/84/50/ff94454e75461d75623e47401ed323d65c10aab8fe9033242c20cd2fdf32/tomli-2.5.0-cp314-cp314-win32.whl", hash = "sha256:6f041843c4d3a37245c0c056fd955b186bf8b1fb85690cbe40b81230891dc34b", size = 103535 }, + { url = "https://files.pythonhosted.org/packages/54/0b/bdacf05f963bd6026ebf6eeb0beda847d1d60e03e440725c64a4e08a0afd/tomli-2.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:f4b653094e18f9031102d3a1da5c729c8f222d85225b18037dac621695e46e1a", size = 114621 }, + { url = "https://files.pythonhosted.org/packages/61/99/53f438fa6ae4f9d4ed0ddde3e7242b3bdc34b48c8f9948b72b9e9b127676/tomli-2.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:3f89d10c1ff6a38d992c27fc8a4816af71a909e08a40ec66934240b1e74347c3", size = 111572 }, + { url = "https://files.pythonhosted.org/packages/b9/20/1f88f19427d380a40e90a770e087489eaafe4aeee070ae88ed2bbec00acd/tomli-2.5.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:e9e15b4a6c7dd6b85b5fbab29488a73f1f70de516942308daa266bf0e0aeb0d4", size = 171814 }, + { url = "https://files.pythonhosted.org/packages/d0/56/cbe5079c9f9a54b9b3e27fc82f08f3cb36edee75561679f53d2380c801d6/tomli-2.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e12bbcd32897272fb05929110362ae9ff4c1b9bb26bd9e971e71dcd3275b4c3d", size = 171324 }, + { url = "https://files.pythonhosted.org/packages/2b/30/1d53fd3b0f1cb3ba542e345ec32c26aefdddc4e829e4f3429af8a4f27782/tomli-2.5.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20aa36de8f2cf87237143bc1fa1aae8d6612c09118f4da21c6a684db5dd1f6f9", size = 297441 }, + { url = "https://files.pythonhosted.org/packages/66/d9/0800acb6a111686f764c1b91ef15cc42a20a66a46013bb42220f1d2c61c1/tomli-2.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22185fad8a1e622f064e78008018a0dd3323550dcb479cb7a1d296888d74024f", size = 307476 }, + { url = "https://files.pythonhosted.org/packages/e8/63/30a8f3cd51b5bec37f04744bad0b0dc6160df84aad4f27b0e9283d66f221/tomli-2.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:984012f71908165449a951de2050d52f276bfe3aa5d5f570f63ddad814370374", size = 296113 }, + { url = "https://files.pythonhosted.org/packages/ab/18/0b9ffc597e69c5a1e20a7823cb60d54b39a9f54e91edcb8574f022186758/tomli-2.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f79203b3965b4000e91808aaa7c040206093f2b8bf86f455982f2274c9ccf442", size = 307725 }, + { url = "https://files.pythonhosted.org/packages/ab/c7/18f8baae0b5607a60e8e19b4a7fedee43a8ff6458e3896dcbbadeeac9c22/tomli-2.5.0-cp314-cp314t-win32.whl", hash = "sha256:91294a9fb94a75542f6e46e4a2ae709bd8d9b51134098cae5cf3bea5478b6d03", size = 108546 }, + { url = "https://files.pythonhosted.org/packages/72/34/4cca9739254130627bde87500b3f2b512154fe2f278efa7e2a5e10ad4bcb/tomli-2.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f15e3e0b835a6d68b10c86bf80a3149780498d6911c93c3ffd1861d19f9200f1", size = 117814 }, + { url = "https://files.pythonhosted.org/packages/7d/fb/afa530d47dd80a78fce43beac6bc6e00f84558eafcffbc6f37b21e80d056/tomli-2.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:6664b7ae7af7294256c53960a6103077f4914cec8ff98479c352f622c6f6b2f0", size = 115188 }, + { url = "https://files.pythonhosted.org/packages/66/98/316fdc00f8c0939e6fe50461dd343c162d3ad51d1286eb25b7db54361d50/tomli-2.5.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:a525685c2f97da40762b8695eb7aa0af4c8344ca1905c73e4e29cb04d34607dc", size = 162775 }, + { url = "https://files.pythonhosted.org/packages/c5/22/7b10fa5bb01c9539f53f69b619361b19350acc73657772ea7ac70ba309a8/tomli-2.5.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:9dbb18c1cfb2f6517942fc9314437f66aa06d94436ffb1f06102ef3572f35276", size = 161406 }, + { url = "https://files.pythonhosted.org/packages/9c/e7/1a069d86dfd20f1f84f71c63faed9f83c1d890bc06c27d82dc7d888fb573/tomli-2.5.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:752e8b1aa6a4367ef8bf6a1a1e005540f7ed055ba36d7193796812ca5404eb52", size = 273855 }, + { url = "https://files.pythonhosted.org/packages/ae/83/d1ef43d1687d092ab9c235455c76e6e709483b346b056f086095c7c263a5/tomli-2.5.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c47300f9bf791808f77d82747691c4bb09cb14bdf3060cca99b42cdc4361d5a7", size = 284910 }, + { url = "https://files.pythonhosted.org/packages/cc/05/f4d9cf7de61822ece0c3873f30d291e324911c71a378b8bfe5ced13fd9f5/tomli-2.5.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:19b0dd8749f4ea2f112c5fcfb3c5248390c899d7e2e173f1d91abee1fa0ff391", size = 277723 }, + { url = "https://files.pythonhosted.org/packages/42/28/78262493141fa543151cf005760c3cb01d09fc28a11f993c05109902cb8c/tomli-2.5.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:57b1c3b01fab802e2899bc3d168dca320e14165e2fd9fd584760fb4ca5826859", size = 285115 }, + { url = "https://files.pythonhosted.org/packages/1a/b9/e1dab9a30bcb677b5cc5cee810609cfd64f24306a3055767dd3fda00b1e0/tomli-2.5.0-cp315-cp315-win32.whl", hash = "sha256:667e521b37a6c5ccaa044202c235b530f90177ffe2cd4a64ecc213c7dd535feb", size = 103475 }, + { url = "https://files.pythonhosted.org/packages/4c/bd/31a3790c11d6ea95fcf5e6022ac0f8d0543c9b61120b730fc481bd43d3b4/tomli-2.5.0-cp315-cp315-win_amd64.whl", hash = "sha256:d747252933c8a65ef6bd8da0fbb7ce28a90eb6119d8cd00772cd528aa07b68d5", size = 114589 }, + { url = "https://files.pythonhosted.org/packages/47/a2/4f6310fa699364f0e3af7ee3af88dddd9af066d33e716a0265bbe2b3ea84/tomli-2.5.0-cp315-cp315-win_arm64.whl", hash = "sha256:75dbcde8751b0a960aa3de173aa5e894d590755c6d7758b7e774c06f1dc3cbdd", size = 111493 }, + { url = "https://files.pythonhosted.org/packages/68/14/00853f0b396d8971107ae1921bb5b322fdee1650d2f16bf06c20adb532e5/tomli-2.5.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:2419c2a189551987b59d80e63ec355671283336f41c6b9b89462df679c7d0c57", size = 171380 }, + { url = "https://files.pythonhosted.org/packages/89/ad/fa6949321dadee46b27363974fb197b94c911c3b0f7a5fd26d7dc18fc2a0/tomli-2.5.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:0dc598040da8d42cf20f0be588ed7004f46db12a0ac6c32e03a59dccedaaadcd", size = 170553 }, + { url = "https://files.pythonhosted.org/packages/53/aa/3056c919eb3e084df3752b2cf5f865dcc04af0b27dba2f66d7b28af4633a/tomli-2.5.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49096930c8d886c9bbdab62d2d0d17ce823ddeea522309a190b36245d5b49e01", size = 294428 }, + { url = "https://files.pythonhosted.org/packages/96/b2/faeeb5d8769ea3832021d73e892c8391eae7b4b4f8b55a789127bd8b18a9/tomli-2.5.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8ade5023067f99fe72b88accd30d0ea05a158e9e32a11f124e731ea9695313f", size = 304909 }, + { url = "https://files.pythonhosted.org/packages/f6/52/f094c09e73fb654b621716d019acb5d29bdfd1be01df80c281d552bda48d/tomli-2.5.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:b69564772b5c8f22ea5f498dff08cfa825045b4d4c4400529000bdf818aa3b2a", size = 293220 }, + { url = "https://files.pythonhosted.org/packages/86/f5/0c30541078ca4b505ce3bd76ed931facbfec524dd018535d691d1af0a6d2/tomli-2.5.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:8ff3a2ca028c7eee0c777f9a092038d0a594a9fa04e215f929a22c329e2cb142", size = 305705 }, + { url = "https://files.pythonhosted.org/packages/05/74/590e7d19d6a118fc5cc5704ff358e21d95b8573f6b9443b1519f29ca8825/tomli-2.5.0-cp315-cp315t-win32.whl", hash = "sha256:62fc1bc8eb03e3a9cadfca713d65614ed8e09d974a283295ffe3a831976b4dc5", size = 108432 }, + { url = "https://files.pythonhosted.org/packages/1c/b8/63a75cfb27a17c38550e44025d3a6e7be64516fd8608a3b75703bf37d81b/tomli-2.5.0-cp315-cp315t-win_amd64.whl", hash = "sha256:f3fcbc57b1791fa6cbe5d8434179d51de12be1a4811469529f47f6e7487a2571", size = 117281 }, + { url = "https://files.pythonhosted.org/packages/72/01/e8c1debb2173973372934c68fc8e46170ab60ef23ed4592dff4dec6e8993/tomli-2.5.0-cp315-cp315t-win_arm64.whl", hash = "sha256:d2ba24db8a9376921b5e87b4762b9adb0f3f1deaea68f2b8b0bb2c11efb9c3e7", size = 115069 }, + { url = "https://files.pythonhosted.org/packages/60/3f/3e3f8fd0919249b0200c80fbc4f9a1e70be19f9883da71dfb7f8b9ab8aca/tomli-2.5.0-py3-none-any.whl", hash = "sha256:32a7b79ac57a2e83670ce329ccf675798bc5a2094783a63676866b70503f2e2b", size = 14765 }, +] + [[package]] name = "typing-extensions" version = "4.16.0" @@ -2336,33 +2100,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/0c/b54a4fdd7f90a3af8b02ebc9ce6712c2c208b7926a2f7bad95c33ebbe943/uvicorn-0.54.0-py3-none-any.whl", hash = "sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf", size = 87427 }, ] -[[package]] -name = "watchdog" -version = "6.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/24/d9be5cd6642a6aa68352ded4b4b10fb0d7889cb7f45814fb92cecd35f101/watchdog-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6eb11feb5a0d452ee41f824e271ca311a09e250441c262ca2fd7ebcf2461a06c", size = 96393 }, - { url = "https://files.pythonhosted.org/packages/63/7a/6013b0d8dbc56adca7fdd4f0beed381c59f6752341b12fa0886fa7afc78b/watchdog-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef810fbf7b781a5a593894e4f439773830bdecb885e6880d957d5b9382a960d2", size = 88392 }, - { url = "https://files.pythonhosted.org/packages/d1/40/b75381494851556de56281e053700e46bff5b37bf4c7267e858640af5a7f/watchdog-6.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:afd0fe1b2270917c5e23c2a65ce50c2a4abb63daafb0d419fde368e272a76b7c", size = 89019 }, - { url = "https://files.pythonhosted.org/packages/39/ea/3930d07dafc9e286ed356a679aa02d777c06e9bfd1164fa7c19c288a5483/watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948", size = 96471 }, - { url = "https://files.pythonhosted.org/packages/12/87/48361531f70b1f87928b045df868a9fd4e253d9ae087fa4cf3f7113be363/watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860", size = 88449 }, - { url = "https://files.pythonhosted.org/packages/5b/7e/8f322f5e600812e6f9a31b75d242631068ca8f4ef0582dd3ae6e72daecc8/watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0", size = 89054 }, - { url = "https://files.pythonhosted.org/packages/68/98/b0345cabdce2041a01293ba483333582891a3bd5769b08eceb0d406056ef/watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c", size = 96480 }, - { url = "https://files.pythonhosted.org/packages/85/83/cdf13902c626b28eedef7ec4f10745c52aad8a8fe7eb04ed7b1f111ca20e/watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134", size = 88451 }, - { url = "https://files.pythonhosted.org/packages/fe/c4/225c87bae08c8b9ec99030cd48ae9c4eca050a59bf5c2255853e18c87b50/watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b", size = 89057 }, - { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079 }, - { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078 }, - { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076 }, - { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077 }, - { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078 }, - { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077 }, - { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078 }, - { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065 }, - { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070 }, - { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067 }, -] - [[package]] name = "werkzeug" version = "3.1.9" @@ -2386,3 +2123,31 @@ sdist = { url = "https://files.pythonhosted.org/packages/c7/79/12135bdf8b9c9367b wheels = [ { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405 }, ] + +[[package]] +name = "zensical" +version = "0.0.70" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "deepmerge" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "pathspec" }, + { name = "pygments" }, + { name = "pymdown-extensions" }, + { name = "pyyaml" }, + { name = "tomli" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/4e/a21d2d91b49766fc64be0d9e9a67782f5bfa0710ce76ae464dbec69d0649/zensical-0.0.70.tar.gz", hash = "sha256:ed5fb8c6ee439ae8d1a27ba606e10c651918499184a50aa135f1d5d83a54c4db", size = 4319716 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/ee/5d4fbe0d24bba9e797b7c4e636184b48cf5eee0d5598ee15fa5231515225/zensical-0.0.70-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2b16dfbc12d2cdf1ab3ef52984831c9db5d202b0e508329de383f0aace277d81", size = 18127509 }, + { url = "https://files.pythonhosted.org/packages/fb/b0/61bb824448493c5db9a5dacf050326e946f91a3dbe704cfa6953d3639660/zensical-0.0.70-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:e9579ada6e14e659b86833d6a1723fa5f947b10ec66e65fc059b05dc8068ec8b", size = 17660685 }, + { url = "https://files.pythonhosted.org/packages/f1/bc/0d9959bf065b8187aaddd42b237e7864d57fc1f30df5b8cf07b2c5284721/zensical-0.0.70-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b187d69ce1563bc24c248b93da468c7bc11b8cc269f231505bff0e45e2e2e32", size = 18166247 }, + { url = "https://files.pythonhosted.org/packages/ca/a9/6f95078f2c16ca014caec8ee986d34ec1f137441c13ec6eb0153cf5033e7/zensical-0.0.70-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:78397d29a2bc2b69b2d46f1f39f64b197e204c0403f343da339c3d73ea692b34", size = 18100648 }, + { url = "https://files.pythonhosted.org/packages/02/fe/ceaeaa19b8caed3978c2d3a65ab97b6ba34659200b20dded0da91b7e3138/zensical-0.0.70-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eec6d8b92a7e00986d8c62d813cf37371ec03a3d5e11383363c979ca98ef0d14", size = 18642573 }, + { url = "https://files.pythonhosted.org/packages/7a/2f/3ba313d98d7aa8b13ce8e1730fdf571ff1bacfe7e1c883870b9a384d0d15/zensical-0.0.70-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d11e08cf0a0e0572aa3cc6adc17f88a691f0c6211f45ea386f5483b79019cf2b", size = 18333292 }, + { url = "https://files.pythonhosted.org/packages/66/25/982a0ccf753e733ab79a889b39491b93c7be2db7479e659f9398dc387dbb/zensical-0.0.70-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:78fe80274362b800d1b4d9d5cbdb3c7459206c48cfbac0f7eea0833ae8c41c1e", size = 18899896 }, + { url = "https://files.pythonhosted.org/packages/43/e5/0c5784851cc179b7ff33810047d2654208bee99997e74205646eed06f602/zensical-0.0.70-cp311-abi3-win_amd64.whl", hash = "sha256:c892d5306eb287a0cccc2f6e1c1206bb5eb7e8d4017a6dea329643de65af81fe", size = 18831199 }, + { url = "https://files.pythonhosted.org/packages/3d/96/09d43fea6eabcf55ae13338f657c08a232d6c6c40e1f7fb3d91723a2eddf/zensical-0.0.70-cp311-abi3-win_arm64.whl", hash = "sha256:d072895c7c4b09a95cc33de259bd540a6891497c79df6f8aefc24c25ad604d5f", size = 18247622 }, +] diff --git a/zensical.toml b/zensical.toml new file mode 100644 index 0000000..cd76dfa --- /dev/null +++ b/zensical.toml @@ -0,0 +1,94 @@ +[project] +site_name = "mongospecs" +site_url = "https://jaykv.github.io/mongospecs/" +repo_url = "https://github.com/jaykv/mongospecs" +edit_uri = "edit/main/docs/" + +nav = [ + { "Home" = "index.md" }, + { "Getting started" = "tutorial.md" }, + { "Guide" = [ + { "Models" = "guide/models.md" }, + { "Choosing a backend" = "guide/backends.md" }, + { "Queries" = "guide/queries.md" }, + { "Writes and transactions" = "guide/writes.md" }, + { "Relations" = "guide/relations.md" }, + { "Hooks and plugins" = "guide/plugins.md" }, + { "Errors" = "guide/errors.md" }, + { "Running in production" = "guide/operations.md" }, + { "Tuning for production" = "guide/production.md" }, + ] }, + { "Web" = [ + { "Resources" = "web/resources.md" }, + { "Authentication and authorization" = "web/auth.md" }, + { "Testing your API" = "web/testing.md" }, + { "Plugs" = "web/plugs.md" }, + ] }, + { "Benchmarks" = "benchmarks.md" }, + { "Coming from Mongoose" = "parity.md" }, + { "Migrating from 0.1.x" = [ + { "Guide" = "migration.md" }, + { "v1 test inventory" = "v1-test-inventory.md" }, + ] }, +] + +[project.theme] +language = "en" +features = [ + "announce.dismiss", + "content.action.edit", + "content.action.view", + "content.code.annotate", + "content.code.copy", + "content.code.select", + "content.footnote.tooltips", + "content.tabs.link", + "content.tooltips", + "navigation.footer", + "navigation.indexes", + "navigation.instant", + "navigation.instant.prefetch", + "navigation.path", + "navigation.sections", + "navigation.tabs", + "navigation.tabs.sticky", + "navigation.top", + "navigation.tracking", + "search.highlight", + "toc.follow", +] + +[[project.theme.palette]] +media = "(prefers-color-scheme)" +toggle.icon = "lucide/sun-moon" +toggle.name = "Switch to light mode" + +[[project.theme.palette]] +media = "(prefers-color-scheme: light)" +scheme = "default" +toggle.icon = "lucide/sun" +toggle.name = "Switch to dark mode" + +[[project.theme.palette]] +media = "(prefers-color-scheme: dark)" +scheme = "slate" +toggle.icon = "lucide/moon" +toggle.name = "Switch to system preference" + +[project.markdown_extensions] +abbr = {} +admonition = {} +attr_list = {} +footnotes = {} +md_in_html = {} +toc.permalink = true +pymdownx.details = {} +pymdownx.emoji.emoji_generator = "zensical.extensions.emoji.to_svg" +pymdownx.emoji.emoji_index = "zensical.extensions.emoji.twemoji" +pymdownx.magiclink = {} +pymdownx.snippets.base_path = ["."] +pymdownx.snippets.check_paths = true +pymdownx.superfences = {} +pymdownx.tabbed.alternate_style = true +pymdownx.tabbed.combine_header_slug = true +pymdownx.tasklist.custom_checkbox = true