diff --git a/graqle/benchmarks/benchmark_runner.py b/graqle/benchmarks/benchmark_runner.py index b88a86b0..576607a7 100644 --- a/graqle/benchmarks/benchmark_runner.py +++ b/graqle/benchmarks/benchmark_runner.py @@ -559,7 +559,8 @@ async def _run_one( graph.set_default_backend(self.backend) start = time.perf_counter() - result: ReasoningResult = await graph.areason(question) + # internal=True (W3, ADR-245): benchmarking is tooling, not user reasoning. + result: ReasoningResult = await graph.areason(question, internal=True) latency = (time.perf_counter() - start) * 1000 # Extract governance stats from metadata diff --git a/graqle/benchmarks/run_multigov_v2.py b/graqle/benchmarks/run_multigov_v2.py index 3208b8d0..f9eff5c4 100644 --- a/graqle/benchmarks/run_multigov_v2.py +++ b/graqle/benchmarks/run_multigov_v2.py @@ -318,7 +318,8 @@ async def run_v2_benchmark( graph._orchestrator = orchestrator cg_start = time.perf_counter() - result: ReasoningResult = await graph.areason(q.question) + # internal=True (W3, ADR-245): benchmarking is tooling, not user reasoning. + result: ReasoningResult = await graph.areason(q.question, internal=True) cg_latency = (time.perf_counter() - cg_start) * 1000 cg_f1 = f1_score(result.answer, q.expected_answer) diff --git a/graqle/benchmarks/run_multigov_v3.py b/graqle/benchmarks/run_multigov_v3.py index 8934afdd..35d1cd81 100644 --- a/graqle/benchmarks/run_multigov_v3.py +++ b/graqle/benchmarks/run_multigov_v3.py @@ -389,7 +389,8 @@ async def run_v3_benchmark( graph._orchestrator = orchestrator cg_start = time.perf_counter() - result: ReasoningResult = await graph.areason(q.question) + # internal=True (W3, ADR-245): benchmarking is tooling, not user reasoning. + result: ReasoningResult = await graph.areason(q.question, internal=True) cg_latency = (time.perf_counter() - cg_start) * 1000 # Get actual cost delta diff --git a/graqle/cli/commands/debate.py b/graqle/cli/commands/debate.py index 974e860b..4b044c20 100644 --- a/graqle/cli/commands/debate.py +++ b/graqle/cli/commands/debate.py @@ -111,6 +111,22 @@ def debate( ) orchestrator = DebateOrchestrator(debate_config, pool, cost_gate) + # W3 (ADR-245): a debate is real (multi-panelist) reasoning that never reaches + # graph.areason(), so the wall must be applied here explicitly — otherwise + # `graq debate` is a free unlimited-reasoning surface. + from graqle.licensing.reasoning_gate import check_reasoning_quota + from graqle.licensing.reasoning_quota import ReasoningQuotaExceeded + + try: + check_reasoning_quota() + except ReasoningQuotaExceeded as exc: + console.print( + f"\n[bold red]Reasoning quota reached.[/bold red] {exc}\n" + " Upgrade for unlimited reasoning: " + "[bold cyan]https://graqle.com/pricing[/bold cyan]" + ) + raise typer.Exit(1) from None + trace = asyncio.run(orchestrator.run(query)) # Display debate results diff --git a/graqle/cli/main.py b/graqle/cli/main.py index ee3c3844..2d308b6d 100644 --- a/graqle/cli/main.py +++ b/graqle/cli/main.py @@ -14,6 +14,13 @@ import sys import typer +from contextlib import contextmanager + +# W3 (ADR-245): the wall itself lives at the SDK primitive; the CLI only needs the +# exception type so it can render an upgrade CTA instead of a traceback. +from graqle.licensing.reasoning_quota import ( + ReasoningQuotaExceeded as _ReasoningQuotaExceeded, +) from graqle.cli.commands.activate import activate_command from graqle.cli.commands.audit import audit_command @@ -147,6 +154,30 @@ def main( console = create_console() +@contextmanager +def _reasoning_quota_cta(): + """W3 (ADR-245): render the upgrade CTA when the reasoning wall fires. + + This does NOT meter — the wall lives at the SDK primitive (``graph.areason``) so + that CLI, MCP, chat and api are all covered by one gate (ADR-245 Decision 8; the + CLI-only placement was the PR #316 bypass). Metering here too would double-charge + every CLI reasoning call. This wrapper only turns the primitive's exception into a + friendly CTA + clean exit instead of a traceback. + """ + try: + yield + except _ReasoningQuotaExceeded as exc: + console.print( + f"\n[bold red]Reasoning quota reached.[/bold red] {exc}\n" + "[dim]The free tier includes a monthly reasoning allowance; your graph and " + "all reads still work.[/dim]\n" + " Upgrade for unlimited reasoning: " + "[bold cyan]https://graqle.com/pricing[/bold cyan] " + "(or register free: https://graqle.com/signup)" + ) + raise typer.Exit(1) from None + + # CR-004 PR-004c (sentinel 1C dedup + regex correction): # ANSI CSI / OSC strip helpers used by the degraded-reasoning warning # in run() and reason(). Extracted to module scope so the regexes are @@ -442,10 +473,13 @@ def run( if coordinator: graph.config.coordinator.enabled = True - # Run reasoning with selected protocol - result = asyncio.run( - graph.areason(query, max_rounds=max_rounds, strategy=strategy) - ) + # W3 (ADR-245): the quota wall fires inside graph.areason(); this only turns it + # into an upgrade CTA + clean exit instead of a traceback. + with _reasoning_quota_cta(): + # Run reasoning with selected protocol + result = asyncio.run( + graph.areason(query, max_rounds=max_rounds, strategy=strategy) + ) # CR-004 PR-004c: probe + degraded-reasoning yellow warning. # The probe never raises (PR-004a 3-deep defence); outer try is @@ -966,12 +1000,14 @@ def bench( console.print(f"[cyan]Benchmarking {queries} queries, {max_rounds} max rounds...[/cyan]") + # W3 (ADR-245): benchmarking is tooling, not user reasoning — internal=True so a + # 50-query bench never burns (or hard-blocks on) a free contributor's quota. # Run queries sequentially with fail-fast: stop on first backend error start = time.perf_counter() results = [] for i, q in enumerate(test_queries): try: - r = asyncio.run(graph.areason(q, max_rounds=max_rounds)) + r = asyncio.run(graph.areason(q, max_rounds=max_rounds, internal=True)) results.append(r) except Exception as e: console.print(f"\n[red]Query {i + 1}/{queries} failed: {e}[/red]") @@ -2663,12 +2699,16 @@ def reason( raise typer.Exit(1) console.print(f"Batch: [green]{len(queries)} queries[/green] (max_concurrent={max_concurrent})") - results = asyncio.run( - graph.areason_batch( - queries, max_rounds=max_rounds, strategy=strategy, - max_concurrent=max_concurrent, + # W3 (ADR-245): areason_batch fans out to areason once per query, so an + # N-query batch costs N quota units — it consumes N LLM invocations, exactly + # like N separate calls. The CTA wrapper renders the wall if it fires mid-batch. + with _reasoning_quota_cta(): + results = asyncio.run( + graph.areason_batch( + queries, max_rounds=max_rounds, strategy=strategy, + max_concurrent=max_concurrent, + ) ) - ) total_cost = sum(r.cost_usd for r in results) total_latency = sum(r.latency_ms for r in results) @@ -2713,9 +2753,12 @@ def reason( # ── Single query mode ───────────────────────────────────────────── console.print(f"Query: [green]{query}[/green]") - result = asyncio.run( - graph.areason(query, max_rounds=max_rounds, strategy=strategy) - ) + # W3 (ADR-245): the quota wall fires inside graph.areason(); this only turns it + # into an upgrade CTA + clean exit instead of a traceback. + with _reasoning_quota_cta(): + result = asyncio.run( + graph.areason(query, max_rounds=max_rounds, strategy=strategy) + ) # CR-004 PR-004c: probe + degraded-reasoning yellow warning. Same # pattern as the run() command above. Skipped silently in JSON diff --git a/graqle/core/graph.py b/graqle/core/graph.py index 2b290cb7..fd8876f0 100644 --- a/graqle/core/graph.py +++ b/graqle/core/graph.py @@ -1853,6 +1853,11 @@ def reason( (Claude Code, Cursor, Codex) for query enhancement. task_type: Optional task type for routing (e.g. "reason", "context"). """ + # NOT internal: this wrapper delegates to areason(), which is the single + # billing wall and charges exactly once for this call. Passing internal=True + # here would EXEMPT the call outright (internal means "never metered", not + # "already metered") — making the whole synchronous reason() API free and + # recreating the PR #316 bypass in mirror image. Delegation charges once. return asyncio.run( self.areason( query, @@ -1873,6 +1878,7 @@ async def areason( node_ids: list[str] | None = None, context: Any = None, task_type: str | None = None, + internal: bool = False, ) -> ReasoningResult: """Run async reasoning query — the core entry point. @@ -1881,7 +1887,19 @@ async def areason( (Claude Code, Cursor, Codex) for query enhancement before PCST activation . task_type: Optional task type for task-based model routing (v0.22). + internal: True when this reasoning is invoked *by* another metered + action (a governance scan, a benchmark runner). Exempt from + the W3 reasoning quota so one user action is never charged + twice. See ADR-245 Decision 8. """ + # W3 (ADR-245): the reasoning-quota wall. This is the SDK primitive every + # surface funnels through — CLI, MCP `graq_reason`, chat agent, api.py — so + # gating here covers all of them. Raises ReasoningQuotaExceeded for a FREE + # user over the monthly cap; fail-open on any metering fault. + from graqle.licensing.reasoning_gate import check_reasoning_quota + + check_reasoning_quota(internal=internal) + from graqle.orchestration.orchestrator import Orchestrator max_rounds = max_rounds or self.config.orchestration.max_rounds @@ -2081,13 +2099,25 @@ async def areason_stream( strategy: str | None = None, node_ids: list[str] | None = None, context: Any = None, + internal: bool = False, ) -> AsyncIterator: """Stream reasoning results as they become available. Usage: async for chunk in graph.areason_stream("query"): print(chunk.content) + + Args: + internal: True when invoked by another metered action — exempt from + the W3 reasoning quota (ADR-245 Decision 8). """ + # W3 (ADR-245): streaming builds its own StreamingOrchestrator and never + # reaches areason(), so it needs the wall explicitly — without this it is a + # free unlimited-reasoning surface (the PR #316 failure mode). + from graqle.licensing.reasoning_gate import check_reasoning_quota + + check_reasoning_quota(internal=internal) + from graqle.orchestration.streaming import StreamingOrchestrator max_rounds = max_rounds or self.config.orchestration.max_rounds diff --git a/graqle/licensing/reasoning_gate.py b/graqle/licensing/reasoning_gate.py new file mode 100644 index 00000000..b3d9d9d8 --- /dev/null +++ b/graqle/licensing/reasoning_gate.py @@ -0,0 +1,148 @@ +"""Reasoning-quota middleware (W3, ADR-245 Decision 8) — one wall, every surface. + +The quota logic lives in :mod:`graqle.licensing.reasoning_quota`; this module is the +single *enforcement point* that the SDK reasoning primitive calls on entry. + +Why a middleware and not a CLI hook: the first W3 attempt (PR #316, CLOSED) placed the +wall in ``graqle/cli/main.py`` only. The MCP server's ``graq_reason`` tool, the chat +agent, and ``api.py`` all reach :meth:`graqle.core.graph.Graqle.areason` *without* going +through the CLI — so a primary surface had free unlimited reasoning. The rule that came +out of that review: **a wall at one surface is not a wall**. Enforce at the primitive. + +``Graqle.areason()`` is the sole chokepoint — sync ``reason()`` delegates to it, and +``areason_batch()`` fans out to it per query — so gating ``areason`` alone covers every +caller exactly once. ``areason_batch`` deliberately does NOT gate: doing so would charge +a 5-query batch six times. + +Exemption: ``internal=True`` — reasoning invoked *by* another metered action (a W2 +PR-Guardian scan, a benchmark runner). Prevents W2/W3 double-charging one user action. +``internal=True`` is the ONLY exemption. A CI environment variable is deliberately not +honoured — ``export CI=true`` is self-attested and would be a one-line bypass of the +whole wall (sentinel BLOCKER-2). A 50-query benchmark stays free because the benchmark +runners pass ``internal=True`` in code, which an end user cannot set from outside. +""" + +from __future__ import annotations + +import logging +import os +import sys +from pathlib import Path + +logger = logging.getLogger("graqle.licensing.reasoning_gate") + +__all__ = [ + "QUOTA_DIR_ENV", + "check_reasoning_quota", + "quota_exempt", + "resolve_quota_dir", +] + +# Test/bench override for the project-local quota directory. +QUOTA_DIR_ENV = "GRAQLE_QUOTA_DIR" + +_DEFAULT_QUOTA_DIR = ".graqle" + +def _under_pytest() -> bool: + """True only when pytest is genuinely loaded in THIS process. + + Deliberately ``sys.modules``, not an environment variable. The first attempt at + this guard used ``PYTEST_CURRENT_TEST``, which is itself a self-attested env var — + it merely replaced one bypass with another, and was proven forgeable: + ``PYTEST_CURRENT_TEST=fake::call GRAQLE_QUOTA_DIR=/tmp/x`` resolved to ``/tmp/x``. + An attacker cannot import pytest into a process that never imported it, so + ``sys.modules`` closes what an env check cannot. + + It is also the only correct answer for TIMING. ``PYTEST_CURRENT_TEST`` is set + per-test (setup/call/teardown) and is absent during module import, collection, and + session-scoped fixtures — measured: at import time it is ``False`` while + ``"pytest" in sys.modules`` is already ``True``. With the env check, a test that + resolved the quota dir at import or in a session fixture would have silently fallen + through to the developer's REAL ``./.graqle`` and mutated their actual quota count. + """ + return "pytest" in sys.modules or "_pytest" in sys.modules + + +def resolve_quota_dir() -> Path: + """Project-local ``.graqle`` directory holding the quota file. + + ``GRAQLE_QUOTA_DIR`` relocates the meter, but ONLY under pytest. + + ⚠️ It is deliberately INERT in normal runs. Honouring it unconditionally made it a + one-line unlimited-reasoning bypass of this very wall: a free user at the cap who + exported ``GRAQLE_QUOTA_DIR=$(mktemp -d)`` got a fresh empty counter on every + invocation. That is categorically worse than the local-bypass class ADR-245 already + accepts (deleting ``.graqle/reasoning_quota.json``), on three counts — it persists + via a shell profile so it is set once and never repeated, it is non-destructive so + nothing looks tampered with, and it needs no repeat action. + + It also contradicted this module's own rule: :func:`quota_exempt` refuses to honour + ``CI=true`` precisely because "self-attested env exemption" is banned by ADR-245 + Decision 8 rule 3. An env var that relocates the counter is the same bypass wearing + a different hat. Tests still need to redirect the meter, so the override survives — + scoped to a real pytest process, which a production run is not. + """ + override = os.environ.get(QUOTA_DIR_ENV) + if override and override.strip() and _under_pytest(): + return Path(override) + return Path(_DEFAULT_QUOTA_DIR) + + +def quota_exempt(internal: bool) -> bool: + """True when this reasoning call must be neither counted nor blocked. + + ``internal`` is the ONLY exemption. A CI environment variable is deliberately + NOT honoured: ``export CI=true`` is self-attested and would be a one-line + unlimited-reasoning bypass of the whole wall. Tooling that legitimately must not + be metered (the benchmark runners, governance sub-calls) passes ``internal=True`` + at the callsite, which is code the user cannot set from the outside. + """ + return bool(internal) + + +def check_reasoning_quota(*, internal: bool = False) -> None: + """Enforce the monthly reasoning quota for ONE reasoning invocation. + + Parameters + ---------- + internal: + True when this reasoning is invoked by another metered action (W2 scan, + benchmark runner, or an internal SDK self-call). Exempt: not counted, never + blocked. + + Raises + ------ + ReasoningQuotaExceeded + when enforcement is on, the verified tier is FREE, and the monthly quota is + spent. This is the ONLY exception this function propagates. + + Any other failure — a missing module, an unreadable quota file, a licensing + error — is swallowed. Reasoning must never break because metering hiccuped. + """ + if quota_exempt(internal): + return + + # Imported lazily: graqle.core.graph imports this module, so a module-level import + # of anything that reaches back into core would be circular. + try: + from graqle.licensing.reasoning_quota import ReasoningQuota, ReasoningQuotaExceeded + except Exception as exc: # noqa: BLE001 — metering unavailable → fail open + # WARNING: an un-importable meter means the wall is off for this call. + logger.warning( + "reasoning quota module unavailable (%s) — allowing this call", exc + ) + return + + try: + ReasoningQuota(resolve_quota_dir()).check_and_record(internal=False) + except ReasoningQuotaExceeded: + # Must escape BEFORE the broad except below — this is the wall doing its job, + # not a metering fault. Swallowing it here would silently disable W3. + raise + except Exception as exc: # noqa: BLE001 — never break reasoning on a meter fault + # WARNING, not debug: this branch grants un-metered reasoning. It must be + # visible in logs, otherwise a corrupt quota file or a permissions problem + # silently disables the wall with no operational signal. + logger.warning( + "reasoning quota could not be enforced (%s) — allowing this call", exc + ) diff --git a/graqle/licensing/reasoning_quota.py b/graqle/licensing/reasoning_quota.py new file mode 100644 index 00000000..e5f8aefa --- /dev/null +++ b/graqle/licensing/reasoning_quota.py @@ -0,0 +1,268 @@ +"""Reasoning-quota wall (W3, ADR-245) — free monthly cap on graph reasoning. + +Free tier gets a bounded number of ``graq run`` / ``graq reason`` invocations per +calendar month; paid tiers (Pro/Team/Enterprise) are unlimited. This is the +*reasoning-frequency* wall in the multi-wall lattice — independent of the node-cap +(size) wall, so a user who stays under the node cap still hits this if they reason a lot. + +Design (consistent with ADR-245 + CR-LIC-03a/03b): + - Tier comes from the VERIFIED licence (``manager.current_tier``), never a raw env/key + presence — an unverified signal must never grant a paid entitlement (CR-LIC-03b rule). + - Enforcement is ON BY DEFAULT; ``GRAQLE_ENFORCE_CAPS`` is an OPT-OUT only (shared with + the node-cap wall). "User sets a flag to be enforced" would be a trivial bypass. + - The count is per calendar month, persisted locally in ``.graqle/reasoning_quota.json``. + Local enforcement is the offline bar (the standard freemium wall); a determined user + can delete the file — server-side accounting is the un-bypassable upgrade (separate CR). + - INTERNAL reasoning (e.g. a PR-Guardian scan that calls reason under the hood) is + EXEMPT via ``internal=True`` so W2 and W3 never double-charge one user action. + - Fail-open on any metering error: a quota-file hiccup must never break reasoning. +""" + +from __future__ import annotations + +import json +import logging +import os +import tempfile +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +try: # POSIX advisory locking + import fcntl +except ImportError: # pragma: no cover — Windows + fcntl = None # type: ignore[assignment] + +try: # Windows mandatory locking + import msvcrt +except ImportError: # pragma: no cover — POSIX + msvcrt = None # type: ignore[assignment] + +logger = logging.getLogger("graqle.licensing.reasoning_quota") + +__all__ = [ + "FREE_REASONS_PER_MONTH", + "QUOTA_FILENAME", + "ReasoningQuotaExceeded", + "ReasoningQuota", + "quota_enforcement_enabled", +] + +QUOTA_FILENAME = "reasoning_quota.json" +FREE_REASONS_PER_MONTH = 30 # generous enough to try; paid = unlimited +_SCHEMA_VERSION = 1 + +# Shared opt-out flag with the node-cap wall (ADR-245): enforcement is ON by default. +_ENFORCE_ENV = "GRAQLE_ENFORCE_CAPS" +_FALSY = frozenset({"0", "false", "no", "off", ""}) + + +class ReasoningQuotaExceeded(Exception): + """Raised when a FREE user exceeds the monthly reasoning quota.""" + + def __init__(self, used: int, limit: int, month: str) -> None: + super().__init__( + f"reasoning quota reached: {used}/{limit} this month ({month})" + ) + self.used = used + self.limit = limit + self.month = month + + +def quota_enforcement_enabled() -> bool: + """True unless explicitly opted out (GRAQLE_ENFORCE_CAPS=falsy). Default ON.""" + raw = os.environ.get(_ENFORCE_ENV) + if raw is None: + return True + return raw.strip().lower() not in _FALSY + + +def _paid_tier() -> bool: + """True iff the VERIFIED licence is a paid tier (Pro/Team/Enterprise). + + Uses manager.current_tier (paid only from a signature+CRL+nonce-verified licence); + never a raw env/key (CR-LIC-03b). Fail-safe: any error → treat as FREE (enforce). + """ + try: + from graqle.licensing.manager import LicenseTier, _get_manager + + return _get_manager().current_tier in ( + LicenseTier.PRO, + LicenseTier.TEAM, + LicenseTier.ENTERPRISE, + ) + except Exception: # noqa: BLE001 + return False + + +# Months of history to keep. One year plus the current month, so a year-on-year +# comparison still has data while the file stays bounded. Without pruning the JSON +# gains a key every month forever — slow, but unbounded, and nothing else trims it. +_RETAIN_MONTHS = 13 + + +def _prune_old_months(data: dict, current: str) -> dict: + """Drop month keys older than ``_RETAIN_MONTHS`` before ``current``. + + Non-month keys (``schema_version``) are preserved untouched, and anything that + does not parse as ``YYYY-MM`` is left alone rather than silently discarded — a + pruner that eats keys it does not understand is a data-loss bug waiting to happen. + """ + try: + cy, cm = (int(p) for p in current.split("-")) + except (ValueError, AttributeError): + return data + cutoff = cy * 12 + cm - _RETAIN_MONTHS + out = {} + for key, value in data.items(): + try: + y, m = (int(p) for p in str(key).split("-")) + except ValueError: + out[key] = value # not a month key (e.g. schema_version) — keep + continue + if y * 12 + m > cutoff: + out[key] = value + return out + + +@dataclass +class QuotaReading: + used: int + limit: int # -1 = unlimited (paid) + month: str + allowed: bool + + +class ReasoningQuota: + """Per-month local reasoning-quota meter under a project's ``.graqle`` dir.""" + + def __init__(self, graqle_dir: Path | str) -> None: + self._path = Path(graqle_dir) / QUOTA_FILENAME + + def _load(self) -> dict: + try: + if self._path.exists(): + data = json.loads(self._path.read_text(encoding="utf-8")) + if isinstance(data, dict): + return data + except Exception: + logger.debug("reasoning quota file unreadable — treating as empty") + return {} + + def _store(self, data: dict) -> None: + try: + self._path.parent.mkdir(parents=True, exist_ok=True) + tmp = self._path.with_suffix(".tmp") + tmp.write_text(json.dumps(data, indent=2), encoding="utf-8") + tmp.replace(self._path) + except Exception: + logger.debug("could not persist reasoning quota (read-only fs?)") + + @staticmethod + def _month() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m") + + def peek(self) -> QuotaReading: + """Current usage without recording. Paid (or enforcement off) → allowed. + + Mirrors ``check_and_record``'s short-circuit exactly. Checking only the tier + here made the two disagree: with ``GRAQLE_ENFORCE_CAPS=0`` at 30/30, ``peek()`` + reported ``allowed=False`` while ``check_and_record`` let the call through, so + any status surface reading ``peek`` would tell a user they were blocked when + they were not. Nothing user-facing consumes ``peek`` today — its only caller is + the exempt-path short-circuit below — but the two must not be allowed to drift. + """ + month = self._month() + if _paid_tier() or not quota_enforcement_enabled(): + return QuotaReading(used=0, limit=-1, month=month, allowed=True) + used = int(self._load().get(month, 0) or 0) + return QuotaReading( + used=used, + limit=FREE_REASONS_PER_MONTH, + month=month, + allowed=used < FREE_REASONS_PER_MONTH, + ) + + @contextmanager + def _locked(self): + """Hold an exclusive cross-process lock on the quota file. + + Concurrent reasoning runs otherwise race the read-modify-write below: two + processes both read ``used=29``, both write ``30``, and the free cap leaks an + extra call per racing process. The lock is advisory and best-effort — if the + platform lock is unavailable we still proceed (fail-open), because a metering + hiccup must never break reasoning. + """ + lock_path = self._path.with_suffix(".lock") + handle = None + try: + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = open(lock_path, "a+b") # noqa: SIM115 — released in finally + if msvcrt is not None: # Windows + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1) + elif fcntl is not None: # POSIX + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + except Exception as exc: # noqa: BLE001 — lock unavailable → proceed unlocked + logger.debug("reasoning quota lock unavailable: %s", exc) + try: + yield + finally: + if handle is not None: + try: + if msvcrt is not None: + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + elif fcntl is not None: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + except Exception: # noqa: BLE001 — closing releases it regardless + logger.debug("reasoning quota lock release failed") + handle.close() + + def check_and_record(self, *, internal: bool = False) -> QuotaReading: + """Enforce + record ONE reasoning invocation. + + Parameters + ---------- + internal: + When True (reasoning invoked by another metered action, e.g. a PR-Guardian + scan), the call is EXEMPT — not counted, never blocked. Prevents W2/W3 + double-charging one user action. + + Raises + ------ + ReasoningQuotaExceeded + when enforcement is on, the tier is FREE, and the monthly quota is used up. + + Never raises on a metering/file error — reasoning must not break on a quota + hiccup (fail-open). + """ + if internal or not quota_enforcement_enabled() or _paid_tier(): + return self.peek() + try: + month = self._month() + # Read-modify-write must be atomic across processes: without the lock two + # concurrent runs both read the same `used` and both write used+1, leaking + # one extra free call per racing process. + with self._locked(): + data = self._load() + used = int(data.get(month, 0) or 0) + if used >= FREE_REASONS_PER_MONTH: + raise ReasoningQuotaExceeded(used, FREE_REASONS_PER_MONTH, month) + data[month] = used + 1 + data["schema_version"] = _SCHEMA_VERSION + self._store(_prune_old_months(data, month)) + return QuotaReading( + used=used + 1, + limit=FREE_REASONS_PER_MONTH, + month=month, + allowed=True, + ) + except ReasoningQuotaExceeded: + raise + except Exception as exc: # noqa: BLE001 — never break reasoning on a meter fault + logger.debug("reasoning quota metering skipped: %s", exc) + return QuotaReading( + used=0, limit=FREE_REASONS_PER_MONTH, month=self._month(), allowed=True + ) diff --git a/tests/test_licensing/test_reasoning_gate.py b/tests/test_licensing/test_reasoning_gate.py new file mode 100644 index 00000000..6384ab4e --- /dev/null +++ b/tests/test_licensing/test_reasoning_gate.py @@ -0,0 +1,811 @@ +"""W3 (ADR-245 Decision 8) — the reasoning wall sits at the SDK primitive. + +PR #316 placed the wall in the CLI only; MCP / chat / api reached ``graph.areason`` +around it. These tests are the regression evidence that the wall is now at the +primitive and that every surface inherits it. + +The suite is deliberately split into two halves: + + 1. Unit tests over ``reasoning_gate`` itself (exemptions, fail-open, escape order). + 2. **Bypass-surface tests** that drive the wall through ``Graqle.areason`` the way + each real consumer does — the CLI, the MCP ``graq_reason`` tool, the chat agent + and ``api.py`` all bottom out in that one call. These are the tests that would + have failed on the #316 architecture. +""" + +from __future__ import annotations + +import asyncio +import json +import re +from pathlib import Path + +import pytest + +from graqle.licensing.reasoning_gate import ( + QUOTA_DIR_ENV, + check_reasoning_quota, + quota_exempt, + resolve_quota_dir, +) +from graqle.licensing.reasoning_quota import ( + FREE_REASONS_PER_MONTH, + QUOTA_FILENAME, + ReasoningQuota, + ReasoningQuotaExceeded, +) + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch, tmp_path): + """FREE tier, enforcement on, no CI, quota file isolated to a temp dir.""" + monkeypatch.setattr("graqle.licensing.reasoning_quota._paid_tier", lambda: False) + monkeypatch.delenv("GRAQLE_ENFORCE_CAPS", raising=False) + for var in ("CI", "GITHUB_ACTIONS"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv(QUOTA_DIR_ENV, str(tmp_path / ".graqle")) + return tmp_path + + +def _paid(monkeypatch): + monkeypatch.setattr("graqle.licensing.reasoning_quota._paid_tier", lambda: True) + + +def _burn(n): + """Consume ``n`` quota units through the gate.""" + for _ in range(n): + check_reasoning_quota() + + +def _used(tmp_path): + """Units recorded in the quota file (0 when the file was never written).""" + path = tmp_path / ".graqle" / QUOTA_FILENAME + if not path.exists(): + return 0 + data = json.loads(path.read_text(encoding="utf-8")) + return sum(v for k, v in data.items() if k != "schema_version") + + +# ── CI detection ──────────────────────────────────────────────────────────── + + + + + + + +# ── exemption matrix ──────────────────────────────────────────────────────── + +def test_internal_is_exempt(): + assert quota_exempt(internal=True) is True + + +def test_plain_user_call_is_not_exempt(): + assert quota_exempt(internal=False) is False + + +def test_ci_env_does_NOT_exempt(monkeypatch): + """SENTINEL BLOCKER-2: `export CI=true` must not buy unlimited free reasoning. + + A CI env var is self-attested — honouring it would be a one-line bypass of the + entire wall. Tooling that must not be metered passes internal=True in code. + """ + monkeypatch.setenv("GITHUB_ACTIONS", "true") + monkeypatch.setenv("CI", "true") + assert quota_exempt(internal=False) is False + + +# ── the wall actually blocks ──────────────────────────────────────────────── + +def test_free_user_blocked_at_cap(tmp_path): + _burn(FREE_REASONS_PER_MONTH) + with pytest.raises(ReasoningQuotaExceeded): + check_reasoning_quota() + + +def test_under_cap_is_allowed(tmp_path): + _burn(FREE_REASONS_PER_MONTH - 1) + check_reasoning_quota() # the last free unit — must not raise + assert _used(tmp_path) == FREE_REASONS_PER_MONTH + + +def test_paid_tier_never_blocked(monkeypatch, tmp_path): + _paid(monkeypatch) + for _ in range(FREE_REASONS_PER_MONTH * 3): + check_reasoning_quota() + assert _used(tmp_path) == 0, "paid tier must not be metered at all" + + +def test_internal_calls_never_counted(tmp_path): + """A 50-query benchmark must not burn a free contributor's quota.""" + for _ in range(50): + check_reasoning_quota(internal=True) + assert _used(tmp_path) == 0 + + +def test_internal_calls_pass_even_when_cap_already_spent(tmp_path): + _burn(FREE_REASONS_PER_MONTH) + check_reasoning_quota(internal=True) # must not raise — bench is exempt + + +def test_ci_env_cannot_lift_a_spent_cap(monkeypatch, tmp_path): + """SENTINEL BLOCKER-2 regression guard: setting CI after the cap must still block.""" + _burn(FREE_REASONS_PER_MONTH) + monkeypatch.setenv("CI", "true") + with pytest.raises(ReasoningQuotaExceeded): + check_reasoning_quota() + + +def test_optout_disables_the_wall(monkeypatch, tmp_path): + _burn(FREE_REASONS_PER_MONTH) + monkeypatch.setenv("GRAQLE_ENFORCE_CAPS", "0") + check_reasoning_quota() # opt-out honoured + + +# ── fail-open + escape ordering ───────────────────────────────────────────── + +def test_metering_fault_fails_open(monkeypatch): + """A meter fault must never break reasoning.""" + def boom(self, *, internal=False): + raise OSError("disk gone") + + monkeypatch.setattr(ReasoningQuota, "check_and_record", boom) + check_reasoning_quota() # swallowed + + +def test_quota_exceeded_escapes_before_the_broad_except(monkeypatch): + """The wall firing is NOT a metering fault — it must propagate. + + Regression guard for the fail-open escape-hatch bug class: if the + ReasoningQuotaExceeded re-raise is moved below the broad `except Exception`, + the wall is silently disabled and this test fails. + """ + def raise_wall(self, *, internal=False): + raise ReasoningQuotaExceeded(30, 30, "2026-07") + + monkeypatch.setattr(ReasoningQuota, "check_and_record", raise_wall) + with pytest.raises(ReasoningQuotaExceeded): + check_reasoning_quota() + + +def test_import_failure_fails_open(monkeypatch): + """If the quota module cannot be imported, reasoning still runs.""" + import builtins + + real_import = builtins.__import__ + + def fake_import(name, *a, **kw): + if name == "graqle.licensing.reasoning_quota": + raise ImportError("simulated") + return real_import(name, *a, **kw) + + monkeypatch.setattr(builtins, "__import__", fake_import) + check_reasoning_quota() # swallowed + + +# ── quota dir resolution ──────────────────────────────────────────────────── + +def test_quota_dir_override(monkeypatch): + monkeypatch.setenv(QUOTA_DIR_ENV, "/tmp/somewhere") + assert resolve_quota_dir() == Path("/tmp/somewhere") + + +def test_quota_dir_defaults_to_project_local(monkeypatch): + monkeypatch.delenv(QUOTA_DIR_ENV, raising=False) + assert resolve_quota_dir() == Path(".graqle") + + +# ═══════════════════════════════════════════════════════════════════════════ +# BYPASS-SURFACE TESTS — the #316 regression guards. +# +# Every surface (CLI, MCP graq_reason, chat agent, api.py) reaches +# Graqle.areason(). These drive that primitive directly, which is what each +# surface does, and assert the wall is inherited. +# ═══════════════════════════════════════════════════════════════════════════ + +@pytest.fixture +def graph(monkeypatch): + """A Graqle instance whose reasoning is stubbed — we test the gate, not the LLM.""" + from graqle.core.graph import Graqle + + g = Graqle() + + async def fake_orchestrate(*a, **kw): + raise AssertionError("orchestrator reached — the wall did not fire") + + return g, fake_orchestrate + + +def _areason_reaches_orchestrator(g): + """True when areason got past the gate (our stub marks that with a sentinel).""" + marker = {"ran": False} + + async def run(): + try: + await g.areason("q") + except ReasoningQuotaExceeded: + raise + except Exception: + # Any post-gate failure (no backend configured, etc.) still proves the + # gate ALLOWED the call through — which is what we are asserting. + marker["ran"] = True + else: + marker["ran"] = True + + asyncio.run(run()) + return marker["ran"] + + +def test_areason_signature_accepts_internal(): + """The primitive must expose the exemption knob its callers rely on.""" + import inspect + + from graqle.core.graph import Graqle + + assert "internal" in inspect.signature(Graqle.areason).parameters + + +def test_areason_blocks_free_user_over_cap(graph, tmp_path): + """THE #316 REGRESSION GUARD. + + On the old architecture the wall lived in the CLI, so calling areason directly — + exactly what MCP graq_reason, the chat agent and api.py do — sailed straight + through. This asserts the primitive itself refuses. + """ + g, _ = graph + _burn(FREE_REASONS_PER_MONTH) + + with pytest.raises(ReasoningQuotaExceeded): + asyncio.run(g.areason("this must not reason")) + + +def test_areason_internal_bypasses_cap(graph, tmp_path): + """Internal reasoning (bench / governance sub-call) is exempt at the primitive.""" + g, _ = graph + _burn(FREE_REASONS_PER_MONTH) + + # Must get PAST the gate. It may fail later for unrelated reasons (no backend); + # what matters is that ReasoningQuotaExceeded is not raised. + assert _areason_reaches_orchestrator_internal(g) + + +def _areason_reaches_orchestrator_internal(g): + async def run(): + try: + await g.areason("q", internal=True) + except ReasoningQuotaExceeded: + return False + except Exception: + return True + return True + + return asyncio.run(run()) + + +def test_sync_reason_is_walled(graph, tmp_path): + """graph.reason() is a public API too — it must not be a free side door. + + It delegates to areason WITHOUT internal=True, so it is charged exactly once. + """ + g, _ = graph + _burn(FREE_REASONS_PER_MONTH) + + with pytest.raises(ReasoningQuotaExceeded): + g.reason("this must not reason") + + +def test_batch_charges_per_query(monkeypatch, tmp_path): + """areason_batch fans out to areason, so N queries cost N units — not 1. + + Charging once per batch would under-bill by N x; charging at BOTH the batch and + the fan-out would double-bill. This pins the agreed semantics. + """ + from graqle.core.graph import Graqle + + g = Graqle() + calls = {"n": 0} + + async def fake_areason(self, query, **kw): + calls["n"] += 1 + check_reasoning_quota(internal=kw.get("internal", False)) + return None + + monkeypatch.setattr(Graqle, "areason", fake_areason) + asyncio.run(g.areason_batch(["q1", "q2", "q3"], max_concurrent=1)) + + assert calls["n"] == 3 + assert _used(tmp_path) == 3, "a 3-query batch must cost exactly 3 quota units" + + +# ── concurrency: the file-locking guard ───────────────────────────────────── + +def test_concurrent_runs_do_not_leak_quota(tmp_path): + """Two racing processes must not both spend the same last unit. + + Without the cross-process lock, both read used=N-1 and both write N, letting a + free user exceed the cap by one call per racing process. + """ + import threading + + _burn(FREE_REASONS_PER_MONTH - 1) # exactly one unit left + + granted, blocked = [], [] + barrier = threading.Barrier(8) + + def worker(): + barrier.wait() # maximise the race + try: + check_reasoning_quota() + granted.append(1) + except ReasoningQuotaExceeded: + blocked.append(1) + + threads = [threading.Thread(target=worker) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert len(granted) == 1, f"exactly one thread may take the last unit, got {len(granted)}" + assert len(blocked) == 7 + assert _used(tmp_path) == FREE_REASONS_PER_MONTH + + +# ── verified-tier boundary (CR-LIC-03b): the real _paid_tier path ─────────── +# The autouse fixture stubs _paid_tier, so these exercise the UNSTUBBED function +# to prove an unverified hint never grants a paid (unlimited) entitlement. + +def _real_paid_tier(monkeypatch, tier=None, raises=False): + """Drive the genuine _paid_tier() against a faked licence manager.""" + import graqle.licensing.reasoning_quota as rq + monkeypatch.undo() # drop the autouse _paid_tier stub + + import sys + import types + + mod = types.ModuleType("graqle.licensing.manager") + + class LicenseTier: + FREE, PRO, TEAM, ENTERPRISE = "free", "pro", "team", "enterprise" + + class _Mgr: + current_tier = tier + + def _get_manager(): + if raises: + raise RuntimeError("licence store unreadable") + return _Mgr() + + mod.LicenseTier = LicenseTier + mod._get_manager = _get_manager + monkeypatch.setitem(sys.modules, "graqle.licensing.manager", mod) + return rq._paid_tier + + +@pytest.mark.parametrize("tier", ["pro", "team", "enterprise"]) +def test_verified_paid_tiers_are_unlimited(monkeypatch, tier): + paid = _real_paid_tier(monkeypatch, tier=tier) + assert paid() is True + + +def test_verified_free_tier_is_metered(monkeypatch): + paid = _real_paid_tier(monkeypatch, tier="free") + assert paid() is False + + +def test_unresolvable_licence_is_treated_as_free(monkeypatch): + """Fail-SAFE, not fail-open: if the tier cannot be verified, meter the user. + + An unverified signal must never grant a paid entitlement (CR-LIC-03b). + """ + paid = _real_paid_tier(monkeypatch, raises=True) + assert paid() is False + + +# ── lock + persistence robustness ─────────────────────────────────────────── + +def test_posix_lock_path_is_exercised(monkeypatch, tmp_path): + """Cover the fcntl branch that POSIX users (and most CI) actually take.""" + import graqle.licensing.reasoning_quota as rq + + calls = [] + + class FakeFcntl: + LOCK_EX, LOCK_UN = 2, 8 + + @staticmethod + def flock(fd, op): + calls.append(op) + + monkeypatch.setattr(rq, "msvcrt", None) # pretend not-Windows + monkeypatch.setattr(rq, "fcntl", FakeFcntl) + + rq.ReasoningQuota(tmp_path).check_and_record() + assert FakeFcntl.LOCK_EX in calls, "exclusive lock never acquired on POSIX" + assert FakeFcntl.LOCK_UN in calls, "lock never released on POSIX" + + +def test_lock_unavailable_still_meters(monkeypatch, tmp_path): + """A platform without working locks must still enforce (degraded, not open).""" + import graqle.licensing.reasoning_quota as rq + + class Boom: + LOCK_EX, LOCK_UN = 2, 8 + + @staticmethod + def flock(fd, op): + raise OSError("flock unsupported on this fs") + + monkeypatch.setattr(rq, "msvcrt", None) + monkeypatch.setattr(rq, "fcntl", Boom) + + r = rq.ReasoningQuota(tmp_path).check_and_record() + assert r.used == 1, "metering must continue when the lock is unavailable" + + +def test_lock_release_failure_is_swallowed(monkeypatch, tmp_path): + """A failed unlock must not surface — closing the handle releases it anyway.""" + import graqle.licensing.reasoning_quota as rq + + class HalfBroken: + LOCK_EX, LOCK_UN = 2, 8 + + @staticmethod + def flock(fd, op): + if op == 8: # only the release fails + raise OSError("unlock failed") + + monkeypatch.setattr(rq, "msvcrt", None) + monkeypatch.setattr(rq, "fcntl", HalfBroken) + + r = rq.ReasoningQuota(tmp_path).check_and_record() + assert r.used == 1 + + +def test_readonly_fs_does_not_break_reasoning(monkeypatch, tmp_path): + """A read-only filesystem must not raise — persistence is best-effort.""" + import graqle.licensing.reasoning_quota as rq + + def no_write(self, data): + raise OSError("read-only filesystem") + + monkeypatch.setattr(rq.ReasoningQuota, "_store", no_write) + r = rq.ReasoningQuota(tmp_path).check_and_record() + assert r.allowed is True + + +def test_store_swallows_write_error(monkeypatch, tmp_path): + """Exercise _store's own handler (not a monkeypatched replacement). + + Covers the read-only-fs branch inside _store: a failed persist is logged and + swallowed, never raised into the reasoning path. + """ + import graqle.licensing.reasoning_quota as rq + + q = rq.ReasoningQuota(tmp_path) + + def bad_replace(self, target): + raise OSError("read-only filesystem") + + monkeypatch.setattr(Path, "replace", bad_replace) + q._store({"2026-07": 1}) # must not raise + + +# ═══════════════════════════════════════════════════════════════════════════ +# SENTINEL BLOCKER-1 regression guards — surfaces that do NOT route through +# areason() and therefore need the wall applied explicitly. +# ═══════════════════════════════════════════════════════════════════════════ + +def test_areason_stream_is_walled(tmp_path): + """areason_stream builds its own StreamingOrchestrator and never calls areason. + + Found by sentinel pass 1: without an explicit gate this is a free unlimited + reasoning surface (it backs the server's SSE /reason/stream endpoint). + """ + from graqle.core.graph import Graqle + + g = Graqle() + _burn(FREE_REASONS_PER_MONTH) + + async def drain(): + async for _ in g.areason_stream("q"): + pass + + with pytest.raises(ReasoningQuotaExceeded): + asyncio.run(drain()) + + +def test_areason_stream_accepts_internal(): + import inspect + + from graqle.core.graph import Graqle + + params = inspect.signature(Graqle.areason_stream).parameters + assert "internal" in params + assert params["internal"].kind is inspect.Parameter.KEYWORD_ONLY + + +def test_every_public_reasoning_entrypoint_is_walled(): + """Structural guard: if someone adds a new public reasoning entrypoint to + Graqle, it must be walled (or explicitly listed as delegating). + + This is the test that would have caught areason_stream before review. + """ + import inspect + + from graqle.core.graph import Graqle + + # areason — gates directly + # areason_stream — gates directly (own orchestrator) + # reason — delegates to areason (charged there) + # areason_batch — fans out to areason (charged per query) + known = {"reason", "areason", "areason_stream", "areason_batch"} + found = { + name + for name, _ in inspect.getmembers(Graqle, callable) + if name.endswith("reason") or name.startswith("areason") or name == "reason" + } + unexpected = found - known + assert not unexpected, ( + f"new public reasoning entrypoint(s) {unexpected} — each must either gate " + "via check_reasoning_quota() or provably delegate to areason()" + ) + + +def test_gating_entrypoints_reference_the_gate(): + """The two self-gating entrypoints must actually call the middleware.""" + import inspect + + from graqle.core.graph import Graqle + + for name in ("areason", "areason_stream"): + src = inspect.getsource(getattr(Graqle, name)) + assert "check_reasoning_quota" in src, f"{name} does not call the wall" + + +def test_internal_is_keyword_only_on_areason(): + """SENTINEL BLOCKER-3 hardening: `internal` must not be positionally settable.""" + import inspect + + from graqle.core.graph import Graqle + + p = inspect.signature(Graqle.areason).parameters["internal"] + assert p.kind is inspect.Parameter.KEYWORD_ONLY + + +def test_server_request_model_cannot_set_internal(): + """SENTINEL BLOCKER-3: no HTTP request body field may map to `internal`. + + The server passes explicit named fields to areason(); `internal` is not one of + them and is not a field on the request model, so a caller cannot inject it. + + Run in a SUBPROCESS, deliberately. Importing a ``graqle.server`` module in-process + leaves it in ``sys.modules`` for the rest of the session, and + ``graqle.governance.tamper_evidence.verifier`` refuses to import when a + server/studio module is already loaded (the moat-M2 ``_assert_isolated`` guard, + WS-A3). That turned this one import into a cross-file failure for every later + test that touches the verifier — it surfaced as 11 unrelated failures in + ``tests/test_cli/test_headless_contract.py`` when the suites ran together, while + each file passed alone. A subprocess keeps the assertion and contains the import. + + The models live in ``graqle.server.models``, NOT ``graqle.server.app``. The + original form of this test imported ``ReasonRequest`` from ``app`` — which does + not define it — under a bare ``except Exception: pytest.skip(...)``, so the + ImportError was swallowed and this security assertion NEVER ran on any machine. + A skip-on-any-exception guard around an import is indistinguishable from a pass. + Only ModuleNotFoundError for the genuinely-optional extra may skip. + """ + import subprocess + import sys + + # Guard the interpreter FIRST. CI runs the bare `pytest` shim (.github/ + # workflows/ci.yml:42), not `python -m pytest`, so sys.executable is not + # guaranteed to be the venv that has graqle installed. If it is not, the probe + # below would exit 3 and SKIP — silently losing this assertion again, which is + # the exact defect this rewrite exists to fix. Fail LOUD instead of skipping: + # a wrong interpreter is an environment bug, never a reason to drop coverage. + base = subprocess.run( + [sys.executable, "-c", "import graqle"], capture_output=True, text=True + ) + assert base.returncode == 0, ( + f"probe interpreter {sys.executable!r} cannot import graqle, so a skip here " + "would be indistinguishable from a pass. Fix the environment (or run pytest " + f"as `python -m pytest`). stderr: {base.stderr.strip()}" + ) + + # ImportError, not ModuleNotFoundError: a genuinely-absent optional extra can + # surface as a plain ImportError rather than the subclass — e.g. a transitive + # C-extension wheel that fails to load, or a broken re-export in __init__. + # Narrowing to ModuleNotFoundError would hard-fail CI on a machine where the + # extra is simply not installed. graqle itself is proven importable above, so + # this cannot mask a broken core install. + probe = ( + "import sys\n" + "try:\n" + " from graqle.server.models import BatchReasonRequest, ReasonRequest\n" + "except ImportError:\n" + " sys.exit(3)\n" # optional server extra genuinely absent → skip + "bad = [m.__name__ for m in (ReasonRequest, BatchReasonRequest)\n" + " if 'internal' in m.model_fields]\n" + "print(','.join(bad))\n" + "sys.exit(1 if bad else 0)\n" + ) + result = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True) + + if result.returncode == 3: + pytest.skip("server extra not installed") + assert result.returncode == 0, ( + "an HTTP request model exposes an 'internal' field, so a caller could set " + f"internal=True and get unmetered reasoning: {result.stdout.strip()} " + f"(stderr: {result.stderr.strip()})" + ) + + +# ═══════════════════════════════════════════════════════════════════════════ +# Sentinel pass-2 refutations, pinned as tests. +# +# Pass 2 raised BLOCKER-4 (app.py + MCP tools might pass internal=True) and +# BLOCKER-5 (chat agent / ReasoningCoordinator might bypass). Both were refuted +# by code inspection. These tests pin the refutations so a future change cannot +# quietly make them true. +# ═══════════════════════════════════════════════════════════════════════════ + +def _sdk_py_files(): + import pathlib + + root = pathlib.Path(__file__).resolve().parents[2] / "graqle" + return list(root.rglob("*.py")) + + +def test_internal_true_only_used_by_sanctioned_tooling(): + """SENTINEL BLOCKER-4: `internal=True` must never appear in a user-facing path. + + Only benchmark tooling may exempt itself. If a server endpoint, MCP tool or + chat handler ever passes internal=True it becomes permanently un-metered — + a billing bypass. This test enumerates every occurrence in the package. + """ + sanctioned = {"benchmark_runner.py", "run_multigov_v2.py", "run_multigov_v3.py"} + offenders = [] + for path in _sdk_py_files(): + if path.name in sanctioned or path.parts[-2:] == ("licensing", "reasoning_gate.py"): + continue + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + continue + for lineno, line in enumerate(text.splitlines(), 1): + # Match a real keyword ARGUMENT (preceded by "(" or ", "), not prose + # in a docstring/comment that merely mentions the flag. + if not re.search(r"[(,]\s*internal\s*=\s*True", line): + continue + if line.lstrip().startswith("#"): + continue + if '"""' in line or line.lstrip().startswith("``"): + continue + if True: + # graq bench is the CLI face of the sanctioned benchmark tooling. + if path.name == "main.py" and "max_rounds=max_rounds" in line: + continue + offenders.append(f"{path.name}:{lineno}: {line.strip()}") + + assert not offenders, ( + "internal=True found outside sanctioned benchmark tooling — these calls " + "would be permanently exempt from the reasoning quota:\n" + "\n".join(offenders) + ) + + +def test_reasoning_coordinator_is_only_reachable_through_the_gate(): + """SENTINEL BLOCKER-5: ReasoningCoordinator must not be a parallel un-gated path. + + It is constructed only inside Graqle._areason_coordinated, which is called only + from areason() — i.e. downstream of the wall. If someone constructs it elsewhere, + that new site would bypass the quota. + """ + import graqle.core.graph as graph_mod + + sites = [] + for path in _sdk_py_files(): + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + continue + for lineno, line in enumerate(text.splitlines(), 1): + if "ReasoningCoordinator(" in line and "class " not in line: + sites.append((path.name, lineno)) + + assert sites, "expected at least the known construction site" + assert all(name == "graph.py" for name, _ in sites), ( + f"ReasoningCoordinator constructed outside core/graph.py: {sites} — " + "each such site is an un-gated reasoning path" + ) + + # ...and that site must sit inside the gated call chain. + import inspect + + src = inspect.getsource(graph_mod.Graqle.areason) + assert "_areason_coordinated" in src, ( + "the coordinator path is no longer reached from the gated areason()" + ) + + +def test_chat_package_has_no_ungated_reasoning(): + """SENTINEL BLOCKER-5: graqle/chat/ must not dispatch reasoning of its own.""" + import pathlib + + chat = pathlib.Path(__file__).resolve().parents[2] / "graqle" / "chat" + if not chat.exists(): + pytest.skip("no chat package") + + offenders = [] + for path in chat.rglob("*.py"): + text = path.read_text(encoding="utf-8", errors="ignore") + for lineno, line in enumerate(text.splitlines(), 1): + stripped = line.strip() + if stripped.startswith("#"): + continue + if "Orchestrator(" in line or ".areason_stream(" in line: + offenders.append(f"{path.name}:{lineno}: {stripped}") + assert not offenders, ( + "chat package dispatches reasoning outside the gated entrypoints:\n" + + "\n".join(offenders) + ) + + +# ═══════════════════════════════════════════════════════════════════════════ +# Pre-merge review findings, pinned as regression tests. +# +# F1 (BLOCKER): GRAQLE_QUOTA_DIR was honoured unconditionally, so a free user at +# the cap got a fresh counter from `export GRAQLE_QUOTA_DIR=$(mktemp -d)` — +# unlimited free reasoning, persistent via shell profile, non-destructive, no +# repeat action. It contradicted this module's own refusal to honour CI=true +# (ADR-245 Decision 8 rule 3: no self-attested env exemptions). +# ═══════════════════════════════════════════════════════════════════════════ + +def test_quota_dir_override_is_inert_outside_pytest(tmp_path): + """The override must NOT relocate the meter in a real (non-pytest) process. + + Checked in a SUBPROCESS: inside this test pytest is necessarily in sys.modules, + so the production path is unreachable in-process. The subprocess also FORGES + PYTEST_CURRENT_TEST, pinning the sentinel's refutation — an env marker must not + be enough to unlock the override, because the first version of this guard used + exactly that var and was proven bypassable. + """ + import subprocess + import sys as _sys + + probe = ( + "import os, sys\n" + "os.environ['GRAQLE_QUOTA_DIR'] = r'{esc}'\n" + "os.environ['PYTEST_CURRENT_TEST'] = 'forged::call'\n" # the attack + "from graqle.licensing.reasoning_gate import resolve_quota_dir\n" + "print(resolve_quota_dir())\n" + ).format(esc=str(tmp_path / "escape")) + out = subprocess.run( + [_sys.executable, "-c", probe], capture_output=True, text=True + ) + + assert out.returncode == 0, out.stderr + resolved = out.stdout.strip() + assert resolved != str(tmp_path / "escape"), ( + "GRAQLE_QUOTA_DIR relocated the quota file in a production process — that is " + "a one-line unlimited-reasoning bypass of the W3 wall. Forging " + "PYTEST_CURRENT_TEST must not unlock it." + ) + assert resolved == str(Path(".graqle")) + + +def test_quota_dir_override_still_works_under_pytest(monkeypatch, tmp_path): + """...but tests must still be able to redirect the meter. + + Note this passes with PYTEST_CURRENT_TEST *deleted*: the guard keys off pytest + being in sys.modules, which is also true during import, collection and + session-scoped fixtures — phases where PYTEST_CURRENT_TEST is unset. With the + old env-based guard, a test resolving the dir in those phases would have silently + written to the developer's real ./.graqle. + """ + from graqle.licensing.reasoning_gate import QUOTA_DIR_ENV, resolve_quota_dir + + monkeypatch.setenv(QUOTA_DIR_ENV, str(tmp_path / "sandbox")) + monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False) + + assert resolve_quota_dir() == tmp_path / "sandbox" diff --git a/tests/test_licensing/test_reasoning_quota.py b/tests/test_licensing/test_reasoning_quota.py new file mode 100644 index 00000000..43db8528 --- /dev/null +++ b/tests/test_licensing/test_reasoning_quota.py @@ -0,0 +1,208 @@ +"""W3 (ADR-245) — free monthly reasoning-quota wall.""" + +from __future__ import annotations + +import json + +import pytest + +from graqle.licensing.reasoning_quota import ( + FREE_REASONS_PER_MONTH, + QUOTA_FILENAME, + ReasoningQuota, + ReasoningQuotaExceeded, + quota_enforcement_enabled, +) + + +@pytest.fixture(autouse=True) +def _free_tier(monkeypatch): + """Default every test to a FREE (unverified) tier unless it overrides.""" + monkeypatch.setattr( + "graqle.licensing.reasoning_quota._paid_tier", lambda: False + ) + monkeypatch.delenv("GRAQLE_ENFORCE_CAPS", raising=False) + + +def _paid(monkeypatch): + monkeypatch.setattr( + "graqle.licensing.reasoning_quota._paid_tier", lambda: True + ) + + +# ── enforcement flag (shared with node-cap) ───────────────────────────────── + +def test_enforcement_on_by_default(monkeypatch): + monkeypatch.delenv("GRAQLE_ENFORCE_CAPS", raising=False) + assert quota_enforcement_enabled() is True + + +@pytest.mark.parametrize("optout", ["0", "false", "no", "off", ""]) +def test_enforcement_opt_out_only(monkeypatch, optout): + monkeypatch.setenv("GRAQLE_ENFORCE_CAPS", optout) + assert quota_enforcement_enabled() is False + + +# ── free-tier counting + block ────────────────────────────────────────────── + +def test_free_records_and_allows_under_cap(tmp_path): + q = ReasoningQuota(tmp_path) + r = q.check_and_record() + assert r.allowed and r.used == 1 and r.limit == FREE_REASONS_PER_MONTH + # persisted + data = json.loads((tmp_path / QUOTA_FILENAME).read_text(encoding="utf-8")) + assert sum(v for k, v in data.items() if k != "schema_version") == 1 + + +def test_free_blocks_after_cap(tmp_path): + q = ReasoningQuota(tmp_path) + for _ in range(FREE_REASONS_PER_MONTH): + q.check_and_record() + with pytest.raises(ReasoningQuotaExceeded): + q.check_and_record() + + +def test_block_carries_numbers(tmp_path): + q = ReasoningQuota(tmp_path) + for _ in range(FREE_REASONS_PER_MONTH): + q.check_and_record() + with pytest.raises(ReasoningQuotaExceeded) as ei: + q.check_and_record() + assert ei.value.used == FREE_REASONS_PER_MONTH + assert ei.value.limit == FREE_REASONS_PER_MONTH + + +# ── paid tier: unlimited, never blocks ────────────────────────────────────── + +def test_paid_tier_unlimited(tmp_path, monkeypatch): + _paid(monkeypatch) + q = ReasoningQuota(tmp_path) + for _ in range(FREE_REASONS_PER_MONTH * 3): + r = q.check_and_record() + assert r.allowed and r.limit == -1 # unlimited + # paid path does not even write the counter + assert not (tmp_path / QUOTA_FILENAME).exists() + + +# ── internal reasoning is EXEMPT (no W2/W3 double-charge) ──────────────────── + +def test_internal_reasoning_exempt(tmp_path): + q = ReasoningQuota(tmp_path) + # Way past the cap, but internal=True → never blocks, never counts. + for _ in range(FREE_REASONS_PER_MONTH * 2): + r = q.check_and_record(internal=True) + assert r.allowed + # A subsequent user (non-internal) call still starts at 1 (internal didn't count). + r = q.check_and_record() + assert r.used == 1 + + +# ── opt-out disables the block ────────────────────────────────────────────── + +def test_opt_out_never_blocks(tmp_path, monkeypatch): + monkeypatch.setenv("GRAQLE_ENFORCE_CAPS", "0") + q = ReasoningQuota(tmp_path) + for _ in range(FREE_REASONS_PER_MONTH + 5): + q.check_and_record() # opted out → never raises + + +# ── fail-open: metering error never breaks reasoning ──────────────────────── + +def test_fail_open_on_meter_error(tmp_path, monkeypatch): + q = ReasoningQuota(tmp_path) + + def _boom(*a, **k): + raise OSError("disk gone") + + monkeypatch.setattr(q, "_load", _boom) + # Must NOT raise (fail-open) — reasoning never breaks on a quota fault. + r = q.check_and_record() + assert r.allowed + + +def test_malformed_json_fails_open(tmp_path): + """A corrupt/partial quota file must fail-open (treated as empty), never crash.""" + (tmp_path / QUOTA_FILENAME).write_text('{"2026-07": "NaN", tru', encoding="utf-8") + q = ReasoningQuota(tmp_path) + # Load treats the bad file as empty → this is the 1st record, allowed. + r = q.check_and_record() + assert r.allowed + + +def test_quota_exceeded_escapes_fail_open(tmp_path): + """MAJOR-2 regression: the block must NOT be swallowed by the fail-open handler. + Even when the quota is exhausted, ReasoningQuotaExceeded propagates.""" + q = ReasoningQuota(tmp_path) + for _ in range(FREE_REASONS_PER_MONTH): + q.check_and_record() + # This must RAISE (not silently pass via fail-open). + with pytest.raises(ReasoningQuotaExceeded): + q.check_and_record() + + +def test_peek_does_not_record(tmp_path): + q = ReasoningQuota(tmp_path) + q.peek() + q.peek() + # peek never writes the counter + assert not (tmp_path / QUOTA_FILENAME).exists() + + +# ── tier-trust: uses VERIFIED tier, not a raw env/key (CR-LIC-03b alignment) ─ + +def test_uses_verified_tier_not_raw_env(tmp_path, monkeypatch): + # Setting a raw env tier must NOT grant unlimited — only a verified paid licence does. + monkeypatch.setenv("GRAQLE_LICENSE_TIER", "enterprise") # unverified + # _paid_tier (verified) is False by the autouse fixture → still FREE-capped. + q = ReasoningQuota(tmp_path) + for _ in range(FREE_REASONS_PER_MONTH): + q.check_and_record() + with pytest.raises(ReasoningQuotaExceeded): + q.check_and_record() # unverified env did NOT buy unlimited + + +# ── Pre-merge review findings (F2, F3), pinned as regression tests ────────── + +def test_peek_matches_check_and_record_when_enforcement_disabled(monkeypatch, tmp_path): + """F3: peek() checked only the tier, so it disagreed with check_and_record. + + With enforcement opted out at the cap, check_and_record ALLOWS the call; peek() + reported allowed=False. Any status surface reading peek would have shown a user + 'blocked' while their calls were succeeding. + """ + import datetime + + from graqle.licensing.reasoning_quota import ( + FREE_REASONS_PER_MONTH, + ReasoningQuota, + ) + + monkeypatch.setenv("GRAQLE_ENFORCE_CAPS", "0") + month = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m") + (tmp_path / "reasoning_quota.json").write_text( + json.dumps({month: FREE_REASONS_PER_MONTH}), encoding="utf-8" + ) + quota = ReasoningQuota(tmp_path) + + assert quota.peek().allowed is True + assert quota.check_and_record().allowed is True # the two must agree + + +def test_old_months_are_pruned(monkeypatch, tmp_path): + """F2: month keys accumulated forever — nothing trimmed the file.""" + import datetime + + from graqle.licensing.reasoning_quota import ReasoningQuota + + monkeypatch.delenv("GRAQLE_ENFORCE_CAPS", raising=False) + seed = {f"{y}-{m:02d}": 1 for y in (2020, 2021, 2022) for m in range(1, 13)} + seed["schema_version"] = 1 + (tmp_path / "reasoning_quota.json").write_text(json.dumps(seed), encoding="utf-8") + + ReasoningQuota(tmp_path).check_and_record() + after = json.loads((tmp_path / "reasoning_quota.json").read_text(encoding="utf-8")) + + stale = [k for k in after if k.startswith(("2020-", "2021-", "2022-"))] + assert not stale, f"old months not pruned: {stale}" + assert after["schema_version"] == 1, "pruning must not eat non-month keys" + assert datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m") in after