From 82e165ce5c64f56208ad15e89aa5611ea5f87882 Mon Sep 17 00:00:00 2001 From: Aviad Date: Sat, 5 Sep 2026 22:34:29 +0300 Subject: [PATCH] feat: complete freshness dependencies and honest scan state (v0.6.0) v0.5.1 fixed results that were wrong. This fixes results that were stale while claiming to be current. Freshness: - Verification recorded only the bounded evidence shown to a user and used it as the invalidation set. Results now record a complete dependency set - handlers, tests, guards, helpers - plus a fingerprint of the claim's surface. - Reproduced and fixed: editing the TEST that justified a route association left the claim FRESH; DELETING an evidence file left it FRESH because the old files row survived with a matching hash; adding a NEW route left a set-level claim FRESH because no recorded file changed. - A dependency that changed, disappeared, or was not seen by the latest scan marks the claim STALE and names it. A result with no recorded dependencies is NEEDS_VERIFICATION, never assumed fresh. Unrelated edits still change nothing. Scan state: - dtc verify recomputes from the last persisted scan, which is not the working tree. It now performs a bounded read-only comparison and reports the relationship before any result. JSON gains evidence_snapshot; the MCP verify_claim tool gains staleness_warning so an agent that cannot see the user's files is told which snapshot it reasoned about. Association: - Tests that request an exact route path (supertest, FastAPI TestClient) now establish association, restoring usefulness that v0.5.1 traded away for correctness. Method identity is preserved: requesting GET /users establishes nothing about POST /users. Name similarity remains a suggestion only. - Routes declared relative to an unresolved router mount prefix are reported as unresolved rather than untested: that gap belongs to DevTime, not the repo. Migration: verifications gains inventory_fingerprint via an idempotent ALTER; pre-v0.6.0 rows keep NULL and are re-verified rather than assumed fresh. Decisions and scan history are untouched. JSON stays schema_version 2. 173 tests (16 new). Version 0.6.0. --- LIMITATIONS.md | 10 +- QUICKSTART.md | 4 +- RELEASE_NOTES_v0.6.0.md | 98 +++++ VERIFICATION.md | 53 ++- pyproject.toml | 4 +- server.json | 4 +- src/devtime/__init__.py | 2 +- src/devtime/cli.py | 31 ++ src/devtime/intelligence/verification.py | 346 +++++++++++++++--- src/devtime/mcp/transport.py | 15 +- src/devtime/scanner/extractors/tests.py | 21 +- tests/integration/test_evidence_precision.py | 8 +- tests/integration/test_verification.py | 353 +++++++++++++++++++ 13 files changed, 878 insertions(+), 71 deletions(-) create mode 100644 RELEASE_NOTES_v0.6.0.md diff --git a/LIMITATIONS.md b/LIMITATIONS.md index fd6bf87..8c2e50e 100644 --- a/LIMITATIONS.md +++ b/LIMITATIONS.md @@ -75,11 +75,11 @@ do. Read this before trusting any single output. - Four built-in claims (route-test-coverage, admin-authorization, billing-webhook-signature, jwt-authentication). User-defined claims are not supported yet, deliberately. -- Route test association is established from test imports only, and a static - import association is not execution coverage. Tests that exercise a route - through a running server (supertest, TestClient) or through an unresolved - helper are reported as unassociated, so this claim currently abstains on many - real repositories. +- Route test association accepts a test that requests the exact route path or + imports the route's implementation. It is still a static association, not + execution coverage. Tests that build URLs dynamically (an f-string or a + settings variable) and routes declared relative to an unresolved router mount + prefix are reported as unresolved rather than untested. - Admin authorization is established only from a guard at the route's own call site. Guards applied by a router mount, a server-wide middleware, or a framework decorator are reported as unresolved. WEAK means DevTime found no diff --git a/QUICKSTART.md b/QUICKSTART.md index e85cdcb..b36c74b 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -27,7 +27,7 @@ python -m venv .venv source .venv/bin/activate # Windows (PowerShell): .venv\Scripts\Activate.ps1 # Windows (Git Bash): source .venv/Scripts/activate pip install -e ".[dev]" -pytest # optional: all tests pass (157 at v0.5.1) +pytest # optional: all tests pass (173 at v0.6.0) ``` ## 3. Create the demo repo @@ -132,7 +132,7 @@ A fresh-clone check was run on the current candidate: - **OS:** Windows 11 (Git Bash) - **Python:** 3.11.9 - **Install:** `pip install -e ".[dev]"` -- **Tests:** all passing (157 at v0.5.1) +- **Tests:** all passing (173 at v0.6.0) - **Demo:** `dtc init` / `dtc scan` / `dtc concepts` / `dtc explain "Billing Webhooks"` all produced the expected output from a clean `git clone`. diff --git a/RELEASE_NOTES_v0.6.0.md b/RELEASE_NOTES_v0.6.0.md new file mode 100644 index 0000000..9d2ba6f --- /dev/null +++ b/RELEASE_NOTES_v0.6.0.md @@ -0,0 +1,98 @@ +# DevTime v0.6.0 - freshness you can rely on, and honest scan state + +v0.5.1 fixed results that were wrong. This release fixes results that were +*stale while claiming to be current*, which is the same failure wearing a +timestamp. + +## A verified claim now goes stale when its real dependencies change + +Verification recorded only the bounded list of evidence shown to a user, and +used that as the invalidation set. The two are not the same thing. Consequences, +all reproduced before being fixed: + +- Editing the **test** that established a route/test association left the result + reported as FRESH. Only route files were fingerprinted. +- **Deleting** an evidence file left the result FRESH, because the old row + survived in the scan database and its hash still matched. +- Adding a **new route** left a claim about "all routes" FRESH, because no + previously recorded file had changed. + +Results now record a complete dependency set, separate from displayed evidence: +the handler files, the tests, the guards and helpers that justified the +conclusion, and a fingerprint of the claim's surface set. A dependency that +changed, disappeared, or was not seen by the latest scan marks the claim STALE +and names the file. A result that recorded nothing to justify it is reported as +NEEDS_VERIFICATION instead of being assumed fresh. + +Precision is preserved: unrelated edits still do not invalidate anything. + +## dtc verify says which snapshot it used + +Verification recomputes from the last persisted scan. That is not your working +tree, and stamping a fresh evaluation timestamp on old evidence implies work +that never happened. DevTime now checks and reports the relationship: + +```text +These results were computed from a scan that no longer matches your working tree. + changed since the scan: src/billing/stripe-webhook.ts +Run dtc scan to verify against current code. +``` + +- JSON output gains `evidence_snapshot`: scan id, scan time, files scanned, and + the working-tree relationship (`matches_scan`, `changed_since_scan`, + `partially_checked`, `never_scanned`). +- The MCP `verify_claim` tool adds an explicit `staleness_warning`, because an + agent cannot see the user's files and must be told which snapshot it is + reasoning about and how to refresh. +- The working-tree check is read-only and bounded, so verification never turns + into a second full scan. + +## Route test association is useful again, legitimately + +v0.5.1 removed the false positives but could only see imports, so it abstained +on the most common real pattern: a test that drives a running app by URL. It now +accepts a test that **requests the exact route path** (supertest, +FastAPI TestClient) as well as one that imports the implementation. + +Route identity still keeps the HTTP method, so a test requesting `GET /users` +establishes nothing about `POST /users`. Name similarity remains a suggestion +that can never raise the status. + +## Unresolved is not the same as untested + +Routes declared relative to a router mount prefix (`APIRouter()` with +`include_router(..., prefix=...)`) have no knowable full URL. Reporting them as +"no test found" blames the repository for a gap in DevTime's analysis. They are +now reported separately: + +```text +10 route(s) are declared relative to a router mount prefix that DevTime did not +resolve, so their full URL is unknown. That is a gap in this analysis, not +evidence that they lack tests. +``` + +## Compatibility + +- The `verifications` table gains `inventory_fingerprint`, added by an + idempotent migration. Existing rows keep NULL and are re-verified rather than + assumed fresh. Recorded decisions and scan history are untouched. +- JSON stays `schema_version: 2`; `evidence_snapshot` and `dependency_count` are + additive. +- No command, claim id, or MCP tool was renamed or removed. +- Test signals gain `requests` metadata; existing databases are re-scanned + normally. + +## Notes + +- 173 passing tests (16 new), including every reproduction above: test edited, + evidence deleted, new route added, unrelated change ignored, empty dependency + set, idempotent migration, stale CLI output, stale MCP disclosure, supertest + and TestClient association, method separation, and non-URL `.get()` calls that + must not be mistaken for requests. +- Known gaps, reported rather than assumed: router-level and application-level + guards, verification reached through an imported helper, dynamic request URLs, + and unresolved mount prefixes. + +## Names + +- PyPI distribution: `devtime-ei`. Python import: `devtime`. CLI: `dtc`. diff --git a/VERIFICATION.md b/VERIFICATION.md index bdd84de..fd301af 100644 --- a/VERIFICATION.md +++ b/VERIFICATION.md @@ -49,12 +49,20 @@ same time: the last verification supported it, but its evidence changed since. | Freshness | Meaning | |-----------|---------| -| FRESH | Evidence files are unchanged since the last verification. | -| STALE | At least one evidence file changed or disappeared. Re-verify. | -| NEEDS_VERIFICATION | The claim has never been verified in this repository. | +| FRESH | Every dependency is unchanged since the last verification. | +| STALE | A dependency changed, disappeared, or the claim's surface set changed. | +| NEEDS_VERIFICATION | Never verified here, or nothing was recorded to justify it. | -Freshness only tracks files that were evidence for the claim. Unrelated changes -never mark a claim stale. +Freshness is computed from a claim's **dependencies**, which are not the same as +the evidence displayed to you. Displayed evidence is a bounded selection; the +dependency set is complete and includes the test files, guards, and helpers that +justified the conclusion. A claim about a set (for example "all routes") also +depends on that set, so a newly added route invalidates it even though no +recorded file changed. + +Only a claim's own dependencies participate, so unrelated edits never mark it +stale. A result that recorded no dependencies is reported as +NEEDS_VERIFICATION rather than assumed fresh. NOT_APPLICABLE matters as much as the others. A repository with no billing code is not "unknown" for a billing claim; the claim simply does not apply, and saying @@ -63,13 +71,17 @@ so plainly is more useful than an ominous UNKNOWN. ## Built-in claims - **route-test-coverage** (id kept for compatibility; now presented as *Route - Test Association*) - "HTTP routes have tests that import their - implementation." Association is established from test imports only, matched on - exact module stems so `users` does not match `superusers`. Route identity keeps - the HTTP method. A test that merely shares a word with a route path is reported - as an unverified suggestion and can never raise the status. Routes defined in - test, example, or fixture files are not application surface and are excluded - from the inventory. A static association is not execution coverage. + Test Association*) - "HTTP routes are referenced by tests that request or + import them." Two evidence levels are accepted: a test that requests the exact + route path (supertest, FastAPI TestClient), or a test that imports the route's + implementation module, matched on exact module stems so `users` does not match + `superusers`. Route identity keeps the HTTP method, so a GET test establishes + nothing about POST on the same path. A test that merely shares a word with a + route path is reported as an unverified suggestion and can never raise the + status. Routes declared relative to an unresolved mount prefix are reported as + unresolved rather than untested. Routes defined in test, example, or fixture + files are excluded from the inventory. A static association is not execution + coverage. - **admin-authorization** (v0.5) - "Administrative routes require an authorization check." Authorization is established only from a guard applied at the route's own call site. Authentication is not authorization: `requireAuth` @@ -120,6 +132,23 @@ When no claim applies, DevTime does not dead-end. It reports what it scanned, what evidence it collected, what would make a claim verifiable, and states plainly that this is a coverage limit rather than a verdict on your code. +## Scan freshness (v0.6) + +Verification recomputes conclusions from the last persisted scan, which is not +the same thing as your working tree. When the two disagree, DevTime says so +before showing any result: + +```text +These results were computed from a scan that no longer matches your working tree. + changed since the scan: src/billing/stripe-webhook.ts +Run dtc scan to verify against current code. +``` + +JSON output carries the same facts under `evidence_snapshot` (scan id, scan +time, files scanned, and the working-tree relationship), and the MCP +`verify_claim` tool adds an explicit `staleness_warning` so an agent that cannot +see your files is told which snapshot it is reasoning about. + ## Trust model - Deterministic and rule-driven. No AI, no network, no code execution. diff --git a/pyproject.toml b/pyproject.toml index cbd0f8c..6d8aad3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "devtime-ei" -version = "0.5.1" +version = "0.6.0" description = "Local-first Engineering Intelligence for software repositories" readme = "README.md" requires-python = ">=3.11" @@ -50,7 +50,7 @@ dev = [ Homepage = "https://github.com/Shakargy/devtime" Repository = "https://github.com/Shakargy/devtime" Issues = "https://github.com/Shakargy/devtime/issues" -"Release Notes" = "https://github.com/Shakargy/devtime/releases/tag/v0.5.1" +"Release Notes" = "https://github.com/Shakargy/devtime/releases/tag/v0.6.0" Demo = "https://youtu.be/1Hiu3Y9J_SI" [project.scripts] diff --git a/server.json b/server.json index b3483ea..6e0b7f3 100644 --- a/server.json +++ b/server.json @@ -8,12 +8,12 @@ "source": "github" }, "websiteUrl": "https://github.com/Shakargy/devtime", - "version": "0.5.1", + "version": "0.6.0", "packages": [ { "registryType": "pypi", "identifier": "devtime-ei", - "version": "0.5.1", + "version": "0.6.0", "transport": { "type": "stdio" } diff --git a/src/devtime/__init__.py b/src/devtime/__init__.py index 410310a..168d17a 100644 --- a/src/devtime/__init__.py +++ b/src/devtime/__init__.py @@ -1,6 +1,6 @@ """DevTime - local-first Engineering Intelligence for repository memory.""" -__version__ = "0.5.1" +__version__ = "0.6.0" # Version metadata (Builder Edition, Chapter 20). EVIDENCE_MODEL = "2026.06.1" diff --git a/src/devtime/cli.py b/src/devtime/cli.py index 2cf28be..3f03e49 100644 --- a/src/devtime/cli.py +++ b/src/devtime/cli.py @@ -157,23 +157,54 @@ def verify( if result.status in ver.APPLICABLE_STATUSES: ver.save_verification(conn, result) + state = ver.scan_state(conn) + if as_json: console.print_json( _json.dumps( { "schema_version": "2", "command": "verify", + "evidence_snapshot": state, "results": [r.to_dict() for r in results], } ) ) return + _print_scan_state(state) _print_report(results, single=bool(claim)) finally: conn.close() +def _print_scan_state(state: dict) -> None: + """Say plainly which snapshot these conclusions came from. + + Verification reads the last persisted scan, not the working tree. When the + two disagree, that must be the first thing a reader sees. + """ + tree = state.get("working_tree") + if tree == "changed_since_scan": + changed = state.get("changed_paths") or [] + console.print( + "[yellow]These results were computed from a scan that no longer " + "matches your working tree.[/yellow]" + ) + for p in changed[:5]: + console.print(f" changed since the scan: {p}", markup=False) + if len(changed) > 5: + console.print(f" ... and {len(changed) - 5} more", markup=False) + console.print("Run [bold]dtc scan[/bold] to verify against current code.") + console.print("") + elif tree == "partially_checked": + console.print( + "[dim]Working-tree comparison was capped; some files were not " + "re-checked against the scan.[/dim]" + ) + console.print("") + + def _print_report(results: list, single: bool) -> None: """Report card: what DevTime can and cannot verify about this repository.""" from devtime.intelligence import verification as ver diff --git a/src/devtime/intelligence/verification.py b/src/devtime/intelligence/verification.py index 3c7d0cb..fe81e44 100644 --- a/src/devtime/intelligence/verification.py +++ b/src/devtime/intelligence/verification.py @@ -31,6 +31,7 @@ from __future__ import annotations +import hashlib import json import re import sqlite3 @@ -123,6 +124,17 @@ class VerificationResult: scan_id: str | None verified_at: str engine_version: str + # v0.6.0: the complete set of files whose content justified this conclusion. + # This is NOT the same as `supporting`, which is a bounded selection shown to + # a human. Displayed evidence may be capped; the dependency set never is, + # because it is what invalidation is computed from. A WEAK result has + # dependencies too: the files it examined are exactly what would change the + # answer. + dependencies: list[str] = field(default_factory=list) + # Fingerprint of the surface the claim reasoned about (e.g. the set of + # application routes). A claim about "all routes" depends on the inventory, + # so adding a new route must invalidate it even if no existing file changed. + inventory: str | None = None def to_dict(self) -> dict: # schema_version 2 (v0.5.0): adds the NOT_APPLICABLE status value. All @@ -142,6 +154,9 @@ def to_dict(self) -> dict: "scan_id": self.scan_id, "verified_at": self.verified_at, "engine_version": self.engine_version, + # Bounded on purpose: the full dependency list is stored locally and + # drives invalidation, but responses stay small. + "dependency_count": len(self.dependencies), } @@ -171,7 +186,7 @@ class ClaimDefinition: "route-test-coverage": ClaimDefinition( slug="route-test-coverage", name="Route Test Association", - statement="HTTP routes have tests that import their implementation.", + statement="HTTP routes are referenced by tests that request or import them.", category="testing", ), "admin-authorization": ClaimDefinition( @@ -219,6 +234,16 @@ def _load_signals(conn: sqlite3.Connection, scan_id: str) -> list[sqlite3.Row]: ).fetchall() +def _inventory_fingerprint(items: list[str]) -> str: + """Stable fingerprint of a surface set (routes, handlers). + + Deterministic and order-independent, so an unchanged repository always + produces the same value while an added or removed member changes it. + """ + joined = "\n".join(sorted(items)) + return hashlib.sha256(joined.encode("utf-8")).hexdigest()[:16] + + def _meta(row: sqlite3.Row) -> dict: try: return json.loads(row["metadata_json"] or "{}") @@ -439,6 +464,14 @@ def _verify_billing_webhook_signature( status = SUPPORTED if n_connected == n_total else WEAK if not signature_tests: missing.append("A test that exercises webhook signature verification.") + dependencies = sorted( + {r["path"] for r in webhook_route_rows} + | verify_files + | {r["path"] for r in verification_rows} + ) + inventory = _inventory_fingerprint( + [_route_label(r) for r in webhook_route_rows] + ) limitations = [ "Signature verification is connected to a handler when both appear in " "the same file. Verification reached through an imported helper is not " @@ -462,6 +495,8 @@ def _verify_billing_webhook_signature( scan_id=scan_id, verified_at=_now(), engine_version=__version__, + dependencies=dependencies, + inventory=inventory, ) if stub_webhooks and not verifications: @@ -537,6 +572,9 @@ def _verify_billing_webhook_signature( scan_id=scan_id, verified_at=_now(), engine_version=__version__, + dependencies=sorted( + {e.path for e in (supporting + verifications + stub_webhooks)} + ), ) @@ -660,6 +698,14 @@ def _verify_jwt_authentication( scan_id=scan_id, verified_at=_now(), engine_version=__version__, + dependencies=sorted( + { + e.path + for e in ( + access_usage + invitation_usage + unclear_usage + jwt_docs + jwt_deps + ) + } + ), ) @@ -681,6 +727,17 @@ def _module_token(path: str) -> str: return stem.lower() +def _is_prefix_relative(route_path: str) -> bool: + """True when a route path cannot identify a full URL on its own. + + FastAPI and Express routers are commonly declared relative to a mount + prefix (`APIRouter()` + `include_router(..., prefix="/api/v1/items")`), so a + declared path of "/" or "/{id}" says nothing about the served URL. + """ + p = (route_path or "").strip() + return p in ("", "/") or p.startswith("/{") or p.startswith("/:") + + def _route_tokens(route_path: str) -> list[str]: """Distinctive, non-generic segments of a route path.""" out = [] @@ -717,6 +774,7 @@ def _verify_route_test_coverage( """ # Aggregate tests per file: imports + a single blob of test names. test_imports: dict[str, set[str]] = {} + test_requests: dict[str, set[tuple[str, str]]] = {} test_blobs: dict[str, list[str]] = {} for row in rows: if row["kind"] != "test": @@ -733,6 +791,14 @@ def _verify_route_test_coverage( imports = test_imports.setdefault(path, set()) for imp in meta.get("imports") or []: imports.add(str(imp).lower()) + reqs = test_requests.setdefault(path, set()) + for req in meta.get("requests") or []: + reqs.add( + ( + str(req.get("method", "")).upper(), + str(req.get("path", "")).lower().rstrip("/") or "/", + ) + ) test_blobs.setdefault(path, []).append(str(row["name"] or "").lower()) # One joined blob per test file keeps name matching linear in test FILES. @@ -777,12 +843,22 @@ def _verify_route_test_coverage( associated: list[tuple[str, str, str]] = [] # (impl path, label, reason) suggested: list[tuple[str, str, str]] = [] # name similarity only unassociated: list[tuple[str, str]] = [] + unresolved: list[tuple[str, str]] = [] # full URL cannot be determined for (impl_path, route_path, method), row in sorted(routes.items()): label = f"{method} {route_path}" if route_path else impl_path token = _module_token(impl_path) reason = "" - # Only level that can support the claim: a test importing this module. - if token and len(token) >= 3: + # Strongest available level: a test that requests this exact endpoint. + norm_path = route_path.rstrip("/") or "/" + for test_path, reqs in test_requests.items(): + if (method, norm_path) in reqs or ("ANY", norm_path) in reqs: + reason = f"{test_path} requests {method} {route_path}" + break + if method == "ANY" and any(p == norm_path for _m, p in reqs): + reason = f"{test_path} requests {route_path}" + break + # Next level: a test importing this module. + if not reason and token and len(token) >= 3: for test_path, stems in test_import_stems.items(): if token in stems: reason = f"{test_path} imports {token}" @@ -799,10 +875,17 @@ def _verify_route_test_coverage( break if hint: suggested.append((impl_path, label, hint)) - unassociated.append((impl_path, label)) + # A prefix-relative path ("/" or "/{id}") does not identify a full URL: + # the router's mount prefix was not resolved, so this is a gap in the + # analysis, not evidence that the route lacks a test. + if _is_prefix_relative(route_path): + unresolved.append((impl_path, label)) + else: + unassociated.append((impl_path, label)) total = len(routes) n_assoc = len(associated) + n_unresolved = len(unresolved) # Evidence shown to the user is bounded; the dependency set used for # invalidation is tracked separately and is not capped. _EVIDENCE_CAP = 25 @@ -820,36 +903,65 @@ def _verify_route_test_coverage( ] why = [ - f"{n_assoc} of {total} routes have a test importing their implementation." + f"{n_assoc} of {total} routes are referenced by a test that requests or " + f"imports them." ] missing: list[str] = [] if n_assoc == total: status = SUPPORTED why.append( - "Every detected route has a test that imports its implementation. " - "This is a static association, not proof the route was executed." + "Every detected route is referenced by a test that requests or imports " + "it. This is a static association, not proof the route was executed " + "or that its assertions cover the behavior." ) else: status = WEAK - why.append( - "Routes without an importing test have no established association." - ) - shown = [label for _, label in unassociated[:8]] - missing.append( - f"Tests importing {total - n_assoc} route(s): " + ", ".join(shown) - + (" ..." if len(unassociated) > 8 else "") - ) + if unassociated: + why.append( + "Routes with no requesting or importing test have no established " + "association." + ) + shown = [label for _, label in unassociated[:8]] + missing.append( + f"A test requesting or importing {len(unassociated)} route(s): " + + ", ".join(shown) + + (" ..." if len(unassociated) > 8 else "") + ) + if unresolved: + why.append( + f"{n_unresolved} route(s) are declared relative to a router mount " + "prefix that DevTime did not resolve, so their full URL is unknown. " + "That is a gap in this analysis, not evidence that they lack tests." + ) + missing.append( + "A resolvable full URL for: " + + ", ".join(label for _, label in unresolved[:6]) + ) if suggested: why.append( f"{len(suggested)} route(s) share vocabulary with a test name but no " - "import was found. Name similarity is a suggestion, not evidence: " + "request or import was found. Name similarity is a suggestion, not " + "evidence: " + suggested[0][2] ) + # Dependencies: every application route file and every test file that was + # considered. Editing a test that justified an association must invalidate + # the result, which v0.5.x did not do because only route files were stored. + dependencies = sorted( + {impl for (impl, _p, _m) in routes.keys()} + | set(test_name_blob.keys()) + | set(test_requests.keys()) + ) + inventory = _inventory_fingerprint( + [f"{m} {p} {impl}" for (impl, p, m) in routes.keys()] + ) + limitations = [ - "Association is established from test imports only. A test that exercises " - "a route indirectly, through a running server, or through a helper that " - "DevTime cannot resolve is reported as unassociated.", + "Association is established from a test that requests the exact route " + "path, or from a test that imports the route's implementation module. A " + "test that reaches a route only through a variable URL, a helper, or a " + "mounted prefix DevTime cannot resolve is reported as unassociated.", "A static import association is not execution coverage. It does not " "establish that the route ran, that assertions covered its behavior, or " "that the test passes.", @@ -875,6 +987,8 @@ def _verify_route_test_coverage( scan_id=scan_id, verified_at=_now(), engine_version=__version__, + dependencies=dependencies, + inventory=inventory, ) @@ -1011,6 +1125,9 @@ def _verify_admin_authorization( + ", ".join(sorted({_route_label(r) for r in unresolved})[:6]) ) + dependencies = sorted({r["path"] for r in admin_routes}) + inventory = _inventory_fingerprint([_route_label(r) for r in admin_routes]) + limitations = [ "Authorization is established only from a guard applied at the route's own " "call site. Guards applied by a router mount, a server-wide middleware, or " @@ -1033,6 +1150,8 @@ def _verify_admin_authorization( scan_id=scan_id, verified_at=_now(), engine_version=__version__, + dependencies=dependencies, + inventory=inventory, ) @@ -1089,6 +1208,7 @@ def _ref(row: sqlite3.Row, observation: str, strength: str) -> EvidenceRef: scan_id TEXT, result_json TEXT NOT NULL, evidence_fingerprints_json TEXT NOT NULL DEFAULT '[]', + inventory_fingerprint TEXT, engine_version TEXT NOT NULL, created_at TEXT NOT NULL ); @@ -1096,32 +1216,54 @@ def _ref(row: sqlite3.Row, observation: str, strength: str) -> EvidenceRef: def ensure_verifications_table(conn: sqlite3.Connection) -> None: - """Idempotent: safe for databases initialized before v0.2.0.""" + """Idempotent: safe for databases initialized before v0.2.0. + + v0.6.0 adds inventory_fingerprint. Existing rows keep NULL, which is read as + "inventory was not tracked when this was recorded" and triggers + re-verification rather than an unjustified FRESH. + """ conn.execute(_VERIFICATIONS_TABLE) + columns = { + row["name"] for row in conn.execute("PRAGMA table_info(verifications)") + } + if "inventory_fingerprint" not in columns: + conn.execute("ALTER TABLE verifications ADD COLUMN inventory_fingerprint TEXT") + conn.commit() def save_verification(conn: sqlite3.Connection, result: VerificationResult) -> str: - """Store an immutable verification result with evidence fingerprints.""" + """Store an immutable verification result with its dependency fingerprints. + + v0.6.0: fingerprints come from the complete dependency set, not from the + bounded list of evidence shown to the user. Previously a route/test claim + stored only route files, so editing the test that justified the association + left the result reported as FRESH. + """ ensure_verifications_table(conn) repo = conn.execute("SELECT id FROM repositories LIMIT 1").fetchone() repo_id = repo["id"] if repo else "unknown" - fingerprints = [ - {"path": e.path, "sha256": e.sha256} - for e in result.supporting - if e.sha256 - ] + + paths: set[str] = set(result.dependencies) + paths |= {e.path for e in result.supporting} # Contradiction evidence participates in freshness too: if the stub changes, # the contradiction must be re-checked. for c in result.contradictions: - fingerprints += [ - {"path": e.path, "sha256": e.sha256} for e in c.evidence if e.sha256 - ] + paths |= {e.path for e in c.evidence} + + sha_by_path = { + row["path"]: row["sha256"] + for row in conn.execute("SELECT path, sha256 FROM files").fetchall() + } + fingerprints = [ + {"path": p, "sha256": sha_by_path.get(p)} for p in sorted(paths) + ] vid = f"ver-{uuid.uuid4().hex[:10]}" conn.execute( "INSERT INTO verifications" "(id, repository_id, claim_slug, status, scan_id, result_json, " - " evidence_fingerprints_json, engine_version, created_at) " - "VALUES (?,?,?,?,?,?,?,?,?)", + " evidence_fingerprints_json, inventory_fingerprint, engine_version, " + " created_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?)", ( vid, repo_id, @@ -1130,6 +1272,7 @@ def save_verification(conn: sqlite3.Connection, result: VerificationResult) -> s result.scan_id, json.dumps(result.to_dict()), json.dumps(fingerprints), + result.inventory, result.engine_version, result.verified_at, ), @@ -1157,6 +1300,82 @@ def load_latest_verification( ) +def scan_state(conn: sqlite3.Connection, root=None) -> dict: + """Describe the evidence snapshot a verification is about to be computed from. + + Verification recomputes conclusions from the last persisted scan. That is + not the same as the working tree, and reporting a fresh evaluation timestamp + without saying so implies evidence that was never collected. This returns + the facts a caller needs to be honest about what was actually examined: + + scan_id / scanned_at which snapshot was used + files_scanned its size + working_tree "unchecked", "matches_scan", or "changed_since_scan" + changed_paths files whose content no longer matches the snapshot + """ + from pathlib import Path + + from devtime import paths as _paths + + row = conn.execute( + "SELECT id, started_at, finished_at, file_count, status FROM scans " + "WHERE status = 'completed' ORDER BY started_at DESC LIMIT 1" + ).fetchone() + if row is None: + return { + "scan_id": None, + "scanned_at": None, + "files_scanned": 0, + "working_tree": "never_scanned", + "changed_paths": [], + } + + state = { + "scan_id": row["id"], + "scanned_at": row["finished_at"] or row["started_at"], + "files_scanned": row["file_count"], + "working_tree": "unchecked", + "changed_paths": [], + } + + # A bounded, read-only check of the files this scan recorded. Hashing is + # capped so a verification never turns into a second full scan. + import hashlib as _hashlib + + repo_root = Path(root) if root else _paths.repo_root() + rows = conn.execute( + "SELECT path, sha256 FROM files WHERE last_seen_scan_id = ? AND sha256 IS NOT NULL", + (row["id"],), + ).fetchall() + if not rows: + return state + + _CHECK_CAP = 400 + changed: list[str] = [] + checked = 0 + for f in rows: + if checked >= _CHECK_CAP: + state["working_tree"] = "partially_checked" + break + target = repo_root / f["path"] + checked += 1 + try: + if not target.exists(): + changed.append(f["path"]) + continue + digest = _hashlib.sha256(target.read_bytes()).hexdigest() + except OSError: + continue + if digest != f["sha256"]: + changed.append(f["path"]) + if changed: + state["working_tree"] = "changed_since_scan" + state["changed_paths"] = sorted(changed)[:20] + elif state["working_tree"] == "unchecked": + state["working_tree"] = "matches_scan" + return state + + def claims_affected_by_paths( conn: sqlite3.Connection, changed_paths: list[str] ) -> list[dict]: @@ -1190,23 +1409,68 @@ def claims_affected_by_paths( def freshness_for(conn: sqlite3.Connection, slug: str) -> tuple[str, list[str]]: - """Compare stored evidence fingerprints against current file hashes. - - Returns (freshness, changed_paths). Never flags unrelated file changes: - only files that were evidence for this claim participate. + """Compare stored dependency fingerprints against the latest scan. + + Returns (freshness, changed_paths). Only files this claim actually depended + on participate, so unrelated edits never mark a claim stale. + + v0.6.0 fixes three ways a result could look current when it was not: + - a dependency file DELETED from the repository (its old row survived in + `files`, so the hash still matched and the claim looked FRESH); + - a dependency file that was never re-seen by the latest scan (ignored, + renamed, or excluded by a policy change); + - a change to the claim's INVENTORY, such as a new route appearing, which + changes the answer without changing any previously recorded file. """ latest = load_latest_verification(conn, slug) if latest is None: return NEEDS_VERIFICATION, [] - _, fingerprints, _ = latest + result_dict, fingerprints, _ = latest + + if not fingerprints: + # Nothing was recorded to justify this result, so "fresh" cannot be + # asserted. Re-verify rather than claim currency we cannot support. + return NEEDS_VERIFICATION, [] + + scan_id = _latest_scan_id(conn) + if scan_id is None: + return NEEDS_VERIFICATION, [] + + current = { + row["path"]: row + for row in conn.execute( + "SELECT path, sha256, last_seen_scan_id FROM files" + ).fetchall() + } + changed: list[str] = [] for fp in fingerprints: - row = conn.execute( - "SELECT sha256 FROM files WHERE path = ? ORDER BY last_seen_scan_id DESC LIMIT 1", - (fp["path"],), - ).fetchone() - if row is None or row["sha256"] != fp["sha256"]: + row = current.get(fp["path"]) + if row is None: + changed.append(fp["path"]) # never seen again + continue + if row["last_seen_scan_id"] != scan_id: + # Present in an older scan only: deleted, renamed, or now ignored. + changed.append(fp["path"]) + continue + if row["sha256"] != fp["sha256"]: changed.append(fp["path"]) + if changed: return STALE, sorted(set(changed)) + + # The recorded surface set must still match what the repository has now. + stored_inventory = conn.execute( + "SELECT inventory_fingerprint FROM verifications WHERE claim_slug = ? " + "ORDER BY created_at DESC LIMIT 1", + (slug,), + ).fetchone() + stored = stored_inventory["inventory_fingerprint"] if stored_inventory else None + if stored is not None: + try: + current_result = verify_claim(conn, slug) + except KeyError: + return FRESH, [] + if current_result.inventory is not None and current_result.inventory != stored: + return STALE, ["(the set of routes or handlers this claim covers changed)"] return FRESH, [] diff --git a/src/devtime/mcp/transport.py b/src/devtime/mcp/transport.py index e9ecc06..10b279c 100644 --- a/src/devtime/mcp/transport.py +++ b/src/devtime/mcp/transport.py @@ -152,7 +152,20 @@ def verify_claim(claim_id: str = "") -> dict: "error": "unknown_claim", "hint": "Call verify_claim with no claim_id to list built-in claims.", } - return result.to_dict() + payload = result.to_dict() + # An agent cannot see the user's working tree, so it must be told + # which snapshot this conclusion came from and whether that snapshot + # is still current. + state = ver.scan_state(conn) + payload["evidence_snapshot"] = state + if state.get("working_tree") == "changed_since_scan": + payload["staleness_warning"] = ( + "This result was computed from a stored scan that no longer " + "matches the working tree. Files changed since the scan: " + + ", ".join(state.get("changed_paths", [])[:5]) + + ". Ask the user to run `dtc scan` before relying on it." + ) + return payload finally: conn.close() diff --git a/src/devtime/scanner/extractors/tests.py b/src/devtime/scanner/extractors/tests.py index 60d5300..9198e7c 100644 --- a/src/devtime/scanner/extractors/tests.py +++ b/src/devtime/scanner/extractors/tests.py @@ -22,6 +22,24 @@ ) +# HTTP requests made by a test: supertest `request(app).get("/users")`, FastAPI +# `client.post("/items")`, fetch-style helpers. Only literals that look like a +# URL path count, so `map.get("key")` is not mistaken for a request (v0.6.0). +_REQUEST_RE = re.compile( + r"""\.(get|post|put|patch|delete|head|options)\(\s*['"](/[^'"]*)['"]""", + re.I, +) + + +def _extract_requests(text: str) -> list[dict]: + seen: list[dict] = [] + for match in _REQUEST_RE.finditer(text): + entry = {"method": match.group(1).upper(), "path": match.group(2)} + if entry not in seen: + seen.append(entry) + return seen[:50] + + def _extract_imports(text: str) -> list[str]: mods: list[str] = [] for m in _IMPORT_RE.finditer(text): @@ -45,6 +63,7 @@ def extract_test_signals(file: WalkedFile) -> list[Signal]: text = read_text(file) e2e = _is_e2e(file.rel_path) imports = _extract_imports(text) + requests = _extract_requests(text) signals: list[Signal] = [] for match in _TEST_NAME_RE.finditer(text): name = match.group(1) or match.group(2) @@ -55,7 +74,7 @@ def extract_test_signals(file: WalkedFile) -> list[Signal]: name=name, file=file, confidence=0.4 if e2e else 0.8, - metadata={"e2e": e2e, "imports": imports}, + metadata={"e2e": e2e, "imports": imports, "requests": requests}, ) ) return signals diff --git a/tests/integration/test_evidence_precision.py b/tests/integration/test_evidence_precision.py index 7c57e80..25ec6e6 100644 --- a/tests/integration/test_evidence_precision.py +++ b/tests/integration/test_evidence_precision.py @@ -14,14 +14,14 @@ # --- version ------------------------------------------------------------------ -def test_version_is_release_0_5_1(): - # v0.5.1 release: package metadata and __version__ agree on the release version. +def test_version_is_release_0_6_0(): + # v0.6.0 release: package metadata and __version__ agree on the release version. import importlib.metadata as m - assert devtime.__version__ == "0.5.1" + assert devtime.__version__ == "0.6.0" # Distribution is published as "devtime-ei" (the name "devtime" is reserved on # PyPI); the import package and the dtc command stay "devtime"/"dtc". - assert m.version("devtime-ei") == "0.5.1" + assert m.version("devtime-ei") == "0.6.0" # --- P0 Authentication headline precision ------------------------------------ diff --git a/tests/integration/test_verification.py b/tests/integration/test_verification.py index eb6f72f..e792a31 100644 --- a/tests/integration/test_verification.py +++ b/tests/integration/test_verification.py @@ -829,3 +829,356 @@ def test_application_routes_are_still_counted_alongside_fixtures(tmp_path, monke result = _verify("route-test-coverage") # Only the application route is in the inventory. assert "of 1 routes" in " ".join(result.why) + + +# --- v0.6.0: complete freshness dependencies ------------------------------------- +# +# v0.5.x stored only the bounded evidence shown to the user, so results stayed +# FRESH after the files that justified them changed or disappeared. + +ROUTE_USERS_V6 = """ +import express from "express"; +const router = express.Router(); +router.get("/users", listUsers); +""" + +TEST_IMPORTS_USERS = """ +import router from "../src/routes/users"; +import { describe, it } from "vitest"; +describe("users", () => { it("lists", () => {}); }); +""" + +WEBHOOK_VERIFIED_V6 = """ +import express from "express"; +import Stripe from "stripe"; +const stripe = new Stripe(process.env.KEY); +const router = express.Router(); +router.post("/api/stripe/webhook", (req, res) => { + stripe.webhooks.constructEvent(req.body, sig, secret); +}); +""" + + +def _freshness(slug): + conn = connection.connect() + try: + return ver.freshness_for(conn, slug) + finally: + conn.close() + + +def _record(slug): + conn = connection.connect() + try: + ver.save_verification(conn, ver.verify_claim(conn, slug)) + finally: + conn.close() + + +def test_editing_the_justifying_test_marks_the_claim_stale(tmp_path, monkeypatch): + # v0.5.x bug: only route files were fingerprinted, so changing the test that + # established the association left the result reported as FRESH. + _repo(tmp_path, { + "src/routes/users.ts": ROUTE_USERS_V6, + "tests/users.test.ts": TEST_IMPORTS_USERS, + }) + _init_scan(tmp_path, monkeypatch) + _record("route-test-coverage") + assert _freshness("route-test-coverage")[0] == ver.FRESH + + (tmp_path / "tests/users.test.ts").write_text( + TEST_IMPORTS_USERS.replace("lists", "renamed"), encoding="utf-8" + ) + assert runner.invoke(app, ["scan", "--refresh"]).exit_code == 0 + + freshness, changed = _freshness("route-test-coverage") + assert freshness == ver.STALE + assert any("users.test.ts" in p for p in changed) + + +def test_deleting_an_evidence_file_marks_the_claim_stale(tmp_path, monkeypatch): + # v0.5.x bug: the old files row survived, so the hash still matched and a + # deleted dependency looked current. + _repo(tmp_path, {"src/billing/wh.ts": WEBHOOK_VERIFIED_V6}) + _init_scan(tmp_path, monkeypatch) + _record("billing-webhook-signature") + assert _freshness("billing-webhook-signature")[0] == ver.FRESH + + (tmp_path / "src/billing/wh.ts").unlink() + assert runner.invoke(app, ["scan", "--refresh"]).exit_code == 0 + + freshness, changed = _freshness("billing-webhook-signature") + assert freshness == ver.STALE + assert any("wh.ts" in p for p in changed) + + +def test_a_new_route_invalidates_a_set_level_claim(tmp_path, monkeypatch): + # A claim about "all routes" depends on the route inventory, not only on the + # files that existed when it was recorded. + _repo(tmp_path, { + "src/routes/users.ts": ROUTE_USERS_V6, + "tests/users.test.ts": TEST_IMPORTS_USERS, + }) + _init_scan(tmp_path, monkeypatch) + _record("route-test-coverage") + assert _freshness("route-test-coverage")[0] == ver.FRESH + + _repo(tmp_path, { + "src/routes/orders.ts": + 'import express from "express";\n' + "const router = express.Router();\n" + 'router.post("/orders", createOrder);\n', + }) + assert runner.invoke(app, ["scan", "--refresh"]).exit_code == 0 + assert _freshness("route-test-coverage")[0] == ver.STALE + + +def test_unrelated_change_still_does_not_invalidate(tmp_path, monkeypatch): + # Precision matters as much as completeness: staleness that fires on every + # commit teaches people to ignore staleness. + _repo(tmp_path, { + "src/routes/users.ts": ROUTE_USERS_V6, + "tests/users.test.ts": TEST_IMPORTS_USERS, + "NOTES.md": "# notes\n", + }) + _init_scan(tmp_path, monkeypatch) + _record("route-test-coverage") + + (tmp_path / "NOTES.md").write_text("# notes changed\n", encoding="utf-8") + assert runner.invoke(app, ["scan", "--refresh"]).exit_code == 0 + assert _freshness("route-test-coverage")[0] == ver.FRESH + + +def test_dependencies_exceed_displayed_evidence(tmp_path, monkeypatch): + # The dependency set must include the test files, which are not part of the + # bounded evidence list shown to a user. + _repo(tmp_path, { + "src/routes/users.ts": ROUTE_USERS_V6, + "tests/users.test.ts": TEST_IMPORTS_USERS, + }) + _init_scan(tmp_path, monkeypatch) + conn = connection.connect() + try: + result = ver.verify_claim(conn, "route-test-coverage") + finally: + conn.close() + assert any("users.test.ts" in d for d in result.dependencies) + assert result.inventory # a set-level claim records its inventory + + +def test_verification_with_no_recorded_dependencies_is_not_fresh(tmp_path, monkeypatch): + # A result that recorded nothing cannot justify a FRESH label. + _repo(tmp_path, {"src/routes/users.ts": ROUTE_USERS_V6}) + _init_scan(tmp_path, monkeypatch) + conn = connection.connect() + try: + ver.ensure_verifications_table(conn) + conn.execute( + "INSERT INTO verifications(id, repository_id, claim_slug, status, " + "scan_id, result_json, evidence_fingerprints_json, engine_version, " + "created_at) VALUES (?,?,?,?,?,?,?,?,?)", + ("ver-empty", "r", "route-test-coverage", ver.SUPPORTED, "s", + "{}", "[]", "0.0.0", "2026-01-01T00:00:00+00:00"), + ) + conn.commit() + finally: + conn.close() + assert _freshness("route-test-coverage")[0] == ver.NEEDS_VERIFICATION + + +def test_verifications_table_migrates_idempotently(tmp_path, monkeypatch): + # Databases created before v0.6.0 lack inventory_fingerprint. + _repo(tmp_path, {"src/routes/users.ts": ROUTE_USERS_V6}) + _init_scan(tmp_path, monkeypatch) + conn = connection.connect() + try: + conn.execute("DROP TABLE IF EXISTS verifications") + conn.execute( + "CREATE TABLE verifications (id TEXT PRIMARY KEY, repository_id TEXT " + "NOT NULL, claim_slug TEXT NOT NULL, status TEXT NOT NULL, scan_id " + "TEXT, result_json TEXT NOT NULL, evidence_fingerprints_json TEXT " + "NOT NULL DEFAULT '[]', engine_version TEXT NOT NULL, created_at " + "TEXT NOT NULL)" + ) + conn.commit() + ver.ensure_verifications_table(conn) + ver.ensure_verifications_table(conn) # idempotent + cols = {r["name"] for r in conn.execute("PRAGMA table_info(verifications)")} + assert "inventory_fingerprint" in cols + # And it still accepts writes. + ver.save_verification(conn, ver.verify_claim(conn, "route-test-coverage")) + finally: + conn.close() + + +# --- v0.6.0: scan staleness is explicit ------------------------------------------ + +def test_scan_state_reports_working_tree_drift(tmp_path, monkeypatch): + _repo(tmp_path, {"src/billing/wh.ts": WEBHOOK_VERIFIED_V6}) + _init_scan(tmp_path, monkeypatch) + conn = connection.connect() + try: + assert ver.scan_state(conn)["working_tree"] == "matches_scan" + finally: + conn.close() + + (tmp_path / "src/billing/wh.ts").write_text( + WEBHOOK_VERIFIED_V6 + "\n// edited\n", encoding="utf-8" + ) + conn = connection.connect() + try: + state = ver.scan_state(conn) + assert state["working_tree"] == "changed_since_scan" + assert any("wh.ts" in p for p in state["changed_paths"]) + assert state["scan_id"] + finally: + conn.close() + + +def test_cli_warns_when_results_come_from_a_stale_scan(tmp_path, monkeypatch): + _repo(tmp_path, {"src/billing/wh.ts": WEBHOOK_VERIFIED_V6}) + _init_scan(tmp_path, monkeypatch) + (tmp_path / "src/billing/wh.ts").write_text( + WEBHOOK_VERIFIED_V6 + "\n// edited after the scan\n", encoding="utf-8" + ) + result = runner.invoke(app, ["verify", "billing-webhook-signature"]) + assert result.exit_code == 0 + # Terminal wrapping is not behavior: compare on normalized whitespace. + out = " ".join(result.stdout.split()) + assert "no longer matches your working tree" in out + assert "dtc scan" in out + assert "wh.ts" in out + + +def test_json_output_carries_the_evidence_snapshot(tmp_path, monkeypatch): + _repo(tmp_path, {"src/billing/wh.ts": WEBHOOK_VERIFIED_V6}) + _init_scan(tmp_path, monkeypatch) + result = runner.invoke(app, ["verify", "--json"]) + assert result.exit_code == 0 + payload = json.loads(result.stdout) + snap = payload["evidence_snapshot"] + assert snap["scan_id"] + assert snap["working_tree"] in ("matches_scan", "partially_checked") + assert "dependency_count" in payload["results"][0] + + +def test_mcp_verify_claim_discloses_stale_snapshot(tmp_path, monkeypatch): + from devtime.mcp.transport import build_server + + _repo(tmp_path, {"src/billing/wh.ts": WEBHOOK_VERIFIED_V6}) + _init_scan(tmp_path, monkeypatch) + (tmp_path / "src/billing/wh.ts").write_text( + WEBHOOK_VERIFIED_V6 + "\n// edited\n", encoding="utf-8" + ) + server = build_server() + out = str(asyncio.run( + server.call_tool("verify_claim", {"claim_id": "billing-webhook-signature"}) + )) + assert "staleness_warning" in out + assert "dtc scan" in out + + +# --- v0.6.0: request-based association ------------------------------------------- +# +# v0.5.1 removed the false positives but could only see imports, so it abstained +# on the common real pattern: a test that drives a running app by URL. + +SUPERTEST_TEST = """ +import request from "supertest"; +import { app } from "../src/app"; +describe("api", () => { + it("returns users", async () => { await request(app).get("/users").expect(200); }); +}); +""" + +TWO_METHOD_ROUTES = """ +import express from "express"; +const router = express.Router(); +router.get("/users", listUsers); +router.post("/users", createUser); +""" + +FASTAPI_ROUTE = """ +from fastapi import APIRouter +router = APIRouter() + +@router.get("/items") +def list_items(): + return [] +""" + +FASTAPI_CLIENT_TEST = """ +from fastapi.testclient import TestClient + +def test_list_items(): + r = client.get("/items") + assert r.status_code == 200 +""" + + +def test_supertest_request_associates_the_exact_route(tmp_path, monkeypatch): + _repo(tmp_path, { + "src/routes/users.ts": TWO_METHOD_ROUTES, + "tests/api.test.ts": SUPERTEST_TEST, + }) + _init_scan(tmp_path, monkeypatch) + result = _verify("route-test-coverage") + # GET is requested by the test; POST on the same path is not. + assert result.status == ver.WEAK + assert "1 of 2" in " ".join(result.why) + assert any("POST /users" in m for m in result.missing) + assert any("requests" in e.observation for e in result.supporting) + + +def test_fastapi_testclient_request_associates(tmp_path, monkeypatch): + _repo(tmp_path, { + "app/api/items.py": FASTAPI_ROUTE, + "tests/test_items.py": FASTAPI_CLIENT_TEST, + }) + _init_scan(tmp_path, monkeypatch) + assert _verify("route-test-coverage").status == ver.SUPPORTED + + +def test_non_url_get_calls_are_not_requests(tmp_path, monkeypatch): + # `map.get("key")` is not an HTTP request: only literals starting with "/" + # are treated as route paths. + _repo(tmp_path, { + "src/routes/users.ts": ROUTE_USERS_V6, + "tests/util.test.ts": + 'import { describe, it } from "vitest";\n' + 'describe("cache", () => { it("reads", () => { cache.get("users"); }); });\n', + }) + _init_scan(tmp_path, monkeypatch) + assert _verify("route-test-coverage").status == ver.WEAK + + +def test_request_to_a_different_path_does_not_associate(tmp_path, monkeypatch): + _repo(tmp_path, { + "src/routes/users.ts": ROUTE_USERS_V6, + "tests/api.test.ts": + 'import request from "supertest";\n' + 'describe("api", () => { it("x", async () => ' + '{ await request(app).get("/orders"); }); });\n', + }) + _init_scan(tmp_path, monkeypatch) + assert _verify("route-test-coverage").status == ver.WEAK + + +def test_prefix_relative_routes_are_unresolved_not_untested(tmp_path, monkeypatch): + # A route declared as "/" or "/{id}" on a router that is mounted elsewhere + # has no knowable full URL. Reporting it as "no test found" would blame the + # repository for a gap in DevTime's analysis. + _repo(tmp_path, { + "app/api/items.py": + "from fastapi import APIRouter\n" + "router = APIRouter()\n\n" + '@router.get("/")\n' + "def list_items():\n" + " return []\n", + }) + _init_scan(tmp_path, monkeypatch) + result = _verify("route-test-coverage") + assert result.status == ver.WEAK + blob = " ".join(result.why).lower() + assert "mount prefix" in blob + assert "not evidence that they lack tests" in blob