diff --git a/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json b/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json index 8ebd08b1e..72eaf506a 100644 --- a/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json +++ b/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "codex-security", - "version": "0.1.78", + "version": "0.1.79", "description": "Codex Security workflows for security scans, analysis, and investigation.", "author": { "name": "OpenAI" diff --git a/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py b/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py index a9de0bffe..9a071f9a9 100644 --- a/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py +++ b/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py @@ -21,6 +21,7 @@ from filesystem_identity import serialize_filesystem_identity from finalize_scan_contract import _read_scan_local_json from workbench.handoff import require_current_continuation +from workbench_source_excerpt import capture_source_scopes from workbench_target import ( directory_content_digest, directory_snapshot_regular_file_count, @@ -775,9 +776,32 @@ def begin_deep_scan_for_target( ) target_device = serialize_filesystem_identity(target_metadata.st_dev) target_inode = serialize_filesystem_identity(target_metadata.st_ino) + target_identity = (revision, target_snapshot_digest, target_device, target_inode) + source_scopes = capture_source_scopes( + target, + target_identity, + [scope], + ) scope_file_count = directory_snapshot_regular_file_count( target if scope == "." else target / scope ) + current_target = require_remediation_target(target_path) + current_metadata = current_target.stat() + current_revision = git_revision(current_target) + current_snapshot_digest = ( + directory_content_digest(current_target) + if current_revision == "unversioned" + else worktree_content_digest(current_target) + ) + if ( + current_revision, + current_snapshot_digest, + serialize_filesystem_identity(current_metadata.st_dev), + serialize_filesystem_identity(current_metadata.st_ino), + ) != target_identity: + raise SystemExit( + "The selected scan target changed while the scan was starting. Try again." + ) connection.execute("BEGIN IMMEDIATE") try: existing = existing_deep_scan_for_target(connection, thread_id, target_path, scope) @@ -799,10 +823,10 @@ def begin_deep_scan_for_target( ) current_target = require_remediation_target(target_path) current_metadata = current_target.stat() - if (current_metadata.st_dev, current_metadata.st_ino) != ( - target_metadata.st_dev, - target_metadata.st_ino, - ): + if ( + serialize_filesystem_identity(current_metadata.st_dev), + serialize_filesystem_identity(current_metadata.st_ino), + ) != (target_device, target_inode): raise SystemExit( "The selected scan target changed while the scan was starting. Try again." ) @@ -870,10 +894,10 @@ def begin_deep_scan_for_target( """ INSERT INTO scans ( id, workspace_id, target_id, target_path, target_revision, target_snapshot_digest, - target_device, target_inode, scope, mode, user_context, + target_device, target_inode, source_scopes_json, scope, mode, user_context, deep_scan_owner_thread_id, scan_dir, model, reasoning_effort, status, phase, handoff_status, started_at, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'deep', ?, ?, ?, ?, ?, + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'deep', ?, ?, ?, ?, ?, 'running', 'preflight', 'delivered', ?, ?, ?) """, ( @@ -885,6 +909,7 @@ def begin_deep_scan_for_target( target_snapshot_digest, target_device, target_inode, + json.dumps(source_scopes, allow_nan=False, separators=(",", ":"), sort_keys=True), scope, user_context, thread_id, diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py index 4a79a8f0b..64732ce78 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py @@ -102,7 +102,13 @@ from workbench_schema import ( sql_statements as sql_statements, ) -from workbench_source_excerpt import finding_source_excerpt, safe_source_path +from workbench_source_excerpt import ( + SourceContext, + capture_source_scopes, + finding_source_excerpt_from_context, + safe_source_path, + source_excerpt_context, +) from workbench_target import ( clean_worktree_content_digest, committed_diff_content_snapshot, @@ -1291,6 +1297,12 @@ def active_scan() -> sqlite3.Row | None: diff_target, metadata=target_metadata, ) + source_scopes = capture_source_scopes( + target, + target_identity, + [scope], + diff_target_kind=diff_target["kind"] if diff_target is not None else None, + ) match_committed_diff_identity: ( Callable[[Path, dict[str, str], str], bool] | None ) = None @@ -1400,6 +1412,7 @@ def target_matches_initial_snapshot() -> bool: target_root=target_root, target_summary=target_summary, scope_file_count=scope_file_count, + source_scopes=source_scopes, timestamp=timestamp, model=args.model, reasoning_effort=args.reasoning_effort, @@ -1455,6 +1468,12 @@ def _start_prompt_driven_scan( ) diff_identity = scan_diff_identity(diff_target) target_identity = scan_target_identity(target, diff_target) + source_scopes = capture_source_scopes( + target, + target_identity, + [scope], + diff_target_kind=diff_target["kind"] if diff_target is not None else None, + ) match_committed_diff_identity: ( Callable[[Path, dict[str, str], str], bool] | None ) = None @@ -1616,6 +1635,7 @@ def target_matches_initial_snapshot() -> bool: target_root=target_root, target_summary=target_summary, scope_file_count=scope_file_count, + source_scopes=source_scopes, timestamp=timestamp, handoff_status="delivered", model=args.model, @@ -2243,6 +2263,12 @@ def register_cli_scan(connection: sqlite3.Connection, args: argparse.Namespace) ) = committed_diff_content_snapshot(repository, base, head) mode = "diff" if diff_target is not None else recipe["mode"] target_identity = scan_target_identity(repository, diff_target) + source_scopes = capture_source_scopes( + repository, + target_identity, + paths or ["."], + diff_target_kind=diff_target["kind"] if diff_target is not None else None, + ) scope_file_count = ( directory_snapshot_regular_file_count(repository) if not paths @@ -2357,6 +2383,7 @@ def target_matches_initial_snapshot() -> bool: timestamp=timestamp, handoff_status="delivered", scan_dir=scan_dir, + source_scopes=source_scopes, ) connection.execute( "UPDATE scans SET recipe_json = ?, parent_scan_id = ?, user_context = ? WHERE id = ?", @@ -3870,7 +3897,7 @@ def list_findings(connection: sqlite3.Connection, args: argparse.Namespace) -> d next_offset = args.offset + len(rows) return { "findingsPage": { - "findings": [finding_result(connection, scan, row) for row in rows], + "findings": finding_results(connection, scan, rows), "limit": limit, "nextOffset": next_offset if next_offset < total else None, "offset": args.offset, @@ -3969,7 +3996,7 @@ def scan_result( "contract": scan_contract(scan), "continuationThreadId": scan["continuation_thread_id"], "failureMessage": scan["failure_message"], - "findings": [finding_result(connection, scan, row) for row in occurrence_rows], + "findings": finding_results(connection, scan, occurrence_rows), "findingCount": finding_count, "findingsTruncated": finding_count > len(occurrence_rows), "severityCounts": severity_counts, @@ -4112,21 +4139,59 @@ def legacy_finding_matches(row: sqlite3.Row, finding: Any) -> bool: ) +def scan_source_excerpt_context( + scan: sqlite3.Row, +) -> tuple[Path | None, SourceContext | None]: + try: + target = require_scan_target_identity(scan) + except SystemExit: + return None, None + try: + selected_paths = requested_scan_paths(scan) + except (IndexError, KeyError, TypeError, ValueError): + selected_paths = [] + if not isinstance(selected_paths, list) or not all( + isinstance(path, str) for path in selected_paths + ): + selected_paths = [] + return target, source_excerpt_context(scan, target, selected_paths) + + +def finding_results( + connection: sqlite3.Connection, + scan: sqlite3.Row, + occurrences: list[sqlite3.Row], +) -> list[dict[str, Any]]: + if not occurrences: + return [] + target, source_context = scan_source_excerpt_context(scan) + return [ + finding_result( + connection, + scan, + occurrence, + target=target, + source_context=source_context, + ) + for occurrence in occurrences + ] + + def finding_result( connection: sqlite3.Connection, scan: sqlite3.Row, occurrence: sqlite3.Row, + *, + target: Path | None, + source_context: SourceContext | None, ) -> dict[str, Any]: details = bounded_finding_details(read_finding_details(occurrence["details_json"])) confidence = details.get("confidence") confidence = confidence if isinstance(confidence, dict) else {} severity = details.get("severity") severity = severity if isinstance(severity, dict) else {} + excerpt_locations = [] locations = [] - try: - target = require_scan_target_identity(scan) - except SystemExit: - target = None for row in connection.execute( """ SELECT relative_path, start_line, end_line, role @@ -4137,6 +4202,13 @@ def finding_result( """, (occurrence["id"], FINDING_LOCATIONS_LIMIT), ): + excerpt_locations.append( + { + "endLine": row["end_line"], + "path": row["relative_path"], + "startLine": row["start_line"], + } + ) absolute_path = safe_source_path(target, row["relative_path"]) if target else None location = { "endLine": row["end_line"], @@ -4181,7 +4253,9 @@ def finding_result( result["knownSince"] = known_since result["knownScanIds"] = known_scan_ids result.pop("artifactPaths", None) - source_excerpt = finding_source_excerpt(scan, target, locations) + source_excerpt = finding_source_excerpt_from_context( + source_context, excerpt_locations + ) if source_excerpt: result["sourceExcerpt"] = source_excerpt artifact_paths = finding_artifact_paths(Path(scan["scan_dir"]), details) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py index 7cb1f6217..7fd3b6f3f 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py @@ -11,6 +11,7 @@ from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path +from typing import Any # Some plugin hosts launch Python with safe-path isolation enabled. sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -160,6 +161,7 @@ def insert_running_scan( target_root: Path, target_summary: str | None, scope_file_count: int, + source_scopes: dict[str, Any], timestamp: str, handoff_status: str = "pending", model: str | None = None, @@ -180,11 +182,11 @@ def insert_running_scan( """ INSERT INTO scans ( id, workspace_id, target_id, target_path, target_revision, target_snapshot_digest, - target_device, target_inode, scope, mode, user_context, + target_device, target_inode, source_scopes_json, scope, mode, user_context, deep_scan_owner_thread_id, diff_target_kind, diff_base_revision, diff_head_revision, diff_content_digest, target_summary, scan_dir, model, reasoning_effort, status, phase, handoff_status, started_at, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'running', 'preflight', ?, ?, ?, ?) """, ( @@ -193,6 +195,7 @@ def insert_running_scan( workspace["target_id"], str(target), *target_identity, + json.dumps(source_scopes, allow_nan=False, separators=(",", ":"), sort_keys=True), scope, workspace["default_mode"], user_context, diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py index 90111fdd7..32b05d177 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py @@ -697,6 +697,13 @@ ALTER TABLE deep_scan_runs ADD COLUMN publication_error_message TEXT; """, ), + ( + 33, + "persist selected source excerpt authority", + """ + ALTER TABLE scans ADD COLUMN source_scopes_json TEXT; + """, + ), ) @@ -777,6 +784,8 @@ def apply_migrations( repair_thread_scoped_workspaces_migration(connection) elif version == 16: should_backfill_targets = repair_stable_targets_migration(connection) + elif version == 33: + add_column_if_missing(connection, "scans", "source_scopes_json", "TEXT") else: for statement in sql_statements(sql): connection.execute(statement) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_source_excerpt.py b/sdk/typescript/_bundled_plugin/scripts/workbench_source_excerpt.py index cea25e23f..6a3ed6816 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_source_excerpt.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_source_excerpt.py @@ -3,88 +3,744 @@ from __future__ import annotations import argparse +import codecs +import json +import os +import re import sqlite3 +import stat +import subprocess import sys +from contextlib import ExitStack +from dataclasses import dataclass, field from pathlib import Path, PurePosixPath -from typing import Any +from typing import IO, Any +from unicodedata import normalize # Some plugin hosts launch Python with safe-path isolation enabled. sys.path.insert(0, str(Path(__file__).resolve().parent)) -from workbench_target import clean_worktree_content_digest, git_bytes +from workbench_constants import GIT_REPOSITORY_ENVIRONMENT +from workbench_target import ( + _replacement_refs_enabled, + clean_worktree_content_digest, + git_bytes, + git_worktree_context, +) CONTEXT_LINES = 3 MAX_BYTES = 16_000 MAX_LINES = 60 +OBJECT_ID = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z") +LINE_BREAK = re.compile(r"\r\n|[\n\v\f\r\x1c-\x1e\x85\u2028\u2029]") +TreeEntry = tuple[str, str, str] +TreeEntries = dict[str, list[TreeEntry]] -def finding_source_excerpt( + +@dataclass +class SourceScopeIndex: + kind: str | None = None + children: dict[str, SourceScopeIndex] = field(default_factory=dict) + tree_entries: dict[str, TreeEntries] = field(default_factory=dict) + filesystem_root: Path | None = None + + +SourceContext = tuple[Path, str, SourceScopeIndex] + + +def normalized_path_component(value: str) -> str: + return normalize("NFC", normalize("NFD", value).casefold()).rstrip(" .") + + +def relative_path(value: str) -> PurePosixPath | None: + path = PurePosixPath(value) + if not value or "\\" in value or path.is_absolute() or ".." in path.parts: + return None + return path + + +def safe_source_path(target: Path, value: str) -> Path | None: + path = relative_path(value) + if path is None: + return None + try: + root = target.resolve() + selected = (root / path.as_posix()).resolve() + selected.relative_to(root) + return selected + except (OSError, RuntimeError, ValueError): + return None + + +def local_git_bytes(repository: Path, *arguments: str) -> bytes | None: + return git_bytes( + repository, + "--no-replace-objects", + *arguments, + local_objects_only=True, + ) + + +def matching_tree_entries( + requests: IO[bytes], + responses: IO[bytes], + object_id: str, + name: str, + filesystem_parent: Path | None = None, + tree_cache: dict[str, TreeEntries] | None = None, +) -> TreeEntry | None: + encoded_object = object_id.encode("ascii") + requests.write(encoded_object + b"\0") + requests.flush() + header = responses.readline() + if not header.endswith(b"\n"): + raise ValueError("unterminated batch header") + fields = header[:-1].split(b" ") + if ( + len(fields) != 3 + or fields[0] != encoded_object + or fields[1] != b"tree" + or not fields[2].isdigit() + ): + raise ValueError("invalid tree response") + unread = int(fields[2]) + buffered = bytearray() + cursor = 0 + + def read_more() -> None: + nonlocal cursor, unread + if cursor: + del buffered[:cursor] + cursor = 0 + chunk = responses.read(min(64 * 1024, unread)) + if not chunk: + raise ValueError("truncated tree response") + buffered.extend(chunk) + unread -= len(chunk) + + def read_field(delimiter: int, maximum_bytes: int | None = None) -> bytearray: + nonlocal cursor + field = bytearray() + while True: + try: + end = buffered.index(delimiter, cursor) + except ValueError: + available = len(buffered) - cursor + if maximum_bytes is not None and len(field) + available > maximum_bytes: + raise ValueError("oversized tree field") from None + field.extend(buffered[cursor:]) + cursor = len(buffered) + if not unread: + raise ValueError("unterminated tree entry") from None + read_more() + continue + if maximum_bytes is not None and len(field) + end - cursor > maximum_bytes: + raise ValueError("oversized tree field") + field.extend(buffered[cursor:end]) + cursor = end + 1 + return field + + def read_object_id(size: int) -> bytes: + nonlocal cursor + while len(buffered) - cursor < size: + if not unread: + raise ValueError("truncated tree entry") + read_more() + value = bytes(buffered[cursor : cursor + size]) + cursor += size + return value + + object_id_bytes = len(object_id) // 2 + expected_name = normalized_path_component(name) + selected = None + ambiguous = False + entries: TreeEntries | None = {} if tree_cache is not None else None + while cursor < len(buffered) or unread: + mode = bytes(read_field(ord(" "), 6)) + decoded_name = read_field(0).decode( + sys.getfilesystemencoding(), errors="surrogateescape" + ) + entry_object = read_object_id(object_id_bytes) + normalized_name = normalized_path_component(decoded_name) + kind = ( + "directory" + if mode in {b"40000", b"040000"} + else "file" + if mode in {b"100644", b"100755"} + else "other" + ) + entry = (decoded_name, kind, entry_object.hex()) + if entries is not None: + entries.setdefault(normalized_name, []).append(entry) + if not ambiguous and normalized_name == expected_name: + if filesystem_parent is not None and decoded_name != name: + try: + if not (filesystem_parent / name).samefile( + filesystem_parent / decoded_name + ): + continue + except OSError: + pass + if selected is not None: + selected = None + ambiguous = True + else: + selected = entry + if responses.read(1) != b"\n": + raise ValueError("missing tree terminator") + if tree_cache is not None and entries is not None: + tree_cache[object_id] = entries + return selected + + +def matching_cached_tree_entry( + entries: TreeEntries, name: str, filesystem_parent: Path +) -> TreeEntry | None: + selected = None + for entry in entries.get(normalized_path_component(name), []): + decoded_name = entry[0] + if decoded_name != name: + try: + if not (filesystem_parent / name).samefile( + filesystem_parent / decoded_name + ): + continue + except OSError: + pass + if selected is not None: + return None + selected = entry + return selected + + +def tree_path( + repository: Path, + tree: str, + value: str, + *, + selected_kinds: dict[int, str] | None = None, + tree_cache: dict[str, TreeEntries] | None = None, + filesystem_root: Path | None = None, +) -> TreeEntry | None: + path = relative_path(value) + if path is None: + return None + if selected_kinds is not None and selected_kinds.get(0, "directory") != "directory": + return None + kind, object_id = "directory", tree + filesystem_parent = repository if filesystem_root is None else filesystem_root + if not path.parts: + return path.as_posix(), kind, object_id + environment = os.environ.copy() + for variable in GIT_REPOSITORY_ENVIRONMENT: + environment.pop(variable, None) + environment["GIT_ALLOW_PROTOCOL"] = "" + environment["GIT_LITERAL_PATHSPECS"] = "1" + environment["GIT_NO_LAZY_FETCH"] = "1" + command = [ + "git", + "-c", + "core.fsmonitor=false", + "-c", + "i18n.logOutputEncoding=UTF-8", + "-C", + str(repository), + "--no-replace-objects", + "cat-file", + "--batch", + "-z", + ] + try: + with ExitStack() as processes: + process = None + pending_tree_entries: dict[str, TreeEntries] = {} + resolved = True + for depth, name in enumerate(path.parts, start=1): + if kind != "directory": + resolved = False + break + cached = ( + tree_cache.get(object_id, pending_tree_entries.get(object_id)) + if tree_cache is not None + else None + ) + if cached is not None: + aliases = matching_cached_tree_entry(cached, name, filesystem_parent) + else: + if process is None: + process = processes.enter_context( + subprocess.Popen( + command, + env=environment, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + ) + if process.stdin is None or process.stdout is None: + return None + aliases = matching_tree_entries( + process.stdin, + process.stdout, + object_id, + name, + filesystem_parent, + pending_tree_entries if tree_cache is not None else None, + ) + # The normalized name must be unique before an exact spelling can win. + if aliases is None or aliases[0] != name: + resolved = False + break + _, kind, object_id = aliases + filesystem_parent /= name + if ( + selected_kinds is not None + and (expected_kind := selected_kinds.get(depth)) is not None + and kind != expected_kind + ): + resolved = False + break + if process is not None: + if process.stdin is None: + return None + process.stdin.close() + if process.wait() != 0: + return None + if tree_cache is not None: + tree_cache.update(pending_tree_entries) + if not resolved: + return None + except (MemoryError, OSError, ValueError): + return None + return path.as_posix(), kind, object_id + + +def replacement_refs_absent(repository: Path) -> bool: + if not _replacement_refs_enabled(repository): + return True + replacement_ref_base = os.environ.get("GIT_REPLACE_REF_BASE", "refs/replace/") + environment = os.environ.copy() + for variable in GIT_REPOSITORY_ENVIRONMENT: + environment.pop(variable, None) + environment["GIT_ALLOW_PROTOCOL"] = "" + environment["GIT_LITERAL_PATHSPECS"] = "1" + environment["GIT_NO_LAZY_FETCH"] = "1" + git_command = [ + "git", + "-c", + "core.fsmonitor=false", + "-c", + "i18n.logOutputEncoding=UTF-8", + "-C", + str(repository), + "--no-replace-objects", + ] + try: + if ( + subprocess.run( + [ + *git_command, + "check-ref-format", + f"{replacement_ref_base}{'0' * 64}", + ], + env=environment, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode + != 0 + ): + return False + command = [ + *git_command, + "for-each-ref", + "--count=1", + "--format=", + f"{replacement_ref_base}*", + ] + with subprocess.Popen( + command, + env=environment, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) as process: + if process.stdout is None: + return False + if process.stdout.read(1): + if process.poll() is None: + process.kill() + return False + return process.wait() == 0 + except (MemoryError, OSError): + return False + + +def target_tree(target: Path, revision: str) -> tuple[Path, str] | None: + if not isinstance(revision, str) or not OBJECT_ID.fullmatch(revision): + return None + repository, prefix = git_worktree_context(target) + if not replacement_refs_absent(repository): + return None + raw_tree = local_git_bytes( + repository, + "rev-parse", + "--verify", + "--end-of-options", + f"{revision}^{{tree}}", + ) + try: + tree = raw_tree.decode("ascii").strip() if raw_tree is not None else "" + except UnicodeDecodeError: + return None + if not OBJECT_ID.fullmatch(tree): + return None + if prefix == ".": + return repository, tree + selected = tree_path(repository, tree, prefix) + if selected is None or selected[1] != "directory": + return None + return repository, selected[2] + + +def capture_source_scopes( + target: Path, + target_identity: tuple[str, str | None, int | str, int | str], + paths: list[str], + *, + diff_target_kind: str | None = None, +) -> dict[str, Any]: + revision, snapshot = target_identity[:2] + authority: dict[str, Any] = {"version": 1, "paths": []} + if ( + (diff_target_kind is not None and diff_target_kind not in {"commit", "range"}) + or revision == "unversioned" + or (snapshot is not None and snapshot != clean_worktree_content_digest()) + ): + return authority + try: + context = target_tree(target, revision) + if context is None: + return authority + repository, tree = context + authority["targetTree"] = tree + captured: set[str] = set() + captured_tree_entries: dict[str, TreeEntries] = {} + for requested in paths: + parsed = relative_path(requested) + if parsed is None or safe_source_path(target, requested) is None: + continue + try: + raw_selected = target + components = parsed.parts + if not components: + metadata = raw_selected.lstat() + for index, component in enumerate(components): + raw_selected /= component + metadata = raw_selected.lstat() + if ( + stat.S_ISLNK(metadata.st_mode) + or getattr(metadata, "st_reparse_tag", 0) & 0x20000000 + or ( + index < len(components) - 1 + and not stat.S_ISDIR(metadata.st_mode) + ) + ): + metadata = None + break + except OSError: + continue + if metadata is None: + continue + kind = ( + "directory" + if stat.S_ISDIR(metadata.st_mode) + else "file" + if stat.S_ISREG(metadata.st_mode) + else None + ) + if kind is None: + continue + selected = parsed.as_posix() + if diff_target_kind in {"commit", "range"}: + committed = tree_path( + repository, + tree, + selected, + tree_cache=captured_tree_entries, + filesystem_root=target, + ) + if committed is None or committed[1] not in {"directory", "file"}: + continue + if kind == "file" or committed[1] == "file": + kind = "file" + if selected not in captured: + captured.add(selected) + authority["paths"].append({"kind": kind, "path": selected}) + except (OSError, RuntimeError, SystemExit, UnicodeError, ValueError): + return {"version": 1, "paths": []} + return authority + + +def load_source_scopes( + scan: sqlite3.Row, target: Path, selected_paths: list[str] +) -> SourceContext | None: + try: + saved = scan["source_scopes_json"] + except (IndexError, KeyError): + return None + if not isinstance(saved, str): + return None + metadata = json.loads(saved) + records = metadata.get("paths") if isinstance(metadata, dict) else None + expected_tree = metadata.get("targetTree") if isinstance(metadata, dict) else None + if ( + not isinstance(metadata, dict) + or metadata.get("version") != 1 + or not isinstance(records, list) + or not records + or not isinstance(expected_tree, str) + or not OBJECT_ID.fullmatch(expected_tree) + ): + return None + expected = { + parsed.as_posix() + for value in selected_paths + if (parsed := relative_path(value)) is not None + } + if not expected: + return None + context = target_tree(target, scan["target_revision"]) + if context is None: + return None + repository, tree = context + if tree != expected_tree: + return None + index = SourceScopeIndex(filesystem_root=target) + for record in records: + if not isinstance(record, dict) or set(record) != {"kind", "path"}: + return None + kind = record.get("kind") + saved_path = record.get("path") + path = relative_path(saved_path) if isinstance(saved_path, str) else None + if path is None: + return None + selected = path.as_posix() + if ( + kind not in {"directory", "file"} + or selected != saved_path + or selected not in expected + ): + return None + node = index + for component in path.parts: + node = node.children.setdefault(component, SourceScopeIndex()) + if node.kind is not None: + return None + node.kind = kind + return repository, tree, index + + +def selected_source_kinds( + index: SourceScopeIndex, value: str +) -> dict[int, str] | None: + path = relative_path(value) + if path is None: + return None + node = index + kinds = {0: node.kind} if node.kind is not None else {} + authorized = node.kind == "directory" + for depth, component in enumerate(path.parts, start=1): + child = node.children.get(component) + if child is None: + return kinds if authorized else None + node = child + if node.kind is not None: + kinds[depth] = node.kind + if node.kind == "directory" or ( + node.kind == "file" and depth == len(path.parts) + ): + authorized = True + return kinds if authorized else None + + +def source_excerpt_context( scan: sqlite3.Row, - target: Path | None, + target: Path, + selected_paths: list[str], +) -> SourceContext | None: + if scan["mode"] == "diff" and scan["diff_target_kind"] not in {"commit", "range"}: + return None + if scan["target_revision"] == "unversioned": + return None + snapshot = scan["target_snapshot_digest"] + if snapshot is not None and snapshot != clean_worktree_content_digest(): + return None + try: + return load_source_scopes(scan, target, selected_paths) + except (OSError, RuntimeError, SystemExit, UnicodeError, ValueError): + return None + + +def finding_source_excerpt_from_context( + context: SourceContext | None, locations: list[dict[str, Any]], ) -> str | None: - if target is None or not locations: - return None - location = next( - ( - candidate - for candidate in locations - if "root_control" in str(candidate.get("role") or "").lower() - ), - locations[0], - ) + if context is None or not locations: + return None + repository, tree, scopes = context + + location = locations[0] path = location.get("path") start_line = location.get("startLine") - end_line = location.get("endLine") if not isinstance(path, str) or not isinstance(start_line, int): return None - source = scanned_source_text(scan, target, path) - if not source or "\0" in source: + try: + selected_kinds = selected_source_kinds(scopes, path) + selected = ( + tree_path( + repository, + tree, + path, + selected_kinds=selected_kinds, + tree_cache=scopes.tree_entries, + filesystem_root=scopes.filesystem_root, + ) + if selected_kinds is not None + else None + ) + if selected is None or selected[1] != "file": + return None + end_line = location.get("endLine") + last_affected_line = end_line if isinstance(end_line, int) else start_line + return scanned_source_excerpt( + repository, + selected[2], + start_line, + last_affected_line, + ) + except ( + MemoryError, + OSError, + RuntimeError, + SystemExit, + UnicodeError, + ValueError, + ): return None - lines = source.splitlines() - if start_line < 1 or start_line > len(lines): + + +def scanned_source_excerpt( + repository: Path, + object_id: str, + start_line: int, + last_affected_line: int, +) -> str | None: + if start_line < 1 or OBJECT_ID.fullmatch(object_id) is None: return None - last_affected_line = end_line if isinstance(end_line, int) else start_line excerpt_start = max(1, start_line - CONTEXT_LINES) - excerpt_end = min( - len(lines), + excerpt_limit = min( max(start_line, last_affected_line) + CONTEXT_LINES, excerpt_start + MAX_LINES - 1, ) - width = len(str(excerpt_end)) - excerpt = "\n".join( - f"{line_number:>{width}} {lines[line_number - 1]}" - for line_number in range(excerpt_start, excerpt_end + 1) - ) - encoded = excerpt.encode("utf-8")[:MAX_BYTES] - return encoded.decode("utf-8", errors="ignore") + environment = os.environ.copy() + for variable in GIT_REPOSITORY_ENVIRONMENT: + environment.pop(variable, None) + environment["GIT_ALLOW_PROTOCOL"] = "" + environment["GIT_LITERAL_PATHSPECS"] = "1" + environment["GIT_NO_LAZY_FETCH"] = "1" + command = [ + "git", + "-c", + "core.fsmonitor=false", + "-c", + "i18n.logOutputEncoding=UTF-8", + "-C", + str(repository), + "--no-replace-objects", + "cat-file", + "blob", + object_id, + ] -def scanned_source_text(scan: sqlite3.Row, target: Path, path: str) -> str | None: - if safe_source_path(target, path) is None: - return None - revision = scan["target_revision"] - if revision == "unversioned": - return None - snapshot_digest = scan["target_snapshot_digest"] - if snapshot_digest is not None and snapshot_digest != clean_worktree_content_digest(): - return None - object_name = f"{revision}:{path}" - content = git_bytes(target, "cat-file", "blob", object_name) - return content.decode("utf-8", errors="replace") if content is not None else None + captured_lines: list[tuple[int, str]] = [] + fragments: list[str] = [] + captured_bytes = 0 + line_number = 1 + last_line = 0 + line_has_content = False + pending_carriage_return = False + def capture(value: str) -> None: + nonlocal captured_bytes, line_has_content + if not value: + return + line_has_content = True + if ( + line_number < excerpt_start + or line_number > excerpt_limit + or captured_bytes >= MAX_BYTES + ): + return + remaining = MAX_BYTES - captured_bytes + selected = value.encode("utf-8")[:remaining] + fragments.append(selected.decode("utf-8", errors="ignore")) + captured_bytes += len(selected) -def safe_source_path(target: Path, relative_path: str) -> Path | None: - if "\\" in relative_path: - return None - parsed = PurePosixPath(relative_path) - if parsed.is_absolute() or ".." in parsed.parts: - return None - try: - path = (target / parsed.as_posix()).resolve() - path.relative_to(target) - except (OSError, RuntimeError, ValueError): + def finish_line() -> None: + nonlocal fragments, last_line, line_has_content, line_number + if excerpt_start <= line_number <= excerpt_limit: + captured_lines.append((line_number, "".join(fragments))) + last_line = line_number + line_number += 1 + fragments = [] + line_has_content = False + + def consume(value: str, *, final: bool = False) -> None: + nonlocal pending_carriage_return + if pending_carriage_return: + value = "\r" + value + pending_carriage_return = False + if not final and value.endswith("\r"): + value = value[:-1] + pending_carriage_return = True + cursor = 0 + for match in LINE_BREAK.finditer(value): + capture(value[cursor : match.start()]) + finish_line() + cursor = match.end() + capture(value[cursor:]) + + decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") + invalid = False + with subprocess.Popen( + command, + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) as process: + if process.stdout is None: + return None + while chunk := process.stdout.read(64 * 1024): + if b"\0" in chunk: + invalid = True + if not invalid: + consume(decoder.decode(chunk)) + if not invalid: + consume(decoder.decode(b"", final=True), final=True) + if line_has_content: + finish_line() + if process.wait() != 0: + return None + + if invalid or last_line < start_line or not captured_lines: return None - return path + width = len(str(captured_lines[-1][0])) + excerpt = "\n".join( + f"{number:>{width}} {line}" for number, line in captured_lines + ) + return excerpt.encode("utf-8")[:MAX_BYTES].decode("utf-8", errors="ignore") def main() -> None: diff --git a/sdk/typescript/src/version.ts b/sdk/typescript/src/version.ts index bb0a179c9..0fcfbbb6d 100644 --- a/sdk/typescript/src/version.ts +++ b/sdk/typescript/src/version.ts @@ -9,7 +9,7 @@ const PACKAGE_VERSIONS = packageVersions( export const VERSION = PACKAGE_VERSIONS.package; export const CODEX_SDK_VERSION = PACKAGE_VERSIONS.sdk; export const CODEX_EXECUTABLE_VERSION = PACKAGE_VERSIONS.executable; -export const BUNDLED_PLUGIN_VERSION = "0.1.78" as const; +export const BUNDLED_PLUGIN_VERSION = "0.1.79" as const; const PACKAGE_NAME = "@openai/codex-security"; diff --git a/sdk/typescript/tests-ts/plugin-report-limits.test.ts b/sdk/typescript/tests-ts/plugin-report-limits.test.ts index 1356466bb..f57299ced 100644 --- a/sdk/typescript/tests-ts/plugin-report-limits.test.ts +++ b/sdk/typescript/tests-ts/plugin-report-limits.test.ts @@ -3,7 +3,7 @@ import { describe, expect, test } from "bun:test"; import { PLUGIN_ROOT } from "./plugin-root.js"; describe("bundled scan report and source limits", () => { - test("accepts large reports, schemas, source files, and late source lines", () => { + test("accepts large reports, schemas, and late source lines", () => { const python = Bun.which("python3") ?? Bun.which("python"); expect(python).not.toBeNull(); const program = [ @@ -19,12 +19,9 @@ describe("bundled scan report and source limits", () => { " schema = pathlib.Path(directory) / 'large.schema.json'", " schema.write_text(json.dumps({'type': 'object', 'description': 'x' * (4 * 1024 * 1024), 'allOf': [{'type': 'object'}] * 129}))", " finalizer.validate_against_schema({'safe': True}, schema)", - " source = b'x' * (1024 * 1024 + 1)", - " excerpts.git_bytes = lambda *args: source", " target = pathlib.Path(directory).resolve()", - " excerpt = excerpts.scanned_source_text({'target_revision': 'deadbeef', 'target_snapshot_digest': None}, target, 'large.py')", " hashes = finalizer._github_line_hashes(io.StringIO('line\\n' * 100001), {100001})", - " print(json.dumps({'documentBytes': len(document), 'sourceBytes': len(excerpt), 'lateSourceLine': 100001 in hashes, 'unsafePathRejected': excerpts.safe_source_path(target, '../outside') is None}))", + " print(json.dumps({'documentBytes': len(document), 'lateSourceLine': 100001 in hashes, 'unsafePathRejected': excerpts.safe_source_path(target, '../outside') is None}))", ].join("\n"); const result = Bun.spawnSync( [python!, "-I", "-B", "-c", program, join(PLUGIN_ROOT, "scripts")], @@ -34,7 +31,6 @@ describe("bundled scan report and source limits", () => { expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0); expect(JSON.parse(new TextDecoder().decode(result.stdout))).toMatchObject({ documentBytes: expect.any(Number), - sourceBytes: 1024 * 1024 + 1, lateSourceLine: true, unsafePathRejected: true, }); diff --git a/sdk/typescript/tests-ts/publication-store.test.ts b/sdk/typescript/tests-ts/publication-store.test.ts index ca85ddcfd..d565f19c2 100644 --- a/sdk/typescript/tests-ts/publication-store.test.ts +++ b/sdk/typescript/tests-ts/publication-store.test.ts @@ -679,9 +679,11 @@ connection.close() fixture, "ALTER TABLE deep_scan_runs DROP COLUMN publication_error_message", ); - databaseRows(fixture, "DELETE FROM schema_migrations WHERE version >= ?", [ - 31, - ]); + databaseRows( + fixture, + "DELETE FROM schema_migrations WHERE version BETWEEN ? AND ?", + [31, 32], + ); await expect( preparePublicationStore(fixture.publication, fixture.environment), @@ -690,8 +692,8 @@ connection.close() expect( databaseRows( fixture, - "SELECT version, name FROM schema_migrations WHERE version >= ? ORDER BY version", - [31], + "SELECT version, name FROM schema_migrations WHERE version BETWEEN ? AND ? ORDER BY version", + [31, 32], ), ).toEqual([ { version: 31, name: "freeze stopped scan source digests" }, diff --git a/sdk/typescript/tests-ts/workbench-source-excerpt.test.ts b/sdk/typescript/tests-ts/workbench-source-excerpt.test.ts new file mode 100644 index 000000000..9825e658d --- /dev/null +++ b/sdk/typescript/tests-ts/workbench-source-excerpt.test.ts @@ -0,0 +1,1310 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, test } from "bun:test"; +import { + BUNDLED_PLUGIN_VERSION, + bootstrapPlugin, + resolveCodexCommand, +} from "../src/index.js"; +import { PLUGIN_ROOT } from "./plugin-root.js"; + +const temporaryRoots: string[] = []; + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } +}); + +function python(): string { + const command = + process.env["PYTHON"] ?? + Bun.which("python3") ?? + Bun.which("python") ?? + Bun.which("py"); + expect(command).not.toBeNull(); + return command!; +} + +function git( + repository: string, + args: string[], + input?: Buffer | string, +): string { + return execFileSync( + "git", + [ + "-c", + "user.name=Synthetic Fixture", + "-c", + "user.email=fixture@example.invalid", + ...args, + ], + { cwd: repository, encoding: "utf8", input }, + ).trim(); +} + +function collisionRepository(root: string): { + caseSensitive: boolean; + deepPath: string; + repository: string; + replacement: string; + revision: string; +} { + const repository = join(root, "repository"); + mkdirSync(repository); + git(repository, ["init", "-q"]); + const blob = (content: string) => + git(repository, ["hash-object", "-w", "--stdin"], content); + const tree = ( + entries: Array<[mode: string, type: string, oid: string, name: string]>, + ) => { + const records = entries + .toSorted((left, right) => + Buffer.from(left[3]).compare(Buffer.from(right[3])), + ) + .map(([mode, type, oid, name]) => + Buffer.concat([ + Buffer.from(`${mode} ${type} ${oid}\t`), + Buffer.from(name), + Buffer.from([0]), + ]), + ); + return git(repository, ["mktree", "-z"], Buffer.concat(records)); + }; + const sourceTree = tree([ + ["100644", "blob", blob("allowed = True\n"), "allowed.py"], + ["100644", "blob", blob("another = True\n"), "another.py"], + ["100644", "blob", blob("x".repeat(2 * 1024 * 1024)), "large.py"], + ["100644", "blob", blob("case_upper = True\n"), "LOWER.py"], + ["100644", "blob", blob("case_lower = True\n"), "lower.py"], + ["100644", "blob", blob("unicode_composed = True\n"), "é.py"], + ["100644", "blob", blob("unicode_decomposed = True\n"), "é.py"], + ["100644", "blob", blob("plain_name = True\n"), "trailing.py"], + ["100644", "blob", blob("trailing_dot = True\n"), "trailing.py."], + ["100644", "blob", blob("third = True\n"), "third.py"], + ["100644", "blob", blob("uncheckoutable = True\n"), "z".repeat(2048)], + ]); + const upperScope = tree([ + ["100644", "blob", blob("selected_scope = True\n"), "selected.py"], + ]); + const lowerScope = tree([ + ["100644", "blob", blob("colliding_scope = True\n"), "sibling.py"], + ]); + const mismatchTree = tree([ + ["100644", "blob", blob("unscanned = True\n"), "secret.py"], + ]); + const linkedSubtree = tree([ + ["100644", "blob", blob("linked_secret = True\n"), "secret.py"], + ]); + const linkedTree = tree([["040000", "tree", linkedSubtree, "subdir"]]); + const deepComponents = Array.from({ length: 128 }, (_, index) => `d${index}`); + let deepTree = tree([["100644", "blob", blob("deep = True\n"), "source.py"]]); + for (const component of deepComponents.toReversed()) { + deepTree = tree([["040000", "tree", deepTree, component]]); + } + const deepPath = ["deep", ...deepComponents, "source.py"].join("/"); + const rootTree = tree([ + ["040000", "tree", upperScope, "Scope"], + ["040000", "tree", linkedTree, "alias"], + ["040000", "tree", deepTree, "deep"], + ["100644", "blob", blob("historical_file = True\n"), "historical"], + ["040000", "tree", mismatchTree, "mismatch"], + ["040000", "tree", lowerScope, "scope"], + ["100644", "blob", blob("outside = True\n"), "outside.py"], + ["040000", "tree", sourceTree, "src"], + ]); + const revision = git(repository, [ + "commit-tree", + rootTree, + "-m", + "synthetic source tree", + ]); + const replacementTree = tree([ + ["100644", "blob", blob("replacement = True\n"), "replacement.py"], + ]); + const replacement = git(repository, [ + "commit-tree", + replacementTree, + "-m", + "synthetic replacement tree", + ]); + git(repository, ["symbolic-ref", "HEAD", "refs/heads/main"]); + git(repository, ["update-ref", "refs/heads/main", revision]); + mkdirSync(join(repository, "src")); + mkdirSync(join(repository, "Scope")); + mkdirSync(join(repository, "deep")); + mkdirSync(join(repository, "historical")); + mkdirSync(join(repository, "real", "subdir"), { recursive: true }); + symlinkSync(join(repository, "real"), join(repository, "alias"), "junction"); + writeFileSync( + join(repository, "real", "subdir", "allowed.py"), + "linked_allowed = True\n", + ); + writeFileSync(join(repository, "src", "allowed.py"), "allowed = True\n"); + writeFileSync(join(repository, "src", "another.py"), "another = True\n"); + writeFileSync(join(repository, "src", "LOWER.py"), "case_upper = True\n"); + const lowercasePath = join(repository, "src", "lower.py"); + const caseSensitive = !existsSync(lowercasePath); + if (caseSensitive) { + writeFileSync(lowercasePath, "case_lower = True\n"); + } + writeFileSync(join(repository, "src", "é.py"), "unicode_composed = True\n"); + writeFileSync(join(repository, "src", "trailing.py"), "plain_name = True\n"); + writeFileSync(join(repository, "src", "third.py"), "third = True\n"); + writeFileSync( + join(repository, "historical", "checkout-only.py"), + "checkout_directory = True\n", + ); + writeFileSync(join(repository, "mismatch"), "selected_file = True\n"); + writeFileSync( + join(repository, "Scope", "selected.py"), + "selected_scope = True\n", + ); + return { caseSensitive, deepPath, repository, replacement, revision }; +} + +function ordinaryRepository(root: string): { + repository: string; + revision: string; +} { + const repository = join(root, "ordinary-repository"); + mkdirSync(join(repository, "src"), { recursive: true }); + mkdirSync(join(repository, "other")); + writeFileSync(join(repository, "src", "allowed.py"), "allowed = True\n"); + writeFileSync(join(repository, "other", "example.py"), "example = True\n"); + git(repository, ["init", "-q"]); + git(repository, ["add", "."]); + git(repository, ["commit", "-qm", "synthetic source tree"]); + return { repository, revision: git(repository, ["rev-parse", "HEAD"]) }; +} + +async function upgradedPlugin(root: string, previousVersion: string) { + const previous = join(root, "previous-plugin"); + cpSync(PLUGIN_ROOT, previous, { recursive: true }); + const manifestPath = join(previous, ".codex-plugin", "plugin.json"); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as { + version: string; + }; + manifest.version = previousVersion; + writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + "\n"); + writeFileSync( + join(previous, "scripts", "workbench_source_excerpt.py"), + "raise RuntimeError('stale excerpt helper must be replaced')\n", + ); + + const home = join(root, "codex-home"); + mkdirSync(home, { mode: 0o700 }); + writeFileSync( + join(home, "config.toml"), + 'cli_auth_credentials_store = "file"\n\n[features]\nplugins = true\n', + ); + const command = resolveCodexCommand(); + const environment = { + ...process.env, + CODEX_HOME: home, + OPENAI_API_KEY: undefined, + CODEX_API_KEY: undefined, + }; + const login = spawnSync(command.command, ["login", "--with-api-key"], { + env: environment, + input: "synthetic-key\n", + encoding: "utf8", + windowsHide: true, + }); + expect(login.error).toBeUndefined(); + expect(login.status, login.stderr).toBe(0); + const options = { codexCommand: command, environment }; + const predecessor = await bootstrapPlugin(home, previous, options); + const upgraded = await bootstrapPlugin(home, PLUGIN_ROOT, options); + const installedMcp = JSON.parse( + readFileSync(join(upgraded.installedRoot, ".mcp.json"), "utf8"), + ) as { mcpServers: Record }; + expect(predecessor.version).toBe(previousVersion); + expect(upgraded.version).toBe(BUNDLED_PLUGIN_VERSION); + expect(upgraded.installedRoot).not.toBe(predecessor.installedRoot); + expect(installedMcp.mcpServers["codex-security"]?.env_vars).toContain( + "CODEX_SAFETY_IDENTIFIER", + ); + expect( + readFileSync( + join(upgraded.installedRoot, "scripts", "workbench_source_excerpt.py"), + ), + ).toEqual( + readFileSync(join(PLUGIN_ROOT, "scripts", "workbench_source_excerpt.py")), + ); + return upgraded.installedRoot; +} + +function collisionProbe( + pluginRoot: string, + fixture: { + deepPath: string; + repository: string; + replacement: string; + revision: string; + }, +) { + const program = String.raw` +import io, json, os, subprocess, sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +import workbench_source_excerpt as excerpts +from workbench_target import clean_worktree_content_digest + +repository = Path(sys.argv[2]).resolve() +revision = sys.argv[3] +replacement = sys.argv[4] +deep_path = sys.argv[5] +metadata = repository.stat() +identity = (revision, clean_worktree_content_digest(), metadata.st_dev, metadata.st_ino) +authority = excerpts.capture_source_scopes(repository, identity, ["src", "src"]) +authority_json = json.dumps(authority) +scan = { + "mode": "standard", + "diff_target_kind": None, + "target_revision": revision, + "target_snapshot_digest": clean_worktree_content_digest(), + "source_scopes_json": authority_json, +} +ordered_excerpt = excerpts.finding_source_excerpt_from_context( + excerpts.source_excerpt_context(scan, repository, ["src"]), + [ + {"path": "src/allowed.py", "startLine": 1, "role": "evidence"}, + {"path": "outside.py", "startLine": 1, "role": "not_root_control"}, + ], +) +def excerpt(path, saved=scan, selected_paths=None): + if selected_paths is None: + selected_paths = ["src"] + context = excerpts.source_excerpt_context(saved, repository, selected_paths) + return excerpts.finding_source_excerpt_from_context( + context, + [{"path": path, "startLine": 1, "endLine": 1, "role": "root_control"}], + ) +original_git = excerpts.local_git_bytes +original_popen = subprocess.Popen +blob_reads = [] +blob_read_sizes = [] +replacement_probe_commands = [] +replacement_probe_read_sizes = [] +watch_replacement_probe = False +class WatchedBlobOutput: + def __init__(self, output): + self.output = output + def read(self, size=-1): + if size < 0 or size > 64 * 1024: + raise AssertionError("blob response was buffered") + blob_read_sizes.append(size) + return self.output.read(size) + def __getattr__(self, name): + return getattr(self.output, name) +class WatchedReplacementOutput: + def __init__(self, output): + self.output = output + def read(self, size=-1): + replacement_probe_read_sizes.append(size) + return self.output.read(size) + def __getattr__(self, name): + return getattr(self.output, name) +def watched_popen(arguments, *positional, **keywords): + process = original_popen(arguments, *positional, **keywords) + if len(arguments) >= 3 and arguments[-3:-1] == ["cat-file", "blob"]: + blob_reads.append(arguments[-1]) + process.stdout = WatchedBlobOutput(process.stdout) + if watch_replacement_probe and "for-each-ref" in arguments: + start = arguments.index("for-each-ref") + replacement_probe_commands.append(arguments[start:]) + process.stdout = WatchedReplacementOutput(process.stdout) + return process +subprocess.Popen = watched_popen +allowed = excerpt("src/allowed.py") +linked_authority = excerpts.capture_source_scopes( + repository, identity, ["alias/subdir"] +) +linked_scan = {**scan, "source_scopes_json": json.dumps(linked_authority)} +before = len(blob_reads) +linked_excerpt = excerpt( + "alias/subdir/secret.py", linked_scan, ["alias/subdir"] +) +linked_blob_reads = blob_reads[before:] +before = len(blob_read_sizes) +large_blob_excerpt = excerpt("src/large.py") +large_blob_sizes = blob_read_sizes[before:] +original_excerpt_reader = excerpts.scanned_source_excerpt +def exhausted_blob(*arguments, **keywords): + raise MemoryError("synthetic blob exhaustion") +excerpts.scanned_source_excerpt = exhausted_blob +try: + blob_memory_error = excerpt("src/allowed.py") +finally: + excerpts.scanned_source_excerpt = original_excerpt_reader +before = len(blob_reads) +collisions = { + path: excerpt(path) + for path in ( + "src/LOWER.py", + "src/lower.py", + "src/é.py", + "src/é.py", + "src/trailing.py", + "src/trailing.py.", + ) +} +collision_blob_reads = blob_reads[before:] +original_samefile = Path.samefile +def distinct_case_samefile(left, right): + right = Path(right) + if left.parent == right.parent and {left.name, right.name} == {"LOWER.py", "lower.py"}: + return False + return original_samefile(left, right) +Path.samefile = distinct_case_samefile +before = len(blob_reads) +try: + distinct_case_excerpts = { + path: excerpt(path) + for path in ("src/LOWER.py", "src/lower.py") + } +finally: + Path.samefile = original_samefile +distinct_case_blob_reads = blob_reads[before:] +outside = excerpt("outside.py") +file_authority = excerpts.capture_source_scopes(repository, identity, ["mismatch"]) +file_scan = {**scan, "source_scopes_json": json.dumps(file_authority)} +before = len(blob_reads) +file_descendant_excerpt = excerpt( + "mismatch/secret.py", file_scan, ["mismatch"] +) +file_descendant_blob_reads = blob_reads[before:] +historical_authority = excerpts.capture_source_scopes( + repository, identity, ["historical"], diff_target_kind="commit" +) +historical_scan = { + **scan, + "mode": "diff", + "diff_target_kind": "commit", + "source_scopes_json": json.dumps(historical_authority), +} +historical_context = excerpts.source_excerpt_context( + historical_scan, repository, ["historical"] +) +historical_excerpt = excerpts.finding_source_excerpt_from_context( + historical_context, + [{"path": "historical", "startLine": 1, "role": "root_control"}], +) +before = len(blob_reads) +broadened = excerpt( + "outside.py", + { + **scan, + "source_scopes_json": json.dumps( + {**authority, "paths": [{"kind": "directory", "path": "."}]} + ), + }, +) +broadened_blob_reads = blob_reads[before:] +scope_collision_authority = excerpts.capture_source_scopes( + repository, identity, ["Scope"] +) +scope_collision_scan = { + **scan, + "source_scopes_json": json.dumps(scope_collision_authority), +} +before = len(blob_reads) +scope_collision_excerpt = excerpt( + "Scope/selected.py", scope_collision_scan, ["Scope"] +) +scope_collision_blob_reads = blob_reads[before:] +subtarget = repository / "src" +subtarget_metadata = subtarget.stat() +subtarget_authority = excerpts.capture_source_scopes( + subtarget, + ( + revision, + clean_worktree_content_digest(), + subtarget_metadata.st_dev, + subtarget_metadata.st_ino, + ), + ["."], +) +subtarget_scan = { + **scan, + "source_scopes_json": json.dumps(subtarget_authority), +} +subtarget_context = excerpts.source_excerpt_context( + subtarget_scan, subtarget, ["."] +) +def distinct_subtarget_samefile(left, right): + right = Path(right) + if ( + left.parent == subtarget + and right.parent == subtarget + and {left.name, right.name} == {"LOWER.py", "lower.py"} + ): + return False + return original_samefile(left, right) +Path.samefile = distinct_subtarget_samefile +try: + subtarget_case_excerpts = { + path: excerpts.finding_source_excerpt_from_context( + subtarget_context, + [{"path": path, "startLine": 1, "role": "root_control"}], + ) + for path in ("LOWER.py", "lower.py") + } + subtarget_unicode_alias = excerpts.finding_source_excerpt_from_context( + subtarget_context, + [{"path": "é.py", "startLine": 1, "role": "root_control"}], + ) +finally: + Path.samefile = original_samefile +subprocess.run(["git", "-C", str(subtarget), "init", "-q"], check=True) +outer_git_dir = Path( + subprocess.check_output( + ["git", "-C", str(repository), "rev-parse", "--absolute-git-dir"], + text=True, + ).strip() +) +alternates = subtarget / ".git" / "objects" / "info" / "alternates" +alternates.write_bytes((str(outer_git_dir / "objects") + "\n").encode()) +subprocess.run( + ["git", "-C", str(subtarget), "update-ref", "refs/heads/main", revision], + check=True, +) +nested_context = excerpts.source_excerpt_context(subtarget_scan, subtarget, ["."]) +before = len(blob_reads) +nested_excerpt = excerpts.finding_source_excerpt_from_context( + nested_context, + [{"path": "outside.py", "startLine": 1, "endLine": 1, "role": "root_control"}], +) +nested_blob_reads = blob_reads[before:] +subprocess.run( + ["git", "-C", str(repository), "update-ref", f"refs/replace/{revision}", replacement], + check=True, +) +before = len(blob_reads) +try: + replaced = excerpt("src/allowed.py") + replacement_blob_reads = blob_reads[before:] + previous_no_replace = os.environ.get("GIT_NO_REPLACE_OBJECTS") + try: + os.environ["GIT_NO_REPLACE_OBJECTS"] = "1" + environment_disabled_excerpt = excerpt("src/allowed.py") + environment_disabled_paths = excerpts.capture_source_scopes(repository, identity, ["src"])["paths"] + finally: + if previous_no_replace is None: + os.environ.pop("GIT_NO_REPLACE_OBJECTS", None) + else: + os.environ["GIT_NO_REPLACE_OBJECTS"] = previous_no_replace + subprocess.run(["git", "-C", str(repository), "config", "core.useReplaceRefs", "false"], check=True) + try: + config_disabled_excerpt = excerpt("src/allowed.py") + config_disabled_paths = excerpts.capture_source_scopes(repository, identity, ["src"])["paths"] + finally: + subprocess.run(["git", "-C", str(repository), "config", "--unset", "core.useReplaceRefs"], check=True) +finally: + subprocess.run( + ["git", "-C", str(repository), "update-ref", "-d", f"refs/replace/{revision}"], + check=True, + ) +custom_replacement_base = "refs/synthetic-replacements/" +custom_replacement_ref = f"{custom_replacement_base}{revision}" +subprocess.run( + ["git", "-C", str(repository), "update-ref", custom_replacement_ref, replacement], + check=True, +) +original_replacement_base = os.environ.get("GIT_REPLACE_REF_BASE") +before = len(blob_reads) +try: + os.environ["GIT_REPLACE_REF_BASE"] = custom_replacement_base + custom_replacement_paths = excerpts.capture_source_scopes( + repository, identity, ["src"] + )["paths"] + custom_replaced = excerpt("src/allowed.py") + os.environ["GIT_REPLACE_REF_BASE"] = "--count=0" + invalid_replacement_base_paths = excerpts.capture_source_scopes( + repository, identity, ["src"] + )["paths"] +finally: + if original_replacement_base is None: + os.environ.pop("GIT_REPLACE_REF_BASE", None) + else: + os.environ["GIT_REPLACE_REF_BASE"] = original_replacement_base + subprocess.run( + ["git", "-C", str(repository), "update-ref", "-d", custom_replacement_ref], + check=True, + ) +custom_replacement_blob_reads = blob_reads[before:] +replace_directory = outer_git_dir / "refs" / "replace" +replace_directory.mkdir(parents=True, exist_ok=True) +broken_replacements = [] +for index in range(128): + broken = replace_directory / f"r{index:04d}" + broken.write_text("not-an-object\n") + broken_replacements.append(broken) +watch_replacement_probe = True +try: + malformed_replacement_paths = excerpts.capture_source_scopes( + repository, identity, ["src"] + )["paths"] +finally: + watch_replacement_probe = False + for broken in broken_replacements: + broken.unlink() +def exhausted_popen(arguments, *positional, **keywords): + if "for-each-ref" in arguments: + raise MemoryError("synthetic replacement probe exhaustion") + return original_popen(arguments, *positional, **keywords) +subprocess.Popen = exhausted_popen +try: + replacement_memory_paths = excerpts.capture_source_scopes( + repository, identity, ["src"] + )["paths"] +finally: + subprocess.Popen = watched_popen +subprocess.Popen = original_popen +legacy_scan = dict(scan) +legacy_scan.pop("source_scopes_json") +legacy = excerpt("src/allowed.py", legacy_scan) +invalid = excerpt("src/allowed.py", {**scan, "source_scopes_json": "{"}) +malformed_revision = excerpt("src/allowed.py", {**scan, "target_revision": 42}) + +original_context = excerpts.git_worktree_context +git_calls = [] +def forbidden_git(*arguments, **kwargs): + git_calls.append(arguments) + raise AssertionError("ineligible diff mode reached Git") +excerpts.local_git_bytes = forbidden_git +excerpts.git_worktree_context = forbidden_git +try: + mutable = { + str(kind): excerpt( + "src/allowed.py", {**scan, "mode": "diff", "diff_target_kind": kind} + ) + for kind in ("working_tree", None) + } +finally: + excerpts.local_git_bytes = original_git + excerpts.git_worktree_context = original_context +immutable = { + kind: excerpt( + "src/allowed.py", {**scan, "mode": "diff", "diff_target_kind": kind} + ) + for kind in ("commit", "range") +} +deep_authority = excerpts.capture_source_scopes(repository, identity, ["deep"]) +deep_scan = {**scan, "source_scopes_json": json.dumps(deep_authority)} +batch_processes = 0 +tree_reads = 0 +original_popen = subprocess.Popen +def watched_popen(arguments, *positional, **keywords): + global batch_processes + if "cat-file" in arguments and "--batch" in arguments: + batch_processes += 1 + return original_popen(arguments, *positional, **keywords) +def counted_git(*arguments, **keywords): + global tree_reads + if len(arguments) >= 3 and arguments[1:3] == ("ls-tree", "-z"): + tree_reads += 1 + return original_git(*arguments, **keywords) +subprocess.Popen = watched_popen +excerpts.local_git_bytes = counted_git +try: + deep_excerpt = excerpt(deep_path, deep_scan, ["deep"]) +finally: + excerpts.local_git_bytes = original_git + subprocess.Popen = original_popen +def read_excerpt_page(paths): + context = excerpts.source_excerpt_context(scan, repository, ["src"]) + batch_processes = 0 + tree_reads = 0 + original_matching_tree_entries = excerpts.matching_tree_entries + def watched_page_popen(arguments, *positional, **keywords): + nonlocal batch_processes + if "cat-file" in arguments and "--batch" in arguments: + batch_processes += 1 + return original_popen(arguments, *positional, **keywords) + def counted_page_tree(*arguments, **keywords): + nonlocal tree_reads + tree_reads += 1 + return original_matching_tree_entries(*arguments, **keywords) + subprocess.Popen = watched_page_popen + excerpts.matching_tree_entries = counted_page_tree + try: + results = { + path: excerpts.finding_source_excerpt_from_context( + context, [{"path": f"src/{path}", "startLine": 1}] + ) + for path in paths + } + finally: + excerpts.matching_tree_entries = original_matching_tree_entries + subprocess.Popen = original_popen + return results, batch_processes, tree_reads +page_excerpts, page_batch_processes, page_tree_reads = read_excerpt_page( + ("allowed.py", "another.py", "third.py") +) +missing_page, missing_batch_processes, missing_tree_reads = read_excerpt_page( + (f"missing-{index}.py" for index in range(3)) +) +missing_excerpts = list(missing_page.values()) +batch_object = "1" * 40 +entry_object = b"\1" * 20 +wide_tree = b"".join( + b"100644 f%04d\0" % index + entry_object for index in range(5_000) +) +wide_tree += b"100644 target.py\0" + entry_object +class CappedRead(io.BytesIO): + largest = 0 + def read(self, size=-1): + if size < 0 or size > 64 * 1024: + raise AssertionError("tree response was buffered") + self.largest = max(self.largest, size) + return super().read(size) +stream_requests = io.BytesIO() +stream_responses = CappedRead( + f"{batch_object} tree {len(wide_tree)}\n".encode() + + wide_tree + + b"\n" +) +streamed_aliases = excerpts.matching_tree_entries( + stream_requests, stream_responses, batch_object, "target.py" +) +alias_tree = (b"100644 target.py\0" + entry_object) * 20_000 +alias_responses = CappedRead( + f"{batch_object} tree {len(alias_tree)}\n".encode() + + alias_tree + + b"\n" +) +ambiguous_alias = excerpts.matching_tree_entries( + io.BytesIO(), alias_responses, batch_object, "target.py" +) +oversized_name = b"x" * (64 * 1024 + 1) +oversized_tree = ( + b"100644 " + oversized_name + b"\0" + entry_object + + b"100644 target.py\0" + entry_object +) +oversized_responses = CappedRead( + f"{batch_object} tree {len(oversized_tree)}\n".encode() + + oversized_tree + + b"\n" +) +oversized_sibling = excerpts.matching_tree_entries( + io.BytesIO(), oversized_responses, batch_object, "target.py" +) +oversized_alias_tree = ( + b"100644 target.py\0" + entry_object + + b"100644 target.py" + b"." * (64 * 1024 + 1) + b"\0" + entry_object +) +oversized_alias_responses = CappedRead( + f"{batch_object} tree {len(oversized_alias_tree)}\n".encode() + + oversized_alias_tree + + b"\n" +) +oversized_alias = excerpts.matching_tree_entries( + io.BytesIO(), oversized_alias_responses, batch_object, "target.py" +) +streamed_wide_tree = ( + streamed_aliases == ("target.py", "file", entry_object.hex()) + and stream_requests.getvalue() == batch_object.encode() + b"\0" + and stream_responses.largest == 64 * 1024 + and ambiguous_alias is None + and alias_responses.largest == 64 * 1024 + and oversized_sibling == ("target.py", "file", entry_object.hex()) + and oversized_responses.largest == 64 * 1024 + and oversized_alias is None + and oversized_alias_responses.largest == 64 * 1024 +) +large_paths = [str(index) for index in range(20_000)] +large_tree = "0" * 40 +large_scan = { + **scan, + "source_scopes_json": json.dumps( + { + "version": 1, + "paths": [ + {"kind": "file", "path": path} for path in large_paths + ], + "targetTree": large_tree, + } + ), +} +large_recipe_bytes = len( + json.dumps( + {"target": {"kind": "paths", "paths": large_paths}}, separators=(",", ":") + ).encode() +) +original_target_tree = excerpts.target_tree +original_tree_path = excerpts.tree_path +tree_path_checks = 0 +def counted_tree_path(_, __, value): + global tree_path_checks + tree_path_checks += 1 + return (value, "file", "1" * 40) +excerpts.target_tree = lambda *_: (repository, large_tree) +excerpts.tree_path = counted_tree_path +try: + large_context = excerpts.source_excerpt_context( + large_scan, repository, large_paths + ) + original_pure_path = excerpts.PurePosixPath + path_parses = 0 + def one_path_parse(*arguments): + global path_parses + path_parses += 1 + if path_parses > 1: + raise RuntimeError("source scope lookup rebuilt the path") + return original_pure_path(*arguments) + excerpts.PurePosixPath = one_path_parse + try: + excerpts.selected_source_kinds( + large_context[2], + "/".join(["nested"] * 20_000 + ["file.py"]), + ) + finally: + excerpts.PurePosixPath = original_pure_path + large_excerpt = excerpts.finding_source_excerpt_from_context( + large_context, [{"path": "unmatched/path.py", "startLine": 1}] + ) +finally: + excerpts.target_tree = original_target_tree + excerpts.tree_path = original_tree_path +print(json.dumps({ + "allowed": allowed, + "broadened": broadened, + "broadenedBlobReads": broadened_blob_reads, + "collisionBlobReads": collision_blob_reads, + "collisions": collisions, + "configDisabledExcerpt": config_disabled_excerpt, + "configDisabledPaths": config_disabled_paths, + "customReplaced": custom_replaced, + "customReplacementBlobReads": custom_replacement_blob_reads, + "customReplacementPaths": custom_replacement_paths, + "deepBatchProcesses": batch_processes, + "deepExcerpt": deep_excerpt, + "deepPathParses": path_parses, + "deepTreeReads": tree_reads, + "distinctCaseBlobReads": distinct_case_blob_reads, + "distinctCaseExcerpts": distinct_case_excerpts, + "duplicatePaths": len(authority["paths"]), + "environmentDisabledExcerpt": environment_disabled_excerpt, + "environmentDisabledPaths": environment_disabled_paths, + "fileAuthorityPaths": file_authority["paths"], + "fileDescendantBlobReads": file_descendant_blob_reads, + "fileDescendantExcerpt": file_descendant_excerpt, + "historicalAuthorityPaths": historical_authority["paths"], + "historicalExcerpt": historical_excerpt, + "invalidReplacementBasePaths": invalid_replacement_base_paths, + "immutable": immutable, + "largeExcerpt": large_excerpt, + "largeBlobStreamed": ( + len(large_blob_excerpt.encode()) == 16_000 + and len(large_blob_sizes) >= 4 + and max(large_blob_sizes) == 64 * 1024 + ), + "blobMemoryError": blob_memory_error, + "largeRecipeFits": large_recipe_bytes < 256 * 1024, + "largeTreePathChecks": tree_path_checks, + "linkedAuthorityPaths": linked_authority["paths"], + "linkedBlobReads": linked_blob_reads, + "linkedExcerpt": linked_excerpt, + "malformedReplacementPaths": malformed_replacement_paths, + "invalid": invalid, + "legacy": legacy, + "malformedRevision": malformed_revision, + "missingBatchProcesses": missing_batch_processes, + "missingExcerpts": missing_excerpts, + "missingTreeReads": missing_tree_reads, + "mutable": {"excerpts": mutable, "gitCalls": len(git_calls)}, + "nestedBlobReads": nested_blob_reads, + "nestedExcerpt": nested_excerpt, + "orderedExcerpt": ordered_excerpt, + "pageBatchProcesses": page_batch_processes, + "pageExcerpts": page_excerpts, + "pageTreeReads": page_tree_reads, + "subtargetCaseExcerpts": subtarget_case_excerpts, + "subtargetPaths": subtarget_authority["paths"], + "subtargetUnicodeAlias": subtarget_unicode_alias, + "outside": outside, + "pathCollisionBlobReads": scope_collision_blob_reads, + "pathCollisionExcerpt": scope_collision_excerpt, + "pathCollisionPaths": len(scope_collision_authority["paths"]), + "replaced": replaced, + "replacementBlobReads": replacement_blob_reads, + "replacementMemoryPaths": replacement_memory_paths, + "replacementProbeCommands": replacement_probe_commands, + "replacementProbeReadSizes": replacement_probe_read_sizes, + "streamedWideTree": streamed_wide_tree, +})) +`; + const result = spawnSync( + python(), + [ + "-I", + "-B", + "-c", + program, + join(pluginRoot, "scripts"), + fixture.repository, + fixture.revision, + fixture.replacement, + fixture.deepPath, + ], + { encoding: "utf8" }, + ); + expect(result.status, result.stderr).toBe(0); + return JSON.parse(result.stdout) as Record; +} + +describe("workbench source excerpts", () => { + test.each(["0.1.60", "0.1.74", "0.1.78"])( + "fails closed on normalized collisions after upgrading cache %s", + async (previousVersion) => { + const root = realpathSync( + mkdtempSync(join(tmpdir(), "codex-security-source-collision-")), + ); + temporaryRoots.push(root); + const fixture = collisionRepository(root); + const installedRoot = await upgradedPlugin(root, previousVersion); + const fixed = collisionProbe(installedRoot, fixture); + + expect(fixed).toEqual({ + allowed: expect.stringContaining("allowed = True"), + broadened: null, + broadenedBlobReads: [], + collisionBlobReads: fixture.caseSensitive + ? [expect.any(String), expect.any(String)] + : [], + collisions: { + "src/LOWER.py": fixture.caseSensitive + ? expect.stringContaining("case_upper = True") + : null, + "src/lower.py": fixture.caseSensitive + ? expect.stringContaining("case_lower = True") + : null, + "src/é.py": null, + "src/é.py": null, + "src/trailing.py": null, + "src/trailing.py.": null, + }, + customReplaced: null, + configDisabledExcerpt: expect.stringContaining("allowed = True"), + configDisabledPaths: [{ kind: "directory", path: "src" }], + customReplacementBlobReads: [], + customReplacementPaths: [], + deepBatchProcesses: 1, + deepExcerpt: expect.stringContaining("deep = True"), + deepPathParses: 1, + deepTreeReads: 0, + distinctCaseBlobReads: [expect.any(String), expect.any(String)], + distinctCaseExcerpts: { + "src/LOWER.py": expect.stringContaining("case_upper = True"), + "src/lower.py": expect.stringContaining("case_lower = True"), + }, + duplicatePaths: 1, + environmentDisabledExcerpt: expect.stringContaining("allowed = True"), + environmentDisabledPaths: [{ kind: "directory", path: "src" }], + fileAuthorityPaths: [{ kind: "file", path: "mismatch" }], + fileDescendantBlobReads: [], + fileDescendantExcerpt: null, + historicalAuthorityPaths: [{ kind: "file", path: "historical" }], + historicalExcerpt: expect.stringContaining("historical_file = True"), + invalidReplacementBasePaths: [], + immutable: { + commit: expect.stringContaining("allowed = True"), + range: expect.stringContaining("allowed = True"), + }, + largeExcerpt: null, + largeBlobStreamed: true, + blobMemoryError: null, + largeRecipeFits: true, + largeTreePathChecks: 0, + linkedAuthorityPaths: [], + linkedBlobReads: [], + linkedExcerpt: null, + malformedReplacementPaths: [], + invalid: null, + legacy: null, + malformedRevision: null, + missingBatchProcesses: 1, + missingExcerpts: [null, null, null], + missingTreeReads: 2, + mutable: { + excerpts: { working_tree: null, None: null }, + gitCalls: 0, + }, + nestedBlobReads: [], + nestedExcerpt: null, + orderedExcerpt: expect.stringContaining("allowed = True"), + pageBatchProcesses: 1, + pageExcerpts: { + "allowed.py": expect.stringContaining("allowed = True"), + "another.py": expect.stringContaining("another = True"), + "third.py": expect.stringContaining("third = True"), + }, + pageTreeReads: 2, + subtargetCaseExcerpts: { + "LOWER.py": expect.stringContaining("case_upper = True"), + "lower.py": expect.stringContaining("case_lower = True"), + }, + subtargetPaths: [{ kind: "directory", path: "." }], + subtargetUnicodeAlias: null, + outside: null, + pathCollisionBlobReads: [], + pathCollisionExcerpt: null, + pathCollisionPaths: 1, + replaced: null, + replacementBlobReads: [], + replacementMemoryPaths: [], + replacementProbeCommands: [ + ["for-each-ref", "--count=1", "--format=", "refs/replace/*"], + ], + replacementProbeReadSizes: [1], + streamedWideTree: true, + }); + }, + 60_000, + ); + + test("persists source authority outside every writer transaction", () => { + const root = realpathSync( + mkdtempSync(join(tmpdir(), "codex-security-source-writers-")), + ); + temporaryRoots.push(root); + const { repository } = ordinaryRepository(root); + const scanRoot = join(root, "scans"); + const cliScanDirectory = join(root, "cli-scan"); + mkdirSync(cliScanDirectory, { mode: 0o700 }); + const workspaceId = randomUUID(); + const program = String.raw` +import contextlib, io, json, os, sqlite3, sys +from pathlib import Path +os.environ["CODEX_SECURITY_STATE_DIR"] = sys.argv[2] +sys.path.insert(0, sys.argv[1]) +import workbench_db + +active_connection = None +current_command = None +race_mutation = False +race_replacement = False +transaction_states = [] +digest_transaction_states = [] +blocked_digest_writers = [] +available_digest_writers = [] +original_connect = workbench_db.connect +original_capture = workbench_db.capture_source_scopes +original_deep_capture = workbench_db.deep_scan.capture_source_scopes +original_deep_worktree_digest = workbench_db.deep_scan.worktree_content_digest +original_deep_remediation_target = workbench_db.deep_scan.require_remediation_target +def connect(): + global active_connection + active_connection = original_connect() + return active_connection +def capture(*arguments, **keywords): + transaction_states.append([current_command, active_connection.in_transaction]) + return original_capture(*arguments, **keywords) +def deep_capture(*arguments, **keywords): + transaction_states.append([current_command, active_connection.in_transaction]) + authority = original_deep_capture(*arguments, **keywords) + if race_mutation: + (Path(repository) / "src" / "allowed.py").write_text("changed_after_capture = True\n") + return authority +def deep_worktree_digest(*arguments, **keywords): + digest_transaction_states.append([current_command, active_connection.in_transaction]) + with sqlite3.connect(workbench_db.database_path(), timeout=0) as concurrent: + try: + concurrent.execute("BEGIN IMMEDIATE") + except sqlite3.OperationalError as error: + blocked_digest_writers.append(str(error)) + else: + available_digest_writers.append(current_command) + concurrent.rollback() + return original_deep_worktree_digest(*arguments, **keywords) +def deep_remediation_target(value): + target = original_deep_remediation_target(value) + if race_replacement and active_connection.in_transaction: + target.rename(target.with_name(f"{target.name}-original")) + target.mkdir() + return original_deep_remediation_target(value) + return target +def run(arguments): + global current_command + current_command = arguments[0] + sys.argv = ["workbench_db.py", *arguments] + output = io.StringIO() + with contextlib.redirect_stdout(output): + workbench_db.main() + return json.loads(output.getvalue()) +workbench_db.connect = connect +workbench_db.capture_source_scopes = capture +workbench_db.deep_scan.capture_source_scopes = deep_capture +workbench_db.deep_scan.worktree_content_digest = deep_worktree_digest +workbench_db.deep_scan.require_remediation_target = deep_remediation_target + +repository, scan_root, cli_scan_dir, workspace_id = sys.argv[3:7] +run(["create-workspace", "--workspace-id", workspace_id, "--thread-id", "workspace-writer"]) +run(["save-workspace", "--workspace-id", workspace_id, "--target-path", repository, "--scope", "src", "--mode", "standard"]) +workspace = run(["start-scan", "--workspace-id", workspace_id, "--scan-root", scan_root]) +prompt = run(["start-prompt-only-scan", "--thread-id", "prompt-writer", "--target-path", repository, "--scope", "src", "--mode", "standard", "--scan-root", scan_root]) +headless = run(["start-headless-standard-scan", "--thread-id", "headless-writer", "--target-path", repository, "--scope", "src", "--scan-root", scan_root]) +recipe = json.dumps({"config": {}, "mode": "standard", "repository": repository, "target": {"kind": "paths", "paths": ["src/allowed.py", "other"]}}) +cli = run(["register-cli-scan", "--repository", repository, "--scan-dir", cli_scan_dir, "--recipe-json", recipe]) +deep = run(["begin-deep-scan", "--thread-id", "deep-writer", "--target-path", repository, "--scan-root", scan_root, "--available-parallelism", "4"]) +race_mutation = True +try: + run(["begin-deep-scan", "--thread-id", "raced-deep-writer", "--target-path", repository, "--scan-root", scan_root, "--available-parallelism", "4"]) +except SystemExit as error: + race_error = str(error) +else: + race_error = None +(Path(repository) / "src" / "allowed.py").write_text("allowed = True\n") +race_mutation = False +race_replacement = True +try: + run(["begin-deep-scan", "--thread-id", "replaced-deep-writer", "--target-path", repository, "--scan-root", scan_root, "--available-parallelism", "4"]) +except SystemExit as error: + replacement_error = str(error) +else: + replacement_error = None +finally: + replacement = Path(repository) + replacement.rmdir() + replacement.with_name(f"{replacement.name}-original").rename(replacement) +with sqlite3.connect(workbench_db.database_path()) as connection: + authorities = {row[0]: json.loads(row[1]) for row in connection.execute("SELECT id, source_scopes_json FROM scans")} +scan_ids = { + "workspace": workspace["results"]["scanId"], + "prompt": prompt["scan"]["scanId"], + "headless": headless["scan"]["scanId"], + "CLI": cli["scanId"], + "deep": deep["deepScan"]["scanId"], +} +print(json.dumps({"authorities": authorities, "availableDigestWriters": available_digest_writers, "blockedDigestWriters": blocked_digest_writers, "digestTransactionStates": digest_transaction_states, "raceError": race_error, "replacementError": replacement_error, "transactionStates": transaction_states, "scanIds": scan_ids})) +`; + const result = spawnSync( + python(), + [ + "-I", + "-B", + "-c", + program, + join(PLUGIN_ROOT, "scripts"), + join(root, "state"), + repository, + scanRoot, + cliScanDirectory, + workspaceId, + ], + { encoding: "utf8" }, + ); + expect(result.status, result.stderr).toBe(0); + const writers = JSON.parse(result.stdout) as Record; + expect(writers["transactionStates"]).toEqual([ + ["start-scan", false], + ["start-prompt-only-scan", false], + ["start-headless-standard-scan", false], + ["register-cli-scan", false], + ["begin-deep-scan", false], + ["begin-deep-scan", false], + ["begin-deep-scan", false], + ]); + expect(writers["digestTransactionStates"]).toEqual( + Array.from({ length: 6 }, () => ["begin-deep-scan", false]), + ); + expect(writers["availableDigestWriters"]).toEqual( + Array.from({ length: 6 }, () => "begin-deep-scan"), + ); + expect(writers["blockedDigestWriters"]).toEqual([]); + expect(writers["raceError"]).toBe( + "The selected scan target changed while the scan was starting. Try again.", + ); + expect(writers["replacementError"]).toBe( + "The selected scan target changed while the scan was starting. Try again.", + ); + const authorities = writers["authorities"] as Record< + string, + { + paths: Array<{ kind: "directory" | "file"; path: string }>; + targetTree: string; + version: number; + } + >; + const scanIds = writers["scanIds"] as Record; + const expected = [ + ["workspace", [{ kind: "directory", path: "src" }]], + ["prompt", [{ kind: "directory", path: "src" }]], + ["headless", [{ kind: "directory", path: "src" }]], + [ + "CLI", + [ + { kind: "file", path: "src/allowed.py" }, + { kind: "directory", path: "other" }, + ], + ], + ["deep", [{ kind: "directory", path: "." }]], + ] as const; + expect(Object.keys(authorities)).toHaveLength(expected.length); + for (const [writer, paths] of expected) { + const authority = authorities[String(scanIds[writer])]; + expect(authority?.version, writer).toBe(1); + expect(authority?.targetTree, writer).toMatch(/^[0-9a-f]{40,64}$/); + expect(authority?.paths, writer).toEqual([...paths]); + } + }, 60_000); + + test.each([false, true])( + "migration 33 appends once with a preexisting source column: %s", + (preexistingColumn) => { + const program = String.raw` +import json, sqlite3, sys +sys.path.insert(0, sys.argv[1]) +from workbench_schema import MIGRATIONS, apply_migrations + +connection = sqlite3.connect(":memory:") +connection.row_factory = sqlite3.Row +counter = 0 +def now(): + global counter + counter += 1 + return f"2026-08-24T00:00:{counter:02d}Z" +def rows(): + return [tuple(row) for row in connection.execute( + "SELECT version, name, applied_at FROM schema_migrations WHERE version >= 29 ORDER BY version" + )] + +predecessor = MIGRATIONS[:-1] +apply_migrations(connection, predecessor, now, lambda database: None) +if sys.argv[2] == "true": + connection.execute("ALTER TABLE scans ADD COLUMN source_scopes_json TEXT") + connection.execute("INSERT INTO schema_migrations VALUES (34, 'persist authorized source excerpt scopes', ?)", (now(),)) +before = rows() +apply_migrations(connection, MIGRATIONS, now, lambda database: None) +once = rows() +column_count = sum( + row[1] == "source_scopes_json" + for row in connection.execute("PRAGMA table_info(scans)") +) +apply_migrations(connection, MIGRATIONS, now, lambda database: None) +twice = rows() +print(json.dumps({ + "predecessorMax": predecessor[-1][0], + "beforeVersions": [row[0] for row in before], + "onceVersions": [row[0] for row in once], + "oldRowsPreserved": [row for row in once if row[0] != 33] == before, + "secondApplyUnchanged": twice == once, + "columnCount": column_count, + "newName": next(row[1] for row in once if row[0] == 33), +})) +`; + const result = spawnSync( + python(), + [ + "-I", + "-B", + "-c", + program, + join(PLUGIN_ROOT, "scripts"), + String(preexistingColumn), + ], + { encoding: "utf8" }, + ); + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + predecessorMax: 32, + beforeVersions: preexistingColumn + ? [29, 30, 31, 32, 34] + : [29, 30, 31, 32], + onceVersions: preexistingColumn + ? [29, 30, 31, 32, 33, 34] + : [29, 30, 31, 32, 33], + oldRowsPreserved: true, + secondApplyUnchanged: true, + columnCount: 1, + newName: "persist selected source excerpt authority", + }); + }, + ); + + test("authorizes raw finding paths before bounding display output", () => { + const program = String.raw` +import json, sqlite3, sys, tempfile +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +import workbench_db + +raw_path = "segment/" * 300 + "source.py" +connection = sqlite3.connect(":memory:") +connection.row_factory = sqlite3.Row +connection.execute("CREATE TABLE finding_locations (occurrence_id TEXT, relative_path TEXT, start_line INTEGER, end_line INTEGER, role TEXT, sort_order INTEGER)") +connection.execute("INSERT INTO finding_locations VALUES ('occurrence', ?, 1, 1, 'root_control', 0)", (raw_path,)) +target = Path(tempfile.mkdtemp()).resolve() +workbench_db.require_scan_target_identity = lambda scan: target +workbench_db.safe_source_path = lambda selected, value: None +workbench_db.finding_remediation_result = lambda database, occurrence_id: None +workbench_db.finding_triage_result = lambda database, occurrence_id: None +workbench_db.scan_history.finding_matches = lambda *arguments: ([], None, []) +seen = [] +selected_paths_seen = [] +authority_context = (target, ()) +def prepare_context(scan, selected, selected_paths): + selected_paths_seen.append(selected_paths) + return authority_context if selected_paths else None +def source_excerpt(context, locations): + seen.append([location["path"] for location in locations]) + if context is authority_context and locations[0]["path"] == raw_path: + return "1 raw_path_authorized = True" + return None +workbench_db.source_excerpt_context = prepare_context +workbench_db.finding_source_excerpt_from_context = source_excerpt +scan = {"id": "scan", "started_at": "now", "scan_dir": str(target), "scope": "."} +occurrence = {"id": "occurrence", "details_json": "{}", "confidence": "high", "severity": "high", "created_at": "now", "finding_id": "finding", "remediation": "fix", "summary": "summary", "title": "title"} +findings = workbench_db.finding_results(connection, scan, [occurrence, occurrence]) +finding = findings[0] +malformed = workbench_db.finding_results( + connection, + {**scan, "recipe_json": json.dumps({"target": {"kind": "paths", "paths": 42}})}, + [occurrence], +)[0] +display_path = finding["locations"][0]["path"] +print(json.dumps({ + "displayBytes": len(display_path.encode()), + "displayDiffers": display_path != raw_path, + "excerpt": finding.get("sourceExcerpt"), + "malformedExcerpt": malformed.get("sourceExcerpt"), + "malformedTitle": malformed.get("title"), + "reusedExcerpt": findings[1].get("sourceExcerpt"), + "sawRawPath": seen == [[raw_path], [raw_path], [raw_path]], + "sawSelectedPaths": selected_paths_seen == [["."], []], +})) +`; + const result = spawnSync( + python(), + ["-I", "-B", "-c", program, join(PLUGIN_ROOT, "scripts")], + { encoding: "utf8" }, + ); + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + displayBytes: 2_048, + displayDiffers: true, + excerpt: "1 raw_path_authorized = True", + malformedExcerpt: null, + malformedTitle: "title", + reusedExcerpt: "1 raw_path_authorized = True", + sawRawPath: true, + sawSelectedPaths: true, + }); + }); +});