diff --git a/plugins/magpie-security-tools/tools/security-management/src/security_management/cve_allocate/recipe.py b/plugins/magpie-security-tools/tools/security-management/src/security_management/cve_allocate/recipe.py index c7e63b56a..ffd8fe1f2 100644 --- a/plugins/magpie-security-tools/tools/security-management/src/security_management/cve_allocate/recipe.py +++ b/plugins/magpie-security-tools/tools/security-management/src/security_management/cve_allocate/recipe.py @@ -18,7 +18,10 @@ - `api` (a governance-authorised user, pre-flight `api`): one proposal to allocate through the CVE tool's API, with the form URL as the fallback and - the one `cve-tool allocate` command to run on a yes. + the one `cve-tool allocate` command to run on a yes. With `vetted_workspace` + (the secure setup's vetted operations), the command is the vetted + `cve-allocate` instead, and `title_file` is the workspace file to write the + title to first. - `recipe` (authorised, any other pre-flight outcome): open the allocation URL, paste the title, submit, paste the ID back. - `relay` (not authorised): a terse request for an authorised member — the @@ -32,6 +35,8 @@ from __future__ import annotations +import hashlib +import json import re from collections.abc import Mapping, Sequence from typing import Any @@ -78,6 +83,7 @@ def render(data: Mapping[str, Any]) -> dict[str, Any]: ) block = fence(title) command = None + title_file = None if member and (data.get("preflight") or {}).get("path") == "api": pmc = one_line(data.get("pmc")) if not pmc: @@ -89,7 +95,20 @@ def render(data: Mapping[str, Any]) -> dict[str, Any]: "This reserves a CVE ID immediately and cannot be undone.\n" f"(Fallback, if the API path fails: the form at {url}.)\n" ) - command = cve_tool.command(data, "allocate", "--pmc", pmc, "--title", title) + workspace = one_line(data.get("vetted_workspace")) + if workspace: + # The vetted operation reads the title from a workspace file; `pmc` lets the + # client refuse a token held for any other PMC. The file is named by a hash of + # tracker, issue and the approved title, so no other run (another tracker, or + # this issue with another title) writes the same file between the write and + # the allocation; a sanitised slug would not do (`a-b/c` and `a/b-c` collide). + key = hashlib.sha256(json.dumps([tracker, str(data["issue"]), title]).encode()).hexdigest()[:24] + title_file = {"path": f"{workspace.rstrip('/')}/cve-title-{key}.txt", "text": title + "\n"} + command = cve_tool.vetted_command( + "security-cve-allocate", "cve-allocate", pmc, title_file["path"] + ) + else: + command = cve_tool.command(data, "allocate", "--pmc", pmc, "--title", title) elif member: path = "recipe" indented = "\n".join(" " + line for line in block.split("\n")) @@ -115,6 +134,7 @@ def render(data: Mapping[str, Any]) -> dict[str, Any]: "path": path, "text": text, "command": command, + "title_file": title_file, "governance_member_path": member, "relay_message_present": path == "relay", "api_allocation_proposed": path == "api", diff --git a/plugins/magpie-security-tools/tools/security-management/src/security_management/shared/cve_tool.py b/plugins/magpie-security-tools/tools/security-management/src/security_management/shared/cve_tool.py index a39fb650f..287106e16 100644 --- a/plugins/magpie-security-tools/tools/security-management/src/security_management/shared/cve_tool.py +++ b/plugins/magpie-security-tools/tools/security-management/src/security_management/shared/cve_tool.py @@ -39,6 +39,8 @@ STABLE = "~/.claude/magpie/security-tools/cve-tool" #: Where a framework checkout or snapshot install keeps the tool, run sandboxed. PROJECT = "/tools/cve-tool" +#: The vetted-ops link, spelled as the sandbox exclusion matches it. +VETTED_OPS = "~/.claude/magpie/vetted-ops" def stable_installed() -> bool: @@ -64,3 +66,16 @@ def command(data: Mapping[str, Any], operation: str, *args: object) -> str: return " ".join(("uv run --project", w(str(project)), "cve-tool", operation, *quoted)) # The literal `~`, unquoted: the sandbox exclusion and the `ask` rule match this spelling. return " ".join(("uvx --from", STABLE, "cve-tool", operation, *quoted)) + + +def vetted_command(caller: str, operation: str, *args: object) -> str: + """`uvx --from vetted-op --caller `. + + The vetted operations (tools/vetted-ops, "The Vulnogram CVE-tool operations") + run the shipped Vulnogram client outside the sandbox with a fixed argv: no + override flag, no other session file. Arguments are shell-quoted here, like + `command`; a value the operation reads from a file (an allocation title, a + record) is passed as that file's workspace path, never inline. + """ + quoted = (shlex.quote(str(arg)) for arg in args) + return " ".join(("uvx --from", VETTED_OPS, "vetted-op --caller", shlex.quote(caller), operation, *quoted)) diff --git a/plugins/magpie-security/skills/cve-allocate/SKILL.md b/plugins/magpie-security/skills/cve-allocate/SKILL.md index 694c3ecc6..622c478cf 100644 --- a/plugins/magpie-security/skills/cve-allocate/SKILL.md +++ b/plugins/magpie-security/skills/cve-allocate/SKILL.md @@ -311,11 +311,12 @@ Full procedure: [`title-normalize.md`](title-normalize.md). **Ask first** whether the user is authorised under `governance.cve_allocation_gate` (Step 0 item 3 may already have the answer). Then write `{tracker, issue, title, governance_member, allocate_url, preflight, pmc, authorised_handles}` and run ` recipe`; -present its `text` as it is. Its `path` is one of: +under the secure setup, with `security-cve-allocate` granted `cve-allocate`, also pass `vetted_workspace` (the vetted-ops policy's `workspace`). +Present its `text` as it is. Its `path` is one of: - **`api`** — a governance-authorised user whose Step 0 pre-flight chose `api`: a proposal to allocate through the CVE tool's API, with `cve_authority.allocate_url` as the fallback. - On the user's yes, run its `command` (one `cve-tool allocate` call, the title shell-quoted) **once**, - write `{exit, stdout, stderr, pmc}` and run ` allocate-result`: its `next` is `step-4` with the `cve_id`, `stop` (the ID comes back by mail), `setup`, `check-records` or `recipe`, + On the user's yes, write a non-null `title_file`'s `text` to its `path` with the Write tool, then run its `command` (one `cve-tool allocate` call, the title shell-quoted, or the vetted `cve-allocate`) **once**, + write `{exit, stdout, stderr, pmc}` and run ` allocate-result` (after a vetted call, read its exit per [`api-allocation.md`](api-allocation.md#allocate-step-3-api-path) first): its `next` is `step-4` with the `cve_id`, `stop` (the ID comes back by mail), `setup`, `check-records` or `recipe`, per [`api-allocation.md` § *Allocate*](api-allocation.md#allocate-step-3-api-path). `rerun_allocate` is false except after an expired token, and even then only on a fresh yes. - **`recipe`** — a governance-authorised user otherwise: open the allocation URL (authenticate per [`/README.md`](../../../../tools/cve-tool/README.md)), paste the title, submit, paste the ID back. - **`relay`** — a user who is not authorised: the CVE tool will not let them submit, so they never get "click *Allocate*". diff --git a/plugins/magpie-security/skills/cve-allocate/api-allocation.md b/plugins/magpie-security/skills/cve-allocate/api-allocation.md index 65a5a0987..c59ac7ef0 100644 --- a/plugins/magpie-security/skills/cve-allocate/api-allocation.md +++ b/plugins/magpie-security/skills/cve-allocate/api-allocation.md @@ -95,6 +95,24 @@ On a yes, run the allocation **once**: Pass the title as a single shell-quoted argument; it is attacker-influenced text, so escape any quote it contains rather than interpolating it raw. +**Under the secure setup**, when the vetted-ops policy grants `security-cve-allocate` the `cve-allocate` operation, +allocate through it instead (` recipe` returns this command when given `vetted_workspace`): +write the recipe's `title_file` `text` to its `path` (`/cve-title-.txt`) with the Write tool, then run + +```bash +uvx --from ~/.claude/magpie/vetted-ops vetted-op --caller security-cve-allocate cve-allocate /cve-title-.txt +``` + +It runs the shipped Vulnogram client with a fixed argv, outside the sandbox, with no override flag reachable +([vetted-ops § *The Vulnogram CVE-tool operations*](../../../../tools/vetted-ops/README.md#the-vulnogram-cve-tool-operations)), +and passes `--pmc `, so the client refuses a token held for any other PMC. +The title travels as a file, so it is never quoted into a command line. +On success it exits 0 and the ID is the first stdout line, as below. +When the client fails, `vetted-op` exits 4 and names the client's own code on stderr (`CVE tool exit N`): read the table below with that `N`. +An exit 3 whose stderr says the caller `is not permitted to run 'cve-allocate'` means the policy does not grant it and nothing was allocated: run the `cve-tool allocate` command above instead, on the same yes. +Any other exit 2 or 3 is `vetted-op` refusing the arguments (the PMC, the title file): nothing was allocated, but do not route around the refusal; show its stderr and stop. +Any other outcome (an exit 4 without a `CVE tool exit` line, an interrupted run) is unclear: treat it like exit 3 / 4 in the table, and check ``'s records in Vulnogram before anything else. + | Exit | Meaning | Next | |---|---|---| | 0 | The first stdout line is the `CVE-YYYY-NNNNN` | Step 4 with that ID; no paste-back needed | diff --git a/plugins/magpie-security/skills/issue-sync/apply-and-push.md b/plugins/magpie-security/skills/issue-sync/apply-and-push.md index 145024bb2..5e407721c 100644 --- a/plugins/magpie-security/skills/issue-sync/apply-and-push.md +++ b/plugins/magpie-security/skills/issue-sync/apply-and-push.md @@ -479,7 +479,7 @@ When the state fetch fails, pass `fetched_state: "unknown"`. or extract it from the body via `awk` between the `` / `` markers — the generator is deterministic, so both yield the same bytes. - Conventional path: `/cve--.json`. + Conventional path: `/cve--.json`, or `/cve--.json` for the vetted push in step 3, which reads only the workspace. `` is the session scratch directory as an absolute path (fall back to `$TMPDIR`); `gh` may run outside the sandbox, where `$TMPDIR` differs, so pass it absolute paths. 3. **Push the update through the adapter's `push_update` method** @@ -491,6 +491,24 @@ When the state fetch fails, pass `fetched_state: "unknown"`. --cve-id --json-file /cve--.json ``` + **Under the secure setup**, when the vetted-ops policy grants `security-issue-sync` the `cve-record-update` operation, + push through it instead, with the JSON at `/cve--.json` (the vetted-ops policy's `workspace`): + + ```bash + uvx --from ~/.claude/magpie/vetted-ops vetted-op --caller security-issue-sync cve-record-update /cve--.json + ``` + + It runs the same Vulnogram push with a fixed argv and no override flag + ([vetted-ops § *The Vulnogram CVE-tool operations*](../../../../tools/vetted-ops/README.md#the-vulnogram-cve-tool-operations)), + so it covers every push above except three, which keep the `cve-tool push` command: + a JSON whose `CNA_private.state` is `PUBLIC` while the live record is not (the vetted push never promotes a record to `PUBLIC`), + and the two overrides below. + When the client fails, `vetted-op` exits 4 and names the client's own code on stderr (`CVE tool exit N`), which reads as the `cve-tool push` exit `N`; + an exit 3 whose stderr says the caller `is not permitted to run 'cve-record-update'` means the policy does not grant it: use `cve-tool push`. + Any other exit 2 or 3 is `vetted-op` rejecting the record (a forbidden key, an ID that does not match ``) or the arguments: + nothing was pushed, and the push stops there; report the stderr, and never send the same JSON through `cve-tool push` to get past the refusal. + Any other outcome (an interrupted run) may or may not have saved: push again, which the idempotence note below makes safe. + For Vulnogram, the push merges into the live record and refuses to rename an affected product or package. Two explicit overrides exist; use them only after confirming with the user: diff --git a/plugins/magpie-vetted-ops/tools/vetted-ops/README.md b/plugins/magpie-vetted-ops/tools/vetted-ops/README.md index a1fe276e2..9dd2b1034 100644 --- a/plugins/magpie-vetted-ops/tools/vetted-ops/README.md +++ b/plugins/magpie-vetted-ops/tools/vetted-ops/README.md @@ -516,7 +516,7 @@ overrides. Three operations run it instead, outside the sandbox through |---|---|---| | `cve-record-fetch` (read) | `cve_id` | `vulnogram-api-record-fetch --cve-id ` | | `cve-record-update` (write) | `cve_id body` | `vulnogram-api-record-update --cve-id --json-file -`, the record on stdin | -| `cve-allocate` (write) | `title` (a workspace file) | `vulnogram-api-allocate --title ` | +| `cve-allocate` (write) | `pmc title` (title a workspace file) | `vulnogram-api-allocate --pmc <pmc> --title <title>`; the client refuses a token held for another PMC | - The client is the copy shipped in this plugin beside vetted-ops (`tools/cve-tool-vulnogram/oauth-api`, resolved like the tracker backends), diff --git a/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cli.py b/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cli.py index 1554908ab..ffe94df16 100644 --- a/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cli.py +++ b/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cli.py @@ -145,6 +145,8 @@ def _validate_params( resolved[name] = ops_mod.commit_hash(raw) elif name == "cve_id": resolved[name] = ops_mod.cve_id(raw) + elif name == "pmc": + resolved[name] = ops_mod.pmc(raw) elif name == "date": resolved[name] = ops_mod.date(raw) elif name == "cursor": diff --git a/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cve_tool.py b/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cve_tool.py index f4d6c2fff..318975564 100644 --- a/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cve_tool.py +++ b/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cve_tool.py @@ -27,7 +27,7 @@ * the client is the copy shipped beside vetted-ops (``tool_dir``), never a path the agent can write; * only the CVE id, the record JSON (piped on stdin, the bytes vetted-ops - validated) and the allocation title are the caller's; every override flag + validated), the allocation PMC and the allocation title are the caller's; every override flag stays off, so the record commands' merge-mode guards always apply; * the child gets an allowlisted environment (``child_env``): ``HOME`` from the password database, a fixed ``PATH``, locale and ``TMPDIR`` only. No diff --git a/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/ops.py b/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/ops.py index 195758b97..b036e306a 100644 --- a/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/ops.py +++ b/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/ops.py @@ -224,6 +224,14 @@ def cve_id(value: str) -> str: return _check(_CVE_ID, value, "CVE id") +#: A Vulnogram PMC / committee id (`airflow`, `incubator-foo`): lowercase, no path or flag. +_PMC = re.compile(r"^[a-z][a-z0-9-]{0,63}$") + + +def pmc(value: str) -> str: + return _check(_PMC, value, "PMC") + + #: Top-level keys a CVE record pushed through `cve-record-update` may not carry: #: Vulnogram reads `go` as a request envelope rather than record content. _CVE_RECORD_FORBIDDEN_KEYS = frozenset({"go"}) @@ -3018,12 +3026,14 @@ def _stats_closed_page(cfg: dict[str, str], cursor: str) -> list[str]: _register( Op( name="cve-allocate", - params=("title",), + params=("pmc", "title"), backend="cve-tool", writes=True, summary="Allocate a CVE id for a title through the Vulnogram CVE tool.", title_files=("title",), - build=lambda cfg, title: cve_tool.argv("vulnogram-api-allocate", "--title", title), + # `--pmc` lets the client refuse a token held for any other PMC, so the + # id is reserved for the PMC the user approved, whatever session is current. + build=lambda cfg, pmc, title: cve_tool.argv("vulnogram-api-allocate", "--pmc", pmc, "--title", title), ) ) diff --git a/tools/security-management/src/security_management/cve_allocate/recipe.py b/tools/security-management/src/security_management/cve_allocate/recipe.py index c7e63b56a..ffd8fe1f2 100644 --- a/tools/security-management/src/security_management/cve_allocate/recipe.py +++ b/tools/security-management/src/security_management/cve_allocate/recipe.py @@ -18,7 +18,10 @@ - `api` (a governance-authorised user, pre-flight `api`): one proposal to allocate through the CVE tool's API, with the form URL as the fallback and - the one `cve-tool allocate` command to run on a yes. + the one `cve-tool allocate` command to run on a yes. With `vetted_workspace` + (the secure setup's vetted operations), the command is the vetted + `cve-allocate` instead, and `title_file` is the workspace file to write the + title to first. - `recipe` (authorised, any other pre-flight outcome): open the allocation URL, paste the title, submit, paste the ID back. - `relay` (not authorised): a terse request for an authorised member — the @@ -32,6 +35,8 @@ from __future__ import annotations +import hashlib +import json import re from collections.abc import Mapping, Sequence from typing import Any @@ -78,6 +83,7 @@ def render(data: Mapping[str, Any]) -> dict[str, Any]: ) block = fence(title) command = None + title_file = None if member and (data.get("preflight") or {}).get("path") == "api": pmc = one_line(data.get("pmc")) if not pmc: @@ -89,7 +95,20 @@ def render(data: Mapping[str, Any]) -> dict[str, Any]: "This reserves a CVE ID immediately and cannot be undone.\n" f"(Fallback, if the API path fails: the form at {url}.)\n" ) - command = cve_tool.command(data, "allocate", "--pmc", pmc, "--title", title) + workspace = one_line(data.get("vetted_workspace")) + if workspace: + # The vetted operation reads the title from a workspace file; `pmc` lets the + # client refuse a token held for any other PMC. The file is named by a hash of + # tracker, issue and the approved title, so no other run (another tracker, or + # this issue with another title) writes the same file between the write and + # the allocation; a sanitised slug would not do (`a-b/c` and `a/b-c` collide). + key = hashlib.sha256(json.dumps([tracker, str(data["issue"]), title]).encode()).hexdigest()[:24] + title_file = {"path": f"{workspace.rstrip('/')}/cve-title-{key}.txt", "text": title + "\n"} + command = cve_tool.vetted_command( + "security-cve-allocate", "cve-allocate", pmc, title_file["path"] + ) + else: + command = cve_tool.command(data, "allocate", "--pmc", pmc, "--title", title) elif member: path = "recipe" indented = "\n".join(" " + line for line in block.split("\n")) @@ -115,6 +134,7 @@ def render(data: Mapping[str, Any]) -> dict[str, Any]: "path": path, "text": text, "command": command, + "title_file": title_file, "governance_member_path": member, "relay_message_present": path == "relay", "api_allocation_proposed": path == "api", diff --git a/tools/security-management/src/security_management/shared/cve_tool.py b/tools/security-management/src/security_management/shared/cve_tool.py index a39fb650f..287106e16 100644 --- a/tools/security-management/src/security_management/shared/cve_tool.py +++ b/tools/security-management/src/security_management/shared/cve_tool.py @@ -39,6 +39,8 @@ STABLE = "~/.claude/magpie/security-tools/cve-tool" #: Where a framework checkout or snapshot install keeps the tool, run sandboxed. PROJECT = "<framework>/tools/cve-tool" +#: The vetted-ops link, spelled as the sandbox exclusion matches it. +VETTED_OPS = "~/.claude/magpie/vetted-ops" def stable_installed() -> bool: @@ -64,3 +66,16 @@ def command(data: Mapping[str, Any], operation: str, *args: object) -> str: return " ".join(("uv run --project", w(str(project)), "cve-tool", operation, *quoted)) # The literal `~`, unquoted: the sandbox exclusion and the `ask` rule match this spelling. return " ".join(("uvx --from", STABLE, "cve-tool", operation, *quoted)) + + +def vetted_command(caller: str, operation: str, *args: object) -> str: + """`uvx --from <VETTED_OPS> vetted-op --caller <caller> <operation> <args>`. + + The vetted operations (tools/vetted-ops, "The Vulnogram CVE-tool operations") + run the shipped Vulnogram client outside the sandbox with a fixed argv: no + override flag, no other session file. Arguments are shell-quoted here, like + `command`; a value the operation reads from a file (an allocation title, a + record) is passed as that file's workspace path, never inline. + """ + quoted = (shlex.quote(str(arg)) for arg in args) + return " ".join(("uvx --from", VETTED_OPS, "vetted-op --caller", shlex.quote(caller), operation, *quoted)) diff --git a/tools/security-management/tests/test_cve_allocate.py b/tools/security-management/tests/test_cve_allocate.py index 6b862f639..9d236e238 100644 --- a/tools/security-management/tests/test_cve_allocate.py +++ b/tools/security-management/tests/test_cve_allocate.py @@ -422,6 +422,51 @@ def test_the_api_command_quotes_the_title(capsys, tmp_path): assert "```text\nx'; rm -rf / #`\n```" in out["text"] +def test_the_vetted_route_reads_the_title_from_a_workspace_file(capsys, tmp_path): + hostile = "x'; rm -rf / #`" + out = recipe(capsys, tmp_path, True, {"path": "api"}, title=hostile, vetted_workspace="/ws/") + title_path = out["title_file"]["path"] + assert title_path.startswith("/ws/cve-title-") and title_path.endswith(".txt") + assert out["command"] == ( + "uvx --from ~/.claude/magpie/vetted-ops vetted-op --caller security-cve-allocate " + f"cve-allocate airflow {title_path}" + ) + assert hostile not in out["command"] + assert out["title_file"] == {"path": title_path, "text": hostile + "\n"} + assert out["api_allocation_proposed"] is True + + +def test_title_files_of_different_trackers_never_collide(capsys, tmp_path): + paths = { + recipe(capsys, tmp_path, True, {"path": "api"}, tracker=tracker, vetted_workspace="/ws")[ + "title_file" + ]["path"] + for tracker in ("a-b/c", "a/b-c", "a/b") + } + assert len(paths) == 3 + + +def test_another_title_for_the_same_issue_gets_its_own_file(capsys, tmp_path): + paths = { + recipe(capsys, tmp_path, True, {"path": "api"}, title=title, vetted_workspace="/ws")["title_file"][ + "path" + ] + for title in ("First title", "Second title") + } + assert len(paths) == 2 + + +def test_without_a_vetted_workspace_the_api_path_keeps_cve_tool(capsys, tmp_path): + out = recipe(capsys, tmp_path, True, {"path": "api"}) + assert "cve-tool allocate --pmc airflow" in out["command"] + assert out["title_file"] is None + + +def test_the_vetted_workspace_is_ignored_off_the_api_path(capsys, tmp_path): + out = recipe(capsys, tmp_path, True, {"path": "recipe"}, vetted_workspace="/ws") + assert out["command"] is None and out["title_file"] is None + + def test_a_backtick_run_in_the_title_lengthens_the_fence(capsys, tmp_path): out = recipe(capsys, tmp_path, True, title="a ``` b") assert "````text\na ``` b\n````" in out["text"].replace(" ", "") diff --git a/tools/skill-evals/evals/security-cve-allocate/README.md b/tools/skill-evals/evals/security-cve-allocate/README.md index ee00c3b92..3e8134528 100644 --- a/tools/skill-evals/evals/security-cve-allocate/README.md +++ b/tools/skill-evals/evals/security-cve-allocate/README.md @@ -59,4 +59,6 @@ Columns and rules: [Recording runs](../../README.md#recording-runs). | Date | Ref | Model | Runs | Result | Notes | |---|---|---|---|---|---| +| 2026-11-18 | `add4ee52` (feat/security-skills-vetted-cve-ops) | `opus[1m]` | 2 | pass | | +| 2026-11-18 | `9e979ff3` (feat/security-skills-vetted-cve-ops) | `opus[1m]` | 1 | pass | | | 2026-11-15 | `1087814b` (perf/trim-allocate-and-rc-cut) | `opus[1m]` | 2 | pass | | diff --git a/tools/skill-evals/evals/security-cve-allocate/runs.jsonl b/tools/skill-evals/evals/security-cve-allocate/runs.jsonl index b5b433805..46009dd08 100644 --- a/tools/skill-evals/evals/security-cve-allocate/runs.jsonl +++ b/tools/skill-evals/evals/security-cve-allocate/runs.jsonl @@ -1,2 +1,5 @@ {"branch": "perf/trim-allocate-and-rc-cut", "cases": 4, "cli": "claude -p", "cost": {"cli_calls": 4, "grader_calls": 1, "wall_seconds": 58.7}, "date": "2026-11-15", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-4-draft-body/case-1-direct-reporter-credit-not-asked", "step-4-draft-body/case-2-credit-already-asked", "step-4-draft-body/case-3-reporter-asks-for-the-record-link", "step-4-draft-body/case-4-second-reporter-of-a-merged-tracker"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 4, "path": "tools/skill-evals/evals/security-cve-allocate", "ref": "1087814b", "schema": 1, "skill": "security-cve-allocate", "tags": [], "timestamp": "2026-11-15T07:01:47Z"} {"branch": "perf/trim-allocate-and-rc-cut", "cases": 4, "cli": "claude -p", "cost": {"cli_calls": 4, "grader_calls": 1, "wall_seconds": 62.4}, "date": "2026-11-15", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-4-draft-body/case-1-direct-reporter-credit-not-asked", "step-4-draft-body/case-2-credit-already-asked", "step-4-draft-body/case-3-reporter-asks-for-the-record-link", "step-4-draft-body/case-4-second-reporter-of-a-merged-tracker"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 4, "path": "tools/skill-evals/evals/security-cve-allocate", "ref": "1087814b", "schema": 1, "skill": "security-cve-allocate", "tags": [], "timestamp": "2026-11-15T07:04:17Z"} +{"branch": "feat/security-skills-vetted-cve-ops", "cases": 4, "cli": "claude -p", "cost": {"cli_calls": 4, "grader_calls": 1, "wall_seconds": 42.9}, "date": "2026-11-18", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-4-draft-body/case-1-direct-reporter-credit-not-asked", "step-4-draft-body/case-2-credit-already-asked", "step-4-draft-body/case-3-reporter-asks-for-the-record-link", "step-4-draft-body/case-4-second-reporter-of-a-merged-tracker"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 4, "path": "tools/skill-evals/evals/security-cve-allocate", "ref": "9e979ff3", "schema": 1, "skill": "security-cve-allocate", "tags": [], "timestamp": "2026-11-18T01:06:05Z"} +{"branch": "feat/security-skills-vetted-cve-ops", "cases": 4, "cli": "claude -p", "cost": {"cli_calls": 4, "grader_calls": 1, "wall_seconds": 44.3}, "date": "2026-11-18", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-4-draft-body/case-1-direct-reporter-credit-not-asked", "step-4-draft-body/case-2-credit-already-asked", "step-4-draft-body/case-3-reporter-asks-for-the-record-link", "step-4-draft-body/case-4-second-reporter-of-a-merged-tracker"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 4, "path": "tools/skill-evals/evals/security-cve-allocate", "ref": "add4ee52", "schema": 1, "skill": "security-cve-allocate", "tags": [], "timestamp": "2026-11-18T01:23:59Z"} +{"branch": "feat/security-skills-vetted-cve-ops", "cases": 4, "cli": "claude -p", "cost": {"cli_calls": 4, "grader_calls": 1, "wall_seconds": 43.4}, "date": "2026-11-18", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-4-draft-body/case-1-direct-reporter-credit-not-asked", "step-4-draft-body/case-2-credit-already-asked", "step-4-draft-body/case-3-reporter-asks-for-the-record-link", "step-4-draft-body/case-4-second-reporter-of-a-merged-tracker"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 4, "path": "tools/skill-evals/evals/security-cve-allocate", "ref": "add4ee52", "schema": 1, "skill": "security-cve-allocate", "tags": [], "timestamp": "2026-11-18T02:37:11Z"} diff --git a/tools/skill-evals/evals/security-issue-sync/README.md b/tools/skill-evals/evals/security-issue-sync/README.md index f6dbc11a6..785f1fd00 100644 --- a/tools/skill-evals/evals/security-issue-sync/README.md +++ b/tools/skill-evals/evals/security-issue-sync/README.md @@ -59,4 +59,6 @@ Columns and rules: [Recording runs](../../README.md#recording-runs). | Date | Ref | Model | Runs | Result | Notes | |---|---|---|---|---|---| +| 2026-11-18 | `add4ee52` (feat/security-skills-vetted-cve-ops) | `opus[1m]` | 2 | pass | | +| 2026-11-18 | `9e979ff3` (feat/security-skills-vetted-cve-ops) | `opus[1m]` | 1 | pass | | | 2026-11-13 | `c22d5769` (feat/cve-tool-interface) | `claude-sonnet-5-5` | 10 | `step-2b-proposed-changes/case-2-milestone-missing-create` failed 1 of 2; `step-2b-proposed-changes/case-5-pre-announce-distributors` failed 2 of 2; `step-2b-proposed-changes/case-6-reporter-unresponsive` errored 1 of 2 | cross-model check for #1800; cross-model check for #1800 (re-run of infra failures) | diff --git a/tools/skill-evals/evals/security-issue-sync/runs.jsonl b/tools/skill-evals/evals/security-issue-sync/runs.jsonl index 1e18777a8..f4636ff2d 100644 --- a/tools/skill-evals/evals/security-issue-sync/runs.jsonl +++ b/tools/skill-evals/evals/security-issue-sync/runs.jsonl @@ -8,3 +8,6 @@ {"branch": "feat/cve-tool-interface", "cases": 1, "cli": "claude -p --model claude-sonnet-5-5", "cost": {"cli_calls": 1, "grader_calls": 1, "wall_seconds": 6.9}, "date": "2026-11-13", "dirty": false, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-2b-proposed-changes/case-6-reporter-unresponsive"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "claude-sonnet-5-5", "model_source": "cli-flag", "notes": "cross-model check for #1800 (re-run of infra failures)", "passed": 1, "path": "tools/skill-evals/evals/security-issue-sync/step-2b-proposed-changes/fixtures/case-6-reporter-unresponsive", "ref": "c22d5769", "schema": 1, "skill": "security-issue-sync", "tags": [], "timestamp": "2026-11-13T04:48:27Z"} {"branch": "feat/cve-tool-interface", "cases": 4, "cli": "claude -p --model claude-sonnet-5-5", "cost": {"cli_calls": 4, "grader_calls": 0, "wall_seconds": 10.9}, "date": "2026-11-13", "dirty": false, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-2c-next-step/case-1-step-3-start-discussion", "step-2c-next-step/case-2-step-6-allocate-cve", "step-2c-next-step/case-3-step-11-pr-merged-parked", "step-2c-next-step/case-4-step-13-fix-released-rm-action"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "claude-sonnet-5-5", "model_source": "cli-flag", "notes": "cross-model check for #1800", "passed": 4, "path": "tools/skill-evals/evals/security-issue-sync/step-2c-next-step/fixtures", "ref": "c22d5769", "schema": 1, "skill": "security-issue-sync", "tags": [], "timestamp": "2026-11-13T04:49:41Z"} {"branch": "feat/cve-tool-interface", "cases": 4, "cli": "claude -p --model claude-sonnet-5-5", "cost": {"cli_calls": 4, "grader_calls": 0, "wall_seconds": 12.4}, "date": "2026-11-13", "dirty": false, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-handoff-delegate/case-1-triage-delegated", "step-handoff-delegate/case-2-cve-allocate-delegated-after", "step-handoff-delegate/case-3-unrelated-delegate", "step-handoff-delegate/case-4-no-overrides-flag"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "claude-sonnet-5-5", "model_source": "cli-flag", "notes": "cross-model check for #1800", "passed": 4, "path": "tools/skill-evals/evals/security-issue-sync/step-handoff-delegate/fixtures", "ref": "c22d5769", "schema": 1, "skill": "security-issue-sync", "tags": [], "timestamp": "2026-11-13T04:49:54Z"} +{"branch": "feat/security-skills-vetted-cve-ops", "cases": 2, "cli": "claude -p", "cost": {"cli_calls": 2, "grader_calls": 2, "wall_seconds": 36.6}, "date": "2026-11-18", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-5b-judged-gates/case-4-private-scanner-credit", "step-5b-judged-gates/case-5-clean-push-fix-released"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 2, "path": "tools/skill-evals/evals/security-issue-sync/step-5b-judged-gates", "ref": "9e979ff3", "schema": 1, "skill": "security-issue-sync", "tags": [], "timestamp": "2026-11-18T01:05:21Z"} +{"branch": "feat/security-skills-vetted-cve-ops", "cases": 2, "cli": "claude -p", "cost": {"cli_calls": 2, "grader_calls": 2, "wall_seconds": 36.6}, "date": "2026-11-18", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-5b-judged-gates/case-4-private-scanner-credit", "step-5b-judged-gates/case-5-clean-push-fix-released"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 2, "path": "tools/skill-evals/evals/security-issue-sync/step-5b-judged-gates", "ref": "add4ee52", "schema": 1, "skill": "security-issue-sync", "tags": [], "timestamp": "2026-11-18T01:24:36Z"} +{"branch": "feat/security-skills-vetted-cve-ops", "cases": 2, "cli": "claude -p", "cost": {"cli_calls": 2, "grader_calls": 2, "wall_seconds": 36.0}, "date": "2026-11-18", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-5b-judged-gates/case-4-private-scanner-credit", "step-5b-judged-gates/case-5-clean-push-fix-released"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 2, "path": "tools/skill-evals/evals/security-issue-sync/step-5b-judged-gates", "ref": "add4ee52", "schema": 1, "skill": "security-issue-sync", "tags": [], "timestamp": "2026-11-18T02:36:27Z"} diff --git a/tools/spec-loop/specs/cve-tooling.md b/tools/spec-loop/specs/cve-tooling.md index 83b0274c0..3296332cb 100644 --- a/tools/spec-loop/specs/cve-tooling.md +++ b/tools/spec-loop/specs/cve-tooling.md @@ -199,6 +199,22 @@ reviewable. push. The title is attacker-influenced and is passed as one shell-quoted argument. +- **Under the secure setup the skills prefer the vetted Vulnogram operations.** + When the vetted-ops policy grants them, `security-cve-allocate` allocates + with `vetted-op cve-allocate <pmc> <title-file>` (the title written to a + workspace file named by a hash of tracker, issue and title, never quoted into a + command; `--pmc` makes the client refuse a token for any other PMC) and + `security-issue-sync`'s Step 5b pushes with `vetted-op cve-record-update`; + `cve-allocate recipe` returns that command and the `title_file` when given + `vetted_workspace`. `cve-tool push` stays for a push that would promote a + record to `PUBLIC` (the vetted push refuses it) and for the two confirmed + overrides; `fetch`, `check`, `setup` and `publish` stay on `cve-tool`. A + failed vetted call exits 4 and names the client's code on stderr, which the + existing exit tables read. Only a refusal naming a missing grant (`is not + permitted to run`) falls back to `cve-tool`; any other `vetted-op` refusal + (a rejected record or argument) stops, so the vetted checks are never + routed around; any other outcome of an allocation is unclear and sends the + user to the PMC's records first, so an allocation is never sent twice. - **Any CVE tool other than Vulnogram takes the recipe path (#2077).** The API pre-flight applies only when `cve_authority.tool` is `vulnogram`. `security-management` `cve_allocate/gate.py` `preflight()` maps a `cve-tool` exit `20` (no tool declared), `21` (no adapter) or `22` (operation not implemented), on either the server or the allocate check, to the `recipe` path with a reason naming the missing piece, instead of an "unexpected exit". diff --git a/tools/vetted-ops/README.md b/tools/vetted-ops/README.md index c774148fd..c53e3ba23 100644 --- a/tools/vetted-ops/README.md +++ b/tools/vetted-ops/README.md @@ -516,7 +516,7 @@ overrides. Three operations run it instead, outside the sandbox through |---|---|---| | `cve-record-fetch` (read) | `cve_id` | `vulnogram-api-record-fetch --cve-id <id>` | | `cve-record-update` (write) | `cve_id body` | `vulnogram-api-record-update --cve-id <id> --json-file -`, the record on stdin | -| `cve-allocate` (write) | `title` (a workspace file) | `vulnogram-api-allocate --title <title>` | +| `cve-allocate` (write) | `pmc title` (title a workspace file) | `vulnogram-api-allocate --pmc <pmc> --title <title>`; the client refuses a token held for another PMC | - The client is the copy shipped in this plugin beside vetted-ops (`tools/cve-tool-vulnogram/oauth-api`, resolved like the tracker backends), diff --git a/tools/vetted-ops/src/vetted_ops/cli.py b/tools/vetted-ops/src/vetted_ops/cli.py index 1554908ab..ffe94df16 100644 --- a/tools/vetted-ops/src/vetted_ops/cli.py +++ b/tools/vetted-ops/src/vetted_ops/cli.py @@ -145,6 +145,8 @@ def _validate_params( resolved[name] = ops_mod.commit_hash(raw) elif name == "cve_id": resolved[name] = ops_mod.cve_id(raw) + elif name == "pmc": + resolved[name] = ops_mod.pmc(raw) elif name == "date": resolved[name] = ops_mod.date(raw) elif name == "cursor": diff --git a/tools/vetted-ops/src/vetted_ops/cve_tool.py b/tools/vetted-ops/src/vetted_ops/cve_tool.py index f4d6c2fff..318975564 100644 --- a/tools/vetted-ops/src/vetted_ops/cve_tool.py +++ b/tools/vetted-ops/src/vetted_ops/cve_tool.py @@ -27,7 +27,7 @@ * the client is the copy shipped beside vetted-ops (``tool_dir``), never a path the agent can write; * only the CVE id, the record JSON (piped on stdin, the bytes vetted-ops - validated) and the allocation title are the caller's; every override flag + validated), the allocation PMC and the allocation title are the caller's; every override flag stays off, so the record commands' merge-mode guards always apply; * the child gets an allowlisted environment (``child_env``): ``HOME`` from the password database, a fixed ``PATH``, locale and ``TMPDIR`` only. No diff --git a/tools/vetted-ops/src/vetted_ops/ops.py b/tools/vetted-ops/src/vetted_ops/ops.py index 195758b97..b036e306a 100644 --- a/tools/vetted-ops/src/vetted_ops/ops.py +++ b/tools/vetted-ops/src/vetted_ops/ops.py @@ -224,6 +224,14 @@ def cve_id(value: str) -> str: return _check(_CVE_ID, value, "CVE id") +#: A Vulnogram PMC / committee id (`airflow`, `incubator-foo`): lowercase, no path or flag. +_PMC = re.compile(r"^[a-z][a-z0-9-]{0,63}$") + + +def pmc(value: str) -> str: + return _check(_PMC, value, "PMC") + + #: Top-level keys a CVE record pushed through `cve-record-update` may not carry: #: Vulnogram reads `go` as a request envelope rather than record content. _CVE_RECORD_FORBIDDEN_KEYS = frozenset({"go"}) @@ -3018,12 +3026,14 @@ def _stats_closed_page(cfg: dict[str, str], cursor: str) -> list[str]: _register( Op( name="cve-allocate", - params=("title",), + params=("pmc", "title"), backend="cve-tool", writes=True, summary="Allocate a CVE id for a title through the Vulnogram CVE tool.", title_files=("title",), - build=lambda cfg, title: cve_tool.argv("vulnogram-api-allocate", "--title", title), + # `--pmc` lets the client refuse a token held for any other PMC, so the + # id is reserved for the PMC the user approved, whatever session is current. + build=lambda cfg, pmc, title: cve_tool.argv("vulnogram-api-allocate", "--pmc", pmc, "--title", title), ) ) diff --git a/tools/vetted-ops/tests/test_cve_tool.py b/tools/vetted-ops/tests/test_cve_tool.py index 938a811c6..70af44daa 100644 --- a/tools/vetted-ops/tests/test_cve_tool.py +++ b/tools/vetted-ops/tests/test_cve_tool.py @@ -145,10 +145,12 @@ def test_allocate_reads_its_title_from_a_workspace_file(policy: config.Config, w title.write_text("Config API: section aliases bypass masking\n") op = ops.resolve("cve-allocate") assert op.writes - params, _ = cli._validate_params(op, [str(title)], policy) + params, _ = cli._validate_params(op, ["airflow", str(title)], policy) assert cli.build_argv(op, params, policy) == [ *_prefix(), "vulnogram-api-allocate", + "--pmc", + "airflow", "--title", "Config API: section aliases bypass masking", ] @@ -162,10 +164,18 @@ def test_a_malformed_cve_id_never_reaches_the_client(policy: config.Config, bad: cli._validate_params(ops.resolve("cve-record-fetch"), [bad], policy) +@pytest.mark.parametrize("bad", ["Airflow", "airflow --credentials", "--credentials", "../x", ""]) +def test_a_malformed_pmc_never_reaches_the_client(policy: config.Config, workspace: Path, bad: str) -> None: + title = workspace / "title.txt" + title.write_text("A title\n") + with pytest.raises(ops.ParamError): + cli._validate_params(ops.resolve("cve-allocate"), [bad, str(title)], policy) + + def test_no_override_flag_is_reachable() -> None: """The caller's values are the only variable parts; no `--allow-*` / `--credentials`.""" for name in ("cve-record-fetch", "cve-record-update", "cve-allocate"): - assert set(ops.resolve(name).params) <= {"cve_id", "body", "title"} + assert set(ops.resolve(name).params) <= {"cve_id", "body", "title", "pmc"} def test_a_tampered_argv_is_refused(policy: config.Config) -> None: @@ -253,6 +263,7 @@ def test_a_dry_run_shows_the_title_only_as_its_size( "security-cve-allocate", "--dry-run", "cve-allocate", + "airflow", str(title), "--config", str(policy_path), diff --git a/tools/vetted-ops/tests/test_vetted_ops.py b/tools/vetted-ops/tests/test_vetted_ops.py index 267cb0559..d404cef83 100644 --- a/tools/vetted-ops/tests/test_vetted_ops.py +++ b/tools/vetted-ops/tests/test_vetted_ops.py @@ -112,6 +112,7 @@ def test_every_op_declares_validators_for_all_its_params() -> None: "version", "commit_hash", "cve_id", + "pmc", "action", "field", "date", @@ -157,6 +158,7 @@ def test_every_builder_produces_a_gh_argv(policy: config.Config) -> None: "version": "1.0.0", "commit_hash": "a1b2c3d", "cve_id": "CVE-2023-1234", + "pmc": "airflow", "ecosystem": "PyPI", "action": "Sync", "field": "CVE tool link",