From 64b52516b13a92b8be244f5747d37cab7aeea832 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 18 Nov 2026 12:10:09 +1100 Subject: [PATCH 1/2] feat(security): use the vetted Vulnogram operations under the secure setup outside the sandbox with a fixed argv and no override flag. The skills still allocated and pushed records through the generic tool interface, which passes every adapter flag through. Under the secure setup, with the operations granted, the allocation skill now allocates through the vetted operation, with the title written to a workspace file instead of quoted into a command, and the sync skill's record push uses the vetted push. The recipe renderer returns that command and the title file when given the workspace. The generic interface stays for what the vetted push refuses by design (a push that promotes a record to its final public state, and the two confirmed overrides) and for fetch, check, setup and publish. A failed vetted call names the client's exit code on stderr, so the existing exit tables still apply; any other refusal means nothing ran, and the skill falls back to the generic interface. Generated-by: Claude Opus 5 --- .../cve_allocate/recipe.py | 19 +++++++++++++-- .../security_management/shared/cve_tool.py | 15 ++++++++++++ .../skills/cve-allocate/SKILL.md | 7 +++--- .../skills/cve-allocate/api-allocation.md | 16 +++++++++++++ .../skills/issue-sync/apply-and-push.md | 17 +++++++++++++- .../cve_allocate/recipe.py | 19 +++++++++++++-- .../security_management/shared/cve_tool.py | 15 ++++++++++++ .../tests/test_cve_allocate.py | 23 +++++++++++++++++++ .../evals/security-cve-allocate/README.md | 1 + .../evals/security-cve-allocate/runs.jsonl | 1 + .../evals/security-issue-sync/README.md | 1 + .../evals/security-issue-sync/runs.jsonl | 1 + tools/spec-loop/specs/cve-tooling.md | 11 +++++++++ 13 files changed, 138 insertions(+), 8 deletions(-) diff --git a/plugins/magpie-security-tools/tools/security-management/src/security_management/cve_allocate/recipe.py b/plugins/magpie-security-tools/tools/security-management/src/security_management/cve_allocate/recipe.py index c7e63b56a..bfbfe3494 100644 --- a/plugins/magpie-security-tools/tools/security-management/src/security_management/cve_allocate/recipe.py +++ b/plugins/magpie-security-tools/tools/security-management/src/security_management/cve_allocate/recipe.py @@ -18,7 +18,10 @@ - `api` (a governance-authorised user, pre-flight `api`): one proposal to allocate through the CVE tool's API, with the form URL as the fallback and - the one `cve-tool allocate` command to run on a yes. + the one `cve-tool allocate` command to run on a yes. With `vetted_workspace` + (the secure setup's vetted operations), the command is the vetted + `cve-allocate` instead, and `title_file` is the workspace file to write the + title to first. - `recipe` (authorised, any other pre-flight outcome): open the allocation URL, paste the title, submit, paste the ID back. - `relay` (not authorised): a terse request for an authorised member — the @@ -78,6 +81,7 @@ def render(data: Mapping[str, Any]) -> dict[str, Any]: ) block = fence(title) command = None + title_file = None if member and (data.get("preflight") or {}).get("path") == "api": pmc = one_line(data.get("pmc")) if not pmc: @@ -89,7 +93,17 @@ def render(data: Mapping[str, Any]) -> dict[str, Any]: "This reserves a CVE ID immediately and cannot be undone.\n" f"(Fallback, if the API path fails: the form at {url}.)\n" ) - command = cve_tool.command(data, "allocate", "--pmc", pmc, "--title", title) + workspace = one_line(data.get("vetted_workspace")) + if workspace: + # The vetted operation reads the title from a workspace file and allocates + # for the PMC its token holds, which the pre-flight checked is `pmc`. + title_file = { + "path": f"{workspace.rstrip('/')}/cve-title-{data['issue']}.txt", + "text": title + "\n", + } + command = cve_tool.vetted_command("security-cve-allocate", "cve-allocate", title_file["path"]) + else: + command = cve_tool.command(data, "allocate", "--pmc", pmc, "--title", title) elif member: path = "recipe" indented = "\n".join(" " + line for line in block.split("\n")) @@ -115,6 +129,7 @@ def render(data: Mapping[str, Any]) -> dict[str, Any]: "path": path, "text": text, "command": command, + "title_file": title_file, "governance_member_path": member, "relay_message_present": path == "relay", "api_allocation_proposed": path == "api", diff --git a/plugins/magpie-security-tools/tools/security-management/src/security_management/shared/cve_tool.py b/plugins/magpie-security-tools/tools/security-management/src/security_management/shared/cve_tool.py index a39fb650f..287106e16 100644 --- a/plugins/magpie-security-tools/tools/security-management/src/security_management/shared/cve_tool.py +++ b/plugins/magpie-security-tools/tools/security-management/src/security_management/shared/cve_tool.py @@ -39,6 +39,8 @@ STABLE = "~/.claude/magpie/security-tools/cve-tool" #: Where a framework checkout or snapshot install keeps the tool, run sandboxed. PROJECT = "/tools/cve-tool" +#: The vetted-ops link, spelled as the sandbox exclusion matches it. +VETTED_OPS = "~/.claude/magpie/vetted-ops" def stable_installed() -> bool: @@ -64,3 +66,16 @@ def command(data: Mapping[str, Any], operation: str, *args: object) -> str: return " ".join(("uv run --project", w(str(project)), "cve-tool", operation, *quoted)) # The literal `~`, unquoted: the sandbox exclusion and the `ask` rule match this spelling. return " ".join(("uvx --from", STABLE, "cve-tool", operation, *quoted)) + + +def vetted_command(caller: str, operation: str, *args: object) -> str: + """`uvx --from vetted-op --caller `. + + The vetted operations (tools/vetted-ops, "The Vulnogram CVE-tool operations") + run the shipped Vulnogram client outside the sandbox with a fixed argv: no + override flag, no other session file. Arguments are shell-quoted here, like + `command`; a value the operation reads from a file (an allocation title, a + record) is passed as that file's workspace path, never inline. + """ + quoted = (shlex.quote(str(arg)) for arg in args) + return " ".join(("uvx --from", VETTED_OPS, "vetted-op --caller", shlex.quote(caller), operation, *quoted)) diff --git a/plugins/magpie-security/skills/cve-allocate/SKILL.md b/plugins/magpie-security/skills/cve-allocate/SKILL.md index 694c3ecc6..b0f237754 100644 --- a/plugins/magpie-security/skills/cve-allocate/SKILL.md +++ b/plugins/magpie-security/skills/cve-allocate/SKILL.md @@ -311,11 +311,12 @@ Full procedure: [`title-normalize.md`](title-normalize.md). **Ask first** whether the user is authorised under `governance.cve_allocation_gate` (Step 0 item 3 may already have the answer). Then write `{tracker, issue, title, governance_member, allocate_url, preflight, pmc, authorised_handles}` and run ` recipe`; -present its `text` as it is. Its `path` is one of: +under the secure setup, with `security-cve-allocate` granted `cve-allocate`, also pass `vetted_workspace` (the vetted-ops policy's `workspace`). +Present its `text` as it is. Its `path` is one of: - **`api`** — a governance-authorised user whose Step 0 pre-flight chose `api`: a proposal to allocate through the CVE tool's API, with `cve_authority.allocate_url` as the fallback. - On the user's yes, run its `command` (one `cve-tool allocate` call, the title shell-quoted) **once**, - write `{exit, stdout, stderr, pmc}` and run ` allocate-result`: its `next` is `step-4` with the `cve_id`, `stop` (the ID comes back by mail), `setup`, `check-records` or `recipe`, + On the user's yes, write a non-null `title_file`'s `text` to its `path` with the Write tool, then run its `command` (one `cve-tool allocate` call, the title shell-quoted, or the vetted `cve-allocate`) **once**, + write `{exit, stdout, stderr, pmc}` and run ` allocate-result` (after a vetted call that failed, `exit` is the `CVE tool exit N` its stderr names): its `next` is `step-4` with the `cve_id`, `stop` (the ID comes back by mail), `setup`, `check-records` or `recipe`, per [`api-allocation.md` § *Allocate*](api-allocation.md#allocate-step-3-api-path). `rerun_allocate` is false except after an expired token, and even then only on a fresh yes. - **`recipe`** — a governance-authorised user otherwise: open the allocation URL (authenticate per [`/README.md`](../../../../tools/cve-tool/README.md)), paste the title, submit, paste the ID back. - **`relay`** — a user who is not authorised: the CVE tool will not let them submit, so they never get "click *Allocate*". diff --git a/plugins/magpie-security/skills/cve-allocate/api-allocation.md b/plugins/magpie-security/skills/cve-allocate/api-allocation.md index 65a5a0987..fbbb123b9 100644 --- a/plugins/magpie-security/skills/cve-allocate/api-allocation.md +++ b/plugins/magpie-security/skills/cve-allocate/api-allocation.md @@ -95,6 +95,22 @@ On a yes, run the allocation **once**: Pass the title as a single shell-quoted argument; it is attacker-influenced text, so escape any quote it contains rather than interpolating it raw. +**Under the secure setup**, when the vetted-ops policy grants `security-cve-allocate` the `cve-allocate` operation, +allocate through it instead (` recipe` returns this command when given `vetted_workspace`): +write the title to `/cve-title-.txt` with the Write tool, then run + +```bash +uvx --from ~/.claude/magpie/vetted-ops vetted-op --caller security-cve-allocate cve-allocate /cve-title-.txt +``` + +It runs the shipped Vulnogram client with a fixed argv, outside the sandbox, with no override flag reachable +([vetted-ops § *The Vulnogram CVE-tool operations*](../../../../tools/vetted-ops/README.md#the-vulnogram-cve-tool-operations)), +and allocates for the PMC its token holds, which the pre-flight confirmed is ``. +The title travels as a file, so it is never quoted into a command line. +On success it exits 0 and the ID is the first stdout line, as below. +When the client fails, `vetted-op` exits 4 and names the client's own code on stderr (`CVE tool exit N`): read the table below with that `N`. +Any other non-zero exit (the caller is not granted the operation, or a usage error) means nothing was allocated: run the `cve-tool allocate` command above instead. + | Exit | Meaning | Next | |---|---|---| | 0 | The first stdout line is the `CVE-YYYY-NNNNN` | Step 4 with that ID; no paste-back needed | diff --git a/plugins/magpie-security/skills/issue-sync/apply-and-push.md b/plugins/magpie-security/skills/issue-sync/apply-and-push.md index 145024bb2..6926d18bc 100644 --- a/plugins/magpie-security/skills/issue-sync/apply-and-push.md +++ b/plugins/magpie-security/skills/issue-sync/apply-and-push.md @@ -479,7 +479,7 @@ When the state fetch fails, pass `fetched_state: "unknown"`. or extract it from the body via `awk` between the `` / `` markers — the generator is deterministic, so both yield the same bytes. - Conventional path: `/cve--.json`. + Conventional path: `/cve--.json`, or `/cve--.json` for the vetted push in step 3, which reads only the workspace. `` is the session scratch directory as an absolute path (fall back to `$TMPDIR`); `gh` may run outside the sandbox, where `$TMPDIR` differs, so pass it absolute paths. 3. **Push the update through the adapter's `push_update` method** @@ -491,6 +491,21 @@ When the state fetch fails, pass `fetched_state: "unknown"`. --cve-id --json-file /cve--.json ``` + **Under the secure setup**, when the vetted-ops policy grants `security-issue-sync` the `cve-record-update` operation, + push through it instead, with the JSON at `/cve--.json` (the vetted-ops policy's `workspace`): + + ```bash + uvx --from ~/.claude/magpie/vetted-ops vetted-op --caller security-issue-sync cve-record-update /cve--.json + ``` + + It runs the same Vulnogram push with a fixed argv and no override flag + ([vetted-ops § *The Vulnogram CVE-tool operations*](../../../../tools/vetted-ops/README.md#the-vulnogram-cve-tool-operations)), + so it covers every push above except three, which keep the `cve-tool push` command: + a JSON whose `CNA_private.state` is `PUBLIC` while the live record is not (the vetted push never promotes a record to `PUBLIC`), + and the two overrides below. + When the client fails, `vetted-op` exits 4 and names the client's own code on stderr (`CVE tool exit N`), which reads as the `cve-tool push` exit `N`; + any other non-zero exit (the caller is not granted the operation, or a usage error) means nothing was pushed: use `cve-tool push`. + For Vulnogram, the push merges into the live record and refuses to rename an affected product or package. Two explicit overrides exist; use them only after confirming with the user: diff --git a/tools/security-management/src/security_management/cve_allocate/recipe.py b/tools/security-management/src/security_management/cve_allocate/recipe.py index c7e63b56a..bfbfe3494 100644 --- a/tools/security-management/src/security_management/cve_allocate/recipe.py +++ b/tools/security-management/src/security_management/cve_allocate/recipe.py @@ -18,7 +18,10 @@ - `api` (a governance-authorised user, pre-flight `api`): one proposal to allocate through the CVE tool's API, with the form URL as the fallback and - the one `cve-tool allocate` command to run on a yes. + the one `cve-tool allocate` command to run on a yes. With `vetted_workspace` + (the secure setup's vetted operations), the command is the vetted + `cve-allocate` instead, and `title_file` is the workspace file to write the + title to first. - `recipe` (authorised, any other pre-flight outcome): open the allocation URL, paste the title, submit, paste the ID back. - `relay` (not authorised): a terse request for an authorised member — the @@ -78,6 +81,7 @@ def render(data: Mapping[str, Any]) -> dict[str, Any]: ) block = fence(title) command = None + title_file = None if member and (data.get("preflight") or {}).get("path") == "api": pmc = one_line(data.get("pmc")) if not pmc: @@ -89,7 +93,17 @@ def render(data: Mapping[str, Any]) -> dict[str, Any]: "This reserves a CVE ID immediately and cannot be undone.\n" f"(Fallback, if the API path fails: the form at {url}.)\n" ) - command = cve_tool.command(data, "allocate", "--pmc", pmc, "--title", title) + workspace = one_line(data.get("vetted_workspace")) + if workspace: + # The vetted operation reads the title from a workspace file and allocates + # for the PMC its token holds, which the pre-flight checked is `pmc`. + title_file = { + "path": f"{workspace.rstrip('/')}/cve-title-{data['issue']}.txt", + "text": title + "\n", + } + command = cve_tool.vetted_command("security-cve-allocate", "cve-allocate", title_file["path"]) + else: + command = cve_tool.command(data, "allocate", "--pmc", pmc, "--title", title) elif member: path = "recipe" indented = "\n".join(" " + line for line in block.split("\n")) @@ -115,6 +129,7 @@ def render(data: Mapping[str, Any]) -> dict[str, Any]: "path": path, "text": text, "command": command, + "title_file": title_file, "governance_member_path": member, "relay_message_present": path == "relay", "api_allocation_proposed": path == "api", diff --git a/tools/security-management/src/security_management/shared/cve_tool.py b/tools/security-management/src/security_management/shared/cve_tool.py index a39fb650f..287106e16 100644 --- a/tools/security-management/src/security_management/shared/cve_tool.py +++ b/tools/security-management/src/security_management/shared/cve_tool.py @@ -39,6 +39,8 @@ STABLE = "~/.claude/magpie/security-tools/cve-tool" #: Where a framework checkout or snapshot install keeps the tool, run sandboxed. PROJECT = "/tools/cve-tool" +#: The vetted-ops link, spelled as the sandbox exclusion matches it. +VETTED_OPS = "~/.claude/magpie/vetted-ops" def stable_installed() -> bool: @@ -64,3 +66,16 @@ def command(data: Mapping[str, Any], operation: str, *args: object) -> str: return " ".join(("uv run --project", w(str(project)), "cve-tool", operation, *quoted)) # The literal `~`, unquoted: the sandbox exclusion and the `ask` rule match this spelling. return " ".join(("uvx --from", STABLE, "cve-tool", operation, *quoted)) + + +def vetted_command(caller: str, operation: str, *args: object) -> str: + """`uvx --from vetted-op --caller `. + + The vetted operations (tools/vetted-ops, "The Vulnogram CVE-tool operations") + run the shipped Vulnogram client outside the sandbox with a fixed argv: no + override flag, no other session file. Arguments are shell-quoted here, like + `command`; a value the operation reads from a file (an allocation title, a + record) is passed as that file's workspace path, never inline. + """ + quoted = (shlex.quote(str(arg)) for arg in args) + return " ".join(("uvx --from", VETTED_OPS, "vetted-op --caller", shlex.quote(caller), operation, *quoted)) diff --git a/tools/security-management/tests/test_cve_allocate.py b/tools/security-management/tests/test_cve_allocate.py index 6b862f639..1a93e5d66 100644 --- a/tools/security-management/tests/test_cve_allocate.py +++ b/tools/security-management/tests/test_cve_allocate.py @@ -422,6 +422,29 @@ def test_the_api_command_quotes_the_title(capsys, tmp_path): assert "```text\nx'; rm -rf / #`\n```" in out["text"] +def test_the_vetted_route_reads_the_title_from_a_workspace_file(capsys, tmp_path): + hostile = "x'; rm -rf / #`" + out = recipe(capsys, tmp_path, True, {"path": "api"}, title=hostile, vetted_workspace="/ws/") + assert out["command"] == ( + "uvx --from ~/.claude/magpie/vetted-ops vetted-op --caller security-cve-allocate " + "cve-allocate /ws/cve-title-242.txt" + ) + assert hostile not in out["command"] + assert out["title_file"] == {"path": "/ws/cve-title-242.txt", "text": hostile + "\n"} + assert out["api_allocation_proposed"] is True + + +def test_without_a_vetted_workspace_the_api_path_keeps_cve_tool(capsys, tmp_path): + out = recipe(capsys, tmp_path, True, {"path": "api"}) + assert "cve-tool allocate --pmc airflow" in out["command"] + assert out["title_file"] is None + + +def test_the_vetted_workspace_is_ignored_off_the_api_path(capsys, tmp_path): + out = recipe(capsys, tmp_path, True, {"path": "recipe"}, vetted_workspace="/ws") + assert out["command"] is None and out["title_file"] is None + + def test_a_backtick_run_in_the_title_lengthens_the_fence(capsys, tmp_path): out = recipe(capsys, tmp_path, True, title="a ``` b") assert "````text\na ``` b\n````" in out["text"].replace(" ", "") diff --git a/tools/skill-evals/evals/security-cve-allocate/README.md b/tools/skill-evals/evals/security-cve-allocate/README.md index ee00c3b92..d1caf56f1 100644 --- a/tools/skill-evals/evals/security-cve-allocate/README.md +++ b/tools/skill-evals/evals/security-cve-allocate/README.md @@ -59,4 +59,5 @@ Columns and rules: [Recording runs](../../README.md#recording-runs). | Date | Ref | Model | Runs | Result | Notes | |---|---|---|---|---|---| +| 2026-11-18 | `9e979ff3` (feat/security-skills-vetted-cve-ops) | `opus[1m]` | 1 | pass | | | 2026-11-15 | `1087814b` (perf/trim-allocate-and-rc-cut) | `opus[1m]` | 2 | pass | | diff --git a/tools/skill-evals/evals/security-cve-allocate/runs.jsonl b/tools/skill-evals/evals/security-cve-allocate/runs.jsonl index b5b433805..6905ee5e4 100644 --- a/tools/skill-evals/evals/security-cve-allocate/runs.jsonl +++ b/tools/skill-evals/evals/security-cve-allocate/runs.jsonl @@ -1,2 +1,3 @@ {"branch": "perf/trim-allocate-and-rc-cut", "cases": 4, "cli": "claude -p", "cost": {"cli_calls": 4, "grader_calls": 1, "wall_seconds": 58.7}, "date": "2026-11-15", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-4-draft-body/case-1-direct-reporter-credit-not-asked", "step-4-draft-body/case-2-credit-already-asked", "step-4-draft-body/case-3-reporter-asks-for-the-record-link", "step-4-draft-body/case-4-second-reporter-of-a-merged-tracker"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 4, "path": "tools/skill-evals/evals/security-cve-allocate", "ref": "1087814b", "schema": 1, "skill": "security-cve-allocate", "tags": [], "timestamp": "2026-11-15T07:01:47Z"} {"branch": "perf/trim-allocate-and-rc-cut", "cases": 4, "cli": "claude -p", "cost": {"cli_calls": 4, "grader_calls": 1, "wall_seconds": 62.4}, "date": "2026-11-15", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-4-draft-body/case-1-direct-reporter-credit-not-asked", "step-4-draft-body/case-2-credit-already-asked", "step-4-draft-body/case-3-reporter-asks-for-the-record-link", "step-4-draft-body/case-4-second-reporter-of-a-merged-tracker"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 4, "path": "tools/skill-evals/evals/security-cve-allocate", "ref": "1087814b", "schema": 1, "skill": "security-cve-allocate", "tags": [], "timestamp": "2026-11-15T07:04:17Z"} +{"branch": "feat/security-skills-vetted-cve-ops", "cases": 4, "cli": "claude -p", "cost": {"cli_calls": 4, "grader_calls": 1, "wall_seconds": 42.9}, "date": "2026-11-18", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-4-draft-body/case-1-direct-reporter-credit-not-asked", "step-4-draft-body/case-2-credit-already-asked", "step-4-draft-body/case-3-reporter-asks-for-the-record-link", "step-4-draft-body/case-4-second-reporter-of-a-merged-tracker"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 4, "path": "tools/skill-evals/evals/security-cve-allocate", "ref": "9e979ff3", "schema": 1, "skill": "security-cve-allocate", "tags": [], "timestamp": "2026-11-18T01:06:05Z"} diff --git a/tools/skill-evals/evals/security-issue-sync/README.md b/tools/skill-evals/evals/security-issue-sync/README.md index f6dbc11a6..da4718c2d 100644 --- a/tools/skill-evals/evals/security-issue-sync/README.md +++ b/tools/skill-evals/evals/security-issue-sync/README.md @@ -59,4 +59,5 @@ Columns and rules: [Recording runs](../../README.md#recording-runs). | Date | Ref | Model | Runs | Result | Notes | |---|---|---|---|---|---| +| 2026-11-18 | `9e979ff3` (feat/security-skills-vetted-cve-ops) | `opus[1m]` | 1 | pass | | | 2026-11-13 | `c22d5769` (feat/cve-tool-interface) | `claude-sonnet-5-5` | 10 | `step-2b-proposed-changes/case-2-milestone-missing-create` failed 1 of 2; `step-2b-proposed-changes/case-5-pre-announce-distributors` failed 2 of 2; `step-2b-proposed-changes/case-6-reporter-unresponsive` errored 1 of 2 | cross-model check for #1800; cross-model check for #1800 (re-run of infra failures) | diff --git a/tools/skill-evals/evals/security-issue-sync/runs.jsonl b/tools/skill-evals/evals/security-issue-sync/runs.jsonl index 1e18777a8..80593c27a 100644 --- a/tools/skill-evals/evals/security-issue-sync/runs.jsonl +++ b/tools/skill-evals/evals/security-issue-sync/runs.jsonl @@ -8,3 +8,4 @@ {"branch": "feat/cve-tool-interface", "cases": 1, "cli": "claude -p --model claude-sonnet-5-5", "cost": {"cli_calls": 1, "grader_calls": 1, "wall_seconds": 6.9}, "date": "2026-11-13", "dirty": false, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-2b-proposed-changes/case-6-reporter-unresponsive"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "claude-sonnet-5-5", "model_source": "cli-flag", "notes": "cross-model check for #1800 (re-run of infra failures)", "passed": 1, "path": "tools/skill-evals/evals/security-issue-sync/step-2b-proposed-changes/fixtures/case-6-reporter-unresponsive", "ref": "c22d5769", "schema": 1, "skill": "security-issue-sync", "tags": [], "timestamp": "2026-11-13T04:48:27Z"} {"branch": "feat/cve-tool-interface", "cases": 4, "cli": "claude -p --model claude-sonnet-5-5", "cost": {"cli_calls": 4, "grader_calls": 0, "wall_seconds": 10.9}, "date": "2026-11-13", "dirty": false, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-2c-next-step/case-1-step-3-start-discussion", "step-2c-next-step/case-2-step-6-allocate-cve", "step-2c-next-step/case-3-step-11-pr-merged-parked", "step-2c-next-step/case-4-step-13-fix-released-rm-action"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "claude-sonnet-5-5", "model_source": "cli-flag", "notes": "cross-model check for #1800", "passed": 4, "path": "tools/skill-evals/evals/security-issue-sync/step-2c-next-step/fixtures", "ref": "c22d5769", "schema": 1, "skill": "security-issue-sync", "tags": [], "timestamp": "2026-11-13T04:49:41Z"} {"branch": "feat/cve-tool-interface", "cases": 4, "cli": "claude -p --model claude-sonnet-5-5", "cost": {"cli_calls": 4, "grader_calls": 0, "wall_seconds": 12.4}, "date": "2026-11-13", "dirty": false, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-handoff-delegate/case-1-triage-delegated", "step-handoff-delegate/case-2-cve-allocate-delegated-after", "step-handoff-delegate/case-3-unrelated-delegate", "step-handoff-delegate/case-4-no-overrides-flag"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "claude-sonnet-5-5", "model_source": "cli-flag", "notes": "cross-model check for #1800", "passed": 4, "path": "tools/skill-evals/evals/security-issue-sync/step-handoff-delegate/fixtures", "ref": "c22d5769", "schema": 1, "skill": "security-issue-sync", "tags": [], "timestamp": "2026-11-13T04:49:54Z"} +{"branch": "feat/security-skills-vetted-cve-ops", "cases": 2, "cli": "claude -p", "cost": {"cli_calls": 2, "grader_calls": 2, "wall_seconds": 36.6}, "date": "2026-11-18", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-5b-judged-gates/case-4-private-scanner-credit", "step-5b-judged-gates/case-5-clean-push-fix-released"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 2, "path": "tools/skill-evals/evals/security-issue-sync/step-5b-judged-gates", "ref": "9e979ff3", "schema": 1, "skill": "security-issue-sync", "tags": [], "timestamp": "2026-11-18T01:05:21Z"} diff --git a/tools/spec-loop/specs/cve-tooling.md b/tools/spec-loop/specs/cve-tooling.md index 83b0274c0..b584e2ee0 100644 --- a/tools/spec-loop/specs/cve-tooling.md +++ b/tools/spec-loop/specs/cve-tooling.md @@ -199,6 +199,17 @@ reviewable. push. The title is attacker-influenced and is passed as one shell-quoted argument. +- **Under the secure setup the skills prefer the vetted Vulnogram operations.** + When the vetted-ops policy grants them, `security-cve-allocate` allocates + with `vetted-op cve-allocate` (the title written to a workspace file, never + quoted into a command) and `security-issue-sync`'s Step 5b pushes with + `vetted-op cve-record-update`; `cve-allocate recipe` returns that command + and the `title_file` when given `vetted_workspace`. `cve-tool push` stays + for a push that would promote a record to `PUBLIC` (the vetted push refuses + it) and for the two confirmed overrides; `fetch`, `check`, `setup` and + `publish` stay on `cve-tool`. A failed vetted call exits 4 and names the + client's code on stderr, which the existing exit tables read; any other + non-zero exit means nothing ran, and the skill falls back to `cve-tool`. - **Any CVE tool other than Vulnogram takes the recipe path (#2077).** The API pre-flight applies only when `cve_authority.tool` is `vulnogram`. `security-management` `cve_allocate/gate.py` `preflight()` maps a `cve-tool` exit `20` (no tool declared), `21` (no adapter) or `22` (operation not implemented), on either the server or the allocate check, to the `recipe` path with a reason naming the missing piece, instead of an "unexpected exit". From fbf1e1eb63f661aa36ac4a963efbac75bb763fe9 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 18 Nov 2026 13:48:43 +1100 Subject: [PATCH 2/2] fix(security): tighten the vetted allocation and push fallbacks Fixes from the pre-PR review of the previous commit: - The vetted allocation names the PMC: the operation gains a validated pmc parameter, passed as --pmc, so the client refuses a token held for any other PMC instead of allocating for whatever the session holds. - Only a refusal naming a missing grant falls back to the generic tool interface. Any other refusal (a rejected record, a malformed argument) stops, so the vetted checks are never routed around, and an unclear allocation outcome sends the user to the PMC's records first, so an allocation is never sent twice. - The title file is named by a hash of tracker, issue and title, so no other run can replace the approved title before the allocation reads it. Generated-by: Claude Opus 5 --- .../cve_allocate/recipe.py | 19 +++++++++----- .../skills/cve-allocate/SKILL.md | 2 +- .../skills/cve-allocate/api-allocation.md | 10 ++++--- .../skills/issue-sync/apply-and-push.md | 5 +++- .../tools/vetted-ops/README.md | 2 +- .../tools/vetted-ops/src/vetted_ops/cli.py | 2 ++ .../vetted-ops/src/vetted_ops/cve_tool.py | 2 +- .../tools/vetted-ops/src/vetted_ops/ops.py | 14 ++++++++-- .../cve_allocate/recipe.py | 19 +++++++++----- .../tests/test_cve_allocate.py | 26 +++++++++++++++++-- .../evals/security-cve-allocate/README.md | 1 + .../evals/security-cve-allocate/runs.jsonl | 2 ++ .../evals/security-issue-sync/README.md | 1 + .../evals/security-issue-sync/runs.jsonl | 2 ++ tools/spec-loop/specs/cve-tooling.md | 23 +++++++++------- tools/vetted-ops/README.md | 2 +- tools/vetted-ops/src/vetted_ops/cli.py | 2 ++ tools/vetted-ops/src/vetted_ops/cve_tool.py | 2 +- tools/vetted-ops/src/vetted_ops/ops.py | 14 ++++++++-- tools/vetted-ops/tests/test_cve_tool.py | 15 +++++++++-- tools/vetted-ops/tests/test_vetted_ops.py | 2 ++ 21 files changed, 126 insertions(+), 41 deletions(-) diff --git a/plugins/magpie-security-tools/tools/security-management/src/security_management/cve_allocate/recipe.py b/plugins/magpie-security-tools/tools/security-management/src/security_management/cve_allocate/recipe.py index bfbfe3494..ffd8fe1f2 100644 --- a/plugins/magpie-security-tools/tools/security-management/src/security_management/cve_allocate/recipe.py +++ b/plugins/magpie-security-tools/tools/security-management/src/security_management/cve_allocate/recipe.py @@ -35,6 +35,8 @@ from __future__ import annotations +import hashlib +import json import re from collections.abc import Mapping, Sequence from typing import Any @@ -95,13 +97,16 @@ def render(data: Mapping[str, Any]) -> dict[str, Any]: ) workspace = one_line(data.get("vetted_workspace")) if workspace: - # The vetted operation reads the title from a workspace file and allocates - # for the PMC its token holds, which the pre-flight checked is `pmc`. - title_file = { - "path": f"{workspace.rstrip('/')}/cve-title-{data['issue']}.txt", - "text": title + "\n", - } - command = cve_tool.vetted_command("security-cve-allocate", "cve-allocate", title_file["path"]) + # The vetted operation reads the title from a workspace file; `pmc` lets the + # client refuse a token held for any other PMC. The file is named by a hash of + # tracker, issue and the approved title, so no other run (another tracker, or + # this issue with another title) writes the same file between the write and + # the allocation; a sanitised slug would not do (`a-b/c` and `a/b-c` collide). + key = hashlib.sha256(json.dumps([tracker, str(data["issue"]), title]).encode()).hexdigest()[:24] + title_file = {"path": f"{workspace.rstrip('/')}/cve-title-{key}.txt", "text": title + "\n"} + command = cve_tool.vetted_command( + "security-cve-allocate", "cve-allocate", pmc, title_file["path"] + ) else: command = cve_tool.command(data, "allocate", "--pmc", pmc, "--title", title) elif member: diff --git a/plugins/magpie-security/skills/cve-allocate/SKILL.md b/plugins/magpie-security/skills/cve-allocate/SKILL.md index b0f237754..622c478cf 100644 --- a/plugins/magpie-security/skills/cve-allocate/SKILL.md +++ b/plugins/magpie-security/skills/cve-allocate/SKILL.md @@ -316,7 +316,7 @@ Present its `text` as it is. Its `path` is one of: - **`api`** — a governance-authorised user whose Step 0 pre-flight chose `api`: a proposal to allocate through the CVE tool's API, with `cve_authority.allocate_url` as the fallback. On the user's yes, write a non-null `title_file`'s `text` to its `path` with the Write tool, then run its `command` (one `cve-tool allocate` call, the title shell-quoted, or the vetted `cve-allocate`) **once**, - write `{exit, stdout, stderr, pmc}` and run ` allocate-result` (after a vetted call that failed, `exit` is the `CVE tool exit N` its stderr names): its `next` is `step-4` with the `cve_id`, `stop` (the ID comes back by mail), `setup`, `check-records` or `recipe`, + write `{exit, stdout, stderr, pmc}` and run ` allocate-result` (after a vetted call, read its exit per [`api-allocation.md`](api-allocation.md#allocate-step-3-api-path) first): its `next` is `step-4` with the `cve_id`, `stop` (the ID comes back by mail), `setup`, `check-records` or `recipe`, per [`api-allocation.md` § *Allocate*](api-allocation.md#allocate-step-3-api-path). `rerun_allocate` is false except after an expired token, and even then only on a fresh yes. - **`recipe`** — a governance-authorised user otherwise: open the allocation URL (authenticate per [`/README.md`](../../../../tools/cve-tool/README.md)), paste the title, submit, paste the ID back. - **`relay`** — a user who is not authorised: the CVE tool will not let them submit, so they never get "click *Allocate*". diff --git a/plugins/magpie-security/skills/cve-allocate/api-allocation.md b/plugins/magpie-security/skills/cve-allocate/api-allocation.md index fbbb123b9..c59ac7ef0 100644 --- a/plugins/magpie-security/skills/cve-allocate/api-allocation.md +++ b/plugins/magpie-security/skills/cve-allocate/api-allocation.md @@ -97,19 +97,21 @@ text, so escape any quote it contains rather than interpolating it raw. **Under the secure setup**, when the vetted-ops policy grants `security-cve-allocate` the `cve-allocate` operation, allocate through it instead (` recipe` returns this command when given `vetted_workspace`): -write the title to `/cve-title-.txt` with the Write tool, then run +write the recipe's `title_file` `text` to its `path` (`/cve-title-.txt`) with the Write tool, then run ```bash -uvx --from ~/.claude/magpie/vetted-ops vetted-op --caller security-cve-allocate cve-allocate /cve-title-.txt +uvx --from ~/.claude/magpie/vetted-ops vetted-op --caller security-cve-allocate cve-allocate /cve-title-.txt ``` It runs the shipped Vulnogram client with a fixed argv, outside the sandbox, with no override flag reachable ([vetted-ops § *The Vulnogram CVE-tool operations*](../../../../tools/vetted-ops/README.md#the-vulnogram-cve-tool-operations)), -and allocates for the PMC its token holds, which the pre-flight confirmed is ``. +and passes `--pmc `, so the client refuses a token held for any other PMC. The title travels as a file, so it is never quoted into a command line. On success it exits 0 and the ID is the first stdout line, as below. When the client fails, `vetted-op` exits 4 and names the client's own code on stderr (`CVE tool exit N`): read the table below with that `N`. -Any other non-zero exit (the caller is not granted the operation, or a usage error) means nothing was allocated: run the `cve-tool allocate` command above instead. +An exit 3 whose stderr says the caller `is not permitted to run 'cve-allocate'` means the policy does not grant it and nothing was allocated: run the `cve-tool allocate` command above instead, on the same yes. +Any other exit 2 or 3 is `vetted-op` refusing the arguments (the PMC, the title file): nothing was allocated, but do not route around the refusal; show its stderr and stop. +Any other outcome (an exit 4 without a `CVE tool exit` line, an interrupted run) is unclear: treat it like exit 3 / 4 in the table, and check ``'s records in Vulnogram before anything else. | Exit | Meaning | Next | |---|---|---| diff --git a/plugins/magpie-security/skills/issue-sync/apply-and-push.md b/plugins/magpie-security/skills/issue-sync/apply-and-push.md index 6926d18bc..5e407721c 100644 --- a/plugins/magpie-security/skills/issue-sync/apply-and-push.md +++ b/plugins/magpie-security/skills/issue-sync/apply-and-push.md @@ -504,7 +504,10 @@ When the state fetch fails, pass `fetched_state: "unknown"`. a JSON whose `CNA_private.state` is `PUBLIC` while the live record is not (the vetted push never promotes a record to `PUBLIC`), and the two overrides below. When the client fails, `vetted-op` exits 4 and names the client's own code on stderr (`CVE tool exit N`), which reads as the `cve-tool push` exit `N`; - any other non-zero exit (the caller is not granted the operation, or a usage error) means nothing was pushed: use `cve-tool push`. + an exit 3 whose stderr says the caller `is not permitted to run 'cve-record-update'` means the policy does not grant it: use `cve-tool push`. + Any other exit 2 or 3 is `vetted-op` rejecting the record (a forbidden key, an ID that does not match ``) or the arguments: + nothing was pushed, and the push stops there; report the stderr, and never send the same JSON through `cve-tool push` to get past the refusal. + Any other outcome (an interrupted run) may or may not have saved: push again, which the idempotence note below makes safe. For Vulnogram, the push merges into the live record and refuses to rename an affected product or package. Two explicit overrides exist; use them only after confirming with the user: diff --git a/plugins/magpie-vetted-ops/tools/vetted-ops/README.md b/plugins/magpie-vetted-ops/tools/vetted-ops/README.md index a1fe276e2..9dd2b1034 100644 --- a/plugins/magpie-vetted-ops/tools/vetted-ops/README.md +++ b/plugins/magpie-vetted-ops/tools/vetted-ops/README.md @@ -516,7 +516,7 @@ overrides. Three operations run it instead, outside the sandbox through |---|---|---| | `cve-record-fetch` (read) | `cve_id` | `vulnogram-api-record-fetch --cve-id ` | | `cve-record-update` (write) | `cve_id body` | `vulnogram-api-record-update --cve-id --json-file -`, the record on stdin | -| `cve-allocate` (write) | `title` (a workspace file) | `vulnogram-api-allocate --title ` | +| `cve-allocate` (write) | `pmc title` (title a workspace file) | `vulnogram-api-allocate --pmc <pmc> --title <title>`; the client refuses a token held for another PMC | - The client is the copy shipped in this plugin beside vetted-ops (`tools/cve-tool-vulnogram/oauth-api`, resolved like the tracker backends), diff --git a/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cli.py b/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cli.py index 1554908ab..ffe94df16 100644 --- a/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cli.py +++ b/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cli.py @@ -145,6 +145,8 @@ def _validate_params( resolved[name] = ops_mod.commit_hash(raw) elif name == "cve_id": resolved[name] = ops_mod.cve_id(raw) + elif name == "pmc": + resolved[name] = ops_mod.pmc(raw) elif name == "date": resolved[name] = ops_mod.date(raw) elif name == "cursor": diff --git a/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cve_tool.py b/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cve_tool.py index f4d6c2fff..318975564 100644 --- a/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cve_tool.py +++ b/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/cve_tool.py @@ -27,7 +27,7 @@ * the client is the copy shipped beside vetted-ops (``tool_dir``), never a path the agent can write; * only the CVE id, the record JSON (piped on stdin, the bytes vetted-ops - validated) and the allocation title are the caller's; every override flag + validated), the allocation PMC and the allocation title are the caller's; every override flag stays off, so the record commands' merge-mode guards always apply; * the child gets an allowlisted environment (``child_env``): ``HOME`` from the password database, a fixed ``PATH``, locale and ``TMPDIR`` only. No diff --git a/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/ops.py b/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/ops.py index 195758b97..b036e306a 100644 --- a/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/ops.py +++ b/plugins/magpie-vetted-ops/tools/vetted-ops/src/vetted_ops/ops.py @@ -224,6 +224,14 @@ def cve_id(value: str) -> str: return _check(_CVE_ID, value, "CVE id") +#: A Vulnogram PMC / committee id (`airflow`, `incubator-foo`): lowercase, no path or flag. +_PMC = re.compile(r"^[a-z][a-z0-9-]{0,63}$") + + +def pmc(value: str) -> str: + return _check(_PMC, value, "PMC") + + #: Top-level keys a CVE record pushed through `cve-record-update` may not carry: #: Vulnogram reads `go` as a request envelope rather than record content. _CVE_RECORD_FORBIDDEN_KEYS = frozenset({"go"}) @@ -3018,12 +3026,14 @@ def _stats_closed_page(cfg: dict[str, str], cursor: str) -> list[str]: _register( Op( name="cve-allocate", - params=("title",), + params=("pmc", "title"), backend="cve-tool", writes=True, summary="Allocate a CVE id for a title through the Vulnogram CVE tool.", title_files=("title",), - build=lambda cfg, title: cve_tool.argv("vulnogram-api-allocate", "--title", title), + # `--pmc` lets the client refuse a token held for any other PMC, so the + # id is reserved for the PMC the user approved, whatever session is current. + build=lambda cfg, pmc, title: cve_tool.argv("vulnogram-api-allocate", "--pmc", pmc, "--title", title), ) ) diff --git a/tools/security-management/src/security_management/cve_allocate/recipe.py b/tools/security-management/src/security_management/cve_allocate/recipe.py index bfbfe3494..ffd8fe1f2 100644 --- a/tools/security-management/src/security_management/cve_allocate/recipe.py +++ b/tools/security-management/src/security_management/cve_allocate/recipe.py @@ -35,6 +35,8 @@ from __future__ import annotations +import hashlib +import json import re from collections.abc import Mapping, Sequence from typing import Any @@ -95,13 +97,16 @@ def render(data: Mapping[str, Any]) -> dict[str, Any]: ) workspace = one_line(data.get("vetted_workspace")) if workspace: - # The vetted operation reads the title from a workspace file and allocates - # for the PMC its token holds, which the pre-flight checked is `pmc`. - title_file = { - "path": f"{workspace.rstrip('/')}/cve-title-{data['issue']}.txt", - "text": title + "\n", - } - command = cve_tool.vetted_command("security-cve-allocate", "cve-allocate", title_file["path"]) + # The vetted operation reads the title from a workspace file; `pmc` lets the + # client refuse a token held for any other PMC. The file is named by a hash of + # tracker, issue and the approved title, so no other run (another tracker, or + # this issue with another title) writes the same file between the write and + # the allocation; a sanitised slug would not do (`a-b/c` and `a/b-c` collide). + key = hashlib.sha256(json.dumps([tracker, str(data["issue"]), title]).encode()).hexdigest()[:24] + title_file = {"path": f"{workspace.rstrip('/')}/cve-title-{key}.txt", "text": title + "\n"} + command = cve_tool.vetted_command( + "security-cve-allocate", "cve-allocate", pmc, title_file["path"] + ) else: command = cve_tool.command(data, "allocate", "--pmc", pmc, "--title", title) elif member: diff --git a/tools/security-management/tests/test_cve_allocate.py b/tools/security-management/tests/test_cve_allocate.py index 1a93e5d66..9d236e238 100644 --- a/tools/security-management/tests/test_cve_allocate.py +++ b/tools/security-management/tests/test_cve_allocate.py @@ -425,15 +425,37 @@ def test_the_api_command_quotes_the_title(capsys, tmp_path): def test_the_vetted_route_reads_the_title_from_a_workspace_file(capsys, tmp_path): hostile = "x'; rm -rf / #`" out = recipe(capsys, tmp_path, True, {"path": "api"}, title=hostile, vetted_workspace="/ws/") + title_path = out["title_file"]["path"] + assert title_path.startswith("/ws/cve-title-") and title_path.endswith(".txt") assert out["command"] == ( "uvx --from ~/.claude/magpie/vetted-ops vetted-op --caller security-cve-allocate " - "cve-allocate /ws/cve-title-242.txt" + f"cve-allocate airflow {title_path}" ) assert hostile not in out["command"] - assert out["title_file"] == {"path": "/ws/cve-title-242.txt", "text": hostile + "\n"} + assert out["title_file"] == {"path": title_path, "text": hostile + "\n"} assert out["api_allocation_proposed"] is True +def test_title_files_of_different_trackers_never_collide(capsys, tmp_path): + paths = { + recipe(capsys, tmp_path, True, {"path": "api"}, tracker=tracker, vetted_workspace="/ws")[ + "title_file" + ]["path"] + for tracker in ("a-b/c", "a/b-c", "a/b") + } + assert len(paths) == 3 + + +def test_another_title_for_the_same_issue_gets_its_own_file(capsys, tmp_path): + paths = { + recipe(capsys, tmp_path, True, {"path": "api"}, title=title, vetted_workspace="/ws")["title_file"][ + "path" + ] + for title in ("First title", "Second title") + } + assert len(paths) == 2 + + def test_without_a_vetted_workspace_the_api_path_keeps_cve_tool(capsys, tmp_path): out = recipe(capsys, tmp_path, True, {"path": "api"}) assert "cve-tool allocate --pmc airflow" in out["command"] diff --git a/tools/skill-evals/evals/security-cve-allocate/README.md b/tools/skill-evals/evals/security-cve-allocate/README.md index d1caf56f1..3e8134528 100644 --- a/tools/skill-evals/evals/security-cve-allocate/README.md +++ b/tools/skill-evals/evals/security-cve-allocate/README.md @@ -59,5 +59,6 @@ Columns and rules: [Recording runs](../../README.md#recording-runs). | Date | Ref | Model | Runs | Result | Notes | |---|---|---|---|---|---| +| 2026-11-18 | `add4ee52` (feat/security-skills-vetted-cve-ops) | `opus[1m]` | 2 | pass | | | 2026-11-18 | `9e979ff3` (feat/security-skills-vetted-cve-ops) | `opus[1m]` | 1 | pass | | | 2026-11-15 | `1087814b` (perf/trim-allocate-and-rc-cut) | `opus[1m]` | 2 | pass | | diff --git a/tools/skill-evals/evals/security-cve-allocate/runs.jsonl b/tools/skill-evals/evals/security-cve-allocate/runs.jsonl index 6905ee5e4..46009dd08 100644 --- a/tools/skill-evals/evals/security-cve-allocate/runs.jsonl +++ b/tools/skill-evals/evals/security-cve-allocate/runs.jsonl @@ -1,3 +1,5 @@ {"branch": "perf/trim-allocate-and-rc-cut", "cases": 4, "cli": "claude -p", "cost": {"cli_calls": 4, "grader_calls": 1, "wall_seconds": 58.7}, "date": "2026-11-15", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-4-draft-body/case-1-direct-reporter-credit-not-asked", "step-4-draft-body/case-2-credit-already-asked", "step-4-draft-body/case-3-reporter-asks-for-the-record-link", "step-4-draft-body/case-4-second-reporter-of-a-merged-tracker"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 4, "path": "tools/skill-evals/evals/security-cve-allocate", "ref": "1087814b", "schema": 1, "skill": "security-cve-allocate", "tags": [], "timestamp": "2026-11-15T07:01:47Z"} {"branch": "perf/trim-allocate-and-rc-cut", "cases": 4, "cli": "claude -p", "cost": {"cli_calls": 4, "grader_calls": 1, "wall_seconds": 62.4}, "date": "2026-11-15", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-4-draft-body/case-1-direct-reporter-credit-not-asked", "step-4-draft-body/case-2-credit-already-asked", "step-4-draft-body/case-3-reporter-asks-for-the-record-link", "step-4-draft-body/case-4-second-reporter-of-a-merged-tracker"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 4, "path": "tools/skill-evals/evals/security-cve-allocate", "ref": "1087814b", "schema": 1, "skill": "security-cve-allocate", "tags": [], "timestamp": "2026-11-15T07:04:17Z"} {"branch": "feat/security-skills-vetted-cve-ops", "cases": 4, "cli": "claude -p", "cost": {"cli_calls": 4, "grader_calls": 1, "wall_seconds": 42.9}, "date": "2026-11-18", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-4-draft-body/case-1-direct-reporter-credit-not-asked", "step-4-draft-body/case-2-credit-already-asked", "step-4-draft-body/case-3-reporter-asks-for-the-record-link", "step-4-draft-body/case-4-second-reporter-of-a-merged-tracker"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 4, "path": "tools/skill-evals/evals/security-cve-allocate", "ref": "9e979ff3", "schema": 1, "skill": "security-cve-allocate", "tags": [], "timestamp": "2026-11-18T01:06:05Z"} +{"branch": "feat/security-skills-vetted-cve-ops", "cases": 4, "cli": "claude -p", "cost": {"cli_calls": 4, "grader_calls": 1, "wall_seconds": 44.3}, "date": "2026-11-18", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-4-draft-body/case-1-direct-reporter-credit-not-asked", "step-4-draft-body/case-2-credit-already-asked", "step-4-draft-body/case-3-reporter-asks-for-the-record-link", "step-4-draft-body/case-4-second-reporter-of-a-merged-tracker"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 4, "path": "tools/skill-evals/evals/security-cve-allocate", "ref": "add4ee52", "schema": 1, "skill": "security-cve-allocate", "tags": [], "timestamp": "2026-11-18T01:23:59Z"} +{"branch": "feat/security-skills-vetted-cve-ops", "cases": 4, "cli": "claude -p", "cost": {"cli_calls": 4, "grader_calls": 1, "wall_seconds": 43.4}, "date": "2026-11-18", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-4-draft-body/case-1-direct-reporter-credit-not-asked", "step-4-draft-body/case-2-credit-already-asked", "step-4-draft-body/case-3-reporter-asks-for-the-record-link", "step-4-draft-body/case-4-second-reporter-of-a-merged-tracker"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 4, "path": "tools/skill-evals/evals/security-cve-allocate", "ref": "add4ee52", "schema": 1, "skill": "security-cve-allocate", "tags": [], "timestamp": "2026-11-18T02:37:11Z"} diff --git a/tools/skill-evals/evals/security-issue-sync/README.md b/tools/skill-evals/evals/security-issue-sync/README.md index da4718c2d..785f1fd00 100644 --- a/tools/skill-evals/evals/security-issue-sync/README.md +++ b/tools/skill-evals/evals/security-issue-sync/README.md @@ -59,5 +59,6 @@ Columns and rules: [Recording runs](../../README.md#recording-runs). | Date | Ref | Model | Runs | Result | Notes | |---|---|---|---|---|---| +| 2026-11-18 | `add4ee52` (feat/security-skills-vetted-cve-ops) | `opus[1m]` | 2 | pass | | | 2026-11-18 | `9e979ff3` (feat/security-skills-vetted-cve-ops) | `opus[1m]` | 1 | pass | | | 2026-11-13 | `c22d5769` (feat/cve-tool-interface) | `claude-sonnet-5-5` | 10 | `step-2b-proposed-changes/case-2-milestone-missing-create` failed 1 of 2; `step-2b-proposed-changes/case-5-pre-announce-distributors` failed 2 of 2; `step-2b-proposed-changes/case-6-reporter-unresponsive` errored 1 of 2 | cross-model check for #1800; cross-model check for #1800 (re-run of infra failures) | diff --git a/tools/skill-evals/evals/security-issue-sync/runs.jsonl b/tools/skill-evals/evals/security-issue-sync/runs.jsonl index 80593c27a..f4636ff2d 100644 --- a/tools/skill-evals/evals/security-issue-sync/runs.jsonl +++ b/tools/skill-evals/evals/security-issue-sync/runs.jsonl @@ -9,3 +9,5 @@ {"branch": "feat/cve-tool-interface", "cases": 4, "cli": "claude -p --model claude-sonnet-5-5", "cost": {"cli_calls": 4, "grader_calls": 0, "wall_seconds": 10.9}, "date": "2026-11-13", "dirty": false, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-2c-next-step/case-1-step-3-start-discussion", "step-2c-next-step/case-2-step-6-allocate-cve", "step-2c-next-step/case-3-step-11-pr-merged-parked", "step-2c-next-step/case-4-step-13-fix-released-rm-action"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "claude-sonnet-5-5", "model_source": "cli-flag", "notes": "cross-model check for #1800", "passed": 4, "path": "tools/skill-evals/evals/security-issue-sync/step-2c-next-step/fixtures", "ref": "c22d5769", "schema": 1, "skill": "security-issue-sync", "tags": [], "timestamp": "2026-11-13T04:49:41Z"} {"branch": "feat/cve-tool-interface", "cases": 4, "cli": "claude -p --model claude-sonnet-5-5", "cost": {"cli_calls": 4, "grader_calls": 0, "wall_seconds": 12.4}, "date": "2026-11-13", "dirty": false, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-handoff-delegate/case-1-triage-delegated", "step-handoff-delegate/case-2-cve-allocate-delegated-after", "step-handoff-delegate/case-3-unrelated-delegate", "step-handoff-delegate/case-4-no-overrides-flag"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "claude-sonnet-5-5", "model_source": "cli-flag", "notes": "cross-model check for #1800", "passed": 4, "path": "tools/skill-evals/evals/security-issue-sync/step-handoff-delegate/fixtures", "ref": "c22d5769", "schema": 1, "skill": "security-issue-sync", "tags": [], "timestamp": "2026-11-13T04:49:54Z"} {"branch": "feat/security-skills-vetted-cve-ops", "cases": 2, "cli": "claude -p", "cost": {"cli_calls": 2, "grader_calls": 2, "wall_seconds": 36.6}, "date": "2026-11-18", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-5b-judged-gates/case-4-private-scanner-credit", "step-5b-judged-gates/case-5-clean-push-fix-released"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 2, "path": "tools/skill-evals/evals/security-issue-sync/step-5b-judged-gates", "ref": "9e979ff3", "schema": 1, "skill": "security-issue-sync", "tags": [], "timestamp": "2026-11-18T01:05:21Z"} +{"branch": "feat/security-skills-vetted-cve-ops", "cases": 2, "cli": "claude -p", "cost": {"cli_calls": 2, "grader_calls": 2, "wall_seconds": 36.6}, "date": "2026-11-18", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-5b-judged-gates/case-4-private-scanner-credit", "step-5b-judged-gates/case-5-clean-push-fix-released"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 2, "path": "tools/skill-evals/evals/security-issue-sync/step-5b-judged-gates", "ref": "add4ee52", "schema": 1, "skill": "security-issue-sync", "tags": [], "timestamp": "2026-11-18T01:24:36Z"} +{"branch": "feat/security-skills-vetted-cve-ops", "cases": 2, "cli": "claude -p", "cost": {"cli_calls": 2, "grader_calls": 2, "wall_seconds": 36.0}, "date": "2026-11-18", "dirty": true, "errored": 0, "errored_cases": [], "failed": 0, "failed_cases": [], "graded_cases": ["step-5b-judged-gates/case-4-private-scanner-credit", "step-5b-judged-gates/case-5-clean-push-fix-released"], "grader_cli": "claude -p --model haiku", "manual": 0, "model": "opus[1m]", "model_source": "settings:user", "notes": "", "passed": 2, "path": "tools/skill-evals/evals/security-issue-sync/step-5b-judged-gates", "ref": "add4ee52", "schema": 1, "skill": "security-issue-sync", "tags": [], "timestamp": "2026-11-18T02:36:27Z"} diff --git a/tools/spec-loop/specs/cve-tooling.md b/tools/spec-loop/specs/cve-tooling.md index b584e2ee0..3296332cb 100644 --- a/tools/spec-loop/specs/cve-tooling.md +++ b/tools/spec-loop/specs/cve-tooling.md @@ -201,15 +201,20 @@ reviewable. argument. - **Under the secure setup the skills prefer the vetted Vulnogram operations.** When the vetted-ops policy grants them, `security-cve-allocate` allocates - with `vetted-op cve-allocate` (the title written to a workspace file, never - quoted into a command) and `security-issue-sync`'s Step 5b pushes with - `vetted-op cve-record-update`; `cve-allocate recipe` returns that command - and the `title_file` when given `vetted_workspace`. `cve-tool push` stays - for a push that would promote a record to `PUBLIC` (the vetted push refuses - it) and for the two confirmed overrides; `fetch`, `check`, `setup` and - `publish` stay on `cve-tool`. A failed vetted call exits 4 and names the - client's code on stderr, which the existing exit tables read; any other - non-zero exit means nothing ran, and the skill falls back to `cve-tool`. + with `vetted-op cve-allocate <pmc> <title-file>` (the title written to a + workspace file named by a hash of tracker, issue and title, never quoted into a + command; `--pmc` makes the client refuse a token for any other PMC) and + `security-issue-sync`'s Step 5b pushes with `vetted-op cve-record-update`; + `cve-allocate recipe` returns that command and the `title_file` when given + `vetted_workspace`. `cve-tool push` stays for a push that would promote a + record to `PUBLIC` (the vetted push refuses it) and for the two confirmed + overrides; `fetch`, `check`, `setup` and `publish` stay on `cve-tool`. A + failed vetted call exits 4 and names the client's code on stderr, which the + existing exit tables read. Only a refusal naming a missing grant (`is not + permitted to run`) falls back to `cve-tool`; any other `vetted-op` refusal + (a rejected record or argument) stops, so the vetted checks are never + routed around; any other outcome of an allocation is unclear and sends the + user to the PMC's records first, so an allocation is never sent twice. - **Any CVE tool other than Vulnogram takes the recipe path (#2077).** The API pre-flight applies only when `cve_authority.tool` is `vulnogram`. `security-management` `cve_allocate/gate.py` `preflight()` maps a `cve-tool` exit `20` (no tool declared), `21` (no adapter) or `22` (operation not implemented), on either the server or the allocate check, to the `recipe` path with a reason naming the missing piece, instead of an "unexpected exit". diff --git a/tools/vetted-ops/README.md b/tools/vetted-ops/README.md index c774148fd..c53e3ba23 100644 --- a/tools/vetted-ops/README.md +++ b/tools/vetted-ops/README.md @@ -516,7 +516,7 @@ overrides. Three operations run it instead, outside the sandbox through |---|---|---| | `cve-record-fetch` (read) | `cve_id` | `vulnogram-api-record-fetch --cve-id <id>` | | `cve-record-update` (write) | `cve_id body` | `vulnogram-api-record-update --cve-id <id> --json-file -`, the record on stdin | -| `cve-allocate` (write) | `title` (a workspace file) | `vulnogram-api-allocate --title <title>` | +| `cve-allocate` (write) | `pmc title` (title a workspace file) | `vulnogram-api-allocate --pmc <pmc> --title <title>`; the client refuses a token held for another PMC | - The client is the copy shipped in this plugin beside vetted-ops (`tools/cve-tool-vulnogram/oauth-api`, resolved like the tracker backends), diff --git a/tools/vetted-ops/src/vetted_ops/cli.py b/tools/vetted-ops/src/vetted_ops/cli.py index 1554908ab..ffe94df16 100644 --- a/tools/vetted-ops/src/vetted_ops/cli.py +++ b/tools/vetted-ops/src/vetted_ops/cli.py @@ -145,6 +145,8 @@ def _validate_params( resolved[name] = ops_mod.commit_hash(raw) elif name == "cve_id": resolved[name] = ops_mod.cve_id(raw) + elif name == "pmc": + resolved[name] = ops_mod.pmc(raw) elif name == "date": resolved[name] = ops_mod.date(raw) elif name == "cursor": diff --git a/tools/vetted-ops/src/vetted_ops/cve_tool.py b/tools/vetted-ops/src/vetted_ops/cve_tool.py index f4d6c2fff..318975564 100644 --- a/tools/vetted-ops/src/vetted_ops/cve_tool.py +++ b/tools/vetted-ops/src/vetted_ops/cve_tool.py @@ -27,7 +27,7 @@ * the client is the copy shipped beside vetted-ops (``tool_dir``), never a path the agent can write; * only the CVE id, the record JSON (piped on stdin, the bytes vetted-ops - validated) and the allocation title are the caller's; every override flag + validated), the allocation PMC and the allocation title are the caller's; every override flag stays off, so the record commands' merge-mode guards always apply; * the child gets an allowlisted environment (``child_env``): ``HOME`` from the password database, a fixed ``PATH``, locale and ``TMPDIR`` only. No diff --git a/tools/vetted-ops/src/vetted_ops/ops.py b/tools/vetted-ops/src/vetted_ops/ops.py index 195758b97..b036e306a 100644 --- a/tools/vetted-ops/src/vetted_ops/ops.py +++ b/tools/vetted-ops/src/vetted_ops/ops.py @@ -224,6 +224,14 @@ def cve_id(value: str) -> str: return _check(_CVE_ID, value, "CVE id") +#: A Vulnogram PMC / committee id (`airflow`, `incubator-foo`): lowercase, no path or flag. +_PMC = re.compile(r"^[a-z][a-z0-9-]{0,63}$") + + +def pmc(value: str) -> str: + return _check(_PMC, value, "PMC") + + #: Top-level keys a CVE record pushed through `cve-record-update` may not carry: #: Vulnogram reads `go` as a request envelope rather than record content. _CVE_RECORD_FORBIDDEN_KEYS = frozenset({"go"}) @@ -3018,12 +3026,14 @@ def _stats_closed_page(cfg: dict[str, str], cursor: str) -> list[str]: _register( Op( name="cve-allocate", - params=("title",), + params=("pmc", "title"), backend="cve-tool", writes=True, summary="Allocate a CVE id for a title through the Vulnogram CVE tool.", title_files=("title",), - build=lambda cfg, title: cve_tool.argv("vulnogram-api-allocate", "--title", title), + # `--pmc` lets the client refuse a token held for any other PMC, so the + # id is reserved for the PMC the user approved, whatever session is current. + build=lambda cfg, pmc, title: cve_tool.argv("vulnogram-api-allocate", "--pmc", pmc, "--title", title), ) ) diff --git a/tools/vetted-ops/tests/test_cve_tool.py b/tools/vetted-ops/tests/test_cve_tool.py index 938a811c6..70af44daa 100644 --- a/tools/vetted-ops/tests/test_cve_tool.py +++ b/tools/vetted-ops/tests/test_cve_tool.py @@ -145,10 +145,12 @@ def test_allocate_reads_its_title_from_a_workspace_file(policy: config.Config, w title.write_text("Config API: section aliases bypass masking\n") op = ops.resolve("cve-allocate") assert op.writes - params, _ = cli._validate_params(op, [str(title)], policy) + params, _ = cli._validate_params(op, ["airflow", str(title)], policy) assert cli.build_argv(op, params, policy) == [ *_prefix(), "vulnogram-api-allocate", + "--pmc", + "airflow", "--title", "Config API: section aliases bypass masking", ] @@ -162,10 +164,18 @@ def test_a_malformed_cve_id_never_reaches_the_client(policy: config.Config, bad: cli._validate_params(ops.resolve("cve-record-fetch"), [bad], policy) +@pytest.mark.parametrize("bad", ["Airflow", "airflow --credentials", "--credentials", "../x", ""]) +def test_a_malformed_pmc_never_reaches_the_client(policy: config.Config, workspace: Path, bad: str) -> None: + title = workspace / "title.txt" + title.write_text("A title\n") + with pytest.raises(ops.ParamError): + cli._validate_params(ops.resolve("cve-allocate"), [bad, str(title)], policy) + + def test_no_override_flag_is_reachable() -> None: """The caller's values are the only variable parts; no `--allow-*` / `--credentials`.""" for name in ("cve-record-fetch", "cve-record-update", "cve-allocate"): - assert set(ops.resolve(name).params) <= {"cve_id", "body", "title"} + assert set(ops.resolve(name).params) <= {"cve_id", "body", "title", "pmc"} def test_a_tampered_argv_is_refused(policy: config.Config) -> None: @@ -253,6 +263,7 @@ def test_a_dry_run_shows_the_title_only_as_its_size( "security-cve-allocate", "--dry-run", "cve-allocate", + "airflow", str(title), "--config", str(policy_path), diff --git a/tools/vetted-ops/tests/test_vetted_ops.py b/tools/vetted-ops/tests/test_vetted_ops.py index 267cb0559..d404cef83 100644 --- a/tools/vetted-ops/tests/test_vetted_ops.py +++ b/tools/vetted-ops/tests/test_vetted_ops.py @@ -112,6 +112,7 @@ def test_every_op_declares_validators_for_all_its_params() -> None: "version", "commit_hash", "cve_id", + "pmc", "action", "field", "date", @@ -157,6 +158,7 @@ def test_every_builder_produces_a_gh_argv(policy: config.Config) -> None: "version": "1.0.0", "commit_hash": "a1b2c3d", "cve_id": "CVE-2023-1234", + "pmc": "airflow", "ecosystem": "PyPI", "action": "Sync", "field": "CVE tool link",