From 783dc24fbff1907f379ea5aabbdfa659d12fcf30 Mon Sep 17 00:00:00 2001 From: Daniel Gaskins Date: Mon, 10 Aug 2026 18:03:36 -0700 Subject: [PATCH] Add coding agent self-equip --- CHANGELOG.md | 18 ++++ README.md | 16 ++- docs/assurance.md | 4 + docs/harness-integrations.md | 57 +++++++++-- pyproject.toml | 2 +- scripts/assure_distribution.py | 17 +++- src/mendmark/__init__.py | 2 +- src/mendmark/cli.py | 14 ++- src/mendmark/equip.py | 150 +++++++++++++++++++++++++++-- tests/test_harness_integrations.py | 61 +++++++++++- 10 files changed, 311 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60f2ab4..173346b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,24 @@ All notable changes to Mendmark are documented here. The project follows Semantic Versioning for its Python and JSON contracts. +## 0.6.1 - 2026-08-10 + +### Added + +- Repo-scoped Mendmark skills for Codex and Claude Code, with explicit + `$mendmark` and `/mendmark` invocation paths and non-destructive auto/all + targeting through `mendmark equip --agent`. +- An always-generated, agent-neutral `SELF-EQUIP.md` protocol for unsupported + coding agents, covering harness discovery, JSON fallback, causal trace + fidelity, reviewed golden behavior, privacy, side effects, baselines, CI, + and final evidence reporting. + +### Changed + +- Self-equip metadata now records coding-agent targets, and distribution + assurance verifies both native skills without modifying existing + `AGENTS.md`, `CLAUDE.md`, or repository policy. + ## 0.6.0 - 2026-08-10 ### Added diff --git a/README.md b/README.md index f3efeb7..93b71a6 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,8 @@ Mendmark has dependency-light adapters for LangChain/LangGraph, CrewAI, and the OpenAI Agents SDK. In an existing agent repository: ```bash -python -m pip install 'mendmark-evals==0.6.0' -mendmark equip --framework auto +python -m pip install 'mendmark-evals==0.6.1' +mendmark equip --framework auto --agent auto ``` The command detects bounded dependency files and creates a reviewed capture @@ -79,10 +79,18 @@ guide, offline evaluator, and inactive CI template under `.mendmark/`. It does not edit application code, upload a trace, overwrite existing work, enable CI, or accept a baseline. -Want the repository's coding agent to perform the integration? +Want Codex or Claude Code to perform the integration? Install its native, +repo-scoped skill (use `all` to install both): ```bash -mendmark equip --print-agent-prompt +mendmark equip --framework auto --agent codex # invoke with $mendmark +mendmark equip --framework auto --agent claude-code # invoke with /mendmark +``` + +For any other repository-capable agent, print a portable self-equip prompt: + +```bash +mendmark equip --agent generic --print-agent-prompt ``` See the [agent harness integration guide](docs/harness-integrations.md) for the diff --git a/docs/assurance.md b/docs/assurance.md index 060a8d6..33e0b42 100644 --- a/docs/assurance.md +++ b/docs/assurance.md @@ -15,6 +15,10 @@ The automated assurance suite checks that: is tested for idempotency, bounded detection, conflict refusal, symlink containment, explicit expectation approval, and a complete generated evaluator journey. +- Codex and Claude Code receive repo-scoped skills in their documented native + locations, while an always-generated portable protocol covers unrecognized + coding agents. Tests verify auto-detection, explicit/all targeting, native + invocation names, existing-instruction preservation, and safety boundaries. - Reports, console output, JUnit, and SARIF do not expose canary values placed in prompts, answers, metadata, tags, tool arguments, tool outputs, or descriptions. - Repeated audits preserve mutation IDs, ordering, decisions, JUnit, SARIF, and diff --git a/docs/harness-integrations.md b/docs/harness-integrations.md index 752331a..20b31fe 100644 --- a/docs/harness-integrations.md +++ b/docs/harness-integrations.md @@ -36,14 +36,15 @@ guide](https://openai.github.io/openai-agents-python/tracing/). From the agent application repository: ```bash -python -m pip install 'mendmark-evals==0.6.0' -mendmark equip --framework auto +python -m pip install 'mendmark-evals==0.6.1' +mendmark equip --framework auto --agent auto ``` Detection reads only bounded dependency files; it does not import or execute the -application. The command creates five reviewable files under `.mendmark/`: +application. The command creates six reviewable files under `.mendmark/`: - `agent-setup.md`: harness-specific capture code and acceptance criteria. +- `SELF-EQUIP.md`: portable instructions for any repository-capable agent. - `evaluator.py`: a deterministic offline evaluator for reviewed snapshots. - `mendmark-ci.yml`: an inactive, pinned CI template. - `config.json`: detected integration metadata. @@ -55,10 +56,39 @@ preview its targets. ## Let a coding agent self-equip the repository +### Codex + +```bash +mendmark equip --framework auto --agent codex +``` + +This installs a repo-scoped skill at `.agents/skills/mendmark/SKILL.md`, the +location Codex discovers from the repository root. Ask Codex to equip or audit +agent evaluations naturally, or invoke the skill explicitly with `$mendmark`. +UI metadata supplies a concise skill title and default prompt. Mendmark does +not create or modify the repository's `AGENTS.md`. +See OpenAI's official [Codex skills documentation](https://learn.chatgpt.com/docs/build-skills) +and [AGENTS.md discovery rules](https://learn.chatgpt.com/docs/agent-configuration/agents-md). + +### Claude Code + +```bash +mendmark equip --framework auto --agent claude-code +``` + +This installs `.claude/skills/mendmark/SKILL.md`. Claude Code can select it when +the request matches its description, or the user can invoke `/mendmark`. +Mendmark does not create or modify `CLAUDE.md`, `CLAUDE.local.md`, or existing +Claude rules. Use `--agent all` to install both native skills. +See Anthropic's official [Claude Code skills documentation](https://docs.claude.com/en/docs/claude-code/skills) +and [project-memory documentation](https://docs.claude.com/en/docs/claude-code/memory). + +### Any other coding agent + Print a prompt that works with repository-capable coding agents: ```bash -mendmark equip --print-agent-prompt +mendmark equip --agent generic --print-agent-prompt ``` The prompt instructs the agent to run detection, read the generated setup file, @@ -66,13 +96,20 @@ capture a real tool-using case, pass the audit, and meet every review criterion. It explicitly forbids uploading trace content or silently treating observed production behavior as correct. -The short prompt can also be copied directly: +Every setup includes `.mendmark/SELF-EQUIP.md`, a framework-neutral procedure +covering capability discovery, JSON 2.0 fallback, stable call correlation, +explicit multi-agent causality, golden-behavior review, privacy, side-effect +isolation, audit execution, baseline review, CI activation, and final reporting. +An unrepresented agent needs no Mendmark-specific plugin: give it the printed +prompt and repository access. + +The short generic prompt can also be copied directly: -> Run `mendmark equip --framework auto`, read `.mendmark/agent-setup.md` -> completely, integrate the detected harness, capture at least one reviewed -> tool-using case, run the local audit, and satisfy every acceptance criterion -> before enabling CI. Do not upload trace content or approve observed behavior -> without human review. +> Run `mendmark equip --framework auto --agent generic`, read +> `.mendmark/SELF-EQUIP.md` and `.mendmark/agent-setup.md` completely, integrate +> the detected harness, capture at least one reviewed tool-using case, run the +> local audit, and satisfy every acceptance criterion before enabling CI. Do +> not upload trace content or approve observed behavior without human review. ## Direct Python API diff --git a/pyproject.toml b/pyproject.toml index 34e4cc9..adaaa1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mendmark-evals" -version = "0.6.0" +version = "0.6.1" description = "Mutation testing for agent evaluation suites" readme = "README.md" requires-python = ">=3.10" diff --git a/scripts/assure_distribution.py b/scripts/assure_distribution.py index c7ea910..b77dbce 100644 --- a/scripts/assure_distribution.py +++ b/scripts/assure_distribution.py @@ -125,7 +125,7 @@ def main() -> int: cwd=workspace, env=clean_env, ) - if "agent-setup.md" not in prompt or "human review" not in prompt: + if "SELF-EQUIP.md" not in prompt or "human review" not in prompt: raise RuntimeError("installed wheel did not expose safe agent self-equip guidance") integration_api = run( [ @@ -150,6 +150,8 @@ def main() -> int: "equip", "--framework", "langgraph", + "--agent", + "all", "--project-root", str(workspace), ], @@ -161,12 +163,25 @@ def main() -> int: for generated in ( "evaluator.py", "agent-setup.md", + "SELF-EQUIP.md", "mendmark-ci.yml", "config.json", ".gitignore", ): if not (workspace / ".mendmark" / generated).is_file(): raise RuntimeError(f"installed wheel omitted equip asset: {generated}") + for skill in ( + workspace / ".agents" / "skills" / "mendmark" / "SKILL.md", + workspace / ".claude" / "skills" / "mendmark" / "SKILL.md", + workspace + / ".agents" + / "skills" + / "mendmark" + / "agents" + / "openai.yaml", + ): + if not skill.is_file(): + raise RuntimeError(f"installed wheel omitted agent skill: {skill}") tasks = run([str(mendmark), "tasks"], cwd=workspace, env=clean_env) if len([line for line in tasks.splitlines() if line.strip()]) != 5: raise RuntimeError("installed wheel did not expose all five ML integrity tasks") diff --git a/src/mendmark/__init__.py b/src/mendmark/__init__.py index ff14598..3a16538 100644 --- a/src/mendmark/__init__.py +++ b/src/mendmark/__init__.py @@ -2,7 +2,7 @@ from .agent_cases import AgentCase, AgentEvent, AgentSpec, ToolCallRecord, ToolSpec -__version__ = "0.6.0" +__version__ = "0.6.1" __all__ = [ "AgentCase", diff --git a/src/mendmark/cli.py b/src/mendmark/cli.py index d86c685..eb4d3ab 100644 --- a/src/mendmark/cli.py +++ b/src/mendmark/cli.py @@ -91,6 +91,12 @@ def build_parser() -> argparse.ArgumentParser: default="auto", help="detect from dependency files or select an integration (default: auto)", ) + equip.add_argument( + "--agent", + choices=("auto", "codex", "claude-code", "generic", "all"), + default="auto", + help="detect or select repository coding-agent instructions (default: auto)", + ) equip.add_argument("--project-root", default=".", help="project to equip") equip.add_argument( "--dry-run", action="store_true", help="show files without writing them" @@ -446,18 +452,22 @@ def main(argv: list[str] | None = None) -> int: return 0 if args.command == "equip": - from .equip import agent_prompt, equip_project + from .equip import agent_prompt, equip_project, resolve_agent_hosts if args.print_agent_prompt: - print(agent_prompt(args.project_root)) + print(agent_prompt(args.project_root, agent=args.agent)) return 0 + root = Path(args.project_root).expanduser().resolve() + agent_hosts = resolve_agent_hosts(root, args.agent) frameworks, created, unchanged = equip_project( args.project_root, framework=args.framework, + agent=args.agent, dry_run=args.dry_run, ) action = "Would create" if args.dry_run else "Created" print("Detected: " + ", ".join(frameworks)) + print("Agent host: " + ", ".join(agent_hosts)) for path in created: print(f"{action}: {path}") for path in unchanged: diff --git a/src/mendmark/equip.py b/src/mendmark/equip.py index 236150a..2ca629a 100644 --- a/src/mendmark/equip.py +++ b/src/mendmark/equip.py @@ -14,6 +14,7 @@ class EquipError(ValueError): SUPPORTED_FRAMEWORKS: Final = ("langgraph", "crewai", "openai-agents", "generic") +SUPPORTED_AGENT_HOSTS: Final = ("codex", "claude-code", "generic") _MARKER = "Generated by Mendmark equip; safe to regenerate." _DEPENDENCY_FILES = ( "pyproject.toml", @@ -43,11 +44,48 @@ def detect_frameworks(project_root: Path) -> tuple[str, ...]: return tuple(found or ("generic",)) -def agent_prompt(project_root: str = ".") -> str: +def detect_agent_hosts(project_root: Path) -> tuple[str, ...]: + """Detect repository coding-agent conventions without reading their contents.""" + found: list[str] = [] + if any( + (project_root / name).exists() + for name in ("AGENTS.md", "AGENTS.override.md", ".agents", ".codex") + ): + found.append("codex") + if any( + (project_root / name).exists() + for name in ("CLAUDE.md", "CLAUDE.local.md", ".claude") + ): + found.append("claude-code") + return tuple(found or ("generic",)) + + +def resolve_agent_hosts(project_root: Path, agent: str) -> tuple[str, ...]: + """Resolve a CLI agent selector to concrete instruction targets.""" + if agent == "auto": + return detect_agent_hosts(project_root) + if agent == "all": + return ("codex", "claude-code", "generic") + if agent not in SUPPORTED_AGENT_HOSTS: + raise EquipError(f"unsupported coding agent: {agent}") + return (agent,) + + +def agent_prompt(project_root: str = ".", *, agent: str = "generic") -> str: """Return the short prompt a user can give any repository coding agent.""" + invocation = { + "codex": "After setup, use `$mendmark` for future Mendmark work. ", + "claude-code": "After setup, use `/mendmark` for future Mendmark work. ", + "generic": "", + "auto": "", + "all": "", + }.get(agent) + if invocation is None: + raise EquipError(f"unsupported coding agent: {agent}") return ( - f"In {project_root}, run `mendmark equip --framework auto`, then read " - "`.mendmark/agent-setup.md` completely. Integrate the detected harness, " + f"In {project_root}, run `mendmark equip --framework auto --agent {agent}`, " + "then read `.mendmark/SELF-EQUIP.md` and `.mendmark/agent-setup.md` " + "completely. " + invocation + "Integrate the detected harness, " "capture at least one reviewed tool-using case, run the local audit, and " "satisfy every acceptance criterion before enabling the generated CI " "workflow. Do not upload trace content or approve observed behavior " @@ -55,6 +93,87 @@ def agent_prompt(project_root: str = ".") -> str: ) +def _universal_protocol() -> str: + return f''' +# Mendmark self-equip protocol + +This is the portable instruction contract for any repository-capable coding +agent, including agents Mendmark does not recognize. + +## Objective + +Add a local, reviewed mutation-test gate for this repository's agent evaluator. +Success means important mistakes in tool selection, arguments, results, +ordering, coordination, and final outcomes make the evaluator fail. + +## Procedure + +1. Read `.mendmark/agent-setup.md` completely and inspect the repository's + dependency manifests, agent entry points, tool definitions, existing tests, + and CI conventions. Do not execute untrusted project code merely to detect a + framework. +2. Use a native adapter when listed in the setup guide. Otherwise implement the + framework-neutral JSON 2.0 contract; preserve stable tool-call IDs and + explicit causal dependencies rather than inferring causality from time. +3. Identify at least one consequential tool-using path. Capture prompts, + arguments, results, side effects, handoffs, and expected outcomes locally. + Redact credentials and personal data before they enter fixtures or reports. +4. Ask a human to review the candidate golden behavior. Never convert an + observed trace into expected behavior silently; `approve_observed=True` is + allowed only during that explicit, one-off review boundary. +5. Run the exact local audit command from `agent-setup.md`. Investigate every + survivor and strengthen the evaluator or document a deliberate exception. +6. Create and review a baseline only after the original cases pass. Do not + activate the generated CI workflow until the local audit and acceptance + checklist pass. +7. Report files changed, cases covered, mutation score, survivors, redactions, + and any remaining risks. Stop and ask for direction if repository policy, + missing credentials, or destructive actions block safe completion. + +## Non-negotiable boundaries + +- Keep traces local; Mendmark requires no hosted service or telemetry upload. +- Do not call production side-effecting tools while building fixtures. +- Do not overwrite existing project or agent instruction files. +- Do not weaken tests, evaluator thresholds, branch protection, or CI to pass. +- Do not commit secrets, raw customer data, generated reports, or credentials. +''' + + +def _agent_skill(agent: str) -> str: + invocation = "$mendmark" if agent == "codex" else "/mendmark" + return f'''--- +name: mendmark +description: Equip, audit, or improve this repository's agent evaluations with Mendmark. Use when asked about agent eval quality, golden traces, tool-call assurance, multi-agent coordination, mutation testing, or Mendmark setup. +--- + +# Mendmark agent-evaluation assurance + +Read `.mendmark/SELF-EQUIP.md` and `.mendmark/agent-setup.md` completely, then +follow them in that order. Treat their human-review, privacy, production-side- +effect, baseline, and CI boundaries as mandatory. Preserve unrelated work and +existing repository instructions. + +When invoked without a narrower request, perform the full self-equip procedure. +Before finishing, run the local audit and summarize coverage, mutation results, +survivors, changed files, and remaining risks. If the generated files are +missing, install the pinned Mendmark version used by the repository and run +`mendmark equip --framework auto --agent {agent}` first. + +Users can invoke this workflow explicitly as `{invocation}`. +''' + + +def _openai_skill_metadata() -> str: + return '''interface: + display_name: "Mendmark Assurance" + short_description: "Mutation-test agent evaluations and golden traces" + default_prompt: "Use $mendmark to equip and audit this repository's agent evaluations." +policy: + allow_implicit_invocation: true +''' + + def _evaluator() -> str: return f'''#!/usr/bin/env python3 """{_MARKER} @@ -277,13 +396,20 @@ def _workflow() -> str: ''' -def _files(frameworks: tuple[str, ...]) -> dict[str, str]: - return { +def _files( + frameworks: tuple[str, ...], agent_hosts: tuple[str, ...] +) -> dict[str, str]: + generated = { ".mendmark/evaluator.py": _evaluator(), ".mendmark/agent-setup.md": _instructions(frameworks), + ".mendmark/SELF-EQUIP.md": _universal_protocol(), ".mendmark/mendmark-ci.yml": _workflow(), ".mendmark/config.json": json.dumps( - {"schema_version": "1.0", "frameworks": list(frameworks)}, + { + "agent_hosts": list(agent_hosts), + "frameworks": list(frameworks), + "schema_version": "1.0", + }, indent=2, sort_keys=True, ) @@ -292,12 +418,21 @@ def _files(frameworks: tuple[str, ...]) -> dict[str, str]: f"# {_MARKER}\nreport.json\nreport.xml\nreport.sarif\n" ), } + if "codex" in agent_hosts: + generated[".agents/skills/mendmark/SKILL.md"] = _agent_skill("codex") + generated[".agents/skills/mendmark/agents/openai.yaml"] = ( + _openai_skill_metadata() + ) + if "claude-code" in agent_hosts: + generated[".claude/skills/mendmark/SKILL.md"] = _agent_skill("claude-code") + return generated def equip_project( project_root: str | Path, *, framework: str = "auto", + agent: str = "auto", dry_run: bool = False, ) -> tuple[tuple[str, ...], tuple[str, ...], tuple[str, ...]]: """Create non-destructive integration assets. @@ -311,7 +446,8 @@ def equip_project( if framework != "auto" and framework not in SUPPORTED_FRAMEWORKS: raise EquipError(f"unsupported framework: {framework}") frameworks = detect_frameworks(root) if framework == "auto" else (framework,) - generated = _files(frameworks) + agent_hosts = resolve_agent_hosts(root, agent) + generated = _files(frameworks, agent_hosts) created: list[str] = [] unchanged: list[str] = [] conflicts: list[str] = [] diff --git a/tests/test_harness_integrations.py b/tests/test_harness_integrations.py index 1ff2664..ccb858a 100644 --- a/tests/test_harness_integrations.py +++ b/tests/test_harness_integrations.py @@ -9,7 +9,13 @@ from mendmark.agent_cases import ToolCallRecord from mendmark.cli import main -from mendmark.equip import EquipError, agent_prompt, detect_frameworks, equip_project +from mendmark.equip import ( + EquipError, + agent_prompt, + detect_agent_hosts, + detect_frameworks, + equip_project, +) from mendmark.integrations import CausalCaseBuilder, HarnessIntegrationError, write_suite from mendmark.integrations.crewai import CrewAIRecorder, case_from_events from mendmark.integrations.langchain import case_from_messages, tool_specs as lc_tools @@ -214,11 +220,11 @@ def test_equip_detects_multiple_harnesses_and_is_idempotent(tmp_path: Path) -> N frameworks, created, unchanged = equip_project(tmp_path) assert frameworks == ("langgraph", "openai-agents") - assert len(created) == 5 + assert len(created) == 6 assert unchanged == () _, created_again, unchanged_again = equip_project(tmp_path) assert created_again == () - assert len(unchanged_again) == 5 + assert len(unchanged_again) == 6 assert "approve_observed=True" in ( tmp_path / ".mendmark" / "agent-setup.md" ).read_text(encoding="utf-8") @@ -241,10 +247,57 @@ def test_equip_dry_run_prompt_and_cli(tmp_path: Path, capsys: pytest.CaptureFixt result = main(["equip", "--print-agent-prompt", "--project-root", str(tmp_path)]) assert result == 0 - assert "mendmark equip --framework auto" in capsys.readouterr().out + assert "mendmark equip --framework auto --agent auto" in capsys.readouterr().out assert ".mendmark/agent-setup.md" in agent_prompt() +def test_equip_generates_native_agent_skills_and_portable_protocol( + tmp_path: Path, +) -> None: + (tmp_path / "AGENTS.md").write_text("# Existing Codex policy\n", encoding="utf-8") + (tmp_path / ".claude").mkdir() + assert detect_agent_hosts(tmp_path) == ("codex", "claude-code") + + _, created, _ = equip_project(tmp_path, framework="generic", agent="auto") + assert ".agents/skills/mendmark/SKILL.md" in created + assert ".agents/skills/mendmark/agents/openai.yaml" in created + assert ".claude/skills/mendmark/SKILL.md" in created + assert (tmp_path / "AGENTS.md").read_text(encoding="utf-8") == ( + "# Existing Codex policy\n" + ) + + codex = (tmp_path / ".agents/skills/mendmark/SKILL.md").read_text( + encoding="utf-8" + ) + claude = (tmp_path / ".claude/skills/mendmark/SKILL.md").read_text( + encoding="utf-8" + ) + portable = (tmp_path / ".mendmark/SELF-EQUIP.md").read_text(encoding="utf-8") + assert "name: mendmark" in codex and "$mendmark" in codex + assert "name: mendmark" in claude and "/mendmark" in claude + assert "agents Mendmark does not recognize" in portable + assert "observed trace into expected behavior silently" in portable + config = json.loads((tmp_path / ".mendmark/config.json").read_text()) + assert config["agent_hosts"] == ["codex", "claude-code"] + metadata = ( + tmp_path / ".agents/skills/mendmark/agents/openai.yaml" + ).read_text(encoding="utf-8") + assert 'display_name: "Mendmark Assurance"' in metadata + assert "allow_implicit_invocation: true" in metadata + + +def test_agent_selector_all_and_targeted_prompts(tmp_path: Path) -> None: + _, created, _ = equip_project(tmp_path, framework="generic", agent="all") + assert ".agents/skills/mendmark/SKILL.md" in created + assert ".claude/skills/mendmark/SKILL.md" in created + assert "--agent codex" in agent_prompt(agent="codex") + assert "$mendmark" in agent_prompt(agent="codex") + assert "--agent claude-code" in agent_prompt(agent="claude-code") + assert "/mendmark" in agent_prompt(agent="claude-code") + with pytest.raises(EquipError, match="unsupported coding agent"): + agent_prompt(agent="unknown") + + def test_equip_refuses_conflicts_and_symlink_escape(tmp_path: Path) -> None: generated = tmp_path / ".mendmark" generated.mkdir()