From de20ee14f0b99809edb32ba31db37e4a2ac63573 Mon Sep 17 00:00:00 2001 From: Daniel Gaskins Date: Mon, 10 Aug 2026 16:46:37 -0700 Subject: [PATCH 1/2] Add agent harness integrations --- .github/workflows/release.yml | 27 +- .github/workflows/tests.yml | 25 ++ CHANGELOG.md | 19 +- README.md | 28 +- docs/assurance.md | 5 + docs/compatibility.md | 7 +- docs/harness-integrations.md | 224 +++++++++++++ pyproject.toml | 2 +- scripts/assure_distribution.py | 48 +++ src/mendmark/__init__.py | 2 +- src/mendmark/cli.py | 40 +++ src/mendmark/equip.py | 345 +++++++++++++++++++ src/mendmark/integrations/__init__.py | 15 + src/mendmark/integrations/common.py | 206 ++++++++++++ src/mendmark/integrations/crewai.py | 135 ++++++++ src/mendmark/integrations/langchain.py | 113 +++++++ src/mendmark/integrations/multi_agent.py | 219 ++++++++++++ src/mendmark/integrations/openai_agents.py | 106 ++++++ tests/test_harness_integrations.py | 373 +++++++++++++++++++++ tests/test_harness_live_compatibility.py | 113 +++++++ 20 files changed, 2046 insertions(+), 6 deletions(-) create mode 100644 docs/harness-integrations.md create mode 100644 src/mendmark/equip.py create mode 100644 src/mendmark/integrations/__init__.py create mode 100644 src/mendmark/integrations/common.py create mode 100644 src/mendmark/integrations/crewai.py create mode 100644 src/mendmark/integrations/langchain.py create mode 100644 src/mendmark/integrations/multi_agent.py create mode 100644 src/mendmark/integrations/openai_agents.py create mode 100644 tests/test_harness_integrations.py create mode 100644 tests/test_harness_live_compatibility.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 26609aa..47d5791 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,8 +44,33 @@ jobs: 'import site; print(site.getsitepackages()[0])') python -m pip_audit --path "$product_site" + harness-assurance: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - profile: langgraph + package: "langchain-core>=1,<2" + - profile: crewai + package: "crewai>=1,<2" + - profile: openai-agents + package: "openai-agents>=0.19,<1" + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.13" + cache: pip + - run: python -m pip install --upgrade pip + - run: python -m pip install -e '.[dev]' '${{ matrix.package }}' + - name: Exercise current public harness objects + env: + MENDMARK_HARNESS: ${{ matrix.profile }} + run: python -m pytest tests/test_harness_live_compatibility.py + build: - needs: assurance + needs: [assurance, harness-assurance] runs-on: ubuntu-latest outputs: version: ${{ steps.package-version.outputs.version }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7b396b8..4aac370 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -92,6 +92,31 @@ jobs: run: python -m pip install -e '.[deepeval,rubric,dev]' - run: python -m pytest tests/test_deepeval.py tests/test_rubric_example.py + harness-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - profile: langgraph + package: "langchain-core>=1,<2" + - profile: crewai + package: "crewai>=1,<2" + - profile: openai-agents + package: "openai-agents>=0.19,<1" + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.13" + cache: pip + - run: python -m pip install --upgrade pip + - run: python -m pip install -e '.[dev]' '${{ matrix.package }}' + - name: Exercise current public harness objects + env: + MENDMARK_HARNESS: ${{ matrix.profile }} + run: python -m pytest tests/test_harness_live_compatibility.py + distribution-assurance: runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index e26415c..60f2ab4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,24 @@ All notable changes to Mendmark are documented here. The project follows Semantic Versioning for its Python and JSON contracts. -## Unreleased +## 0.6.0 - 2026-08-10 + +### Added + +- Dependency-light, public-object adapters for LangChain/LangGraph messages, + CrewAI events, and OpenAI Agents SDK run items, including tool-schema and + side-effect metadata conversion. +- `mendmark equip` for bounded harness detection, non-destructive local + scaffolding, an offline evaluator, an inactive pinned CI template, and a + copyable coding-agent self-equip prompt. +- Live compatibility assurance against current releases of all three harness + paths, plus explicit human approval before observed traces can become + expected behavior. +- A fluent causal-case builder for reviewed multi-agent delegation, parallel + dependencies, tool authority, state changes, results, and aggregation without + hand-authoring schema 2.0 JSON. + +## 0.5.0 - 2026-08-06 ### Added diff --git a/README.md b/README.md index f394f05..f3efeb7 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@

Quick start · + Harnesses · Golden set · Multi-agent · How it works · @@ -63,6 +64,31 @@ mendmark audit-json examples/multi_agent_suite.json \ --evaluator-command "python3 examples/multi_agent_evaluator.py" ``` +## Equip an agent harness + +Mendmark has dependency-light adapters for LangChain/LangGraph, CrewAI, and the +OpenAI Agents SDK. In an existing agent repository: + +```bash +python -m pip install 'mendmark-evals==0.6.0' +mendmark equip --framework auto +``` + +The command detects bounded dependency files and creates a reviewed capture +guide, offline evaluator, and inactive CI template under `.mendmark/`. It does +not edit application code, upload a trace, overwrite existing work, enable CI, +or accept a baseline. + +Want the repository's coding agent to perform the integration? + +```bash +mendmark equip --print-agent-prompt +``` + +See the [agent harness integration guide](docs/harness-integrations.md) for the +direct Python APIs, explicit trace-approval boundary, framework compatibility, +and multi-agent guidance. + ## See the blind spot in two minutes **[▶ Watch the narrated weak-eval demonstration (original v1 fault inventory)](docs/assets/mendmark-weak-eval-demo.mp4)** @@ -346,7 +372,7 @@ and the [ML evaluation card](https://github.com/danielgaskins/mendmark/blob/main ## Current boundary -Version 0.5 is a local, open-source engine. It does not yet provide a hosted +Version 0.6 is a local, open-source engine. It does not yet provide a hosted dashboard, team accounts, remote trace ingestion, or a secrets service. The planned control plane is described in [the product design](https://github.com/danielgaskins/mendmark/blob/main/docs/product.md). diff --git a/docs/assurance.md b/docs/assurance.md index 2073e1b..060a8d6 100644 --- a/docs/assurance.md +++ b/docs/assurance.md @@ -10,6 +10,11 @@ The automated assurance suite checks that: - `mendmark --version`, `--help`, packaged tasks, packaged schemas, and the complete single- and multi-agent JSON audit journeys work from that clean installation. +- Harness converters are tested with privacy-safe facsimiles and current public + objects from LangChain Core, CrewAI, and OpenAI Agents SDK. `mendmark equip` + is tested for idempotency, bounded detection, conflict refusal, symlink + containment, explicit expectation approval, and a complete generated + evaluator journey. - Reports, console output, JUnit, and SARIF do not expose canary values placed in prompts, answers, metadata, tags, tool arguments, tool outputs, or descriptions. - Repeated audits preserve mutation IDs, ordering, decisions, JUnit, SARIF, and diff --git a/docs/compatibility.md b/docs/compatibility.md index 7e9eaa1..f3d2658 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -18,9 +18,14 @@ Mendmark 0.x is evolving, but automation still needs predictable contracts. - Historical benchmarks select immutable `agent-eval-v1` or `multi-agent-v1` mutation profiles. Ordinary audits use `current`; adding a new operator never rewrites an older golden-set contract. +- Harness adapters are dependency-light and use documented public object + fields. Hosted CI exercises LangChain Core 1.5.3, CrewAI 1.15.14, and OpenAI + Agents SDK 0.19.4 as the 2026-08-10 compatibility snapshot. Newer supported + releases are tested through the latest-version harness matrix; Mendmark does + not install or pin a harness in customer environments. - A custom operator name is globally unique within an audit and is part of the customer's accepted baseline contract. -- `audit`, `audit-json`, `prepare`, `grade`, and `show` exit with 0 for success, +- `audit`, `audit-json`, `equip`, `prepare`, `grade`, and `show` exit with 0 for success, 1 for a failed product gate, and 2 for invalid input or infrastructure failure. The JSON Schemas under `mendmark/schemas` are the machine-readable contract. diff --git a/docs/harness-integrations.md b/docs/harness-integrations.md new file mode 100644 index 0000000..752331a --- /dev/null +++ b/docs/harness-integrations.md @@ -0,0 +1,224 @@ +# Agent harness integrations + +Mendmark converts public harness traces into its stable local JSON contract. It +does not import a harness at package import time, constrain the harness version, +or send trace content to a hosted service. + +## Supported first-class paths + +The initial targets were selected from active Python agent projects on +2026-08-10. GitHub stars are an imperfect adoption signal, so activity and a +stable tool/trace interface were considered as well. + +| Harness path | Adoption signal at selection | Mendmark input | +| --- | ---: | --- | +| LangChain / LangGraph | 143,910 / 39,385 GitHub stars | `AIMessage` and `ToolMessage` history | +| CrewAI | 56,908 GitHub stars | Public tool-usage and completion events | +| OpenAI Agents SDK | 28,542 GitHub stars | Public `RunResult.new_items` | + +The project sources are the [LangChain](https://github.com/langchain-ai/langchain), +[LangGraph](https://github.com/langchain-ai/langgraph), +[CrewAI](https://github.com/crewAIInc/crewAI), and +[OpenAI Agents SDK](https://github.com/openai/openai-agents-python) +repositories. Mendmark's compatibility job exercises current real objects from +each framework instead of relying only on local facsimiles. + +These paths follow the frameworks' documented structures: LangGraph exposes +tool lifecycle events and correlated message tool calls, CrewAI exposes an +event bus with tool-completion events, and the OpenAI Agents SDK exposes public +run items and trace processors. See the official [LangGraph event-streaming +guide](https://docs.langchain.com/oss/python/langgraph/event-streaming), +[CrewAI documentation](https://docs.crewai.com/), and [OpenAI Agents SDK tracing +guide](https://openai.github.io/openai-agents-python/tracing/). + +## One-command setup + +From the agent application repository: + +```bash +python -m pip install 'mendmark-evals==0.6.0' +mendmark equip --framework auto +``` + +Detection reads only bounded dependency files; it does not import or execute the +application. The command creates five reviewable files under `.mendmark/`: + +- `agent-setup.md`: harness-specific capture code and acceptance criteria. +- `evaluator.py`: a deterministic offline evaluator for reviewed snapshots. +- `mendmark-ci.yml`: an inactive, pinned CI template. +- `config.json`: detected integration metadata. +- `.gitignore`: excludes generated report artifacts. + +It never edits application code, activates CI, overwrites an existing differing +file, or accepts a baseline. Running it again is idempotent. Use `--dry-run` to +preview its targets. + +## Let a coding agent self-equip the repository + +Print a prompt that works with repository-capable coding agents: + +```bash +mendmark equip --print-agent-prompt +``` + +The prompt instructs the agent to run detection, read the generated setup file, +capture a real tool-using case, pass the audit, and meet every review criterion. +It explicitly forbids uploading trace content or silently treating observed +production behavior as correct. + +The short prompt can also be copied directly: + +> Run `mendmark equip --framework auto`, read `.mendmark/agent-setup.md` +> completely, integrate the detected harness, capture at least one reviewed +> tool-using case, run the local audit, and satisfy every acceptance criterion +> before enabling CI. Do not upload trace content or approve observed behavior +> without human review. + +## Direct Python API + +### LangChain and LangGraph + +```python +from mendmark.integrations import write_suite +from mendmark.integrations.langchain import case_from_messages, tool_specs + +case = case_from_messages( + result["messages"], + case_id="refund-reviewed", + input=test_input, + expected_output=expected_output, + approve_observed=True, +) +write_suite( + ".mendmark/suite.json", + [case], + tool_specs(tools, side_effecting=["refund_order"]), +) +``` + +Tool calls are joined to `ToolMessage` results by call ID. Both framework +objects and their documented dictionary forms are supported. + +### OpenAI Agents SDK + +```python +from mendmark.integrations import write_suite +from mendmark.integrations.openai_agents import case_from_result, tool_specs + +result = Runner.run_sync(agent, test_input) +case = case_from_result( + result, + case_id="refund-reviewed", + input=test_input, + expected_output=expected_output, + approve_observed=True, +) +write_suite( + ".mendmark/suite.json", + [case], + tool_specs(agent.tools, side_effecting=["refund_order"]), +) +``` + +The adapter pairs public `ToolCallItem` and `ToolCallOutputItem` objects using +their call ID. No OpenAI API call is made by the adapter. + +### CrewAI + +```python +from mendmark.integrations import write_suite +from mendmark.integrations.crewai import CrewAIRecorder, tool_specs + +recorder = CrewAIRecorder().attach() +result = crew.kickoff(inputs=test_inputs) +case = recorder.case( + case_id="refund-reviewed", + input=test_input, + expected_output=expected_output, + approve_observed=True, +) +write_suite( + ".mendmark/suite.json", + [case], + tool_specs(all_tools, side_effecting=["refund_order"]), +) +``` + +Attach one recorder in an isolated capture/test process and clear it between +cases. CrewAI's event bus is process-global, so concurrent crews should capture +in separate processes or pass their already-separated event sequences directly +to `case_from_events`. + +## Approval boundary + +All converters require `expected_tools` unless the caller explicitly supplies +`approve_observed=True`. That escape hatch exists for a one-off, human-reviewed +snapshot. It must not be applied automatically to arbitrary production traces. + +For maintained capture code, build and pass an explicit sequence of +`ToolCallRecord` expectations. Mark side-effecting tools explicitly: duplicate +payment, refund, email, write, and deployment calls are otherwise treated as +ordinary calls and receive weaker mutation severity. + +The adapters above produce the stable ordered-trace schema `1.0`. If agent +identity, delegation, parallel branches, shared state, or causal dependencies +matter, use `CausalCaseBuilder`, which emits Mendmark's native [multi-agent +schema 2.0](multi-agent.md). Do not flatten a coordination test and then claim +coverage of coordination failures. + +```python +from mendmark.integrations import CausalCaseBuilder, write_suite + +case = ( + CausalCaseBuilder( + case_id="parallel-review", + input=test_input, + root_agent_id="supervisor", + ) + .agent("supervisor") + .agent("billing", allowed_tools=["lookup_order", "refund_order"]) + .agent("risk", allowed_tools=["assess_risk"]) + .delegation("delegate-billing", "supervisor", "billing") + .delegation("delegate-risk", "supervisor", "risk") + .tool_call( + "lookup", + "billing", + "lookup_order", + input_parameters={"order_id": "104"}, + output={"status": "paid"}, + depends_on=["delegate-billing"], + ) + .result("billing-result", "billing", "supervisor", depends_on=["lookup"]) + .result("risk-result", "risk", "supervisor", depends_on=["delegate-risk"]) + .message( + "aggregate", + "supervisor", + depends_on=["billing-result", "risk-result"], + ) + .build( + actual_output=actual_output, + expected_output=expected_output, + approve_observed=True, + ) +) +write_suite(".mendmark/multi-agent-suite.json", [case], tool_contracts) +``` + +The builder validates agent identities, tool authority, unique event IDs, +dependency targets, and acyclicity. Dependencies remain explicit; it never +turns unreliable wall-clock ordering into a causal claim. + +## Run and activate CI + +```bash +mendmark audit-json .mendmark/suite.json \ + --evaluator-command "python .mendmark/evaluator.py" \ + --output .mendmark/report.json \ + --junit .mendmark/report.xml \ + --sarif .mendmark/report.sarif +``` + +Review survivors and create an accepted baseline only from a passing audit. +Then copy `.mendmark/mendmark-ci.yml` to +`.github/workflows/mendmark.yml`, open a pull request, and require its +`agent-eval-assurance` check after the first successful run. diff --git a/pyproject.toml b/pyproject.toml index e6457b2..34e4cc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mendmark-evals" -version = "0.5.0" +version = "0.6.0" description = "Mutation testing for agent evaluation suites" readme = "README.md" requires-python = ">=3.10" diff --git a/scripts/assure_distribution.py b/scripts/assure_distribution.py index b8eb4e0..c7ea910 100644 --- a/scripts/assure_distribution.py +++ b/scripts/assure_distribution.py @@ -94,6 +94,7 @@ def main() -> int: ("scripts", "assure_distribution.py"), (".github", "allowed_signers"), ("docs", "assurance.md"), + ("docs", "harness-integrations.md"), ("golden", "agent-eval-v1", "manifest.json"), ("golden", "agent-eval-v1", "suite.json"), ("golden", "agent-eval-v1", "results.json"), @@ -119,6 +120,53 @@ def main() -> int: help_text = run([str(mendmark), "--help"], cwd=workspace, env=clean_env) if "mutation-test" not in help_text.lower() or "audit-json" not in help_text: raise RuntimeError("installed CLI help is missing the primary product journey") + prompt = run( + [str(mendmark), "equip", "--print-agent-prompt"], + cwd=workspace, + env=clean_env, + ) + if "agent-setup.md" not in prompt or "human review" not in prompt: + raise RuntimeError("installed wheel did not expose safe agent self-equip guidance") + integration_api = run( + [ + str(python), + "-c", + ( + "from mendmark.integrations import CausalCaseBuilder, write_suite; " + "from mendmark.integrations.langchain import case_from_messages; " + "from mendmark.integrations.crewai import CrewAIRecorder; " + "from mendmark.integrations.openai_agents import case_from_result; " + "print('harness adapters ready')" + ), + ], + cwd=workspace, + env=clean_env, + ) + if integration_api.strip() != "harness adapters ready": + raise RuntimeError("installed wheel omitted the harness integration API") + equip_output = run( + [ + str(mendmark), + "equip", + "--framework", + "langgraph", + "--project-root", + str(workspace), + ], + cwd=workspace, + env=clean_env, + ) + if "Created: .mendmark/evaluator.py" not in equip_output: + raise RuntimeError("installed wheel did not scaffold a harness integration") + for generated in ( + "evaluator.py", + "agent-setup.md", + "mendmark-ci.yml", + "config.json", + ".gitignore", + ): + if not (workspace / ".mendmark" / generated).is_file(): + raise RuntimeError(f"installed wheel omitted equip asset: {generated}") tasks = run([str(mendmark), "tasks"], cwd=workspace, env=clean_env) if len([line for line in tasks.splitlines() if line.strip()]) != 5: raise RuntimeError("installed wheel did not expose all five ML integrity tasks") diff --git a/src/mendmark/__init__.py b/src/mendmark/__init__.py index d7b84fa..ff14598 100644 --- a/src/mendmark/__init__.py +++ b/src/mendmark/__init__.py @@ -2,7 +2,7 @@ from .agent_cases import AgentCase, AgentEvent, AgentSpec, ToolCallRecord, ToolSpec -__version__ = "0.5.0" +__version__ = "0.6.0" __all__ = [ "AgentCase", diff --git a/src/mendmark/cli.py b/src/mendmark/cli.py index 68f6c00..d86c685 100644 --- a/src/mendmark/cli.py +++ b/src/mendmark/cli.py @@ -82,6 +82,25 @@ def build_parser() -> argparse.ArgumentParser: subparsers.add_parser("tasks", help="list and validate available tasks") + equip = subparsers.add_parser( + "equip", help="scaffold a reviewed agent-harness integration" + ) + equip.add_argument( + "--framework", + choices=("auto", "langgraph", "crewai", "openai-agents", "generic"), + default="auto", + help="detect from dependency files or select an integration (default: auto)", + ) + equip.add_argument("--project-root", default=".", help="project to equip") + equip.add_argument( + "--dry-run", action="store_true", help="show files without writing them" + ) + equip.add_argument( + "--print-agent-prompt", + action="store_true", + help="print a prompt for a repository coding agent without writing files", + ) + prepare = subparsers.add_parser("prepare", help="create a public agent workspace") prepare.add_argument("task_id") prepare.add_argument("--runs-root", default="runs") @@ -426,6 +445,27 @@ def main(argv: list[str] | None = None) -> int: ) return 0 + if args.command == "equip": + from .equip import agent_prompt, equip_project + + if args.print_agent_prompt: + print(agent_prompt(args.project_root)) + return 0 + frameworks, created, unchanged = equip_project( + args.project_root, + framework=args.framework, + dry_run=args.dry_run, + ) + action = "Would create" if args.dry_run else "Created" + print("Detected: " + ", ".join(frameworks)) + for path in created: + print(f"{action}: {path}") + for path in unchanged: + print(f"Unchanged: {path}") + if not args.dry_run: + print("Next: ask your coding agent to complete .mendmark/agent-setup.md") + return 0 + if args.command == "prepare": task = _task(tasks_root, args.task_id) run_dir = prepare_run( diff --git a/src/mendmark/equip.py b/src/mendmark/equip.py new file mode 100644 index 0000000..236150a --- /dev/null +++ b/src/mendmark/equip.py @@ -0,0 +1,345 @@ +"""Safe, review-first scaffolding for agent harness integrations.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Final + +from . import __version__ + + +class EquipError(ValueError): + """Raised when a project cannot be equipped without unsafe assumptions.""" + + +SUPPORTED_FRAMEWORKS: Final = ("langgraph", "crewai", "openai-agents", "generic") +_MARKER = "Generated by Mendmark equip; safe to regenerate." +_DEPENDENCY_FILES = ( + "pyproject.toml", + "requirements.txt", + "requirements-dev.txt", + "uv.lock", + "poetry.lock", + "pdm.lock", +) + + +def detect_frameworks(project_root: Path) -> tuple[str, ...]: + """Detect supported harness names from bounded dependency-file reads.""" + corpus: list[str] = [] + for name in _DEPENDENCY_FILES: + path = project_root / name + if path.is_file() and path.stat().st_size <= 2_000_000: + corpus.append(path.read_text(encoding="utf-8", errors="replace").lower()) + text = "\n".join(corpus) + found: list[str] = [] + if "langgraph" in text or "langchain" in text: + found.append("langgraph") + if "crewai" in text: + found.append("crewai") + if "openai-agents" in text or "openai_agents" in text: + found.append("openai-agents") + return tuple(found or ("generic",)) + + +def agent_prompt(project_root: str = ".") -> str: + """Return the short prompt a user can give any repository coding agent.""" + return ( + f"In {project_root}, run `mendmark equip --framework auto`, then read " + "`.mendmark/agent-setup.md` completely. Integrate the detected harness, " + "capture at least one reviewed tool-using case, run the local audit, and " + "satisfy every acceptance criterion before enabling the generated CI " + "workflow. Do not upload trace content or approve observed behavior " + "without human review." + ) + + +def _evaluator() -> str: + return f'''#!/usr/bin/env python3 +"""{_MARKER} + +Offline exact evaluator for reviewed Mendmark trace snapshots. +""" +from __future__ import annotations + +import json +import sys + + +def main() -> int: + request = json.load(sys.stdin) + evaluations = [] + for requested in request["evaluations"]: + case = requested["case"] + tools_match = case.get("tools_called", []) == case.get("expected_tools", []) + output_matches = ( + case.get("expected_output") is None + or case.get("actual_output") == case.get("expected_output") + ) + events_match = case.get("events", []) == case.get("expected_events", []) + is_multi = "events" in case or "expected_events" in case + results = [ + {{ + "name": "Reviewed tool trace", + "score": float(tools_match), + "passed": tools_match, + "reason": "Actual tool calls match the reviewed expected calls.", + }}, + {{ + "name": "Reviewed outcome", + "score": float(output_matches), + "passed": output_matches, + "reason": "The outcome matches when an expected outcome is configured.", + }}, + ] + if is_multi: + results.append( + {{ + "name": "Reviewed coordination graph", + "score": float(events_match), + "passed": events_match, + "reason": "Actual causal events match the reviewed graph.", + }} + ) + evaluations.append( + {{"evaluation_id": requested["evaluation_id"], "results": results}} + ) + json.dump( + {{"schema_version": request["schema_version"], "evaluations": evaluations}}, + sys.stdout, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +''' + + +def _adapter_example(framework: str) -> str: + if framework == "langgraph": + return '''```python +from mendmark.integrations.langchain import case_from_messages, tool_specs +from mendmark.integrations import write_suite + +# `result` is the graph/agent state and `tools` is the configured tool list. +case = case_from_messages( + result["messages"], + case_id="reviewed-order-flow", + input=test_input, + expected_output=reviewed_expected_output, + approve_observed=True, # Only in the one-off capture after human review. +) +write_suite( + ".mendmark/suite.json", + [case], + tool_specs(tools, side_effecting=["refund_order"]), +) +```''' + if framework == "openai-agents": + return '''```python +from mendmark.integrations.openai_agents import case_from_result, tool_specs +from mendmark.integrations import write_suite + +# `result` is returned by Runner.run/run_sync and `agent` is the configured Agent. +case = case_from_result( + result, + case_id="reviewed-order-flow", + input=test_input, + expected_output=reviewed_expected_output, + approve_observed=True, # Only in the one-off capture after human review. +) +write_suite( + ".mendmark/suite.json", + [case], + tool_specs(agent.tools, side_effecting=["refund_order"]), +) +```''' + if framework == "crewai": + return '''```python +from mendmark.integrations.crewai import CrewAIRecorder, tool_specs +from mendmark.integrations import write_suite + +recorder = CrewAIRecorder().attach() # Attach before crew.kickoff(). +result = crew.kickoff(inputs=test_inputs) +case = recorder.case( + case_id="reviewed-order-flow", + input=test_input, + expected_output=reviewed_expected_output, + approve_observed=True, # Only in the one-off capture after human review. +) +write_suite( + ".mendmark/suite.json", + [case], + tool_specs(all_tools, side_effecting=["refund_order"]), +) +```''' + return '''Use the framework-neutral JSON contract documented at +`https://github.com/danielgaskins/mendmark/blob/main/docs/json-adapter.md`. +Create a reviewed suite at `.mendmark/suite.json`; do not send trace content to +Mendmark or any hosted service.''' + + +def _instructions(frameworks: tuple[str, ...]) -> str: + detected = ", ".join(frameworks) + examples = "\n\n".join( + f"### {framework}\n\n{_adapter_example(framework)}" for framework in frameworks + ) + return f''' +# Equip this repository with Mendmark + +Detected integration: **{detected}**. + +Mendmark mutation-tests the evaluator, not the agent. Keep all trace content +local. The generated exact evaluator is a safe starting point for reviewed +snapshots; replace or supplement it with the project's real evaluator when +available. + +## Capture a reviewed case + +{examples} + +`approve_observed=True` is deliberately explicit. Inspect tool names, +arguments, results, ordering, side effects, and the expected outcome before +running the capture. Prefer passing an explicit `expected_tools` sequence in a +maintained capture test. Never turn arbitrary production traces into expected +behavior automatically. + +## Run locally + +```bash +mendmark audit-json .mendmark/suite.json \\ + --evaluator-command "python .mendmark/evaluator.py" \\ + --output .mendmark/report.json \\ + --junit .mendmark/report.xml \\ + --sarif .mendmark/report.sarif +``` + +Review every survivor. When the suite is accepted, create its baseline once +with `--write-baseline`, review `.mendmark/baseline.json`, and commit it. Copy +`.mendmark/mendmark-ci.yml` to `.github/workflows/mendmark.yml` only after the +local command passes. + +## Acceptance criteria + +- [ ] At least one consequential, tool-using case is captured. +- [ ] A human reviewed expected calls, arguments, outputs, order, and outcome. +- [ ] Every side-effecting tool is declared with `side_effecting=True`. +- [ ] The original case passes before mutation. +- [ ] Critical survivors were fixed or explicitly documented and accepted. +- [ ] No prompt, arguments, result content, credentials, or PII appears in the report. +- [ ] The baseline was generated only from a passing audit and reviewed in diff. +- [ ] The generated CI workflow passes on a pull request before becoming required. + +For native multi-agent coordination mutation testing, model the causal graph +with `mendmark.integrations.CausalCaseBuilder` instead of flattening it into an +ordered tool trace. See `docs/multi-agent.md` in Mendmark's repository. +''' + + +def _workflow() -> str: + return f'''# {_MARKER} +name: mendmark + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + security-events: write + +jobs: + agent-eval-assurance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.13" + cache: pip + - run: python -m pip install . 'mendmark-evals=={__version__}' + - name: Mutation-test agent evals + run: >- + mendmark audit-json .mendmark/suite.json + --evaluator-command "python .mendmark/evaluator.py" + --baseline .mendmark/baseline.json + --output .mendmark/report.json + --junit .mendmark/report.xml + --sarif .mendmark/report.sarif + --maximum-mutants 10000 + - uses: github/codeql-action/upload-sarif@b374143c1149a911a7c303ebe6abb5c19149a01c # v3 + if: always() && hashFiles('.mendmark/report.sarif') != '' + continue-on-error: true + with: + sarif_file: .mendmark/report.sarif +''' + + +def _files(frameworks: tuple[str, ...]) -> dict[str, str]: + return { + ".mendmark/evaluator.py": _evaluator(), + ".mendmark/agent-setup.md": _instructions(frameworks), + ".mendmark/mendmark-ci.yml": _workflow(), + ".mendmark/config.json": json.dumps( + {"schema_version": "1.0", "frameworks": list(frameworks)}, + indent=2, + sort_keys=True, + ) + + "\n", + ".mendmark/.gitignore": ( + f"# {_MARKER}\nreport.json\nreport.xml\nreport.sarif\n" + ), + } + + +def equip_project( + project_root: str | Path, + *, + framework: str = "auto", + dry_run: bool = False, +) -> tuple[tuple[str, ...], tuple[str, ...], tuple[str, ...]]: + """Create non-destructive integration assets. + + Returns ``(frameworks, created, unchanged)``. Existing differing files are + never overwritten; the user must review and resolve them manually. + """ + root = Path(project_root).expanduser().resolve() + if not root.is_dir(): + raise EquipError(f"project root is not a directory: {root}") + if framework != "auto" and framework not in SUPPORTED_FRAMEWORKS: + raise EquipError(f"unsupported framework: {framework}") + frameworks = detect_frameworks(root) if framework == "auto" else (framework,) + generated = _files(frameworks) + created: list[str] = [] + unchanged: list[str] = [] + conflicts: list[str] = [] + for relative, content in generated.items(): + destination = root / relative + resolved_destination = destination.resolve() + if root not in resolved_destination.parents: + conflicts.append(relative) + continue + if destination.exists(): + if destination.is_symlink() or not destination.is_file(): + conflicts.append(relative) + elif destination.read_text(encoding="utf-8") == content: + unchanged.append(relative) + else: + conflicts.append(relative) + else: + created.append(relative) + if conflicts: + raise EquipError( + "refusing to overwrite existing integration file(s): " + + ", ".join(conflicts) + ) + if not dry_run: + for relative in created: + destination = root / relative + destination.parent.mkdir(parents=True, exist_ok=True) + if root not in destination.resolve().parents: + raise EquipError(f"integration path escapes project root: {relative}") + destination.write_text(generated[relative], encoding="utf-8") + return frameworks, tuple(created), tuple(unchanged) diff --git a/src/mendmark/integrations/__init__.py b/src/mendmark/integrations/__init__.py new file mode 100644 index 0000000..2989cd1 --- /dev/null +++ b/src/mendmark/integrations/__init__.py @@ -0,0 +1,15 @@ +"""Dependency-light adapters for popular Python agent harnesses. + +The adapters use public, duck-typed harness objects so importing Mendmark never +imports or constrains the customer's agent framework. +""" + +from .common import HarnessIntegrationError, suite_to_json, write_suite +from .multi_agent import CausalCaseBuilder + +__all__ = [ + "HarnessIntegrationError", + "CausalCaseBuilder", + "suite_to_json", + "write_suite", +] diff --git a/src/mendmark/integrations/common.py b/src/mendmark/integrations/common.py new file mode 100644 index 0000000..a6f42b1 --- /dev/null +++ b/src/mendmark/integrations/common.py @@ -0,0 +1,206 @@ +"""Shared helpers for converting harness traces to Mendmark suites.""" + +from __future__ import annotations + +import json +import os +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import asdict, is_dataclass +from pathlib import Path +from typing import Any + +from ..agent_cases import AgentCase, ToolCallRecord, ToolSpec +from ..json_adapter import case_to_json, load_json_suite + + +class HarnessIntegrationError(ValueError): + """Raised when a harness object cannot be converted without guessing.""" + + +def member(value: Any, name: str, default: Any = None) -> Any: + """Read a public field from either an object or mapping.""" + if isinstance(value, Mapping): + return value.get(name, default) + return getattr(value, name, default) + + +def json_value(value: Any, *, _depth: int = 0) -> Any: + """Convert common harness/Pydantic values to deterministic JSON values.""" + if _depth > 32: + raise HarnessIntegrationError("harness value nesting exceeds 32 levels") + if value is None or isinstance(value, (str, int, float, bool)): + return value + if hasattr(value, "model_dump"): + value = value.model_dump(mode="json") + elif is_dataclass(value) and not isinstance(value, type): + value = asdict(value) + if isinstance(value, Mapping): + return { + str(key): json_value(item, _depth=_depth + 1) + for key, item in value.items() + } + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return [json_value(item, _depth=_depth + 1) for item in value] + return str(value) + + +def json_arguments(value: Any, *, location: str) -> dict[str, Any]: + """Normalize the JSON-object arguments used by agent tool calls.""" + if value is None or value == "": + return {} + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError as error: + raise HarnessIntegrationError( + f"{location} contains invalid JSON arguments" + ) from error + normalized = json_value(value) + if not isinstance(normalized, dict): + raise HarnessIntegrationError(f"{location} arguments must be a JSON object") + return normalized + + +def text_output(value: Any) -> str: + """Normalize a harness final output while preserving structured content.""" + if value is None: + return "" + if isinstance(value, str): + return value + normalized = json_value(value) + if isinstance(normalized, str): + return normalized + return json.dumps(normalized, sort_keys=True, separators=(",", ":")) + + +def schema_from_tool(tool: Any, *, framework: str) -> dict[str, Any]: + """Extract a public JSON input schema from common harness tool objects.""" + for attribute in ("params_json_schema", "args_schema", "tool_call_schema"): + candidate = member(tool, attribute) + if candidate is None: + continue + if callable(candidate) and not isinstance(candidate, type): + candidate = candidate() + if isinstance(candidate, type): + if hasattr(candidate, "model_json_schema"): + candidate = candidate.model_json_schema() + elif hasattr(candidate, "schema"): + candidate = candidate.schema() + if hasattr(candidate, "model_json_schema"): + candidate = candidate.model_json_schema() + if isinstance(candidate, Mapping): + return json_value(candidate) + raise HarnessIntegrationError( + f"{framework} tool {member(tool, 'name', '')!r} does not expose " + "a JSON input schema" + ) + + +def tool_spec_from_object( + tool: Any, + *, + framework: str, + side_effecting: bool = False, +) -> ToolSpec: + name = member(tool, "name") or member(tool, "tool_name") + if not isinstance(name, str) or not name.strip(): + raise HarnessIntegrationError(f"{framework} tool has no stable name") + description = member(tool, "description") + if description is not None: + description = str(description).strip() or None + return ToolSpec( + name=name, + input_schema=schema_from_tool(tool, framework=framework), + description=description, + side_effecting=side_effecting, + ) + + +def approved_expected_calls( + observed: Sequence[ToolCallRecord], + expected: Sequence[ToolCallRecord] | None, + *, + approve_observed: bool, +) -> tuple[ToolCallRecord, ...]: + if expected is not None: + return tuple(expected) + if approve_observed: + return tuple(observed) + raise HarnessIntegrationError( + "expected tool calls are required; after reviewing the trace, pass " + "expected_tools=... or explicitly set approve_observed=True" + ) + + +def _tool_to_json(tool: ToolSpec) -> dict[str, Any]: + value: dict[str, Any] = { + "name": tool.name, + "input_schema": tool.input_schema, + "side_effecting": tool.side_effecting, + } + if tool.description is not None: + value["description"] = tool.description + return value + + +def suite_to_json( + cases: Sequence[AgentCase], + tools: Sequence[ToolSpec], + *, + policy: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Create a JSON suite from converted cases without writing to disk.""" + if not cases: + raise HarnessIntegrationError("a Mendmark suite needs at least one case") + multi_agent = any(case.is_multi_agent for case in cases) + if multi_agent and not all(case.is_multi_agent for case in cases): + raise HarnessIntegrationError( + "single-agent and multi-agent cases must be written to separate suites" + ) + return { + "schema_version": "2.0" if multi_agent else "1.0", + "policy": dict( + policy + or { + "minimum_kill_rate": 0.9, + "fail_on_critical_survivor": True, + "fail_on_untested_tools": True, + "fail_on_tool_contract_issues": True, + "fail_on_regression": True, + } + ), + "tools": [_tool_to_json(tool) for tool in tools], + "cases": [case_to_json(case) for case in cases], + } + + +def write_suite( + path: str | Path, + cases: Sequence[AgentCase], + tools: Sequence[ToolSpec], + *, + policy: Mapping[str, Any] | None = None, + overwrite: bool = False, +) -> Path: + """Validate and atomically write a harness-derived Mendmark suite.""" + destination = Path(path).expanduser().resolve() + if destination.exists() and not overwrite: + raise HarnessIntegrationError( + f"refusing to overwrite existing suite: {destination}" + ) + destination.parent.mkdir(parents=True, exist_ok=True) + payload = suite_to_json(cases, tools, policy=policy) + descriptor, raw_temp = tempfile.mkstemp( + prefix=f".{destination.name}.", suffix=".tmp", dir=destination.parent + ) + temporary = Path(raw_temp) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, sort_keys=True) + handle.write("\n") + load_json_suite(temporary) + temporary.replace(destination) + finally: + temporary.unlink(missing_ok=True) + return destination diff --git a/src/mendmark/integrations/crewai.py b/src/mendmark/integrations/crewai.py new file mode 100644 index 0000000..ce85f81 --- /dev/null +++ b/src/mendmark/integrations/crewai.py @@ -0,0 +1,135 @@ +"""CrewAI event adapters and a one-line event collector.""" + +from __future__ import annotations + +import threading +from collections.abc import Sequence +from typing import Any + +from ..agent_cases import AgentCase, ToolCallRecord, ToolSpec +from .common import ( + HarnessIntegrationError, + approved_expected_calls, + json_arguments, + json_value, + member, + text_output, + tool_spec_from_object, +) + + +def case_from_events( + events: Sequence[Any], + *, + case_id: str, + input: str, + expected_output: str | None, + expected_tools: Sequence[ToolCallRecord] | None = None, + approve_observed: bool = False, + tags: Sequence[str] = (), +) -> AgentCase: + """Convert CrewAI public tool-usage and agent-completion events.""" + calls: list[ToolCallRecord] = [] + final_output = "" + for event in events: + event_type = member(event, "type") + if event_type == "tool_usage_finished": + name = member(event, "tool_name") + if not isinstance(name, str) or not name: + raise HarnessIntegrationError("CrewAI tool event has no stable name") + calls.append( + ToolCallRecord( + name=name, + input_parameters=json_arguments( + member(event, "tool_args", {}), + location=f"CrewAI tool call {name!r}", + ), + output=json_value(member(event, "output")), + ) + ) + elif event_type == "agent_execution_completed": + final_output = text_output(member(event, "output")) + elif event_type == "crew_kickoff_completed": + output = member(event, "output") + final_output = text_output(member(output, "raw", output)) + if not calls: + raise HarnessIntegrationError("CrewAI event collection contains no tool calls") + expected = approved_expected_calls( + calls, expected_tools, approve_observed=approve_observed + ) + return AgentCase( + case_id=case_id, + input=input, + actual_output=final_output, + expected_output=expected_output, + tools_called=tuple(calls), + expected_tools=expected, + tags=tuple(tags), + metadata={"harness": "crewai"}, + ) + + +class CrewAIRecorder: + """Thread-safe collector registered against CrewAI's public event bus.""" + + def __init__(self) -> None: + self._events: list[Any] = [] + self._lock = threading.Lock() + self._attached = False + + @property + def events(self) -> tuple[Any, ...]: + with self._lock: + return tuple(self._events) + + def clear(self) -> None: + with self._lock: + self._events.clear() + + def record(self, event: Any) -> None: + with self._lock: + self._events.append(event) + + def attach(self) -> "CrewAIRecorder": + """Subscribe once to stable CrewAI completion event classes.""" + if self._attached: + return self + try: + from crewai.events import crewai_event_bus + from crewai.events.types.agent_events import AgentExecutionCompletedEvent + from crewai.events.types.crew_events import CrewKickoffCompletedEvent + from crewai.events.types.tool_usage_events import ToolUsageFinishedEvent + except ImportError as error: + raise HarnessIntegrationError( + "CrewAI is not installed; install it in the agent application's environment" + ) from error + + for event_class in ( + ToolUsageFinishedEvent, + AgentExecutionCompletedEvent, + CrewKickoffCompletedEvent, + ): + def handler(_source: Any, event: Any, *, recorder: CrewAIRecorder = self) -> None: + recorder.record(event) + + crewai_event_bus.on(event_class)(handler) + self._attached = True + return self + + def case(self, **kwargs: Any) -> AgentCase: + return case_from_events(self.events, **kwargs) + + +def tool_specs( + tools: Sequence[Any], *, side_effecting: Sequence[str] = () +) -> tuple[ToolSpec, ...]: + """Convert CrewAI BaseTool-compatible objects.""" + side_effect_names = set(side_effecting) + return tuple( + tool_spec_from_object( + tool, + framework="CrewAI", + side_effecting=str(member(tool, "name")) in side_effect_names, + ) + for tool in tools + ) diff --git a/src/mendmark/integrations/langchain.py b/src/mendmark/integrations/langchain.py new file mode 100644 index 0000000..270ec71 --- /dev/null +++ b/src/mendmark/integrations/langchain.py @@ -0,0 +1,113 @@ +"""LangChain and LangGraph message adapters.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from ..agent_cases import AgentCase, ToolCallRecord, ToolSpec +from .common import ( + HarnessIntegrationError, + approved_expected_calls, + json_arguments, + json_value, + member, + text_output, + tool_spec_from_object, +) + + +def _message_kind(message: Any) -> str: + return str(member(message, "type", member(message, "role", ""))).lower() + + +def _call_id(value: Any) -> str | None: + identifier = member(value, "id") or member(value, "tool_call_id") + return str(identifier) if identifier is not None else None + + +def case_from_messages( + messages: Sequence[Any], + *, + case_id: str, + input: str, + expected_output: str | None, + expected_tools: Sequence[ToolCallRecord] | None = None, + approve_observed: bool = False, + tags: Sequence[str] = (), +) -> AgentCase: + """Convert LangChain/LangGraph AIMessage and ToolMessage history. + + Tool calls are correlated with ToolMessage results by their public call IDs. + Dict-form messages using ``role`` are supported alongside message objects. + """ + outputs: dict[str, Any] = {} + for message in messages: + kind = _message_kind(message) + if kind in {"tool", "toolmessage"}: + identifier = member(message, "tool_call_id") + if identifier is not None: + output = member(message, "artifact", None) + if output is None: + output = member(message, "content") + outputs[str(identifier)] = json_value(output) + + calls: list[ToolCallRecord] = [] + final_output = "" + for message in messages: + kind = _message_kind(message) + raw_calls = member(message, "tool_calls", ()) or () + if kind in {"ai", "assistant", "aimessage"}: + content = member(message, "content", "") + if content and not raw_calls: + final_output = text_output(content) + for index, raw_call in enumerate(raw_calls): + function = member(raw_call, "function", {}) or {} + name = member(raw_call, "name") or member(function, "name") + if not isinstance(name, str) or not name: + raise HarnessIntegrationError( + f"LangChain tool call {index} has no stable name" + ) + arguments = member(raw_call, "args", None) + if arguments is None: + arguments = member(function, "arguments", {}) + identifier = _call_id(raw_call) + calls.append( + ToolCallRecord( + name=name, + input_parameters=json_arguments( + arguments, location=f"LangChain tool call {name!r}" + ), + output=outputs.get(identifier) if identifier else None, + ) + ) + if not calls: + raise HarnessIntegrationError("LangChain trace contains no tool calls") + expected = approved_expected_calls( + calls, expected_tools, approve_observed=approve_observed + ) + return AgentCase( + case_id=case_id, + input=input, + actual_output=final_output, + expected_output=expected_output, + tools_called=tuple(calls), + expected_tools=expected, + tags=tuple(tags), + metadata={"harness": "langchain-langgraph"}, + ) + + +def tool_specs( + tools: Sequence[Any], *, side_effecting: Sequence[str] = () +) -> tuple[ToolSpec, ...]: + """Convert LangChain BaseTool-compatible objects to Mendmark contracts.""" + side_effect_names = set(side_effecting) + return tuple( + tool_spec_from_object( + tool, + framework="LangChain", + side_effecting=str(member(tool, "name")) in side_effect_names, + ) + for tool in tools + ) diff --git a/src/mendmark/integrations/multi_agent.py b/src/mendmark/integrations/multi_agent.py new file mode 100644 index 0000000..66401d1 --- /dev/null +++ b/src/mendmark/integrations/multi_agent.py @@ -0,0 +1,219 @@ +"""A fluent, framework-neutral builder for reviewed causal agent traces.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any + +from ..agent_cases import ( + AgentCase, + AgentEvent, + AgentSpec, + ToolCallRecord, + case_graph_issues, +) +from .common import HarnessIntegrationError, json_arguments, json_value + + +class CausalCaseBuilder: + """Build a validated schema 2.0 case without hand-writing JSON. + + Dependencies are always explicit. The builder never infers causality from + wall-clock ordering, which would be incorrect for parallel harness runs. + """ + + def __init__(self, *, case_id: str, input: str, root_agent_id: str) -> None: + self.case_id = case_id + self.input = input + self.root_agent_id = root_agent_id + self._agents: list[AgentSpec] = [] + self._events: list[AgentEvent] = [] + + def agent( + self, + agent_id: str, + *, + role: str | None = None, + description: str | None = None, + allowed_tools: Sequence[str] = (), + ) -> "CausalCaseBuilder": + self._agents.append( + AgentSpec( + agent_id=agent_id, + role=role, + description=description, + allowed_tools=tuple(allowed_tools), + ) + ) + return self + + def event( + self, + event_id: str, + kind: str, + actor_id: str, + *, + target_agent_id: str | None = None, + depends_on: Sequence[str] = (), + tool_call: ToolCallRecord | None = None, + payload: Any = None, + ) -> "CausalCaseBuilder": + self._events.append( + AgentEvent( + event_id=event_id, + kind=kind, + actor_id=actor_id, + target_agent_id=target_agent_id, + depends_on=tuple(depends_on), + tool_call=tool_call, + payload=json_value(payload), + ) + ) + return self + + def delegation( + self, + event_id: str, + actor_id: str, + target_agent_id: str, + *, + depends_on: Sequence[str] = (), + payload: Any = None, + ) -> "CausalCaseBuilder": + return self.event( + event_id, + "delegation", + actor_id, + target_agent_id=target_agent_id, + depends_on=depends_on, + payload=payload, + ) + + def tool_call( + self, + event_id: str, + actor_id: str, + name: str, + *, + input_parameters: Mapping[str, Any] | str | None = None, + output: Any = None, + depends_on: Sequence[str] = (), + description: str | None = None, + ) -> "CausalCaseBuilder": + return self.event( + event_id, + "tool_call", + actor_id, + depends_on=depends_on, + tool_call=ToolCallRecord( + name=name, + input_parameters=json_arguments( + input_parameters, + location=f"multi-agent tool call {name!r}", + ), + output=json_value(output), + description=description, + ), + ) + + def result( + self, + event_id: str, + actor_id: str, + target_agent_id: str, + *, + depends_on: Sequence[str] = (), + payload: Any = None, + ) -> "CausalCaseBuilder": + return self.event( + event_id, + "agent_result", + actor_id, + target_agent_id=target_agent_id, + depends_on=depends_on, + payload=payload, + ) + + def message( + self, + event_id: str, + actor_id: str, + *, + target_agent_id: str | None = None, + depends_on: Sequence[str] = (), + payload: Any = None, + ) -> "CausalCaseBuilder": + return self.event( + event_id, + "message", + actor_id, + target_agent_id=target_agent_id, + depends_on=depends_on, + payload=payload, + ) + + def state_update( + self, + event_id: str, + actor_id: str, + *, + depends_on: Sequence[str] = (), + payload: Any = None, + ) -> "CausalCaseBuilder": + return self.event( + event_id, + "state_update", + actor_id, + depends_on=depends_on, + payload=payload, + ) + + def build( + self, + *, + actual_output: str, + expected_output: str | None, + expected_events: Sequence[AgentEvent] | None = None, + approve_observed: bool = False, + tags: Sequence[str] = (), + metadata: Mapping[str, Any] | None = None, + ) -> AgentCase: + if expected_events is None and not approve_observed: + raise HarnessIntegrationError( + "expected causal events are required; after reviewing the graph, " + "pass expected_events=... or explicitly set approve_observed=True" + ) + expected = tuple(self._events if expected_events is None else expected_events) + case = AgentCase( + case_id=self.case_id, + input=self.input, + actual_output=actual_output, + expected_output=expected_output, + agents=tuple(self._agents), + events=tuple(self._events), + expected_events=expected, + root_agent_id=self.root_agent_id, + tags=tuple(tags), + metadata={"harness": "causal-builder", **dict(metadata or {})}, + ) + issues = case_graph_issues(case) + if issues: + issue = issues[0] + event = f" at event {issue['event_id']!r}" if "event_id" in issue else "" + raise HarnessIntegrationError( + f"invalid causal graph{event}: {issue['issue']}" + ) + allowed = {agent.agent_id: set(agent.allowed_tools) for agent in case.agents} + for trace_name, events in (("actual", case.events), ("expected", case.expected_events)): + for event in events: + if ( + event.kind == "tool_call" + and event.tool_call is not None + and event.tool_call.name not in allowed[event.actor_id] + ): + raise HarnessIntegrationError( + f"{trace_name} event {event.event_id!r} calls tool " + f"{event.tool_call.name!r} outside agent " + f"{event.actor_id!r}'s allow-list" + ) + return case diff --git a/src/mendmark/integrations/openai_agents.py b/src/mendmark/integrations/openai_agents.py new file mode 100644 index 0000000..a6060a7 --- /dev/null +++ b/src/mendmark/integrations/openai_agents.py @@ -0,0 +1,106 @@ +"""OpenAI Agents SDK result adapters.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from ..agent_cases import AgentCase, ToolCallRecord, ToolSpec +from .common import ( + HarnessIntegrationError, + approved_expected_calls, + json_arguments, + json_value, + member, + text_output, + tool_spec_from_object, +) + + +def _raw(item: Any) -> Any: + return member(item, "raw_item", {}) or {} + + +def _call_id(item: Any) -> str | None: + identifier = member(item, "call_id") + if identifier is None: + raw = _raw(item) + identifier = member(raw, "call_id") or member(raw, "id") + return str(identifier) if identifier is not None else None + + +def case_from_result( + result: Any, + *, + case_id: str, + input: str, + expected_output: str | None, + expected_tools: Sequence[ToolCallRecord] | None = None, + approve_observed: bool = False, + tags: Sequence[str] = (), +) -> AgentCase: + """Convert an OpenAI Agents SDK RunResult using its public run items.""" + items = member(result, "new_items") + if not isinstance(items, Sequence): + raise HarnessIntegrationError( + "OpenAI Agents result does not expose a new_items sequence" + ) + outputs: dict[str, Any] = {} + for item in items: + if member(item, "type") == "tool_call_output_item": + identifier = _call_id(item) + if identifier: + outputs[identifier] = json_value(member(item, "output")) + + calls: list[ToolCallRecord] = [] + for index, item in enumerate(items): + if member(item, "type") != "tool_call_item": + continue + raw = _raw(item) + name = member(item, "tool_name") or member(raw, "name") + if not isinstance(name, str) or not name: + raise HarnessIntegrationError( + f"OpenAI Agents tool call {index} has no stable name" + ) + arguments = member(raw, "arguments", {}) + identifier = _call_id(item) + calls.append( + ToolCallRecord( + name=name, + input_parameters=json_arguments( + arguments, location=f"OpenAI Agents tool call {name!r}" + ), + output=outputs.get(identifier) if identifier else None, + description=member(item, "description"), + ) + ) + if not calls: + raise HarnessIntegrationError("OpenAI Agents result contains no tool calls") + expected = approved_expected_calls( + calls, expected_tools, approve_observed=approve_observed + ) + return AgentCase( + case_id=case_id, + input=input, + actual_output=text_output(member(result, "final_output")), + expected_output=expected_output, + tools_called=tuple(calls), + expected_tools=expected, + tags=tuple(tags), + metadata={"harness": "openai-agents"}, + ) + + +def tool_specs( + tools: Sequence[Any], *, side_effecting: Sequence[str] = () +) -> tuple[ToolSpec, ...]: + """Convert OpenAI Agents FunctionTool-compatible objects.""" + side_effect_names = set(side_effecting) + return tuple( + tool_spec_from_object( + tool, + framework="OpenAI Agents", + side_effecting=str(member(tool, "name")) in side_effect_names, + ) + for tool in tools + ) diff --git a/tests/test_harness_integrations.py b/tests/test_harness_integrations.py new file mode 100644 index 0000000..1ff2664 --- /dev/null +++ b/tests/test_harness_integrations.py @@ -0,0 +1,373 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from mendmark.agent_cases import ToolCallRecord +from mendmark.cli import main +from mendmark.equip import EquipError, agent_prompt, detect_frameworks, equip_project +from mendmark.integrations import CausalCaseBuilder, HarnessIntegrationError, write_suite +from mendmark.integrations.crewai import CrewAIRecorder, case_from_events +from mendmark.integrations.langchain import case_from_messages, tool_specs as lc_tools +from mendmark.integrations.openai_agents import ( + case_from_result, + tool_specs as openai_tools, +) +from mendmark.json_adapter import load_json_suite + + +@dataclass +class FakeTool: + name: str + description: str + args_schema: dict[str, object] + + +def test_langchain_messages_pair_tool_results_and_write_valid_suite(tmp_path: Path) -> None: + messages = [ + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call-1", "name": "lookup_order", "args": {"order_id": "104"}} + ], + }, + { + "role": "tool", + "tool_call_id": "call-1", + "content": {"status": "paid"}, + }, + {"role": "assistant", "content": "Order 104 is paid."}, + ] + case = case_from_messages( + messages, + case_id="paid-order", + input="Check order 104", + expected_output="Order 104 is paid.", + approve_observed=True, + ) + tools = lc_tools( + [ + FakeTool( + "lookup_order", + "Look up one order", + { + "type": "object", + "properties": {"order_id": {"type": "string"}}, + "required": ["order_id"], + }, + ) + ] + ) + path = write_suite(tmp_path / "suite.json", [case], tools) + + suite = load_json_suite(path) + assert suite.cases[0].tools_called == suite.cases[0].expected_tools + assert suite.cases[0].tools_called[0].output == {"status": "paid"} + assert suite.cases[0].metadata == {"harness": "langchain-langgraph"} + + +def test_harness_converter_never_implicitly_approves_observed_trace() -> None: + with pytest.raises(HarnessIntegrationError, match="expected tool calls are required"): + case_from_messages( + [ + { + "role": "assistant", + "tool_calls": [{"id": "1", "name": "charge", "args": {}}], + }, + {"role": "tool", "tool_call_id": "1", "content": "ok"}, + ], + case_id="charge", + input="charge it", + expected_output=None, + ) + + +def test_openai_agents_result_uses_public_run_items() -> None: + agent = SimpleNamespace(name="billing") + call = SimpleNamespace( + type="tool_call_item", + agent=agent, + tool_name="refund_order", + description="Refund a paid order", + call_id="call-2", + raw_item={ + "name": "refund_order", + "call_id": "call-2", + "arguments": '{"order_id":"104","amount":29.99}', + }, + ) + output = SimpleNamespace( + type="tool_call_output_item", + call_id="call-2", + output={"status": "accepted"}, + raw_item={"call_id": "call-2"}, + ) + result = SimpleNamespace( + new_items=[call, output], final_output="Refund accepted." + ) + + case = case_from_result( + result, + case_id="refund", + input="Refund order 104", + expected_output="Refund accepted.", + approve_observed=True, + ) + + assert case.tools_called[0] == ToolCallRecord( + name="refund_order", + input_parameters={"order_id": "104", "amount": 29.99}, + output={"status": "accepted"}, + description="Refund a paid order", + ) + assert case.metadata == {"harness": "openai-agents"} + + +def test_openai_tool_schema_and_side_effect_are_preserved() -> None: + tool = SimpleNamespace( + name="send_email", + description="Send an email", + params_json_schema={ + "type": "object", + "properties": {"to": {"type": "string"}}, + "required": ["to"], + }, + ) + converted = openai_tools([tool], side_effecting=["send_email"]) + assert converted[0].side_effecting is True + assert converted[0].input_schema["required"] == ["to"] + + +def test_crewai_events_and_recorder_capture_public_event_fields() -> None: + recorder = CrewAIRecorder() + recorder.record( + SimpleNamespace( + type="tool_usage_finished", + tool_name="search", + tool_args='{"query":"mendmark"}', + output={"hits": 3}, + ) + ) + recorder.record( + SimpleNamespace(type="agent_execution_completed", output="Found three results.") + ) + case = recorder.case( + case_id="search", + input="Search for Mendmark", + expected_output="Found three results.", + approve_observed=True, + ) + assert case.tools_called[0].input_parameters == {"query": "mendmark"} + assert case.tools_called[0].output == {"hits": 3} + assert recorder.events + recorder.clear() + assert recorder.events == () + + +def test_crewai_converter_rejects_invalid_json_arguments() -> None: + with pytest.raises(HarnessIntegrationError, match="invalid JSON arguments"): + case_from_events( + [ + SimpleNamespace( + type="tool_usage_finished", + tool_name="search", + tool_args="{bad", + output="none", + ) + ], + case_id="bad", + input="bad", + expected_output=None, + approve_observed=True, + ) + + +def test_write_suite_is_atomic_and_refuses_overwrite(tmp_path: Path) -> None: + case = case_from_messages( + [ + {"role": "assistant", "tool_calls": [{"id": "1", "name": "x", "args": {}}]}, + {"role": "tool", "tool_call_id": "1", "content": "ok"}, + ], + case_id="one", + input="x", + expected_output=None, + approve_observed=True, + ) + tools = lc_tools([FakeTool("x", "x", {"type": "object"})]) + destination = write_suite(tmp_path / "suite.json", [case], tools) + original = destination.read_bytes() + with pytest.raises(HarnessIntegrationError, match="refusing to overwrite"): + write_suite(destination, [case], tools) + assert destination.read_bytes() == original + + +def test_equip_detects_multiple_harnesses_and_is_idempotent(tmp_path: Path) -> None: + (tmp_path / "pyproject.toml").write_text( + '[project]\ndependencies=["langgraph", "openai-agents"]\n', encoding="utf-8" + ) + assert detect_frameworks(tmp_path) == ("langgraph", "openai-agents") + + frameworks, created, unchanged = equip_project(tmp_path) + assert frameworks == ("langgraph", "openai-agents") + assert len(created) == 5 + assert unchanged == () + _, created_again, unchanged_again = equip_project(tmp_path) + assert created_again == () + assert len(unchanged_again) == 5 + assert "approve_observed=True" in ( + tmp_path / ".mendmark" / "agent-setup.md" + ).read_text(encoding="utf-8") + + +def test_equip_dry_run_prompt_and_cli(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + result = main( + [ + "equip", + "--framework", + "crewai", + "--project-root", + str(tmp_path), + "--dry-run", + ] + ) + assert result == 0 + assert not (tmp_path / ".mendmark").exists() + assert "Would create: .mendmark/evaluator.py" in capsys.readouterr().out + + result = main(["equip", "--print-agent-prompt", "--project-root", str(tmp_path)]) + assert result == 0 + assert "mendmark equip --framework auto" in capsys.readouterr().out + assert ".mendmark/agent-setup.md" in agent_prompt() + + +def test_equip_refuses_conflicts_and_symlink_escape(tmp_path: Path) -> None: + generated = tmp_path / ".mendmark" + generated.mkdir() + (generated / "evaluator.py").write_text("customer code\n", encoding="utf-8") + with pytest.raises(EquipError, match="refusing to overwrite"): + equip_project(tmp_path, framework="generic") + + clean = tmp_path / "clean" + outside = tmp_path / "outside" + clean.mkdir() + outside.mkdir() + (clean / ".mendmark").symlink_to(outside, target_is_directory=True) + with pytest.raises(EquipError, match="refusing to overwrite"): + equip_project(clean, framework="generic") + + +def test_generated_evaluator_completes_an_offline_audit(tmp_path: Path) -> None: + equip_project(tmp_path, framework="langgraph") + case = case_from_messages( + [ + { + "role": "assistant", + "tool_calls": [{"id": "1", "name": "lookup", "args": {"id": "7"}}], + }, + {"role": "tool", "tool_call_id": "1", "content": {"status": "ok"}}, + {"role": "assistant", "content": "Found it."}, + ], + case_id="lookup", + input="Find 7", + expected_output="Found it.", + approve_observed=True, + ) + tools = lc_tools( + [ + FakeTool( + "lookup", + "Look up an item", + { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + }, + ) + ] + ) + write_suite(tmp_path / ".mendmark" / "suite.json", [case], tools) + result = main( + [ + "audit-json", + str(tmp_path / ".mendmark" / "suite.json"), + "--evaluator-command", + f"python3 {tmp_path / '.mendmark' / 'evaluator.py'}", + "--output", + str(tmp_path / "report.json"), + ] + ) + assert result == 0 + report = json.loads((tmp_path / "report.json").read_text(encoding="utf-8")) + assert report["summary"]["survived"] == 0 + assert report["gate"]["passed"] is True + + +def test_causal_builder_preserves_parallel_dependencies_and_writes_v2( + tmp_path: Path, +) -> None: + builder = ( + CausalCaseBuilder( + case_id="parallel", input="review", root_agent_id="supervisor" + ) + .agent("supervisor") + .agent("billing", allowed_tools=["lookup"]) + .agent("risk", allowed_tools=["risk"]) + .delegation("d-billing", "supervisor", "billing") + .delegation("d-risk", "supervisor", "risk") + .tool_call( + "lookup", + "billing", + "lookup", + input_parameters={"id": "7"}, + output={"paid": True}, + depends_on=["d-billing"], + ) + .tool_call( + "risk", + "risk", + "risk", + input_parameters={"id": "7"}, + output={"level": "low"}, + depends_on=["d-risk"], + ) + .result("billing-result", "billing", "supervisor", depends_on=["lookup"]) + .result("risk-result", "risk", "supervisor", depends_on=["risk"]) + .message( + "aggregate", + "supervisor", + depends_on=["billing-result", "risk-result"], + ) + ) + with pytest.raises(HarnessIntegrationError, match="expected causal events"): + builder.build(actual_output="ok", expected_output="ok") + case = builder.build( + actual_output="ok", expected_output="ok", approve_observed=True + ) + tools = lc_tools( + [ + FakeTool("lookup", "lookup", {"type": "object"}), + FakeTool("risk", "risk", {"type": "object"}), + ] + ) + path = write_suite(tmp_path / "multi.json", [case], tools) + suite = load_json_suite(path) + assert suite.schema_version == "2.0" + aggregate = suite.cases[0].events[-1] + assert set(aggregate.depends_on) == {"billing-result", "risk-result"} + + +def test_causal_builder_rejects_unauthorized_tool_use() -> None: + builder = ( + CausalCaseBuilder(case_id="bad", input="bad", root_agent_id="root") + .agent("root") + .tool_call("charge", "root", "charge", input_parameters={}) + ) + with pytest.raises(HarnessIntegrationError, match="outside agent .* allow-list"): + builder.build( + actual_output="bad", expected_output=None, approve_observed=True + ) diff --git a/tests/test_harness_live_compatibility.py b/tests/test_harness_live_compatibility.py new file mode 100644 index 0000000..b19a538 --- /dev/null +++ b/tests/test_harness_live_compatibility.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import os +from datetime import datetime, timezone +from types import SimpleNamespace + +import pytest + +from mendmark.integrations.crewai import case_from_events, tool_specs as crewai_tools +from mendmark.integrations.langchain import case_from_messages, tool_specs as lc_tools +from mendmark.integrations.openai_agents import ( + case_from_result, + tool_specs as openai_tools, +) + + +PROFILE = os.environ.get("MENDMARK_HARNESS") + + +@pytest.mark.skipif(PROFILE != "langgraph", reason="LangGraph compatibility profile") +def test_current_langchain_core_public_objects() -> None: + from langchain_core.messages import AIMessage, ToolMessage + from langchain_core.tools import tool + + @tool + def lookup_order(order_id: str) -> str: + """Look up an order.""" + return order_id + + messages = [ + AIMessage( + content="", + tool_calls=[{"name": "lookup_order", "args": {"order_id": "7"}, "id": "c1"}], + ), + ToolMessage(content='{"status":"paid"}', tool_call_id="c1"), + AIMessage(content="Paid."), + ] + case = case_from_messages( + messages, + case_id="live", + input="check 7", + expected_output="Paid.", + approve_observed=True, + ) + assert case.tools_called[0].name == "lookup_order" + assert lc_tools([lookup_order])[0].input_schema["required"] == ["order_id"] + + +@pytest.mark.skipif(PROFILE != "openai-agents", reason="OpenAI Agents compatibility profile") +def test_current_openai_agents_public_objects() -> None: + from agents import Agent, ToolCallItem, ToolCallOutputItem, function_tool + + @function_tool + def lookup_order(order_id: str) -> str: + """Look up an order.""" + return order_id + + agent = Agent(name="billing", tools=[lookup_order]) + call = ToolCallItem( + agent=agent, + raw_item={ + "type": "function_call", + "name": "lookup_order", + "arguments": '{"order_id":"7"}', + "call_id": "c1", + }, + ) + output = ToolCallOutputItem( + agent=agent, + raw_item={"type": "function_call_output", "call_id": "c1", "output": "paid"}, + output="paid", + ) + case = case_from_result( + SimpleNamespace(new_items=[call, output], final_output="Paid."), + case_id="live", + input="check 7", + expected_output="Paid.", + approve_observed=True, + ) + assert case.tools_called[0].output == "paid" + assert openai_tools([lookup_order])[0].input_schema["required"] == ["order_id"] + + +@pytest.mark.skipif(PROFILE != "crewai", reason="CrewAI compatibility profile") +def test_current_crewai_public_objects() -> None: + from crewai.events.types.tool_usage_events import ToolUsageFinishedEvent + from crewai.tools import BaseTool + + class LookupOrder(BaseTool): + name: str = "lookup_order" + description: str = "Look up an order" + + def _run(self, order_id: str) -> str: + return order_id + + tool = LookupOrder() + now = datetime.now(timezone.utc) + event = ToolUsageFinishedEvent( + tool_name="lookup_order", + tool_args={"order_id": "7"}, + output="paid", + started_at=now, + finished_at=now, + ) + case = case_from_events( + [event, SimpleNamespace(type="agent_execution_completed", output="Paid.")], + case_id="live", + input="check 7", + expected_output="Paid.", + approve_observed=True, + ) + assert case.tools_called[0].output == "paid" + assert crewai_tools([tool])[0].input_schema["type"] == "object" From a0a805f517e80c3ec623027c817b2127d288fb45 Mon Sep 17 00:00:00 2001 From: Daniel Gaskins Date: Mon, 10 Aug 2026 16:47:22 -0700 Subject: [PATCH 2/2] Avoid duplicate branch checks --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4aac370..049ee77 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,6 +2,7 @@ name: tests on: push: + branches: [main] pull_request: workflow_dispatch: