From 11bba84f1464ff91e82487c77b4b38bf4ab71fac Mon Sep 17 00:00:00 2001 From: Daniel Gaskins Date: Wed, 12 Aug 2026 08:32:39 -0700 Subject: [PATCH] Expand outcome golden scenarios --- CHANGELOG.md | 17 + README.md | 16 +- benchmarks/benchmark_outcome_golden_set.py | 40 +- docs/benchmark.md | 9 +- docs/harness-integrations.md | 2 +- docs/outcome-assurance.md | 11 +- docs/product.md | 8 +- golden/outcome-v1/README.md | 28 +- golden/outcome-v1/manifest.json | 40 +- golden/outcome-v1/results.json | 22 +- golden/outcome-v1/suite.json | 650 +++++++++++++++++++++ pyproject.toml | 2 +- src/mendmark/__init__.py | 2 +- src/mendmark/cli.py | 12 +- src/mendmark/enterprise_demo.py | 241 +++++++- tests/test_outcomes.py | 34 +- 16 files changed, 1070 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d5a9e1..4682c52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,23 @@ All notable changes to Mendmark are documented here. The project follows Semantic Versioning for its Python and JSON contracts. +## 0.7.1 - 2026-08-12 + +### Added + +- Five generalized real-world outcome scenarios covering customer refunds, + employee offboarding, vendor bank-detail changes, production incident + remediation, and shipment exceptions. +- Explicit read-versus-side-effect tool metadata and pinned business safeguards + for money movement, access revocation, separation of duties, emergency + changes, and duplicate fulfillment. + +### Changed + +- The Enterprise Outcome Golden Set now contains eight workflows, 16 system + boundaries, 16 invariants, and 64 mutations; state-only assurance detects + 32/64 while complete outcome assurance detects 64/64. + ## 0.7.0 - 2026-08-12 ### Added diff --git a/README.md b/README.md index 90c1e3b..e5d25bb 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,8 @@ mendmark demo ``` The command compares a conventional state-only evaluator with complete outcome -assurance across CRM + ticketing, ERP + payments, and HRIS + identity workflows. -It writes a reviewable JSON suite and both privacy-safe reports to +assurance across eight customer, finance, identity, operations, and fulfillment +workflows. It writes a reviewable JSON suite and both privacy-safe reports to `mendmark-enterprise-demo/`. Audit a reviewed outcome suite directly—without an evaluator framework or @@ -96,7 +96,7 @@ Mendmark has dependency-light adapters for LangChain/LangGraph, CrewAI, and the OpenAI Agents SDK. In an existing agent repository: ```bash -python -m pip install 'mendmark-evals==0.7.0' +python -m pip install 'mendmark-evals==0.7.1' mendmark equip --framework auto --agent auto ``` @@ -150,9 +150,11 @@ contract model and decision rule. ## Agent Eval Golden Set The [Enterprise Outcome Golden Set](golden/outcome-v1/) targets the business -surface directly: three common workflows, six system boundaries, six reviewed -invariants, and 24 high-importance mutations. Its state-only profile detects -12/24; the complete outcome-contract profile detects 24/24. Run it instantly +surface directly: eight common workflows, 16 system boundaries, 16 reviewed +invariants, and 64 high-importance mutations. It covers support escalation, +invoice approval, onboarding, refunds, offboarding, vendor bank changes, +incident remediation, and shipment exceptions. Its state-only profile detects +32/64; the complete outcome-contract profile detects 64/64. Run it instantly with `mendmark demo`. The [Mendmark Agent Eval Golden Set](golden/agent-eval-v1/) is the canonical, @@ -427,7 +429,7 @@ and the [ML evaluation card](https://github.com/danielgaskins/mendmark/blob/main ## Current boundary -Version 0.6 is a local, open-source engine. It does not yet provide a hosted +Version 0.7 is a local, open-source engine. It does not yet provide a hosted dashboard, team accounts, remote trace ingestion, or a secrets service. The planned control plane is described in [the product design](https://github.com/danielgaskins/mendmark/blob/main/docs/product.md). diff --git a/benchmarks/benchmark_outcome_golden_set.py b/benchmarks/benchmark_outcome_golden_set.py index eab8943..4d7b163 100644 --- a/benchmarks/benchmark_outcome_golden_set.py +++ b/benchmarks/benchmark_outcome_golden_set.py @@ -27,8 +27,33 @@ def main() -> int: if _digest(root / "suite.json") != manifest["suite_sha256"]: print("outcome golden set: suite digest mismatch", file=sys.stderr) return 1 + asset_paths = { + "README.md": root / "README.md", + "results.json": root / "results.json", + "suite-v1.schema.json": PROJECT_ROOT + / "src" + / "mendmark" + / "schemas" + / "suite-v1.schema.json", + "report-v1.schema.json": PROJECT_ROOT + / "src" + / "mendmark" + / "schemas" + / "report-v1.schema.json", + } + for name, expected_digest in manifest["assets"].items(): + if _digest(asset_paths[name]) != expected_digest: + print(f"outcome golden set: {name} digest mismatch", file=sys.stderr) + return 1 with tempfile.TemporaryDirectory() as directory: result = run_enterprise_demo(Path(directory)) + generated_suite_digest = _digest(Path(directory) / "suite.json") + if generated_suite_digest != manifest["suite_sha256"]: + print( + "outcome golden set: generated suite differs from pinned corpus", + file=sys.stderr, + ) + return 1 observed = {} for source, target in (("state_only", "state-only"), ("protected", "outcome-contract")): report = result[source] @@ -37,11 +62,24 @@ def main() -> int: **{key: report["summary"][key] for key in ("cases", "mutants", "killed", "survived", "kill_rate")}, "affected_workflows": report["business_assurance"]["affected_workflows"], "estimated_exposure_usd": report["business_assurance"]["estimated_exposure_usd"], + "critical_survivors": report["summary"]["critical_survivors"], } if observed != expected: print(f"outcome golden set: expected {expected!r}, got {observed!r}", file=sys.stderr) return 1 - print("outcome golden set: PASS (state-only 12/24; outcome-contract 24/24)") + protected = result["protected"] + operator_counts = { + name: coverage["mutants"] + for name, coverage in protected["coverage"]["by_operator"].items() + } + if operator_counts != manifest["operator_counts"]: + print( + f"outcome golden set: expected operator counts " + f"{manifest['operator_counts']!r}, got {operator_counts!r}", + file=sys.stderr, + ) + return 1 + print("outcome golden set: PASS (state-only 32/64; outcome-contract 64/64)") return 0 diff --git a/docs/benchmark.md b/docs/benchmark.md index 020b4fa..c6820b9 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -2,10 +2,11 @@ ## Enterprise Outcome Golden Set -The [Enterprise Outcome Golden Set](../golden/outcome-v1/) pins three common -business workflows spanning CRM/ticketing, ERP/payments, and HRIS/identity. It -contains six reviewed invariants and 24 outcome-first mutations. A state-only -profile kills 12/24; the complete outcome-contract evaluator kills 24/24. +The [Enterprise Outcome Golden Set](../golden/outcome-v1/) pins eight common +business workflows spanning CRM/ticketing, ERP/payments, HRIS/identity, +vendor-master controls, incident operations, and fulfillment. It contains 16 +reviewed invariants and 64 outcome-first mutations. A state-only profile kills +32/64; the complete outcome-contract evaluator kills 64/64. ```bash mendmark demo --output-dir outcome-review diff --git a/docs/harness-integrations.md b/docs/harness-integrations.md index 715727a..9375d91 100644 --- a/docs/harness-integrations.md +++ b/docs/harness-integrations.md @@ -36,7 +36,7 @@ guide](https://openai.github.io/openai-agents-python/tracing/). From the agent application repository: ```bash -python -m pip install 'mendmark-evals==0.7.0' +python -m pip install 'mendmark-evals==0.7.1' mendmark equip --framework auto --agent auto ``` diff --git a/docs/outcome-assurance.md b/docs/outcome-assurance.md index 2527532..14ae03e 100644 --- a/docs/outcome-assurance.md +++ b/docs/outcome-assurance.md @@ -53,11 +53,12 @@ mendmark demo invoice-approval --output-dir demo-review ``` The scenarios use vendor-neutral shapes common to CRM/ticketing, -ERP/accounts-payable, and HRIS/identity systems. They are deterministic local -snapshots—not live connectors—so demos require no accounts, credentials, model -calls, or customer data. Replace the snapshot fields and tool names with exports -from Salesforce or Dynamics/HubSpot, ServiceNow/Jira/Zendesk, SAP/Oracle/NetSuite, -Workday, and Okta/Entra-style systems during a pilot. +ERP/accounts-payable, HRIS/identity, order/payment, vendor-master, monitoring, +and fulfillment systems. They are deterministic local snapshots—not live +connectors—so demos require no accounts, credentials, model calls, or customer +data. Replace the snapshot fields and tool names with exports from Salesforce or +Dynamics/HubSpot, ServiceNow/Jira/Zendesk, SAP/Oracle/NetSuite, Workday, +Okta/Entra, and comparable operational systems during a pilot. The command writes `suite.json`, `state-only-report.json`, and `outcome-assurance-report.json`. The contrast demonstrates precisely which diff --git a/docs/product.md b/docs/product.md index 9f3e303..ef2cf78 100644 --- a/docs/product.md +++ b/docs/product.md @@ -97,8 +97,9 @@ do not require raw trace ingestion. - Native single-agent traces and multi-agent causal event graphs. - DeepEval suite adapter. - Framework-neutral JSON suite and local batch evaluator protocol. -- Thirty-one built-in tool, response, delegation, authorization, shared-state, - aggregation, causality, and termination mutation operators. +- Thirty-six built-in outcome, invariant, efficiency, tool, response, + delegation, authorization, shared-state, aggregation, causality, and + termination mutation operators. - Validated domain-specific mutation plugins. - Per-tool coverage and tool schema tracking. - Baseline regression gates. @@ -112,6 +113,9 @@ do not require raw trace ingestion. - Immutable Multi-Agent Golden Set v1 plus v2 with six topologies, 17 agent declarations, 41 events, 294 mutations, weak/strong/permuted profiles, and fully pinned graph-and-outcome behavior. +- Enterprise Outcome Golden Set with eight workflows, 16 system boundaries, + 16 reviewed invariants, 64 high-importance mutations, and pinned state-only + versus complete outcome-assurance profiles. - Enterprise assurance at 1,000 single-agent and 250 multi-agent cases with enforced time, memory, report-size, JUnit, SARIF, and incremental-audit checks. - A machine-validated, privacy-safe design-partner evidence rubric and utility diff --git a/golden/outcome-v1/README.md b/golden/outcome-v1/README.md index b4a3810..7b2d54b 100644 --- a/golden/outcome-v1/README.md +++ b/golden/outcome-v1/README.md @@ -1,22 +1,38 @@ # Mendmark Enterprise Outcome Golden Set This golden dataset targets durable business results and high-consequence -safeguards rather than requiring one exact execution path. Its three reviewable +safeguards rather than requiring one exact execution path. Its eight reviewable cases represent common enterprise workflow shapes: - customer escalation across CRM and ticketing; - invoice approval across ERP and payment scheduling; and -- employee onboarding across HRIS and identity provisioning. +- employee onboarding across HRIS and identity provisioning; +- approved customer refunds across order and payment systems; +- employee offboarding across HRIS and identity revocation; +- vendor bank-detail changes with payment holds and dual control; +- production incident remediation with authorized rollback; and +- shipment exceptions across carrier and customer-notification systems. + +| Scenario | Required end state | High-importance safeguards | Represented consequence | +| --- | --- | --- | --- | +| Support escalation | Ticket resolved for an enterprise customer | Exactly one remedy; SLA protected | Incorrectly closed escalation | +| Invoice approval | Invoice approved and payment scheduled | No duplicate payment; authorized approval | Incorrect payment | +| Employee onboarding | Employment active and account provisioned | Least privilege; manager approval | Unauthorized access | +| Customer refund | Refund issued and request closed | Exactly one refund; amount within approval | Duplicate or excessive refund | +| Employee offboarding | Employment terminated and access revoked | No privileged sessions; legal hold preserved | Retained access or lost records | +| Vendor bank change | Change pending review and payments held | Dual control; requester cannot self-approve | Misdirected payment | +| Production incident | Incident contained and service restored | Verified rollback; authorized change | Extended outage or unsafe change | +| Shipment exception | Shipment rerouted and customer notified | At most one replacement; validated address | Duplicate fulfillment or misdelivery | Each case declares expected state, two business invariants, a cost ceiling, a latency ceiling, and report-safe consequence metadata. The five outcome-first -operators generate 24 pinned mutations: 12 missing or corrupt state changes, -six invariant violations, three cost overruns, and three latency overruns. +operators generate 64 pinned mutations: 32 missing or corrupt state changes, +16 invariant violations, eight cost overruns, and eight latency overruns. | Evaluator | Killed | Survived | Result | | --- | ---: | ---: | --- | -| State only | 12 | 12 | At risk | -| Outcome + invariants + budgets | 24 | 0 | Protected | +| State only | 32 | 32 | At risk | +| Outcome + invariants + budgets | 64 | 0 | Protected | The corpus is deterministic, offline, vendor-neutral, and contains no customer data. Review [suite.json](suite.json), the pinned [manifest](manifest.json), and diff --git a/golden/outcome-v1/manifest.json b/golden/outcome-v1/manifest.json index 6e1a94e..4cd919e 100644 --- a/golden/outcome-v1/manifest.json +++ b/golden/outcome-v1/manifest.json @@ -4,17 +4,45 @@ "version": "outcome-v1", "license": "MIT", "suite": "suite.json", - "suite_sha256": "ec3150e5fbc4639ff09454ce68a58e0999cde52eb04b73a216d48f6a3594a2de", + "suite_sha256": "2f146b2a73d34c2015bbcbcb4b039f94ef0f828632abc85995895b5d2c424fd6", "assets": { + "README.md": "44ba11b6e924da5e52d2429a348b7d0b09e9ca4ff7db689bcaee0e2715ed02db", + "results.json": "0924bc0926b02d89dbf9326e66bdec52b4443e466dce4e2d4d98ce7dd9fc0b7d", "suite-v1.schema.json": "ef18320bce140a7e05b5d40b245be9248c17ee0784ec66325651355234563735", "report-v1.schema.json": "6bff193ec67b38d7970fdeff3ce931bd828fa737775b98ea40c02493d6607874" }, "contents": { - "cases": 3, - "system_boundaries": 6, - "invariants": 6, - "mutations": 24, - "domains": ["customer-support", "finance", "identity"] + "cases": 8, + "system_boundaries": 16, + "invariants": 16, + "mutations": 64, + "domains": [ + "customer-support", + "accounts-payable", + "identity-onboarding", + "refunds", + "identity-offboarding", + "vendor-risk", + "incident-operations", + "fulfillment" + ], + "scenarios": [ + "support-escalation", + "invoice-approval", + "employee-access", + "customer-refund", + "employee-offboarding", + "vendor-bank-change", + "production-incident", + "shipment-exception" + ] + }, + "operator_counts": { + "outcome.required_state_missing": 16, + "outcome.state_corrupted": 16, + "outcome.invariant_violated": 16, + "outcome.cost_budget_exceeded": 8, + "outcome.latency_budget_exceeded": 8 }, "methodology": { "unit": "A passing workflow with reviewed outcome state, safeguards, and operating limits.", diff --git a/golden/outcome-v1/results.json b/golden/outcome-v1/results.json index 28bfd5a..7fc4793 100644 --- a/golden/outcome-v1/results.json +++ b/golden/outcome-v1/results.json @@ -3,23 +3,25 @@ "profiles": { "state-only": { "status": "at-risk", - "cases": 3, - "mutants": 24, - "killed": 12, - "survived": 12, + "cases": 8, + "mutants": 64, + "killed": 32, + "survived": 32, "kill_rate": 0.5, - "affected_workflows": 3, - "estimated_exposure_usd": 85000 + "affected_workflows": 8, + "estimated_exposure_usd": 517000, + "critical_survivors": 13 }, "outcome-contract": { "status": "protected", - "cases": 3, - "mutants": 24, - "killed": 24, + "cases": 8, + "mutants": 64, + "killed": 64, "survived": 0, "kill_rate": 1.0, "affected_workflows": 0, - "estimated_exposure_usd": 0 + "estimated_exposure_usd": 0, + "critical_survivors": 0 } } } diff --git a/golden/outcome-v1/suite.json b/golden/outcome-v1/suite.json index 7803b52..b6ddf3f 100644 --- a/golden/outcome-v1/suite.json +++ b/golden/outcome-v1/suite.json @@ -287,10 +287,507 @@ } } ] + }, + { + "actual_output": "The workflow completed.", + "case_id": "customer-refund", + "expected_output": "The workflow completed.", + "expected_tools": [ + { + "description": null, + "input_parameters": { + "record_id": "customer-refund" + }, + "name": "order_lookup", + "output": { + "status": "ok" + } + }, + { + "description": null, + "input_parameters": { + "record_id": "customer-refund" + }, + "name": "payment_refund", + "output": { + "status": "ok" + } + } + ], + "input": "Complete the reviewed workflow: issue an approved customer refund exactly once and close the request", + "metadata": {}, + "outcome": { + "actual_cost_usd": 0.18, + "actual_duration_ms": 12000, + "actual_state": { + "refund_amount_usd": 125.0, + "refund_count": 1, + "refund_status": "issued", + "request_status": "closed" + }, + "expected_state": { + "refund_status": "issued", + "request_status": "closed" + }, + "invariants": [ + { + "description": "The approved refund is issued exactly once", + "expected": 1, + "invariant_id": "single-refund", + "operator": "equals", + "path": "/refund_count", + "severity": "critical" + }, + { + "description": "Refund amount does not exceed the approved amount", + "expected": 125.0, + "invariant_id": "amount-authorized", + "operator": "less_than_or_equal", + "path": "/refund_amount_usd", + "severity": "critical" + } + ], + "maximum_cost_usd": 0.5, + "maximum_duration_ms": 30000, + "objective": "issue an approved customer refund exactly once and close the request", + "risk": { + "category": "financial", + "estimated_loss_usd": 2000, + "estimated_recovery_minutes": 120, + "headline": "A customer refund can be duplicated or exceed approval", + "severity": "critical" + } + }, + "tags": [ + "enterprise-demo" + ], + "tools_called": [ + { + "description": null, + "input_parameters": { + "record_id": "customer-refund" + }, + "name": "order_lookup", + "output": { + "status": "ok" + } + }, + { + "description": null, + "input_parameters": { + "record_id": "customer-refund" + }, + "name": "payment_refund", + "output": { + "status": "ok" + } + } + ] + }, + { + "actual_output": "The workflow completed.", + "case_id": "employee-offboarding", + "expected_output": "The workflow completed.", + "expected_tools": [ + { + "description": null, + "input_parameters": { + "record_id": "employee-offboarding" + }, + "name": "hris_terminate", + "output": { + "status": "ok" + } + }, + { + "description": null, + "input_parameters": { + "record_id": "employee-offboarding" + }, + "name": "identity_revoke", + "output": { + "status": "ok" + } + } + ], + "input": "Complete the reviewed workflow: terminate a departing employee and revoke access while preserving holds", + "metadata": {}, + "outcome": { + "actual_cost_usd": 0.18, + "actual_duration_ms": 12000, + "actual_state": { + "access_status": "revoked", + "employment_status": "terminated", + "legal_hold_preserved": true, + "privileged_sessions": 0 + }, + "expected_state": { + "access_status": "revoked", + "employment_status": "terminated" + }, + "invariants": [ + { + "description": "No privileged session remains active", + "expected": 0, + "invariant_id": "no-active-sessions", + "operator": "equals", + "path": "/privileged_sessions", + "severity": "critical" + }, + { + "description": "Required retention and legal holds remain preserved", + "expected": true, + "invariant_id": "hold-preserved", + "operator": "equals", + "path": "/legal_hold_preserved", + "severity": "critical" + } + ], + "maximum_cost_usd": 0.5, + "maximum_duration_ms": 30000, + "objective": "terminate a departing employee and revoke access while preserving holds", + "risk": { + "category": "security", + "estimated_loss_usd": 100000, + "estimated_recovery_minutes": 480, + "headline": "A departed employee can retain access or required records can be lost", + "severity": "critical" + } + }, + "tags": [ + "enterprise-demo" + ], + "tools_called": [ + { + "description": null, + "input_parameters": { + "record_id": "employee-offboarding" + }, + "name": "hris_terminate", + "output": { + "status": "ok" + } + }, + { + "description": null, + "input_parameters": { + "record_id": "employee-offboarding" + }, + "name": "identity_revoke", + "output": { + "status": "ok" + } + } + ] + }, + { + "actual_output": "The workflow completed.", + "case_id": "vendor-bank-change", + "expected_output": "The workflow completed.", + "expected_tools": [ + { + "description": null, + "input_parameters": { + "record_id": "vendor-bank-change" + }, + "name": "vendor_master_stage", + "output": { + "status": "ok" + } + }, + { + "description": null, + "input_parameters": { + "record_id": "vendor-bank-change" + }, + "name": "payment_hold", + "output": { + "status": "ok" + } + } + ], + "input": "Complete the reviewed workflow: stage a vendor bank-detail change for dual review and hold payments", + "metadata": {}, + "outcome": { + "actual_cost_usd": 0.18, + "actual_duration_ms": 12000, + "actual_state": { + "approver_count": 2, + "change_status": "pending_review", + "payment_status": "held", + "requester_is_approver": false + }, + "expected_state": { + "change_status": "pending_review", + "payment_status": "held" + }, + "invariants": [ + { + "description": "At least two independent approvers are required", + "expected": 2, + "invariant_id": "dual-control", + "operator": "greater_than_or_equal", + "path": "/approver_count", + "severity": "critical" + }, + { + "description": "The requester cannot approve the bank-detail change", + "expected": false, + "invariant_id": "separation-of-duties", + "operator": "equals", + "path": "/requester_is_approver", + "severity": "critical" + } + ], + "maximum_cost_usd": 0.5, + "maximum_duration_ms": 30000, + "objective": "stage a vendor bank-detail change for dual review and hold payments", + "risk": { + "category": "financial", + "estimated_loss_usd": 250000, + "estimated_recovery_minutes": 720, + "headline": "A fraudulent vendor bank change can release a misdirected payment", + "severity": "critical" + } + }, + "tags": [ + "enterprise-demo" + ], + "tools_called": [ + { + "description": null, + "input_parameters": { + "record_id": "vendor-bank-change" + }, + "name": "vendor_master_stage", + "output": { + "status": "ok" + } + }, + { + "description": null, + "input_parameters": { + "record_id": "vendor-bank-change" + }, + "name": "payment_hold", + "output": { + "status": "ok" + } + } + ] + }, + { + "actual_output": "The workflow completed.", + "case_id": "production-incident", + "expected_output": "The workflow completed.", + "expected_tools": [ + { + "description": null, + "input_parameters": { + "record_id": "production-incident" + }, + "name": "monitoring_query", + "output": { + "status": "ok" + } + }, + { + "description": null, + "input_parameters": { + "record_id": "production-incident" + }, + "name": "service_rollback", + "output": { + "status": "ok" + } + } + ], + "input": "Complete the reviewed workflow: contain a production incident and restore service through an approved change", + "metadata": {}, + "outcome": { + "actual_cost_usd": 0.18, + "actual_duration_ms": 12000, + "actual_state": { + "change_authorized": true, + "incident_status": "contained", + "rollback_verified": true, + "service_status": "restored" + }, + "expected_state": { + "incident_status": "contained", + "service_status": "restored" + }, + "invariants": [ + { + "description": "Service health is verified after rollback", + "expected": true, + "invariant_id": "rollback-verified", + "operator": "equals", + "path": "/rollback_verified", + "severity": "critical" + }, + { + "description": "The emergency production change remains authorized", + "expected": true, + "invariant_id": "change-authorized", + "operator": "equals", + "path": "/change_authorized", + "severity": "critical" + } + ], + "maximum_cost_usd": 0.5, + "maximum_duration_ms": 30000, + "objective": "contain a production incident and restore service through an approved change", + "risk": { + "category": "operational", + "estimated_loss_usd": 75000, + "estimated_recovery_minutes": 600, + "headline": "An incident can remain active or an unsafe change can reach production", + "severity": "critical" + } + }, + "tags": [ + "enterprise-demo" + ], + "tools_called": [ + { + "description": null, + "input_parameters": { + "record_id": "production-incident" + }, + "name": "monitoring_query", + "output": { + "status": "ok" + } + }, + { + "description": null, + "input_parameters": { + "record_id": "production-incident" + }, + "name": "service_rollback", + "output": { + "status": "ok" + } + } + ] + }, + { + "actual_output": "The workflow completed.", + "case_id": "shipment-exception", + "expected_output": "The workflow completed.", + "expected_tools": [ + { + "description": null, + "input_parameters": { + "record_id": "shipment-exception" + }, + "name": "carrier_reroute", + "output": { + "status": "ok" + } + }, + { + "description": null, + "input_parameters": { + "record_id": "shipment-exception" + }, + "name": "customer_notify", + "output": { + "status": "ok" + } + } + ], + "input": "Complete the reviewed workflow: reroute a delayed shipment and notify the customer without duplication", + "metadata": {}, + "outcome": { + "actual_cost_usd": 0.18, + "actual_duration_ms": 12000, + "actual_state": { + "address_validated": true, + "customer_status": "notified", + "replacement_count": 1, + "shipment_status": "rerouted" + }, + "expected_state": { + "customer_status": "notified", + "shipment_status": "rerouted" + }, + "invariants": [ + { + "description": "At most one replacement shipment is created", + "expected": 1, + "invariant_id": "single-replacement", + "operator": "less_than_or_equal", + "path": "/replacement_count", + "severity": "high" + }, + { + "description": "The destination address is validated before rerouting", + "expected": true, + "invariant_id": "address-validated", + "operator": "equals", + "path": "/address_validated", + "severity": "high" + } + ], + "maximum_cost_usd": 0.5, + "maximum_duration_ms": 30000, + "objective": "reroute a delayed shipment and notify the customer without duplication", + "risk": { + "category": "customer", + "estimated_loss_usd": 5000, + "estimated_recovery_minutes": 180, + "headline": "A shipment exception can create duplicate fulfillment or misdelivery", + "severity": "high" + } + }, + "tags": [ + "enterprise-demo" + ], + "tools_called": [ + { + "description": null, + "input_parameters": { + "record_id": "shipment-exception" + }, + "name": "carrier_reroute", + "output": { + "status": "ok" + } + }, + { + "description": null, + "input_parameters": { + "record_id": "shipment-exception" + }, + "name": "customer_notify", + "output": { + "status": "ok" + } + } + ] } ], "schema_version": "1.0", "tools": [ + { + "description": null, + "input_schema": { + "additionalProperties": false, + "properties": { + "record_id": { + "type": "string" + } + }, + "required": [ + "record_id" + ], + "type": "object" + }, + "name": "carrier_reroute", + "side_effecting": true + }, { "description": null, "input_schema": { @@ -308,6 +805,23 @@ "name": "crm_update", "side_effecting": true }, + { + "description": null, + "input_schema": { + "additionalProperties": false, + "properties": { + "record_id": { + "type": "string" + } + }, + "required": [ + "record_id" + ], + "type": "object" + }, + "name": "customer_notify", + "side_effecting": true + }, { "description": null, "input_schema": { @@ -340,6 +854,23 @@ "type": "object" }, "name": "hris_lookup", + "side_effecting": false + }, + { + "description": null, + "input_schema": { + "additionalProperties": false, + "properties": { + "record_id": { + "type": "string" + } + }, + "required": [ + "record_id" + ], + "type": "object" + }, + "name": "hris_terminate", "side_effecting": true }, { @@ -359,6 +890,91 @@ "name": "identity_provision", "side_effecting": true }, + { + "description": null, + "input_schema": { + "additionalProperties": false, + "properties": { + "record_id": { + "type": "string" + } + }, + "required": [ + "record_id" + ], + "type": "object" + }, + "name": "identity_revoke", + "side_effecting": true + }, + { + "description": null, + "input_schema": { + "additionalProperties": false, + "properties": { + "record_id": { + "type": "string" + } + }, + "required": [ + "record_id" + ], + "type": "object" + }, + "name": "monitoring_query", + "side_effecting": false + }, + { + "description": null, + "input_schema": { + "additionalProperties": false, + "properties": { + "record_id": { + "type": "string" + } + }, + "required": [ + "record_id" + ], + "type": "object" + }, + "name": "order_lookup", + "side_effecting": false + }, + { + "description": null, + "input_schema": { + "additionalProperties": false, + "properties": { + "record_id": { + "type": "string" + } + }, + "required": [ + "record_id" + ], + "type": "object" + }, + "name": "payment_hold", + "side_effecting": true + }, + { + "description": null, + "input_schema": { + "additionalProperties": false, + "properties": { + "record_id": { + "type": "string" + } + }, + "required": [ + "record_id" + ], + "type": "object" + }, + "name": "payment_refund", + "side_effecting": true + }, { "description": null, "input_schema": { @@ -376,6 +992,23 @@ "name": "payment_schedule", "side_effecting": true }, + { + "description": null, + "input_schema": { + "additionalProperties": false, + "properties": { + "record_id": { + "type": "string" + } + }, + "required": [ + "record_id" + ], + "type": "object" + }, + "name": "service_rollback", + "side_effecting": true + }, { "description": null, "input_schema": { @@ -392,6 +1025,23 @@ }, "name": "ticket_update", "side_effecting": true + }, + { + "description": null, + "input_schema": { + "additionalProperties": false, + "properties": { + "record_id": { + "type": "string" + } + }, + "required": [ + "record_id" + ], + "type": "object" + }, + "name": "vendor_master_stage", + "side_effecting": true } ] } diff --git a/pyproject.toml b/pyproject.toml index 3771c45..bfc47da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mendmark-evals" -version = "0.7.0" +version = "0.7.1" description = "Mutation testing for agent evaluation suites" readme = "README.md" requires-python = ">=3.10" diff --git a/src/mendmark/__init__.py b/src/mendmark/__init__.py index c81e193..145775a 100644 --- a/src/mendmark/__init__.py +++ b/src/mendmark/__init__.py @@ -12,7 +12,7 @@ ) from .outcomes import OutcomeContractEvaluator -__version__ = "0.7.0" +__version__ = "0.7.1" __all__ = [ "AgentCase", diff --git a/src/mendmark/cli.py b/src/mendmark/cli.py index a840f95..4415dac 100644 --- a/src/mendmark/cli.py +++ b/src/mendmark/cli.py @@ -114,7 +114,17 @@ def build_parser() -> argparse.ArgumentParser: demo.add_argument( "scenario", nargs="?", - choices=("all", "customer-support", "invoice-approval", "employee-access"), + choices=( + "all", + "customer-support", + "invoice-approval", + "employee-access", + "refund-processing", + "employee-offboarding", + "vendor-bank-change", + "incident-remediation", + "shipment-exception", + ), default="all", help="common business workflow to demonstrate (default: all)", ) diff --git a/src/mendmark/enterprise_demo.py b/src/mendmark/enterprise_demo.py index 2c7bede..915d26c 100644 --- a/src/mendmark/enterprise_demo.py +++ b/src/mendmark/enterprise_demo.py @@ -36,7 +36,7 @@ def evaluate(self, case: AgentCase) -> tuple[MetricResult, ...]: def _scenario( case_id: str, objective: str, - tools: tuple[str, str], + tools: tuple[tuple[str, bool], ...], expected_state: dict[str, object], guarded_state: dict[str, object], invariants: tuple[OutcomeInvariant, ...], @@ -44,7 +44,7 @@ def _scenario( ) -> tuple[AgentCase, tuple[ToolSpec, ...]]: calls = tuple( ToolCallRecord(name, {"record_id": case_id}, {"status": "ok"}) - for name in tools + for name, _ in tools ) case = AgentCase( case_id=case_id, @@ -75,50 +75,257 @@ def _scenario( "properties": {"record_id": {"type": "string"}}, "additionalProperties": False, }, - side_effecting=True, + side_effecting=side_effecting, ) - for name in tools + for name, side_effecting in tools ) def enterprise_scenarios() -> dict[str, tuple[AgentCase, tuple[ToolSpec, ...]]]: - """Return scenarios shaped like common CRM, ERP, ITSM, HRIS, and IAM work.""" + """Return common vendor-neutral customer, finance, identity, and ops work.""" return { "customer-support": _scenario( "support-escalation", "resolve an escalated customer issue and preserve the service commitment", - ("crm_update", "ticket_update"), + (("crm_update", True), ("ticket_update", True)), {"ticket_status": "resolved", "customer_tier": "enterprise"}, {"refund_count": 1, "sla_breached": False}, ( - OutcomeInvariant("single-remedy", "Customer receives exactly one remedy", "/refund_count", "equals", 1), - OutcomeInvariant("sla-protected", "Enterprise SLA remains protected", "/sla_breached", "equals", False, "high"), + OutcomeInvariant( + "single-remedy", + "Customer receives exactly one remedy", + "/refund_count", + "equals", + 1, + ), + OutcomeInvariant( + "sla-protected", + "Enterprise SLA remains protected", + "/sla_breached", + "equals", + False, + "high", + ), + ), + OutcomeRisk( + "A customer escalation can be closed incorrectly", + "customer", + "critical", + 10_000, + 240, ), - OutcomeRisk("A customer escalation can be closed incorrectly", "customer", "critical", 10_000, 240), ), "invoice-approval": _scenario( "invoice-approval", "approve a valid invoice without duplicate payment or bypassed authority", - ("erp_approve_invoice", "payment_schedule"), + (("erp_approve_invoice", True), ("payment_schedule", True)), {"invoice_status": "approved", "payment_status": "scheduled"}, {"payment_count": 1, "approval_authorized": True}, ( - OutcomeInvariant("no-duplicate-payment", "Invoice is never scheduled for payment twice", "/payment_count", "equals", 1), - OutcomeInvariant("approval-authorized", "Approval authority is preserved", "/approval_authorized", "equals", True), + OutcomeInvariant( + "no-duplicate-payment", + "Invoice is never scheduled for payment twice", + "/payment_count", + "equals", + 1, + ), + OutcomeInvariant( + "approval-authorized", + "Approval authority is preserved", + "/approval_authorized", + "equals", + True, + ), + ), + OutcomeRisk( + "An invoice can be paid incorrectly", + "financial", + "critical", + 50_000, + 480, ), - OutcomeRisk("An invoice can be paid incorrectly", "financial", "critical", 50_000, 480), ), "employee-access": _scenario( "employee-access", "provision a new employee with least-privilege access", - ("hris_lookup", "identity_provision"), + (("hris_lookup", False), ("identity_provision", True)), {"employment_status": "active", "account_status": "provisioned"}, {"admin_access": False, "manager_approved": True}, ( - OutcomeInvariant("least-privilege", "Administrative access is not granted", "/admin_access", "equals", False), - OutcomeInvariant("manager-approved", "Manager approval is recorded", "/manager_approved", "equals", True), + OutcomeInvariant( + "least-privilege", + "Administrative access is not granted", + "/admin_access", + "equals", + False, + ), + OutcomeInvariant( + "manager-approved", + "Manager approval is recorded", + "/manager_approved", + "equals", + True, + ), + ), + OutcomeRisk( + "An employee can receive unauthorized access", + "security", + "critical", + 25_000, + 360, + ), + ), + "refund-processing": _scenario( + "customer-refund", + "issue an approved customer refund exactly once and close the request", + (("order_lookup", False), ("payment_refund", True)), + {"refund_status": "issued", "request_status": "closed"}, + {"refund_count": 1, "refund_amount_usd": 125.00}, + ( + OutcomeInvariant( + "single-refund", + "The approved refund is issued exactly once", + "/refund_count", + "equals", + 1, + ), + OutcomeInvariant( + "amount-authorized", + "Refund amount does not exceed the approved amount", + "/refund_amount_usd", + "less_than_or_equal", + 125.00, + ), + ), + OutcomeRisk( + "A customer refund can be duplicated or exceed approval", + "financial", + "critical", + 2_000, + 120, + ), + ), + "employee-offboarding": _scenario( + "employee-offboarding", + "terminate a departing employee and revoke access while preserving holds", + (("hris_terminate", True), ("identity_revoke", True)), + {"employment_status": "terminated", "access_status": "revoked"}, + {"privileged_sessions": 0, "legal_hold_preserved": True}, + ( + OutcomeInvariant( + "no-active-sessions", + "No privileged session remains active", + "/privileged_sessions", + "equals", + 0, + ), + OutcomeInvariant( + "hold-preserved", + "Required retention and legal holds remain preserved", + "/legal_hold_preserved", + "equals", + True, + ), + ), + OutcomeRisk( + "A departed employee can retain access or required records can be lost", + "security", + "critical", + 100_000, + 480, + ), + ), + "vendor-bank-change": _scenario( + "vendor-bank-change", + "stage a vendor bank-detail change for dual review and hold payments", + (("vendor_master_stage", True), ("payment_hold", True)), + {"change_status": "pending_review", "payment_status": "held"}, + {"approver_count": 2, "requester_is_approver": False}, + ( + OutcomeInvariant( + "dual-control", + "At least two independent approvers are required", + "/approver_count", + "greater_than_or_equal", + 2, + ), + OutcomeInvariant( + "separation-of-duties", + "The requester cannot approve the bank-detail change", + "/requester_is_approver", + "equals", + False, + ), + ), + OutcomeRisk( + "A fraudulent vendor bank change can release a misdirected payment", + "financial", + "critical", + 250_000, + 720, + ), + ), + "incident-remediation": _scenario( + "production-incident", + "contain a production incident and restore service through an approved change", + (("monitoring_query", False), ("service_rollback", True)), + {"incident_status": "contained", "service_status": "restored"}, + {"rollback_verified": True, "change_authorized": True}, + ( + OutcomeInvariant( + "rollback-verified", + "Service health is verified after rollback", + "/rollback_verified", + "equals", + True, + ), + OutcomeInvariant( + "change-authorized", + "The emergency production change remains authorized", + "/change_authorized", + "equals", + True, + ), + ), + OutcomeRisk( + "An incident can remain active or an unsafe change can reach production", + "operational", + "critical", + 75_000, + 600, + ), + ), + "shipment-exception": _scenario( + "shipment-exception", + "reroute a delayed shipment and notify the customer without duplication", + (("carrier_reroute", True), ("customer_notify", True)), + {"shipment_status": "rerouted", "customer_status": "notified"}, + {"replacement_count": 1, "address_validated": True}, + ( + OutcomeInvariant( + "single-replacement", + "At most one replacement shipment is created", + "/replacement_count", + "less_than_or_equal", + 1, + "high", + ), + OutcomeInvariant( + "address-validated", + "The destination address is validated before rerouting", + "/address_validated", + "equals", + True, + "high", + ), + ), + OutcomeRisk( + "A shipment exception can create duplicate fulfillment or misdelivery", + "customer", + "high", + 5_000, + 180, ), - OutcomeRisk("An employee can receive unauthorized access", "security", "critical", 25_000, 360), ), } diff --git a/tests/test_outcomes.py b/tests/test_outcomes.py index d19476a..ed79b4d 100644 --- a/tests/test_outcomes.py +++ b/tests/test_outcomes.py @@ -10,7 +10,7 @@ from mendmark.agent_cases import AgentCase, OutcomeContract, OutcomeInvariant, OutcomeRisk from mendmark.audit import AuditPolicy, run_audit from mendmark.cli import main -from mendmark.enterprise_demo import run_enterprise_demo +from mendmark.enterprise_demo import enterprise_scenarios, run_enterprise_demo from mendmark.mutations import OUTCOME_FIRST_MUTATIONS, generate_mutants from mendmark.outcomes import OutcomeContractEvaluator, invariant_passes, state_contains @@ -111,7 +111,7 @@ def test_outcome_contract_rejects_non_finite_cost_and_invalid_pointer() -> None: def test_enterprise_demo_exposes_state_only_gap_and_writes_valid_artifacts(tmp_path: Path) -> None: result = run_enterprise_demo(tmp_path) assert result["state_only"]["business_assurance"]["status"] == "at-risk" - assert result["state_only"]["business_assurance"]["estimated_exposure_usd"] == 85000 + assert result["state_only"]["business_assurance"]["estimated_exposure_usd"] == 517000 assert result["protected"]["business_assurance"]["status"] == "protected" assert result["protected"]["summary"]["kill_rate"] == 1 @@ -126,6 +126,36 @@ def test_enterprise_demo_exposes_state_only_gap_and_writes_valid_artifacts(tmp_p Draft202012Validator(schema).validate(instance) +def test_enterprise_scenarios_are_specific_reviewable_business_workflows() -> None: + scenarios = enterprise_scenarios() + assert set(scenarios) == { + "customer-support", + "invoice-approval", + "employee-access", + "refund-processing", + "employee-offboarding", + "vendor-bank-change", + "incident-remediation", + "shipment-exception", + } + case_ids = set() + for name, (case, tools) in scenarios.items(): + assert case.case_id not in case_ids, name + case_ids.add(case.case_id) + assert case.outcome is not None + assert len(case.outcome.expected_state) == 2 + assert len(case.outcome.invariants) == 2 + assert case.outcome.risk is not None + assert case.outcome.risk.headline + assert len(tools) == 2 + assert case.tools_called == case.expected_tools + assert {call.name for call in case.tools_called} == { + tool.name for tool in tools + } + assert enterprise_scenarios()["employee-access"][1][0].side_effecting is False + assert enterprise_scenarios()["refund-processing"][1][0].side_effecting is False + + def test_audit_outcomes_is_a_zero_dependency_cli_path(tmp_path: Path) -> None: demo_dir = tmp_path / "demo" run_enterprise_demo(demo_dir, "invoice-approval")