diff --git a/app/agent/graph/assessment_graph.py b/app/agent/graph/assessment_graph.py index 9d834e8..0871c25 100644 --- a/app/agent/graph/assessment_graph.py +++ b/app/agent/graph/assessment_graph.py @@ -10,6 +10,7 @@ from app.core.config import settings from app.core.db import engine +from app.models.agent_execution import AgentTaskContract from app.models.assessment import AssessmentReport from app.models.governance import ControlEvidenceItem, ControlInstance from app.models.parser import ParsedDocument @@ -141,6 +142,17 @@ async def _build_context(state: AssessmentGraphState) -> AssessmentGraphState: } +async def _plan(state: AssessmentGraphState) -> AssessmentGraphState: + from app.agent.task_contract import build_plan_artifact + + return { + "plan_artifact": build_plan_artifact( + state["task_contract"], + skill_id=state.get("skill_id"), + ) + } + + async def _gather_context(state: AssessmentGraphState) -> AssessmentGraphState: from app.agent import orchestrator as legacy @@ -233,6 +245,23 @@ async def _verify_threat_evidence( return {"report": report} +async def _evaluate(state: AssessmentGraphState) -> AssessmentGraphState: + from app.agent.task_contract import evaluate_assessment_report + + evaluation = evaluate_assessment_report( + state["report"], + state["task_contract"], + ) + report = state["report"].model_copy( + update={ + "task_contract": state["task_contract"], + "plan_artifact": state["plan_artifact"], + "evaluation": evaluation, + } + ) + return {"evaluation": evaluation, "report": report} + + async def _persist_governance(state: AssessmentGraphState) -> AssessmentGraphState: try: count = persist_assessment_control_evidence( @@ -248,21 +277,25 @@ async def _persist_governance(state: AssessmentGraphState) -> AssessmentGraphSta def compile_assessment_graph(): graph = StateGraph(AssessmentGraphState) graph.add_node("load_skill", _load_skill) + graph.add_node("plan_assessment", _plan) graph.add_node("build_document_context", _build_context) graph.add_node("gather_policy_history_and_evidence", _gather_context) graph.add_node("draft_assessment", _draft) graph.add_node("review_assessment", _review) graph.add_node("parse_report", _parse_report) graph.add_node("verify_threat_evidence", _verify_threat_evidence) + graph.add_node("evaluate_assessment", _evaluate) graph.add_node("persist_gate3_control_evidence", _persist_governance) graph.add_edge(START, "load_skill") - graph.add_edge("load_skill", "build_document_context") + graph.add_edge("load_skill", "plan_assessment") + graph.add_edge("plan_assessment", "build_document_context") graph.add_edge("build_document_context", "gather_policy_history_and_evidence") graph.add_edge("gather_policy_history_and_evidence", "draft_assessment") graph.add_edge("draft_assessment", "review_assessment") graph.add_edge("review_assessment", "parse_report") graph.add_edge("parse_report", "verify_threat_evidence") - graph.add_edge("verify_threat_evidence", "persist_gate3_control_evidence") + graph.add_edge("verify_threat_evidence", "evaluate_assessment") + graph.add_edge("evaluate_assessment", "persist_gate3_control_evidence") graph.add_edge("persist_gate3_control_evidence", END) return graph.compile() @@ -275,7 +308,16 @@ async def run_assessment_graph( project_id: str | None = None, phase: str | None = None, skill_id: str | None = None, + task_contract: AgentTaskContract | None = None, ) -> AssessmentReport: + if task_contract is None: + from app.agent.task_contract import build_task_contract + + task_contract = build_task_contract( + task_id=task_id, + parsed_documents=parsed_documents, + phase=phase, + ) compiled = compile_assessment_graph() final_state = await compiled.ainvoke( { @@ -285,6 +327,7 @@ async def run_assessment_graph( "project_id": project_id, "phase": phase, "skill_id": skill_id, + "task_contract": task_contract, } ) return final_state["report"] diff --git a/app/agent/graph/state_types.py b/app/agent/graph/state_types.py index 73befad..cfa5784 100644 --- a/app/agent/graph/state_types.py +++ b/app/agent/graph/state_types.py @@ -6,6 +6,11 @@ from langchain_core.messages import BaseMessage from langgraph.graph.message import add_messages +from app.models.agent_execution import ( + AgentTaskContract, + EvaluationArtifact, + PlanArtifact, +) from app.models.assessment import AssessmentReport from app.models.parser import ParsedDocument @@ -58,3 +63,6 @@ class AssessmentGraphState(TypedDict, total=False): reviewed_raw: str report: AssessmentReport persisted_controls: int + task_contract: AgentTaskContract + plan_artifact: PlanArtifact + evaluation: EvaluationArtifact diff --git a/app/agent/orchestrator.py b/app/agent/orchestrator.py index dec8b85..bf7fc33 100644 --- a/app/agent/orchestrator.py +++ b/app/agent/orchestrator.py @@ -14,6 +14,7 @@ from app.core.guardrails import UNTRUSTED_CONTENT_INSTRUCTION, wrap_untrusted_content from app.kb.service import get_kb_service from app.llm.base import invoke_llm +from app.models.agent_execution import AgentTaskContract from app.models.assessment import ( AssessmentReport, ComplianceGap, @@ -119,6 +120,7 @@ async def run_assessment( project_id: str | None = None, phase: str | None = None, skill_id: str | None = None, + task_contract: AgentTaskContract | None = None, ) -> AssessmentReport: from app.agent.graph.assessment_graph import run_assessment_graph @@ -129,6 +131,7 @@ async def run_assessment( project_id=project_id, phase=phase, skill_id=skill_id, + task_contract=task_contract, ) diff --git a/app/agent/task_contract.py b/app/agent/task_contract.py new file mode 100644 index 0000000..2afe269 --- /dev/null +++ b/app/agent/task_contract.py @@ -0,0 +1,190 @@ +"""Deterministic Task Contract, planning, and evaluation helpers.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from hashlib import sha256 +from uuid import UUID + +from app.models.agent_execution import ( + AgentTaskContract, + EvaluationArtifact, + EvaluationCheck, + PlanArtifact, + PlanStep, + TaskInputReference, +) +from app.models.assessment import AssessmentReport +from app.models.parser import ParsedDocument + + +def build_task_contract( + *, + task_id: UUID, + parsed_documents: list[ParsedDocument], + phase: str | None, + created_at: datetime | None = None, +) -> AgentTaskContract: + """Create the bounded contract shared by REST, MCP, and A2A assessments.""" + inputs = tuple(_input_reference(document) for document in parsed_documents) + normalized_phase = phase or "auto" + risk_tier = ( + "high" + if normalized_phase in {"deployment", "operations", "full_ssdlc"} + else "medium" + ) + return AgentTaskContract( + task_id=task_id, + created_at=created_at or datetime.now(UTC), + goal=( + "Assess the submitted documents for security, compliance, and " + f"evidence gaps in the {normalized_phase} SSDLC phase." + ), + inputs=inputs, + allowed_paths=tuple(item.uri for item in inputs), + allowed_tools=( + "document.read", + "knowledge_base.search", + "llm.generate", + "evidence.verify", + ), + expected_outputs=("assessment_report.v2", "evaluation_artifact.v1"), + success_criteria=( + "The report satisfies the AssessmentReport schema.", + "Every supported citation resolves to an allowed input or trusted source.", + "Threat evidence is evaluated independently from report drafting.", + "The result remains pending until the configured human review completes.", + ), + risk_tier=risk_tier, + # Plan-first enforcement is a later M1 policy slice. Until then every + # task remains under the existing mandatory human-review boundary. + approval_mode="human_review", + retry_limit=2, + escalation_owner="security_reviewer", + ) + + +def build_plan_artifact( + contract: AgentTaskContract, + *, + skill_id: str | None, +) -> PlanArtifact: + """Produce a visible plan without invoking tools or reading document content.""" + return PlanArtifact( + task_id=contract.task_id, + created_at=datetime.now(UTC), + skill_id=skill_id, + steps=( + PlanStep( + id="plan-1", + phase="plan", + action="Validate the task contract and select the assessment skill.", + inputs=("agent_task_contract.v1",), + outputs=("plan_artifact.v1",), + success_check="Contract fields are valid and all inputs are in scope.", + ), + PlanStep( + id="act-1", + phase="act", + action="Build bounded document context and retrieve policy evidence.", + inputs=contract.allowed_paths, + outputs=("document_context", "policy_context", "history_context"), + success_check=( + "Only allowed inputs and read-only retrieval tools are used." + ), + ), + PlanStep( + id="act-2", + phase="act", + action="Draft and independently review the structured assessment.", + inputs=("document_context", "policy_context", "history_context"), + outputs=("assessment_report.v2",), + success_check="The reviewer emits a schema-valid assessment report.", + ), + PlanStep( + id="evaluate-1", + phase="evaluate", + action="Verify evidence and apply deterministic success checks.", + inputs=("assessment_report.v2",), + outputs=("evaluation_artifact.v1",), + success_check=( + "Required checks pass or the result is marked for review." + ), + ), + ), + ) + + +def evaluate_assessment_report( + report: AssessmentReport, + contract: AgentTaskContract, +) -> EvaluationArtifact: + """Evaluate generated output with deterministic checks outside the drafter.""" + schema_valid = True + schema_details = "AssessmentReport schema validation passed." + try: + AssessmentReport.model_validate(report.model_dump()) + except ValueError as exc: + schema_valid = False + schema_details = f"AssessmentReport schema validation failed: {exc}" + + finding_count = len(report.risk_items) + len(report.compliance_gaps) + evidence_present = finding_count == 0 or bool(report.sources) + threats = report.threat_model.threats if report.threat_model else [] + threats_verified = all(threat.verification is not None for threat in threats) + + checks = ( + EvaluationCheck( + name="schema_valid", + passed=schema_valid, + details=schema_details, + ), + EvaluationCheck( + name="task_identity_matches", + passed=report.task_id == str(contract.task_id), + details="Report task ID must match the immutable task contract.", + ), + EvaluationCheck( + name="required_output_present", + passed=bool(report.summary.strip()) and report.status != "failed", + details="A non-empty, non-failed assessment report is required.", + ), + EvaluationCheck( + name="finding_evidence_present", + passed=evidence_present, + required=False, + details="Findings should include at least one structured source citation.", + ), + EvaluationCheck( + name="threat_evidence_verified", + passed=threats_verified, + details="Every generated threat must pass through the evidence critic.", + ), + ) + required_failed = any(not check.passed for check in checks if check.required) + advisory_failed = any(not check.passed for check in checks if not check.required) + if required_failed: + outcome = "failed" + elif advisory_failed: + outcome = "needs_review" + else: + outcome = "passed" + return EvaluationArtifact( + task_id=contract.task_id, + created_at=datetime.now(UTC), + outcome=outcome, + checks=checks, + ) + + +def _input_reference(document: ParsedDocument) -> TaskInputReference: + content = ( + document.content if isinstance(document.content, str) else str(document.content) + ) + digest = document.metadata.file_hash or sha256(content.encode("utf-8")).hexdigest() + return TaskInputReference( + uri=f"document://{digest}", + filename=document.metadata.filename, + media_type=document.metadata.type, + sha256=digest, + ) diff --git a/app/models/__init__.py b/app/models/__init__.py index 281716f..0ba6a56 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -1,3 +1,11 @@ +from .agent_execution import ( + AgentTaskContract, + EvaluationArtifact, + EvaluationCheck, + PlanArtifact, + PlanStep, + TaskInputReference, +) from .assessment import ( AssessmentReport, AssessmentTaskCreated, @@ -38,6 +46,7 @@ __all__ = [ "AgentIntegrationStatus", "AgentProtocolEndpoint", + "AgentTaskContract", "AssessmentReport", "AssessmentTaskCreated", "AssessmentTaskResult", @@ -48,12 +57,16 @@ "ControlInstance", "CrossPhaseRef", "DreadScore", + "EvaluationArtifact", + "EvaluationCheck", "GateSubmission", "GovernanceAuditLog", "OrgFrameworkConfig", "ParsedDocument", "PolicyDocument", "PolicyEmbedding", + "PlanArtifact", + "PlanStep", "Project", "PromptAuditLog", "QuestionInstance", @@ -66,6 +79,7 @@ "SubAgentRun", "Threat", "ThreatModel", + "TaskInputReference", "TrackedRemediation", "User", "Vulnerability", diff --git a/app/models/agent_execution.py b/app/models/agent_execution.py new file mode 100644 index 0000000..5ef566d --- /dev/null +++ b/app/models/agent_execution.py @@ -0,0 +1,69 @@ +"""Versioned, immutable artifacts for bounded agent execution.""" + +from datetime import datetime +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + + +class _ImmutableArtifact(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + +class TaskInputReference(_ImmutableArtifact): + uri: str + filename: str + media_type: str + sha256: str + + +class AgentTaskContract(_ImmutableArtifact): + version: Literal["1.0"] = "1.0" + task_id: UUID + created_at: datetime + goal: str + inputs: tuple[TaskInputReference, ...] = Field(min_length=1) + allowed_paths: tuple[str, ...] = Field(min_length=1) + allowed_tools: tuple[str, ...] = Field(min_length=1) + expected_outputs: tuple[str, ...] = Field(min_length=1) + success_criteria: tuple[str, ...] = Field(min_length=1) + risk_tier: Literal["low", "medium", "high", "critical"] + approval_mode: Literal["human_review", "plan_first"] + retry_limit: int = Field(ge=0, le=5) + escalation_owner: str + + +class PlanStep(_ImmutableArtifact): + id: str + phase: Literal["plan", "act", "evaluate"] + action: str + inputs: tuple[str, ...] = () + outputs: tuple[str, ...] = () + success_check: str + + +class PlanArtifact(_ImmutableArtifact): + version: Literal["1.0"] = "1.0" + task_id: UUID + contract_version: Literal["1.0"] = "1.0" + created_at: datetime + mode: Literal["read_only"] = "read_only" + skill_id: str | None = None + steps: tuple[PlanStep, ...] = Field(min_length=1) + + +class EvaluationCheck(_ImmutableArtifact): + name: str + passed: bool + required: bool = True + details: str + + +class EvaluationArtifact(_ImmutableArtifact): + version: Literal["1.0"] = "1.0" + task_id: UUID + created_at: datetime + evaluator: Literal["deterministic_policy"] = "deterministic_policy" + outcome: Literal["passed", "needs_review", "failed"] + checks: tuple[EvaluationCheck, ...] = Field(min_length=1) diff --git a/app/models/assessment.py b/app/models/assessment.py index 73ec42d..1b7e940 100644 --- a/app/models/assessment.py +++ b/app/models/assessment.py @@ -10,6 +10,12 @@ from pydantic import BaseModel, Field +from app.models.agent_execution import ( + AgentTaskContract, + EvaluationArtifact, + PlanArtifact, +) + class RiskItem(BaseModel): id: str @@ -168,6 +174,9 @@ class AssessmentReport(BaseModel): confidence: float = Field(default=0.0, ge=0.0, le=1.0) sources: list[SourceCitation] = Field(default_factory=list) metadata: ReportMetadata | None = None + task_contract: AgentTaskContract | None = None + plan_artifact: PlanArtifact | None = None + evaluation: EvaluationArtifact | None = None format: Literal["json", "markdown"] = "json" @@ -175,6 +184,7 @@ class AssessmentTaskCreated(BaseModel): task_id: UUID status: Literal["accepted", "queued"] message: str | None = None + task_contract: AgentTaskContract class AssessmentTaskResult(BaseModel): @@ -196,6 +206,9 @@ class AssessmentTaskResult(BaseModel): version: int = 1 assignee: str | None = None comments: list[dict] = Field(default_factory=list) + task_contract: AgentTaskContract + plan_artifact: PlanArtifact | None = None + evaluation: EvaluationArtifact | None = None class RemediationTracking(BaseModel): diff --git a/app/services/assessment_service.py b/app/services/assessment_service.py index c8372ae..7e79e05 100644 --- a/app/services/assessment_service.py +++ b/app/services/assessment_service.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import inspect import logging from collections.abc import Awaitable, Callable from datetime import UTC, datetime @@ -10,6 +11,7 @@ from uuid import UUID, uuid4 from app.agent.orchestrator import run_assessment +from app.agent.task_contract import build_task_contract from app.kb.service import get_kb_service from app.models.assessment import ( AssessmentReport, @@ -42,6 +44,21 @@ class InvalidTaskStateError(ValueError): """Raised when an operation is invalid for the current task state.""" +def _runner_accepts_task_contract(runner: AssessmentRunner) -> bool: + """Detect the optional contract parameter without executing a runner twice.""" + side_effect = getattr(runner, "side_effect", None) + target = side_effect if callable(side_effect) else runner + try: + parameters = inspect.signature(target).parameters.values() + except (TypeError, ValueError): + return False + return any( + parameter.name == "task_contract" + or parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) + + class AssessmentService: def __init__(self) -> None: self._tasks: dict[str, dict[str, Any]] = {} @@ -64,6 +81,12 @@ async def submit( task_id = uuid4() task_id_str = str(task_id) created_at = datetime.now(UTC) + task_contract = build_task_contract( + task_id=task_id, + parsed_documents=parsed_documents, + phase=phase, + created_at=created_at, + ) self._tasks[task_id_str] = { "task_id": task_id, "status": "pending", @@ -81,6 +104,7 @@ async def submit( "revisions": [], "comments": [], "remediation_tracking": {}, + "task_contract": task_contract, } asyncio.create_task( self._run( @@ -99,6 +123,7 @@ async def submit( task_id=task_id, status="accepted", message="Assessment task created.", + task_contract=task_contract, ) async def _run( @@ -123,14 +148,19 @@ async def _run( } ) try: - report = await runner( - task_id, - parsed_documents, - scenario_id=scenario_id, - project_id=project_id, - phase=phase, - skill_id=skill_id, - ) + runner_kwargs = { + "scenario_id": scenario_id, + "project_id": project_id, + "phase": phase, + "skill_id": skill_id, + } + if _runner_accepts_task_contract(runner): + runner_kwargs["task_contract"] = task["task_contract"] + report = await runner(task_id, parsed_documents, **runner_kwargs) + if report.task_contract is None: + report = report.model_copy( + update={"task_contract": task["task_contract"]} + ) if report.metadata: report.metadata.ssdlc_stage = phase report.metadata.ssdlc_phase = phase @@ -151,6 +181,8 @@ async def _run( "report": report.model_dump(), "completed_at": now, "remediation_tracking": tracking, + "plan_artifact": report.plan_artifact, + "evaluation": report.evaluation, } ) task["revisions"].append( @@ -215,6 +247,9 @@ def get(self, task_id: str) -> AssessmentTaskResult: version=task.get("version", 1), assignee=task.get("assignee"), comments=task.get("comments", []), + task_contract=task["task_contract"], + plan_artifact=task.get("plan_artifact"), + evaluation=task.get("evaluation"), ) def list( diff --git a/docs/openapi.json b/docs/openapi.json index fb168b6..18c80e0 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -136,6 +136,115 @@ "title": "AgentProtocolEndpoint", "type": "object" }, + "AgentTaskContract": { + "additionalProperties": false, + "properties": { + "allowed_paths": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Allowed Paths", + "type": "array" + }, + "allowed_tools": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Allowed Tools", + "type": "array" + }, + "approval_mode": { + "enum": [ + "human_review", + "plan_first" + ], + "title": "Approval Mode", + "type": "string" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "escalation_owner": { + "title": "Escalation Owner", + "type": "string" + }, + "expected_outputs": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Expected Outputs", + "type": "array" + }, + "goal": { + "title": "Goal", + "type": "string" + }, + "inputs": { + "items": { + "$ref": "#/components/schemas/TaskInputReference" + }, + "minItems": 1, + "title": "Inputs", + "type": "array" + }, + "retry_limit": { + "maximum": 5.0, + "minimum": 0.0, + "title": "Retry Limit", + "type": "integer" + }, + "risk_tier": { + "enum": [ + "low", + "medium", + "high", + "critical" + ], + "title": "Risk Tier", + "type": "string" + }, + "success_criteria": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Success Criteria", + "type": "array" + }, + "task_id": { + "format": "uuid", + "title": "Task Id", + "type": "string" + }, + "version": { + "const": "1.0", + "default": "1.0", + "title": "Version", + "type": "string" + } + }, + "required": [ + "task_id", + "created_at", + "goal", + "inputs", + "allowed_paths", + "allowed_tools", + "expected_outputs", + "success_criteria", + "risk_tier", + "approval_mode", + "retry_limit", + "escalation_owner" + ], + "title": "AgentTaskContract", + "type": "object" + }, "AssessmentReport": { "properties": { "compliance_gaps": { @@ -159,6 +268,16 @@ "title": "Cross Phase Refs", "type": "array" }, + "evaluation": { + "anyOf": [ + { + "$ref": "#/components/schemas/EvaluationArtifact" + }, + { + "type": "null" + } + ] + }, "format": { "default": "json", "enum": [ @@ -189,6 +308,16 @@ ], "title": "Phase" }, + "plan_artifact": { + "anyOf": [ + { + "$ref": "#/components/schemas/PlanArtifact" + }, + { + "type": "null" + } + ] + }, "remediations": { "items": { "$ref": "#/components/schemas/Remediation" @@ -223,6 +352,16 @@ "title": "Summary", "type": "string" }, + "task_contract": { + "anyOf": [ + { + "$ref": "#/components/schemas/AgentTaskContract" + }, + { + "type": "null" + } + ] + }, "task_id": { "title": "Task Id", "type": "string" @@ -279,6 +418,9 @@ "title": "Status", "type": "string" }, + "task_contract": { + "$ref": "#/components/schemas/AgentTaskContract" + }, "task_id": { "format": "uuid", "title": "Task Id", @@ -287,7 +429,8 @@ }, "required": [ "task_id", - "status" + "status", + "task_contract" ], "title": "AssessmentTaskCreated", "type": "object" @@ -341,6 +484,26 @@ ], "title": "Error Message" }, + "evaluation": { + "anyOf": [ + { + "$ref": "#/components/schemas/EvaluationArtifact" + }, + { + "type": "null" + } + ] + }, + "plan_artifact": { + "anyOf": [ + { + "$ref": "#/components/schemas/PlanArtifact" + }, + { + "type": "null" + } + ] + }, "report": { "anyOf": [ { @@ -365,6 +528,9 @@ "title": "Status", "type": "string" }, + "task_contract": { + "$ref": "#/components/schemas/AgentTaskContract" + }, "task_id": { "format": "uuid", "title": "Task Id", @@ -379,7 +545,8 @@ "required": [ "task_id", "status", - "created_at" + "created_at", + "task_contract" ], "title": "AssessmentTaskResult", "type": "object" @@ -717,6 +884,87 @@ "title": "DreadScore", "type": "object" }, + "EvaluationArtifact": { + "additionalProperties": false, + "properties": { + "checks": { + "items": { + "$ref": "#/components/schemas/EvaluationCheck" + }, + "minItems": 1, + "title": "Checks", + "type": "array" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "evaluator": { + "const": "deterministic_policy", + "default": "deterministic_policy", + "title": "Evaluator", + "type": "string" + }, + "outcome": { + "enum": [ + "passed", + "needs_review", + "failed" + ], + "title": "Outcome", + "type": "string" + }, + "task_id": { + "format": "uuid", + "title": "Task Id", + "type": "string" + }, + "version": { + "const": "1.0", + "default": "1.0", + "title": "Version", + "type": "string" + } + }, + "required": [ + "task_id", + "created_at", + "outcome", + "checks" + ], + "title": "EvaluationArtifact", + "type": "object" + }, + "EvaluationCheck": { + "additionalProperties": false, + "properties": { + "details": { + "title": "Details", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "passed": { + "title": "Passed", + "type": "boolean" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "passed", + "details" + ], + "title": "EvaluationCheck", + "type": "object" + }, "EvidenceCriticSummary": { "properties": { "contradicted": { @@ -1079,6 +1327,115 @@ "title": "OrgFrameworkConfigUpdate", "type": "object" }, + "PlanArtifact": { + "additionalProperties": false, + "properties": { + "contract_version": { + "const": "1.0", + "default": "1.0", + "title": "Contract Version", + "type": "string" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "mode": { + "const": "read_only", + "default": "read_only", + "title": "Mode", + "type": "string" + }, + "skill_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Skill Id" + }, + "steps": { + "items": { + "$ref": "#/components/schemas/PlanStep" + }, + "minItems": 1, + "title": "Steps", + "type": "array" + }, + "task_id": { + "format": "uuid", + "title": "Task Id", + "type": "string" + }, + "version": { + "const": "1.0", + "default": "1.0", + "title": "Version", + "type": "string" + } + }, + "required": [ + "task_id", + "created_at", + "steps" + ], + "title": "PlanArtifact", + "type": "object" + }, + "PlanStep": { + "additionalProperties": false, + "properties": { + "action": { + "title": "Action", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "inputs": { + "default": [], + "items": { + "type": "string" + }, + "title": "Inputs", + "type": "array" + }, + "outputs": { + "default": [], + "items": { + "type": "string" + }, + "title": "Outputs", + "type": "array" + }, + "phase": { + "enum": [ + "plan", + "act", + "evaluate" + ], + "title": "Phase", + "type": "string" + }, + "success_check": { + "title": "Success Check", + "type": "string" + } + }, + "required": [ + "id", + "phase", + "action", + "success_check" + ], + "title": "PlanStep", + "type": "object" + }, "ProjectCreate": { "properties": { "ai_risk_class": { @@ -2391,6 +2748,35 @@ "title": "SubmitAnswersRequest", "type": "object" }, + "TaskInputReference": { + "additionalProperties": false, + "properties": { + "filename": { + "title": "Filename", + "type": "string" + }, + "media_type": { + "title": "Media Type", + "type": "string" + }, + "sha256": { + "title": "Sha256", + "type": "string" + }, + "uri": { + "title": "Uri", + "type": "string" + } + }, + "required": [ + "uri", + "filename", + "media_type", + "sha256" + ], + "title": "TaskInputReference", + "type": "object" + }, "Threat": { "properties": { "affected_component": { diff --git a/docs/schemas/assessment-report.json b/docs/schemas/assessment-report.json index cc9230c..b04060d 100644 --- a/docs/schemas/assessment-report.json +++ b/docs/schemas/assessment-report.json @@ -1,5 +1,114 @@ { "$defs": { + "AgentTaskContract": { + "additionalProperties": false, + "properties": { + "allowed_paths": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Allowed Paths", + "type": "array" + }, + "allowed_tools": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Allowed Tools", + "type": "array" + }, + "approval_mode": { + "enum": [ + "human_review", + "plan_first" + ], + "title": "Approval Mode", + "type": "string" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "escalation_owner": { + "title": "Escalation Owner", + "type": "string" + }, + "expected_outputs": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Expected Outputs", + "type": "array" + }, + "goal": { + "title": "Goal", + "type": "string" + }, + "inputs": { + "items": { + "$ref": "#/$defs/TaskInputReference" + }, + "minItems": 1, + "title": "Inputs", + "type": "array" + }, + "retry_limit": { + "maximum": 5, + "minimum": 0, + "title": "Retry Limit", + "type": "integer" + }, + "risk_tier": { + "enum": [ + "low", + "medium", + "high", + "critical" + ], + "title": "Risk Tier", + "type": "string" + }, + "success_criteria": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Success Criteria", + "type": "array" + }, + "task_id": { + "format": "uuid", + "title": "Task Id", + "type": "string" + }, + "version": { + "const": "1.0", + "default": "1.0", + "title": "Version", + "type": "string" + } + }, + "required": [ + "task_id", + "created_at", + "goal", + "inputs", + "allowed_paths", + "allowed_tools", + "expected_outputs", + "success_criteria", + "risk_tier", + "approval_mode", + "retry_limit", + "escalation_owner" + ], + "title": "AgentTaskContract", + "type": "object" + }, "ComplianceGap": { "properties": { "citation_ids": { @@ -201,6 +310,87 @@ "title": "DreadScore", "type": "object" }, + "EvaluationArtifact": { + "additionalProperties": false, + "properties": { + "checks": { + "items": { + "$ref": "#/$defs/EvaluationCheck" + }, + "minItems": 1, + "title": "Checks", + "type": "array" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "evaluator": { + "const": "deterministic_policy", + "default": "deterministic_policy", + "title": "Evaluator", + "type": "string" + }, + "outcome": { + "enum": [ + "passed", + "needs_review", + "failed" + ], + "title": "Outcome", + "type": "string" + }, + "task_id": { + "format": "uuid", + "title": "Task Id", + "type": "string" + }, + "version": { + "const": "1.0", + "default": "1.0", + "title": "Version", + "type": "string" + } + }, + "required": [ + "task_id", + "created_at", + "outcome", + "checks" + ], + "title": "EvaluationArtifact", + "type": "object" + }, + "EvaluationCheck": { + "additionalProperties": false, + "properties": { + "details": { + "title": "Details", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "passed": { + "title": "Passed", + "type": "boolean" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "passed", + "details" + ], + "title": "EvaluationCheck", + "type": "object" + }, "EvidenceCriticSummary": { "properties": { "contradicted": { @@ -293,6 +483,116 @@ "title": "EvidenceVerification", "type": "object" }, + "PlanArtifact": { + "additionalProperties": false, + "properties": { + "contract_version": { + "const": "1.0", + "default": "1.0", + "title": "Contract Version", + "type": "string" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "mode": { + "const": "read_only", + "default": "read_only", + "title": "Mode", + "type": "string" + }, + "skill_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Skill Id" + }, + "steps": { + "items": { + "$ref": "#/$defs/PlanStep" + }, + "minItems": 1, + "title": "Steps", + "type": "array" + }, + "task_id": { + "format": "uuid", + "title": "Task Id", + "type": "string" + }, + "version": { + "const": "1.0", + "default": "1.0", + "title": "Version", + "type": "string" + } + }, + "required": [ + "task_id", + "created_at", + "steps" + ], + "title": "PlanArtifact", + "type": "object" + }, + "PlanStep": { + "additionalProperties": false, + "properties": { + "action": { + "title": "Action", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "inputs": { + "default": [], + "items": { + "type": "string" + }, + "title": "Inputs", + "type": "array" + }, + "outputs": { + "default": [], + "items": { + "type": "string" + }, + "title": "Outputs", + "type": "array" + }, + "phase": { + "enum": [ + "plan", + "act", + "evaluate" + ], + "title": "Phase", + "type": "string" + }, + "success_check": { + "title": "Success Check", + "type": "string" + } + }, + "required": [ + "id", + "phase", + "action", + "success_check" + ], + "title": "PlanStep", + "type": "object" + }, "Remediation": { "properties": { "action": { @@ -675,6 +975,35 @@ "title": "SourceCitation", "type": "object" }, + "TaskInputReference": { + "additionalProperties": false, + "properties": { + "filename": { + "title": "Filename", + "type": "string" + }, + "media_type": { + "title": "Media Type", + "type": "string" + }, + "sha256": { + "title": "Sha256", + "type": "string" + }, + "uri": { + "title": "Uri", + "type": "string" + } + }, + "required": [ + "uri", + "filename", + "media_type", + "sha256" + ], + "title": "TaskInputReference", + "type": "object" + }, "Threat": { "properties": { "affected_component": { @@ -958,6 +1287,17 @@ "title": "Cross Phase Refs", "type": "array" }, + "evaluation": { + "anyOf": [ + { + "$ref": "#/$defs/EvaluationArtifact" + }, + { + "type": "null" + } + ], + "default": null + }, "format": { "default": "json", "enum": [ @@ -990,6 +1330,17 @@ "default": null, "title": "Phase" }, + "plan_artifact": { + "anyOf": [ + { + "$ref": "#/$defs/PlanArtifact" + }, + { + "type": "null" + } + ], + "default": null + }, "remediations": { "items": { "$ref": "#/$defs/Remediation" @@ -1024,6 +1375,17 @@ "title": "Summary", "type": "string" }, + "task_contract": { + "anyOf": [ + { + "$ref": "#/$defs/AgentTaskContract" + }, + { + "type": "null" + } + ], + "default": null + }, "task_id": { "title": "Task Id", "type": "string" diff --git a/frontend/src/api/schema.d.ts b/frontend/src/api/schema.d.ts index b45aab4..8981088 100644 --- a/frontend/src/api/schema.d.ts +++ b/frontend/src/api/schema.d.ts @@ -1129,6 +1129,51 @@ export interface components { /** Transport */ transport: string; }; + /** AgentTaskContract */ + AgentTaskContract: { + /** Allowed Paths */ + allowed_paths: string[]; + /** Allowed Tools */ + allowed_tools: string[]; + /** + * Approval Mode + * @enum {string} + */ + approval_mode: "human_review" | "plan_first"; + /** + * Created At + * Format: date-time + */ + created_at: string; + /** Escalation Owner */ + escalation_owner: string; + /** Expected Outputs */ + expected_outputs: string[]; + /** Goal */ + goal: string; + /** Inputs */ + inputs: components["schemas"]["TaskInputReference"][]; + /** Retry Limit */ + retry_limit: number; + /** + * Risk Tier + * @enum {string} + */ + risk_tier: "low" | "medium" | "high" | "critical"; + /** Success Criteria */ + success_criteria: string[]; + /** + * Task Id + * Format: uuid + */ + task_id: string; + /** + * Version + * @default 1.0 + * @constant + */ + version: "1.0"; + }; /** AssessmentReport */ AssessmentReport: { /** Compliance Gaps */ @@ -1140,6 +1185,7 @@ export interface components { confidence: number; /** Cross Phase Refs */ cross_phase_refs?: components["schemas"]["CrossPhaseRef"][]; + evaluation?: components["schemas"]["EvaluationArtifact"] | null; /** * Format * @default json @@ -1149,6 +1195,7 @@ export interface components { metadata?: components["schemas"]["ReportMetadata"] | null; /** Phase */ phase?: string | null; + plan_artifact?: components["schemas"]["PlanArtifact"] | null; /** Remediations */ remediations?: components["schemas"]["Remediation"][]; /** Risk Items */ @@ -1162,6 +1209,7 @@ export interface components { status: "completed" | "partial" | "failed"; /** Summary */ summary: string; + task_contract?: components["schemas"]["AgentTaskContract"] | null; /** Task Id */ task_id: string; threat_model?: components["schemas"]["ThreatModel"] | null; @@ -1182,6 +1230,7 @@ export interface components { * @enum {string} */ status: "accepted" | "queued"; + task_contract: components["schemas"]["AgentTaskContract"]; /** * Task Id * Format: uuid @@ -1205,12 +1254,15 @@ export interface components { created_at: string; /** Error Message */ error_message?: string | null; + evaluation?: components["schemas"]["EvaluationArtifact"] | null; + plan_artifact?: components["schemas"]["PlanArtifact"] | null; report?: components["schemas"]["AssessmentReport"] | null; /** * Status * @enum {string} */ status: "pending" | "running" | "review_pending" | "approved" | "rejected" | "escalated" | "completed" | "failed"; + task_contract: components["schemas"]["AgentTaskContract"]; /** * Task Id * Format: uuid @@ -1336,6 +1388,52 @@ export interface components { /** Total */ total?: number | null; }; + /** EvaluationArtifact */ + EvaluationArtifact: { + /** Checks */ + checks: components["schemas"]["EvaluationCheck"][]; + /** + * Created At + * Format: date-time + */ + created_at: string; + /** + * Evaluator + * @default deterministic_policy + * @constant + */ + evaluator: "deterministic_policy"; + /** + * Outcome + * @enum {string} + */ + outcome: "passed" | "needs_review" | "failed"; + /** + * Task Id + * Format: uuid + */ + task_id: string; + /** + * Version + * @default 1.0 + * @constant + */ + version: "1.0"; + }; + /** EvaluationCheck */ + EvaluationCheck: { + /** Details */ + details: string; + /** Name */ + name: string; + /** Passed */ + passed: boolean; + /** + * Required + * @default true + */ + required: boolean; + }; /** EvidenceCriticSummary */ EvidenceCriticSummary: { /** @@ -1489,6 +1587,65 @@ export interface components { /** Updated By Id */ updated_by_id?: number | null; }; + /** PlanArtifact */ + PlanArtifact: { + /** + * Contract Version + * @default 1.0 + * @constant + */ + contract_version: "1.0"; + /** + * Created At + * Format: date-time + */ + created_at: string; + /** + * Mode + * @default read_only + * @constant + */ + mode: "read_only"; + /** Skill Id */ + skill_id?: string | null; + /** Steps */ + steps: components["schemas"]["PlanStep"][]; + /** + * Task Id + * Format: uuid + */ + task_id: string; + /** + * Version + * @default 1.0 + * @constant + */ + version: "1.0"; + }; + /** PlanStep */ + PlanStep: { + /** Action */ + action: string; + /** Id */ + id: string; + /** + * Inputs + * @default [] + */ + inputs: string[]; + /** + * Outputs + * @default [] + */ + outputs: string[]; + /** + * Phase + * @enum {string} + */ + phase: "plan" | "act" | "evaluate"; + /** Success Check */ + success_check: string; + }; /** ProjectCreate */ ProjectCreate: { /** Ai Risk Class */ @@ -1839,6 +1996,17 @@ export interface components { [key: string]: unknown; }; }; + /** TaskInputReference */ + TaskInputReference: { + /** Filename */ + filename: string; + /** Media Type */ + media_type: string; + /** Sha256 */ + sha256: string; + /** Uri */ + uri: string; + }; /** Threat */ Threat: { /** Affected Component */ diff --git a/tests/test_agent_gateway.py b/tests/test_agent_gateway.py index 4aaf11a..98c2a16 100644 --- a/tests/test_agent_gateway.py +++ b/tests/test_agent_gateway.py @@ -1,5 +1,6 @@ import json from unittest.mock import AsyncMock +from uuid import UUID import httpx import pytest @@ -50,6 +51,7 @@ async def test_agent_submission_uses_shared_task_and_requires_review( assert result["status"] == "review_pending" assert result["report"]["phase"] == "design" runner.assert_awaited_once() + assert runner.await_args.kwargs["task_contract"].task_id == UUID(created["task_id"]) def test_agent_gateway_status_and_token_boundary(monkeypatch): diff --git a/tests/test_agent_task_contract.py b/tests/test_agent_task_contract.py new file mode 100644 index 0000000..fa7840c --- /dev/null +++ b/tests/test_agent_task_contract.py @@ -0,0 +1,98 @@ +"""Task Contract and Plan–Act–Evaluate artifact tests.""" + +from uuid import uuid4 + +import pytest +from pydantic import ValidationError + +from app.agent.task_contract import ( + build_plan_artifact, + build_task_contract, + evaluate_assessment_report, +) +from app.models.assessment import AssessmentReport, RiskItem +from app.models.parser import ParsedDocument, ParsedDocumentMetadata + + +def _document() -> ParsedDocument: + return ParsedDocument( + metadata=ParsedDocumentMetadata( + filename="design.md", + type="md", + file_hash="a" * 64, + ), + content="# Design\nThe API uses OIDC.", + ) + + +def test_task_contract_is_versioned_bounded_and_immutable(): + task_id = uuid4() + contract = build_task_contract( + task_id=task_id, + parsed_documents=[_document()], + phase="design", + ) + + assert contract.version == "1.0" + assert contract.task_id == task_id + assert contract.inputs[0].sha256 == "a" * 64 + assert contract.allowed_paths == (f"document://{'a' * 64}",) + assert contract.risk_tier == "medium" + assert contract.approval_mode == "human_review" + assert contract.retry_limit == 2 + assert contract.escalation_owner == "security_reviewer" + + with pytest.raises(ValidationError): + contract.goal = "Replace the approved goal" + + +def test_plan_exposes_plan_act_evaluate_order_without_mutating_contract(): + contract = build_task_contract( + task_id=uuid4(), + parsed_documents=[_document()], + phase="design", + ) + plan = build_plan_artifact(contract, skill_id="secure-design-review") + + assert plan.mode == "read_only" + assert tuple(step.phase for step in plan.steps) == ( + "plan", + "act", + "act", + "evaluate", + ) + assert plan.steps[1].inputs == contract.allowed_paths + + with pytest.raises(ValidationError): + plan.steps[0].action = "Skip planning" + + +def test_evaluation_is_independent_and_flags_findings_without_evidence(): + task_id = uuid4() + contract = build_task_contract( + task_id=task_id, + parsed_documents=[_document()], + phase="design", + ) + report = AssessmentReport( + task_id=str(task_id), + status="completed", + summary="One risk needs review.", + risk_items=[ + RiskItem( + id="R1", + title="Missing authorization detail", + severity="high", + ) + ], + ) + + evaluation = evaluate_assessment_report(report, contract) + + assert evaluation.evaluator == "deterministic_policy" + assert evaluation.outcome == "needs_review" + evidence_check = next( + check for check in evaluation.checks if check.name == "finding_evidence_present" + ) + assert evidence_check.passed is False + assert evidence_check.required is False diff --git a/tests/test_assessments_api.py b/tests/test_assessments_api.py index 86df165..e7d66a7 100644 --- a/tests/test_assessments_api.py +++ b/tests/test_assessments_api.py @@ -94,6 +94,9 @@ async def mock_run_assessment( data = r.json() assert data.get("status") == "accepted" assert "task_id" in data + assert data["task_contract"]["version"] == "1.0" + assert data["task_contract"]["goal"] + assert data["task_contract"]["allowed_paths"] task_id = data["task_id"] r2 = client.get(f"/api/v1/assessments/{task_id}") assert r2.status_code == 200 @@ -102,6 +105,8 @@ async def mock_run_assessment( assert r2_data.get("report") is not None assert "confidence" in r2_data.get("report", {}) assert "sources" in r2_data.get("report", {}) + assert r2_data["task_contract"] == data["task_contract"] + assert r2_data["report"]["task_contract"] == data["task_contract"] def test_submit_assessment_with_skill(client): diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index aa4f950..224f9fc 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -75,11 +75,21 @@ def _ctx(): async def test_orchestrator_uses_skill_prompt(): skill = _make_skill() with _patch_deps(skill) as mock_invoke: - await run_assessment(uuid4(), [_make_doc()], skill_id="test-skill") + report = await run_assessment( + uuid4(), + [_make_doc()], + skill_id="test-skill", + ) calls = mock_invoke.call_args_list found = any("YOU ARE A TEST SKILL" in call.args[0] for call in calls) assert found, "Skill system prompt was not injected into LLM calls" + assert report.task_contract is not None + assert report.plan_artifact is not None + assert report.plan_artifact.skill_id == "test-skill" + assert report.evaluation is not None + assert report.evaluation.evaluator == "deterministic_policy" + assert report.evaluation.outcome in {"passed", "needs_review"} # --------------------------------------------------------------------------- @@ -268,9 +278,15 @@ def test_graph_registry_exports_assessment_mermaid(): assert "docsentinel_assessment" in GRAPH_REGISTRY compiled = GRAPH_REGISTRY["docsentinel_assessment"][1]() mermaid = compiled.get_graph(xray=True).draw_mermaid() + assert "plan_assessment" in mermaid assert "draft_assessment" in mermaid assert "verify_threat_evidence" in mermaid + assert "evaluate_assessment" in mermaid assert "persist_gate3_control_evidence" in mermaid + assert "load_skill --> plan_assessment" in mermaid + assert "plan_assessment --> build_document_context" in mermaid + assert "verify_threat_evidence --> evaluate_assessment" in mermaid + assert "evaluate_assessment --> persist_gate3_control_evidence" in mermaid def test_normalize_threat_model_accepts_common_stride_variations():