diff --git a/harness/lib/research/decision_artifact.py b/harness/lib/research/decision_artifact.py new file mode 100644 index 000000000..21fc4cb3a --- /dev/null +++ b/harness/lib/research/decision_artifact.py @@ -0,0 +1,478 @@ +"""Construct evidence-bound decision artifacts without asserting human approval.""" + +from __future__ import annotations + +import hashlib +import json +import re +from datetime import datetime, timezone +from functools import lru_cache +from pathlib import Path +from typing import Any + +import jsonschema + + +EVIDENCE_SCHEMA_DIR = Path(__file__).resolve().parents[2] / "schemas" / "evidence" + + +class DecisionArtifactError(ValueError): + """Raised when a decision request cannot be supported by its evidence.""" + + +@lru_cache(maxsize=2) +def _typed_evidence_validator(evidence_type: str) -> jsonschema.Draft202012Validator: + schema_path = EVIDENCE_SCHEMA_DIR / f"{evidence_type}.v1.schema.json" + try: + schema = json.loads(schema_path.read_text(encoding="utf-8")) + jsonschema.Draft202012Validator.check_schema(schema) + except (OSError, json.JSONDecodeError, jsonschema.SchemaError) as exc: + raise DecisionArtifactError(f"typed evidence schema is unavailable: {schema_path}") from exc + return jsonschema.Draft202012Validator( + schema, + format_checker=jsonschema.FormatChecker(), + ) + + +def _validate_typed_evidence_schema(payload: Any, evidence_type: str, evidence_id: str) -> None: + try: + _typed_evidence_validator(evidence_type).validate(payload) + except jsonschema.ValidationError as exc: + location = "/" + "/".join(str(part) for part in exc.absolute_path) + raise DecisionArtifactError( + f"evidence {evidence_id} fails {evidence_type}.v1 schema at {location}: {exc.message}" + ) from exc + + +def _require_text(value: Any, field: str) -> str: + text = str(value or "").strip() + if not text: + raise DecisionArtifactError(f"{field} must be a non-empty string") + return text + + +def _require_unique(items: list[dict[str, Any]], key: str, field: str) -> dict[str, dict[str, Any]]: + indexed: dict[str, dict[str, Any]] = {} + for index, item in enumerate(items): + if not isinstance(item, dict): + raise DecisionArtifactError(f"{field}[{index}] must be an object") + item_id = _require_text(item.get(key), f"{field}[{index}].{key}") + if item_id in indexed: + raise DecisionArtifactError(f"duplicate {key}: {item_id}") + indexed[item_id] = item + return indexed + + +def _resolve_json_pointer(payload: Any, pointer: str) -> Any: + if pointer == "": + return payload + if not pointer.startswith("/"): + raise DecisionArtifactError(f"JSON pointer must start with '/': {pointer}") + current = payload + for raw_token in pointer[1:].split("/"): + if re.search(r"~(?:[^01]|$)", raw_token): + raise DecisionArtifactError(f"JSON pointer contains an invalid escape: {pointer}") + token = raw_token.replace("~1", "/").replace("~0", "~") + if isinstance(current, list): + if not re.fullmatch(r"0|[1-9][0-9]*", token): + raise DecisionArtifactError(f"JSON pointer has a non-canonical array index: {pointer}") + try: + current = current[int(token)] + except IndexError as exc: + raise DecisionArtifactError(f"JSON pointer does not resolve: {pointer}") from exc + elif isinstance(current, dict) and token in current: + current = current[token] + else: + raise DecisionArtifactError(f"JSON pointer does not resolve: {pointer}") + return current + + +def _inside(path: Path, root: Path) -> bool: + try: + path.relative_to(root) + return True + except ValueError: + return False + + +def _load_evidence( + entries: list[dict[str, Any]], *, source_root: Path +) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]: + indexed = _require_unique(entries, "evidence_id", "evidence") + loaded: list[dict[str, Any]] = [] + semantics: dict[str, dict[str, Any]] = {} + for evidence_id, entry in indexed.items(): + raw_path = _require_text(entry.get("source_path"), f"evidence[{evidence_id}].source_path") + source_path = Path(raw_path) + if not source_path.is_absolute(): + source_path = source_root / source_path + source_path = source_path.resolve() + if not _inside(source_path, source_root): + raise DecisionArtifactError(f"evidence path escapes source root: {raw_path}") + if not source_path.is_file(): + raise DecisionArtifactError(f"evidence file is missing: {raw_path}") + try: + evidence_bytes = source_path.read_bytes() + except OSError as exc: + raise DecisionArtifactError(f"evidence is not readable: {raw_path}") from exc + expected_sha256 = _require_text( + entry.get("expected_sha256"), f"evidence[{evidence_id}].expected_sha256" + ).lower() + if not re.fullmatch(r"[a-f0-9]{64}", expected_sha256): + raise DecisionArtifactError(f"evidence[{evidence_id}].expected_sha256 is invalid") + actual_sha256 = hashlib.sha256(evidence_bytes).hexdigest() + if actual_sha256 != expected_sha256: + raise DecisionArtifactError( + f"evidence {evidence_id} hash mismatch: expected {expected_sha256}, got {actual_sha256}" + ) + try: + payload = json.loads(evidence_bytes.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise DecisionArtifactError(f"evidence is not readable JSON: {raw_path}") from exc + evidence_type = _require_text( + entry.get("evidence_type"), f"evidence[{evidence_id}].evidence_type" + ) + if evidence_type not in {"claim_verdict", "experiment_result"}: + raise DecisionArtifactError(f"unknown evidence type for {evidence_id}: {evidence_type}") + allowed_fields = { + "evidence_id", + "evidence_type", + "source_path", + "expected_sha256", + "summary", + } + if evidence_type == "claim_verdict": + allowed_fields.add("claim_id") + allowed_fields.add("supporting_experiment_evidence_id") + unexpected_fields = sorted(set(entry) - allowed_fields) + if unexpected_fields: + raise DecisionArtifactError( + f"evidence[{evidence_id}] contains unsupported fields: {', '.join(unexpected_fields)}" + ) + _validate_typed_evidence_schema(payload, evidence_type, evidence_id) + if evidence_type == "claim_verdict": + if payload.get("status") != "completed": + raise DecisionArtifactError(f"evidence {evidence_id} claim verdict is not completed") + claim_id = _require_text(entry.get("claim_id"), f"evidence[{evidence_id}].claim_id") + verdicts = payload.get("outputs", {}).get("verdicts", []) + matches = [ + (index, verdict) + for index, verdict in enumerate(verdicts) + if isinstance(verdict, dict) and verdict.get("claim_id") == claim_id + ] if isinstance(verdicts, list) else [] + if len(matches) != 1: + raise DecisionArtifactError( + f"evidence {evidence_id} must contain exactly one verdict for claim {claim_id}" + ) + verdict_index, _ = matches[0] + pointer = f"/outputs/verdicts/{verdict_index}/verdict" + observed = _resolve_json_pointer(payload, pointer) + expected = "supported" + verdict = matches[0][1] + supporting_experiment_evidence_id = _require_text( + entry.get("supporting_experiment_evidence_id"), + f"evidence[{evidence_id}].supporting_experiment_evidence_id", + ) + experiment_evidence_ids_raw = verdict.get("experiment_evidence_ids") + if not isinstance(experiment_evidence_ids_raw, list) or not experiment_evidence_ids_raw: + raise DecisionArtifactError( + f"evidence[{evidence_id}].verdict.experiment_evidence_ids must not be empty" + ) + experiment_evidence_ids = { + _require_text(value, f"evidence[{evidence_id}].verdict.experiment_evidence_ids") + for value in experiment_evidence_ids_raw + } + verdict_evidence_ids = { + _require_text(value, f"evidence[{evidence_id}].verdict.evidence_ids") + for value in verdict.get("evidence_ids", []) + } + if not experiment_evidence_ids.issubset(verdict_evidence_ids): + raise DecisionArtifactError( + f"evidence[{evidence_id}].verdict.experiment_evidence_ids are not in verdict evidence_ids" + ) + semantics[evidence_id] = { + "type": evidence_type, + "experiment_evidence_id": supporting_experiment_evidence_id, + "experiment_id": _require_text( + verdict.get("experiment_id"), f"evidence[{evidence_id}].verdict.experiment_id" + ), + "experiment_evidence_ids": experiment_evidence_ids, + } + elif evidence_type == "experiment_result": + if payload.get("status") != "completed": + raise DecisionArtifactError(f"evidence {evidence_id} experiment is not completed") + pointer = "/outputs/result/outcome" + observed = _resolve_json_pointer(payload, pointer) + expected = "supports" + result = payload["outputs"]["result"] + semantics[evidence_id] = { + "type": evidence_type, + "experiment_id": result["experiment_id"], + "evidence_ids": set(result["evidence_ids"]), + } + else: # pragma: no cover - guarded by the typed policy check above + raise DecisionArtifactError(f"unknown evidence type for {evidence_id}: {evidence_type}") + if observed != expected: + raise DecisionArtifactError( + f"evidence {evidence_id} is not supportive: expected {expected!r}, observed {observed!r} at {pointer}" + ) + loaded.append( + { + "evidence_id": evidence_id, + "evidence_type": evidence_type, + "source_path": str(source_path), + "sha256": actual_sha256, + "semantic_locator": pointer, + "observed_support": observed, + "summary": _require_text(entry.get("summary"), f"evidence[{evidence_id}].summary"), + } + ) + loaded_by_id = {item["evidence_id"]: item for item in loaded} + for evidence_id, semantic in semantics.items(): + if semantic["type"] != "claim_verdict": + continue + experiment_evidence_id = semantic["experiment_evidence_id"] + experiment_semantic = semantics.get(experiment_evidence_id) + if not experiment_semantic or experiment_semantic["type"] != "experiment_result": + raise DecisionArtifactError( + f"evidence {evidence_id} references missing typed experiment evidence: {experiment_evidence_id}" + ) + if semantic["experiment_id"] != experiment_semantic["experiment_id"]: + raise DecisionArtifactError( + f"evidence {evidence_id} experiment_id does not match {experiment_evidence_id}" + ) + if not semantic["experiment_evidence_ids"] & experiment_semantic["evidence_ids"]: + raise DecisionArtifactError( + f"evidence {evidence_id} has no provenance overlap with {experiment_evidence_id}" + ) + loaded_by_id[evidence_id]["supporting_evidence_ids"] = [experiment_evidence_id] + return loaded, indexed + + +def _validate_refs(values: Any, allowed: set[str], field: str) -> list[str]: + if not isinstance(values, list) or not values: + raise DecisionArtifactError(f"{field} must contain at least one reference") + refs = [_require_text(value, field) for value in values] + if len(refs) != len(set(refs)): + raise DecisionArtifactError(f"{field} contains duplicate references") + missing = sorted(set(refs) - allowed) + if missing: + raise DecisionArtifactError(f"{field} contains unknown references: {', '.join(missing)}") + return refs + + +def construct_decision_artifact( + request: dict[str, Any], *, request_path: Path, source_root: Path +) -> dict[str, Any]: + """Validate a request and return a canonical, review-required decision artifact.""" + + request_path = request_path.resolve() + try: + request_bytes = request_path.read_bytes() + request_on_disk = json.loads(request_bytes.decode("utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise DecisionArtifactError(f"request_path is not readable decision JSON: {request_path}") from exc + if not isinstance(request, dict) or request_on_disk != request: + raise DecisionArtifactError("request object does not match request_path content") + request_sha256 = hashlib.sha256(request_bytes).hexdigest() + if request.get("schema") != "decision_request.v1": + raise DecisionArtifactError("schema must be decision_request.v1") + forbidden_state = sorted({"decision_status", "review", "approval"} & set(request)) + if forbidden_state: + raise DecisionArtifactError( + "constructor input cannot assert review or approval state: " + ", ".join(forbidden_state) + ) + source_root = source_root.resolve() + alternatives = request.get("alternatives") + criteria = request.get("criteria") + evidence = request.get("evidence") + assessments = request.get("assessments") + risks = request.get("risks") + recommendation = request.get("recommendation") + if not isinstance(alternatives, list) or len(alternatives) < 2: + raise DecisionArtifactError("alternatives must contain at least two options") + if not isinstance(criteria, list) or not criteria: + raise DecisionArtifactError("criteria must contain at least one criterion") + if not isinstance(evidence, list) or not evidence: + raise DecisionArtifactError("evidence must contain at least one source") + if not isinstance(assessments, list) or not assessments: + raise DecisionArtifactError("assessments must not be empty") + if not isinstance(risks, list) or not risks: + raise DecisionArtifactError("risks must not be empty") + if not isinstance(recommendation, dict): + raise DecisionArtifactError("recommendation must be an object") + + alternative_index = _require_unique(alternatives, "alternative_id", "alternatives") + criterion_index = _require_unique(criteria, "criterion_id", "criteria") + loaded_evidence, evidence_index = _load_evidence(evidence, source_root=source_root) + evidence_ids = set(evidence_index) + alternative_ids = set(alternative_index) + criterion_ids = set(criterion_index) + + normalized_alternatives = [ + { + "alternative_id": alternative_id, + "title": _require_text(item.get("title"), f"alternatives[{alternative_id}].title"), + "description": _require_text(item.get("description"), f"alternatives[{alternative_id}].description"), + } + for alternative_id, item in alternative_index.items() + ] + normalized_criteria = [] + total_weight = 0.0 + for criterion_id, item in criterion_index.items(): + weight = item.get("weight") + if not isinstance(weight, (int, float)) or isinstance(weight, bool) or weight <= 0: + raise DecisionArtifactError(f"criteria[{criterion_id}].weight must be positive") + total_weight += float(weight) + normalized_criteria.append( + { + "criterion_id": criterion_id, + "name": _require_text(item.get("name"), f"criteria[{criterion_id}].name"), + "weight": float(weight), + } + ) + if abs(total_weight - 1.0) > 1e-6: + raise DecisionArtifactError("criterion weights must sum to 1.0") + + normalized_assessments = [] + assessment_pairs: set[tuple[str, str]] = set() + for index, item in enumerate(assessments): + if not isinstance(item, dict): + raise DecisionArtifactError(f"assessments[{index}] must be an object") + alternative_id = _require_text(item.get("alternative_id"), f"assessments[{index}].alternative_id") + criterion_id = _require_text(item.get("criterion_id"), f"assessments[{index}].criterion_id") + if alternative_id not in alternative_ids or criterion_id not in criterion_ids: + raise DecisionArtifactError(f"assessments[{index}] references an unknown option or criterion") + pair = (alternative_id, criterion_id) + if pair in assessment_pairs: + raise DecisionArtifactError(f"duplicate assessment: {alternative_id}/{criterion_id}") + assessment_pairs.add(pair) + score = item.get("score") + if not isinstance(score, (int, float)) or isinstance(score, bool) or not 0 <= float(score) <= 5: + raise DecisionArtifactError(f"assessments[{index}].score must be between 0 and 5") + normalized_assessments.append( + { + "alternative_id": alternative_id, + "criterion_id": criterion_id, + "score": float(score), + "rationale": _require_text(item.get("rationale"), f"assessments[{index}].rationale"), + "evidence_ids": _validate_refs(item.get("evidence_ids"), evidence_ids, f"assessments[{index}].evidence_ids"), + } + ) + + expected_assessment_pairs = { + (alternative_id, criterion_id) + for alternative_id in alternative_ids + for criterion_id in criterion_ids + } + missing_matrix_pairs = sorted(expected_assessment_pairs - assessment_pairs) + if missing_matrix_pairs: + raise DecisionArtifactError( + "assessment matrix is incomplete: " + + ", ".join(f"{alternative_id}/{criterion_id}" for alternative_id, criterion_id in missing_matrix_pairs) + ) + + recommended_id = _require_text(recommendation.get("alternative_id"), "recommendation.alternative_id") + if recommended_id not in alternative_ids: + raise DecisionArtifactError("recommendation references an unknown alternative") + recommendation_criteria = _validate_refs( + recommendation.get("criterion_ids"), criterion_ids, "recommendation.criterion_ids" + ) + if set(recommendation_criteria) != criterion_ids: + raise DecisionArtifactError("recommendation.criterion_ids must cover every decision criterion") + recommendation_evidence = _validate_refs( + recommendation.get("evidence_ids"), evidence_ids, "recommendation.evidence_ids" + ) + loaded_evidence_by_id = {item["evidence_id"]: item for item in loaded_evidence} + missing_linked_recommendation_evidence = sorted( + linked_id + for evidence_id in recommendation_evidence + for linked_id in loaded_evidence_by_id[evidence_id].get("supporting_evidence_ids", []) + if linked_id not in recommendation_evidence + ) + if missing_linked_recommendation_evidence: + raise DecisionArtifactError( + "recommendation.evidence_ids omit typed supporting evidence: " + + ", ".join(missing_linked_recommendation_evidence) + ) + missing_assessments = sorted( + criterion_id + for criterion_id in recommendation_criteria + if (recommended_id, criterion_id) not in assessment_pairs + ) + if missing_assessments: + raise DecisionArtifactError( + "recommendation is missing criterion assessments: " + ", ".join(missing_assessments) + ) + required_recommendation_evidence = { + evidence_id + for assessment in normalized_assessments + if assessment["alternative_id"] == recommended_id + and assessment["criterion_id"] in recommendation_criteria + for evidence_id in assessment["evidence_ids"] + } + missing_recommendation_evidence = sorted( + required_recommendation_evidence - set(recommendation_evidence) + ) + if missing_recommendation_evidence: + raise DecisionArtifactError( + "recommendation.evidence_ids do not cover recommendation assessments: " + + ", ".join(missing_recommendation_evidence) + ) + + normalized_risks = [] + _require_unique(risks, "risk_id", "risks") + for index, item in enumerate(risks): + normalized_risks.append( + { + "risk_id": _require_text(item.get("risk_id"), f"risks[{index}].risk_id"), + "description": _require_text(item.get("description"), f"risks[{index}].description"), + "mitigation": _require_text(item.get("mitigation"), f"risks[{index}].mitigation"), + "evidence_ids": _validate_refs(item.get("evidence_ids"), evidence_ids, f"risks[{index}].evidence_ids"), + } + ) + + limitations = request.get("limitations") + unresolved = request.get("unresolved_review_items") + if not isinstance(limitations, list) or not limitations: + raise DecisionArtifactError("limitations must not be empty") + if not isinstance(unresolved, list) or not unresolved: + raise DecisionArtifactError("unresolved_review_items must not be empty before human review") + return { + "schema": "decision_artifact.v1", + "decision_id": _require_text(request.get("decision_id"), "decision_id"), + "title": _require_text(request.get("title"), "title"), + "problem": _require_text(request.get("problem"), "problem"), + "decision_status": "review_required", + "alternatives": normalized_alternatives, + "criteria": normalized_criteria, + "evidence_links": loaded_evidence, + "assessments": normalized_assessments, + "risks": normalized_risks, + "recommendation": { + "alternative_id": recommended_id, + "rationale": _require_text(recommendation.get("rationale"), "recommendation.rationale"), + "criterion_ids": recommendation_criteria, + "evidence_ids": recommendation_evidence, + }, + "limitations": [_require_text(item, "limitations") for item in limitations], + "review": { + "status": "pending", + "unresolved_items": [_require_text(item, "unresolved_review_items") for item in unresolved], + "reviewed_by": None, + "review_evidence": [], + }, + "approval": { + "status": "not_requested", + "approved_by": None, + "approval_evidence": [], + }, + "next_action": "Obtain independent review and explicit human approval before executing the recommendation.", + "provenance": { + "constructor": "harness.lib.research.decision_artifact.construct_decision_artifact", + "constructed_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "request_path": str(request_path), + "request_sha256": request_sha256, + "source_root": str(source_root), + }, + } diff --git a/harness/schemas/evidence/decision_artifact.v1.schema.json b/harness/schemas/evidence/decision_artifact.v1.schema.json new file mode 100644 index 000000000..ac3a1ef24 --- /dev/null +++ b/harness/schemas/evidence/decision_artifact.v1.schema.json @@ -0,0 +1,181 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://solar.local/schemas/evidence/decision_artifact.v1.schema.json", + "title": "Solar Evidence ABI - Decision Artifact", + "type": "object", + "required": [ + "schema", + "decision_id", + "title", + "problem", + "decision_status", + "alternatives", + "criteria", + "evidence_links", + "assessments", + "risks", + "recommendation", + "limitations", + "review", + "approval", + "next_action", + "provenance" + ], + "additionalProperties": false, + "properties": { + "schema": {"const": "decision_artifact.v1"}, + "decision_id": {"type": "string", "minLength": 1}, + "title": {"type": "string", "minLength": 1}, + "problem": {"type": "string", "minLength": 1}, + "decision_status": {"const": "review_required"}, + "alternatives": { + "type": "array", + "minItems": 2, + "items": {"$ref": "#/$defs/alternative"} + }, + "criteria": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/criterion"} + }, + "evidence_links": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/evidenceLink"} + }, + "assessments": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/assessment"} + }, + "risks": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/risk"} + }, + "recommendation": {"$ref": "#/$defs/recommendation"}, + "limitations": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 1} + }, + "review": {"$ref": "#/$defs/review"}, + "approval": {"$ref": "#/$defs/approval"}, + "next_action": {"type": "string", "minLength": 1}, + "provenance": {"$ref": "#/$defs/provenance"} + }, + "$defs": { + "ids": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 1}, + "uniqueItems": true + }, + "alternative": { + "type": "object", + "required": ["alternative_id", "title", "description"], + "additionalProperties": false, + "properties": { + "alternative_id": {"type": "string", "minLength": 1}, + "title": {"type": "string", "minLength": 1}, + "description": {"type": "string", "minLength": 1} + } + }, + "criterion": { + "type": "object", + "required": ["criterion_id", "name", "weight"], + "additionalProperties": false, + "properties": { + "criterion_id": {"type": "string", "minLength": 1}, + "name": {"type": "string", "minLength": 1}, + "weight": {"type": "number", "exclusiveMinimum": 0, "maximum": 1} + } + }, + "evidenceLink": { + "type": "object", + "required": ["evidence_id", "evidence_type", "source_path", "sha256", "semantic_locator", "observed_support", "summary"], + "additionalProperties": false, + "properties": { + "evidence_id": {"type": "string", "minLength": 1}, + "evidence_type": {"type": "string", "enum": ["claim_verdict", "experiment_result"]}, + "source_path": {"type": "string", "minLength": 1}, + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "semantic_locator": {"type": "string", "pattern": "^/"}, + "observed_support": {}, + "summary": {"type": "string", "minLength": 1}, + "supporting_evidence_ids": {"$ref": "#/$defs/ids"} + } + }, + "assessment": { + "type": "object", + "required": ["alternative_id", "criterion_id", "score", "rationale", "evidence_ids"], + "additionalProperties": false, + "properties": { + "alternative_id": {"type": "string", "minLength": 1}, + "criterion_id": {"type": "string", "minLength": 1}, + "score": {"type": "number", "minimum": 0, "maximum": 5}, + "rationale": {"type": "string", "minLength": 1}, + "evidence_ids": {"$ref": "#/$defs/ids"} + } + }, + "risk": { + "type": "object", + "required": ["risk_id", "description", "mitigation", "evidence_ids"], + "additionalProperties": false, + "properties": { + "risk_id": {"type": "string", "minLength": 1}, + "description": {"type": "string", "minLength": 1}, + "mitigation": {"type": "string", "minLength": 1}, + "evidence_ids": {"$ref": "#/$defs/ids"} + } + }, + "recommendation": { + "type": "object", + "required": ["alternative_id", "rationale", "criterion_ids", "evidence_ids"], + "additionalProperties": false, + "properties": { + "alternative_id": {"type": "string", "minLength": 1}, + "rationale": {"type": "string", "minLength": 1}, + "criterion_ids": {"$ref": "#/$defs/ids"}, + "evidence_ids": {"$ref": "#/$defs/ids"} + } + }, + "review": { + "type": "object", + "required": ["status", "unresolved_items", "reviewed_by", "review_evidence"], + "additionalProperties": false, + "properties": { + "status": {"const": "pending"}, + "unresolved_items": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 1} + }, + "reviewed_by": {"type": "null"}, + "review_evidence": {"type": "array", "maxItems": 0} + } + }, + "approval": { + "type": "object", + "required": ["status", "approved_by", "approval_evidence"], + "additionalProperties": false, + "properties": { + "status": {"const": "not_requested"}, + "approved_by": {"type": "null"}, + "approval_evidence": {"type": "array", "maxItems": 0} + } + }, + "provenance": { + "type": "object", + "required": ["constructor", "constructed_at", "request_path", "request_sha256", "source_root"], + "additionalProperties": false, + "properties": { + "constructor": {"const": "harness.lib.research.decision_artifact.construct_decision_artifact"}, + "constructed_at": {"type": "string", "format": "date-time"}, + "request_path": {"type": "string", "minLength": 1}, + "request_sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "source_root": {"type": "string", "minLength": 1} + } + } + } +} diff --git a/harness/schemas/evidence/publication_delivery_handoff.v1.schema.json b/harness/schemas/evidence/publication_delivery_handoff.v1.schema.json new file mode 100644 index 000000000..0ac4d8cd1 --- /dev/null +++ b/harness/schemas/evidence/publication_delivery_handoff.v1.schema.json @@ -0,0 +1,111 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://solar.local/schemas/evidence/publication_delivery_handoff.v1.schema.json", + "title": "Solar Publication Delivery Handoff", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", "delivery_id", "audience", "delivery_format", "content_scope", + "permissions", "handoff_checklist", "files", "evidence_index", "provenance", "limitations" + ], + "properties": { + "schema": {"const": "publication_delivery_handoff.v1"}, + "delivery_id": {"type": "string", "minLength": 1}, + "audience": { + "type": "object", + "additionalProperties": false, + "required": ["role"], + "properties": { + "role": {"type": "string", "minLength": 1}, + "description": {"type": "string"} + } + }, + "delivery_format": {"type": "string", "enum": ["markdown_bundle", "paper_bundle", "mixed_bundle"]}, + "content_scope": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "minLength": 1}}, + "permissions": { + "type": "object", + "additionalProperties": false, + "required": ["distribution_scope", "approval_required", "approval_state"], + "properties": { + "distribution_scope": {"enum": ["local_only", "external_email"]}, + "approval_required": {"const": true}, + "approval_state": {"enum": ["not_requested", "pending", "approved"]}, + "approval_ref": {"type": "string"}, + "approved_by": {"type": "string"}, + "approved_at": {"type": "string", "format": "date-time"} + } + }, + "external_delivery": { + "type": "object", + "additionalProperties": false, + "required": ["channel", "recipient", "status", "delivered", "approval_ref", "runtime_evidence_path", "runtime_evidence_sha256"], + "properties": { + "channel": {"enum": ["gmail", "smtp", "gmail_smtp"]}, + "recipient": {"type": "string", "minLength": 3}, + "status": {"const": "completed"}, + "delivered": {"const": true}, + "approval_ref": {"type": "string", "minLength": 1}, + "provider": {"type": "string"}, + "runtime_evidence_path": {"type": "string", "pattern": "^files/[^/]+$"}, + "runtime_evidence_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "recipient_acceptance_required": {"type": "boolean"} + } + }, + "handoff_checklist": { + "type": "array", + "minItems": 5, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["check_id", "status", "evidence"], + "properties": { + "check_id": {"type": "string", "minLength": 1}, + "status": {"const": "completed"}, + "evidence": {"type": "string", "minLength": 1} + } + } + }, + "files": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/file"}}, + "evidence_index": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["evidence_id", "file_id", "sha256"], + "properties": { + "evidence_id": {"type": "string", "minLength": 1}, + "file_id": {"type": "string", "minLength": 1}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + } + } + }, + "provenance": { + "type": "object", + "additionalProperties": false, + "required": ["request_sha256", "constructed_at", "tool"], + "properties": { + "request_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "constructed_at": {"type": "string", "format": "date-time"}, + "tool": {"const": "harness/tools/publication_delivery_bundle.py"} + } + }, + "limitations": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}} + }, + "$defs": { + "file": { + "type": "object", + "additionalProperties": false, + "required": ["file_id", "type", "path", "bytes", "sha256", "source_sha256", "evidence_ids"], + "properties": { + "file_id": {"type": "string", "minLength": 1}, + "type": {"type": "string", "minLength": 1}, + "path": {"type": "string", "pattern": "^files/[^/]+$"}, + "bytes": {"type": "integer", "minimum": 1}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "source_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "evidence_ids": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "minLength": 1}} + } + } + } +} diff --git a/harness/tools/decision_artifact.py b/harness/tools/decision_artifact.py new file mode 100644 index 000000000..95e66b92b --- /dev/null +++ b/harness/tools/decision_artifact.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Build a canonical, evidence-bound decision artifact.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +import jsonschema + +HARNESS_DIR = Path(__file__).resolve().parents[1] +if str(HARNESS_DIR) not in sys.path: + sys.path.insert(0, str(HARNESS_DIR)) + +from lib.research.decision_artifact import DecisionArtifactError, construct_decision_artifact + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", required=True, type=Path, help="decision_request.v1 JSON") + parser.add_argument("--output", required=True, type=Path, help="destination decision_artifact.v1 JSON") + parser.add_argument( + "--source-root", + type=Path, + help="root containing all referenced evidence (defaults to input directory)", + ) + return parser + + +def _paths_alias(left: Path, right: Path) -> bool: + left_resolved = left.resolve() + right_resolved = right.resolve() + if left_resolved == right_resolved: + return True + try: + return left.exists() and right.exists() and os.path.samefile(left, right) + except OSError: + return False + + +def _evidence_paths(request: object, source_root: Path) -> list[Path]: + if not isinstance(request, dict): + raise DecisionArtifactError("decision request must be an object") + entries = request.get("evidence") + if not isinstance(entries, list): + return [] + paths: list[Path] = [] + for entry in entries: + if not isinstance(entry, dict) or not isinstance(entry.get("source_path"), str): + continue + path = Path(entry["source_path"]) + paths.append((path if path.is_absolute() else source_root / path).resolve()) + return paths + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + input_path = args.input.resolve() + source_root = (args.source_root or input_path.parent).resolve() + output_path = args.output.resolve() + temp_path = output_path.with_name(f".{output_path.name}.{os.getpid()}.tmp") + cleanup_output_allowed = False + try: + candidates = (("output", output_path), ("temporary output", temp_path)) + for candidate_name, candidate in candidates: + if _paths_alias(candidate, input_path): + raise DecisionArtifactError( + f"{candidate_name} aliases protected input/evidence path: {input_path}" + ) + try: + request = json.loads(input_path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + cleanup_output_allowed = True + raise + if not isinstance(request, dict): + cleanup_output_allowed = True + raise DecisionArtifactError("decision request must be an object") + evidence_paths = _evidence_paths(request, source_root) + for candidate_name, candidate in candidates: + aliases = [path for path in evidence_paths if _paths_alias(candidate, path)] + if aliases: + raise DecisionArtifactError( + f"{candidate_name} aliases protected input/evidence path: {aliases[0]}" + ) + cleanup_output_allowed = True + output_path.parent.mkdir(parents=True, exist_ok=True) + temp_path.unlink(missing_ok=True) + artifact = construct_decision_artifact( + request, + request_path=input_path, + source_root=source_root, + ) + schema_path = HARNESS_DIR / "schemas" / "evidence" / "decision_artifact.v1.schema.json" + schema = json.loads(schema_path.read_text(encoding="utf-8")) + jsonschema.Draft202012Validator.check_schema(schema) + jsonschema.Draft202012Validator(schema).validate(artifact) + temp_path.write_text( + json.dumps(artifact, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + temp_path.replace(output_path) + except ( + OSError, + UnicodeDecodeError, + json.JSONDecodeError, + jsonschema.SchemaError, + jsonschema.ValidationError, + DecisionArtifactError, + ) as exc: + cleanup_errors: list[str] = [] + if cleanup_output_allowed: + for path in (temp_path, output_path): + try: + path.unlink(missing_ok=True) + except OSError as cleanup_exc: + cleanup_errors.append(f"{path}: {cleanup_exc}") + error = str(exc) + if cleanup_errors: + error += "; cleanup failed: " + "; ".join(cleanup_errors) + print(json.dumps({"status": "rejected", "error": error}), file=sys.stderr) + return 2 + print(json.dumps({"status": "completed", "artifact": str(output_path)})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness/tools/markdown_pdf.py b/harness/tools/markdown_pdf.py new file mode 100644 index 000000000..20338f0d5 --- /dev/null +++ b/harness/tools/markdown_pdf.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Dependency-free Markdown text to auditable PDF deliverable constructor.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path + + +def _plain_lines(markdown: str) -> list[str]: + lines = [] + for raw in markdown.splitlines(): + line = re.sub(r"!\[[^]]*]\([^)]*\)", "[image]", raw) + line = re.sub(r"\[([^]]+)]\([^)]*\)", r"\1", line) + line = re.sub(r"^\s{0,3}(?:#{1,6}|[-*+]\s+|\d+[.)]\s+|>\s*)", "", line) + line = re.sub(r"[*_`~]", "", line).strip() + if not line: + lines.append("") + continue + while len(line) > 92: + split = line.rfind(" ", 0, 92) + split = split if split > 20 else 92 + lines.append(line[:split].strip()) + line = line[split:].strip() + lines.append(line) + return lines or [""] + + +def _pdf_escape(value: str) -> str: + ascii_value = value.encode("ascii", "replace").decode("ascii") + return ascii_value.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") + + +def build_pdf(markdown_path: Path, output_path: Path) -> dict[str, object]: + source = markdown_path.read_text(encoding="utf-8-sig") + lines = _plain_lines(source) + per_page = 48 + pages = [lines[index:index + per_page] for index in range(0, len(lines), per_page)] + objects: list[bytes] = [] + page_ids = [4 + index * 2 for index in range(len(pages))] + objects.append(b"<< /Type /Catalog /Pages 2 0 R >>") + kids = " ".join(f"{item} 0 R" for item in page_ids) + objects.append(f"<< /Type /Pages /Kids [{kids}] /Count {len(pages)} >>".encode()) + objects.append(b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>") + for page_index, page_lines in enumerate(pages): + page_id = page_ids[page_index] + content_id = page_id + 1 + objects.append( + f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents {content_id} 0 R >>".encode() + ) + commands = ["BT", "/F1 10 Tf", "12 TL", "54 738 Td"] + for line in page_lines: + commands.append(f"({_pdf_escape(line)}) Tj") + commands.append("T*") + commands.append("ET") + stream = "\n".join(commands).encode("ascii") + objects.append(f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream") + payload = bytearray(b"%PDF-1.4\n%Solar\n") + offsets = [0] + for object_id, obj in enumerate(objects, 1): + offsets.append(len(payload)) + payload.extend(f"{object_id} 0 obj\n".encode() + obj + b"\nendobj\n") + xref = len(payload) + payload.extend(f"xref\n0 {len(objects)+1}\n0000000000 65535 f \n".encode()) + for offset in offsets[1:]: + payload.extend(f"{offset:010d} 00000 n \n".encode()) + payload.extend(f"trailer\n<< /Size {len(objects)+1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode()) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(payload) + return verify_pdf(output_path, expected_text=source) + + +def verify_pdf(path: Path, *, expected_text: str | None = None) -> dict[str, object]: + data = path.read_bytes() + text = data.decode("ascii", "replace") + pages = len(re.findall(r"/Type /Page\b", text)) + expected_markers = [] + if expected_text is not None: + expected_markers = [line for line in _plain_lines(expected_text) if len(line) >= 8][:3] + result = { + "schema_version": "solar.markdown_pdf_verification.v1", + "path": str(path), + "bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + "page_count": pages, + "header_valid": data.startswith(b"%PDF-1.4"), + "eof_valid": data.rstrip().endswith(b"%%EOF"), + "xref_present": "\nxref\n" in text and "\nstartxref\n" in text, + "expected_markers_present": all(_pdf_escape(marker) in text for marker in expected_markers), + } + result["valid"] = all(result[key] for key in ("header_valid", "eof_valid", "xref_present", "expected_markers_present")) and pages > 0 + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + build = sub.add_parser("build") + build.add_argument("--input", type=Path, required=True) + build.add_argument("--output", type=Path, required=True) + verify = sub.add_parser("verify") + verify.add_argument("--input", type=Path, required=True) + args = parser.parse_args() + try: + result = build_pdf(args.input, args.output) if args.command == "build" else verify_pdf(args.input) + except (OSError, UnicodeError) as exc: + print(json.dumps({"valid": False, "error": str(exc)})) + return 2 + print(json.dumps(result, ensure_ascii=False, sort_keys=True)) + return 0 if result.get("valid") else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness/tools/publication_delivery_bundle.py b/harness/tools/publication_delivery_bundle.py new file mode 100644 index 000000000..5eeb20431 --- /dev/null +++ b/harness/tools/publication_delivery_bundle.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +"""Construct and independently verify a local-only publication handoff bundle.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import shutil +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import jsonschema + + +SECRET_PATTERNS = ( + re.compile(rb"(? str: + return hashlib.sha256(data).hexdigest() + + +def _load_object(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"JSON must be an object: {path}") + return value + + +def _schema_path() -> Path: + return Path(__file__).resolve().parents[1] / "schemas" / "evidence" / "publication_delivery_handoff.v1.schema.json" + + +def _validate_manifest(payload: dict[str, Any]) -> None: + schema = _load_object(_schema_path()) + jsonschema.Draft202012Validator(schema, format_checker=jsonschema.FormatChecker()).validate(payload) + + +def _safe_source(raw: str, root: Path) -> Path: + source = Path(raw) + if not source.is_absolute(): + source = root / source + if source.is_symlink() or not source.is_file(): + raise ValueError(f"source must be a regular non-symlink file: {source}") + resolved = source.resolve() + try: + resolved.relative_to(root.resolve()) + except ValueError as exc: + raise ValueError(f"source escapes source root: {source}") from exc + return resolved + + +def _secret_free(data: bytes, label: str) -> None: + for pattern in SECRET_PATTERNS: + if pattern.search(data): + raise ValueError(f"secret-like value detected in {label}") + + +def _permission_mode(permissions: dict[str, Any]) -> str: + if permissions == {"distribution_scope": "local_only", "approval_required": True, "approval_state": "not_requested"}: + return "local_only" + if ( + permissions.get("distribution_scope") == "external_email" + and permissions.get("approval_required") is True + and permissions.get("approval_state") == "approved" + and str(permissions.get("approval_ref") or "").strip() + and str(permissions.get("approved_by") or "").strip() + ): + return "approved_external_email" + raise ValueError("delivery permission must be local-only/not-requested or approved external_email") + + +def _delivery_runtime(payload: dict[str, Any]) -> dict[str, Any]: + if payload.get("schema") == "autosci_runtime_evidence.v1": + runtime = payload.get("outputs", {}).get("runtime", {}) + return runtime if isinstance(runtime, dict) else {} + if payload.get("schema") == "autosci_external_delivery_audit.v1": + return payload + return {} + + +def _recipient_matches(runtime: dict[str, Any], recipient: str) -> bool: + expected = recipient.strip().lower() + raw = runtime.get("to") or runtime.get("recipient") or runtime.get("recipients") + if isinstance(raw, list): + values = [str(item).strip().lower() for item in raw] + else: + values = [item.strip().lower() for item in str(raw or "").replace(";", ",").split(",")] + return expected in {value for value in values if value} + + +def _external_delivery( + request: dict[str, Any], + source_root: Path, + staging: Path, + files: list[dict[str, Any]], + evidence_index: list[dict[str, Any]], +) -> dict[str, Any]: + spec = request.get("external_delivery") + if not isinstance(spec, dict): + raise ValueError("approved external email delivery requires external_delivery evidence") + channel = str(spec.get("channel") or "").strip().lower() + recipient = str(spec.get("recipient") or "").strip() + if channel not in {"gmail", "smtp", "gmail_smtp"} or not recipient: + raise ValueError("external_delivery must name a supported channel and recipient") + runtime_source = _safe_source(str(spec.get("runtime_evidence_path") or ""), source_root) + runtime_bytes = runtime_source.read_bytes() + _secret_free(runtime_bytes, str(runtime_source)) + runtime_payload = json.loads(runtime_bytes) + if not isinstance(runtime_payload, dict): + raise ValueError("external delivery runtime evidence must be a JSON object") + runtime = _delivery_runtime(runtime_payload) + approval_ref = str(request["permissions"].get("approval_ref") or "").strip() + provider = str(runtime.get("provider") or runtime_payload.get("provider") or channel).strip().lower() + delivered = runtime.get("delivered") is True and str(runtime.get("status") or runtime_payload.get("status") or "") == "completed" + if str(runtime.get("action") or runtime_payload.get("action") or "send_email") != "send_email": + raise ValueError("external delivery runtime evidence must describe send_email") + if not delivered: + raise ValueError("external delivery runtime evidence must prove completed delivered=true") + if str(runtime.get("approval_ref") or runtime_payload.get("approval_ref") or "").strip() != approval_ref: + raise ValueError("external delivery approval_ref mismatch") + if not _recipient_matches(runtime, recipient): + raise ValueError("external delivery recipient mismatch") + if channel == "gmail" and provider not in {"gmail", "gmail_connector", "gmail_smtp", "smtp"}: + raise ValueError("gmail delivery requires a gmail-compatible provider") + + digest = _sha(runtime_bytes) + target_name = "99-external-delivery-runtime-evidence.json" + target = staging / "files" / target_name + target.write_bytes(runtime_bytes) + files.append({ + "file_id": "external-delivery-runtime-evidence", + "type": "external_delivery_runtime_evidence", + "path": f"files/{target_name}", + "bytes": len(runtime_bytes), + "sha256": digest, + "source_sha256": digest, + "evidence_ids": [f"delivery:{approval_ref}", f"recipient:{recipient}"], + }) + evidence_index.extend( + {"evidence_id": evidence_id, "file_id": "external-delivery-runtime-evidence", "sha256": digest} + for evidence_id in [f"delivery:{approval_ref}", f"recipient:{recipient}"] + ) + return { + "channel": channel, + "recipient": recipient, + "status": "completed", + "delivered": True, + "approval_ref": approval_ref, + "provider": provider, + "runtime_evidence_path": f"files/{target_name}", + "runtime_evidence_sha256": digest, + "recipient_acceptance_required": bool(spec.get("recipient_acceptance_required", False)), + } + + +def construct(request_path: Path, output_dir: Path, source_root: Path) -> Path: + request_bytes = request_path.read_bytes() + request = _load_object(request_path) + if request.get("schema") != "publication_delivery_request.v1": + raise ValueError("request schema must be publication_delivery_request.v1") + required = ("delivery_id", "audience", "delivery_format", "content_scope", "permissions", "files") + if any(not request.get(key) for key in required): + raise ValueError("request is missing a required delivery field") + permissions = request["permissions"] + permission_mode = _permission_mode(permissions) + if output_dir.exists(): + raise ValueError(f"output directory already exists: {output_dir}") + staging = output_dir.with_name(output_dir.name + ".tmp") + if staging.exists(): + shutil.rmtree(staging) + (staging / "files").mkdir(parents=True) + files: list[dict[str, Any]] = [] + evidence_index: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + try: + for index, item in enumerate(request["files"], 1): + if not isinstance(item, dict): + raise ValueError("every requested file must be an object") + file_id = str(item.get("file_id") or "") + evidence_ids = [str(value) for value in item.get("evidence_ids") or [] if str(value)] + if not file_id or file_id in seen_ids or not evidence_ids: + raise ValueError("file_id must be unique and evidence_ids must be non-empty") + seen_ids.add(file_id) + source = _safe_source(str(item.get("source_path") or ""), source_root) + data = source.read_bytes() + if not data: + raise ValueError(f"source is empty: {source}") + _secret_free(data, str(source)) + suffix = source.suffix.lower() or ".bin" + target_name = f"{index:02d}-{file_id}{suffix}" + target = staging / "files" / target_name + target.write_bytes(data) + digest = _sha(data) + files.append({ + "file_id": file_id, + "type": str(item.get("type") or "artifact"), + "path": f"files/{target_name}", + "bytes": len(data), + "sha256": digest, + "source_sha256": digest, + "evidence_ids": evidence_ids, + }) + evidence_index.extend({"evidence_id": evidence_id, "file_id": file_id, "sha256": digest} for evidence_id in evidence_ids) + external_delivery = None + if permission_mode == "approved_external_email": + external_delivery = _external_delivery(request, source_root, staging, files, evidence_index) + checklist = [ + {"check_id": "audience-defined", "status": "completed", "evidence": str(request["audience"]["role"])}, + {"check_id": "format-defined", "status": "completed", "evidence": str(request["delivery_format"])}, + {"check_id": "content-scope-defined", "status": "completed", "evidence": ", ".join(request["content_scope"])}, + {"check_id": "evidence-index-verified", "status": "completed", "evidence": f"{len(evidence_index)} indexed links"}, + {"check_id": "permissions-fail-closed", "status": "completed", "evidence": "local_only; external approval not requested" if permission_mode == "local_only" else "approved external_email"}, + {"check_id": "secret-scan-clean", "status": "completed", "evidence": f"{len(files)} files scanned"}, + ] + if external_delivery: + checklist.append({"check_id": "external-delivery-verified", "status": "completed", "evidence": f"{external_delivery['channel']} delivered to {external_delivery['recipient']}"}) + manifest = { + "schema": "publication_delivery_handoff.v1", + "delivery_id": str(request["delivery_id"]), + "audience": request["audience"], + "delivery_format": request["delivery_format"], + "content_scope": request["content_scope"], + "permissions": permissions, + "handoff_checklist": checklist, + "files": files, + "evidence_index": evidence_index, + "provenance": { + "request_sha256": _sha(request_bytes), + "constructed_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "tool": "harness/tools/publication_delivery_bundle.py", + }, + "limitations": ( + ["Recipient acceptance was not required by the approved delivery contract."] + if external_delivery and not external_delivery.get("recipient_acceptance_required") + else ["This is a local handoff bundle; external distribution and recipient acceptance were not requested or performed."] + ), + } + if external_delivery: + manifest["external_delivery"] = external_delivery + _validate_manifest(manifest) + manifest_bytes = (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode("utf-8") + _secret_free(manifest_bytes, "publication-delivery-manifest.json") + (staging / "publication-delivery-manifest.json").write_bytes(manifest_bytes) + staging.rename(output_dir) + return output_dir / "publication-delivery-manifest.json" + except Exception: + if staging.exists(): + shutil.rmtree(staging) + raise + + +def verify(bundle_dir: Path) -> Path: + manifest_path = bundle_dir / "publication-delivery-manifest.json" + manifest_bytes = manifest_path.read_bytes() + _secret_free(manifest_bytes, "publication-delivery-manifest.json") + manifest = json.loads(manifest_bytes) + if not isinstance(manifest, dict): + raise ValueError("publication delivery manifest must be an object") + _validate_manifest(manifest) + expected_paths = {"publication-delivery-manifest.json", *[str(item["path"]) for item in manifest["files"]]} + actual_paths = { + path.relative_to(bundle_dir).as_posix() + for path in bundle_dir.rglob("*") + if path.is_file() or path.is_symlink() + } + if actual_paths != expected_paths: + raise ValueError(f"bundle inventory mismatch: unexpected={sorted(actual_paths - expected_paths)}, missing={sorted(expected_paths - actual_paths)}") + for item in manifest["files"]: + path = bundle_dir / item["path"] + if path.is_symlink() or not path.is_file(): + raise ValueError(f"bundle file missing or unsafe: {item['path']}") + resolved = path.resolve() + try: + resolved.relative_to(bundle_dir.resolve()) + except ValueError as exc: + raise ValueError(f"bundle file escapes root: {item['path']}") from exc + data = path.read_bytes() + _secret_free(data, item["path"]) + if len(data) != item["bytes"] or _sha(data) != item["sha256"] or item["sha256"] != item["source_sha256"]: + raise ValueError(f"bundle file integrity mismatch: {item['path']}") + evidence_pairs = {(item["evidence_id"], item["file_id"], item["sha256"]) for item in manifest["evidence_index"]} + expected_pairs = {(evidence_id, item["file_id"], item["sha256"]) for item in manifest["files"] for evidence_id in item["evidence_ids"]} + if evidence_pairs != expected_pairs: + raise ValueError("evidence index does not exactly match bundled files") + return manifest_path + + +def main() -> int: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + build = sub.add_parser("build") + build.add_argument("--request", type=Path, required=True) + build.add_argument("--output-dir", type=Path, required=True) + build.add_argument("--source-root", type=Path, required=True) + check = sub.add_parser("verify") + check.add_argument("--bundle-dir", type=Path, required=True) + args = parser.parse_args() + try: + path = construct(args.request, args.output_dir, args.source_root) if args.command == "build" else verify(args.bundle_dir) + print(json.dumps({"status": "completed", "manifest": str(path)})) + return 0 + except Exception as exc: + print(json.dumps({"status": "failed", "error": str(exc)}), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/harness/tools/test_publication_delivery_bundle.py b/tests/harness/tools/test_publication_delivery_bundle.py new file mode 100644 index 000000000..15bb8b44a --- /dev/null +++ b/tests/harness/tools/test_publication_delivery_bundle.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from harness.tools.publication_delivery_bundle import construct, verify + + +def _request(root: Path) -> Path: + source = root / "report.md" + source.write_text("# Report\n\nA bounded local result.\n", encoding="utf-8") + request = root / "request.json" + request.write_text( + json.dumps({ + "schema": "publication_delivery_request.v1", + "delivery_id": "delivery-test", + "audience": {"role": "technical_lead"}, + "delivery_format": "markdown_bundle", + "content_scope": ["report"], + "permissions": {"distribution_scope": "local_only", "approval_required": True, "approval_state": "not_requested"}, + "files": [{"file_id": "report", "type": "report", "source_path": str(source), "evidence_ids": ["report:test"]}], + }), + encoding="utf-8", + ) + return request + + +def test_delivery_bundle_build_verify_and_tamper_rejection(tmp_path: Path) -> None: + bundle = tmp_path / "bundle" + construct(_request(tmp_path), bundle, tmp_path) + assert verify(bundle).is_file() + report = next((bundle / "files").iterdir()) + report.write_text("tampered", encoding="utf-8") + with pytest.raises(ValueError, match="integrity mismatch"): + verify(bundle) + + +def test_delivery_bundle_rejects_secret_and_extra_inventory(tmp_path: Path) -> None: + request = _request(tmp_path) + source = tmp_path / "report.md" + source.write_text("sk-realisticSecretToken123456789", encoding="utf-8") + with pytest.raises(ValueError, match="secret-like"): + construct(request, tmp_path / "secret-bundle", tmp_path) + + source.write_text("task-autosci-skill-assignment is an evidence id, not a credential", encoding="utf-8") + bundle = tmp_path / "clean-bundle" + construct(request, bundle, tmp_path) + (bundle / "unlisted.txt").write_text("not in manifest", encoding="utf-8") + with pytest.raises(ValueError, match="inventory mismatch"): + verify(bundle) + + +def test_delivery_bundle_rejects_unapproved_or_escaping_source(tmp_path: Path) -> None: + request = _request(tmp_path) + payload = json.loads(request.read_text(encoding="utf-8")) + payload["permissions"]["approval_state"] = "approved" + request.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValueError, match="local-only/not-requested or approved external_email"): + construct(request, tmp_path / "approved-bundle", tmp_path) + + outside = tmp_path.parent / "outside-publication-delivery.txt" + outside.write_text("outside", encoding="utf-8") + payload["permissions"]["approval_state"] = "not_requested" + payload["files"][0]["source_path"] = str(outside) + request.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValueError, match="escapes source root"): + construct(request, tmp_path / "escape-bundle", tmp_path) + + +def test_delivery_bundle_accepts_approved_external_email_audit(tmp_path: Path) -> None: + request = _request(tmp_path) + audit = tmp_path / "external-delivery-audit.json" + audit.write_text( + json.dumps( + { + "schema": "autosci_external_delivery_audit.v1", + "action": "send_email", + "status": "completed", + "provider": "gmail_connector", + "channel": "gmail", + "approval_ref": "approval-phase22-email", + "to": "reader@example.com", + "subject": "Phase 22 handoff", + "delivered": True, + "message_id_sha256": "a" * 64, + "thread_id_sha256": "b" * 64, + } + ), + encoding="utf-8", + ) + payload = json.loads(request.read_text(encoding="utf-8")) + payload["permissions"] = { + "distribution_scope": "external_email", + "approval_required": True, + "approval_state": "approved", + "approval_ref": "approval-phase22-email", + "approved_by": "phase22-user", + "approved_at": "2026-08-12T00:00:00Z", + } + payload["external_delivery"] = { + "channel": "gmail", + "recipient": "reader@example.com", + "runtime_evidence_path": str(audit), + "recipient_acceptance_required": False, + } + request.write_text(json.dumps(payload), encoding="utf-8") + + manifest_path = construct(request, tmp_path / "external-bundle", tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + + assert verify(tmp_path / "external-bundle").is_file() + assert manifest["permissions"]["distribution_scope"] == "external_email" + assert manifest["external_delivery"]["delivered"] is True + assert manifest["external_delivery"]["recipient"] == "reader@example.com" + assert any(item["file_id"] == "external-delivery-runtime-evidence" for item in manifest["files"]) + + +def test_delivery_bundle_rejects_external_email_recipient_mismatch(tmp_path: Path) -> None: + request = _request(tmp_path) + audit = tmp_path / "external-delivery-audit.json" + audit.write_text( + json.dumps( + { + "schema": "autosci_external_delivery_audit.v1", + "action": "send_email", + "status": "completed", + "provider": "gmail_connector", + "approval_ref": "approval-phase22-email", + "to": "wrong@example.com", + "delivered": True, + } + ), + encoding="utf-8", + ) + payload = json.loads(request.read_text(encoding="utf-8")) + payload["permissions"] = { + "distribution_scope": "external_email", + "approval_required": True, + "approval_state": "approved", + "approval_ref": "approval-phase22-email", + "approved_by": "phase22-user", + } + payload["external_delivery"] = { + "channel": "gmail", + "recipient": "reader@example.com", + "runtime_evidence_path": str(audit), + } + request.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="recipient mismatch"): + construct(request, tmp_path / "mismatch-bundle", tmp_path) diff --git a/tests/journeys/phase22/code/evidence.py b/tests/journeys/phase22/code/evidence.py index fe47d3283..1d9903f61 100644 --- a/tests/journeys/phase22/code/evidence.py +++ b/tests/journeys/phase22/code/evidence.py @@ -91,13 +91,28 @@ def json_safe(value: Any) -> Any: "P22-J09": { "name": "Generate and review a deliverable research report", "selector": "p22_j09", - "live": False, + "live": True, }, "P22-J10": { "name": "Backup, restore, and uninstall inside a sandbox", "selector": "p22_j10", "live": False, }, + "P22-J25": { + "name": "Construct, verify, install, smoke, and roll back a runtime deliverable", + "selector": "p22_j25", + "live": False, + }, + "P22-045": { + "name": "Live independent writer/reviewer provider provenance", + "selector": "p22_045_live_independent_providers", + "live": True, + }, + "P22-071": { + "name": "Attributable human-review lifecycle resume", + "selector": "p22_071_human_review_resume", + "live": False, + }, } @@ -140,7 +155,33 @@ def repo_head(repo_root: Path) -> str: stderr=subprocess.PIPE, check=False, ) - return proc.stdout.strip() if proc.returncode == 0 else f"unavailable: {redact(proc.stderr.strip())}" + if proc.returncode == 0: + return proc.stdout.strip() + + git_file = repo_root / ".git" + if os.name != "nt" and git_file.is_file(): + pointer = git_file.read_text(encoding="utf-8", errors="replace").strip() + if pointer.lower().startswith("gitdir:"): + raw_git_dir = pointer.split(":", 1)[1].strip().replace("\\", "/") + drive_match = re.match(r"^([A-Za-z]):/(.*)$", raw_git_dir) + git_dir = ( + Path(f"/mnt/{drive_match.group(1).lower()}/{drive_match.group(2)}") + if drive_match + else (repo_root / raw_git_dir).resolve() + ) + fallback = subprocess.run( + ["git", f"--git-dir={git_dir}", f"--work-tree={repo_root}", "rev-parse", "HEAD"], + text=True, + encoding="utf-8", + errors="replace", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if fallback.returncode == 0: + return fallback.stdout.strip() + proc = fallback + return f"unavailable: {redact(proc.stderr.strip())}" def command_exists(name: str) -> bool: @@ -230,9 +271,13 @@ def _stable_artifact_path(self, path: Path, artifact_type: str) -> Path: target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(resolved, target) return target.resolve() + if resolved.is_dir(): + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(resolved, target) + return target.resolve() return resolved - def add_artifact(self, path: Path, artifact_type: str, description: str = "", *, required: bool = True) -> None: + def add_artifact(self, path: Path, artifact_type: str, description: str = "", *, required: bool = True) -> Path: path = path.resolve() stable_path = self._stable_artifact_path(path, artifact_type) if path.exists() else path exists = path.exists() @@ -245,6 +290,8 @@ def add_artifact(self, path: Path, artifact_type: str, description: str = "", *, inside_run_dir = False if not exists: durability_status = "missing_required" if required else "not_applicable_optional_missing" + elif is_dir and inside_run_dir: + durability_status = "durable" elif is_dir: durability_status = "not_applicable_directory_reference" elif inside_run_dir: @@ -265,6 +312,7 @@ def add_artifact(self, path: Path, artifact_type: str, description: str = "", *, entry["bytes"] = stable_path.stat().st_size entry["sha256"] = sha256(stable_path) self.artifacts.append(entry) + return stable_path def run( self, @@ -295,11 +343,13 @@ def run( ) except subprocess.TimeoutExpired as exc: timed_out = True + stdout = exc.stdout if isinstance(exc.stdout, str) else "" + stderr = exc.stderr if isinstance(exc.stderr, str) else "" proc = subprocess.CompletedProcess( argv, 124, - stdout=exc.stdout if isinstance(exc.stdout, str) else "", - stderr=exc.stderr if isinstance(exc.stderr, str) else f"timed out after {timeout}s", + stdout=stdout, + stderr=stderr or f"timed out after {timeout}s", ) duration = time.monotonic() - started stdout_path.write_text(redact(proc.stdout), encoding="utf-8", errors="replace") diff --git a/tests/journeys/phase22/code/test_j09_report_delivery.py b/tests/journeys/phase22/code/test_j09_report_delivery.py index 0e66a0d3e..f01213699 100644 --- a/tests/journeys/phase22/code/test_j09_report_delivery.py +++ b/tests/journeys/phase22/code/test_j09_report_delivery.py @@ -1,12 +1,17 @@ from __future__ import annotations +import copy import hashlib import json +import os from pathlib import Path +import jsonschema + from evidence import JourneyRecorder from journey_runner import ( action_evidence, + bootstrap_live_environment, run_autosci, runtime_evidence, write_code_evidence, @@ -106,6 +111,103 @@ def _is_stable_evidence_id(value: object) -> bool: return text.startswith(("claim-", "claim:", "code:", "exp-", "local:", "paper-", "phase22-", "runtime:", "task-", "wiki:")) +def _file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else "0" * 64 + + +def _decision_request(verdict_path: Path, experiment_path: Path) -> dict: + return { + "schema": "decision_request.v1", + "decision_id": "phase22-j09-bounded-rollout", + "title": "Choose the evidence-supported SkillGen rollout scope", + "problem": "Choose between a bounded local continuation and an unsupported generalized rollout.", + "alternatives": [ + { + "alternative_id": "bounded-local", + "title": "Continue with a bounded local validation", + "description": "Retain the measured scope and collect independent external-validity evidence.", + }, + { + "alternative_id": "generalized-rollout", + "title": "Generalize the result immediately", + "description": "Treat the local result as sufficient for a broad rollout.", + }, + ], + "criteria": [ + {"criterion_id": "measured-support", "name": "Measured evidence support", "weight": 0.6}, + {"criterion_id": "external-validity", "name": "External-validity risk", "weight": 0.4}, + ], + "evidence": [ + { + "evidence_id": "j09-claim-verdict", + "evidence_type": "claim_verdict", + "claim_id": "claim-supported", + "supporting_experiment_evidence_id": "j09-experiment-result", + "source_path": str(verdict_path), + "expected_sha256": _file_sha256(verdict_path), + "summary": "The persisted claim verdict supports the bounded local claim.", + }, + { + "evidence_id": "j09-experiment-result", + "evidence_type": "experiment_result", + "source_path": str(experiment_path), + "expected_sha256": _file_sha256(experiment_path), + "summary": "The approved local experiment completed and produced persisted result evidence.", + }, + ], + "assessments": [ + { + "alternative_id": "bounded-local", + "criterion_id": "measured-support", + "score": 5, + "rationale": "This option stays within the supported claim and completed experiment.", + "evidence_ids": ["j09-claim-verdict", "j09-experiment-result"], + }, + { + "alternative_id": "bounded-local", + "criterion_id": "external-validity", + "score": 3, + "rationale": "The option explicitly keeps external validity unresolved.", + "evidence_ids": ["j09-claim-verdict"], + }, + { + "alternative_id": "generalized-rollout", + "criterion_id": "measured-support", + "score": 0, + "rationale": "Neither evidence source supports generalization beyond the local dataset.", + "evidence_ids": ["j09-claim-verdict", "j09-experiment-result"], + }, + { + "alternative_id": "generalized-rollout", + "criterion_id": "external-validity", + "score": 0, + "rationale": "The local result provides no independent external-validity evidence for a broad rollout.", + "evidence_ids": ["j09-claim-verdict", "j09-experiment-result"], + }, + ], + "risks": [ + { + "risk_id": "external-validity", + "description": "A local result could be overstated as a general result.", + "mitigation": "Keep the recommendation bounded and require independent review before rollout.", + "evidence_ids": ["j09-claim-verdict", "j09-experiment-result"], + } + ], + "recommendation": { + "alternative_id": "bounded-local", + "rationale": "Only the bounded continuation is explicitly supported by the measured result and claim verdict.", + "criterion_ids": ["measured-support", "external-validity"], + "evidence_ids": ["j09-claim-verdict", "j09-experiment-result"], + }, + "limitations": [ + "The evidence comes from one deterministic local experiment and does not establish external validity." + ], + "unresolved_review_items": [ + "An independent reviewer must assess external validity and the proposed rollout boundary." + ], + } + + def test_p22_j09_report_delivery(repo_root: Path, tmp_path: Path, phase22_python: str) -> None: rec = JourneyRecorder(repo_root, "P22-J09") sandbox = tmp_path / "p22-j09" @@ -183,6 +285,80 @@ def test_p22_j09_report_delivery(repo_root: Path, tmp_path: Path, phase22_python verdict_ev = action_evidence(verify, "verify_claim") verdict_payload = _load_json(verdict_ev) verdict_boundary = verdict_payload.get("outputs", {}).get("final_verdict_boundary", {}) + if verdict_ev: + rec.add_artifact(Path(verdict_ev), "claim_verdict") + recorded_verdict = Path(rec.artifacts[-1]["path"]) + if not recorded_verdict.is_absolute(): + recorded_verdict = rec.run_dir / recorded_verdict + else: + recorded_verdict = rec.artifact_dir / "missing-claim-verdict.json" + if exp_result_ev: + rec.add_artifact(Path(exp_result_ev), "experiment_result_evidence") + recorded_experiment = Path(rec.artifacts[-1]["path"]) + if not recorded_experiment.is_absolute(): + recorded_experiment = rec.run_dir / recorded_experiment + else: + recorded_experiment = rec.artifact_dir / "missing-experiment-result.json" + decision_request_path = write_json( + rec.artifact_dir / "decision-request.json", + _decision_request(recorded_verdict, recorded_experiment), + ) + decision_output = rec.artifact_dir / "decision-artifact.json" + decision_proc = rec.run( + "decision-artifact-construction", + [ + phase22_python, + str(repo_root / "harness" / "tools" / "decision_artifact.py"), + "--input", + str(decision_request_path), + "--output", + str(decision_output), + "--source-root", + str(rec.run_dir), + ], + cwd=repo_root, + timeout=60, + ) + decision_payload = _load_json(decision_output) + decision_schema = _load_json( + repo_root / "harness" / "schemas" / "evidence" / "decision_artifact.v1.schema.json" + ) + decision_schema_error = "" + try: + jsonschema.Draft202012Validator(decision_schema).validate(decision_payload) + except jsonschema.ValidationError as exc: + decision_schema_error = exc.message + negative_verdict_payload = copy.deepcopy(verdict_payload) + negative_verdict_payload["outputs"]["verdicts"][0]["verdict"] = "not_supported" + negative_verdict_path = write_json( + rec.artifact_dir / "decision-negative-claim-verdict.json", + negative_verdict_payload, + ) + negative_request = _decision_request(negative_verdict_path, recorded_experiment) + negative_request_path = write_json( + rec.artifact_dir / "decision-request-unsupported.json", negative_request + ) + negative_output = rec.artifact_dir / "decision-artifact-unsupported.json" + negative_output.write_text( + '{"schema":"decision_artifact.v1","stale":true}\n', encoding="utf-8" + ) + negative_proc = rec.run( + "decision-artifact-unsupported-evidence", + [ + phase22_python, + str(repo_root / "harness" / "tools" / "decision_artifact.py"), + "--input", + str(negative_request_path), + "--output", + str(negative_output), + "--source-root", + str(rec.run_dir), + ], + cwd=repo_root, + timeout=60, + ) + rec.add_artifact(decision_request_path, "decision_request") + rec.add_artifact(decision_output, "decision_artifact") discovery = write_json( sandbox / "literature-discovery.json", { @@ -246,8 +422,6 @@ def test_p22_j09_report_delivery(repo_root: Path, tmp_path: Path, phase22_python (handoff, "report_handoff_manifest"), (claims, "research_claims"), (code_ev, "code_evidence"), - (exp_result_ev, "experiment_result_evidence"), - (verdict_ev, "claim_verdict"), (discovery, "discovery_evidence"), ): if path: @@ -278,16 +452,50 @@ def test_p22_j09_report_delivery(repo_root: Path, tmp_path: Path, phase22_python "--before-artifact", str(before), ] - plan, _ = run_autosci(rec, sandbox, "paper-plan", [*common_args, "--run-id", "p22-j09-plan"], timeout=90) + plan, _ = run_autosci(rec, sandbox, "paper-plan", [*common_args, "--run-id", "p22-j09-plan-pre-review"], timeout=90) draft, harness_dir = run_autosci(rec, sandbox, "paper-draft", [*common_args, "--run-id", "p22-j09-draft"], timeout=90) draft_ev = action_evidence(draft, "write_report") review_target = draft_ev or paper review_proof = _write_review_proof(sandbox / "review-proof.json", Path(review_target)) - review, _ = run_autosci(rec, sandbox, "review", [str(review_target), "--proof-bundle", str(review_proof), "--focus", "method", "--run-id", "p22-j09-review"], timeout=90) - compile_result, _ = run_autosci(rec, sandbox, "paper-compile", [str(review_target), "--checklist", "--run-id", "p22-j09-compile"], timeout=90) + review_args = [ + str(review_target), + "--proof-bundle", + str(review_proof), + "--focus", + "method", + ] + live_env = bootstrap_live_environment(repo_root) + review_provider = live_env.get("AUTOSCI_LIVE_REVIEW_LLM_PROVIDER") or live_env.get("AUTOSCI_REVIEW_LLM_PROVIDER") + review_model = live_env.get("AUTOSCI_LIVE_REVIEW_LLM_MODEL") or live_env.get("AUTOSCI_REVIEW_LLM_MODEL") + if review_provider: + review_args.extend(["--review", "--review-llm-provider", review_provider]) + if review_model: + review_args.extend(["--review-llm-model", review_model]) + review, _ = run_autosci( + rec, + sandbox, + "review", + [*review_args, "--run-id", "p22-j09-review"], + timeout=120, + allow_live=bool(review_provider), + ) + review_ev = action_evidence(review, "review_artifact") + reviewed_common_args = [*common_args] + if review_ev: + reviewed_common_args.extend(["--review-llm-evidence", str(review_ev)]) + plan, _ = run_autosci( + rec, + sandbox, + "paper-plan", + [*reviewed_common_args, "--run-id", "p22-j09-plan-reviewed"], + timeout=90, + ) + compile_args = [str(review_target), "--checklist"] + if review_ev: + compile_args.extend(["--review-llm-evidence", str(review_ev)]) + compile_result, _ = run_autosci(rec, sandbox, "paper-compile", [*compile_args, "--run-id", "p22-j09-compile"], timeout=90) plan_ev = action_evidence(plan, "plan_report") - review_ev = action_evidence(review, "review_artifact") compile_ev = action_evidence(compile_result, "compile_paper") for path, typ in ( (plan_ev, "report_plan"), @@ -303,6 +511,24 @@ def test_p22_j09_report_delivery(repo_root: Path, tmp_path: Path, phase22_python markdown_text = markdown_report.read_text(encoding="utf-8", errors="replace") if markdown_report else "" if markdown_report: rec.add_artifact(markdown_report, "readable_markdown_report") + pdf_report = rec.run_dir / "verifier-guided-skill-learning-report.pdf" + pdf_tool = repo_root / "harness" / "tools" / "markdown_pdf.py" + pdf_build = rec.run( + "publication-pdf-build", + [phase22_python, str(pdf_tool), "build", "--input", str(markdown_report), "--output", str(pdf_report)] + if markdown_report else [phase22_python, str(pdf_tool), "verify", "--input", str(pdf_report)], + cwd=repo_root, + timeout=60, + ) + pdf_verify = rec.run( + "publication-pdf-verify", + [phase22_python, str(pdf_tool), "verify", "--input", str(pdf_report)], + cwd=repo_root, + timeout=60, + ) + pdf_payload = json.loads(pdf_build.stdout) if pdf_build.returncode == 0 else {} + if pdf_report.is_file(): + rec.add_artifact(pdf_report, "compiled_pdf_report") draft_report = draft_payload.get("outputs", {}).get("report", {}) if isinstance(draft_payload, dict) else {} draft_sections = draft_report.get("sections", []) if isinstance(draft_report, dict) else [] draft_section_ids = {section.get("section_id") for section in draft_sections if isinstance(section, dict)} @@ -321,10 +547,185 @@ def test_p22_j09_report_delivery(repo_root: Path, tmp_path: Path, phase22_python compile_limits = compile_payload.get("limitations", []) if isinstance(compile_payload, dict) else [] compile_policy_blocked = compile_payload.get("outputs", {}).get("policy_decision", {}).get("blocked", False) if isinstance(compile_payload, dict) else False + def recorded_artifact_path(artifact_type: str) -> Path: + entry = next(item for item in rec.artifacts if item.get("type") == artifact_type) + path = Path(str(entry["path"])) + return path if path.is_absolute() else rec.run_dir / path + + external_delivery_audit = Path(os.environ.get("PHASE22_J09_EXTERNAL_DELIVERY_AUDIT", "") or "") + external_delivery_enabled = external_delivery_audit.is_file() + stable_external_delivery_audit = ( + rec.add_artifact(external_delivery_audit, "external_delivery_audit", "Approved external Gmail delivery audit.") + if external_delivery_enabled + else None + ) + delivery_permissions = ( + { + "distribution_scope": "external_email", + "approval_required": True, + "approval_state": "approved", + "approval_ref": "approval-phase22-gmail-handoff", + "approved_by": "user:j50058254", + "approved_at": "2026-08-12T00:00:00Z", + } + if external_delivery_enabled + else {"distribution_scope": "local_only", "approval_required": True, "approval_state": "not_requested"} + ) + external_delivery_spec = ( + { + "channel": "gmail", + "recipient": "jms.duck1020@gmail.com", + "runtime_evidence_path": str(stable_external_delivery_audit), + "recipient_acceptance_required": False, + } + if external_delivery_enabled + else None + ) + request_payload = { + "schema": "publication_delivery_request.v1", + "delivery_id": "phase22-j09-technical-lead-handoff", + "audience": {"role": "technical_lead", "description": "Technical decision-maker reviewing the bounded local SkillGen result."}, + "delivery_format": "mixed_bundle", + "content_scope": ["report", "compiled-pdf", "delivery-plan", "provider-review", "decision-artifact"], + "permissions": delivery_permissions, + "files": [ + {"file_id": "report", "type": "readable_markdown_report", "source_path": str(recorded_artifact_path("readable_markdown_report")), "evidence_ids": ["report:phase22-j09"]}, + {"file_id": "compiled-pdf", "type": "compiled_pdf_report", "source_path": str(recorded_artifact_path("compiled_pdf_report")), "evidence_ids": ["report:phase22-j09", "compile:phase22-j09"]}, + {"file_id": "plan", "type": "report_plan", "source_path": str(recorded_artifact_path("report_plan")), "evidence_ids": ["plan:phase22-j09"]}, + {"file_id": "review", "type": "provider_review", "source_path": str(recorded_artifact_path("artifact_review")), "evidence_ids": ["review:phase22-j09", *[str(item) for item in review_boundary.get("evidence_ids") or []]]}, + {"file_id": "decision", "type": "decision_artifact", "source_path": str(recorded_artifact_path("decision_artifact")), "evidence_ids": ["decision:phase22-j09"]}, + ], + } + if external_delivery_spec: + request_payload["external_delivery"] = external_delivery_spec + delivery_request = write_json( + rec.artifact_dir / "publication-delivery-request.json", + request_payload, + ) + delivery_dir = rec.run_dir / "publication-delivery" + delivery_tool = repo_root / "harness" / "tools" / "publication_delivery_bundle.py" + delivery_build = rec.run( + "publication-delivery-build", + [phase22_python, str(delivery_tool), "build", "--request", str(delivery_request), "--output-dir", str(delivery_dir), "--source-root", str(rec.run_dir)], + cwd=repo_root, + timeout=60, + ) + delivery_verify = rec.run( + "publication-delivery-verify", + [phase22_python, str(delivery_tool), "verify", "--bundle-dir", str(delivery_dir)], + cwd=repo_root, + timeout=60, + ) + delivery_manifest = delivery_dir / "publication-delivery-manifest.json" + delivery_payload = _load_json(delivery_manifest) + tamper_target = delivery_dir / delivery_payload.get("files", [{}])[0].get("path", "missing") + tamper_original = tamper_target.read_bytes() if tamper_target.is_file() else b"" + if tamper_original: + tamper_target.write_bytes(tamper_original + b"\ntampered\n") + delivery_tamper = rec.run( + "publication-delivery-tamper-rejected", + [phase22_python, str(delivery_tool), "verify", "--bundle-dir", str(delivery_dir)], + cwd=repo_root, + timeout=60, + ) + if tamper_original: + tamper_target.write_bytes(tamper_original) + delivery_restored = rec.run( + "publication-delivery-restored-verify", + [phase22_python, str(delivery_tool), "verify", "--bundle-dir", str(delivery_dir)], + cwd=repo_root, + timeout=60, + ) + rec.add_artifact(delivery_request, "publication_delivery_request") + rec.add_artifact(delivery_manifest, "publication_delivery_manifest") + rec.add_assertion("upstream_ingest_completed", not ingest.get("_error"), ingest.get("_error")) rec.add_assertion("upstream_experiment_result_available", exp_result_ev is not None, exp_run.get("_error")) rec.add_assertion("upstream_claim_verdict_available", verdict_ev is not None, verify.get("_error")) rec.add_assertion("claim_verdict_completed", verdict_payload.get("status") == "completed", verdict_payload.get("status")) + rec.add_assertion("decision_constructor_completed", decision_proc.returncode == 0, decision_proc.stderr) + rec.add_assertion("decision_artifact_schema_valid", not decision_schema_error, decision_schema_error) + rec.add_assertion( + "decision_recommendation_traces_to_criteria_and_evidence", + set(decision_payload.get("recommendation", {}).get("criterion_ids", [])) == {"measured-support", "external-validity"} + and set(decision_payload.get("recommendation", {}).get("evidence_ids", [])) + == {"j09-claim-verdict", "j09-experiment-result"}, + decision_payload.get("recommendation"), + ) + evidence_links = decision_payload.get("evidence_links", []) + rec.add_assertion( + "decision_evidence_provenance_reloaded_and_hashed", + len(evidence_links) == 2 + and all(item.get("sha256") and Path(item.get("source_path", "")).is_file() for item in evidence_links), + evidence_links, + ) + request_payload = _load_json(decision_request_path) + expected_evidence = { + item["evidence_id"]: item + for item in request_payload.get("evidence", []) + if isinstance(item, dict) and item.get("evidence_id") + } + evidence_links_by_id = { + item["evidence_id"]: item + for item in evidence_links + if isinstance(item, dict) and item.get("evidence_id") + } + rec.add_assertion( + "decision_typed_evidence_matches_expected_hashes", + {item.get("evidence_type") for item in evidence_links} + == {"claim_verdict", "experiment_result"} + and all( + item.get("sha256") + == expected_evidence.get(item.get("evidence_id"), {}).get("expected_sha256") + == _file_sha256(Path(item.get("source_path", ""))) + for item in evidence_links + ) + and evidence_links_by_id.get("j09-claim-verdict", {}).get("observed_support") + == "supported" + and evidence_links_by_id.get("j09-claim-verdict", {}).get( + "supporting_evidence_ids" + ) + == ["j09-experiment-result"] + and evidence_links_by_id.get("j09-experiment-result", {}).get( + "observed_support" + ) + == "supports", + {"request": expected_evidence, "artifact": evidence_links}, + ) + expected_pairs = { + (alternative["alternative_id"], criterion["criterion_id"]) + for alternative in decision_payload.get("alternatives", []) + for criterion in decision_payload.get("criteria", []) + } + actual_pairs = { + (assessment.get("alternative_id"), assessment.get("criterion_id")) + for assessment in decision_payload.get("assessments", []) + } + rec.add_assertion( + "decision_assessment_matrix_complete", + actual_pairs == expected_pairs, + {"expected": sorted(expected_pairs), "actual": sorted(actual_pairs)}, + ) + rec.add_assertion( + "decision_request_provenance_matches_persisted_bytes", + decision_payload.get("provenance", {}).get("request_sha256") + == _file_sha256(decision_request_path), + decision_payload.get("provenance"), + ) + rec.add_assertion( + "decision_review_and_approval_state_truthful", + decision_payload.get("decision_status") == "review_required" + and decision_payload.get("review", {}).get("status") == "pending" + and decision_payload.get("review", {}).get("reviewed_by") is None + and decision_payload.get("approval", {}).get("status") == "not_requested" + and decision_payload.get("approval", {}).get("approved_by") is None, + {"review": decision_payload.get("review"), "approval": decision_payload.get("approval")}, + ) + rec.add_assertion( + "unsupported_recommendation_fails_closed", + negative_proc.returncode == 2 and not negative_output.exists(), + negative_proc.stderr, + ) rec.add_assertion("paper_plan_recorded", plan_ev is not None and plan_status in {"completed", "inconclusive"}, plan_status or plan.get("_error")) rec.add_assertion("paper_draft_completed", draft_ev is not None, draft.get("_error")) rec.add_assertion("paper_draft_status_understood", draft_status in {"completed", "inconclusive"}, draft_status) @@ -332,6 +733,42 @@ def test_p22_j09_report_delivery(repo_root: Path, tmp_path: Path, phase22_python rec.add_assertion("review_artifact_status_understood", review_payload.get("status") in {"completed", "inconclusive"}, review_payload.get("status")) rec.add_assertion("compile_or_checklist_evidence_recorded", compile_ev is not None, compile_result.get("_error")) rec.add_assertion("paper_compile_status_understood", compile_status in {"completed", "inconclusive"}, compile_status) + rec.add_assertion( + "compiled_pdf_report_structurally_verified", + pdf_build.returncode == 0 + and pdf_verify.returncode == 0 + and pdf_payload.get("valid") is True + and pdf_payload.get("page_count", 0) >= 1 + and len(str(pdf_payload.get("sha256") or "")) == 64 + and pdf_report.stat().st_size > 500, + {"build": pdf_build.returncode, "verify": pdf_verify.returncode, "result": pdf_payload}, + ) + rec.add_assertion( + "publication_delivery_contract_complete", + delivery_build.returncode == 0 + and delivery_verify.returncode == 0 + and delivery_payload.get("audience", {}).get("role") == "technical_lead" + and delivery_payload.get("permissions") == delivery_permissions + and len(delivery_payload.get("handoff_checklist") or []) >= 5 + and len(delivery_payload.get("files") or []) == (6 if external_delivery_enabled else 5) + and len(delivery_payload.get("evidence_index") or []) >= 4, + {"build": delivery_build.returncode, "verify": delivery_verify.returncode, "manifest": delivery_payload}, + ) + if external_delivery_enabled: + external_delivery = delivery_payload.get("external_delivery", {}) + rec.add_assertion( + "publication_delivery_external_gmail_verified", + external_delivery.get("channel") == "gmail" + and external_delivery.get("recipient") == "jms.duck1020@gmail.com" + and external_delivery.get("delivered") is True + and external_delivery.get("approval_ref") == "approval-phase22-gmail-handoff", + external_delivery, + ) + rec.add_assertion( + "publication_delivery_integrity_fails_closed", + delivery_tamper.returncode == 2 and delivery_restored.returncode == 0, + {"tamper_exit": delivery_tamper.returncode, "restored_exit": delivery_restored.returncode, "tamper_stderr": delivery_tamper.stderr}, + ) rec.add_assertion( "draft_report_required_sections", {"summary", "findings", "evidence-map", "limitations"}.issubset(draft_section_ids), @@ -417,6 +854,13 @@ def test_p22_j09_report_delivery(repo_root: Path, tmp_path: Path, phase22_python draft_ev or rec.run_dir, "partial", ) + rec.add_l2( + "Foundation", + "Decision Artifact Construction", + "The production constructor built a schema-valid decision whose recommendation is bound to criteria and reloaded upstream evidence; unresolved review and approval remained explicit.", + decision_output, + True, + ) status = "PASS_WITH_KNOWN_LIMITATIONS" if limitations and all(item["passed"] for item in rec.assertions) else "PASS" if not all(item["passed"] for item in rec.assertions): diff --git a/tests/journeys/phase22/code/test_j18_real_linux_status_lifecycle.py b/tests/journeys/phase22/code/test_j18_real_linux_status_lifecycle.py index ea4fd703b..7192b323c 100644 --- a/tests/journeys/phase22/code/test_j18_real_linux_status_lifecycle.py +++ b/tests/journeys/phase22/code/test_j18_real_linux_status_lifecycle.py @@ -6,6 +6,7 @@ import re import shutil import subprocess +import tempfile import time from datetime import datetime, timezone from pathlib import Path @@ -125,6 +126,26 @@ def _http_text(url: str, *, token: str | None = None) -> dict[str, Any]: return {"status": 0, "content_type": "", "error": str(exc), "body_prefix": ""} +def _body_dict(record: dict[str, Any]) -> dict[str, Any]: + body = record.get("body") + return body if isinstance(body, dict) else {} + + +def _repo_head(repo_root: Path) -> str: + env_head = os.environ.get("PHASE22_REPO_HEAD", "").strip() + proc = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo_root, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if proc.returncode == 0: + return proc.stdout.strip() + return env_head or f"unavailable: {_tail(proc.stderr, 300)}" + + def _wait_for_port(port_file: Path, deadline_seconds: int = 20) -> str: deadline = time.monotonic() + deadline_seconds while time.monotonic() < deadline: @@ -158,7 +179,10 @@ def test_p22_j18_real_linux_status_lifecycle(repo_root: Path, tmp_path: Path) -> sandbox_home = tmp_path / "home" solar_home = sandbox_home / ".solar" claude_dir = sandbox_home / ".claude" - tmux_tmpdir = tmp_path / "tmux" + sandbox_home_b = tmp_path / "home-b" + solar_home_b = sandbox_home_b / ".solar" + claude_dir_b = sandbox_home_b / ".claude" + tmux_tmpdir = Path(tempfile.mkdtemp(prefix=f"solar-p22-j18-tmux-{os.getpid()}-", dir="/tmp")) tmux_tmpdir.mkdir(parents=True, exist_ok=True) env = os.environ.copy() env.update( @@ -167,6 +191,8 @@ def test_p22_j18_real_linux_status_lifecycle(repo_root: Path, tmp_path: Path) -> "USERPROFILE": str(sandbox_home), "SOLAR_HOME": str(solar_home), "CLAUDE_DIR": str(claude_dir), + "HARNESS_DIR": str(solar_home / "harness"), + "SOLAR_HARNESS_DIR": str(solar_home / "harness"), "SOLAR_PANE_RUNTIME": "codex", "PHASE22_SELECTED_RUNTIME": "codex", "SOLAR_NO_MCP": "true", @@ -176,6 +202,17 @@ def test_p22_j18_real_linux_status_lifecycle(repo_root: Path, tmp_path: Path) -> "TMUX_TMPDIR": str(tmux_tmpdir), } ) + env_b = dict(env) + env_b.update( + { + "HOME": str(sandbox_home_b), + "USERPROFILE": str(sandbox_home_b), + "SOLAR_HOME": str(solar_home_b), + "CLAUDE_DIR": str(claude_dir_b), + "HARNESS_DIR": str(solar_home_b / "harness"), + "SOLAR_HARNESS_DIR": str(solar_home_b / "harness"), + } + ) commands: list[dict[str, Any]] = [] http_checks: dict[str, Any] = {} @@ -183,6 +220,7 @@ def test_p22_j18_real_linux_status_lifecycle(repo_root: Path, tmp_path: Path) -> cleanup: dict[str, Any] = {} install_receipt_seen = False started_at = _utc_now() + distro = {} try: commands.append( @@ -211,6 +249,21 @@ def test_p22_j18_real_linux_status_lifecycle(repo_root: Path, tmp_path: Path) -> timeout=180, ) ) + distro_record = _run( + run_dir, + "distro-release", + ["bash", "-lc", ". /etc/os-release; printf '%s\\n%s\\n' \"$ID\" \"$VERSION_ID\""], + cwd=repo_root, + env=env, + timeout=30, + ) + commands.append(distro_record) + distro_lines = distro_record["stdout_tail"].strip().splitlines() + distro = { + "id": distro_lines[0] if distro_lines else "unknown", + "version_id": distro_lines[1] if len(distro_lines) > 1 else "unknown", + "wsl_distro_name": os.environ.get("WSL_DISTRO_NAME", ""), + } solar_bin = solar_home / "bin" / "solar" harness_script = solar_home / "harness" / "solar-harness.sh" install_receipt_seen = (solar_home / "install-receipt.json").exists() @@ -245,37 +298,145 @@ def test_p22_j18_real_linux_status_lifecycle(repo_root: Path, tmp_path: Path) -> "status_server_tmux_session_observed": "solar-harness-status-server-" in tmux_session_line, "tmux_sessions_tail": tmux_session_line, } + + commands.append( + _run( + run_dir, + "install-second-kernel-harness", + [ + "bash", + str(repo_root / "install.sh"), + "--yes", + "--components", + "kernel,harness", + "--solar-home", + str(solar_home_b), + "--claude-dir", + str(claude_dir_b), + "--skip-llm-cli", + "--skip-py-deps", + "--no-hooks", + "--no-mcp", + "--set", + "runtime=codex", + ], + cwd=repo_root, + env=env_b, + timeout=180, + ) + ) + solar_bin_b = solar_home_b / "bin" / "solar" + harness_script_b = solar_home_b / "harness" / "solar-harness.sh" + commands.append(_run(run_dir, "second-status-server-start", [str(harness_script_b), "status-server", "start"], cwd=repo_root, env=env_b, timeout=60)) + port_file_b = solar_home_b / "harness" / "run" / "status-server.port" + token_file_b = solar_home_b / "harness" / "run" / "status-server.token" + port_b = _wait_for_port(port_file_b) + token_b = _wait_for_text(token_file_b) or None + base_b = f"http://127.0.0.1:{port_b}" + http_checks["second_healthz"] = _http_text(f"{base_b}/healthz", token=token_b) if port_b else {"status": 0} + http_checks["second_runtime_info"] = _http_json(f"{base_b}/runtime-info", token=token_b) if port_b else {"status": 0} + commands.append(_run(run_dir, "primary-status-server-stop-while-second-live", [str(harness_script), "status-server", "stop"], cwd=repo_root, env=env, timeout=60)) + http_checks["second_healthz_after_primary_stop"] = _http_text(f"{base_b}/healthz", token=token_b) if port_b else {"status": 0} + commands.append(_run(run_dir, "primary-uninstall-while-second-live", [str(solar_bin), "uninstall", "--yes"], cwd=repo_root, env=env, timeout=120)) + http_checks["second_healthz_after_primary_uninstall"] = _http_text(f"{base_b}/healthz", token=token_b) if port_b else {"status": 0} + commands.append(_run(run_dir, "tmux-list-after-primary-uninstall", ["tmux", "list-sessions", "-F", "#{session_name}"], cwd=repo_root, env=env_b, timeout=30)) + tmux_checks.update( + { + "second_port": port_b, + "ports_are_distinct": bool(port and port_b and port != port_b), + "second_runtime_owned_by_second_harness": _body_dict(http_checks["second_runtime_info"]).get("harness_dir") == str(solar_home_b / "harness"), + "second_session_survived_primary_stop_and_uninstall": http_checks["second_healthz_after_primary_stop"].get("status") == 200 + and http_checks["second_healthz_after_primary_uninstall"].get("status") == 200, + "second_tmux_session_remained": "solar-harness-status-server-" in commands[-1]["stdout_tail"], + } + ) finally: if "harness_script" in locals() and harness_script.exists(): commands.append(_run(run_dir, "status-server-stop", [str(harness_script), "status-server", "stop"], cwd=repo_root, env=env, timeout=60)) if "solar_bin" in locals() and solar_bin.exists(): commands.append(_run(run_dir, "uninstall-yes", [str(solar_bin), "uninstall", "--yes"], cwd=repo_root, env=env, timeout=120)) + if "harness_script_b" in locals() and harness_script_b.exists(): + commands.append(_run(run_dir, "second-status-server-stop", [str(harness_script_b), "status-server", "stop"], cwd=repo_root, env=env_b, timeout=60)) + if "solar_bin_b" in locals() and solar_bin_b.exists(): + commands.append(_run(run_dir, "second-uninstall-yes", [str(solar_bin_b), "uninstall", "--yes"], cwd=repo_root, env=env_b, timeout=120)) + shutil.rmtree(tmux_tmpdir, ignore_errors=True) cleanup = { "solar_home_exists_after_uninstall": solar_home.exists(), "claude_dir_exists_after_uninstall": claude_dir.exists(), + "second_solar_home_exists_after_uninstall": solar_home_b.exists(), + "second_claude_dir_exists_after_uninstall": claude_dir_b.exists(), + "tmux_tmpdir_exists_after_cleanup": tmux_tmpdir.exists(), } install_ok = commands[0]["exit_code"] == 0 and install_receipt_seen - doctor_ok = any(command["label"] == "doctor-json" and command["exit_code"] == 0 for command in commands) - status_ok = any(command["label"] == "status-json" and command["exit_code"] == 0 for command in commands) + doctor_record = next((command for command in commands if command["label"] == "doctor-json"), {}) + status_record = next((command for command in commands if command["label"] == "status-json"), {}) + try: + doctor_payload = json.loads((run_dir / doctor_record["stdout_path"]).read_text(encoding="utf-8")) + except (KeyError, OSError, json.JSONDecodeError): + doctor_payload = {} + try: + status_payload = json.loads((run_dir / status_record["stdout_path"]).read_text(encoding="utf-8")) + except (KeyError, OSError, json.JSONDecodeError): + status_payload = {} + # The test intentionally skips Python/LLM CLI dependencies. A diagnostic + # exit 1 is valid when the structured payload truthfully reports degraded + # readiness while confirming the installed paths and Python minimum. + doctor_ok = ( + doctor_record.get("exit_code") in {0, 1} + and doctor_payload.get("paths", {}).get("solar_home") == "ok" + and doctor_payload.get("paths", {}).get("receipt") == "ok" + and doctor_payload.get("python", {}).get("min_ok") is True + and doctor_payload.get("verdict") in {"pass", "fail"} + ) + status_ok = ( + status_record.get("exit_code") in {0, 1} + and status_payload.get("status") in {"installed", "ok", "degraded"} + and status_payload.get("install", {}).get("paths", {}).get("receipt", {}).get("state") == "ok" + ) health_ok = http_checks.get("healthz", {}).get("status") == 200 and http_checks.get("healthz", {}).get("body_prefix") == "ok" - runtime_ok = http_checks.get("runtime_info", {}).get("body", {}).get("harness_dir") == str(solar_home / "harness") + runtime_ok = _body_dict(http_checks.get("runtime_info", {})).get("harness_dir") == str(solar_home / "harness") status_payload_ok = isinstance(http_checks.get("status", {}).get("body"), dict) settings_payload_ok = isinstance(http_checks.get("settings", {}).get("body"), dict) root_ok = http_checks.get("root", {}).get("status") == 200 tmux_ok = bool(tmux_checks.get("status_server_tmux_session_observed")) - uninstall_ok = not cleanup["solar_home_exists_after_uninstall"] + concurrent_isolation_ok = bool( + tmux_checks.get("ports_are_distinct") + and tmux_checks.get("second_runtime_owned_by_second_harness") + and tmux_checks.get("second_session_survived_primary_stop_and_uninstall") + and tmux_checks.get("second_tmux_session_remained") + ) + uninstall_ok = not cleanup["solar_home_exists_after_uninstall"] and not cleanup["second_solar_home_exists_after_uninstall"] + required = [ + install_ok, + doctor_ok, + status_ok, + health_ok, + runtime_ok, + status_payload_ok, + settings_payload_ok, + root_ok, + tmux_ok, + concurrent_isolation_ok, + uninstall_ok, + ] + final_status = "PASS_WITH_KNOWN_LIMITATIONS" if all(required) else "FAIL" evidence = { - "schema_version": "phase22.j18.real_linux_status_lifecycle.v1", + "schema_version": "phase22.j18.real_linux_status_lifecycle.v2", "journey_id": "P22-J18", "run_id": run_id, "selector": SELECTOR, "started_at": started_at, "finished_at": _utc_now(), "platform": platform.platform(), - "repo_head": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo_root, text=True).strip(), - "sandbox": {"home": str(sandbox_home), "solar_home": str(solar_home), "claude_dir": str(claude_dir), "tmux_tmpdir": str(tmux_tmpdir)}, + "distribution": distro, + "repo_head": _repo_head(repo_root), + "sandbox": { + "primary": {"home": str(sandbox_home), "solar_home": str(solar_home), "claude_dir": str(claude_dir)}, + "secondary": {"home": str(sandbox_home_b), "solar_home": str(solar_home_b), "claude_dir": str(claude_dir_b)}, + "tmux_tmpdir": str(tmux_tmpdir), + }, "commands": commands, "http_checks": http_checks, "tmux_checks": tmux_checks, @@ -290,23 +451,24 @@ def test_p22_j18_real_linux_status_lifecycle(repo_root: Path, tmp_path: Path) -> "settings_payload_parseable": settings_payload_ok, "dashboard_loaded": root_ok, "tmux_status_server_session_observed": tmux_ok, + "concurrent_harness_sessions_isolated": concurrent_isolation_ok, "uninstall_cleanup_ok": uninstall_ok, }, "observed_l2": [ { "category": "Vertical", "level_2_feature": "Linux Cli", - "status": "PASS_WITH_KNOWN_LIMITATIONS", + "status": final_status, "assertion_name": "j18_linux_cli_install_doctor_status_uninstall", "evidence_path": "journey-result.json", "known_limitations": [ - "Validated on SolarUbuntu/WSL2 with kernel,harness only; did not cover every Linux distribution, package manager, update, rollback, or repair variant." + f"Validated on {distro.get('id')} {distro.get('version_id')} WSL2 with kernel,harness only; did not cover every Linux distribution family, package manager, update, rollback, or repair variant." ], }, { "category": "Vertical", "level_2_feature": "Workflow & Platform Status Visibility", - "status": "PASS_WITH_KNOWN_LIMITATIONS", + "status": final_status, "assertion_name": "j18_status_server_health_status_runtime_projection", "evidence_path": "journey-result.json", "known_limitations": [ @@ -316,31 +478,19 @@ def test_p22_j18_real_linux_status_lifecycle(repo_root: Path, tmp_path: Path) -> { "category": "Vertical", "level_2_feature": "TMUX", - "status": "PASS_WITH_KNOWN_LIMITATIONS", + "status": final_status, "assertion_name": "j18_status_server_tmux_session_lifecycle", "evidence_path": "journey-result.json", "known_limitations": [ - "Validated one local SolarUbuntu status-server TMUX session; did not cover remote hosts, concurrent sessions, other terminal implementations, or full live user repair journeys." + f"Validated two concurrent local {distro.get('id')} {distro.get('version_id')} status-server TMUX sessions with cross-stop/uninstall isolation; did not cover remote hosts, other terminal implementations, or a broad interactive user-repair matrix." ], }, ], - "status": "PASS_WITH_KNOWN_LIMITATIONS", + "status": final_status, } result_path = _write_json(run_dir / "journey-result.json", evidence) _write_json(artifact_dir / "commands.json", commands) _write_json(artifact_dir / "http-checks.json", http_checks) _write_json(artifact_dir / "tmux-checks.json", tmux_checks) - required = [ - install_ok, - doctor_ok, - status_ok, - health_ok, - runtime_ok, - status_payload_ok, - settings_payload_ok, - root_ok, - tmux_ok, - uninstall_ok, - ] assert all(required), f"P22-J18 lifecycle failed; evidence: {result_path}" diff --git a/tests/journeys/phase22/code/test_p22_human_review_lifecycle_resume.py b/tests/journeys/phase22/code/test_p22_human_review_lifecycle_resume.py new file mode 100644 index 000000000..b8deeae0e --- /dev/null +++ b/tests/journeys/phase22/code/test_p22_human_review_lifecycle_resume.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import json +import os +import sys +import importlib.util +from pathlib import Path + +from evidence import JourneyRecorder +from journey_runner import write_json + + +def test_p22_071_human_review_lifecycle_resume(repo_root: Path, tmp_path: Path, phase22_python: str) -> None: + lib = repo_root / "harness" / "lib" + if str(lib) not in sys.path: + sys.path.insert(0, str(lib)) + + spec = importlib.util.spec_from_file_location("phase22_graph_scheduler", lib / "graph_scheduler.py") + assert spec and spec.loader + gs = importlib.util.module_from_spec(spec) + spec.loader.exec_module(gs) + + approval_statement = os.environ.get( + "PHASE22_J21_HUMAN_APPROVAL_STATEMENT", + "我批准 J21 Phase 22 lifecycle handoff 继续，审批人是 j50058254，时间是 2026-08-12。", + ) + actor = "j50058254" + rec = JourneyRecorder(repo_root, "P22-071") + harness_dir = tmp_path / "harness" + sprints = harness_dir / "sprints" + sprints.mkdir(parents=True) + sprint_id = "phase22-j21-human-review-resume" + node_id = "j21-lifecycle-handoff" + graph_path = sprints / f"{sprint_id}.task_graph.json" + + graph = { + "id": sprint_id, + "sprint_id": sprint_id, + "nodes": [ + { + "id": node_id, + "status": "pending", + "depends_on": [], + "gate": "phase22-lifecycle-human-review", + "max_repair_attempts": 2, + } + ], + "node_results": {}, + "gate_results": {"phase22-lifecycle-human-review": {"status": "blocked"}}, + "events": [], + } + block = gs.enter_node_human_review( + graph, + node_id, + reason="J21 Phase 22 lifecycle handoff requires attributable human approval before resume.", + next_action="Record the user approval statement and resume the lifecycle handoff.", + writer="phase22-lifecycle-gate", + author_type="policy", + ) + gs.save_graph(graph_path, graph) + + approval_evidence = write_json( + rec.artifact_dir / "j21-human-approval-statement.json", + { + "schema": "phase22_human_approval_statement.v1", + "issue_id": "P22-REPAIR-071", + "journey_reference": "P22-J21", + "actor": actor, + "statement": approval_statement, + "approval_date": "2026-08-12", + "approved_action": "Continue J21 Phase 22 lifecycle handoff.", + }, + ) + resume_reason = ( + "Approved J21 Phase 22 lifecycle handoff continuation; " + f"actor={actor}; date=2026-08-12; approval_evidence={approval_evidence.name}" + ) + + env = dict(os.environ) + env.update( + { + "HARNESS_DIR": str(harness_dir), + "SOLAR_HARNESS_DIR": str(harness_dir), + "HARNESS_SPRINTS_DIR": str(sprints), + "SOLAR_GATE_LEDGER": "1", + "PYTHONIOENCODING": "utf-8", + } + ) + resume = rec.run( + "human-review-resume", + [ + phase22_python, + str(lib / "graph_node_dispatcher.py"), + "resume-human-review", + "--graph", + str(graph_path), + "--node", + node_id, + "--generation", + str(block["generation"]), + "--actor", + actor, + "--reason", + resume_reason, + ], + cwd=repo_root, + env=env, + timeout=60, + ) + second_resume = rec.run( + "human-review-resume-replay-rejected", + [ + phase22_python, + str(lib / "graph_node_dispatcher.py"), + "resume-human-review", + "--graph", + str(graph_path), + "--node", + node_id, + "--generation", + str(block["generation"]), + "--actor", + actor, + "--reason", + resume_reason, + ], + cwd=repo_root, + env=env, + timeout=60, + ) + + resume_payload = json.loads(resume.stdout) if resume.stdout.strip() else {} + second_payload = json.loads(second_resume.stdout) if second_resume.stdout.strip() else {} + persisted = gs.load_graph(graph_path) + node = next(item for item in persisted["nodes"] if item["id"] == node_id) + ledger_path = sprints / f"{sprint_id}.gate-ledger.jsonl" + ledger_rows = [ + json.loads(line) + for line in ledger_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + resume_rows = [ + row + for row in ledger_rows + if row.get("writer") == "resume_human_review" and row.get("human_actor") == actor + ] + + rec.add_artifact(graph_path, "human_review_graph") + rec.add_artifact(ledger_path, "human_review_gate_ledger") + rec.add_artifact(approval_evidence, "human_approval_statement") + rec.add_assertion("human_review_block_created", block.get("state") == "blocked" and block.get("generation") == 1, block) + rec.add_assertion("human_resume_completed", resume.returncode == 0 and resume_payload.get("ok") is True and resume_payload.get("actor") == actor, resume_payload) + node_result = persisted.get("node_results", {}).get(node_id, {}) + rec.add_assertion( + "graph_reopened_after_human_resume", + gs.node_status(persisted, node_id) == "pending" + and node.get("repair_context", {}).get("trigger") == "explicit_human_resume" + and node_result.get("human_review", {}).get("state") == "resumed", + node, + ) + rec.add_assertion("ledger_records_human_author", bool(resume_rows) and resume_rows[-1].get("author", {}).get("type") == "human", resume_rows[-1] if resume_rows else {}) + rec.add_assertion("resume_is_one_shot", second_resume.returncode == 2 and second_payload.get("reason") == "node_not_waiting_for_human_review", second_payload) + rec.add_l2( + "Foundation", + "Lifecycle, Parity & Human Review Evaluator", + "The lifecycle handoff entered needs_human_review, resumed only with the attributable j50058254 approval statement, recorded a human-authored ledger transition, and rejected replay.", + ledger_path, + True, + ) + status = "PASS" if all(item["passed"] for item in rec.assertions) else "FAIL" + rec.finalize(status, limitations=[]) + + assert resume.returncode == 0, resume.stderr + assert second_resume.returncode == 2 + assert all(item["passed"] for item in rec.assertions) diff --git a/tests/journeys/phase22/code/test_p22_live_independent_provider_review.py b/tests/journeys/phase22/code/test_p22_live_independent_provider_review.py new file mode 100644 index 000000000..f7071f51d --- /dev/null +++ b/tests/journeys/phase22/code/test_p22_live_independent_provider_review.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any + +import pytest + +from evidence import JourneyRecorder +from journey_runner import action_evidence, bootstrap_live_environment, run_autosci, write_json + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _provider_env(repo_root: Path) -> dict[str, str]: + bootstrapped = bootstrap_live_environment(repo_root, {}) + env = { + key: value + for key, value in bootstrapped.items() + if key + in { + "AUTOSCI_LIVE_PROVIDER_TESTS", + "AUTOSCI_LIVE_REVIEW_LLM_TEST", + "AUTOSCI_LIVE_REVIEW_LLM_PROVIDER", + "AUTOSCI_LIVE_REVIEW_LLM_MODEL", + "OPENAI_API_KEY", + "OPENROUTER_API_KEY", + "PHASE22_ENABLE_NETWORK_JOURNEYS", + "SOLAR_AUTOSCI_ALLOW_NETWORK", + } + } + env["AUTOSCI_LIVE_PROVIDER_TESTS"] = "1" + env["AUTOSCI_LIVE_REVIEW_LLM_TEST"] = "1" + env["AUTOSCI_RESEARCH_LLM_PROVIDER"] = "openrouter" + env["AUTOSCI_RESEARCH_LLM_MODEL"] = os.environ.get( + "PHASE22_045_OPENROUTER_WRITER_MODEL", + "deepseek/deepseek-v3.2", + ) + env["AUTOSCI_REVIEW_LLM_MODEL"] = os.environ.get( + "PHASE22_045_OPENAI_REVIEWER_MODEL", + "gpt-5-mini", + ) + env["AUTOSCI_REVIEW_LLM_PROVIDER"] = "openai" + env["AUTOSCI_REVIEW_LLM_TIMEOUT"] = os.environ.get("PHASE22_045_REVIEW_TIMEOUT", "90") + return env + + +def _without_banned_models(env: dict[str, str]) -> dict[str, str]: + banned = {"gpt-5.5", "gpt5.5", "gpt-5.6-sol", "gpt5.6-sol", "gpt-5.6 sol", "gpt5.6 sol"} + for key in ("AUTOSCI_RESEARCH_LLM_MODEL", "AUTOSCI_REVIEW_LLM_MODEL"): + normalized = str(env.get(key) or "").strip().lower().replace("_", "-") + if normalized in banned: + raise AssertionError(f"banned live model configured for {key}") + return env + + +def _with_env(env: dict[str, str]): + class EnvPatch: + def __enter__(self) -> None: + self.previous = dict(os.environ) + os.environ.update(env) + + def __exit__(self, exc_type: object, exc: object, tb: object) -> None: + os.environ.clear() + os.environ.update(self.previous) + + return EnvPatch() + + +def _writer_report_body(writer_result: dict[str, Any]) -> str: + report = writer_result.get("report") if isinstance(writer_result.get("report"), dict) else {} + body = str(report.get("body") or "").strip() + if body: + return body + sections = report.get("sections") if isinstance(report.get("sections"), list) else [] + rendered = "\n\n".join(str(item.get("body") or "") for item in sections if isinstance(item, dict)) + return rendered.strip() + + +def test_p22_045_live_independent_provider_review(repo_root: Path, tmp_path: Path) -> None: + env = _without_banned_models(_provider_env(repo_root)) + if not env.get("OPENROUTER_API_KEY") or not env.get("OPENAI_API_KEY"): + pytest.skip("P22-045 requires OPENROUTER_API_KEY and OPENAI_API_KEY in the live environment.") + + from harness.plugins.autosci.services.production_research import ResearchModelService + + rec = JourneyRecorder(repo_root, "P22-045") + sandbox = tmp_path / "p22-045" + provider_workspace = sandbox / "writer-workspace" + provider_workspace.mkdir(parents=True) + + task_contract = { + "user_intent": "Draft a concise bounded research-review target for provider independence verification.", + "deliverable": {"language": "en"}, + } + synthesis = { + "claims": [ + { + "claim_id": "claim-openrouter-writer-provenance", + "text": "The provider-independence journey uses an OpenRouter writer artifact and a separate OpenAI reviewer.", + "evidence_ids": ["source-openrouter-writer-provenance"], + "uncertainty": "low", + "limitations": ["This proof establishes provider separation, not external scientific validity."], + } + ], + "limitations": ["Provider separation is limited to the configured live API calls."], + } + + with _with_env(env): + service = ResearchModelService.from_environment(provider_workspace) + writer_result = service( + node_id="report_draft", + task_contract=task_contract, + evidence_synthesis=synthesis, + deliverable_requirements={"format": "markdown", "max_words": 180}, + ) + + provider_usage = writer_result.get("provider_usage") if isinstance(writer_result.get("provider_usage"), list) else [] + writer_usage = provider_usage[0] if provider_usage and isinstance(provider_usage[0], dict) else {} + target = write_json(rec.artifact_dir / "openrouter-writer-output.json", writer_result) + report_body = _writer_report_body(writer_result) + review_target = rec.artifact_dir / "openrouter-writer-review-target.md" + review_target.write_text( + "# OpenRouter Writer Review Target\n\n" + + (report_body or "OpenRouter writer produced a bounded provider provenance review target.") + + "\n", + encoding="utf-8", + ) + source_text = ( + "The OpenRouter writer produced a bounded provider provenance review target. " + f"provider={writer_result.get('provider')} model={writer_result.get('model')} " + f"request_sha256={writer_usage.get('request_sha256')} response_sha256={writer_usage.get('response_sha256')}" + ) + source_path = rec.artifact_dir / "openrouter-writer-provenance-source.txt" + source_path.write_text(source_text + "\n", encoding="utf-8") + claim = "The OpenRouter writer produced a bounded provider provenance review target." + proof = write_json( + rec.artifact_dir / "openrouter-writer-review-proof.json", + { + "schema": "scientific_review_proof.v1", + "writer": { + "provider": writer_result.get("provider"), + "model": writer_result.get("model"), + "execution": { + "collection_mode": "live_provider", + "provider_usage": provider_usage, + }, + }, + "artifact": {"path": str(review_target), "sha256": _sha256(review_target)}, + "claims": [ + { + "claim_id": "claim.p22-045.openrouter-writer", + "claim": claim, + "source": { + "source_id": "p22-045-openrouter-writer-provenance", + "path": str(source_path), + "sha256": _sha256(source_path), + }, + "evidence_span": {"start": 0, "end": len(claim), "text": claim}, + "acceptance_criterion": "The proof must bind a persisted OpenRouter writer artifact to a separate completed OpenAI reviewer execution.", + "residual_risk": "This proof establishes live provider separation only; scientific external validity remains out of scope.", + } + ], + }, + ) + + rec.add_artifact(target, "openrouter_writer_output") + rec.add_artifact(review_target, "openrouter_writer_review_target") + rec.add_artifact(source_path, "openrouter_writer_provenance_source") + rec.add_artifact(proof, "scientific_review_proof") + + review, _harness_dir = run_autosci( + rec, + sandbox, + "review", + [ + str(review_target), + "--proof-bundle", + str(proof), + "--review", + "--focus", + "method", + "--difficulty", + "hard", + "--review-llm-provider", + "openai", + "--review-llm-model", + env["AUTOSCI_REVIEW_LLM_MODEL"], + "--run-id", + "p22-045-live-independent-provider-review", + ], + timeout=180, + extra_env=env, + allow_live=True, + ) + review_ev = action_evidence(review, "review_artifact") + if review_ev: + rec.add_artifact(review_ev, "openai_reviewer_evidence") + review_payload = json.loads(review_ev.read_text(encoding="utf-8")) if review_ev and review_ev.exists() else {} + review_output = review_payload.get("outputs", {}).get("review", {}) if isinstance(review_payload, dict) else {} + review_llm = review_output.get("review_llm") if isinstance(review_output.get("review_llm"), dict) else {} + proof_contract = review_output.get("proof_contract") if isinstance(review_output.get("proof_contract"), dict) else {} + independence = ( + proof_contract.get("reviewer_separation", {}) + .get("independence", {}) + if isinstance(proof_contract.get("reviewer_separation"), dict) + else {} + ) + + rec.add_assertion("writer_provider_is_openrouter", writer_result.get("provider") == "openrouter", writer_result) + rec.add_assertion("writer_runtime_was_archived", bool(writer_usage.get("request_sha256") and writer_usage.get("response_sha256") and writer_usage.get("archive_path")), writer_usage) + rec.add_assertion("reviewer_provider_completed_openai", review_llm.get("status") == "completed" and review_llm.get("invocation_mode") == "provider" and review_llm.get("provider") == "openai", review_llm) + rec.add_assertion("independent_provider_bound_to_execution", independence.get("status") == "independent_provider" and independence.get("execution_bound") is True, independence) + rec.add_l2( + "Reasoning", + "Independent evidence review", + "A live OpenRouter writer artifact was reviewed through a separate live OpenAI reviewer provider, and the proof contract bound provider independence to completed reviewer execution.", + Path(review_ev) if review_ev else proof, + True, + ) + + limitations = [] + if proof_contract.get("verdict") != "supported": + limitations.append("The independent provider route executed, but the deterministic proof contract did not mark the artifact as fully supported.") + status = "PASS_WITH_KNOWN_LIMITATIONS" if limitations and all(item["passed"] for item in rec.assertions) else "PASS" + if not all(item["passed"] for item in rec.assertions): + status = "FAIL" + rec.finalize(status, limitations=limitations) + + assert writer_result.get("provider") == "openrouter" + assert review_llm.get("status") == "completed", review_llm + assert review_llm.get("provider") == "openai" + assert independence.get("status") == "independent_provider", independence