diff --git a/docs/ci/runtime_intelligence_gitlab_artifacts.md b/docs/ci/runtime_intelligence_gitlab_artifacts.md index 7906e15..7fba8a4 100644 --- a/docs/ci/runtime_intelligence_gitlab_artifacts.md +++ b/docs/ci/runtime_intelligence_gitlab_artifacts.md @@ -97,6 +97,10 @@ The initial gate is conservative: - AIGuard coverage evidence raw context must also preserve the same Orchestrator producer markers, so the diagnostic artifact remains traceable to the Orchestrator feed without making AIGuard the producer or decision owner +- AIGuard coverage evidence raw context must preserve Orchestrator producer + markers and mapping hints for EdgeEnv history `missing_telemetry` entries + when such context exists, while keeping the entry an evidence gap rather than + successful Runtime telemetry - Orchestrator candidate context must include `run_id`, `telemetry_source`, `operation`, and `resource`, so CI can catch incomplete handoffs without making Orchestrator a regression owner diff --git a/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md b/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md index bb9740d..015b08e 100644 --- a/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md +++ b/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md @@ -131,6 +131,7 @@ Expected Lab behavior: - The same handoff gate verifies that the referenced `runtime_telemetry_history` artifact exists and preserves EdgeEnv history schema, telemetry coverage, and Runtime history seed ownership markers. - The same handoff gate verifies that missing telemetry entries remain evidence gaps while preserving Orchestrator producer markers, owner boundary flags, and EdgeEnv mapping hints when Orchestrator context is attached. - The bundle gate also requires AIGuard coverage evidence raw context to preserve the same Orchestrator mapping hint and producer markers, proving that AIGuard kept EdgeEnv/Orchestrator ownership markers as diagnosis context rather than recomputing coverage or owning deployment policy. +- The same gate requires AIGuard replay raw context to preserve Orchestrator producer markers and mapping hints for EdgeEnv history `missing_telemetry` entries when present, keeping missing telemetry as replay evidence gap context. - Additional Lab test fixtures under `tests/fixtures/edgeenv_regression/` mirror EdgeEnv replay examples for candidate telemetry gaps and execution sequence inversion. These fixture smokes verify that replay warnings become Lab-owned report context without making Lab recompute EdgeEnv comparability. - Markdown/HTML reports include a `Runtime Intelligence Risk Summary` that summarizes EdgeEnv comparability/regression, telemetry replay gaps, Runtime history seed traceability, AIGuard deterministic evidence, and the Lab-owned deployment decision in one reviewer-facing table. - When EdgeEnv includes preserved Orchestrator feed context, the `Runtime Intelligence Risk Summary` surfaces queue, thermal, throttling, memory, and fallback context as supplemental runtime evidence. diff --git a/examples/runtime_intelligence_chain/aiguard_runtime_operation_guard_analysis.json b/examples/runtime_intelligence_chain/aiguard_runtime_operation_guard_analysis.json index 1e60767..4abdcea 100644 --- a/examples/runtime_intelligence_chain/aiguard_runtime_operation_guard_analysis.json +++ b/examples/runtime_intelligence_chain/aiguard_runtime_operation_guard_analysis.json @@ -47,8 +47,8 @@ "runtime_telemetry_context_present": true, "runtime_telemetry_source": "result_artifacts+runtime_telemetry_history", "runtime_telemetry_history_schema_version": "edgeenv.runtime-telemetry-history.v1", - "history_orchestrator_feed_runs": 1.0, - "history_registered_runs": 2.0, + "history_orchestrator_feed_runs": 2.0, + "history_registered_runs": 3.0, "history_telemetry_runs": 2.0, "history_telemetry_seed_runs": 2.0, "baseline_runtime_telemetry_history_seed_schema_version": "inferedge-runtime-telemetry-history-seed-v1", @@ -60,7 +60,7 @@ "candidate_runtime_telemetry_history_seed_production_monitoring": false, "candidate_runtime_telemetry_history_seed_missing_telemetry_is_failure": false, "candidate_runtime_telemetry_history_seed_point_count": 1.0, - "history_missing_telemetry_runs": 0.0, + "history_missing_telemetry_runs": 1.0, "telemetry_coverage_source": "history_telemetry_coverage", "history_telemetry_coverage_missing_field_run_count": 1.0, "history_telemetry_coverage_missing_field_runs": [ @@ -133,7 +133,81 @@ "baseline_queue_depth": null, "candidate_queue_depth": 7.0, "evidence_gap_count": 1.0, - "evidence_gaps": [] + "evidence_gaps": [], + "history_missing_orchestrator_context_count": 1.0, + "history_missing_orchestrator_context_run_ids": [ + "edgeenv-smoke-missing" + ], + "history_missing_orchestrator_contexts": [ + { + "schema_version": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "role": "orchestrator_operation_context_for_edgeenv", + "source_repository": "InferEdgeOrchestrator", + "artifact_role": "orchestrator-supplemental-operation-context", + "producer_contract": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "source": "orchestration_summary", + "run_id": "edgeenv-smoke-missing", + "not_a_regression_judgement": true, + "not_a_comparability_gate": true, + "decision_owner": "lab", + "regression_owner": "edgeenv", + "candidate_context": { + "run_id": "edgeenv-smoke-missing", + "telemetry_source": "inferedge_orchestrator_operation_summary", + "queue_depth": 4, + "operation": { + "queue_depth": 4, + "deadline_missed_count": 1, + "fallback_count": 0 + }, + "resource": { + "source": "runtime_health_snapshot", + "gpu_temperature": 72.0, + "throttling_detected": false + } + }, + "edgeenv_mapping_hint": { + "copy_candidate_context_to": "runtime_telemetry_context.candidate", + "operation_context_role": "supplemental", + "coverage_summary_owner": "edgeenv", + "coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", + "candidate_context_required_fields": [ + "run_id", + "telemetry_source", + "operation", + "resource" + ], + "aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability" + ] + } + } + ], + "history_missing_orchestrator_source_repository": "InferEdgeOrchestrator", + "history_missing_orchestrator_artifact_role": "orchestrator-supplemental-operation-context", + "history_missing_orchestrator_producer_contract": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "history_missing_orchestrator_candidate_context_telemetry_source": "inferedge_orchestrator_operation_summary", + "history_missing_orchestrator_edgeenv_mapping_hint": { + "copy_candidate_context_to": "runtime_telemetry_context.candidate", + "operation_context_role": "supplemental", + "coverage_summary_owner": "edgeenv", + "coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", + "candidate_context_required_fields": [ + "run_id", + "telemetry_source", + "operation", + "resource" + ], + "aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability" + ] + }, + "history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability" + ] } } }, @@ -175,8 +249,8 @@ "runtime_telemetry_context_present": true, "runtime_telemetry_source": "result_artifacts+runtime_telemetry_history", "runtime_telemetry_history_schema_version": "edgeenv.runtime-telemetry-history.v1", - "history_orchestrator_feed_runs": 1.0, - "history_registered_runs": 2.0, + "history_orchestrator_feed_runs": 2.0, + "history_registered_runs": 3.0, "history_telemetry_runs": 2.0, "history_telemetry_seed_runs": 2.0, "baseline_runtime_telemetry_history_seed_schema_version": "inferedge-runtime-telemetry-history-seed-v1", @@ -188,7 +262,7 @@ "candidate_runtime_telemetry_history_seed_production_monitoring": false, "candidate_runtime_telemetry_history_seed_missing_telemetry_is_failure": false, "candidate_runtime_telemetry_history_seed_point_count": 1.0, - "history_missing_telemetry_runs": 0.0, + "history_missing_telemetry_runs": 1.0, "telemetry_coverage_source": "history_telemetry_coverage", "history_telemetry_coverage_missing_field_run_count": 1.0, "history_telemetry_coverage_missing_field_runs": [ @@ -261,7 +335,81 @@ "baseline_queue_depth": null, "candidate_queue_depth": 7.0, "evidence_gap_count": 1.0, - "evidence_gaps": [] + "evidence_gaps": [], + "history_missing_orchestrator_context_count": 1.0, + "history_missing_orchestrator_context_run_ids": [ + "edgeenv-smoke-missing" + ], + "history_missing_orchestrator_contexts": [ + { + "schema_version": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "role": "orchestrator_operation_context_for_edgeenv", + "source_repository": "InferEdgeOrchestrator", + "artifact_role": "orchestrator-supplemental-operation-context", + "producer_contract": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "source": "orchestration_summary", + "run_id": "edgeenv-smoke-missing", + "not_a_regression_judgement": true, + "not_a_comparability_gate": true, + "decision_owner": "lab", + "regression_owner": "edgeenv", + "candidate_context": { + "run_id": "edgeenv-smoke-missing", + "telemetry_source": "inferedge_orchestrator_operation_summary", + "queue_depth": 4, + "operation": { + "queue_depth": 4, + "deadline_missed_count": 1, + "fallback_count": 0 + }, + "resource": { + "source": "runtime_health_snapshot", + "gpu_temperature": 72.0, + "throttling_detected": false + } + }, + "edgeenv_mapping_hint": { + "copy_candidate_context_to": "runtime_telemetry_context.candidate", + "operation_context_role": "supplemental", + "coverage_summary_owner": "edgeenv", + "coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", + "candidate_context_required_fields": [ + "run_id", + "telemetry_source", + "operation", + "resource" + ], + "aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability" + ] + } + } + ], + "history_missing_orchestrator_source_repository": "InferEdgeOrchestrator", + "history_missing_orchestrator_artifact_role": "orchestrator-supplemental-operation-context", + "history_missing_orchestrator_producer_contract": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "history_missing_orchestrator_candidate_context_telemetry_source": "inferedge_orchestrator_operation_summary", + "history_missing_orchestrator_edgeenv_mapping_hint": { + "copy_candidate_context_to": "runtime_telemetry_context.candidate", + "operation_context_role": "supplemental", + "coverage_summary_owner": "edgeenv", + "coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", + "candidate_context_required_fields": [ + "run_id", + "telemetry_source", + "operation", + "resource" + ], + "aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability" + ] + }, + "history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability" + ] } } }, @@ -303,8 +451,8 @@ "runtime_telemetry_context_present": true, "runtime_telemetry_source": "result_artifacts+runtime_telemetry_history", "runtime_telemetry_history_schema_version": "edgeenv.runtime-telemetry-history.v1", - "history_orchestrator_feed_runs": 1.0, - "history_registered_runs": 2.0, + "history_orchestrator_feed_runs": 2.0, + "history_registered_runs": 3.0, "history_telemetry_runs": 2.0, "history_telemetry_seed_runs": 2.0, "baseline_runtime_telemetry_history_seed_schema_version": "inferedge-runtime-telemetry-history-seed-v1", @@ -316,7 +464,7 @@ "candidate_runtime_telemetry_history_seed_production_monitoring": false, "candidate_runtime_telemetry_history_seed_missing_telemetry_is_failure": false, "candidate_runtime_telemetry_history_seed_point_count": 1.0, - "history_missing_telemetry_runs": 0.0, + "history_missing_telemetry_runs": 1.0, "telemetry_coverage_source": "history_telemetry_coverage", "history_telemetry_coverage_missing_field_run_count": 1.0, "history_telemetry_coverage_missing_field_runs": [ @@ -389,7 +537,81 @@ "baseline_queue_depth": null, "candidate_queue_depth": 7.0, "evidence_gap_count": 1.0, - "evidence_gaps": [] + "evidence_gaps": [], + "history_missing_orchestrator_context_count": 1.0, + "history_missing_orchestrator_context_run_ids": [ + "edgeenv-smoke-missing" + ], + "history_missing_orchestrator_contexts": [ + { + "schema_version": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "role": "orchestrator_operation_context_for_edgeenv", + "source_repository": "InferEdgeOrchestrator", + "artifact_role": "orchestrator-supplemental-operation-context", + "producer_contract": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "source": "orchestration_summary", + "run_id": "edgeenv-smoke-missing", + "not_a_regression_judgement": true, + "not_a_comparability_gate": true, + "decision_owner": "lab", + "regression_owner": "edgeenv", + "candidate_context": { + "run_id": "edgeenv-smoke-missing", + "telemetry_source": "inferedge_orchestrator_operation_summary", + "queue_depth": 4, + "operation": { + "queue_depth": 4, + "deadline_missed_count": 1, + "fallback_count": 0 + }, + "resource": { + "source": "runtime_health_snapshot", + "gpu_temperature": 72.0, + "throttling_detected": false + } + }, + "edgeenv_mapping_hint": { + "copy_candidate_context_to": "runtime_telemetry_context.candidate", + "operation_context_role": "supplemental", + "coverage_summary_owner": "edgeenv", + "coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", + "candidate_context_required_fields": [ + "run_id", + "telemetry_source", + "operation", + "resource" + ], + "aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability" + ] + } + } + ], + "history_missing_orchestrator_source_repository": "InferEdgeOrchestrator", + "history_missing_orchestrator_artifact_role": "orchestrator-supplemental-operation-context", + "history_missing_orchestrator_producer_contract": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "history_missing_orchestrator_candidate_context_telemetry_source": "inferedge_orchestrator_operation_summary", + "history_missing_orchestrator_edgeenv_mapping_hint": { + "copy_candidate_context_to": "runtime_telemetry_context.candidate", + "operation_context_role": "supplemental", + "coverage_summary_owner": "edgeenv", + "coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", + "candidate_context_required_fields": [ + "run_id", + "telemetry_source", + "operation", + "resource" + ], + "aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability" + ] + }, + "history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability" + ] } } } @@ -468,8 +690,8 @@ "runtime_telemetry_context_present": true, "runtime_telemetry_source": "result_artifacts+runtime_telemetry_history", "runtime_telemetry_history_schema_version": "edgeenv.runtime-telemetry-history.v1", - "history_orchestrator_feed_runs": 1.0, - "history_registered_runs": 2.0, + "history_orchestrator_feed_runs": 2.0, + "history_registered_runs": 3.0, "history_telemetry_runs": 2.0, "history_telemetry_seed_runs": 2.0, "baseline_runtime_telemetry_history_seed_schema_version": "inferedge-runtime-telemetry-history-seed-v1", @@ -481,7 +703,7 @@ "candidate_runtime_telemetry_history_seed_production_monitoring": false, "candidate_runtime_telemetry_history_seed_missing_telemetry_is_failure": false, "candidate_runtime_telemetry_history_seed_point_count": 1.0, - "history_missing_telemetry_runs": 0.0, + "history_missing_telemetry_runs": 1.0, "telemetry_coverage_source": "history_telemetry_coverage", "history_telemetry_coverage_missing_field_run_count": 1.0, "history_telemetry_coverage_missing_field_runs": [ @@ -522,31 +744,31 @@ "operation_context_role": "supplemental", "coverage_summary_owner": "edgeenv", "coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", - "candidate_context_required_fields": [ + "candidate_context_required_fields": [ + "run_id", + "telemetry_source", + "operation", + "resource" + ], + "aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability" + ] + }, + "orchestrator_mapping_hint_copy_candidate_context_to": "runtime_telemetry_context.candidate", + "orchestrator_mapping_hint_operation_context_role": "supplemental", + "orchestrator_mapping_hint_coverage_summary_owner": "edgeenv", + "orchestrator_mapping_hint_coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", + "orchestrator_mapping_hint_candidate_context_required_fields": [ "run_id", "telemetry_source", "operation", "resource" ], - "aiguard_evidence_candidates": [ + "orchestrator_mapping_hint_aiguard_evidence_candidates": [ "runtime_queue_overload", "runtime_thermal_instability" - ] - }, - "orchestrator_mapping_hint_copy_candidate_context_to": "runtime_telemetry_context.candidate", - "orchestrator_mapping_hint_operation_context_role": "supplemental", - "orchestrator_mapping_hint_coverage_summary_owner": "edgeenv", - "orchestrator_mapping_hint_coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", - "orchestrator_mapping_hint_candidate_context_required_fields": [ - "run_id", - "telemetry_source", - "operation", - "resource" - ], - "orchestrator_mapping_hint_aiguard_evidence_candidates": [ - "runtime_queue_overload", - "runtime_thermal_instability" - ], + ], "baseline_max_temperature_c": null, "candidate_max_temperature_c": 78.5, "baseline_throttling_detected": null, @@ -554,7 +776,81 @@ "baseline_queue_depth": null, "candidate_queue_depth": 7.0, "evidence_gap_count": 1.0, - "evidence_gaps": [] + "evidence_gaps": [], + "history_missing_orchestrator_context_count": 1.0, + "history_missing_orchestrator_context_run_ids": [ + "edgeenv-smoke-missing" + ], + "history_missing_orchestrator_contexts": [ + { + "schema_version": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "role": "orchestrator_operation_context_for_edgeenv", + "source_repository": "InferEdgeOrchestrator", + "artifact_role": "orchestrator-supplemental-operation-context", + "producer_contract": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "source": "orchestration_summary", + "run_id": "edgeenv-smoke-missing", + "not_a_regression_judgement": true, + "not_a_comparability_gate": true, + "decision_owner": "lab", + "regression_owner": "edgeenv", + "candidate_context": { + "run_id": "edgeenv-smoke-missing", + "telemetry_source": "inferedge_orchestrator_operation_summary", + "queue_depth": 4, + "operation": { + "queue_depth": 4, + "deadline_missed_count": 1, + "fallback_count": 0 + }, + "resource": { + "source": "runtime_health_snapshot", + "gpu_temperature": 72.0, + "throttling_detected": false + } + }, + "edgeenv_mapping_hint": { + "copy_candidate_context_to": "runtime_telemetry_context.candidate", + "operation_context_role": "supplemental", + "coverage_summary_owner": "edgeenv", + "coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", + "candidate_context_required_fields": [ + "run_id", + "telemetry_source", + "operation", + "resource" + ], + "aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability" + ] + } + } + ], + "history_missing_orchestrator_source_repository": "InferEdgeOrchestrator", + "history_missing_orchestrator_artifact_role": "orchestrator-supplemental-operation-context", + "history_missing_orchestrator_producer_contract": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "history_missing_orchestrator_candidate_context_telemetry_source": "inferedge_orchestrator_operation_summary", + "history_missing_orchestrator_edgeenv_mapping_hint": { + "copy_candidate_context_to": "runtime_telemetry_context.candidate", + "operation_context_role": "supplemental", + "coverage_summary_owner": "edgeenv", + "coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", + "candidate_context_required_fields": [ + "run_id", + "telemetry_source", + "operation", + "resource" + ], + "aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability" + ] + }, + "history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability" + ] }, "runtime_telemetry_context": { "role": "supplemental_runtime_telemetry_context", diff --git a/examples/runtime_intelligence_chain/edgeenv_regression_with_orchestrator_context.json b/examples/runtime_intelligence_chain/edgeenv_regression_with_orchestrator_context.json index 0e35360..01e5d50 100644 --- a/examples/runtime_intelligence_chain/edgeenv_regression_with_orchestrator_context.json +++ b/examples/runtime_intelligence_chain/edgeenv_regression_with_orchestrator_context.json @@ -194,10 +194,10 @@ "history": { "schema_version": "edgeenv.runtime-telemetry-history.v1", "summary": { - "registered_runs": 2, + "registered_runs": 3, "telemetry_runs": 2, - "missing_telemetry_runs": 0, - "orchestrator_feed_runs": 1, + "missing_telemetry_runs": 1, + "orchestrator_feed_runs": 2, "history_seed_runs": 2 }, "telemetry_coverage": { @@ -344,6 +344,58 @@ ] } } + ], + "missing_telemetry": [ + { + "run_id": "edgeenv-smoke-missing", + "reason": "runtime_telemetry_missing", + "missing_telemetry_is_failure": false, + "orchestrator_context_present": true, + "orchestrator_operation_context": { + "schema_version": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "role": "orchestrator_operation_context_for_edgeenv", + "source_repository": "InferEdgeOrchestrator", + "artifact_role": "orchestrator-supplemental-operation-context", + "producer_contract": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "source": "orchestration_summary", + "run_id": "edgeenv-smoke-missing", + "not_a_regression_judgement": true, + "not_a_comparability_gate": true, + "decision_owner": "lab", + "regression_owner": "edgeenv", + "candidate_context": { + "run_id": "edgeenv-smoke-missing", + "telemetry_source": "inferedge_orchestrator_operation_summary", + "queue_depth": 4, + "operation": { + "queue_depth": 4, + "deadline_missed_count": 1, + "fallback_count": 0 + }, + "resource": { + "source": "runtime_health_snapshot", + "gpu_temperature": 72.0, + "throttling_detected": false + } + }, + "edgeenv_mapping_hint": { + "copy_candidate_context_to": "runtime_telemetry_context.candidate", + "operation_context_role": "supplemental", + "coverage_summary_owner": "edgeenv", + "coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", + "candidate_context_required_fields": [ + "run_id", + "telemetry_source", + "operation", + "resource" + ], + "aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability" + ] + } + } + } ] }, "notes": [ diff --git a/scripts/check_runtime_intelligence_artifact_bundle.py b/scripts/check_runtime_intelligence_artifact_bundle.py index 2ff7353..5ded9bb 100644 --- a/scripts/check_runtime_intelligence_artifact_bundle.py +++ b/scripts/check_runtime_intelligence_artifact_bundle.py @@ -21,7 +21,7 @@ "aiguard_coverage_gap_recommendation": ( "Inspect telemetry coverage missing fields" ), - "orchestrator_feed": "| Orchestrator operation feed context | 1 |", + "orchestrator_feed": "| Orchestrator operation feed context | 2 |", "runtime_history_seed": "| Runtime telemetry history seed | 2 |", "orchestrator_attached_run": "| Orchestrator context attached runs | candidate |", "aiguard_evidence": "| AIGuard deterministic evidence | warning / suspicious |", @@ -30,7 +30,7 @@ "runtime_queue_overload, runtime_thermal_instability |" ), "aiguard_orchestrator_handoff": ( - "| AIGuard Orchestrator context handoff | feeds=1.0, candidate |" + "| AIGuard Orchestrator context handoff | feeds=2.0, candidate |" ), "aiguard_history_seed_handoff": "| AIGuard history seed handoff | seeds=2.0", "guard_warning_rule": "guard_warning_review", diff --git a/scripts/check_runtime_intelligence_bundle_manifest.py b/scripts/check_runtime_intelligence_bundle_manifest.py index a95a0f2..e37bf24 100644 --- a/scripts/check_runtime_intelligence_bundle_manifest.py +++ b/scripts/check_runtime_intelligence_bundle_manifest.py @@ -121,6 +121,7 @@ "aiguard_raw_context: telemetry_coverage_source=history_telemetry_coverage", "aiguard_raw_context: orchestrator_mapping_hint preserved", "aiguard_raw_context: orchestrator_producer_markers preserved", + "aiguard_raw_context: missing_telemetry_orchestrator_context preserved", ) EDGEENV_HANDOFF_SUMMARY_CONTRACT_MARKERS = ( "edgeenv_handoff: lab_bundle_alignment validated", @@ -470,7 +471,11 @@ def _validate_edgeenv_runtime_history_artifact( if isinstance(coverage, dict): _validate_edgeenv_history_coverage_summary(coverage, errors) _validate_edgeenv_history_seed_runs(history, errors) - _validate_edgeenv_missing_telemetry_orchestrator_context(history, errors) + _validate_edgeenv_missing_telemetry_orchestrator_context( + history, + errors, + "EdgeEnv handoff runtime_telemetry_history", + ) def _validate_edgeenv_report(edgeenv_report: dict[str, Any], errors: list[str]) -> None: @@ -503,9 +508,14 @@ def _validate_edgeenv_report(edgeenv_report: dict[str, Any], errors: list[str]) ) summary = history.get("summary") or {} _record( - summary.get("orchestrator_feed_runs") == 1, + summary.get("orchestrator_feed_runs") == 2, + errors, + "runtime_telemetry_context.history.summary.orchestrator_feed_runs must be 2", + ) + _record( + summary.get("missing_telemetry_runs") == 1, errors, - "runtime_telemetry_context.history.summary.orchestrator_feed_runs must be 1", + "runtime_telemetry_context.history.summary.missing_telemetry_runs must be 1", ) _record( summary.get("history_seed_runs") == 2, @@ -521,6 +531,11 @@ def _validate_edgeenv_report(edgeenv_report: dict[str, Any], errors: list[str]) if isinstance(history_coverage, dict): _validate_edgeenv_history_coverage_summary(history_coverage, errors) _validate_edgeenv_history_seed_runs(history, errors) + _validate_edgeenv_missing_telemetry_orchestrator_context( + history, + errors, + "runtime_telemetry_context.history", + ) candidate = context.get("candidate") or {} _record( @@ -779,12 +794,13 @@ def _validate_edgeenv_history_seed_runs( def _validate_edgeenv_missing_telemetry_orchestrator_context( history: dict[str, Any], errors: list[str], + label: str, ) -> None: missing_telemetry = history.get("missing_telemetry") _record( isinstance(missing_telemetry, list), errors, - "EdgeEnv handoff runtime_telemetry_history.missing_telemetry must be a list", + f"{label}.missing_telemetry must be a list", ) if not isinstance(missing_telemetry, list): return @@ -801,8 +817,7 @@ def _validate_edgeenv_missing_telemetry_orchestrator_context( _record( isinstance(missing_run, dict), errors, - "EdgeEnv handoff runtime_telemetry_history.missing_telemetry must " - "include edgeenv-smoke-missing", + f"{label}.missing_telemetry must include edgeenv-smoke-missing", ) if not isinstance(missing_run, dict): return @@ -810,22 +825,22 @@ def _validate_edgeenv_missing_telemetry_orchestrator_context( _record( missing_run.get("reason") == "runtime_telemetry_missing", errors, - "EdgeEnv handoff runtime_telemetry_history.missing_telemetry" - "[edgeenv-smoke-missing].reason must be runtime_telemetry_missing", + f"{label}.missing_telemetry[edgeenv-smoke-missing].reason " + "must be runtime_telemetry_missing", ) operation_context = missing_run.get("orchestrator_operation_context") _record( isinstance(operation_context, dict), errors, - "EdgeEnv handoff runtime_telemetry_history.missing_telemetry" - "[edgeenv-smoke-missing] must include orchestrator_operation_context", + f"{label}.missing_telemetry[edgeenv-smoke-missing] must include " + "orchestrator_operation_context", ) if not isinstance(operation_context, dict): return prefix = ( - "EdgeEnv handoff runtime_telemetry_history.missing_telemetry" - "[edgeenv-smoke-missing].orchestrator_operation_context" + f"{label}.missing_telemetry[edgeenv-smoke-missing]." + "orchestrator_operation_context" ) _record( operation_context.get("schema_version") @@ -1077,6 +1092,7 @@ def _validate_coverage_gap_evidence( "must be ['queue_depth']", ) _validate_aiguard_orchestrator_mapping_hint(edgeenv, errors) + _validate_aiguard_missing_orchestrator_context(edgeenv, errors) def _validate_aiguard_history_seed_context( @@ -1286,6 +1302,103 @@ def _validate_aiguard_orchestrator_mapping_hint( ) +def _validate_aiguard_missing_orchestrator_context( + edgeenv_context: dict[str, Any], + errors: list[str], +) -> None: + _record( + edgeenv_context.get("history_missing_telemetry_runs") == 1.0, + errors, + "AIGuard coverage evidence history_missing_telemetry_runs must be 1.0", + ) + _record( + edgeenv_context.get("history_missing_orchestrator_context_count") == 1.0, + errors, + "AIGuard coverage evidence " + "history_missing_orchestrator_context_count must be 1.0", + ) + run_ids = edgeenv_context.get("history_missing_orchestrator_context_run_ids") + _record( + isinstance(run_ids, list) and "edgeenv-smoke-missing" in run_ids, + errors, + "AIGuard coverage evidence " + "history_missing_orchestrator_context_run_ids must include " + "edgeenv-smoke-missing", + ) + _record( + edgeenv_context.get("history_missing_orchestrator_source_repository") + == REQUIRED_SOURCE_REPOSITORIES["orchestrator_operation_context"], + errors, + "AIGuard coverage evidence " + "history_missing_orchestrator_source_repository must be " + f"{REQUIRED_SOURCE_REPOSITORIES['orchestrator_operation_context']}", + ) + _record( + edgeenv_context.get("history_missing_orchestrator_artifact_role") + == REQUIRED_ARTIFACT_ROLES["orchestrator_operation_context"], + errors, + "AIGuard coverage evidence history_missing_orchestrator_artifact_role " + f"must be {REQUIRED_ARTIFACT_ROLES['orchestrator_operation_context']}", + ) + _record( + edgeenv_context.get("history_missing_orchestrator_producer_contract") + == REQUIRED_PRODUCER_CONTRACTS["orchestrator_feed_schema"], + errors, + "AIGuard coverage evidence history_missing_orchestrator_producer_contract " + f"must be {REQUIRED_PRODUCER_CONTRACTS['orchestrator_feed_schema']}", + ) + _record( + edgeenv_context.get( + "history_missing_orchestrator_candidate_context_telemetry_source" + ) + == "inferedge_orchestrator_operation_summary", + errors, + "AIGuard coverage evidence " + "history_missing_orchestrator_candidate_context_telemetry_source must be " + "inferedge_orchestrator_operation_summary", + ) + mapping_hint = edgeenv_context.get( + "history_missing_orchestrator_edgeenv_mapping_hint" + ) + _record( + isinstance(mapping_hint, dict), + errors, + "AIGuard coverage evidence " + "history_missing_orchestrator_edgeenv_mapping_hint must be an object", + ) + if isinstance(mapping_hint, dict): + for key, expected in REQUIRED_ORCHESTRATOR_MAPPING_HINT.items(): + _record( + mapping_hint.get(key) == expected, + errors, + "AIGuard coverage evidence " + f"history_missing_orchestrator_edgeenv_mapping_hint.{key} " + f"must be {expected}", + ) + evidence_candidates = edgeenv_context.get( + "history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates" + ) + _record( + isinstance(evidence_candidates, list), + errors, + "AIGuard coverage evidence " + "history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates " + "must be a list", + ) + if isinstance(evidence_candidates, list): + missing_candidates = sorted( + REQUIRED_ORCHESTRATOR_AIGUARD_EVIDENCE_CANDIDATES + - set(evidence_candidates) + ) + _record( + not missing_candidates, + errors, + "AIGuard coverage evidence " + "history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates " + f"is missing {missing_candidates}", + ) + + def _write_summary( path: str, *, diff --git a/scripts/check_runtime_intelligence_ci_artifacts.py b/scripts/check_runtime_intelligence_ci_artifacts.py index 0a0e918..521abbd 100644 --- a/scripts/check_runtime_intelligence_ci_artifacts.py +++ b/scripts/check_runtime_intelligence_ci_artifacts.py @@ -37,6 +37,7 @@ "aiguard_raw_context: telemetry_coverage_source=history_telemetry_coverage", "aiguard_raw_context: orchestrator_mapping_hint preserved", "aiguard_raw_context: orchestrator_producer_markers preserved", + "aiguard_raw_context: missing_telemetry_orchestrator_context preserved", "edgeenv_handoff: lab_bundle_alignment validated", "edgeenv_handoff: runtime_telemetry_history validated", "edgeenv_handoff: missing_telemetry_orchestrator_context validated", diff --git a/tests/test_runtime_intelligence_bundle_manifest.py b/tests/test_runtime_intelligence_bundle_manifest.py index 05f5423..ca504fe 100644 --- a/tests/test_runtime_intelligence_bundle_manifest.py +++ b/tests/test_runtime_intelligence_bundle_manifest.py @@ -72,6 +72,10 @@ def test_runtime_intelligence_bundle_manifest_gate_cli_passes(tmp_path): "aiguard_raw_context: telemetry_coverage_source=history_telemetry_coverage" in summary ) + assert ( + "aiguard_raw_context: missing_telemetry_orchestrator_context preserved" + in summary + ) def test_runtime_intelligence_bundle_manifest_gate_validates_edgeenv_handoff( @@ -633,6 +637,58 @@ def test_runtime_intelligence_bundle_manifest_gate_fails_for_bad_guard_producer_ ) in summary +def test_runtime_intelligence_bundle_manifest_gate_fails_for_bad_guard_missing_context( + tmp_path, +): + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + guard_path = ( + REPO_ROOT + / "examples" + / "runtime_intelligence_chain" + / manifest["files"]["aiguard_guard_analysis"] + ) + guard_analysis = json.loads(guard_path.read_text(encoding="utf-8")) + coverage_evidence = next( + item + for item in guard_analysis["evidence"] + if item.get("type") == "runtime_telemetry_context_coverage" + ) + edgeenv_context = coverage_evidence["raw_context"]["edgeenv_regression"] + edgeenv_context["history_missing_orchestrator_artifact_role"] = ( + "edgeenv-regression-context" + ) + edgeenv_context["history_missing_orchestrator_edgeenv_mapping_hint"][ + "coverage_summary_owner" + ] = "aiguard" + edgeenv_context[ + "history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates" + ] = ["runtime_queue_overload"] + + guard_copy = tmp_path / "aiguard_guard_analysis.json" + guard_copy.write_text(json.dumps(guard_analysis), encoding="utf-8") + manifest["files"]["aiguard_guard_analysis"] = str(guard_copy) + manifest_path = tmp_path / "bundle_manifest.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + summary_path = tmp_path / "bundle_manifest_gate_summary.md" + + result = manifest_gate(manifest=str(manifest_path), summary_out=str(summary_path)) + + assert result == 2 + summary = summary_path.read_text(encoding="utf-8") + assert ( + "history_missing_orchestrator_artifact_role must be " + "orchestrator-supplemental-operation-context" + ) in summary + assert ( + "history_missing_orchestrator_edgeenv_mapping_hint." + "coverage_summary_owner must be edgeenv" + ) in summary + assert ( + "history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates " + "is missing ['runtime_thermal_instability']" + ) in summary + + def test_runtime_intelligence_bundle_manifest_gate_fails_for_bad_guard_history_seed( tmp_path, ): diff --git a/tests/test_runtime_intelligence_ci_template.py b/tests/test_runtime_intelligence_ci_template.py index bb3a329..4f2465f 100644 --- a/tests/test_runtime_intelligence_ci_template.py +++ b/tests/test_runtime_intelligence_ci_template.py @@ -115,6 +115,7 @@ def test_runtime_intelligence_ci_artifact_gate_passes_for_expected_outputs(tmp_p "- aiguard_raw_context: telemetry_coverage_source=history_telemetry_coverage", "- aiguard_raw_context: orchestrator_mapping_hint preserved", "- aiguard_raw_context: orchestrator_producer_markers preserved", + "- aiguard_raw_context: missing_telemetry_orchestrator_context preserved", "- edgeenv_handoff: lab_bundle_alignment validated", "- edgeenv_handoff: runtime_telemetry_history validated", "- edgeenv_handoff: missing_telemetry_orchestrator_context validated", @@ -354,6 +355,7 @@ def test_runtime_intelligence_ci_artifact_gate_fails_for_failed_deployment_risk( "- aiguard_raw_context: telemetry_coverage_source=history_telemetry_coverage", "- aiguard_raw_context: orchestrator_mapping_hint preserved", "- aiguard_raw_context: orchestrator_producer_markers preserved", + "- aiguard_raw_context: missing_telemetry_orchestrator_context preserved", "- edgeenv_handoff: lab_bundle_alignment validated", "- edgeenv_handoff: runtime_telemetry_history validated", "- edgeenv_handoff: missing_telemetry_orchestrator_context validated", diff --git a/tests/test_runtime_intelligence_evidence_chain_smoke.py b/tests/test_runtime_intelligence_evidence_chain_smoke.py index 4213ef2..3ad10c0 100644 --- a/tests/test_runtime_intelligence_evidence_chain_smoke.py +++ b/tests/test_runtime_intelligence_evidence_chain_smoke.py @@ -149,6 +149,23 @@ def test_runtime_intelligence_chain_smoke_ingests_precomputed_guard_artifact(): ] is False ) + assert guard_edgeenv_context["history_missing_telemetry_runs"] == 1.0 + assert guard_edgeenv_context["history_missing_orchestrator_context_run_ids"] == [ + "edgeenv-smoke-missing" + ] + assert ( + guard_edgeenv_context["history_missing_orchestrator_source_repository"] + == "InferEdgeOrchestrator" + ) + assert ( + guard_edgeenv_context["history_missing_orchestrator_artifact_role"] + == "orchestrator-supplemental-operation-context" + ) + assert set( + guard_edgeenv_context[ + "history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates" + ] + ) >= {"runtime_queue_overload", "runtime_thermal_instability"} assert bundle["deployment_decision"]["decision"] == "review_required" assert bundle["deployment_decision"]["guard_status"] == "warning" assert "guard_warning_review" in bundle["deployment_decision"]["triggered_rules"] @@ -159,11 +176,11 @@ def test_runtime_intelligence_chain_smoke_ingests_precomputed_guard_artifact(): "| Runtime telemetry coverage gaps | baseline=none; candidate=queue_depth |" in bundle["markdown"] ) - assert "| Orchestrator operation feed context | 1 |" in bundle["markdown"] + assert "| Orchestrator operation feed context | 2 |" in bundle["markdown"] assert "| Runtime telemetry history seed | 2 |" in bundle["markdown"] assert "| Orchestrator context attached runs | candidate |" in bundle["markdown"] assert "runtime_queue_overload, runtime_thermal_instability" in bundle["markdown"] - assert "| AIGuard Orchestrator context handoff | feeds=1.0, candidate |" in bundle[ + assert "| AIGuard Orchestrator context handoff | feeds=2.0, candidate |" in bundle[ "markdown" ] assert "| AIGuard history seed handoff | seeds=2.0" in bundle["markdown"] diff --git a/tests/test_runtime_intelligence_smoke_script.py b/tests/test_runtime_intelligence_smoke_script.py index 785492d..3b04c9e 100644 --- a/tests/test_runtime_intelligence_smoke_script.py +++ b/tests/test_runtime_intelligence_smoke_script.py @@ -74,6 +74,10 @@ def test_runtime_intelligence_smoke_script_runs_artifact_chain(tmp_path): "aiguard_raw_context: orchestrator_producer_markers preserved" in bundle_summary ) + assert ( + "aiguard_raw_context: missing_telemetry_orchestrator_context preserved" + in bundle_summary + ) assert "edgeenv_handoff: lab_bundle_alignment validated" in bundle_summary assert "edgeenv_handoff: runtime_telemetry_history validated" in bundle_summary assert (