diff --git a/docs/ci/runtime_intelligence_gitlab_artifacts.md b/docs/ci/runtime_intelligence_gitlab_artifacts.md index 42871ea..a463bfa 100644 --- a/docs/ci/runtime_intelligence_gitlab_artifacts.md +++ b/docs/ci/runtime_intelligence_gitlab_artifacts.md @@ -57,6 +57,7 @@ Expected artifacts are intentionally file-based and local-first: - `BENCHMARKS.md` - Runtime Intelligence bundle manifest under `examples/runtime_intelligence_chain/bundle_manifest.json` - EdgeEnv producer-side handoff manifest under `examples/runtime_intelligence_chain/edgeenv_lab_handoff_manifest.json` +- EdgeEnv telemetry history artifact under `examples/runtime_intelligence_chain/runtime_telemetry_history.json` - Runtime Intelligence bundle manifest gate summary - EdgeEnv regression Markdown / HTML report under `reports/runtime_intelligence_ci/` - deterministic Runtime Intelligence summary Markdown / HTML with precomputed AIGuard runtime operation evidence @@ -109,6 +110,9 @@ The initial gate is conservative: - the optional EdgeEnv handoff input must keep `lab_bundle_alignment` metadata aligned with the Lab bundle manifest, while leaving `aiguard_guard_analysis` as an external AIGuard artifact +- the EdgeEnv handoff `runtime_telemetry_history` file must exist and preserve + the EdgeEnv history schema, telemetry coverage summary, and Runtime history + seed ownership markers The bundle manifest gate is implemented by `scripts/check_runtime_intelligence_bundle_manifest.py`. It verifies that the bundle contains baseline/candidate Runtime results, EdgeEnv regression evidence, AIGuard guard evidence, and explicit owner/boundary metadata before Lab generates the report. In this template it also consumes `--edgeenv-handoff examples/runtime_intelligence_chain/edgeenv_lab_handoff_manifest.json` to verify EdgeEnv producer-side file/source/role/schema alignment. The same gate now also checks `source_repositories`, `artifact_roles`, and `producer_contracts` so the smoke remains a cross-repo handoff fixture rather than a Lab-only report sample. diff --git a/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md b/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md index 0350557..6431a8e 100644 --- a/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md +++ b/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md @@ -105,6 +105,7 @@ This second smoke uses committed lightweight artifacts to represent the cross-re - `examples/runtime_intelligence_chain/bundle_manifest.json` declares the local-first artifact bundle, file paths, source repositories, artifact roles, producer contracts, owners, and boundary flags. - `examples/runtime_intelligence_chain/edgeenv_lab_handoff_manifest.json` mirrors the EdgeEnv producer-side handoff manifest and its `lab_bundle_alignment` metadata, so Lab can verify EdgeEnv-produced file keys separately from external AIGuard evidence. +- `examples/runtime_intelligence_chain/runtime_telemetry_history.json` is the EdgeEnv producer-side telemetry history artifact referenced by the handoff manifest. - Orchestrator context is preserved inside the EdgeEnv regression artifact as `orchestrator_operation_context`. - AIGuard deterministic queue/thermal evidence is passed as a precomputed `guard_analysis` artifact that mirrors the AIGuard producer-side diagnosis v1 evidence shape. - Lab owns the combined report and deployment decision. @@ -126,6 +127,7 @@ Expected Lab behavior: - The same gate requires Orchestrator's AIGuard evidence candidate hint to preserve `runtime_queue_overload` and `runtime_thermal_instability`, keeping runtime operation anomaly evidence deterministic and supplemental. - The same gate requires Orchestrator candidate context to carry `run_id`, `telemetry_source`, `operation`, and `resource`, so Lab can verify handoff completeness without treating Orchestrator context as a regression judgement. - When an EdgeEnv handoff manifest is provided, the bundle gate requires EdgeEnv-produced file keys, external AIGuard file keys, source repository mapping, artifact roles, producer contracts, and boundary flags to match Lab's Runtime Intelligence bundle contract. +- The same handoff gate verifies that the referenced `runtime_telemetry_history` artifact exists and preserves EdgeEnv history schema, telemetry coverage, and Runtime history seed ownership markers. - The bundle gate also requires AIGuard coverage evidence raw context to preserve the same Orchestrator mapping hint, proving that AIGuard kept EdgeEnv/Orchestrator ownership markers as diagnosis context rather than recomputing coverage or owning deployment policy. - Additional Lab test fixtures under `tests/fixtures/edgeenv_regression/` mirror EdgeEnv replay examples for candidate telemetry gaps and execution sequence inversion. These fixture smokes verify that replay warnings become Lab-owned report context without making Lab recompute EdgeEnv comparability. - Markdown/HTML reports include a `Runtime Intelligence Risk Summary` that summarizes EdgeEnv comparability/regression, telemetry replay gaps, Runtime history seed traceability, AIGuard deterministic evidence, and the Lab-owned deployment decision in one reviewer-facing table. diff --git a/examples/runtime_intelligence_chain/runtime_telemetry_history.json b/examples/runtime_intelligence_chain/runtime_telemetry_history.json new file mode 100644 index 0000000..f8fc155 --- /dev/null +++ b/examples/runtime_intelligence_chain/runtime_telemetry_history.json @@ -0,0 +1,155 @@ +{ + "schema_version": "edgeenv.runtime-telemetry-history.v1", + "summary": { + "registered_runs": 2, + "telemetry_runs": 2, + "missing_telemetry_runs": 0, + "orchestrator_feed_runs": 1, + "history_seed_runs": 2 + }, + "telemetry_coverage": { + "runs_with_coverage": 2, + "runs_without_coverage": 0, + "expected_fields": [ + "gpu_temperature", + "queue_depth", + "telemetry_timestamp" + ], + "observed_fields": [ + "gpu_temperature", + "queue_depth", + "telemetry_timestamp" + ], + "missing_fields": [ + "queue_depth" + ], + "coverage_ratio_min": 0.666667, + "coverage_ratio_max": 1.0, + "missing_telemetry_is_failure_values": [ + false + ], + "any_missing_telemetry_is_failure": false, + "missing_field_run_count": 1, + "missing_field_runs": [ + { + "run_id": "edgeenv-smoke-candidate", + "missing_fields": [ + "queue_depth" + ], + "missing_field_count": 1, + "missing_telemetry_is_failure": false + } + ], + "run_summaries": [ + { + "run_id": "edgeenv-smoke-baseline", + "coverage_present": true, + "expected_fields": [ + "gpu_temperature", + "queue_depth", + "telemetry_timestamp" + ], + "observed_fields": [ + "gpu_temperature", + "queue_depth", + "telemetry_timestamp" + ], + "missing_fields": [], + "expected_field_count": 3, + "observed_field_count": 3, + "missing_field_count": 0, + "coverage_ratio": 1.0, + "missing_telemetry_is_failure": false + }, + { + "run_id": "edgeenv-smoke-candidate", + "coverage_present": true, + "expected_fields": [ + "gpu_temperature", + "queue_depth", + "telemetry_timestamp" + ], + "observed_fields": [ + "gpu_temperature", + "telemetry_timestamp" + ], + "missing_fields": [ + "queue_depth" + ], + "expected_field_count": 3, + "observed_field_count": 2, + "missing_field_count": 1, + "coverage_ratio": 0.666667, + "missing_telemetry_is_failure": false + } + ] + }, + "runs": [ + { + "run_id": "edgeenv-smoke-baseline", + "runtime_telemetry_history_seed": { + "schema_version": "inferedge-runtime-telemetry-history-seed-v1", + "evidence_role": "runtime_telemetry_history_seed", + "registry_owner": "edgeenv", + "decision_owner": "lab", + "source_result_schema_version": "inferedge-runtime-result-v1", + "source_telemetry_schema_version": "inferedge-runtime-telemetry-v1", + "replay_scope": "single_result_to_history", + "replay_ready": true, + "production_monitoring": false, + "missing_telemetry_is_failure": false, + "source_result": { + "run_id": "edgeenv-smoke-baseline", + "compare_key": "yolov8n__b1__h640w640__fp32", + "backend_key": "onnxruntime__cpu", + "engine_backend": "onnxruntime", + "device": "cpu", + "precision": "fp32", + "power_mode": "unknown" + }, + "points": [ + { + "execution_sequence_id": 1, + "telemetry_timestamp": "2026-05-21T00:00:01Z", + "mean_ms": 100.0, + "p99_ms": 130.0, + "timeout_observed": false + } + ] + } + }, + { + "run_id": "edgeenv-smoke-candidate", + "runtime_telemetry_history_seed": { + "schema_version": "inferedge-runtime-telemetry-history-seed-v1", + "evidence_role": "runtime_telemetry_history_seed", + "registry_owner": "edgeenv", + "decision_owner": "lab", + "source_result_schema_version": "inferedge-runtime-result-v1", + "source_telemetry_schema_version": "inferedge-runtime-telemetry-v1", + "replay_scope": "single_result_to_history", + "replay_ready": true, + "production_monitoring": false, + "missing_telemetry_is_failure": false, + "source_result": { + "run_id": "edgeenv-smoke-candidate", + "compare_key": "yolov8n__b1__h640w640__fp32", + "backend_key": "onnxruntime__cpu", + "engine_backend": "onnxruntime", + "device": "cpu", + "precision": "fp32", + "power_mode": "unknown" + }, + "points": [ + { + "execution_sequence_id": 2, + "telemetry_timestamp": "2026-05-21T00:05:01Z", + "mean_ms": 118.0, + "p99_ms": 171.6, + "timeout_observed": false + } + ] + } + } + ] +} diff --git a/scripts/check_runtime_intelligence_bundle_manifest.py b/scripts/check_runtime_intelligence_bundle_manifest.py index 4a06964..5b22d94 100644 --- a/scripts/check_runtime_intelligence_bundle_manifest.py +++ b/scripts/check_runtime_intelligence_bundle_manifest.py @@ -11,6 +11,7 @@ EXPECTED_SCHEMA_VERSION = "inferedge.runtime-intelligence-artifact-bundle.v1" EDGEENV_HANDOFF_SCHEMA_VERSION = "edgeenv.runtime-intelligence-lab-handoff.v1" EDGEENV_HANDOFF_ROLE = "edgeenv-runtime-intelligence-lab-handoff" +EDGEENV_HANDOFF_RUNTIME_HISTORY_KEY = "runtime_telemetry_history" REQUIRED_FILES = { "baseline_result", "candidate_result", @@ -118,6 +119,7 @@ ) EDGEENV_HANDOFF_SUMMARY_CONTRACT_MARKERS = ( "edgeenv_handoff: lab_bundle_alignment validated", + "edgeenv_handoff: runtime_telemetry_history validated", ) @@ -220,6 +222,7 @@ def _validate_manifest_shape(manifest: dict[str, Any], errors: list[str]) -> Non def _validate_edgeenv_handoff_alignment( handoff: dict[str, Any], *, + handoff_path: Path, manifest: dict[str, Any], errors: list[str], ) -> None: @@ -256,6 +259,11 @@ def _validate_edgeenv_handoff_alignment( errors, "EdgeEnv handoff files must not include aiguard_guard_analysis", ) + _validate_edgeenv_handoff_runtime_history_artifact( + handoff_files, + handoff_path=handoff_path, + errors=errors, + ) alignment = handoff.get("lab_bundle_alignment") _record( @@ -359,6 +367,99 @@ def _validate_edgeenv_handoff_alignment( ) +def _validate_edgeenv_handoff_runtime_history_artifact( + handoff_files: dict[str, Any], + *, + handoff_path: Path, + errors: list[str], +) -> None: + raw_path = handoff_files.get(EDGEENV_HANDOFF_RUNTIME_HISTORY_KEY) + _record( + isinstance(raw_path, str), + errors, + "EdgeEnv handoff files.runtime_telemetry_history must be a string path", + ) + if not isinstance(raw_path, str): + return + + resolved = _resolve_bundle_path(handoff_path, raw_path) + _record( + resolved.exists(), + errors, + f"EdgeEnv handoff files.runtime_telemetry_history does not exist: {resolved}", + ) + if not resolved.exists(): + return + + try: + history = json.loads(resolved.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + errors.append( + "EdgeEnv handoff files.runtime_telemetry_history is invalid JSON: " + f"{resolved}: {exc}" + ) + return + except OSError as exc: + errors.append( + "EdgeEnv handoff files.runtime_telemetry_history could not be read: " + f"{resolved}: {exc}" + ) + return + + _record( + isinstance(history, dict), + errors, + "EdgeEnv handoff files.runtime_telemetry_history must be a JSON object", + ) + if not isinstance(history, dict): + return + _validate_edgeenv_runtime_history_artifact(history, errors) + + +def _validate_edgeenv_runtime_history_artifact( + history: dict[str, Any], + errors: list[str], +) -> None: + _record( + history.get("schema_version") + == REQUIRED_PRODUCER_CONTRACTS["edgeenv_history_schema"], + errors, + "EdgeEnv handoff runtime_telemetry_history.schema_version must be " + f"{REQUIRED_PRODUCER_CONTRACTS['edgeenv_history_schema']}", + ) + summary = history.get("summary") or {} + _record( + summary.get("registered_runs") == 2, + errors, + "EdgeEnv handoff runtime_telemetry_history.summary.registered_runs must be 2", + ) + _record( + summary.get("telemetry_runs") == 2, + errors, + "EdgeEnv handoff runtime_telemetry_history.summary.telemetry_runs must be 2", + ) + _record( + summary.get("orchestrator_feed_runs") == 1, + errors, + "EdgeEnv handoff runtime_telemetry_history.summary." + "orchestrator_feed_runs must be 1", + ) + _record( + summary.get("history_seed_runs") == 2, + errors, + "EdgeEnv handoff runtime_telemetry_history.summary.history_seed_runs must be 2", + ) + coverage = history.get("telemetry_coverage") + _record( + isinstance(coverage, dict), + errors, + "EdgeEnv handoff runtime_telemetry_history must include telemetry_coverage", + ) + if isinstance(coverage, dict): + _validate_edgeenv_history_coverage_summary(coverage, errors) + _validate_edgeenv_history_seed_runs(history, errors) + + def _validate_edgeenv_report(edgeenv_report: dict[str, Any], errors: list[str]) -> None: _record( edgeenv_report.get("mode") == "same-condition", @@ -1079,8 +1180,10 @@ def main( manifest_payload = _load_json(manifest_path, "Runtime Intelligence bundle manifest") _validate_manifest_shape(manifest_payload, errors) if edgeenv_handoff: + edgeenv_handoff_path = Path(edgeenv_handoff).resolve() _validate_edgeenv_handoff_alignment( - _load_json(Path(edgeenv_handoff).resolve(), "EdgeEnv handoff manifest"), + _load_json(edgeenv_handoff_path, "EdgeEnv handoff manifest"), + handoff_path=edgeenv_handoff_path, manifest=manifest_payload, errors=errors, ) diff --git a/scripts/check_runtime_intelligence_ci_artifacts.py b/scripts/check_runtime_intelligence_ci_artifacts.py index 6a0568d..5c0660a 100644 --- a/scripts/check_runtime_intelligence_ci_artifacts.py +++ b/scripts/check_runtime_intelligence_ci_artifacts.py @@ -33,6 +33,7 @@ "aiguard_raw_context: telemetry_coverage_source=history_telemetry_coverage", "aiguard_raw_context: orchestrator_mapping_hint preserved", "edgeenv_handoff: lab_bundle_alignment validated", + "edgeenv_handoff: runtime_telemetry_history validated", ) diff --git a/tests/test_runtime_intelligence_bundle_manifest.py b/tests/test_runtime_intelligence_bundle_manifest.py index 703a209..eacb9ea 100644 --- a/tests/test_runtime_intelligence_bundle_manifest.py +++ b/tests/test_runtime_intelligence_bundle_manifest.py @@ -82,6 +82,7 @@ def test_runtime_intelligence_bundle_manifest_gate_validates_edgeenv_handoff( ) summary = summary_path.read_text(encoding="utf-8") assert "edgeenv_handoff: lab_bundle_alignment validated" in summary + assert "edgeenv_handoff: runtime_telemetry_history validated" in summary def test_runtime_intelligence_bundle_manifest_gate_fails_for_bad_edgeenv_handoff( @@ -116,6 +117,26 @@ def test_runtime_intelligence_bundle_manifest_gate_fails_for_bad_edgeenv_handoff ) in summary +def test_runtime_intelligence_bundle_manifest_gate_fails_for_missing_handoff_history( + tmp_path, +): + handoff = json.loads(EDGEENV_HANDOFF.read_text(encoding="utf-8")) + handoff["files"]["runtime_telemetry_history"] = "missing_runtime_history.json" + handoff_path = tmp_path / "edgeenv_lab_handoff_manifest.json" + handoff_path.write_text(json.dumps(handoff), encoding="utf-8") + summary_path = tmp_path / "bundle_manifest_gate_summary.md" + + result = manifest_gate( + manifest=str(MANIFEST), + edgeenv_handoff=str(handoff_path), + summary_out=str(summary_path), + ) + + assert result == 2 + summary = summary_path.read_text(encoding="utf-8") + assert "files.runtime_telemetry_history does not exist" in summary + + def test_runtime_intelligence_bundle_manifest_gate_fails_for_bad_owner(tmp_path): manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) manifest["ownership"]["deployment_decision_owner"] = "aiguard" diff --git a/tests/test_runtime_intelligence_ci_template.py b/tests/test_runtime_intelligence_ci_template.py index 1f47cf1..57ff9b3 100644 --- a/tests/test_runtime_intelligence_ci_template.py +++ b/tests/test_runtime_intelligence_ci_template.py @@ -111,6 +111,7 @@ def test_runtime_intelligence_ci_artifact_gate_passes_for_expected_outputs(tmp_p "- aiguard_raw_context: telemetry_coverage_source=history_telemetry_coverage", "- aiguard_raw_context: orchestrator_mapping_hint preserved", "- edgeenv_handoff: lab_bundle_alignment validated", + "- edgeenv_handoff: runtime_telemetry_history validated", ] ), encoding="utf-8", @@ -343,6 +344,7 @@ def test_runtime_intelligence_ci_artifact_gate_fails_for_failed_deployment_risk( "- aiguard_raw_context: telemetry_coverage_source=history_telemetry_coverage", "- aiguard_raw_context: orchestrator_mapping_hint preserved", "- edgeenv_handoff: lab_bundle_alignment validated", + "- edgeenv_handoff: runtime_telemetry_history validated", ] ), encoding="utf-8", diff --git a/tests/test_runtime_intelligence_smoke_script.py b/tests/test_runtime_intelligence_smoke_script.py index 940851c..d0b7fab 100644 --- a/tests/test_runtime_intelligence_smoke_script.py +++ b/tests/test_runtime_intelligence_smoke_script.py @@ -67,6 +67,7 @@ def test_runtime_intelligence_smoke_script_runs_artifact_chain(tmp_path): ).read_text(encoding="utf-8") assert "- Status: passed" in bundle_summary assert "edgeenv_handoff: lab_bundle_alignment validated" in bundle_summary + assert "edgeenv_handoff: runtime_telemetry_history validated" in bundle_summary ci_summary = ( output_dir / "runtime_intelligence_ci_artifact_gate_summary.md"