Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/ci/runtime_intelligence_gitlab_artifacts.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,10 @@ The initial gate is conservative:
- AIGuard coverage evidence raw context must also preserve the same
Orchestrator producer markers, so the diagnostic artifact remains traceable
to the Orchestrator feed without making AIGuard the producer or decision owner
- AIGuard coverage evidence raw context must preserve Orchestrator producer
markers and mapping hints for EdgeEnv history `missing_telemetry` entries
when such context exists, while keeping the entry an evidence gap rather than
successful Runtime telemetry
- Orchestrator candidate context must include `run_id`, `telemetry_source`,
`operation`, and `resource`, so CI can catch incomplete handoffs without
making Orchestrator a regression owner
Expand Down
1 change: 1 addition & 0 deletions docs/portfolio/edgeenv_runtime_regression_lab_handoff.md
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ Expected Lab behavior:
- The same handoff gate verifies that the referenced `runtime_telemetry_history` artifact exists and preserves EdgeEnv history schema, telemetry coverage, and Runtime history seed ownership markers.
- The same handoff gate verifies that missing telemetry entries remain evidence gaps while preserving Orchestrator producer markers, owner boundary flags, and EdgeEnv mapping hints when Orchestrator context is attached.
- The bundle gate also requires AIGuard coverage evidence raw context to preserve the same Orchestrator mapping hint and producer markers, proving that AIGuard kept EdgeEnv/Orchestrator ownership markers as diagnosis context rather than recomputing coverage or owning deployment policy.
- The same gate requires AIGuard replay raw context to preserve Orchestrator producer markers and mapping hints for EdgeEnv history `missing_telemetry` entries when present, keeping missing telemetry as replay evidence gap context.
- Additional Lab test fixtures under `tests/fixtures/edgeenv_regression/` mirror EdgeEnv replay examples for candidate telemetry gaps and execution sequence inversion. These fixture smokes verify that replay warnings become Lab-owned report context without making Lab recompute EdgeEnv comparability.
- Markdown/HTML reports include a `Runtime Intelligence Risk Summary` that summarizes EdgeEnv comparability/regression, telemetry replay gaps, Runtime history seed traceability, AIGuard deterministic evidence, and the Lab-owned deployment decision in one reviewer-facing table.
- When EdgeEnv includes preserved Orchestrator feed context, the `Runtime Intelligence Risk Summary` surfaces queue, thermal, throttling, memory, and fallback context as supplemental runtime evidence.
Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -194,10 +194,10 @@
"history": {
"schema_version": "edgeenv.runtime-telemetry-history.v1",
"summary": {
"registered_runs": 2,
"registered_runs": 3,
"telemetry_runs": 2,
"missing_telemetry_runs": 0,
"orchestrator_feed_runs": 1,
"missing_telemetry_runs": 1,
"orchestrator_feed_runs": 2,
"history_seed_runs": 2
},
"telemetry_coverage": {
Expand Down Expand Up @@ -344,6 +344,58 @@
]
}
}
],
"missing_telemetry": [
{
"run_id": "edgeenv-smoke-missing",
"reason": "runtime_telemetry_missing",
"missing_telemetry_is_failure": false,
"orchestrator_context_present": true,
"orchestrator_operation_context": {
"schema_version": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1",
"role": "orchestrator_operation_context_for_edgeenv",
"source_repository": "InferEdgeOrchestrator",
"artifact_role": "orchestrator-supplemental-operation-context",
"producer_contract": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1",
"source": "orchestration_summary",
"run_id": "edgeenv-smoke-missing",
"not_a_regression_judgement": true,
"not_a_comparability_gate": true,
"decision_owner": "lab",
"regression_owner": "edgeenv",
"candidate_context": {
"run_id": "edgeenv-smoke-missing",
"telemetry_source": "inferedge_orchestrator_operation_summary",
"queue_depth": 4,
"operation": {
"queue_depth": 4,
"deadline_missed_count": 1,
"fallback_count": 0
},
"resource": {
"source": "runtime_health_snapshot",
"gpu_temperature": 72.0,
"throttling_detected": false
}
},
"edgeenv_mapping_hint": {
"copy_candidate_context_to": "runtime_telemetry_context.candidate",
"operation_context_role": "supplemental",
"coverage_summary_owner": "edgeenv",
"coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage",
"candidate_context_required_fields": [
"run_id",
"telemetry_source",
"operation",
"resource"
],
"aiguard_evidence_candidates": [
"runtime_queue_overload",
"runtime_thermal_instability"
]
}
}
}
]
},
"notes": [
Expand Down
4 changes: 2 additions & 2 deletions scripts/check_runtime_intelligence_artifact_bundle.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"aiguard_coverage_gap_recommendation": (
"Inspect telemetry coverage missing fields"
),
"orchestrator_feed": "| Orchestrator operation feed context | 1 |",
"orchestrator_feed": "| Orchestrator operation feed context | 2 |",
"runtime_history_seed": "| Runtime telemetry history seed | 2 |",
"orchestrator_attached_run": "| Orchestrator context attached runs | candidate |",
"aiguard_evidence": "| AIGuard deterministic evidence | warning / suspicious |",
Expand All @@ -30,7 +30,7 @@
"runtime_queue_overload, runtime_thermal_instability |"
),
"aiguard_orchestrator_handoff": (
"| AIGuard Orchestrator context handoff | feeds=1.0, candidate |"
"| AIGuard Orchestrator context handoff | feeds=2.0, candidate |"
),
"aiguard_history_seed_handoff": "| AIGuard history seed handoff | seeds=2.0",
"guard_warning_rule": "guard_warning_review",
Expand Down
137 changes: 125 additions & 12 deletions scripts/check_runtime_intelligence_bundle_manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@
"aiguard_raw_context: telemetry_coverage_source=history_telemetry_coverage",
"aiguard_raw_context: orchestrator_mapping_hint preserved",
"aiguard_raw_context: orchestrator_producer_markers preserved",
"aiguard_raw_context: missing_telemetry_orchestrator_context preserved",
)
EDGEENV_HANDOFF_SUMMARY_CONTRACT_MARKERS = (
"edgeenv_handoff: lab_bundle_alignment validated",
Expand Down Expand Up @@ -470,7 +471,11 @@ def _validate_edgeenv_runtime_history_artifact(
if isinstance(coverage, dict):
_validate_edgeenv_history_coverage_summary(coverage, errors)
_validate_edgeenv_history_seed_runs(history, errors)
_validate_edgeenv_missing_telemetry_orchestrator_context(history, errors)
_validate_edgeenv_missing_telemetry_orchestrator_context(
history,
errors,
"EdgeEnv handoff runtime_telemetry_history",
)


def _validate_edgeenv_report(edgeenv_report: dict[str, Any], errors: list[str]) -> None:
Expand Down Expand Up @@ -503,9 +508,14 @@ def _validate_edgeenv_report(edgeenv_report: dict[str, Any], errors: list[str])
)
summary = history.get("summary") or {}
_record(
summary.get("orchestrator_feed_runs") == 1,
summary.get("orchestrator_feed_runs") == 2,
errors,
"runtime_telemetry_context.history.summary.orchestrator_feed_runs must be 2",
)
_record(
summary.get("missing_telemetry_runs") == 1,
errors,
"runtime_telemetry_context.history.summary.orchestrator_feed_runs must be 1",
"runtime_telemetry_context.history.summary.missing_telemetry_runs must be 1",
)
_record(
summary.get("history_seed_runs") == 2,
Expand All @@ -521,6 +531,11 @@ def _validate_edgeenv_report(edgeenv_report: dict[str, Any], errors: list[str])
if isinstance(history_coverage, dict):
_validate_edgeenv_history_coverage_summary(history_coverage, errors)
_validate_edgeenv_history_seed_runs(history, errors)
_validate_edgeenv_missing_telemetry_orchestrator_context(
history,
errors,
"runtime_telemetry_context.history",
)

candidate = context.get("candidate") or {}
_record(
Expand Down Expand Up @@ -779,12 +794,13 @@ def _validate_edgeenv_history_seed_runs(
def _validate_edgeenv_missing_telemetry_orchestrator_context(
history: dict[str, Any],
errors: list[str],
label: str,
) -> None:
missing_telemetry = history.get("missing_telemetry")
_record(
isinstance(missing_telemetry, list),
errors,
"EdgeEnv handoff runtime_telemetry_history.missing_telemetry must be a list",
f"{label}.missing_telemetry must be a list",
)
if not isinstance(missing_telemetry, list):
return
Expand All @@ -801,31 +817,30 @@ def _validate_edgeenv_missing_telemetry_orchestrator_context(
_record(
isinstance(missing_run, dict),
errors,
"EdgeEnv handoff runtime_telemetry_history.missing_telemetry must "
"include edgeenv-smoke-missing",
f"{label}.missing_telemetry must include edgeenv-smoke-missing",
)
if not isinstance(missing_run, dict):
return

_record(
missing_run.get("reason") == "runtime_telemetry_missing",
errors,
"EdgeEnv handoff runtime_telemetry_history.missing_telemetry"
"[edgeenv-smoke-missing].reason must be runtime_telemetry_missing",
f"{label}.missing_telemetry[edgeenv-smoke-missing].reason "
"must be runtime_telemetry_missing",
)
operation_context = missing_run.get("orchestrator_operation_context")
_record(
isinstance(operation_context, dict),
errors,
"EdgeEnv handoff runtime_telemetry_history.missing_telemetry"
"[edgeenv-smoke-missing] must include orchestrator_operation_context",
f"{label}.missing_telemetry[edgeenv-smoke-missing] must include "
"orchestrator_operation_context",
)
if not isinstance(operation_context, dict):
return

prefix = (
"EdgeEnv handoff runtime_telemetry_history.missing_telemetry"
"[edgeenv-smoke-missing].orchestrator_operation_context"
f"{label}.missing_telemetry[edgeenv-smoke-missing]."
"orchestrator_operation_context"
)
_record(
operation_context.get("schema_version")
Expand Down Expand Up @@ -1077,6 +1092,7 @@ def _validate_coverage_gap_evidence(
"must be ['queue_depth']",
)
_validate_aiguard_orchestrator_mapping_hint(edgeenv, errors)
_validate_aiguard_missing_orchestrator_context(edgeenv, errors)


def _validate_aiguard_history_seed_context(
Expand Down Expand Up @@ -1286,6 +1302,103 @@ def _validate_aiguard_orchestrator_mapping_hint(
)


def _validate_aiguard_missing_orchestrator_context(
edgeenv_context: dict[str, Any],
errors: list[str],
) -> None:
_record(
edgeenv_context.get("history_missing_telemetry_runs") == 1.0,
errors,
"AIGuard coverage evidence history_missing_telemetry_runs must be 1.0",
)
_record(
edgeenv_context.get("history_missing_orchestrator_context_count") == 1.0,
errors,
"AIGuard coverage evidence "
"history_missing_orchestrator_context_count must be 1.0",
)
run_ids = edgeenv_context.get("history_missing_orchestrator_context_run_ids")
_record(
isinstance(run_ids, list) and "edgeenv-smoke-missing" in run_ids,
errors,
"AIGuard coverage evidence "
"history_missing_orchestrator_context_run_ids must include "
"edgeenv-smoke-missing",
)
_record(
edgeenv_context.get("history_missing_orchestrator_source_repository")
== REQUIRED_SOURCE_REPOSITORIES["orchestrator_operation_context"],
errors,
"AIGuard coverage evidence "
"history_missing_orchestrator_source_repository must be "
f"{REQUIRED_SOURCE_REPOSITORIES['orchestrator_operation_context']}",
)
_record(
edgeenv_context.get("history_missing_orchestrator_artifact_role")
== REQUIRED_ARTIFACT_ROLES["orchestrator_operation_context"],
errors,
"AIGuard coverage evidence history_missing_orchestrator_artifact_role "
f"must be {REQUIRED_ARTIFACT_ROLES['orchestrator_operation_context']}",
)
_record(
edgeenv_context.get("history_missing_orchestrator_producer_contract")
== REQUIRED_PRODUCER_CONTRACTS["orchestrator_feed_schema"],
errors,
"AIGuard coverage evidence history_missing_orchestrator_producer_contract "
f"must be {REQUIRED_PRODUCER_CONTRACTS['orchestrator_feed_schema']}",
)
_record(
edgeenv_context.get(
"history_missing_orchestrator_candidate_context_telemetry_source"
)
== "inferedge_orchestrator_operation_summary",
errors,
"AIGuard coverage evidence "
"history_missing_orchestrator_candidate_context_telemetry_source must be "
"inferedge_orchestrator_operation_summary",
)
mapping_hint = edgeenv_context.get(
"history_missing_orchestrator_edgeenv_mapping_hint"
)
_record(
isinstance(mapping_hint, dict),
errors,
"AIGuard coverage evidence "
"history_missing_orchestrator_edgeenv_mapping_hint must be an object",
)
if isinstance(mapping_hint, dict):
for key, expected in REQUIRED_ORCHESTRATOR_MAPPING_HINT.items():
_record(
mapping_hint.get(key) == expected,
errors,
"AIGuard coverage evidence "
f"history_missing_orchestrator_edgeenv_mapping_hint.{key} "
f"must be {expected}",
)
evidence_candidates = edgeenv_context.get(
"history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates"
)
_record(
isinstance(evidence_candidates, list),
errors,
"AIGuard coverage evidence "
"history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates "
"must be a list",
)
if isinstance(evidence_candidates, list):
missing_candidates = sorted(
REQUIRED_ORCHESTRATOR_AIGUARD_EVIDENCE_CANDIDATES
- set(evidence_candidates)
)
_record(
not missing_candidates,
errors,
"AIGuard coverage evidence "
"history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates "
f"is missing {missing_candidates}",
)


def _write_summary(
path: str,
*,
Expand Down
1 change: 1 addition & 0 deletions scripts/check_runtime_intelligence_ci_artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
"aiguard_raw_context: telemetry_coverage_source=history_telemetry_coverage",
"aiguard_raw_context: orchestrator_mapping_hint preserved",
"aiguard_raw_context: orchestrator_producer_markers preserved",
"aiguard_raw_context: missing_telemetry_orchestrator_context preserved",
"edgeenv_handoff: lab_bundle_alignment validated",
"edgeenv_handoff: runtime_telemetry_history validated",
"edgeenv_handoff: missing_telemetry_orchestrator_context validated",
Expand Down
Loading
Loading