feat(adapters/openclaw): surface nemoclaw onboarding OTel trace artifacts (#5193)

clawmetry/adapters/openclaw.py

@@ -1541,6 +1541,118 @@ def _model_router_live() -> dict:
     return result
 
 
+def _nemoclaw_onboard_trace() -> dict:
+    """Read NemoClaw onboarding OTel trace artifacts (#5193).
+
+    When ``NEMOCLAW_TRACE`` is set the harness writes OpenTelemetry-style spans
+    for each onboarding phase (e.g. ``nemoclaw.onboard.phase.gateway``,
+    ``nemoclaw.onboard.phase.inference``) including span status (OK/ERROR/UNSET),
+    duration_ms, events, sanitised attributes, and a ``summary.slowest_spans``
+    list.  ClawMetry surfaces the worst-case status, error phase names, and the
+    slowest-span summary so a failed or slow onboarding step is diagnosable from
+    the dashboard rather than silently invisible.
+
+    Path resolution (first match wins):
+    1. ``NEMOCLAW_TRACE_FILE`` env var.
+    2. ``NEMOCLAW_TRACE_DIR/trace.json``.
+    3. ``.e2e/traces/trace.json`` (harness default, relative to cwd).
+
+    Handles both the flat harness shape ``{spans:[...], summary:{...}}`` and the
+    standard OTel ``resource_spans`` export.  Returns ``{}`` when
+    ``NEMOCLAW_TRACE`` is unset/disabled or no file is found.  Never raises.
+    """
+    import json as _json
+
+    trace_env = os.environ.get("NEMOCLAW_TRACE", "")
+    if not trace_env or trace_env.lower() in ("0", "false", "no"):
+        return {}
+
+    candidates = []
+    tf = os.environ.get("NEMOCLAW_TRACE_FILE", "")
+    if tf:
+        candidates.append(tf)
+    td = os.environ.get("NEMOCLAW_TRACE_DIR", "")
+    if td:
+        candidates.append(os.path.join(td, "trace.json"))
+    candidates.append(os.path.join(".e2e", "traces", "trace.json"))
+
+    data = None
+    for path in candidates:
+        try:
+            with open(path, "r", encoding="utf-8", errors="replace") as fh:
+                data = _json.load(fh)
+            break
+        except (OSError, ValueError):
+            continue
+        except Exception:
+            continue
+
+    if data is None or not isinstance(data, dict):
+        return {}
+
+    try:
+        spans: list = []
+        if "spans" in data:
+            raw = data["spans"]
+            if isinstance(raw, list):
+                spans = raw
+        elif "resource_spans" in data:
+            for rs in data.get("resource_spans", []):
+                for ss in (rs.get("scope_spans") or rs.get("scopeSpans") or []):
+                    spans.extend(ss.get("spans", []))
+
+        _STATUS_RANK = {"ERROR": 2, "UNSET": 1, "OK": 0}
+        # OTel status codes: 0 = UNSET, 1 = OK, 2 = ERROR (the JSON export
+        # writes ``{"code": N}``; some exporters write ``STATUS_CODE_OK``).
+        _CODE_TO_STATUS = {0: "UNSET", 1: "OK", 2: "ERROR"}
+
+        def _norm_status(raw) -> str:
+            if isinstance(raw, dict):
+                raw = raw.get("code", raw.get("status_code", "UNSET"))
+            if isinstance(raw, bool):
+                return "UNSET"
+            if isinstance(raw, (int, float)):
+                return _CODE_TO_STATUS.get(int(raw), "UNSET")
+            text = str(raw or "UNSET").upper().strip()
+            if text.startswith("STATUS_CODE_"):
+                text = text[len("STATUS_CODE_"):]
+            return text if text in _STATUS_RANK else "UNSET"
+
+        worst_rank = -1
+        worst_status = "UNSET"
+        error_names: list = []
+
+        for span in spans:
+            if not isinstance(span, dict):
+                continue
+            status = _norm_status(span.get("status", "UNSET"))
+            rank = _STATUS_RANK[status]
+            if rank > worst_rank:
+                worst_rank = rank
+                worst_status = status
+            if status == "ERROR":
+                name = span.get("name") or span.get("spanName") or ""
+                if name:
+                    error_names.append(str(name))
+
+        result: dict = {}
+        if spans:
+            result["nemoclawOnboardTraceStatus"] = worst_status
+            result["nemoclawOnboardTraceSpanCount"] = len(spans)
+        if error_names:
+            result["nemoclawOnboardTraceErrors"] = error_names[:10]
+
+        summary = data.get("summary")
+        if isinstance(summary, dict):
+            slow = summary.get("slowest_spans")
+            if isinstance(slow, list) and slow:
+                result["nemoclawOnboardSlowSpans"] = slow[:5]
+
+        return result
+    except Exception:
+        return {}
+
+
 def _parse_proxy_config_model_list(content: str) -> Optional[List[str]]:
     """Extract model names from a LiteLLM-style proxy-config YAML (#2960).
 
@@ -2451,6 +2563,13 @@ def detect(self) -> DetectResult:
             # fleet views know whether autonomous skill actions are gated by
             # human approval.  Returns {} on installs without the key.
             meta.update(_workshop_approval_config())
+            # NemoClaw onboarding OTel trace artifacts (#5193): surfaces
+            # nemoclawOnboardTraceStatus/SpanCount/Errors/SlowSpans when
+            # NEMOCLAW_TRACE is set and the harness wrote a trace file.
+            # Returns {} when disabled or file absent — no guard needed.
+            _ot = _nemoclaw_onboard_trace()
+            if _ot:
+                meta.update(_ot)
             return DetectResult(
                 name=self.name,
                 display_name=self.display_name,

tests/test_obs_gap_nemoclaw_onboard_trace_5193.py

@@ -0,0 +1,226 @@
+"""Tests for issue #5193 — nemoclaw: onboarding OTel trace artifacts not ingested.
+
+Verifies that _nemoclaw_onboard_trace() reads the trace file written by the
+NemoClaw harness (NEMOCLAW_TRACE_FILE / NEMOCLAW_TRACE_DIR / .e2e/traces/trace.json)
+and surfaces nemoclawOnboardTraceStatus, nemoclawOnboardTraceSpanCount,
+nemoclawOnboardTraceErrors, and nemoclawOnboardSlowSpans on DetectResult.meta.
+
+Fingerprint: hgap-1f53053124 (used to dedupe — keep it in the body).
+"""
+from __future__ import annotations
+
+import importlib
+import json
+import os
+import sys
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def _restore_sys_modules():
+    saved = sys.modules.get("clawmetry.adapters.openclaw")
+    yield
+    if saved is None:
+        sys.modules.pop("clawmetry.adapters.openclaw", None)
+    else:
+        sys.modules["clawmetry.adapters.openclaw"] = saved
+
+
+def _reload_adapter():
+    import clawmetry.adapters.openclaw as oc_mod
+    importlib.reload(oc_mod)
+    return oc_mod
+
+
+def _write_trace(path, spans, summary=None):
+    data: dict = {"spans": spans}
+    if summary is not None:
+        data["summary"] = summary
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "w") as fh:
+        json.dump(data, fh)
+
+
+def test_disabled_when_env_not_set(monkeypatch):
+    """Returns {} when NEMOCLAW_TRACE is absent."""
+    monkeypatch.delenv("NEMOCLAW_TRACE", raising=False)
+    oc = _reload_adapter()
+    assert oc._nemoclaw_onboard_trace() == {}
+
+
+def test_disabled_when_env_zero(monkeypatch):
+    """Returns {} when NEMOCLAW_TRACE=0 (explicitly disabled)."""
+    monkeypatch.setenv("NEMOCLAW_TRACE", "0")
+    oc = _reload_adapter()
+    assert oc._nemoclaw_onboard_trace() == {}
+
+
+def test_returns_empty_when_file_missing(monkeypatch, tmp_path):
+    """Returns {} gracefully when trace file does not exist — never raises."""
+    monkeypatch.setenv("NEMOCLAW_TRACE", "1")
+    monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(tmp_path / "no_such_file.json"))
+    monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False)
+    monkeypatch.chdir(tmp_path)  # prevents fallback to .e2e/traces/trace.json
+    oc = _reload_adapter()
+    assert oc._nemoclaw_onboard_trace() == {}
+
+
+def test_ok_spans_surface_status_and_count(monkeypatch, tmp_path):
+    """All-OK spans produce status=OK and the correct span count."""
+    trace_file = tmp_path / "trace.json"
+    _write_trace(str(trace_file), [
+        {"name": "nemoclaw.onboard.phase.gateway", "status": "OK", "duration_ms": 120},
+        {"name": "nemoclaw.onboard.phase.inference", "status": "OK", "duration_ms": 340},
+    ])
+    monkeypatch.setenv("NEMOCLAW_TRACE", "1")
+    monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(trace_file))
+    monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False)
+    oc = _reload_adapter()
+    result = oc._nemoclaw_onboard_trace()
+    assert result["nemoclawOnboardTraceStatus"] == "OK"
+    assert result["nemoclawOnboardTraceSpanCount"] == 2
+    assert "nemoclawOnboardTraceErrors" not in result
+
+
+def test_error_span_surfaces_status_and_name(monkeypatch, tmp_path):
+    """An ERROR span sets worst-case status=ERROR and includes the phase name."""
+    trace_file = tmp_path / "trace.json"
+    _write_trace(str(trace_file), [
+        {"name": "nemoclaw.onboard.phase.gateway", "status": "OK", "duration_ms": 80},
+        {"name": "nemoclaw.onboard.phase.inference", "status": "ERROR", "duration_ms": 5000},
+    ])
+    monkeypatch.setenv("NEMOCLAW_TRACE", "1")
+    monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(trace_file))
+    monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False)
+    oc = _reload_adapter()
+    result = oc._nemoclaw_onboard_trace()
+    assert result["nemoclawOnboardTraceStatus"] == "ERROR"
+    assert result["nemoclawOnboardTraceSpanCount"] == 2
+    assert "nemoclaw.onboard.phase.inference" in result["nemoclawOnboardTraceErrors"]
+
+
+def test_slowest_spans_forwarded_from_summary(monkeypatch, tmp_path):
+    """summary.slowest_spans is forwarded as nemoclawOnboardSlowSpans (capped at 5)."""
+    slow = [
+        {"name": "nemoclaw.onboard.phase.inference", "duration_ms": 4200},
+        {"name": "nemoclaw.onboard.phase.gateway", "duration_ms": 900},
+    ]
+    trace_file = tmp_path / "trace.json"
+    _write_trace(str(trace_file), [
+        {"name": "nemoclaw.onboard.phase.inference", "status": "OK", "duration_ms": 4200},
+    ], summary={"slowest_spans": slow})
+    monkeypatch.setenv("NEMOCLAW_TRACE", "1")
+    monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(trace_file))
+    monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False)
+    oc = _reload_adapter()
+    result = oc._nemoclaw_onboard_trace()
+    assert result["nemoclawOnboardSlowSpans"] == slow
+
+
+def test_trace_dir_env_resolves_to_trace_json(monkeypatch, tmp_path):
+    """NEMOCLAW_TRACE_DIR is used as a directory and trace.json is appended."""
+    trace_dir = tmp_path / "traces"
+    trace_dir.mkdir()
+    _write_trace(str(trace_dir / "trace.json"), [
+        {"name": "nemoclaw.onboard.phase.gateway", "status": "OK", "duration_ms": 50},
+    ])
+    monkeypatch.setenv("NEMOCLAW_TRACE", "1")
+    monkeypatch.delenv("NEMOCLAW_TRACE_FILE", raising=False)
+    monkeypatch.setenv("NEMOCLAW_TRACE_DIR", str(trace_dir))
+    oc = _reload_adapter()
+    result = oc._nemoclaw_onboard_trace()
+    assert result["nemoclawOnboardTraceStatus"] == "OK"
+    assert result["nemoclawOnboardTraceSpanCount"] == 1
+
+
+def test_resource_spans_otel_export_format(monkeypatch, tmp_path):
+    """Standard OTel resource_spans export shape is parsed correctly (snake_case keys)."""
+    trace_file = tmp_path / "trace.json"
+    data = {
+        "resource_spans": [
+            {
+                "scope_spans": [
+                    {
+                        "spans": [
+                            {"name": "nemoclaw.onboard.phase.gateway", "status": "OK"},
+                            {"name": "nemoclaw.onboard.phase.inference", "status": "ERROR"},
+                        ]
+                    }
+                ]
+            }
+        ]
+    }
+    with open(str(trace_file), "w") as fh:
+        json.dump(data, fh)
+    monkeypatch.setenv("NEMOCLAW_TRACE", "1")
+    monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(trace_file))
+    monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False)
+    oc = _reload_adapter()
+    result = oc._nemoclaw_onboard_trace()
+    assert result["nemoclawOnboardTraceStatus"] == "ERROR"
+    assert result["nemoclawOnboardTraceSpanCount"] == 2
+    assert "nemoclaw.onboard.phase.inference" in result["nemoclawOnboardTraceErrors"]
+
+
+def test_resource_spans_camel_case_scope_spans(monkeypatch, tmp_path):
+    """Standard OTel resource_spans export with camelCase scopeSpans key is parsed correctly."""
+    trace_file = tmp_path / "trace.json"
+    data = {
+        "resource_spans": [
+            {
+                "scopeSpans": [
+                    {
+                        "spans": [
+                            {"name": "nemoclaw.onboard.phase.gateway", "status": "OK"},
+                        ]
+                    }
+                ]
+            }
+        ]
+    }
+    with open(str(trace_file), "w") as fh:
+        json.dump(data, fh)
+    monkeypatch.setenv("NEMOCLAW_TRACE", "1")
+    monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(trace_file))
+    monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False)
+    oc = _reload_adapter()
+    result = oc._nemoclaw_onboard_trace()
+    assert result["nemoclawOnboardTraceStatus"] == "OK"
+    assert result["nemoclawOnboardTraceSpanCount"] == 1
+
+
+def test_otel_export_shape_with_status_code_objects(monkeypatch, tmp_path):
+    """Standard OTel JSON export: resource_spans/scope_spans and {code: N} status."""
+    trace_file = tmp_path / "trace.json"
+    data = {"resource_spans": [{"scope_spans": [{"spans": [
+        {"name": "nemoclaw.onboard.phase.gateway", "status": {"code": 1}},
+        {"name": "nemoclaw.onboard.phase.inference", "status": {"code": 2}},
+        {"name": "nemoclaw.onboard.phase.sandbox", "status": "STATUS_CODE_OK"},
+        {"name": "nemoclaw.onboard.phase.unknown", "status": "weird-value"},
+    ]}]}]}
+    with open(trace_file, "w") as fh:
+        json.dump(data, fh)
+    monkeypatch.setenv("NEMOCLAW_TRACE", "1")
+    monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(trace_file))
+    monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False)
+    oc = _reload_adapter()
+    out = oc._nemoclaw_onboard_trace()
+    assert out["nemoclawOnboardTraceStatus"] == "ERROR"
+    assert out["nemoclawOnboardTraceSpanCount"] == 4
+    assert out["nemoclawOnboardTraceErrors"] == ["nemoclaw.onboard.phase.inference"]
+
+
+def test_unrecognised_status_strings_report_unset_not_raw(monkeypatch, tmp_path):
+    """Status is always one of OK/ERROR/UNSET, never an arbitrary raw string."""
+    trace_file = tmp_path / "trace.json"
+    _write_trace(str(trace_file), [
+        {"name": "nemoclaw.onboard.phase.gateway", "status": "bogus"},
+    ])
+    monkeypatch.setenv("NEMOCLAW_TRACE", "1")
+    monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(trace_file))
+    monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False)
+    oc = _reload_adapter()
+    out = oc._nemoclaw_onboard_trace()
+    assert out["nemoclawOnboardTraceStatus"] == "UNSET"
+    assert "nemoclawOnboardTraceErrors" not in out