From fa9dca1f3a2786095f4fd51b3496fdd63e30cd7f Mon Sep 17 00:00:00 2001 From: clawmetry-autofix Date: Tue, 25 Aug 2026 09:30:41 +0000 Subject: [PATCH 1/4] feat(adapters/openclaw): surface nemoclaw onboarding OTel trace artifacts (#5193) The NemoClaw harness writes OpenTelemetry-style spans for each onboarding phase when NEMOCLAW_TRACE is set, but the adapter discarded the file entirely. Adds _nemoclaw_onboard_trace() which resolves the file via NEMOCLAW_TRACE_FILE / NEMOCLAW_TRACE_DIR / .e2e/traces/trace.json, parses both the flat harness shape and the OTel resource_spans format, and surfaces worst-case status, error phase names, and summary.slowest_spans so a failed or slow onboarding step is diagnosable from the dashboard. Closes #5193 Co-Authored-By: ClawMetry Autofix Bot --- clawmetry/adapters/openclaw.py | 103 ++++++++++++++ ...est_obs_gap_nemoclaw_onboard_trace_5193.py | 134 ++++++++++++++++++ 2 files changed, 237 insertions(+) create mode 100644 tests/test_obs_gap_nemoclaw_onboard_trace_5193.py diff --git a/clawmetry/adapters/openclaw.py b/clawmetry/adapters/openclaw.py index 741e22281f..7368f8c95d 100644 --- a/clawmetry/adapters/openclaw.py +++ b/clawmetry/adapters/openclaw.py @@ -1541,6 +1541,102 @@ def _model_router_live() -> dict: return result +def _nemoclaw_onboard_trace() -> dict: + """Read NemoClaw onboarding OTel trace artifacts (#5193). + + When ``NEMOCLAW_TRACE`` is set the harness writes OpenTelemetry-style spans + for each onboarding phase (e.g. ``nemoclaw.onboard.phase.gateway``, + ``nemoclaw.onboard.phase.inference``) including span status (OK/ERROR/UNSET), + duration_ms, events, sanitised attributes, and a ``summary.slowest_spans`` + list. ClawMetry surfaces the worst-case status, error phase names, and the + slowest-span summary so a failed or slow onboarding step is diagnosable from + the dashboard rather than silently invisible. + + Path resolution (first match wins): + 1. ``NEMOCLAW_TRACE_FILE`` env var. + 2. ``NEMOCLAW_TRACE_DIR/trace.json``. + 3. ``.e2e/traces/trace.json`` (harness default, relative to cwd). + + Handles both the flat harness shape ``{spans:[...], summary:{...}}`` and the + standard OTel ``resource_spans`` export. Returns ``{}`` when + ``NEMOCLAW_TRACE`` is unset/disabled or no file is found. Never raises. + """ + import json as _json + + trace_env = os.environ.get("NEMOCLAW_TRACE", "") + if not trace_env or trace_env.lower() in ("0", "false", "no"): + return {} + + candidates = [] + tf = os.environ.get("NEMOCLAW_TRACE_FILE", "") + if tf: + candidates.append(tf) + td = os.environ.get("NEMOCLAW_TRACE_DIR", "") + if td: + candidates.append(os.path.join(td, "trace.json")) + candidates.append(os.path.join(".e2e", "traces", "trace.json")) + + data = None + for path in candidates: + try: + with open(path, "r", encoding="utf-8", errors="replace") as fh: + data = _json.load(fh) + break + except (OSError, ValueError): + continue + except Exception: + continue + + if data is None or not isinstance(data, dict): + return {} + + try: + spans: list = [] + if "spans" in data: + raw = data["spans"] + if isinstance(raw, list): + spans = raw + elif "resource_spans" in data: + for rs in data.get("resource_spans", []): + for ss in (rs.get("scope_spans") or rs.get("scopeSpans") or []): + spans.extend(ss.get("spans", [])) + + _STATUS_RANK = {"ERROR": 2, "UNSET": 1, "OK": 0} + worst_rank = -1 + worst_status = "UNKNOWN" + error_names: list = [] + + for span in spans: + if not isinstance(span, dict): + continue + status = str(span.get("status", "UNSET")).upper() + rank = _STATUS_RANK.get(status, 0) + if rank > worst_rank: + worst_rank = rank + worst_status = status + if status == "ERROR": + name = span.get("name") or span.get("spanName") or "" + if name: + error_names.append(str(name)) + + result: dict = {} + if spans: + result["nemoclawOnboardTraceStatus"] = worst_status + result["nemoclawOnboardTraceSpanCount"] = len(spans) + if error_names: + result["nemoclawOnboardTraceErrors"] = error_names[:10] + + summary = data.get("summary") + if isinstance(summary, dict): + slow = summary.get("slowest_spans") + if isinstance(slow, list) and slow: + result["nemoclawOnboardSlowSpans"] = slow[:5] + + return result + except Exception: + return {} + + def _parse_proxy_config_model_list(content: str) -> Optional[List[str]]: """Extract model names from a LiteLLM-style proxy-config YAML (#2960). @@ -2451,6 +2547,13 @@ def detect(self) -> DetectResult: # fleet views know whether autonomous skill actions are gated by # human approval. Returns {} on installs without the key. meta.update(_workshop_approval_config()) + # NemoClaw onboarding OTel trace artifacts (#5193): surfaces + # nemoclawOnboardTraceStatus/SpanCount/Errors/SlowSpans when + # NEMOCLAW_TRACE is set and the harness wrote a trace file. + # Returns {} when disabled or file absent — no guard needed. + _ot = _nemoclaw_onboard_trace() + if _ot: + meta.update(_ot) return DetectResult( name=self.name, display_name=self.display_name, diff --git a/tests/test_obs_gap_nemoclaw_onboard_trace_5193.py b/tests/test_obs_gap_nemoclaw_onboard_trace_5193.py new file mode 100644 index 0000000000..f4d67601e4 --- /dev/null +++ b/tests/test_obs_gap_nemoclaw_onboard_trace_5193.py @@ -0,0 +1,134 @@ +"""Tests for issue #5193 — nemoclaw: onboarding OTel trace artifacts not ingested. + +Verifies that _nemoclaw_onboard_trace() reads the trace file written by the +NemoClaw harness (NEMOCLAW_TRACE_FILE / NEMOCLAW_TRACE_DIR / .e2e/traces/trace.json) +and surfaces nemoclawOnboardTraceStatus, nemoclawOnboardTraceSpanCount, +nemoclawOnboardTraceErrors, and nemoclawOnboardSlowSpans on DetectResult.meta. + +Fingerprint: hgap-1f53053124 (used to dedupe — keep it in the body). +""" +from __future__ import annotations + +import importlib +import json +import os +import sys + +import pytest + + +@pytest.fixture(autouse=True) +def _restore_sys_modules(): + saved = sys.modules.get("clawmetry.adapters.openclaw") + yield + if saved is None: + sys.modules.pop("clawmetry.adapters.openclaw", None) + else: + sys.modules["clawmetry.adapters.openclaw"] = saved + + +def _reload_adapter(): + import clawmetry.adapters.openclaw as oc_mod + importlib.reload(oc_mod) + return oc_mod + + +def _write_trace(path, spans, summary=None): + data: dict = {"spans": spans} + if summary is not None: + data["summary"] = summary + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as fh: + json.dump(data, fh) + + +def test_disabled_when_env_not_set(monkeypatch): + """Returns {} when NEMOCLAW_TRACE is absent.""" + monkeypatch.delenv("NEMOCLAW_TRACE", raising=False) + oc = _reload_adapter() + assert oc._nemoclaw_onboard_trace() == {} + + +def test_disabled_when_env_zero(monkeypatch): + """Returns {} when NEMOCLAW_TRACE=0 (explicitly disabled).""" + monkeypatch.setenv("NEMOCLAW_TRACE", "0") + oc = _reload_adapter() + assert oc._nemoclaw_onboard_trace() == {} + + +def test_returns_empty_when_file_missing(monkeypatch, tmp_path): + """Returns {} gracefully when trace file does not exist — never raises.""" + monkeypatch.setenv("NEMOCLAW_TRACE", "1") + monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(tmp_path / "no_such_file.json")) + monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False) + monkeypatch.chdir(tmp_path) # prevents fallback to .e2e/traces/trace.json + oc = _reload_adapter() + assert oc._nemoclaw_onboard_trace() == {} + + +def test_ok_spans_surface_status_and_count(monkeypatch, tmp_path): + """All-OK spans produce status=OK and the correct span count.""" + trace_file = tmp_path / "trace.json" + _write_trace(str(trace_file), [ + {"name": "nemoclaw.onboard.phase.gateway", "status": "OK", "duration_ms": 120}, + {"name": "nemoclaw.onboard.phase.inference", "status": "OK", "duration_ms": 340}, + ]) + monkeypatch.setenv("NEMOCLAW_TRACE", "1") + monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(trace_file)) + monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False) + oc = _reload_adapter() + result = oc._nemoclaw_onboard_trace() + assert result["nemoclawOnboardTraceStatus"] == "OK" + assert result["nemoclawOnboardTraceSpanCount"] == 2 + assert "nemoclawOnboardTraceErrors" not in result + + +def test_error_span_surfaces_status_and_name(monkeypatch, tmp_path): + """An ERROR span sets worst-case status=ERROR and includes the phase name.""" + trace_file = tmp_path / "trace.json" + _write_trace(str(trace_file), [ + {"name": "nemoclaw.onboard.phase.gateway", "status": "OK", "duration_ms": 80}, + {"name": "nemoclaw.onboard.phase.inference", "status": "ERROR", "duration_ms": 5000}, + ]) + monkeypatch.setenv("NEMOCLAW_TRACE", "1") + monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(trace_file)) + monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False) + oc = _reload_adapter() + result = oc._nemoclaw_onboard_trace() + assert result["nemoclawOnboardTraceStatus"] == "ERROR" + assert result["nemoclawOnboardTraceSpanCount"] == 2 + assert "nemoclaw.onboard.phase.inference" in result["nemoclawOnboardTraceErrors"] + + +def test_slowest_spans_forwarded_from_summary(monkeypatch, tmp_path): + """summary.slowest_spans is forwarded as nemoclawOnboardSlowSpans (capped at 5).""" + slow = [ + {"name": "nemoclaw.onboard.phase.inference", "duration_ms": 4200}, + {"name": "nemoclaw.onboard.phase.gateway", "duration_ms": 900}, + ] + trace_file = tmp_path / "trace.json" + _write_trace(str(trace_file), [ + {"name": "nemoclaw.onboard.phase.inference", "status": "OK", "duration_ms": 4200}, + ], summary={"slowest_spans": slow}) + monkeypatch.setenv("NEMOCLAW_TRACE", "1") + monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(trace_file)) + monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False) + oc = _reload_adapter() + result = oc._nemoclaw_onboard_trace() + assert result["nemoclawOnboardSlowSpans"] == slow + + +def test_trace_dir_env_resolves_to_trace_json(monkeypatch, tmp_path): + """NEMOCLAW_TRACE_DIR is used as a directory and trace.json is appended.""" + trace_dir = tmp_path / "traces" + trace_dir.mkdir() + _write_trace(str(trace_dir / "trace.json"), [ + {"name": "nemoclaw.onboard.phase.gateway", "status": "OK", "duration_ms": 50}, + ]) + monkeypatch.setenv("NEMOCLAW_TRACE", "1") + monkeypatch.delenv("NEMOCLAW_TRACE_FILE", raising=False) + monkeypatch.setenv("NEMOCLAW_TRACE_DIR", str(trace_dir)) + oc = _reload_adapter() + result = oc._nemoclaw_onboard_trace() + assert result["nemoclawOnboardTraceStatus"] == "OK" + assert result["nemoclawOnboardTraceSpanCount"] == 1 From e3869480cf0962eb2e50a827570c4089d44c17f4 Mon Sep 17 00:00:00 2001 From: Vivek Chand Date: Thu, 27 Aug 2026 12:30:42 +0000 Subject: [PATCH 2/4] ci: re-trigger E2E Gate (stale required check) From e6304fe4c9d5a0c7275a37f3b26db83839c7c51d Mon Sep 17 00:00:00 2001 From: clawmetry-autofix Date: Sun, 30 Aug 2026 12:48:21 +0000 Subject: [PATCH 3/4] test(adapters/openclaw): add OTel resource_spans format coverage (#5193) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add two tests covering the standard OTel export shape (resource_spans → scope_spans/scopeSpans → spans) to close the gap identified in babysit review — only the flat harness format was previously tested. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01751f3a-c1f5-5543-a2cf-cc88f096e807 --- ...est_obs_gap_nemoclaw_onboard_trace_5193.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test_obs_gap_nemoclaw_onboard_trace_5193.py b/tests/test_obs_gap_nemoclaw_onboard_trace_5193.py index f4d67601e4..d56b1e10c6 100644 --- a/tests/test_obs_gap_nemoclaw_onboard_trace_5193.py +++ b/tests/test_obs_gap_nemoclaw_onboard_trace_5193.py @@ -132,3 +132,59 @@ def test_trace_dir_env_resolves_to_trace_json(monkeypatch, tmp_path): result = oc._nemoclaw_onboard_trace() assert result["nemoclawOnboardTraceStatus"] == "OK" assert result["nemoclawOnboardTraceSpanCount"] == 1 + + +def test_resource_spans_otel_export_format(monkeypatch, tmp_path): + """Standard OTel resource_spans export shape is parsed correctly (snake_case keys).""" + trace_file = tmp_path / "trace.json" + data = { + "resource_spans": [ + { + "scope_spans": [ + { + "spans": [ + {"name": "nemoclaw.onboard.phase.gateway", "status": "OK"}, + {"name": "nemoclaw.onboard.phase.inference", "status": "ERROR"}, + ] + } + ] + } + ] + } + with open(str(trace_file), "w") as fh: + json.dump(data, fh) + monkeypatch.setenv("NEMOCLAW_TRACE", "1") + monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(trace_file)) + monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False) + oc = _reload_adapter() + result = oc._nemoclaw_onboard_trace() + assert result["nemoclawOnboardTraceStatus"] == "ERROR" + assert result["nemoclawOnboardTraceSpanCount"] == 2 + assert "nemoclaw.onboard.phase.inference" in result["nemoclawOnboardTraceErrors"] + + +def test_resource_spans_camel_case_scope_spans(monkeypatch, tmp_path): + """Standard OTel resource_spans export with camelCase scopeSpans key is parsed correctly.""" + trace_file = tmp_path / "trace.json" + data = { + "resource_spans": [ + { + "scopeSpans": [ + { + "spans": [ + {"name": "nemoclaw.onboard.phase.gateway", "status": "OK"}, + ] + } + ] + } + ] + } + with open(str(trace_file), "w") as fh: + json.dump(data, fh) + monkeypatch.setenv("NEMOCLAW_TRACE", "1") + monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(trace_file)) + monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False) + oc = _reload_adapter() + result = oc._nemoclaw_onboard_trace() + assert result["nemoclawOnboardTraceStatus"] == "OK" + assert result["nemoclawOnboardTraceSpanCount"] == 1 From 10548b9681750aff39fe55b26980ad886881354b Mon Sep 17 00:00:00 2001 From: vivekchand Date: Thu, 3 Sep 2026 19:19:12 +0200 Subject: [PATCH 4/4] fix(adapters/openclaw): normalise OTel span status objects in NemoClaw onboarding trace The standard OTel JSON export writes span status as {"code": N} (0 UNSET, 1 OK, 2 ERROR) and some exporters as STATUS_CODE_*. The ranking compared raw strings, so an export-shaped trace reported "{'CODE': 2}" as its status. Map both forms (and unrecognised values) onto OK/ERROR/UNSET so nemoclawOnboardTraceStatus is always one of the three documented values. Spec: REQ-OBS-RSO-034 / AC-OBS-RSO-002.3 and the "NemoClaw onboarding trace artifacts" blueprint section (Runtime and Session Observability). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01KpoJsfi8kkY29uvPWNQj3P --- clawmetry/adapters/openclaw.py | 22 ++++++++++-- ...est_obs_gap_nemoclaw_onboard_trace_5193.py | 36 +++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/clawmetry/adapters/openclaw.py b/clawmetry/adapters/openclaw.py index 7368f8c95d..26273dc363 100644 --- a/clawmetry/adapters/openclaw.py +++ b/clawmetry/adapters/openclaw.py @@ -1602,15 +1602,31 @@ def _nemoclaw_onboard_trace() -> dict: spans.extend(ss.get("spans", [])) _STATUS_RANK = {"ERROR": 2, "UNSET": 1, "OK": 0} + # OTel status codes: 0 = UNSET, 1 = OK, 2 = ERROR (the JSON export + # writes ``{"code": N}``; some exporters write ``STATUS_CODE_OK``). + _CODE_TO_STATUS = {0: "UNSET", 1: "OK", 2: "ERROR"} + + def _norm_status(raw) -> str: + if isinstance(raw, dict): + raw = raw.get("code", raw.get("status_code", "UNSET")) + if isinstance(raw, bool): + return "UNSET" + if isinstance(raw, (int, float)): + return _CODE_TO_STATUS.get(int(raw), "UNSET") + text = str(raw or "UNSET").upper().strip() + if text.startswith("STATUS_CODE_"): + text = text[len("STATUS_CODE_"):] + return text if text in _STATUS_RANK else "UNSET" + worst_rank = -1 - worst_status = "UNKNOWN" + worst_status = "UNSET" error_names: list = [] for span in spans: if not isinstance(span, dict): continue - status = str(span.get("status", "UNSET")).upper() - rank = _STATUS_RANK.get(status, 0) + status = _norm_status(span.get("status", "UNSET")) + rank = _STATUS_RANK[status] if rank > worst_rank: worst_rank = rank worst_status = status diff --git a/tests/test_obs_gap_nemoclaw_onboard_trace_5193.py b/tests/test_obs_gap_nemoclaw_onboard_trace_5193.py index d56b1e10c6..c9a095e63f 100644 --- a/tests/test_obs_gap_nemoclaw_onboard_trace_5193.py +++ b/tests/test_obs_gap_nemoclaw_onboard_trace_5193.py @@ -188,3 +188,39 @@ def test_resource_spans_camel_case_scope_spans(monkeypatch, tmp_path): result = oc._nemoclaw_onboard_trace() assert result["nemoclawOnboardTraceStatus"] == "OK" assert result["nemoclawOnboardTraceSpanCount"] == 1 + + +def test_otel_export_shape_with_status_code_objects(monkeypatch, tmp_path): + """Standard OTel JSON export: resource_spans/scope_spans and {code: N} status.""" + trace_file = tmp_path / "trace.json" + data = {"resource_spans": [{"scope_spans": [{"spans": [ + {"name": "nemoclaw.onboard.phase.gateway", "status": {"code": 1}}, + {"name": "nemoclaw.onboard.phase.inference", "status": {"code": 2}}, + {"name": "nemoclaw.onboard.phase.sandbox", "status": "STATUS_CODE_OK"}, + {"name": "nemoclaw.onboard.phase.unknown", "status": "weird-value"}, + ]}]}]} + with open(trace_file, "w") as fh: + json.dump(data, fh) + monkeypatch.setenv("NEMOCLAW_TRACE", "1") + monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(trace_file)) + monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False) + oc = _reload_adapter() + out = oc._nemoclaw_onboard_trace() + assert out["nemoclawOnboardTraceStatus"] == "ERROR" + assert out["nemoclawOnboardTraceSpanCount"] == 4 + assert out["nemoclawOnboardTraceErrors"] == ["nemoclaw.onboard.phase.inference"] + + +def test_unrecognised_status_strings_report_unset_not_raw(monkeypatch, tmp_path): + """Status is always one of OK/ERROR/UNSET, never an arbitrary raw string.""" + trace_file = tmp_path / "trace.json" + _write_trace(str(trace_file), [ + {"name": "nemoclaw.onboard.phase.gateway", "status": "bogus"}, + ]) + monkeypatch.setenv("NEMOCLAW_TRACE", "1") + monkeypatch.setenv("NEMOCLAW_TRACE_FILE", str(trace_file)) + monkeypatch.delenv("NEMOCLAW_TRACE_DIR", raising=False) + oc = _reload_adapter() + out = oc._nemoclaw_onboard_trace() + assert out["nemoclawOnboardTraceStatus"] == "UNSET" + assert "nemoclawOnboardTraceErrors" not in out