Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions src/python/isaaccapture/cloudxr/oob_teleop_env.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from urllib.parse import urlencode, urljoin
from urllib.request import Request, urlopen

from ..logging_config._core import logging_enabled
from .oob_teleop_hub import OOB_WS_PATH

log = logging.getLogger("isaaccapture.cloudxr.oob_teleop_env")
Expand Down Expand Up @@ -567,14 +568,13 @@ def resolve_lan_host_for_oob() -> str:
def oob_progress(stage: str, msg: str) -> None:
"""One-line progress update for ``--setup-oob`` / ``--usb-local`` steps.

Goes to stderr in dim cyan so the operator can see *where* the launcher
is in its sequence of steps without these lines competing with the
success banner (stdout) or error prints (red).

This stays a print so progress remains visible when the console log level
is set to warning.
Logged at INFO as ``[<stage>] <msg>``; printed to stderr instead when
``ISAACCAPTURE_LOGGING=off``, where no handler of ours would show it.
"""
print(f"\033[36m[{stage}]\033[0m {msg}", file=sys.stderr, flush=True)
if logging_enabled():
log.info("[%s] %s", stage, msg)
else:
print(f"\033[36m[{stage}]\033[0m {msg}", file=sys.stderr, flush=True)


def print_oob_hub_startup_banner(
Expand Down
57 changes: 55 additions & 2 deletions src/python/isaaccapture/cloudxr/oob_teleop_lifecycle.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from .oob_teleop_env import (
USB_TURN_CREDENTIAL,
USB_TURN_USER,
oob_progress,
redact_control_token,
usb_backend_port,
)
Expand Down Expand Up @@ -187,6 +188,9 @@ def __init__(
self._last_health: str | None = None
self._readiness_stage: str | None = None
self._readiness_since: float | None = None
# Whether the operator was told the headset streams now, and ever did.
self._streaming = False
self._streamed = False
self._prerequisite_signature: tuple | None = None
self.snapshot: dict = {}

Expand Down Expand Up @@ -260,6 +264,23 @@ async def _publish(self, health: str, state: str, reason: str, **flags) -> None:
self._restart_episode()
self._last_health = health
reason = re.sub(r"https?://\S+", "<URL>", redact_control_token(reason))[:300]
# The operator sees the headset connect once ACTIVE confirms the stream, and
# disconnect only on an explicit streaming=False: stale metrics drop health
# below ACTIVE while the browser still streams, and a publication that omits
# the flag does not know. The explicit False matches the snapshot default but
# is what announces the loss. Other transitions stay on this module's logger.
stage = "usb-local" if self.usb_local else "setup-oob"
if health == "active" and not self._streaming:
oob_progress(
stage,
"headset reconnected; streaming resumed"
if self._streamed
else "headset connected; streaming",
)
self._streaming = self._streamed = True
elif self._streaming and flags.get("streaming") is False:
oob_progress(stage, f"headset disconnected: {reason}")
self._streaming = False
snapshot = {
"schemaVersion": 1,
"generation": self.generation,
Expand Down Expand Up @@ -458,8 +479,21 @@ async def _enter_transport_recovery(
adb_ready: bool,
network_present: bool,
adb_diagnostic: str = "",
transport_verified: bool = False,
) -> None:
"""Preserve browser intent and publish one USB-loss transition."""
"""Preserve browser intent and publish one USB-loss transition.

*transport_verified* means this iteration's USB transport checks passed, so
the disruption came from observing the browser, not from the stream's path.
"""
# USB-local streams over this transport; once it is verified, only the
# browser's own report shows the stream lost. Over Wi-Fi only a headset that
# ADB reports without a network has certainly lost its stream. Read the
# report before the browser client is forgotten below.
stream_lost = (adb_ready and not network_present) or (
self.usb_local
and not (transport_verified and await self._browser_reports_streaming())
)
await self._remember_transport_loss()
if self._transport_lost:
self._invalidate_completed_repair(
Expand All @@ -472,6 +506,17 @@ async def _enter_transport_recovery(
adbReady=adb_ready,
networkPresent=network_present,
adbDiagnostic=redact_control_token(adb_diagnostic),
**({"streaming": False} if stream_lost else {}),
)

async def _browser_reports_streaming(self) -> bool:
"""Whether this lifecycle's browser still reports a running stream to the hub."""
state = await self.hub.get_snapshot()
return any(
headset["clientId"] == self.browser_client
and headset.get("streaming")
and headset.get("streamPhase") == "streaming"
for headset in state["headsets"]
)

def _transport_retry_interval(self) -> float:
Expand Down Expand Up @@ -989,6 +1034,7 @@ async def _automate(self) -> None:
networkPresent=True,
reverseRulesVerified=self.usb_local,
turnPrerequisitesReady=self.usb_local,
streaming=False,
)

async def on_client_loaded() -> None:
Expand Down Expand Up @@ -1078,6 +1124,7 @@ async def on_client_loaded() -> None:
networkPresent=True,
reverseRulesVerified=self.usb_local,
turnPrerequisitesReady=self.usb_local,
streaming=False,
)

def on_dispatched() -> Awaitable[None]:
Expand All @@ -1096,6 +1143,7 @@ async def publish() -> None:
reverseRulesVerified=self.usb_local,
turnPrerequisitesReady=self.usb_local,
connectDispatched=True,
streaming=False,
)

return publish()
Expand All @@ -1121,6 +1169,7 @@ async def publish() -> None:
"Selected headset must be Awake before CONNECT; wear or unlock it",
adbReady=True,
networkPresent=True,
streaming=False,
)
return None
except adb.OobAdbError:
Expand Down Expand Up @@ -1290,7 +1339,8 @@ async def _recover_existing_browser(self) -> None:
turnPrerequisitesReady=True,
browserRegistered=True,
healthProbeAcknowledged=True,
streaming=False,
# Unconfirmed is not lost: announce only if the browser stopped streaming.
**({} if report.get("streaming") else {"streaming": False}),
clientMetricsFresh=False,
streamPhase=phase,
terminalEventId=terminal_event,
Expand Down Expand Up @@ -1859,10 +1909,12 @@ async def run(self) -> None:
)
await self.sleep(self.config.interval_sec)
continue
transport_verified = False
try:
# In USB-local mode, ACTIVE/VERIFYING_BROWSER revalidate USB first.
if self.browser_ready or self.connect_dispatched:
await self._check_usb_prerequisites()
transport_verified = True
# Checked after prerequisites (a genuine USB/ADB/network failure keeps
# priority) but before per-state dispatch below, so a CDP-observed
# terminal error is handled the same way regardless of which state
Expand Down Expand Up @@ -1943,6 +1995,7 @@ async def run(self) -> None:
"REBUILDING_USB",
adb_ready=exc.adb_ready,
network_present=exc.network_present,
transport_verified=transport_verified,
)
continue
except asyncio.CancelledError:
Expand Down
5 changes: 4 additions & 1 deletion src/python/isaaccapture/logging_config/_console.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,10 @@ def filter(self, record: logging.LogRecord) -> bool:
_SGR_ESCAPE = re.compile(r"(?:\x1b\[[0-9;]*m)+")

# Exact logger name to ANSI emphasis.
_logger_colors: dict[str, str] = {}
_logger_colors: dict[str, str] = {
# cloudxr.oob_teleop_env contains [setup-oob] / [usb-local] progress.
"isaaccapture.cloudxr.oob_teleop_env": "\033[36m",
}


#: Color only WARNING and ERROR+ so severity remains distinctive.
Expand Down
26 changes: 26 additions & 0 deletions tests/python/core/cloudxr/test_oob_teleop_env.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

from __future__ import annotations

import logging
from urllib.parse import parse_qs, urlparse

import cloudxr_py_test_ns.oob_teleop_env as oob_teleop_env_under_test
Expand Down Expand Up @@ -611,6 +612,31 @@ def test_startup_banner_hides_control_and_turn_credentials(
assert "turnCredential=<REDACTED>" in out


def test_oob_progress_logs_at_info(
monkeypatch: pytest.MonkeyPatch, capsys, caplog
) -> None:
"""With logging on, progress is an INFO record on the module logger, not a print."""
monkeypatch.delenv("ISAACCAPTURE_LOGGING", raising=False)
with caplog.at_level(logging.INFO, logger="isaaccapture.cloudxr.oob_teleop_env"):
oob_teleop_env_under_test.oob_progress("setup-oob", "teleop tab found")
[record] = caplog.records
assert record.name == "isaaccapture.cloudxr.oob_teleop_env"
assert record.levelno == logging.INFO
assert record.getMessage() == "[setup-oob] teleop tab found"
assert capsys.readouterr().err == ""


def test_oob_progress_prints_when_logging_is_off(
monkeypatch: pytest.MonkeyPatch, capsys, caplog
) -> None:
"""``ISAACCAPTURE_LOGGING=off`` keeps the cyan stderr line and logs nothing."""
monkeypatch.setenv("ISAACCAPTURE_LOGGING", "off")
with caplog.at_level(logging.INFO, logger="isaaccapture.cloudxr.oob_teleop_env"):
oob_teleop_env_under_test.oob_progress("setup-oob", "teleop tab found")
assert capsys.readouterr().err == "\033[36m[setup-oob]\033[0m teleop tab found\n"
assert not caplog.records


# Host preflight (H5) ------------------------------------------------------


Expand Down
Loading
Loading