Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions docs/rest-api.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,32 @@ curl -sS http://127.0.0.1:20050/health
Node status: public host, miner info, hardware descriptor, peer count, latest
block index, and uptime.

Submission liveness fields (see gh-18 / gh-20 / gh-27). A node can report
`is_mining: true` while landing zero extrinsics; these fields make that
visible without waiting for `proofs_won` to stall:

| Field | Type | Meaning |
|---|---|---|
| `is_mining` | bool | The controller snapshot was refreshed recently, i.e. workers are producing solutions. Says nothing about whether submits land. |
| `last_successful_submission` | ISO-8601 UTC string or `null` | Time of the last **accepted** submission (proof or mempool solution). `null` until the first one lands. Updated only on accepted submits, never on mined-but-unsubmitted solutions. Stale or `null` while `is_mining` is `true` means the submit path is broken. |
| `last_successful_submission_epoch` | number or `null` | Same instant as epoch seconds, for consumers that compute staleness arithmetically. |
| `consecutive_submit_failures` | int or `null` | Failed submits since the last accepted one; reset to 0 on success. |
| `runtime_incompatible` | string or `null` | Set when this build is too old for the chain runtime (submits will keep failing until upgraded). |

```json
{
"success": true,
"data": {
"is_mining": true,
"last_successful_submission": "2026-09-07T18:42:11.503219+00:00",
"last_successful_submission_epoch": 1788806531.503219,
"consecutive_submit_failures": 0,
"runtime_incompatible": null,
"...": "..."
}
}
```

### GET /api/v1/system

Hardware survey plus whitelisted config (same `descriptor` block returned by
Expand Down
36 changes: 29 additions & 7 deletions substrate/telemetry_process.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import multiprocessing.synchronize
import signal
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Optional, Tuple

Expand Down Expand Up @@ -480,6 +481,22 @@ async def _handle_miner_survey(request: web.Request) -> web.Response:
return _success(survey)


def _epoch_to_iso(ts: Any) -> Optional[str]:
"""Render an epoch-seconds value as an ISO-8601 UTC string, or None.

The controller snapshot stores ``last_successful_submission`` as a
``time.time()`` float; gh-27 asks for an ISO timestamp on the status
endpoint. Anything that is not a finite number maps to None so a
corrupt or missing value never breaks the whole status response.
"""
if isinstance(ts, bool) or not isinstance(ts, (int, float)):
return None
try:
return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat()
except (OverflowError, OSError, ValueError):
return None


async def _handle_status(request: web.Request) -> web.Response:
"""Aggregate chain head + miner identity status.

Expand Down Expand Up @@ -528,19 +545,24 @@ async def _handle_status(request: web.Request) -> web.Response:
type(exc).__name__, exc,
)

last_submit_epoch = snapshot.get("controller", {}).get(
"last_successful_submission"
)

return _success(
{
"ss58_address": snapshot.get("ss58_address"),
"account_id_hex": account_hex,
"node_id": snapshot.get("node_id"),
"is_mining": is_mining,
# Wall-clock (epoch seconds) of the last landed submit, or None
# if the node has never landed one. A node reporting is_mining
# True with a stale/None value here is mining but not winning
# (QUI-829 / gh-18) — the one field that makes that self-evident.
"last_successful_submission": snapshot.get("controller", {}).get(
"last_successful_submission"
),
# ISO-8601 UTC timestamp of the last *accepted* submit, or None
# if the node has never landed one (gh-27). A node reporting
# is_mining True with a stale/None value here is mining but not
# winning (QUI-829 / gh-18) — the one field that makes that
# self-evident. The raw epoch value is kept alongside for
# consumers that compute staleness arithmetically.
"last_successful_submission": _epoch_to_iso(last_submit_epoch),
"last_successful_submission_epoch": last_submit_epoch,
# Consecutive failed submits since the last landed proof, and a
# reason string when this build is too old for the chain runtime.
# A large counter or a non-null reason means "mining but landing
Expand Down
78 changes: 78 additions & 0 deletions tests/test_telemetry_process.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,84 @@ def test_telemetry_status_passes_through_sync_state(tmp_path: Path):
proc.join()


def test_epoch_to_iso_renders_utc_or_none():
"""Epoch floats become ISO-8601 UTC; missing/invalid values become None."""
from substrate.telemetry_process import _epoch_to_iso

assert _epoch_to_iso(0) == "1970-01-01T00:00:00+00:00"
assert _epoch_to_iso(1_700_000_000.5) == "2023-11-14T22:13:20.500000+00:00"
assert _epoch_to_iso(None) is None
assert _epoch_to_iso("1700000000") is None
assert _epoch_to_iso(True) is None
assert _epoch_to_iso(float("nan")) is None
assert _epoch_to_iso(1e20) is None


def test_telemetry_status_reports_last_successful_submission_as_iso(tmp_path: Path):
"""/api/v1/status renders last_successful_submission as ISO-8601 (gh-27).

The controller snapshot carries the raw ``time.time()`` value; the
endpoint must expose it as an ISO timestamp (None before the first
accepted submission) and keep the epoch alongside.
"""
from substrate.telemetry_process import telemetry_main

epoch = 1_700_000_000.0
stats_path = tmp_path / "telemetry-stats.json"
stats_path.write_text(json.dumps({
"controller": {
"heads_observed": 1,
"last_successful_submission": epoch,
},
}))

port = _free_port()
shutdown_event = mp.Event()
proc = mp.Process(
target=telemetry_main,
kwargs={
"listen_host": "127.0.0.1",
"listen_port": port,
"stats_snapshot_path": str(stats_path),
"validator_urls": ["http://example.invalid"],
"shutdown_event": shutdown_event,
},
)
proc.start()
try:
import urllib.request
deadline = time.time() + 5.0
while time.time() < deadline:
try:
resp = urllib.request.urlopen(
f"http://127.0.0.1:{port}/api/v1/status", timeout=0.5,
).read()
break
except Exception:
time.sleep(0.1)
else:
raise RuntimeError("telemetry process did not start in 5s")

data = json.loads(resp)["data"]
assert data["last_successful_submission"] == "2023-11-14T22:13:20+00:00"
assert data["last_successful_submission_epoch"] == epoch

# Before the first accepted submission both fields are null.
stats_path.write_text(json.dumps({"controller": {"heads_observed": 1}}))
resp = urllib.request.urlopen(
f"http://127.0.0.1:{port}/api/v1/status", timeout=2,
).read()
data = json.loads(resp)["data"]
assert data["last_successful_submission"] is None
assert data["last_successful_submission_epoch"] is None
finally:
shutdown_event.set()
proc.join(timeout=5)
if proc.is_alive():
proc.terminate()
proc.join()


def test_telemetry_process_returns_503_when_snapshot_missing(tmp_path: Path):
"""If the snapshot file doesn't exist yet, /api/v1/stats returns 503."""
from substrate.telemetry_process import telemetry_main
Expand Down