Skip to content

Commit c40fdcb

Browse files
authored
Merge pull request #7827 from loopx-project/codex/health-observation-snapshot-20261101
fix(chat): bound quota-backlog health observations
2 parents 38fcfec + ff9da09 commit c40fdcb

3 files changed

Lines changed: 123 additions & 10 deletions

File tree

‎loopx/capabilities/native_chat/external_conversations.py‎

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -88,29 +88,53 @@ def _context_recovery_plan(self, row: dict[str, Any], *,
8888
})
8989

9090
def _source_stop_observed(self, row: dict[str, Any], turn: dict[str, Any],
91-
events: list[dict[str, Any]]) -> bool:
91+
events: list[dict[str, Any]], *,
92+
source_controls: Callable[[], list[dict[str, Any]]] | None = None) -> bool:
9293
"""Read the same durable source controls for recovery and display."""
9394
return (any(event.get("kind") in {"turn.interrupt_requested", "turn.interrupted"} for event in events)
9495
or (turn.get("status") in {"failed", "completed"} and any(
9596
item.get("binding_id") == row["binding_id"] and item.get("source") == row["source"]
9697
and item.get("command") in {"stop", "new"}
9798
and datetime.fromisoformat(item["created_at"]) >= datetime.fromisoformat(row["created_at"])
98-
for item in self.pending())))
99+
for item in (source_controls or self.pending)())))
99100

100-
def quota_backlog_pending(self, row: dict[str, Any]) -> bool:
101+
def quota_backlog_observer(self) -> Callable[[dict[str, Any]], bool]:
102+
"""Share lazy journal IO within one read-only observation, never admission."""
103+
controls: list[dict[str, Any]] | None = None
104+
105+
def read_controls() -> list[dict[str, Any]]:
106+
nonlocal controls
107+
if controls is None:
108+
controls = self.pending()
109+
return controls
110+
111+
return lambda row: self.quota_backlog_pending(row, source_controls=read_controls)
112+
113+
def quota_backlog_pending(self, row: dict[str, Any], *,
114+
source_controls: Callable[[], list[dict[str, Any]]] | None = None) -> bool:
101115
"""Project existing work without resolving providers or admitting recovery."""
102116
from ...control_plane.effect_runtime import effect_runtime_result
103117

104118
store = self.controller.store
105119
session = store.load_session(row["session_id"])
106120
turn = store.load_turn(row["session_id"], row["turn_id"])
107-
events = store.events_after(row["session_id"], row["turn_id"], None) if turn else []
108-
return effect_runtime_result("collaboration.conversation.request", {
121+
observation = {"turn": turn, "stop_observed": False}
122+
params = {
109123
"request": {key: value for key, value in row.items() if key != "attachments"},
110124
"current_session": session,
111-
"pending_observation": {"turn": turn,
112-
"stop_observed": self._source_stop_observed(row, turn, events) if turn else False},
113-
}).get("pending") is True
125+
"pending_observation": observation,
126+
}
127+
# The typed owner first excludes work that is not quota backlog even
128+
# without a stop. Stop evidence can only remove a pending request;
129+
# never expose a positive result before reading those native facts.
130+
if effect_runtime_result("collaboration.conversation.request", params).get("pending") is not True:
131+
return False
132+
assert turn is not None # Required by the typed pending projection.
133+
events = store.events_after(row["session_id"], row["turn_id"], None)
134+
if self._source_stop_observed(row, turn, events, source_controls=source_controls):
135+
observation["stop_observed"] = True
136+
return effect_runtime_result("collaboration.conversation.request", params).get("pending") is True
137+
return True
114138

115139
def context_recovery_candidate(self, request_ref: str) -> bool:
116140
"""Discovery only; every authority and native fact is re-read at admission."""

‎loopx/extensions/lark/private_conversations.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from ...capabilities.native_chat.external_conversations import ChatExternalConversations
1919
from ...chat_store import TERMINAL_TURN_STATES, _atomic_write_json, _read_json
2020
from ...control_plane.collaboration import conversation_reply_context
21+
from ...control_plane.effect_runtime import effect_runtime_request_scope
2122
from ...file_lock import exclusive_file_lock
2223
from ...presentation.markdown import markdown_scalar
2324
from ...presentation.renderers.conversation_status_markdown import render_conversation_status
@@ -219,8 +220,10 @@ def profiles(self) -> dict[str, dict[str, str]]:
219220
"binding_id": row["binding_id"], "consumer_ref": hashlib.sha256(app_id.encode("utf-8")).hexdigest()[:32]}
220221
return profiles
221222

223+
@effect_runtime_request_scope()
222224
def health(self) -> dict[str, dict[str, int]]:
223225
health: dict[str, dict[str, int]] = {}
226+
quota_pending = self.core.quota_backlog_observer()
224227
for path in [*self.root.glob("*.json"), *self.root.glob("web-projections/*.json")]:
225228
row = _read_json(path)
226229
entry = health.setdefault(row["binding_id"], {"pending_count": 0, "recovery_count": 0})
@@ -235,7 +238,7 @@ def health(self) -> dict[str, dict[str, int]]:
235238
if (native.get("status") == "accepted" and native.get("command") is None
236239
and all(native.get(key) == row.get(key) for key in
237240
("binding_id", "source", "session_id", "turn_id"))):
238-
pending = self.core.quota_backlog_pending(native)
241+
pending = quota_pending(native)
239242
if pending:
240243
entry["pending_count"] += 1
241244
entry["recovery_count"] += int(any(phase.get("started") and not phase.get("verified")

‎tests/test_lark_private_conversations.py‎

Lines changed: 87 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,7 @@ def connect_group(fixture):
242242

243243

244244
@pytest.mark.parametrize("info, pending", [("usageLimitExceeded", 1), ("cyberPolicy", 0), (None, 0)])
245-
def test_delivered_quota_failure_stays_pending_across_restart_without_replaying(ordinary, info, pending): # noqa: F811
245+
def test_delivered_quota_failure_stays_pending_across_restart_without_replaying(ordinary, monkeypatch, info, pending): # noqa: F811
246246
store, runtime, provider, transport = connect(ordinary)
247247
fake = ordinary[-2]
248248
fake.write_text(fake.read_text().replace('"codexErrorInfo": "cyberPolicy"',
@@ -259,6 +259,11 @@ def test_delivered_quota_failure_stays_pending_across_restart_without_replaying(
259259
transport.reconcile()
260260
saved = json.loads((transport.root / f"{row['request_ref']}.json").read_text())
261261
assert saved["status"] == "delivered" and saved["deliveries"]["terminal"]["verified"]
262+
if not pending:
263+
# The typed owner can exclude completed/non-quota work without
264+
# scanning unrelated events or source-control journals.
265+
monkeypatch.setattr(store, "events_after", lambda *_: pytest.fail("not quota backlog"))
266+
monkeypatch.setattr(transport.core, "pending", lambda: pytest.fail("not quota backlog"))
262267
assert transport.health()[row["binding_id"]]["pending_count"] == pending
263268
before = ordinary[4].read_bytes(), list(provider.writes)
264269
restarted = LarkPrivateConversations(controller=runtime, runtime_root=transport.runtime_root,
@@ -270,6 +275,87 @@ def test_delivered_quota_failure_stays_pending_across_restart_without_replaying(
270275
runtime.close()
271276

272277

278+
@pytest.mark.parametrize("command", ["stop", "new"])
279+
def test_health_reads_source_controls_once_and_keeps_fresh_recovery(ordinary, monkeypatch, command): # noqa: F811
280+
from loopx.capabilities.native_chat import external_conversations
281+
from loopx.control_plane import effect_runtime
282+
283+
store, runtime, provider, transport = connect(ordinary)
284+
fake = ordinary[-2]
285+
fake.write_text(fake.read_text().replace('"codexErrorInfo": "cyberPolicy"',
286+
'"codexErrorInfo": "usageLimitExceeded"').replace(
287+
' print(json.dumps({"method": "item/agentMessage/delta", "params": {"threadId": "durable-thread", "turnId": active_turn, "delta": "Partial answer."}}), flush=True)\n', ''))
288+
try:
289+
binding = transport._binding("notes-app")
290+
rows = []
291+
for index in range(8):
292+
event = provider.event("notes-app", f"health-quota-{index}", "typed terminal failure")
293+
transport.admit("notes-app", event)
294+
row = transport.core.read_request(identity_ref(binding["provider_ref"], event["message_id"]))
295+
turn = runtime.wait_for_turn(session_id=row["session_id"], turn_id=row["turn_id"], timeout_sec=10)
296+
assert turn["status"] == "failed" and turn["error_code"] == "usage_limit_exceeded"
297+
transport.reconcile()
298+
rows.append(row)
299+
300+
def durable_state():
301+
paths = {path for root in (transport.root, transport.core.root) for path in root.glob("*.json")}
302+
paths.update(path for path in store.sessions_root.rglob("*") if path.suffix in {".json", ".jsonl"})
303+
return {path: path.read_bytes() for path in paths}
304+
305+
before = durable_state()
306+
effects = ordinary[4].read_bytes(), list(provider.writes)
307+
reads = []
308+
revisions = []
309+
read_json = external_conversations._read_json
310+
fingerprint = effect_runtime._runtime_fingerprint
311+
312+
def counted_read(path):
313+
if path.parent == transport.core.root and re.fullmatch(r"[a-f0-9]{24}", path.stem):
314+
reads.append(path)
315+
return read_json(path)
316+
317+
monkeypatch.setattr(external_conversations, "_read_json", counted_read)
318+
def counted_revision():
319+
revisions.append(True)
320+
return fingerprint()
321+
322+
monkeypatch.setattr(effect_runtime, "_runtime_fingerprint", counted_revision)
323+
expected = {"pending_count": len(rows), "recovery_count": 0}
324+
assert transport.health()[binding["binding_id"]] == expected
325+
# Correlation reads plus one complete control observation, rather than
326+
# a complete journal scan for each historical terminal request.
327+
assert len(reads) <= 3 * len(rows)
328+
assert len(revisions) <= 1 # One managed runtime revision for the health request.
329+
assert durable_state() == before
330+
assert (ordinary[4].read_bytes(), provider.writes) == effects
331+
332+
# Durable control observations must keep all three existing fences.
333+
# These synthetic journal records are inputs, not new admissions.
334+
for index, change in enumerate([
335+
{"binding_id": "f" * 24},
336+
{"source": dict(rows[0]["source"], source_ref="e" * 24)},
337+
{"created_at": "2000-01-01T00:00:00+00:00"},
338+
]):
339+
control_row = dict(rows[0], command=command, **change)
340+
(transport.core.root / f"{index + 1:024x}.json").write_text(json.dumps(control_row))
341+
# The journal reader already tolerates damaged JSON and ignores
342+
# non-request metadata; observation reuse must retain that behavior.
343+
(transport.core.root / ("d" * 24 + ".json")).write_text("{")
344+
(transport.core.root / "metadata.json").write_text(json.dumps(dict(rows[0], command=command)))
345+
assert transport.health()[binding["binding_id"]] == expected
346+
assert all(transport.core.quota_backlog_pending(row) for row in rows)
347+
348+
# An actual later control must reach both a new health observation and
349+
# the default recovery check, independently of the previous snapshot.
350+
control = provider.event("notes-app", "health-control", f"/{command}")
351+
transport.admit("notes-app", control)
352+
assert transport.health()[binding["binding_id"]]["pending_count"] == 1 # undelivered control
353+
assert all(not transport.core.quota_backlog_pending(row) for row in rows)
354+
assert all(not transport.core.context_recovery_candidate(row["request_ref"]) for row in rows)
355+
finally:
356+
runtime.close()
357+
358+
273359
@pytest.mark.parametrize("fault", [None, "changed-source", "revoked", "stop", "unknown-native",
274360
"background", "later-user", "reservation-crash", "post-enqueue-crash", "queued-newer", "queued-input", "private-path", "still-incomplete"])
275361
def test_original_worker_continues_explicit_unfinished_project_work_once_after_restart(ordinary, monkeypatch, fault): # noqa: F811

0 commit comments

Comments
 (0)