@@ -242,7 +242,7 @@ def connect_group(fixture):
242242
243243
244244@pytest .mark .parametrize ("info, pending" , [("usageLimitExceeded" , 1 ), ("cyberPolicy" , 0 ), (None , 0 )])
245- def test_delivered_quota_failure_stays_pending_across_restart_without_replaying (ordinary , info , pending ): # noqa: F811
245+ def test_delivered_quota_failure_stays_pending_across_restart_without_replaying (ordinary , monkeypatch , info , pending ): # noqa: F811
246246 store , runtime , provider , transport = connect (ordinary )
247247 fake = ordinary [- 2 ]
248248 fake .write_text (fake .read_text ().replace ('"codexErrorInfo": "cyberPolicy"' ,
@@ -259,6 +259,11 @@ def test_delivered_quota_failure_stays_pending_across_restart_without_replaying(
259259 transport .reconcile ()
260260 saved = json .loads ((transport .root / f"{ row ['request_ref' ]} .json" ).read_text ())
261261 assert saved ["status" ] == "delivered" and saved ["deliveries" ]["terminal" ]["verified" ]
262+ if not pending :
263+ # The typed owner can exclude completed/non-quota work without
264+ # scanning unrelated events or source-control journals.
265+ monkeypatch .setattr (store , "events_after" , lambda * _ : pytest .fail ("not quota backlog" ))
266+ monkeypatch .setattr (transport .core , "pending" , lambda : pytest .fail ("not quota backlog" ))
262267 assert transport .health ()[row ["binding_id" ]]["pending_count" ] == pending
263268 before = ordinary [4 ].read_bytes (), list (provider .writes )
264269 restarted = LarkPrivateConversations (controller = runtime , runtime_root = transport .runtime_root ,
@@ -270,6 +275,87 @@ def test_delivered_quota_failure_stays_pending_across_restart_without_replaying(
270275 runtime .close ()
271276
272277
278+ @pytest .mark .parametrize ("command" , ["stop" , "new" ])
279+ def test_health_reads_source_controls_once_and_keeps_fresh_recovery (ordinary , monkeypatch , command ): # noqa: F811
280+ from loopx .capabilities .native_chat import external_conversations
281+ from loopx .control_plane import effect_runtime
282+
283+ store , runtime , provider , transport = connect (ordinary )
284+ fake = ordinary [- 2 ]
285+ fake .write_text (fake .read_text ().replace ('"codexErrorInfo": "cyberPolicy"' ,
286+ '"codexErrorInfo": "usageLimitExceeded"' ).replace (
287+ ' print(json.dumps({"method": "item/agentMessage/delta", "params": {"threadId": "durable-thread", "turnId": active_turn, "delta": "Partial answer."}}), flush=True)\n ' , '' ))
288+ try :
289+ binding = transport ._binding ("notes-app" )
290+ rows = []
291+ for index in range (8 ):
292+ event = provider .event ("notes-app" , f"health-quota-{ index } " , "typed terminal failure" )
293+ transport .admit ("notes-app" , event )
294+ row = transport .core .read_request (identity_ref (binding ["provider_ref" ], event ["message_id" ]))
295+ turn = runtime .wait_for_turn (session_id = row ["session_id" ], turn_id = row ["turn_id" ], timeout_sec = 10 )
296+ assert turn ["status" ] == "failed" and turn ["error_code" ] == "usage_limit_exceeded"
297+ transport .reconcile ()
298+ rows .append (row )
299+
300+ def durable_state ():
301+ paths = {path for root in (transport .root , transport .core .root ) for path in root .glob ("*.json" )}
302+ paths .update (path for path in store .sessions_root .rglob ("*" ) if path .suffix in {".json" , ".jsonl" })
303+ return {path : path .read_bytes () for path in paths }
304+
305+ before = durable_state ()
306+ effects = ordinary [4 ].read_bytes (), list (provider .writes )
307+ reads = []
308+ revisions = []
309+ read_json = external_conversations ._read_json
310+ fingerprint = effect_runtime ._runtime_fingerprint
311+
312+ def counted_read (path ):
313+ if path .parent == transport .core .root and re .fullmatch (r"[a-f0-9]{24}" , path .stem ):
314+ reads .append (path )
315+ return read_json (path )
316+
317+ monkeypatch .setattr (external_conversations , "_read_json" , counted_read )
318+ def counted_revision ():
319+ revisions .append (True )
320+ return fingerprint ()
321+
322+ monkeypatch .setattr (effect_runtime , "_runtime_fingerprint" , counted_revision )
323+ expected = {"pending_count" : len (rows ), "recovery_count" : 0 }
324+ assert transport .health ()[binding ["binding_id" ]] == expected
325+ # Correlation reads plus one complete control observation, rather than
326+ # a complete journal scan for each historical terminal request.
327+ assert len (reads ) <= 3 * len (rows )
328+ assert len (revisions ) <= 1 # One managed runtime revision for the health request.
329+ assert durable_state () == before
330+ assert (ordinary [4 ].read_bytes (), provider .writes ) == effects
331+
332+ # Durable control observations must keep all three existing fences.
333+ # These synthetic journal records are inputs, not new admissions.
334+ for index , change in enumerate ([
335+ {"binding_id" : "f" * 24 },
336+ {"source" : dict (rows [0 ]["source" ], source_ref = "e" * 24 )},
337+ {"created_at" : "2000-01-01T00:00:00+00:00" },
338+ ]):
339+ control_row = dict (rows [0 ], command = command , ** change )
340+ (transport .core .root / f"{ index + 1 :024x} .json" ).write_text (json .dumps (control_row ))
341+ # The journal reader already tolerates damaged JSON and ignores
342+ # non-request metadata; observation reuse must retain that behavior.
343+ (transport .core .root / ("d" * 24 + ".json" )).write_text ("{" )
344+ (transport .core .root / "metadata.json" ).write_text (json .dumps (dict (rows [0 ], command = command )))
345+ assert transport .health ()[binding ["binding_id" ]] == expected
346+ assert all (transport .core .quota_backlog_pending (row ) for row in rows )
347+
348+ # An actual later control must reach both a new health observation and
349+ # the default recovery check, independently of the previous snapshot.
350+ control = provider .event ("notes-app" , "health-control" , f"/{ command } " )
351+ transport .admit ("notes-app" , control )
352+ assert transport .health ()[binding ["binding_id" ]]["pending_count" ] == 1 # undelivered control
353+ assert all (not transport .core .quota_backlog_pending (row ) for row in rows )
354+ assert all (not transport .core .context_recovery_candidate (row ["request_ref" ]) for row in rows )
355+ finally :
356+ runtime .close ()
357+
358+
273359@pytest .mark .parametrize ("fault" , [None , "changed-source" , "revoked" , "stop" , "unknown-native" ,
274360 "background" , "later-user" , "reservation-crash" , "post-enqueue-crash" , "queued-newer" , "queued-input" , "private-path" , "still-incomplete" ])
275361def test_original_worker_continues_explicit_unfinished_project_work_once_after_restart (ordinary , monkeypatch , fault ): # noqa: F811
0 commit comments