fix(native-accounts): bound graceful-to-force App handoff

.github/workflows/package-smoke.yml

@@ -42,6 +42,7 @@ jobs:
       - name: Validate recovery and existing operator contracts
         run: |
           python -m compileall -q packages/loopx-codex-provider-routing/src
+          python packages/loopx-codex-provider-routing/smoke/native_host_smoke.py
           python packages/loopx-codex-provider-routing/smoke/codex_provider_routing_smoke.py
           python packages/loopx-codex-provider-routing/smoke/recovery_contracts_smoke.py
           python packages/loopx-codex-provider-routing/smoke/operator_smoke.py

packages/loopx-codex-provider-routing/NATIVE.md

@@ -19,10 +19,13 @@ windows, waiting cause and recovery. Tokens and identity claims stay private.
 restart permission, the provider requests a normal AppKit quit of only the
 inspected GPT instance, waits for exit and native idle, saves/switches the login,
 then reopens that same route. It never addresses a shared bundle id or quits a
-different route. A refused quit, timeout, new process owner or remaining CLI/file
-owner leaves credentials unchanged and pauses automatic retries until a check.
+different route. With the separate force permission below, a refused or timed-out
+normal quit can escalate against that same inspected instance. Otherwise those
+failures, or remaining CLI/file owners, leave credentials unchanged and pause
+automatic shutdowns until a check. Changed process ownership leaves credentials
+unchanged and consumes the bounded retry budget.
 Without this additional permission it continues to wait for closed clients.
-It does not implement hot login replacement, force termination, job replay or
+It does not implement hot login replacement, job replay or
 resumption of an interrupted turn. If every
 account is unavailable, it waits for the observed reset and checks again.
 
@@ -52,7 +55,11 @@ and reopen the GPT instance. Neither permission is enabled by installation.
 When enabled, normal refusal or timeout automatically force-quits only the
 observed GPT launch, then continues switching and reopening without another
 confirmation. The PID, executable, launch date, selection and quota freshness
-must still match. Other CLI/file owners remain holds. Force quit can interrupt
+must still match. The launch probe must explicitly contain `launch_time`: a
+positive finite timestamp retains the launch-date fence; explicit `null` from
+Launch Services uses the inspected PID and executable instead. Missing fields,
+invalid types and non-finite timestamps cannot authorize force quit. A successful
+normal quit never escalates. Other CLI/file owners remain holds. Force quit can interrupt
 work in that instance; it does not authorize replay of submitted jobs.
 An operator-owned service independent of the App can run the same observer:
 
@@ -112,9 +119,12 @@ are distinct. The observer never buys credits or uses reset coupons.
 Only fresh active-account exhaustion starts automatic selection. Destinations
 must be allowlisted and freshly available. Selection follows the configured
 ring with a five-minute cooldown. An external native/manual choice wins and
-starts a cooldown. Busy clients defer; refused/changed clients and remaining
-file owners pause further shutdowns. Other repeated failures stop after three
-attempts until an explicit check/reconnect. Resets require new observations.
+starts a cooldown. Busy clients defer; refused/timed-out clients and remaining
+file owners pause further shutdowns. Changed clients and other repeated failures
+stop after three attempts until an explicit check/reconnect. Successful quota
+reads do not reset that budget, which survives observer restart. A completed
+switch, explicit check/reconfiguration or later native choice can reset it.
+Quota resets require new observations.
 A paused observer still reads native selection metadata: a later native/manual
 choice invalidates the old account observations and failure pause, then starts
 the usual cooldown without another shutdown attempt.

packages/loopx-codex-provider-routing/extension.toml

@@ -1,6 +1,6 @@
 schema_version = "loopx_extension_manifest_v0"
 id = "loopx-codex-provider-routing"
-version = "0.12.1"
+version = "0.12.2"
 requires_loopx_api = ">=1,<2"
 permissions = []
 

packages/loopx-codex-provider-routing/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "loopx-codex-provider-routing"
-version = "0.12.1"
+version = "0.12.2"
 description = "Native Codex account slots and exhausted-account handoff for LoopX"
 readme = "README.md"
 requires-python = ">=3.11"

packages/loopx-codex-provider-routing/smoke/native_host_smoke.py

@@ -0,0 +1,96 @@
+"""Offline smoke for native App shutdown during an exhausted-account handoff."""
+
+from __future__ import annotations
+
+import importlib
+import json
+from pathlib import Path
+import sys
+
+PACKAGE_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(PACKAGE_ROOT / "src"))
+
+native_host = importlib.import_module("loopx_codex_provider_routing.native_host")
+
+EXE = "/Applications/ChatGPT.app/Contents/MacOS/ChatGPT"
+
+
+class Apps:
+    def __init__(self):
+        self.pid = 4242
+
+    def executable(self, _bundle):
+        return EXE
+
+    def app_bundle(self, _spec, _role, _home):
+        return Path("/Applications/ChatGPT.app")
+
+    def running(self, _exe, _spec):
+        return self.pid
+
+
+def adapter():
+    value = native_host.NativeHost.__new__(native_host.NativeHost)
+    value.apps = Apps()
+    value.spec = {}
+    value.home = Path("/tmp/native-host-smoke")
+    value.expected = "revision"
+    value.not_after = None
+    value.revision = lambda: "revision"
+    return value
+
+
+def close_with(launch_time, *, force, refuse=False):
+    calls = []
+    host = adapter()
+
+    def fake_run(command, **kwargs):
+        calls.append((list(command), kwargs.get("input", "")))
+        if kwargs["input"] == native_host.FORCE_QUIT or (kwargs["input"] == native_host.QUIT and not refuse):
+            host.apps.pid = None
+
+        class Result:
+            stdout = json.dumps({"launch_time": launch_time, "accepted": not refuse})
+            returncode = 0
+
+        return Result()
+
+    original = native_host.subprocess.run
+    native_host.subprocess.run = fake_run
+    try:
+        closed = host.close_client("revision", force=force)
+    finally:
+        native_host.subprocess.run = original
+    return closed, calls
+
+
+# An App started outside Launch Services reports a nil launchDate. The force
+# path must still terminate it instead of failing the handoff as changed.
+closed, calls = close_with(None, force=True, refuse=True)
+assert closed is True
+assert len(calls) == 3, calls
+assert "launch_time" in calls[0][1] and "terminate" not in calls[0][1], calls[0]
+assert "app.terminate" in calls[1][1] and "forceTerminate" not in calls[1][1], calls[1]
+assert "forceTerminate" in calls[2][1], calls[2]
+assert calls[2][0][-1] == "unknown", calls[2][0]
+
+# A registered App keeps the launch-time fence.
+closed, calls = close_with(123.0, force=True, refuse=True)
+assert closed is True
+assert len(calls) == 3, calls
+assert "forceTerminate" in calls[2][1], calls[2]
+assert calls[2][0][-1] == "123.0", calls[2][0]
+
+# Force permission does not bypass a successful normal quit.
+closed, calls = close_with(123.0, force=True)
+assert closed is True
+assert len(calls) == 2, calls
+assert "app.terminate" in calls[1][1] and "forceTerminate" not in calls[1][1], calls[1]
+
+# The default path still asks the App to quit normally.
+closed, calls = close_with(None, force=False)
+assert closed is True
+assert len(calls) == 1, calls
+assert "terminate" in calls[0][1] and "forceTerminate" not in calls[0][1], calls[0]
+
+print("native host force handoff smoke passed")

packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native.py

@@ -213,6 +213,10 @@ def tick(self, *, refresh: bool = False, manual_slot: str | None = None,
                 return self._projection(state)
             if not refresh and not manual_slot and now < (state.get("retry_at") or 0):
                 return self._projection(state)
+            if refresh or manual_slot:
+                # A deliberate check/choice starts a new retry budget; healthy
+                # quota alone does not prove that a handoff succeeded.
+                state["failure_count"] = 0
             if "adapter" not in state:
                 # An explicit check/manual choice can connect existing slots
                 # without turning automatic effects on first.
@@ -290,7 +294,7 @@ def tick(self, *, refresh: bool = False, manual_slot: str | None = None,
                                    **({"manual_slot": manual_slot} if manual_slot else {})})
                 state.update(observations=observed["observations"], accounts=decision["accounts"],
                              last_checked_at=self.clock(), action=decision["action"], target=decision["target"],
-                             retry_at=decision["retry_at"] or self.clock() + 60, code=None, failure_count=0)
+                             retry_at=decision["retry_at"] or self.clock() + 60, code=None)
                 if decision["action"] == "switch":
                     if not manual_slot and not self.switch_allowed():
                         raise HostError("observer_stopped")
@@ -329,7 +333,7 @@ def tick(self, *, refresh: bool = False, manual_slot: str | None = None,
                 # Unrelated/remaining owners still require a safe-boundary wait.
                 # Other repeated failures stop automatic attempts until Refresh.
                 failures = state.get("failure_count", 0) + (code not in ("waiting_for_idle", "handoff_pending"))
-                if code in ("other_clients_busy", "client_quit_refused", "client_quit_timeout", "client_changed"):
+                if code in ("other_clients_busy", "client_quit_refused", "client_quit_timeout"):
                     failures = 3  # Do not repeatedly close/reopen a refused client.
                 state.update(action=code if code in ("waiting_for_idle", "selection_changed", "adapter_changed", "client_quit_refused", "client_quit_timeout", "client_changed", "handoff_pending") else "unavailable",
                              code=code, retry_at=self.clock() + (5 if code == "handoff_pending" else min(300, 60 * 2 ** min(failures, 3))), failure_count=failures)

packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native_host.py

@@ -9,6 +9,7 @@
 import hashlib
 import importlib
 import json
+import math
 import os
 from pathlib import Path
 import selectors
@@ -45,12 +46,20 @@ def __init__(self, code: str):
   if (!app || !app.executableURL || !app.isFinishedLaunching) throw Error("Client startup is still incomplete");
   if (ObjC.unwrap(app.executableURL.path) !== argv[1]) throw Error("Client executable changed");
   var started = Number(app.launchDate.timeIntervalSince1970);
+  if (argv.length > 2 && argv[2] !== "unknown" && started !== Number(argv[2])) throw Error("Client launch changed");
   return JSON.stringify({launch_time: started, accepted: Boolean(app.terminate)});
 }'''
 
 FORCE_QUIT = QUIT.replace(
-    'var started = Number(app.launchDate.timeIntervalSince1970);\n  return JSON.stringify({launch_time: started, accepted: Boolean(app.terminate)});',
-    'if (Number(app.launchDate.timeIntervalSince1970) !== Number(argv[2])) throw Error("Client launch changed");\n  if (!app.forceTerminate) throw Error("Client force quit failed");',
+    'return JSON.stringify({launch_time: started, accepted: Boolean(app.terminate)});',
+    'if (!app.forceTerminate) throw Error("Client force quit failed");',
+)
+
+# Freeze launch identity before graceful quit so a refused/timed-out quit can
+# escalate only against that same instance, without another confirmation.
+PROBE = QUIT.replace(
+    'return JSON.stringify({launch_time: started, accepted: Boolean(app.terminate)});',
+    'return JSON.stringify({launch_time: started});',
 )
 
 
@@ -129,48 +138,57 @@ def close_client(self, expected: str, *, timeout: float = 25, force: bool = Fals
             raise HostError("selection_changed")
         if pid is None:
             return False
-        # Recheck exact ownership immediately before requesting a normal quit.
+        # Recheck exact ownership immediately before requesting a quit.
         if self.apps.running(exe, self.spec) != pid:
             raise HostError("client_changed")
-        try:
-            result = subprocess.run(["/usr/bin/osascript", "-l", "JavaScript", "-", str(pid), str(exe)],
-                                    input=QUIT, text=True, check=True, capture_output=True, timeout=10)
-        except (OSError, subprocess.SubprocessError):
-            raise HostError("client_quit_refused") from None
-        launch_time = None
-        accepted = True
-        if force or getattr(result, "stdout", ""):
+        if force:
             try:
+                result = subprocess.run(["/usr/bin/osascript", "-l", "JavaScript", "-", str(pid), str(exe)],
+                                        input=PROBE, text=True, check=True, capture_output=True, timeout=10)
                 receipt = json.loads(result.stdout)
-                launch_time, accepted = receipt["launch_time"], receipt["accepted"]
-                if (type(accepted) is not bool or force
-                        and (type(launch_time) not in (int, float) or launch_time <= 0)):
+                launch_time = receipt["launch_time"]
+                # Launch Services can report a nil launchDate for an App started
+                # outside the normal Finder/open registration path. Fall back to
+                # the inspected PID + executable identity instead of stalling.
+                if launch_time is not None and (type(launch_time) not in (int, float)
+                                               or not math.isfinite(launch_time) or launch_time <= 0):
                     raise ValueError()
-            except (AttributeError, KeyError, TypeError, ValueError):
-                raise HostError("client_changed") from None
-        if not accepted and not force:
-            raise HostError("client_quit_refused")
-        deadline = time.monotonic() + timeout
-        while accepted and time.monotonic() < deadline:
+            except (OSError, subprocess.SubprocessError, AttributeError, KeyError, TypeError, ValueError, OverflowError):
+                if self.apps.running(exe, self.spec) is None:
+                    return True
+                raise HostError("client_quit_refused") from None
+            if self.revision() != expected:
+                raise HostError("selection_changed")
             current = self.apps.running(exe, self.spec)
             if current is None:
                 return True
             if current != pid:
                 raise HostError("client_changed")
-            time.sleep(0.1)
-        if force:
-            # Force only this observed launch, under the same selection and
-            # fresh-quota fence. A recycled PID or a later manual choice wins.
+            if self.not_after is not None and time.time() >= self.not_after:
+                raise HostError("quota_stale")
+            identity = str(launch_time) if launch_time is not None else "unknown"
+            try:
+                return self._graceful_quit(pid, exe, timeout, identity=identity)
+            except HostError as error:
+                if error.code not in ("client_quit_refused", "client_quit_timeout"):
+                    raise
+            # Opt-in covers fallback only; a quit dialog must not bypass a
+            # later selection/process/quota change before force termination.
             if self.revision() != expected:
                 raise HostError("selection_changed")
-            if self.apps.running(exe, self.spec) != pid:
+            current = self.apps.running(exe, self.spec)
+            if current is None:
+                return True
+            if current != pid:
                 raise HostError("client_changed")
             if self.not_after is not None and time.time() >= self.not_after:
                 raise HostError("quota_stale")
             try:
-                subprocess.run(["/usr/bin/osascript", "-l", "JavaScript", "-", str(pid), str(exe), str(launch_time)],
+                subprocess.run(["/usr/bin/osascript", "-l", "JavaScript", "-", str(pid), str(exe), identity],
                                input=FORCE_QUIT, text=True, check=True, capture_output=True, timeout=10)
             except (OSError, subprocess.SubprocessError):
+                if self.apps.running(exe, self.spec) is None:
+                    return True
                 raise HostError("client_quit_refused") from None
             deadline = time.monotonic() + 10
             while time.monotonic() < deadline:
@@ -180,6 +198,37 @@ def close_client(self, expected: str, *, timeout: float = 25, force: bool = Fals
                 if current != pid:
                     raise HostError("client_changed")
                 time.sleep(0.1)
+            raise HostError("client_quit_timeout")
+        return self._graceful_quit(pid, exe, timeout)
+
+    def _graceful_quit(self, pid: int, exe: Path, timeout: float, *, identity: str | None = None) -> bool:
+        command = ["/usr/bin/osascript", "-l", "JavaScript", "-", str(pid), str(exe)]
+        if identity is not None:
+            command.append(identity)
+        try:
+            result = subprocess.run(command,
+                                    input=QUIT, text=True, check=True, capture_output=True, timeout=10)
+        except (OSError, subprocess.SubprocessError):
+            raise HostError("client_quit_refused") from None
+        accepted = True
+        if getattr(result, "stdout", ""):
+            try:
+                receipt = json.loads(result.stdout)
+                accepted = receipt["accepted"]
+                if type(accepted) is not bool:
+                    raise ValueError()
+            except (AttributeError, KeyError, TypeError, ValueError):
+                raise HostError("client_changed") from None
+        if not accepted:
+            raise HostError("client_quit_refused")
+        deadline = time.monotonic() + timeout
+        while accepted and time.monotonic() < deadline:
+            current = self.apps.running(exe, self.spec)
+            if current is None:
+                return True
+            if current != pid:
+                raise HostError("client_changed")
+            time.sleep(0.1)
         raise HostError("client_quit_timeout")
 
     def reopen(self, expected: str) -> dict:
@@ -220,11 +269,11 @@ def switch(self, slot: str, expected: str, *, reopen: bool, restart_clients: boo
                             raise HostError("other_clients_busy") from None
                         time.sleep(0.2)
             result = self.manager.switch(slot)
-        except Exception as error:
+        except Exception:
             # A refused/failed transaction must not unnecessarily leave a
             # normally closed source App down. Native preflight blocks opening
             # an incomplete credential journal; a later manual choice wins.
-            if restart_clients and before.get("client_running") and getattr(error, "code", None) != "client_changed":
+            if restart_clients and before.get("client_running"):
                 try:
                     if self.status().get("client_running") is False:
                         self.reopen(expected)