From 732cda427ba6472cebf85ca796df699a8e491dcb Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 15:00:27 +0300 Subject: [PATCH 001/154] Do not leave a background Bash task running past the end of a turn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `claude-acp:*` agent that launches a Bash command with `run_in_background` and never collects it leaves the task pending inside the Claude Code subprocess. Over ACP there is no live turn between user messages, so the bridge holds the completion wake and delivers it prepended to the next `session/prompt` — and the user's new question gets answered only after the previous turn's leftovers, which reads as the chat finishing old work instead of listening. Seen on `brigado` (conversation 82f39c9e1841): one turn launched two background `nc` port scans and failed to collect either, and the two following turns each opened by `cat`-ing one of the orphaned task outputs. There is no ACP primitive to drain the bridge's pending wake queue, so the instruction layer is the only lever. Adds the rule to the shared core rules every agent session is given. Closes CORR-611. --- agents/_defaults/core_rules.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/agents/_defaults/core_rules.md b/agents/_defaults/core_rules.md index 8a4438499..cabd3468b 100644 --- a/agents/_defaults/core_rules.md +++ b/agents/_defaults/core_rules.md @@ -17,6 +17,10 @@ instead of reimplementing what it already does by hand. - **Short tool chains.** 1–5 calls per response or tick. One skill-driven flow beats a long chain of raw calls that reconstructs what the playbook says. +- **Never end a turn with a background task outstanding.** If you launch a Bash + command with `run_in_background`, collect its output before you answer. Prefer + a foreground command with a generous `timeout` — a task that finishes after + your turn ends will interrupt the user's *next* question with stale work. - **Confirm before you move money.** Orders, swaps, LP mutations and anything destructive get confirmed with the user first. The rule is the guard, not the prompt you happen to be in. From ffe9e5af3fd35c15543499c229df4f1d498085e9 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 16:16:47 +0300 Subject: [PATCH 002/154] Tell a bound Agent who it is on the pydantic-ai backend too ACPClient took a system_prompt and put it on the only system-level channel ACP has; PydanticAIClient had no such parameter, so the factory computed the identity header for every bound Agent and then dropped it on the ollama:/lmstudio:/openrouter:/custom@ half of the fleet. Those agents answered as Condor. PydanticAIClient now takes the same keyword and hands it to pydantic-ai as Agent(instructions=...), and build_llm_client forwards it on both branches instead of only the ACP one. Its MCP servers also ask for their own instructions, which pydantic-ai drops by default: that is the second system-level channel the ACP host forwards and this one silently lost, so the condor server's routing rules now reach these models as well. --- condor/acp/pydantic_ai_client.py | 15 +++++- condor/runtime/llm_client.py | 8 +-- tests/test_agent_identity.py | 87 ++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 4 deletions(-) diff --git a/condor/acp/pydantic_ai_client.py b/condor/acp/pydantic_ai_client.py index 6cf587800..33e4d064a 100644 --- a/condor/acp/pydantic_ai_client.py +++ b/condor/acp/pydantic_ai_client.py @@ -461,6 +461,7 @@ def __init__( allowed_tools: ( list[str] | None ) = None, # restrict the agent to these tool names + system_prompt: str = "", ): self.model_name = model self.mcp_server_configs = mcp_servers or [] @@ -468,6 +469,11 @@ def __init__( self.extra_env = extra_env self.base_url = base_url self.api_key = api_key + # Who the model is told it is, delivered at system level as pydantic-ai + # ``instructions``. The twin of ACPClient's ``_meta.systemPrompt.append`` + # (client.py): without it a bound Agent answers as the host instead of + # as itself, and the weaker channels do not fix that (FEAT-025). + self.system_prompt = system_prompt # When set, the agent only sees tools whose name is in this allowlist # (used by delegated domain agents to scope an agent to one domain). self.allowed_tools = set(allowed_tools) if allowed_tools else None @@ -724,6 +730,10 @@ async def start(self) -> None: args=args, env=env, timeout=30, + # pydantic-ai drops a server's ``instructions`` by default; the + # ACP host forwards them, so ask for them here too or the condor + # server's routing rules never reach a pydantic-ai model. + include_instructions=True, ) toolsets.append(mcp_server) @@ -732,7 +742,10 @@ async def start(self) -> None: model = self._build_model() prepare = self._prepare_tools if self.allowed_tools else None self._agent = Agent( - model, toolsets=self._gate_toolsets(toolsets), prepare_tools=prepare + model, + instructions=self.system_prompt or None, + toolsets=self._gate_toolsets(toolsets), + prepare_tools=prepare, ) # Resolve the global semaphore for this server's base URL so all client diff --git a/condor/runtime/llm_client.py b/condor/runtime/llm_client.py index c2adbeb6b..d826148e8 100644 --- a/condor/runtime/llm_client.py +++ b/condor/runtime/llm_client.py @@ -63,9 +63,10 @@ def build_llm_client( ``PYDANTIC_AI_TOOL_FILTER`` env > ``None`` (auto-detect by model size). ``extra_env``, ``system_prompt`` and ``allowed_tools`` are forwarded to - whichever client understands them: the ACP subprocess takes the env and the - system prompt but cannot enforce an allowlist; PydanticAI enforces the - allowlist (and takes the env for its MCP subprocesses). + whichever client understands them. Both clients take the env and the system + prompt — each over its own system-level channel (``_meta.systemPrompt`` for + ACP, ``instructions`` for pydantic-ai), so a bound Agent keeps its identity + on either backend (ARCH-331). Only PydanticAI enforces the tool allowlist. """ if pydantic_ai.is_pydantic_ai_model(agent_key): custom_url, api_key = resolve_custom_endpoint( @@ -85,6 +86,7 @@ def build_llm_client( tool_filter_mode or os.environ.get("PYDANTIC_AI_TOOL_FILTER") or None ), allowed_tools=allowed_tools, + system_prompt=system_prompt, ) # ACP subprocess models: claude-code, gemini, codex. A Claude model can be diff --git a/tests/test_agent_identity.py b/tests/test_agent_identity.py index 89a5f8b50..9824f8d99 100644 --- a/tests/test_agent_identity.py +++ b/tests/test_agent_identity.py @@ -7,6 +7,8 @@ the session's opening context) and both must say the same thing. """ +import asyncio + import pytest from condor.agents import agent as agent_module @@ -196,3 +198,88 @@ def test_session_new_appends_the_system_prompt(): def test_session_new_omits_meta_when_unbound(): """The Condor chat sends exactly what it sends today — no `_meta` at all.""" assert _session_new_params() == {"cwd": "/tmp", "mcpServers": []} + + +# --- The same identity, on the pydantic-ai backend (ARCH-331) -------------- +# +# ACP is only half the fleet: ollama:/lmstudio:/openrouter:/custom@ models run +# in-process through PydanticAIClient, which used to build its Agent with no +# system prompt at all. A bound Agent on those backends was therefore anonymous +# — the identity header the caller had already computed was dropped on the floor +# by the factory. pydantic-ai's system-level channel is `instructions=`. + + +def _make_client(**kwargs): + from condor.acp.pydantic_ai_client import PydanticAIClient + + return PydanticAIClient("openai:gpt-4o", **kwargs) + + +async def _instructions_on_the_wire(client) -> str | None: + """Run one turn against a stub model and report what it was instructed.""" + from pydantic_ai.messages import ModelResponse, TextPart + from pydantic_ai.models.function import FunctionModel + + seen: dict = {} + + def respond(messages, info): + seen["instructions"] = messages[0].instructions + return ModelResponse(parts=[TextPart("ok")]) + + client._build_model = lambda: FunctionModel(respond) + await client.start() + try: + await client._agent.run("who are you?") + finally: + await client.stop() + return seen["instructions"] + + +def test_pydantic_ai_agent_is_instructed_with_the_system_prompt(): + header = identity_header("backpack_mm", "Backpack MM") + client = _make_client(system_prompt=header) + assert client.system_prompt == header + assert asyncio.run(_instructions_on_the_wire(client)) == header + + +def test_pydantic_ai_agent_unbound_carries_no_instructions(): + """The Condor chat is unchanged: an empty prompt must not become a blank one.""" + assert asyncio.run(_instructions_on_the_wire(_make_client())) is None + + +def test_pydantic_ai_mcp_servers_ask_for_their_instructions(): + """The second system-level channel: pydantic-ai drops MCP `instructions` + unless asked, so the condor server's routing rules never reached these + models either.""" + import pydantic_ai.mcp as mcp_module + + class _Stop(Exception): + pass + + seen: dict = {} + + def _record(command, **kwargs): + seen.update(kwargs) + raise _Stop # abort start() before anything is spawned + + original = mcp_module.MCPServerStdio + mcp_module.MCPServerStdio = _record + try: + client = _make_client(mcp_servers=[{"command": "condor-mcp", "args": []}]) + with pytest.raises(_Stop): + asyncio.run(client.start()) + finally: + mcp_module.MCPServerStdio = original + + assert seen["include_instructions"] is True + + +def test_factory_forwards_the_system_prompt_to_both_backends(): + """`build_llm_client` used to hand `system_prompt` to ACP only.""" + from condor.runtime.llm_client import build_llm_client + + header = identity_header("brigado", "Brigado") + assert build_llm_client("ollama:llama3.1", system_prompt=header).system_prompt == ( + header + ) + assert build_llm_client("claude-code", system_prompt=header).system_prompt == header From 60e860ee5248faa4a926cf8186d13cbdee1be0bc Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 16:24:55 +0300 Subject: [PATCH 003/154] Let a reasoning model's thinking reach the screen on the pydantic-ai path The module promises "the same ACPEvent types so TickEngine can consume it identically", but the response fold handled exactly two part types: TextPart and ToolCallPart. The ThinkingPart that reasoning models return -- deepseek-r1 and qwq over ollama, gpt-oss over openrouter -- matched neither branch and was dropped on the floor. Every consumer of that thinking already existed: ThoughtChunk is a member of ACPEvent, the ACP client emits it from agent_thought_chunk, and both the dashboard and Telegram render it. Only the producer was missing, so the thought panel stayed permanently empty for every pydantic-ai model while the identical panel filled up for a Claude ACP agent. Import ThoughtChunk and translate ThinkingPart into it, next to the TextPart branch. A response carrying no thinking streams exactly as it did before. --- condor/acp/pydantic_ai_client.py | 19 +++++- tests/test_pydantic_ai_thought_chunks.py | 86 ++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 tests/test_pydantic_ai_thought_chunks.py diff --git a/condor/acp/pydantic_ai_client.py b/condor/acp/pydantic_ai_client.py index 33e4d064a..261198c27 100644 --- a/condor/acp/pydantic_ai_client.py +++ b/condor/acp/pydantic_ai_client.py @@ -26,6 +26,7 @@ PermissionCallback, PromptDone, TextChunk, + ThoughtChunk, ToolCallEvent, ToolCallUpdate, ) @@ -921,7 +922,12 @@ async def prompt_stream( try: from pydantic_ai.agent import CallToolsNode, ModelRequestNode - from pydantic_ai.messages import TextPart, ToolCallPart, ToolReturnPart + from pydantic_ai.messages import ( + TextPart, + ThinkingPart, + ToolCallPart, + ToolReturnPart, + ) from pydantic_graph import End self._permission_gate.reset() @@ -985,6 +991,17 @@ async def prompt_stream( if isinstance(part, TextPart) and part.content: yield TextChunk(text=part.content) + elif isinstance(part, ThinkingPart) and part.content: + # Reasoning models (deepseek-r1/qwq via ollama, + # gpt-oss via openrouter) return their thinking + # as a third part type. The ACP path already + # translates the same thing from + # ``agent_thought_chunk``; without this branch + # the thinking stream is silently dropped and + # the dashboard/Telegram thought panel stays + # empty for every pydantic-ai model (ARCH-333). + yield ThoughtChunk(text=part.content) + elif isinstance(part, ToolCallPart): tool_id = part.tool_call_id or uuid.uuid4().hex[:12] tool_name = part.tool_name diff --git a/tests/test_pydantic_ai_thought_chunks.py b/tests/test_pydantic_ai_thought_chunks.py new file mode 100644 index 000000000..705a23b7c --- /dev/null +++ b/tests/test_pydantic_ai_thought_chunks.py @@ -0,0 +1,86 @@ +"""A reasoning model's thinking must reach the shared event vocabulary (ARCH-333). + +``PydanticAIClient`` folded a model response into ACPEvents by handling exactly +two part types — ``TextPart`` and ``ToolCallPart`` — so the ``ThinkingPart`` that +reasoning models return (deepseek-r1/qwq via ollama, gpt-oss via openrouter) fell +through both branches and was dropped. The thought panel that the ACP path fills +from ``agent_thought_chunk`` therefore stayed permanently empty for every +pydantic-ai model, even though ``ThoughtChunk`` is already a member of +``ACPEvent`` and is already rendered by both surfaces. + +These tests drive a real ``Agent`` run over a stub model, the same way the +permission-gate tests do, and assert on the events the client actually yields. +""" + +import asyncio + +from pydantic_ai import Agent +from pydantic_ai.messages import ModelResponse, TextPart, ThinkingPart +from pydantic_ai.models.function import AgentInfo, FunctionModel + +from condor.acp.client import TextChunk, ThoughtChunk +from condor.acp.pydantic_ai_client import PydanticAIClient + + +def _client_returning(parts: list) -> PydanticAIClient: + """A started-enough client whose model answers once with ``parts``.""" + + def respond(messages: list, info: AgentInfo) -> ModelResponse: + return ModelResponse(parts=list(parts)) + + client = PydanticAIClient("openai:gpt-4o") + client._agent = Agent(FunctionModel(respond)) + return client + + +def _drive(client: PydanticAIClient) -> list: + async def run() -> list: + return [event async for event in client.prompt_stream("go")] + + return asyncio.run(run()) + + +def test_thinking_part_is_yielded_as_a_thought_chunk(): + """The reasoning content reaches the caller instead of being dropped.""" + client = _client_returning( + [ThinkingPart(content="weighing the two pools"), TextPart("done")] + ) + + events = _drive(client) + + thoughts = [e for e in events if isinstance(e, ThoughtChunk)] + assert thoughts, "ThinkingPart produced no ThoughtChunk" + assert thoughts[0].text == "weighing the two pools" + + +def test_thinking_is_not_confused_with_the_answer(): + """Thinking goes to ThoughtChunk; the answer still goes to TextChunk.""" + client = _client_returning( + [ThinkingPart(content="weighing the two pools"), TextPart("SOL-USDC")] + ) + + events = _drive(client) + + assert [e.text for e in events if isinstance(e, ThoughtChunk)] == [ + "weighing the two pools" + ] + assert "SOL-USDC" in "".join(e.text for e in events if isinstance(e, TextChunk)) + + +def test_empty_thinking_part_yields_nothing(): + """A content-less ThinkingPart must not open an empty thought bubble.""" + client = _client_returning([ThinkingPart(content=""), TextPart("done")]) + + events = _drive(client) + + assert not [e for e in events if isinstance(e, ThoughtChunk)] + + +def test_run_without_thinking_streams_as_before(): + """No ThinkingPart, no behaviour change.""" + client = _client_returning([TextPart("done")]) + + events = _drive(client) + + assert not [e for e in events if isinstance(e, ThoughtChunk)] + assert [e.text for e in events if isinstance(e, TextChunk)] == ["done"] From 8da554b69097eaa0f2e825bb78959ef1ce2c7cfe Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 16:26:46 +0300 Subject: [PATCH 004/154] Tell the delete dialog why a strategy would not delete MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A strategy whose strategy.md still resolves to the shipped library is refused by StrategyStore.delete — an update would only bring it back. That refusal is a ValueError, and delete_strategy never caught it, so it left the route as a 500. The dialog has one canned line for a failed delete, "It may be running", so a stopped strategy reported the one thing it certainly was not and the real reason appeared nowhere. Map the refusal to a 400 carrying the store's own message, the way delete_agent already does for the reserved `condor` agent, and let the three delete dialogs show the server's detail instead of the canned line. A delete() that returns False is now a 500 rather than a {"deleted": true} that removed nothing. --- condor/web/routes/agents.py | 14 ++- .../src/components/agent/AgentStrategies.tsx | 6 +- .../components/agent/StrategyWorkbench.tsx | 6 +- .../agent/workspace/PlaybookView.tsx | 6 +- tests/test_strategy_delete_refusals.py | 93 +++++++++++++++++++ 5 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 tests/test_strategy_delete_refusals.py diff --git a/condor/web/routes/agents.py b/condor/web/routes/agents.py index fd433248c..2020dfaea 100644 --- a/condor/web/routes/agents.py +++ b/condor/web/routes/agents.py @@ -2649,7 +2649,19 @@ async def delete_strategy( status_code=400, detail="Cannot delete a running strategy. Stop all instances first.", ) - _strategy_store().delete(slug, sslug) + try: + removed = _strategy_store().delete(slug, sslug) + except ValueError as exc: + # A strategy whose ``strategy.md`` is still the shipped one is refused + # by the store (layering.stock_delete_error). Unhandled here that + # refusal reached the browser as a 500, which the delete dialog reports + # as "it may be running" — the one thing it certainly was not. + raise HTTPException(status_code=400, detail=str(exc)) from exc + if not removed: + raise HTTPException( + status_code=500, + detail="Could not remove the strategy folder — see the server log.", + ) return {"deleted": True} diff --git a/frontend/src/components/agent/AgentStrategies.tsx b/frontend/src/components/agent/AgentStrategies.tsx index 714e79d16..04e2c4004 100644 --- a/frontend/src/components/agent/AgentStrategies.tsx +++ b/frontend/src/components/agent/AgentStrategies.tsx @@ -207,7 +207,11 @@ export function AgentStrategies({ title="Delete Strategy" isPending={deleteMut.isPending} isError={deleteMut.isError} - errorText="Failed to delete strategy. It may be running." + errorText={ + deleteMut.error instanceof Error + ? deleteMut.error.message + : "Failed to delete strategy. It may be running." + } onConfirm={() => deleteMut.mutate()} onClose={() => setDeleteStrategy(null)} > diff --git a/frontend/src/components/agent/StrategyWorkbench.tsx b/frontend/src/components/agent/StrategyWorkbench.tsx index f4cc71030..3e9b9699d 100644 --- a/frontend/src/components/agent/StrategyWorkbench.tsx +++ b/frontend/src/components/agent/StrategyWorkbench.tsx @@ -504,7 +504,11 @@ export function StrategyWorkbench({ title="Delete Strategy" isPending={deleteMutation.isPending} isError={deleteMutation.isError} - errorText="Failed to delete strategy. It may be running." + errorText={ + deleteMutation.error instanceof Error + ? deleteMutation.error.message + : "Failed to delete strategy. It may be running." + } onConfirm={() => deleteMutation.mutate()} onClose={() => setShowDeleteConfirm(false)} > diff --git a/frontend/src/components/agent/workspace/PlaybookView.tsx b/frontend/src/components/agent/workspace/PlaybookView.tsx index 5aeeafc2d..438d5c9e4 100644 --- a/frontend/src/components/agent/workspace/PlaybookView.tsx +++ b/frontend/src/components/agent/workspace/PlaybookView.tsx @@ -195,7 +195,11 @@ export function PlaybookView({ title="Delete Strategy" isPending={deleteMutation.isPending} isError={deleteMutation.isError} - errorText="Failed to delete strategy. It may be running." + errorText={ + deleteMutation.error instanceof Error + ? deleteMutation.error.message + : "Failed to delete strategy. It may be running." + } onConfirm={() => deleteMutation.mutate()} onClose={() => setShowDeleteConfirm(false)} > diff --git a/tests/test_strategy_delete_refusals.py b/tests/test_strategy_delete_refusals.py new file mode 100644 index 000000000..a02da97d4 --- /dev/null +++ b/tests/test_strategy_delete_refusals.py @@ -0,0 +1,93 @@ +"""What the delete-strategy route says when it refuses (CORR). + +``StrategyStore.delete`` refuses a strategy whose ``strategy.md`` still +resolves to the shipped library — a delete would only be undone by the next +update (``layering.stock_delete_error``). That refusal is a ``ValueError``, and +unhandled it left the route as a 500, so the browser's delete dialog fell back +to its only canned line: "It may be running." A stopped strategy then reported +the one thing it certainly was not, with the real reason nowhere on screen. + +The route now maps the refusal to a 400 carrying the store's own message, the +way ``delete_agent`` already does for the reserved ``condor`` agent. +""" + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +from condor.agents.agent import AgentStore +from condor.agents.strategy import StrategyStore +from condor.web.auth import get_current_user +from condor.web.models import WebUser +from condor.web.routes import agents as routes + +USER = WebUser(id=555, username="u", first_name="U", role="user") + + +@pytest.fixture +def roots(tmp_path, monkeypatch): + """A local root and a stock one, both empty, with an agent in each.""" + local, stock = tmp_path / "local", tmp_path / "stock" + monkeypatch.setenv("CONDOR_AGENTS_ROOT", str(local)) + monkeypatch.setenv("CONDOR_STOCK_AGENTS_ROOT", str(stock)) + AgentStore().create(name="Brigado", description="BRL market making") + return local, stock + + +def _ship(stock, sslug: str, name: str) -> None: + """Put a strategy in the shipped library, the way a release would.""" + home = stock / "brigado" / "strategies" / sslug + home.mkdir(parents=True) + (home / "strategy.md").write_text(f"---\nname: {name}\n---\n\nTick.\n") + + +def _client() -> TestClient: + app = FastAPI() + app.include_router(routes.router) + app.dependency_overrides[get_current_user] = lambda: USER + return TestClient(app) + + +def test_deleting_a_shipped_strategy_is_a_400_that_says_why(roots): + """The refusal reaches the dialog instead of a bare 500.""" + _, stock = roots + _ship(stock, "brl_mm", "BRL MM") + + res = _client().delete("/agents/brigado/strategies/brl_mm") + + assert res.status_code == 400 + detail = res.json()["detail"] + assert "ships with Condor" in detail + assert "running" not in detail.lower(), "the one thing it is not" + + +def test_a_shipped_strategy_with_local_runtime_output_is_still_refused(roots): + """A local ``learnings.md`` beside it is not a fork of the playbook. + + This is the shape that reported "it may be running": the strategy had been + run, so its local home existed, but ``strategy.md`` was still stock. + """ + local, stock = roots + _ship(stock, "brl_mm", "BRL MM") + home = local / "brigado" / "strategies" / "brl_mm" + home.mkdir(parents=True) + (home / "learnings.md").write_text("# Learnings\n") + + res = _client().delete("/agents/brigado/strategies/brl_mm") + + assert res.status_code == 400 + assert home.exists(), "a refused delete removes nothing" + + +def test_a_local_strategy_still_deletes(roots): + """The refusal is scoped to shipped playbooks, not to deletes at large.""" + local, _ = roots + StrategyStore().create(agent_slug="brigado", name="Scalp") + home = local / "brigado" / "strategies" / "scalp" + assert home.exists() + + res = _client().delete("/agents/brigado/strategies/scalp") + + assert res.status_code == 200 + assert res.json() == {"deleted": True} + assert not home.exists() From ecf400e205fdd4cee69148bc7d2d70e8e58e5715 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 16:35:59 +0300 Subject: [PATCH 005/154] Let the prompt stream's hard ceiling come from the timeout policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit prompt_stream carried its own 1860 literal, with a comment claiming it sat "slightly above session-level timeout" — true only at defaults. Raising CONDOR_TIMEOUT_PROMPT_OVERALL moved the session deadline and left the stream cutting the answer at 31 minutes, and the comment became false. The ceiling is now derived by TimeoutPolicy as prompt_overall + 60 and read through the same deferred TIMEOUTS import _settle_previous_turn and abort_prompt already use, so both deadlines move together. The delegate budget test reads the constant instead of regex-scraping the module source. --- condor/acp/client.py | 11 +++++++--- condor/runtime/timeouts.py | 14 +++++++++++++ tests/runtime/test_runtime_state.py | 29 +++++++++++++++++++++++++++ tests/test_delegate_timeout_budget.py | 16 +++++---------- 4 files changed, 56 insertions(+), 14 deletions(-) diff --git a/condor/acp/client.py b/condor/acp/client.py index fb6e153da..a06ecb545 100644 --- a/condor/acp/client.py +++ b/condor/acp/client.py @@ -983,11 +983,16 @@ def _on_response(fut: asyncio.Future) -> None: future.add_done_callback(_on_response) + # Imported here, not at module scope: condor.runtime.events imports + # condor.acp, so a top-level import would close the cycle. + from condor.runtime.timeouts import TIMEOUTS + loop = asyncio.get_event_loop() start_time = loop.time() - max_duration = ( - 1860 # 31 min hard ceiling (slightly above session-level timeout) - ) + # Hard ceiling for this stream, kept slightly above the session-level + # budget by the policy itself so a deployment that raises + # CONDOR_TIMEOUT_PROMPT_OVERALL is not silently cut short here. + max_duration = TIMEOUTS.prompt_hard_stop try: while True: diff --git a/condor/runtime/timeouts.py b/condor/runtime/timeouts.py index 844f90896..61919fe22 100644 --- a/condor/runtime/timeouts.py +++ b/condor/runtime/timeouts.py @@ -61,6 +61,20 @@ class TimeoutPolicy: # 0 disables the sweep. 1 hour. session_idle: int = 3600 + @property + def prompt_hard_stop(self) -> int: + """The ACP stream's own ceiling: one minute above the turn budget. + + ``prompt_overall`` is the deadline the *session* enforces; this is the + backstop under it inside ``ACPClient.prompt_stream``, so a subprocess + that stops answering ends even when nobody is watching the session. + Derived rather than stored so that raising + ``CONDOR_TIMEOUT_PROMPT_OVERALL`` moves both together — a stored copy + is exactly how a ``1860`` literal in the ACP client drifted out of + reach of this policy in the first place. + """ + return self.prompt_overall + 60 + @classmethod def load(cls) -> "TimeoutPolicy": """Build the policy, applying CONDOR_TIMEOUT_* overrides. diff --git a/tests/runtime/test_runtime_state.py b/tests/runtime/test_runtime_state.py index 8fc8befd5..1cd93971b 100644 --- a/tests/runtime/test_runtime_state.py +++ b/tests/runtime/test_runtime_state.py @@ -327,6 +327,35 @@ def test_timeout_bad_override_is_ignored(monkeypatch): assert TimeoutPolicy.load().prompt_overall == 1800 +def test_the_stream_hard_stop_sits_just_above_the_turn_budget(): + """The ACP stream's backstop is a minute past the session's own deadline.""" + assert TimeoutPolicy().prompt_hard_stop == 1860 + assert TimeoutPolicy().prompt_hard_stop == TimeoutPolicy().prompt_overall + 60 + + +def test_the_stream_hard_stop_follows_the_turn_budget_override(monkeypatch): + """A deployment that buys a longer turn must not be cut short by the stream. + + ``prompt_stream`` used to carry its own ``1860`` literal, so raising + ``CONDOR_TIMEOUT_PROMPT_OVERALL`` to an hour moved the session deadline and + left the stream cutting the answer at 31 minutes. + """ + monkeypatch.setenv("CONDOR_TIMEOUT_PROMPT_OVERALL", "3600") + + assert TimeoutPolicy.load().prompt_hard_stop == 3660 + + +def test_prompt_stream_has_no_hardcoded_ceiling(): + """The deadline the stream enforces is the policy's, not a copy of it.""" + import inspect + + from condor.acp.client import ACPClient + + src = inspect.getsource(ACPClient.prompt_stream) + assert "TIMEOUTS.prompt_hard_stop" in src + assert "1860" not in src + + def test_resolve_tick_timeout_precedence(): """Caller override beats strategy config beats the default.""" assert resolve_tick_timeout("loop", caller=42, strategy=99) == 42 diff --git a/tests/test_delegate_timeout_budget.py b/tests/test_delegate_timeout_budget.py index fdb78e59e..df94319c0 100644 --- a/tests/test_delegate_timeout_budget.py +++ b/tests/test_delegate_timeout_budget.py @@ -21,7 +21,6 @@ """ import asyncio -from pathlib import Path from types import SimpleNamespace import pytest @@ -146,18 +145,13 @@ def test_a_budget_past_the_session_ceiling_is_refused_with_the_limit(monkeypatch def test_the_ceiling_stays_under_the_acp_prompt_hard_stop(): """A budget the agent session cannot honour would be a promise, not a knob. - ``ACPClient.prompt_stream`` stops a prompt at its own hard ceiling, so an - outer budget past that only delays the same cut-off. Read from the source - rather than copied, so raising one and not the other fails here. + ``ACPClient.prompt_stream`` stops a prompt at ``TIMEOUTS.prompt_hard_stop``, + so an outer budget past that only delays the same cut-off. Read from the + policy the stream itself reads, so raising one and not the other fails here. """ - import re + from condor.runtime.timeouts import TIMEOUTS - from condor.acp import client as acp_client - - src = Path(acp_client.__file__).read_text() - ceiling = int(re.search(r"max_duration = \(?\s*(\d+)", src).group(1)) - - assert MAX_DELEGATE_TIMEOUT_S <= ceiling + assert MAX_DELEGATE_TIMEOUT_S <= TIMEOUTS.prompt_hard_stop # -- The MCP tool: can a caller reach it at all -- From f69ae474e62fe7d0c3a1aa6963ef11afb98e3019 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 16:43:59 +0300 Subject: [PATCH 006/154] Declare the profile and mute flags once, where both servers read them ARCH-289 moved the profile machinery into mcp_servers/_profiles.py because it was byte-for-byte identical in both servers, but it took only the enforcement half. The argv half stayed written twice: hummingbot_api/settings.py had _parse_tool_profile() and _parse_muted_tools(), and condor/settings.py declared the same two flags on its own parser with the comma contract renamed to _split_names. Three copies of one wire format had to agree, so renaming a flag or widening the separator was a two-file change a reviewer could half-land -- and a half-landed change silently widens or narrows one seat, the exact failure the shared module exists to prevent. parse_profile_flags(default_profile) now holds the parser, the parse_known_args and the comma split, with the import-time timing rationale in its docstring. Each server keeps its own DEFAULT_TOOL_PROFILE and passes it in. The module stays a leaf: argparse is stdlib and it still imports no server. --- mcp_servers/_profiles.py | 31 ++++++++++++++++++++ mcp_servers/condor/settings.py | 17 ++++++----- mcp_servers/hummingbot_api/settings.py | 40 +++----------------------- 3 files changed, 43 insertions(+), 45 deletions(-) diff --git a/mcp_servers/_profiles.py b/mcp_servers/_profiles.py index 648786021..8dcc07b5e 100644 --- a/mcp_servers/_profiles.py +++ b/mcp_servers/_profiles.py @@ -19,6 +19,7 @@ from __future__ import annotations +import argparse from collections.abc import Callable, Iterable, Mapping from typing import TYPE_CHECKING, Any @@ -26,6 +27,36 @@ from mcp.server.fastmcp import FastMCP +def parse_profile_flags(default_profile: str) -> tuple[str, tuple[str, ...]]: + """``(--profile, --mute-tools)`` off argv, read at import. + + Both flags have to be resolved *here* rather than once a server is already + starting its stdio loop: which tools exist is decided when the module + registers them, which is import time, and the mute subtracts from the + profile inside ``register_tools``, which runs at import too. Hence + ``parse_known_args`` — every other flag a spawner passes (``--url``, + ``--server-name``, ``--chat-id``, …) stays inert here, and a run under + pytest, whose argv is the test runner's, resolves the defaults. + + ``default_profile`` is the caller's, not this module's: each server owns its + own ``DEFAULT_TOOL_PROFILE`` and its own rings. An empty or absent + ``--mute-tools`` is the norm — the spawner only puts it on the line when the + operator has actually switched something off — and blanks in it are not + names, so ``"a, b,,c"`` is ``("a", "b", "c")``. + + The single declaration of the wire format both servers and + ``condor.runtime.toolsets`` have to agree on: renaming a flag or widening + the separator contract is one edit here, not a two-file change a reviewer + can half-land. + """ + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--profile", default=default_profile) + parser.add_argument("--mute-tools", default="") + args, _ = parser.parse_known_args() + muted = tuple(n.strip() for n in (args.mute_tools or "").split(",") if n.strip()) + return args.profile, muted + + def make_resolver(namespace: dict[str, Any]) -> Callable[[str], Any]: """A ``_resolve(name)`` that looks tools up in one server's ``globals()``. diff --git a/mcp_servers/condor/settings.py b/mcp_servers/condor/settings.py index c29e55b05..ae178a8dd 100644 --- a/mcp_servers/condor/settings.py +++ b/mcp_servers/condor/settings.py @@ -5,6 +5,8 @@ import os from dataclasses import dataclass +from mcp_servers._profiles import parse_profile_flags + # Imported for its ``load_dotenv()`` side effect as much as for the helper: # ``_parse_settings()`` runs at import, before anything else in the process # pulls this module in, so without it ``.env`` is not yet in ``os.environ`` and @@ -113,11 +115,6 @@ def _resolve_user_id(argv_user_id: int | None) -> int: return 0 -def _split_names(raw: str) -> tuple[str, ...]: - """``"a, b,,c"`` → ``("a", "b", "c")``. Blanks are not names.""" - return tuple(name.strip() for name in (raw or "").split(",") if name.strip()) - - def _parse_settings() -> Settings: parser = argparse.ArgumentParser(add_help=False) parser.add_argument("--chat-id", type=int, default=None) @@ -127,9 +124,11 @@ def _parse_settings() -> Settings: parser.add_argument("--session-key", default=None) parser.add_argument("--delegate-worker", action="store_true", default=False) parser.add_argument("--ask-target", action="store_true", default=False) - parser.add_argument("--profile", default=DEFAULT_TOOL_PROFILE) - parser.add_argument("--mute-tools", default="") args, _ = parser.parse_known_args() + # ``--profile``/``--mute-tools`` are declared once, in the shared leaf, so + # this seat and the hummingbot-api seat cannot drift on the wire format + # ``condor.runtime.toolsets`` writes (ARCH-570). + tool_profile, muted_tools = parse_profile_flags(DEFAULT_TOOL_PROFILE) return Settings( chat_id=( @@ -152,8 +151,8 @@ def _parse_settings() -> Settings: delegate_worker=( args.delegate_worker or os.environ.get("CONDOR_DELEGATE_WORKER", "") == "1" ), - tool_profile=args.profile, - muted_tools=_split_names(args.mute_tools), + tool_profile=tool_profile, + muted_tools=muted_tools, ) diff --git a/mcp_servers/hummingbot_api/settings.py b/mcp_servers/hummingbot_api/settings.py index be831a9c2..98d55b6a6 100644 --- a/mcp_servers/hummingbot_api/settings.py +++ b/mcp_servers/hummingbot_api/settings.py @@ -2,7 +2,6 @@ Configuration settings for Hummingbot MCP Server """ -import argparse import os from pathlib import Path @@ -10,6 +9,7 @@ import yaml from pydantic import BaseModel, Field, field_validator +from mcp_servers._profiles import parse_profile_flags from mcp_servers.hummingbot_api.exceptions import ConfigurationError CONFIG_DIR = Path.home() / ".hummingbot_mcp" @@ -21,39 +21,6 @@ DEFAULT_TOOL_PROFILE = "full" -def _parse_tool_profile() -> str: - """``--profile`` off argv, read at import. - - It has to be resolved *here* rather than in ``server._apply_cli_args``: which - tools exist is decided when the module registers them, which is import time, - and ``_apply_cli_args`` only runs once ``_run()`` is already starting the - stdio loop. ``parse_known_args`` so every other flag the spawner passes - (``--url``, ``--server-name``, ``--bot-id``) stays inert here, and so a run - under pytest — whose argv is the test runner's — resolves the default. - """ - parser = argparse.ArgumentParser(add_help=False) - parser.add_argument("--profile", default=DEFAULT_TOOL_PROFILE) - args, _ = parser.parse_known_args() - return args.profile - - -def _parse_muted_tools() -> tuple[str, ...]: - """``--mute-tools a,b,c`` off argv, read at import (FEAT-091). - - Same timing argument as ``_parse_tool_profile`` above — the mute subtracts - from the profile inside ``register_tools``, which runs at import — and the - same ``parse_known_args``, so a run under pytest resolves to nothing muted. - - An empty or absent flag is the norm: the spawner only puts it on the line - when the operator has actually switched something off, so an uncurated - install keeps the argv it always had. - """ - parser = argparse.ArgumentParser(add_help=False) - parser.add_argument("--mute-tools", default="") - args, _ = parser.parse_known_args() - return tuple(n.strip() for n in (args.mute_tools or "").split(",") if n.strip()) - - class ServerConfig(BaseModel): """Active server configuration""" @@ -156,6 +123,7 @@ def get_settings() -> Settings: """Get application settings from server configuration""" try: server_config = _load_server_config() + tool_profile, muted_tools = parse_profile_flags(DEFAULT_TOOL_PROFILE) return Settings( api_url=server_config.url, @@ -166,8 +134,8 @@ def get_settings() -> Settings: max_retries=int(os.getenv("HUMMINGBOT_MAX_RETRIES", "3")), retry_delay=float(os.getenv("HUMMINGBOT_RETRY_DELAY", "2.0")), log_level=os.getenv("HUMMINGBOT_LOG_LEVEL", "INFO"), - tool_profile=_parse_tool_profile(), - muted_tools=_parse_muted_tools(), + tool_profile=tool_profile, + muted_tools=muted_tools, ) except Exception as e: raise ConfigurationError(f"Failed to load configuration: {e}") From 9cd13553ec99739c725ad629a3c9507995e4ffe8 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 16:49:50 +0300 Subject: [PATCH 007/154] Read a balance row's USD value in one guarded place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The portfolio route re-inlined the naive half of the fetcher's balance reader three times, so a row whose "value" key is present but null raised TypeError out of the live-portfolio loop and out of both history extractors — uncaught in each case, 500ing the whole response for one bad row. Promote the fetcher's guarded reader to balance_value and call it from the route instead of re-parsing by hand. --- condor/fetchers/portfolio.py | 4 ++-- condor/web/routes/portfolio.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/condor/fetchers/portfolio.py b/condor/fetchers/portfolio.py index 2671e170d..98a3de3cb 100644 --- a/condor/fetchers/portfolio.py +++ b/condor/fetchers/portfolio.py @@ -35,7 +35,7 @@ # more thing that has to agree with that one. -def _balance_value(item: Dict[str, Any]) -> float: +def balance_value(item: Dict[str, Any]) -> float: """USD value of one raw balance row, tolerating both payload spellings.""" try: return float(item.get("value", item.get("usd_value", 0)) or 0) @@ -46,7 +46,7 @@ def _balance_value(item: Dict[str, Any]) -> float: def _stable_value(balances: List[Any]) -> float: """Total USD held in Hyperliquid's shared stable collateral within one connector.""" return sum( - _balance_value(item) + balance_value(item) for item in balances if isinstance(item, dict) and item.get("token", item.get("asset", "")) in HL_STABLES diff --git a/condor/web/routes/portfolio.py b/condor/web/routes/portfolio.py index 78a415ee9..6d0b3fc27 100644 --- a/condor/web/routes/portfolio.py +++ b/condor/web/routes/portfolio.py @@ -9,6 +9,7 @@ from condor.fetchers.portfolio import ( PORTFOLIO_HISTORY_RANGES, UNIFIED_ACCOUNT_NOTE, + balance_value, dedupe_unified_accounts, ) from condor.web.auth import require_server_access @@ -102,7 +103,7 @@ async def get_portfolio( "available_units", item.get("available_balance", total_bal) ) ) - usd_val = float(item.get("value", item.get("usd_value", 0))) + usd_val = balance_value(item) if not token: continue @@ -338,7 +339,7 @@ def _extract_token_values(data: object) -> dict[str, float]: for item in inner: if isinstance(item, dict): token = item.get("token", item.get("asset", "")) - usd = float(item.get("value", item.get("usd_value", 0))) + usd = balance_value(item) if token and usd > 0: tokens[token] = tokens.get(token, 0) + usd return tokens @@ -357,7 +358,7 @@ def _extract_connector_totals(data: object) -> dict[str, float]: s = 0.0 for item in inner: if isinstance(item, dict): - s += float(item.get("value", item.get("usd_value", 0))) + s += balance_value(item) totals[key] = s elif isinstance(inner, (int, float)): totals[key] = float(inner) From 8c4b5107f3234ca8779c9cad97512b4cba2e856c Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 17:00:35 +0300 Subject: [PATCH 008/154] Make the config.yml sync structural, so no setter can skip it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six of the eighteen preference mutators never called _sync_section_to_cm, so the wallets, DEX pools and executor defaults a user picked in Telegram lived only in the pickle. The sharing scrubber builds its known_wallet redaction set from the config.yml copy, which meant it never redacted a wallet the user had actually configured. Every write now goes through a _mutate context manager that hydrates, yields the section and syncs it on the way out; hydration moves into _migrate_legacy_data, which every accessor already calls, so a cold pickle no longer reads defaults over a populated config.yml. clear_preferences clears the config.yml side too — of this module's sections only, leaving the code_run grant alone — otherwise the next read restored what was just cleared. --- condor/preferences.py | 218 ++++++++++++++------------ tests/test_preferences_config_sync.py | 135 ++++++++++++++++ 2 files changed, 256 insertions(+), 97 deletions(-) create mode 100644 tests/test_preferences_config_sync.py diff --git a/condor/preferences.py b/condor/preferences.py index c193acf2b..0acb391df 100644 --- a/condor/preferences.py +++ b/condor/preferences.py @@ -15,14 +15,15 @@ - Fallback: context.user_data pickle (for session-level state) When a user_data dict contains '_user_id', setters sync the whole affected -preference section to ConfigManager (via _sync_section_to_cm) so the web -dashboard can also read them. +preference section to ConfigManager (via the _mutate context manager) so the +web dashboard, the MCP subprocess and the sharing scrubber can also read them. """ import logging import re +from contextlib import contextmanager from copy import deepcopy -from typing import Any, Dict, List, Optional, TypedDict +from typing import Any, Dict, Iterator, List, Optional, TypedDict from urllib.parse import urlparse logger = logging.getLogger(__name__) @@ -85,6 +86,45 @@ def _load_from_cm(user_data: Dict) -> None: logger.debug("Failed to hydrate preferences from config: %s", e) +def _clear_sections_in_cm(user_data: Dict) -> None: + """Drop this module's preference sections from ConfigManager. + + Deliberately section by section rather than wiping the user's whole + ``user_preferences`` map: that map also carries reserved keys such as the + ``code_run`` capability grant, which a preference reset has no business + revoking. + """ + user_id = _get_user_id(user_data) + if user_id is None: + return + try: + from config_manager import get_config_manager + + cm = get_config_manager() + for section in _get_default_preferences(): + cm.delete_user_preference(user_id, section) + except Exception as e: + logger.debug("Failed to clear preferences in config: %s", e) + + +@contextmanager +def _mutate(user_data: Dict, section: str) -> Iterator[Any]: + """Edit one preference section, then persist it to config.yml. + + The sync used to be opt-in per setter, and six of the eighteen forgot to + call it — so the wallets, pools and executor defaults a user picked in + Telegram never reached config.yml. That is the copy the sharing scrubber + reads to learn which wallets are the user's own (condor/sharing/scrub.py), + and the durable one that survives a reset of the pickle. Routing every + write through here makes the sync structural: a new setter cannot omit + what it does not have to call. + """ + _load_from_cm(user_data) + prefs = _ensure_preferences(user_data) + yield prefs[section] + _sync_section_to_cm(user_data, section) + + # ============================================ # CONSTANTS AND DEFAULTS # ============================================ @@ -418,6 +458,12 @@ def _migrate_legacy_data(user_data: Dict) -> None: - trading_context: old CLOB/DEX trading context - portfolio_config: old portfolio settings """ + # Every accessor comes through here, so this is where hydration from + # config.yml belongs: bolted onto individual getters it reached only two of + # them, and the rest returned defaults on a cold pickle. Its own + # _prefs_hydrated guard keeps it one-shot. + _load_from_cm(user_data) + # Always guarantee the preferences structure exists (cheap fast path), # even when migration already ran — e.g. after clear_preferences(). prefs = _ensure_preferences(user_data) @@ -502,7 +548,6 @@ def get_preferences(user_data: Dict) -> UserPreferences: Returns: Complete user preferences dictionary """ - _load_from_cm(user_data) _migrate_legacy_data(user_data) return deepcopy(user_data[USER_PREFERENCES_KEY]) @@ -558,9 +603,8 @@ def get_general_prefs(user_data: Dict) -> GeneralPrefs: def set_portfolio_days(user_data: Dict, days: int) -> None: """Set portfolio graph days""" - prefs = _ensure_preferences(user_data) - prefs["portfolio"]["days"] = days - _sync_section_to_cm(user_data, "portfolio") + with _mutate(user_data, "portfolio") as portfolio: + portfolio["days"] = days logger.info(f"Set portfolio days to {days}") @@ -576,9 +620,8 @@ def get_clob_account(user_data: Dict) -> str: def set_clob_last_order(user_data: Dict, params: CLOBOrderParams) -> None: """Set last CLOB order parameters (for quick trading)""" - prefs = _ensure_preferences(user_data) - prefs["clob"]["last_order"] = dict(params) - _sync_section_to_cm(user_data, "clob") + with _mutate(user_data, "clob") as clob: + clob["last_order"] = dict(params) logger.info(f"Updated CLOB last_order params") @@ -648,9 +691,8 @@ def get_dex_slippage(user_data: Dict) -> Optional[str]: def set_dex_slippage(user_data: Dict, slippage: str) -> None: """Set default DEX slippage percentage""" - prefs = _ensure_preferences(user_data) - prefs["dex"]["default_slippage"] = slippage - _sync_section_to_cm(user_data, "dex") + with _mutate(user_data, "dex") as dex: + dex["default_slippage"] = slippage logger.info(f"Set DEX slippage to {slippage}%") @@ -661,9 +703,8 @@ def get_dex_last_swap(user_data: Dict) -> DEXSwapParams: def set_dex_last_swap(user_data: Dict, params: DEXSwapParams) -> None: """Set last DEX swap parameters (for quick trading)""" - prefs = _ensure_preferences(user_data) - prefs["dex"]["last_swap"] = dict(params) - _sync_section_to_cm(user_data, "dex") + with _mutate(user_data, "dex") as dex: + dex["last_swap"] = dict(params) logger.info(f"Updated DEX last_swap params") @@ -674,8 +715,8 @@ def get_dex_last_pool(user_data: Dict) -> DEXPoolParams: def set_dex_last_pool(user_data: Dict, params: DEXPoolParams) -> None: """Set last DEX pool parameters""" - prefs = _ensure_preferences(user_data) - prefs["dex"]["last_pool"] = dict(params) + with _mutate(user_data, "dex") as dex: + dex["last_pool"] = dict(params) logger.info(f"Updated DEX last_pool params") @@ -724,9 +765,8 @@ def get_active_server(user_data: Dict) -> Optional[str]: def set_active_server(user_data: Dict, server_name: Optional[str]) -> None: """Set active server name""" - prefs = _ensure_preferences(user_data) - prefs["general"]["active_server"] = server_name - _sync_section_to_cm(user_data, "general") + with _mutate(user_data, "general") as general: + general["active_server"] = server_name logger.info(f"Set active server to {server_name}") @@ -773,12 +813,8 @@ def set_wallet_networks(user_data: Dict, wallet_address: str, networks: list) -> wallet_address: The wallet address networks: List of enabled network IDs """ - prefs = _ensure_preferences(user_data) - if "gateway" not in prefs: - prefs["gateway"] = {"wallet_networks": {}} - if "wallet_networks" not in prefs["gateway"]: - prefs["gateway"]["wallet_networks"] = {} - prefs["gateway"]["wallet_networks"][wallet_address] = networks + with _mutate(user_data, "gateway") as gateway: + gateway.setdefault("wallet_networks", {})[wallet_address] = networks logger.info(f"Set wallet {wallet_address[:10]}... networks to {networks}") @@ -789,10 +825,9 @@ def remove_wallet_networks(user_data: Dict, wallet_address: str) -> None: user_data: User data dict wallet_address: The wallet address to remove """ - prefs = _ensure_preferences(user_data) - if "gateway" in prefs and "wallet_networks" in prefs["gateway"]: - prefs["gateway"]["wallet_networks"].pop(wallet_address, None) - logger.info(f"Removed wallet {wallet_address[:10]}... network preferences") + with _mutate(user_data, "gateway") as gateway: + gateway.get("wallet_networks", {}).pop(wallet_address, None) + logger.info(f"Removed wallet {wallet_address[:10]}... network preferences") def get_default_networks_for_chain(chain: str) -> list: @@ -914,12 +949,9 @@ def set_last_trade_connector( connector_name: For DEX: network ID (e.g., "solana-mainnet-beta") For CEX: connector name (e.g., "binance_perpetual") """ - prefs = _ensure_preferences(user_data) - if "unified_trade" not in prefs: - prefs["unified_trade"] = {} - prefs["unified_trade"]["last_connector_type"] = connector_type - prefs["unified_trade"]["last_connector_name"] = connector_name - _sync_section_to_cm(user_data, "unified_trade") + with _mutate(user_data, "unified_trade") as unified_trade: + unified_trade["last_connector_type"] = connector_type + unified_trade["last_connector_name"] = connector_name logger.info(f"Set last trade connector: {connector_type}:{connector_name}") @@ -941,12 +973,12 @@ def get_executor_deployed_pairs(user_data: Dict) -> List[str]: def add_executor_deployed_pair(user_data: Dict, pair: str) -> None: """Add a trading pair to the front of the deployed pairs list""" - prefs = _ensure_preferences(user_data) - deployed = list(prefs["executors"].get("deployed_pairs", [])) - if pair in deployed: - deployed.remove(pair) - deployed.insert(0, pair) - prefs["executors"]["deployed_pairs"] = deployed[:8] + with _mutate(user_data, "executors") as executors: + deployed = list(executors.get("deployed_pairs", [])) + if pair in deployed: + deployed.remove(pair) + deployed.insert(0, pair) + executors["deployed_pairs"] = deployed[:8] def get_executor_last_config(user_data: Dict, executor_type: str) -> Dict[str, Any]: @@ -974,9 +1006,8 @@ def set_executor_last_config( executor_type: 'grid' or 'position' params: Config params to save """ - prefs = _ensure_preferences(user_data) - key = f"last_{executor_type}" - prefs["executors"][key] = params + with _mutate(user_data, "executors") as executors: + executors[f"last_{executor_type}"] = params logger.info(f"Updated executor last_{executor_type} config") @@ -1026,12 +1057,10 @@ def secret_notices_enabled(user_data: Dict) -> bool: def set_secret_notices(user_data: Dict, enabled: bool) -> None: """Turn the ambiguous-shape notice on or off for this user.""" - prefs = _ensure_preferences(user_data) - agent = prefs.setdefault("agent", {}) - if agent.get("secret_notices", True) == bool(enabled): + if secret_notices_enabled(user_data) == bool(enabled): return - agent["secret_notices"] = bool(enabled) - _sync_section_to_cm(user_data, "agent") + with _mutate(user_data, "agent") as agent: + agent["secret_notices"] = bool(enabled) def set_chat_binding(user_data: Dict, binding: "ChatBindingPrefs") -> None: @@ -1041,13 +1070,12 @@ def set_chat_binding(user_data: Dict, binding: "ChatBindingPrefs") -> None: written by whichever handler owns it, without that handler having to know (or preserve) the rest of the record. """ - prefs = _ensure_preferences(user_data) - agent = prefs.setdefault("agent", {}) - merged = {**(agent.get("chat_binding") or {}), **binding} - if merged == agent.get("chat_binding"): + current = get_chat_binding(user_data) + merged = {**current, **binding} + if merged == current: return - agent["chat_binding"] = merged - _sync_section_to_cm(user_data, "agent") + with _mutate(user_data, "agent") as agent: + agent["chat_binding"] = merged # ============================================ @@ -1130,48 +1158,42 @@ def save_custom_provider( Returns the stored record (with the sanitized name), so callers can build agent keys from a value that round-trips. """ - prefs = _ensure_preferences(user_data) - agent = prefs.setdefault("agent", {}) - providers: List[CustomProviderPrefs] = agent.setdefault("custom_providers", []) - safe_name = sanitize_provider_name(name) record: CustomProviderPrefs = { "name": safe_name, "base_url": base_url, "api_key": api_key, } - for i, existing in enumerate(providers): - if sanitize_provider_name(existing.get("name", "")) == safe_name: - providers[i] = record - break - else: - if len(providers) >= MAX_CUSTOM_PROVIDERS: - raise ValueError( - f"You already have {MAX_CUSTOM_PROVIDERS} saved endpoints. " - "Remove one before adding another." - ) - providers.append(record) + with _mutate(user_data, "agent") as agent: + providers: List[CustomProviderPrefs] = agent.setdefault("custom_providers", []) + for i, existing in enumerate(providers): + if sanitize_provider_name(existing.get("name", "")) == safe_name: + providers[i] = record + break + else: + if len(providers) >= MAX_CUSTOM_PROVIDERS: + raise ValueError( + f"You already have {MAX_CUSTOM_PROVIDERS} saved endpoints. " + "Remove one before adding another." + ) + providers.append(record) - _sync_section_to_cm(user_data, "agent") logger.info("Saved custom provider '%s' (%s)", safe_name, base_url) return deepcopy(record) def remove_custom_provider(user_data: Dict, name: str) -> bool: """Forget an endpoint. Returns True if one was removed.""" - prefs = _ensure_preferences(user_data) - agent = prefs.setdefault("agent", {}) - providers: List[CustomProviderPrefs] = agent.setdefault("custom_providers", []) - target = sanitize_provider_name(name) + providers = get_custom_providers(user_data) remaining = [ p for p in providers if sanitize_provider_name(p.get("name", "")) != target ] if len(remaining) == len(providers): return False - agent["custom_providers"] = remaining - _sync_section_to_cm(user_data, "agent") + with _mutate(user_data, "agent") as agent: + agent["custom_providers"] = remaining logger.info("Removed custom provider '%s'", target) return True @@ -1267,12 +1289,10 @@ def get_active_agent_key(user_id: int) -> Optional[str]: def set_active_agent_key(user_data: Dict, agent_key: str) -> None: """Record the user's current model selection in the shared preference store.""" - prefs = _ensure_preferences(user_data) - agent = prefs.setdefault("agent", {}) - if agent.get("active_agent_key") == agent_key: + if get_agent_prefs(user_data).get("active_agent_key") == agent_key: return - agent["active_agent_key"] = agent_key - _sync_section_to_cm(user_data, "agent") + with _mutate(user_data, "agent") as agent: + agent["active_agent_key"] = agent_key def _migrate_legacy_custom_llm(user_data: Dict, prefs: Dict) -> None: @@ -1295,14 +1315,14 @@ def _migrate_legacy_custom_llm(user_data: Dict, prefs: Dict) -> None: taken = {sanitize_provider_name(p.get("name", "")) for p in providers} while name in taken: name = f"{name}-2" - providers.append( - { - "name": name, - "base_url": base_url, - "api_key": legacy.get("api_key", ""), - } - ) - _sync_section_to_cm(user_data, "agent") + with _mutate(user_data, "agent"): + providers.append( + { + "name": name, + "base_url": base_url, + "api_key": legacy.get("api_key", ""), + } + ) logger.info("Migrated legacy custom_llm endpoint %s → '%s'", base_url, name) user_data.pop("custom_llm", None) @@ -1391,10 +1411,8 @@ def get_note(user_data: Dict, key: str) -> Optional[str]: def set_note(user_data: Dict, key: str, value: str) -> None: """Set a note (key-value pair).""" - prefs = _ensure_preferences(user_data) - if "notes" not in prefs: - prefs["notes"] = {} - prefs["notes"][key] = value + with _mutate(user_data, "notes") as notes: + notes[key] = value logger.info(f"Set note '{key}'") @@ -1404,7 +1422,13 @@ def set_note(user_data: Dict, key: str, value: str) -> None: def clear_preferences(user_data: Dict) -> None: - """Clear all user preferences (reset to defaults)""" + """Clear all user preferences (reset to defaults). + + config.yml is cleared too, and only of the sections this module owns: it is + the durable copy, so dropping the in-memory one alone would let the next + hydration restore exactly what was just cleared. + """ + _clear_sections_in_cm(user_data) if USER_PREFERENCES_KEY in user_data: del user_data[USER_PREFERENCES_KEY] # Reset bookkeeping flags so migration/hydration rebuild consistently diff --git a/tests/test_preferences_config_sync.py b/tests/test_preferences_config_sync.py new file mode 100644 index 000000000..39dd1437a --- /dev/null +++ b/tests/test_preferences_config_sync.py @@ -0,0 +1,135 @@ +"""Every preference write reaches config.yml, and every read hydrates from it. + +The sync to ConfigManager used to be opt-in per setter, and six setters never +called it — so the wallets, DEX pools and executor defaults chosen in Telegram +existed only in the pickle. The sharing scrubber reads the config.yml copy to +learn which wallets are the user's own, so those wallets were never redacted +(ARCH-604). Hydration had the mirror hole: only two of ~15 accessors called it, +so a cold pickle read defaults over a populated config.yml. + +Both directions are now structural: writes go through ``_mutate``, reads through +``_migrate_legacy_data``. +""" + +import inspect + +import pytest + +import condor.preferences as prefs_module +import config_manager as cm_module +from condor.preferences import ( + DEFAULT_PORTFOLIO_DAYS, + add_executor_deployed_pair, + clear_preferences, + get_dex_prefs, + get_executor_prefs, + get_gateway_prefs, + get_portfolio_prefs, + get_preferences, + set_dex_last_pool, + set_executor_last_config, + set_portfolio_days, + set_wallet_networks, +) + +USER_ID = 4242 +WALLET = "7xKXtg2CW87d97TXJSDpbD5jBkheTqA83TZRuJosgAsU" + + +@pytest.fixture +def cm(tmp_path, monkeypatch): + """A real ConfigManager on a throwaway config.yml — persistence included.""" + monkeypatch.chdir(tmp_path) # audit_log.yml is written relative to cwd + manager = cm_module.ConfigManager(config_path=str(tmp_path / "config.yml")) + manager._data["user_preferences"] = {} + manager._audit_log = [] + manager._save_config() + monkeypatch.setattr(cm_module, "get_config_manager", lambda: manager) + return manager + + +def test_previously_unsynced_setters_reach_config(cm): + """The four setters that silently skipped the sync now persist.""" + user_data = {"_user_id": USER_ID} + + set_wallet_networks(user_data, WALLET, ["solana-mainnet-beta"]) + set_dex_last_pool(user_data, {"trading_pair": "SOL-USDC"}) + add_executor_deployed_pair(user_data, "SOL-USDC") + set_executor_last_config(user_data, "grid", {"total_amount_quote": 100}) + + stored = cm.get_user_preferences(USER_ID) + assert stored["gateway"]["wallet_networks"] == {WALLET: ["solana-mainnet-beta"]} + assert stored["dex"]["last_pool"] == {"trading_pair": "SOL-USDC"} + assert stored["executors"]["deployed_pairs"] == ["SOL-USDC"] + assert stored["executors"]["last_grid"] == {"total_amount_quote": 100} + + +def test_wallets_survive_a_reset_of_the_pickle(cm): + """What the scrubber reads: wallets written in Telegram, read config-only.""" + set_wallet_networks({"_user_id": USER_ID}, WALLET, ["solana-mainnet-beta"]) + + fresh = prefs_module.load_user_data_for(USER_ID) + + assert list(get_gateway_prefs(fresh).get("wallet_networks")) == [WALLET] + + +def test_accessor_hydrates_from_config_without_get_preferences(cm): + """A cold pickle reads config.yml through any accessor, not just two.""" + cm.set_user_preference(USER_ID, "dex", {"default_slippage": "3.5"}) + + user_data = {"_user_id": USER_ID} + assert get_dex_prefs(user_data).get("default_slippage") == "3.5" + + cm.set_user_preference(USER_ID, "executors", {"deployed_pairs": ["ETH-USDC"]}) + assert get_executor_prefs({"_user_id": USER_ID})["deployed_pairs"] == ["ETH-USDC"] + + +def test_clear_preferences_clears_the_config_copy(cm): + """Otherwise the next hydration restores exactly what was cleared.""" + user_data = {"_user_id": USER_ID} + set_portfolio_days(user_data, 30) + set_wallet_networks(user_data, WALLET, ["base"]) + assert cm.get_user_preferences(USER_ID)["portfolio"]["days"] == 30 + + clear_preferences(user_data) + + assert get_portfolio_prefs(user_data)["days"] == DEFAULT_PORTFOLIO_DAYS + assert get_gateway_prefs(user_data)["wallet_networks"] == {} + assert get_preferences({"_user_id": USER_ID})["portfolio"]["days"] == ( + DEFAULT_PORTFOLIO_DAYS + ) + assert cm.get_user_preferences(USER_ID) == {} + + +def test_clear_preferences_keeps_reserved_keys(cm): + """A preference reset must not revoke the code_run capability grant.""" + cm.set_user_preference(USER_ID, cm_module.CODE_RUN_PREFERENCE, True) + set_portfolio_days({"_user_id": USER_ID}, 30) + + clear_preferences({"_user_id": USER_ID}) + + stored = cm.get_user_preferences(USER_ID) + assert stored == {cm_module.CODE_RUN_PREFERENCE: True} + + +def test_every_mutator_routes_through_mutate(): + """The sync is structural: a setter cannot forget what it never calls.""" + mutators = [ + name + for name in dir(prefs_module) + if name.startswith(("set_", "add_", "remove_")) + and inspect.isfunction(getattr(prefs_module, name)) + and getattr(prefs_module, name).__module__ == prefs_module.__name__ + ] + assert mutators, "no mutators found — the check would pass vacuously" + + missing = [ + name + for name in mutators + if "_mutate(" not in inspect.getsource(getattr(prefs_module, name)) + ] + assert missing == [] + + # ...and there is exactly one place left that syncs a section. + source = inspect.getsource(prefs_module) + assert source.count("_sync_section_to_cm(user_data, section)") == 1 From 702fae54a089b712ca72bf959c37001406702f7c Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 17:08:08 +0300 Subject: [PATCH 009/154] Give the pydantic-ai turn's permission decision a name of its own prompt_stream carried every concern of a turn at once: the semaphore, the gate reset, image assembly, graph iteration, the whole permission decision, the event projection for three node types, message-history accumulation and error mapping. The permission block sat eight levels deep, which is why black had to spread its denial reason over seven physical lines, and its load-bearing rule -- that approved/reason are on the gate before the next graph step runs the tool -- existed only as a comment in the middle of the nest. The decision now lives in _authorize, whose docstring states the fail-closed contract and why the record has to happen there; the events it produces live in _tool_events, the two node branches in _response_events and _tool_return_events, and the images in _build_user_prompt. prompt_stream keeps the graph loop and nothing else. Pure refactor, same events in the same order. --- condor/acp/pydantic_ai_client.py | 327 ++++++++++++++++--------------- 1 file changed, 173 insertions(+), 154 deletions(-) diff --git a/condor/acp/pydantic_ai_client.py b/condor/acp/pydantic_ai_client.py index 261198c27..95e2a978d 100644 --- a/condor/acp/pydantic_ai_client.py +++ b/condor/acp/pydantic_ai_client.py @@ -16,7 +16,7 @@ import os import time import uuid -from typing import Any, AsyncIterator +from typing import Any, AsyncIterator, Iterator from urllib.parse import urlparse from urllib.request import Request, urlopen @@ -922,12 +922,6 @@ async def prompt_stream( try: from pydantic_ai.agent import CallToolsNode, ModelRequestNode - from pydantic_ai.messages import ( - TextPart, - ThinkingPart, - ToolCallPart, - ToolReturnPart, - ) from pydantic_graph import End self._permission_gate.reset() @@ -936,20 +930,9 @@ async def prompt_stream( # event the dashboard already received. blocked_ids: set[str] = set() - user_prompt: Any = text - if images: - from pydantic_ai.messages import BinaryContent - - user_prompt = [ - *( - BinaryContent(data=image.data, media_type=image.mime) - for image in images - ), - text, - ] - async with self._agent.iter( - user_prompt, message_history=self._message_history + self._build_user_prompt(text, images), + message_history=self._message_history, ) as run: async for node in run: if self._abort_requested: @@ -967,142 +950,12 @@ async def prompt_stream( if isinstance(node, ModelRequestNode): elapsed = time.monotonic() - start_time yield Heartbeat(elapsed_seconds=elapsed) - # Extract tool return results from request parts - if hasattr(node, "request") and node.request: - for part in node.request.parts: - if isinstance(part, ToolReturnPart): - if (part.tool_call_id or "") in blocked_ids: - continue - content = part.content - output_str = ( - content - if isinstance(content, str) - else str(content) - ) - yield ToolCallUpdate( - tool_call_id=part.tool_call_id or "", - status="completed", - output=output_str, - ) + for event in self._tool_return_events(node, blocked_ids): + yield event elif isinstance(node, CallToolsNode): - # Emit text and tool-call events from model response - for part in node.model_response.parts: - if isinstance(part, TextPart) and part.content: - yield TextChunk(text=part.content) - - elif isinstance(part, ThinkingPart) and part.content: - # Reasoning models (deepseek-r1/qwq via ollama, - # gpt-oss via openrouter) return their thinking - # as a third part type. The ACP path already - # translates the same thing from - # ``agent_thought_chunk``; without this branch - # the thinking stream is silently dropped and - # the dashboard/Telegram thought panel stays - # empty for every pydantic-ai model (ARCH-333). - yield ThoughtChunk(text=part.content) - - elif isinstance(part, ToolCallPart): - tool_id = part.tool_call_id or uuid.uuid4().hex[:12] - tool_name = part.tool_name - - # Risk check via permission callback - if self.permission_callback: - # Unparseable args stay None rather than - # collapsing to {}: the gate reads that - # as "unknown" and fails closed, where - # an empty dict would have read as a - # harmless no-argument call (SEC-093). - tool_call_info = { - "tool": tool_name, - "title": tool_name, - "input": _tool_args_to_dict(part.args), - } - options = [ - {"optionId": "allow", "kind": "allow_once"}, - {"optionId": "deny", "kind": "deny"}, - ] - # Don't hold the per-server slot while a - # human decides — release it for the wait - # so other sessions/ticks on this backend - # aren't blocked (PERF-029). - # Fail closed: only an explicit "selected" - # outcome allows the call. A check that - # raises or times out is a denial, never - # a pass (SEC-080). - reason = "" - try: - async with self._release_request_slot(): - result = await self.permission_callback( - tool_call_info, options - ) - outcome = ( - result.get("outcome", {}) - if isinstance(result, dict) - else {} - ) - approved = ( - isinstance(outcome, dict) - and outcome.get("outcome") == "selected" - ) - if not approved: - # The gate says why when it can - # (condor.agents.risk attaches a - # ``reason``); an agent told only - # "denied" reads its own refusal - # as a missing approval and waits - # for a human it may not have. - reason = ( - ( - result.get("reason") - if isinstance(result, dict) - else "" - ) - or "denied by the risk/confirmation gate" - ) - except Exception as exc: - log.exception( - "Permission check failed for %s — " - "blocking the call", - tool_name, - ) - approved = False - reason = f"permission check failed ({exc})" - - # Record before the node runs: the tool - # executes on the next graph step, where - # the gated toolset consumes this. - self._permission_gate.record( - part.tool_call_id, - tool_name, - approved, - reason, - ) - - if not approved: - if part.tool_call_id: - blocked_ids.add(part.tool_call_id) - yield ToolCallEvent( - tool_call_id=tool_id, - title=tool_name, - status="blocked", - kind="mcp", - input=_tool_args_to_dict(part.args), - ) - continue - - yield ToolCallEvent( - tool_call_id=tool_id, - title=tool_name, - status="in_progress", - kind="mcp", - input=_tool_args_to_dict(part.args), - ) - - yield ToolCallUpdate( - tool_call_id=tool_id, - status="completed", - ) + async for event in self._response_events(node, blocked_ids): + yield event # Accumulate messages so the next prompt_stream() call sees # this turn's context via message_history. An aborted run @@ -1123,6 +976,172 @@ async def prompt_stream( yield TextChunk(text=self._format_error(e)) yield PromptDone(stop_reason="error") + def _build_user_prompt(self, text: str, images: list | None) -> Any: + """Assemble the user turn: images first, then the text. + + Plain text when there are no images, so the common path stays the shape + pydantic-ai documents. + """ + if not images: + return text + + from pydantic_ai.messages import BinaryContent + + return [ + *( + BinaryContent(data=image.data, media_type=image.mime) + for image in images + ), + text, + ] + + def _tool_return_events( + self, node: Any, blocked_ids: set[str] + ) -> Iterator[ACPEvent]: + """Project a model request's tool results as ``completed`` updates. + + A refused call still produces a (synthetic) refusal result on the next + request; projecting it would paint "completed" over the "blocked" event + the dashboard already has, so ``blocked_ids`` filters those out. + """ + from pydantic_ai.messages import ToolReturnPart + + request = getattr(node, "request", None) + if not request: + return + + for part in request.parts: + if not isinstance(part, ToolReturnPart): + continue + if (part.tool_call_id or "") in blocked_ids: + continue + content = part.content + yield ToolCallUpdate( + tool_call_id=part.tool_call_id or "", + status="completed", + output=content if isinstance(content, str) else str(content), + ) + + async def _response_events( + self, node: Any, blocked_ids: set[str] + ) -> AsyncIterator[ACPEvent]: + """Project one model response into text, thought and tool-call events. + + Tool calls are authorized here, one graph step before pydantic-graph + executes them; ``blocked_ids`` collects the refused ones so their + refusal result is not later reported as a completion. + """ + from pydantic_ai.messages import TextPart, ThinkingPart, ToolCallPart + + for part in node.model_response.parts: + if isinstance(part, TextPart) and part.content: + yield TextChunk(text=part.content) + + elif isinstance(part, ThinkingPart) and part.content: + # Reasoning models (deepseek-r1/qwq via ollama, gpt-oss via + # openrouter) return their thinking as a third part type. The + # ACP path already translates the same thing from + # ``agent_thought_chunk``; without this branch the thinking + # stream is silently dropped and the dashboard/Telegram thought + # panel stays empty for every pydantic-ai model (ARCH-333). + yield ThoughtChunk(text=part.content) + + elif isinstance(part, ToolCallPart): + approved = True + if self.permission_callback: + approved, _reason = await self._authorize(part) + if not approved and part.tool_call_id: + blocked_ids.add(part.tool_call_id) + for event in self._tool_events(part, approved): + yield event + + async def _authorize(self, part: Any) -> tuple[bool, str]: + """Decide whether a tool call may run, and record the decision. + + Fail closed: only an explicit "selected" outcome approves the call. A + callback that raises, times out or answers in any other shape is a + denial, never a pass (SEC-080). + + The decision is recorded on ``self._permission_gate`` *before* this + returns, because the tool itself executes on the next graph step, where + the gated toolset consumes exactly that record. Moving the record after + the caller's event projection would let the tool run undecided. + + Returns ``(approved, reason)``; ``reason`` is empty when approved. + """ + tool_name = part.tool_name + # Unparseable args stay None rather than collapsing to {}: the gate + # reads that as "unknown" and fails closed, where an empty dict would + # have read as a harmless no-argument call (SEC-093). + tool_call_info = { + "tool": tool_name, + "title": tool_name, + "input": _tool_args_to_dict(part.args), + } + options = [ + {"optionId": "allow", "kind": "allow_once"}, + {"optionId": "deny", "kind": "deny"}, + ] + + reason = "" + try: + # Don't hold the per-server slot while a human decides — release it + # for the wait so other sessions/ticks on this backend aren't + # blocked (PERF-029). + async with self._release_request_slot(): + result = await self.permission_callback(tool_call_info, options) + outcome = result.get("outcome", {}) if isinstance(result, dict) else {} + approved = ( + isinstance(outcome, dict) and outcome.get("outcome") == "selected" + ) + if not approved: + # The gate says why when it can (condor.agents.risk attaches a + # ``reason``); an agent told only "denied" reads its own refusal + # as a missing approval and waits for a human it may not have. + given = result.get("reason") if isinstance(result, dict) else "" + reason = given or "denied by the risk/confirmation gate" + except Exception as exc: + log.exception( + "Permission check failed for %s — blocking the call", tool_name + ) + approved = False + reason = f"permission check failed ({exc})" + + self._permission_gate.record(part.tool_call_id, tool_name, approved, reason) + return approved, reason + + def _tool_events(self, part: Any, approved: bool) -> list[ACPEvent]: + """Project one tool call into the events the UI shows. + + A refused call gets a single terminal ``blocked`` event; an approved one + opens ``in_progress`` and closes ``completed`` right away, with the real + output arriving later as the ``ToolReturnPart`` update for the same id. + """ + tool_id = part.tool_call_id or uuid.uuid4().hex[:12] + args = _tool_args_to_dict(part.args) + + if not approved: + return [ + ToolCallEvent( + tool_call_id=tool_id, + title=part.tool_name, + status="blocked", + kind="mcp", + input=args, + ) + ] + + return [ + ToolCallEvent( + tool_call_id=tool_id, + title=part.tool_name, + status="in_progress", + kind="mcp", + input=args, + ), + ToolCallUpdate(tool_call_id=tool_id, status="completed"), + ] + def _format_error(self, e: Exception) -> str: """Translate provider HTTP errors into actionable user-facing text. From b5e048d546b82240939dafe216a8813fa8858b0c Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 17:17:12 +0300 Subject: [PATCH 010/154] Let condor/paths.py own reports/, the root it never named reports/ was the one runtime output root built outside the module that owns roots, and it had made both of the mistakes paths.py was written to close, at once. condor/reports/store.py fixed it at import from Path(__file__).parents[2] -- anchored at the repo, but a module constant no env var could move, which cost a re-import shim so monkeypatching still worked and made eighteen test modules patch a *pair* of names by hand. handlers/bots/archived_report.py built the bare name "reports" instead, relative to the working directory, so a process launched from anywhere but the checkout wrote its JSON and PNGs into a second, different report store. reports_dir() and reports_index_path() join backtests_dir() and code_runs_dir(), with $CONDOR_REPORTS_DIR read on every call. The location does not move: /reports/ has always had its own .gitignore line and every install's history is already there. Both producers now resolve through it, so they land in one directory whatever the cwd, and the store's _charts_dir()/_index_file() call the resolvers instead of re-importing the package attribute. Thirty-six monkeypatch.setattr calls across eighteen test modules collapse into eighteen setenv lines, and the suite's autouse fixture repoints the root for every test -- the fixture tax paths.py exists to remove. test_suite_isolation pins the fourth root the way it already pins the other three. --- condor/paths.py | 56 +++++++++++++++++--- condor/reports/__init__.py | 4 -- condor/reports/store.py | 16 ++---- handlers/bots/archived_report.py | 24 ++++----- tests/conftest.py | 8 ++- tests/test_agent_session_report.py | 3 +- tests/test_archived_report_lookup.py | 3 +- tests/test_call_routine.py | 5 +- tests/test_code_runner.py | 5 +- tests/test_journal_retention.py | 7 +-- tests/test_report_builder.py | 3 +- tests/test_report_component_gallery.py | 3 +- tests/test_report_egress_hydration.py | 4 +- tests/test_report_subject.py | 4 +- tests/test_reports_attribution.py | 3 +- tests/test_reports_ownership.py | 12 ++--- tests/test_routine_instance_retention.py | 3 +- tests/test_routine_result_retention.py | 3 +- tests/test_routine_run_report_link.py | 3 +- tests/test_routine_telegram_report_source.py | 3 +- tests/test_share_report_path_containment.py | 6 +-- tests/test_suite_isolation.py | 29 ++++++++-- tests/test_web_report_assets.py | 3 +- tests/test_web_report_serving.py | 3 +- 24 files changed, 122 insertions(+), 91 deletions(-) diff --git a/condor/paths.py b/condor/paths.py index 14ae538d8..3e92050b4 100644 --- a/condor/paths.py +++ b/condor/paths.py @@ -6,9 +6,9 @@ deployment's data inside the code, and it meant a test run wrote into the developer's live install because there was no single knob to turn. -There are **three** roots, not one, and this module is where all three are -named. They are separate on purpose (``.gitignore`` has always listed the first -two apart) and the line between them is *who writes and when*: +There are **four** roots, not one, and this module is where all four are +named. They are separate on purpose (``.gitignore`` has always listed them +apart) and the line between them is *who writes and when*: /.condor/ # or $CONDOR_RUNTIME_ROOT ├── users/{user_id}/ # the runtime store: one conversation's @@ -25,6 +25,11 @@ ├── backtests/ └── code_runs/ + /reports/ # or $CONDOR_REPORTS_DIR + ├── reports_index.json # what a run had to show for itself: + ├── .html # one rendered report per file, indexed + └── assets/ # and served to Telegram and the web + /agents/ # or $CONDOR_STOCK_AGENTS_ROOT ├── /AGENT.md # the shipped library: the curated ├── /skills/ # agents, their playbooks and the @@ -38,7 +43,7 @@ ├── /strategies// # journals, stores, mutes. Git has └── _shared/{skills,routines}/ # never heard of it. -``data/`` is the older of the three and is named literally in agent-facing text +``data/`` is the oldest of them and is named literally in agent-facing text (``data/code_runs/`` in the ``run_code`` tool description), so folding it into ``.condor/`` would open a new split rather than close one. What it did lack was a resolver: three of its stores built their path at import from the bare name @@ -46,6 +51,18 @@ pickle and ``code_runs`` were already anchored at the repo. Both now come from :func:`data_dir`. +``reports/`` is what a run had to show for itself, and it arrived here last +(ARCH-605) having made *both* of the mistakes the paragraph above describes, at +once. ``condor/reports/store.py`` built it at import from +``Path(__file__).parents[2]`` -- anchored at the repo, but a module constant no +env var could move, which cost a re-import shim so monkeypatching still worked +and made eighteen test modules patch a pair of names by hand. A second +producer, ``handlers/bots/archived_report.py``, built the bare name ``reports`` +instead, so a process launched from anywhere but the checkout split the store +in two. It keeps its own root rather than folding into ``data/``: ``/reports/`` +has always had its own ``.gitignore`` line and every install's history is +already there. + ``agents/`` used to be **one** root that was *half version-controlled*: an agent's definition and its skill library were committed, the ``store/user_{id}/`` under them was not, and the product rewrote both. That made every ``/update`` @@ -72,8 +89,8 @@ Two rules this module keeps: -* :func:`runtime_root`, :func:`data_dir`, :func:`stock_agents_root` and - :func:`local_agents_root` are a +* :func:`runtime_root`, :func:`data_dir`, :func:`reports_dir`, + :func:`stock_agents_root` and :func:`local_agents_root` are a *function call at every use*, never a module constant. The env override has to be observable after import, or the test fixture that isolates the suite cannot work and the MCP/ACP subprocesses cannot inherit it. @@ -110,10 +127,12 @@ DATA_DIR_ENV = "CONDOR_DATA_DIR" AGENTS_ROOT_ENV = "CONDOR_AGENTS_ROOT" STOCK_AGENTS_ROOT_ENV = "CONDOR_STOCK_AGENTS_ROOT" +REPORTS_DIR_ENV = "CONDOR_REPORTS_DIR" RUNTIME_DIRNAME = ".condor" DATA_DIRNAME = "data" AGENTS_DIRNAME = "agents" +REPORTS_DIRNAME = "reports" # : condor/paths.py -> condor/ -> _PROJECT_ROOT = Path(__file__).resolve().parent.parent @@ -290,6 +309,31 @@ def code_runs_dir() -> Path: return data_dir() / "code_runs" +def reports_dir() -> Path: + """Every rendered report, plus ``reports_index.json`` and the shared assets. + + A root of its own and not ``data_dir() / "reports"``: ``/reports/`` has its + own ``.gitignore`` entry and every install already has its history there, so + folding it in would move files rather than close a split. What it lacked was + a resolver -- ``condor/reports/store.py`` built it at import from + ``Path(__file__).parents[2]``, which no env var could move, and + ``handlers/bots/archived_report.py`` built the bare name ``reports``, which + the working directory could. Both come from here now (ARCH-605). + + ``$CONDOR_REPORTS_DIR`` overrides it, read on every call for the same reason + :func:`runtime_root` reads its own. + """ + override = os.environ.get(REPORTS_DIR_ENV) + if override: + return Path(override).expanduser() + return _PROJECT_ROOT / REPORTS_DIRNAME + + +def reports_index_path() -> Path: + """The report index: one row per saved report (``condor.reports.store``).""" + return reports_dir() / "reports_index.json" + + def users_root() -> Path: return runtime_root() / USERS_DIRNAME diff --git a/condor/reports/__init__.py b/condor/reports/__init__.py index ee858619c..9141925f4 100644 --- a/condor/reports/__init__.py +++ b/condor/reports/__init__.py @@ -4,8 +4,6 @@ from .builder import LiveReport, ReportBuilder from .rendering import hydrate from .store import ( - CHARTS_DIR, - INDEX_FILE, MAX_REPORTS, attribute_owner, attribute_to, @@ -26,8 +24,6 @@ "ReportBuilder", "LiveReport", "subjects", - "CHARTS_DIR", - "INDEX_FILE", "MAX_REPORTS", "attribute_owner", "attribute_to", diff --git a/condor/reports/store.py b/condor/reports/store.py index 107fc4d45..001cb7c75 100644 --- a/condor/reports/store.py +++ b/condor/reports/store.py @@ -9,11 +9,9 @@ from contextlib import contextmanager from pathlib import Path +from condor import paths from condor.fsutil import atomic_write_json, atomic_write_text -# reports/ is a repository-root output directory, not this source package. -CHARTS_DIR = Path(__file__).resolve().parents[2] / "reports" -INDEX_FILE = CHARTS_DIR / "reports_index.json" MAX_REPORTS = int(os.environ.get("CONDOR_MAX_REPORTS", "100")) _index_lock = asyncio.Lock() @@ -32,17 +30,13 @@ def _charts_dir() -> Path: - # Resolve through the public package so existing runtime overrides of - # condor.reports.CHARTS_DIR keep working after the module-to-package split. - from . import CHARTS_DIR as configured_dir - - return configured_dir + # reports/ is a repository-root output directory, not this source package; + # condor.paths owns it, so $CONDOR_REPORTS_DIR moves it after import. + return paths.reports_dir() def _index_file() -> Path: - from . import INDEX_FILE as configured_file - - return configured_file + return paths.reports_index_path() def reset_last_report_id() -> None: diff --git a/handlers/bots/archived_report.py b/handlers/bots/archived_report.py index cf38af6a5..821957d09 100644 --- a/handlers/bots/archived_report.py +++ b/handlers/bots/archived_report.py @@ -13,16 +13,16 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple -logger = logging.getLogger(__name__) +from condor.paths import reports_dir -# Reports directory in project root -REPORTS_DIR = Path("reports") +logger = logging.getLogger(__name__) def ensure_reports_dir() -> Path: - """Create reports directory if it doesn't exist.""" - REPORTS_DIR.mkdir(exist_ok=True) - return REPORTS_DIR + """Create the report output directory if it doesn't exist.""" + directory = reports_dir() + directory.mkdir(parents=True, exist_ok=True) + return directory def _extract_bot_name(db_path: str) -> str: @@ -218,7 +218,7 @@ async def save_full_report( ) # Save JSON - json_path = REPORTS_DIR / f"{filename}.json" + json_path = reports_dir() / f"{filename}.json" with open(json_path, "w", encoding="utf-8") as f: json.dump(report_data, f, indent=2, default=_serialize_datetime) logger.info(f"Saved JSON report to {json_path}") @@ -238,7 +238,7 @@ async def save_full_report( ) if chart_bytes: - png_path = REPORTS_DIR / f"{filename}.png" + png_path = reports_dir() / f"{filename}.png" with open(png_path, "wb") as f: f.write(chart_bytes.read()) logger.info(f"Saved chart to {png_path}") @@ -262,10 +262,10 @@ def list_reports() -> List[Dict[str, Any]]: """ reports = [] try: - if not REPORTS_DIR.exists(): + if not reports_dir().exists(): return [] - for json_file in REPORTS_DIR.glob("*.json"): + for json_file in reports_dir().glob("*.json"): try: with open(json_file, "r", encoding="utf-8") as f: data = json.load(f) @@ -314,7 +314,7 @@ def load_report(filename: str) -> Optional[Dict[str, Any]]: if not filename.endswith(".json"): filename = f"{filename}.json" - report_path = REPORTS_DIR / filename + report_path = reports_dir() / filename if not report_path.exists(): return None @@ -341,7 +341,7 @@ def delete_report(filename: str) -> bool: if not filename.endswith(".json"): filename = f"{filename}.json" - json_path = REPORTS_DIR / filename + json_path = reports_dir() / filename png_path = json_path.with_suffix(".png") deleted = False diff --git a/tests/conftest.py b/tests/conftest.py index 8270c9b23..807e787d3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -52,7 +52,7 @@ def _reset_gecko_throttle(): @pytest.fixture(autouse=True) def _isolated_runtime_root(tmp_path, monkeypatch): - """Keep all three durable roots out of the developer's live install. + """Keep every durable root out of the developer's live install. For as long as every store derived its own root there was nothing to repoint, so four test modules each had to remember to monkeypatch a private @@ -81,6 +81,11 @@ def _isolated_runtime_root(tmp_path, monkeypatch): wants the shipped tree names it (``load_shared_routine`` above, and the few that assert on ``agents/condor/AGENT.md``). + The last line covers ``reports/``, the root that arrived last (ARCH-605). + It used to be a module constant in ``condor/reports/store.py``, so the only + way to isolate it was to monkeypatch a *pair* of names -- and eighteen test + modules did, by hand. One env var replaces all thirty-six. + ``tmp_path / "agents"`` and not ``condor-agents`` for the writable root: ``tmp_path`` stands in for the repo root in the agent tests, so the registry and the stores stay one tree. @@ -91,3 +96,4 @@ def _isolated_runtime_root(tmp_path, monkeypatch): monkeypatch.setenv(paths.DATA_DIR_ENV, str(tmp_path / "condor-data")) monkeypatch.setenv(paths.AGENTS_ROOT_ENV, str(tmp_path / "agents")) monkeypatch.setenv(paths.STOCK_AGENTS_ROOT_ENV, str(tmp_path / "stock-agents")) + monkeypatch.setenv(paths.REPORTS_DIR_ENV, str(tmp_path / "reports")) diff --git a/tests/test_agent_session_report.py b/tests/test_agent_session_report.py index 75f3324d3..731b03d41 100644 --- a/tests/test_agent_session_report.py +++ b/tests/test_agent_session_report.py @@ -18,8 +18,7 @@ @pytest.fixture def reports_dir(tmp_path, monkeypatch): - monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path / "reports") - monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports" / "reports_index.json") + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path / "reports")) return tmp_path diff --git a/tests/test_archived_report_lookup.py b/tests/test_archived_report_lookup.py index 7a326aefb..0f8c367ba 100644 --- a/tests/test_archived_report_lookup.py +++ b/tests/test_archived_report_lookup.py @@ -33,8 +33,7 @@ @pytest.fixture def reports_dir(tmp_path, monkeypatch): - monkeypatch.setattr(reports, "CHARTS_DIR", tmp_path) - monkeypatch.setattr(reports, "INDEX_FILE", tmp_path / "reports_index.json") + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path)) return tmp_path diff --git a/tests/test_call_routine.py b/tests/test_call_routine.py index 1c49a5558..267ba9704 100644 --- a/tests/test_call_routine.py +++ b/tests/test_call_routine.py @@ -83,10 +83,7 @@ async def _report(self, *a, **kw): @pytest.fixture def reports_dir(tmp_path, monkeypatch): - monkeypatch.setattr(reports, "CHARTS_DIR", tmp_path / "reports") - monkeypatch.setattr( - reports, "INDEX_FILE", tmp_path / "reports" / "reports_index.json" - ) + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path / "reports")) monkeypatch.setattr( rendering, "plotly_bundle", diff --git a/tests/test_code_runner.py b/tests/test_code_runner.py index c6270a1c0..ad3592e9f 100644 --- a/tests/test_code_runner.py +++ b/tests/test_code_runner.py @@ -54,10 +54,7 @@ async def _raise(*a, **kw): @pytest.fixture def reports_dir(tmp_path, monkeypatch): - monkeypatch.setattr(reports, "CHARTS_DIR", tmp_path / "reports") - monkeypatch.setattr( - reports, "INDEX_FILE", tmp_path / "reports" / "reports_index.json" - ) + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path / "reports")) monkeypatch.setattr( rendering, "plotly_bundle", diff --git a/tests/test_journal_retention.py b/tests/test_journal_retention.py index 63d7221d0..0bd711be3 100644 --- a/tests/test_journal_retention.py +++ b/tests/test_journal_retention.py @@ -19,7 +19,6 @@ import pytest import condor.agents.journal as journal_mod -import condor.reports as rep from condor.agents.journal import JournalManager, count_journal_ticks @@ -155,8 +154,7 @@ async def stop(self): def test_one_real_engine_tick_rewrites_the_journal_once(tmp_path, monkeypatch, writes): - monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path / "reports") - monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports" / "reports_index.json") + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path / "reports")) engine = _engine(tmp_path / "agents", monkeypatch) monkeypatch.setattr(engine, "_get_client", _async(object())) monkeypatch.setattr(engine, "_adopt_running_bots", _async(None)) @@ -178,8 +176,7 @@ def test_a_risk_blocked_tick_rewrites_the_journal_once(tmp_path, monkeypatch, wr """The blocked path wrote twice (append_action + record_tick).""" from condor.agents.risk import RiskState - monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path / "reports") - monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports" / "reports_index.json") + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path / "reports")) engine = _engine(tmp_path / "agents", monkeypatch) monkeypatch.setattr(engine, "_get_client", _async(object())) monkeypatch.setattr(engine, "_adopt_running_bots", _async(None)) diff --git a/tests/test_report_builder.py b/tests/test_report_builder.py index d82379ed7..f1f769415 100644 --- a/tests/test_report_builder.py +++ b/tests/test_report_builder.py @@ -18,8 +18,7 @@ @pytest.fixture def reports_dir(tmp_path, monkeypatch): - monkeypatch.setattr(reports, "CHARTS_DIR", tmp_path) - monkeypatch.setattr(reports, "INDEX_FILE", tmp_path / "reports_index.json") + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path)) monkeypatch.setattr( rendering, "plotly_bundle", diff --git a/tests/test_report_component_gallery.py b/tests/test_report_component_gallery.py index d3a97e7ae..397bdd09e 100644 --- a/tests/test_report_component_gallery.py +++ b/tests/test_report_component_gallery.py @@ -11,8 +11,7 @@ @pytest.fixture def reports_dir(tmp_path, monkeypatch): - monkeypatch.setattr(reports, "CHARTS_DIR", tmp_path) - monkeypatch.setattr(reports, "INDEX_FILE", tmp_path / "reports_index.json") + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path)) monkeypatch.setattr( rendering, "plotly_bundle", diff --git a/tests/test_report_egress_hydration.py b/tests/test_report_egress_hydration.py index fa6dff344..f04f0aae0 100644 --- a/tests/test_report_egress_hydration.py +++ b/tests/test_report_egress_hydration.py @@ -14,7 +14,6 @@ import pytest -import condor.reports as rep import condor.routine_hooks as routine_hooks import handlers.routines as hr from condor.reports import rendering @@ -68,8 +67,7 @@ def reports_dir(tmp_path, monkeypatch): ), encoding="utf-8", ) - monkeypatch.setattr(rep, "CHARTS_DIR", directory) - monkeypatch.setattr(rep, "INDEX_FILE", index) + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(directory)) return directory diff --git a/tests/test_report_subject.py b/tests/test_report_subject.py index dee595b21..6ad64a804 100644 --- a/tests/test_report_subject.py +++ b/tests/test_report_subject.py @@ -13,7 +13,6 @@ import pytest -import condor.reports as reports from condor.reports import ReportBuilder, store, subjects RUN = subjects.bot_run("brigado", "/data/bots/archive/run.sqlite") @@ -22,8 +21,7 @@ @pytest.fixture def reports_dir(tmp_path, monkeypatch): - monkeypatch.setattr(reports, "CHARTS_DIR", tmp_path) - monkeypatch.setattr(reports, "INDEX_FILE", tmp_path / "reports_index.json") + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path)) return tmp_path diff --git a/tests/test_reports_attribution.py b/tests/test_reports_attribution.py index 57ad412b3..272cc3c3a 100644 --- a/tests/test_reports_attribution.py +++ b/tests/test_reports_attribution.py @@ -9,8 +9,7 @@ @pytest.fixture def reports_dir(tmp_path, monkeypatch): - monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path) - monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports_index.json") + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path)) return tmp_path diff --git a/tests/test_reports_ownership.py b/tests/test_reports_ownership.py index bb0fba0e0..0f122a14f 100644 --- a/tests/test_reports_ownership.py +++ b/tests/test_reports_ownership.py @@ -70,8 +70,7 @@ def reports_dir(tmp_path, monkeypatch): ), encoding="utf-8", ) - monkeypatch.setattr(rep, "CHARTS_DIR", directory) - monkeypatch.setattr(rep, "INDEX_FILE", index) + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(directory)) monkeypatch.setattr(routes, "get_config_manager", lambda: FakeConfigManager()) return directory @@ -162,8 +161,7 @@ def test_missing_report_is_404_not_403(reports_dir): def test_save_records_the_owner_from_attribute_owner(tmp_path, monkeypatch): - monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path) - monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports_index.json") + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path)) async def go(): with rep.attribute_owner(USER.id): @@ -184,8 +182,7 @@ async def go(): def test_list_reports_owner_filter_drops_foreign_and_ownerless(tmp_path, monkeypatch): - monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path) - monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports_index.json") + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path)) async def go(): for owner, title in ((USER.id, "Mine"), (OTHER.id, "Theirs"), (None, "Old")): @@ -205,8 +202,7 @@ async def go(): def test_live_report_update_preserves_the_owner(tmp_path, monkeypatch): """An in-place update keeps the user_id stamped at first save.""" - monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path) - monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports_index.json") + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path)) async def go(): live = rep.LiveReport("Live", source_name="loop") diff --git a/tests/test_routine_instance_retention.py b/tests/test_routine_instance_retention.py index a77a022ab..aa00c11e7 100644 --- a/tests/test_routine_instance_retention.py +++ b/tests/test_routine_instance_retention.py @@ -28,8 +28,7 @@ @pytest.fixture def reports_dir(tmp_path, monkeypatch): - monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path) - monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports_index.json") + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path)) return tmp_path diff --git a/tests/test_routine_result_retention.py b/tests/test_routine_result_retention.py index cd086fb4f..e80f374c3 100644 --- a/tests/test_routine_result_retention.py +++ b/tests/test_routine_result_retention.py @@ -27,8 +27,7 @@ @pytest.fixture def reports_dir(tmp_path, monkeypatch): - monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path) - monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports_index.json") + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path)) return tmp_path diff --git a/tests/test_routine_run_report_link.py b/tests/test_routine_run_report_link.py index 0b5069795..e9ee782e0 100644 --- a/tests/test_routine_run_report_link.py +++ b/tests/test_routine_run_report_link.py @@ -17,8 +17,7 @@ @pytest.fixture def reports_dir(tmp_path, monkeypatch): - monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path) - monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports_index.json") + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path)) return tmp_path diff --git a/tests/test_routine_telegram_report_source.py b/tests/test_routine_telegram_report_source.py index 887ace080..0f0b00fce 100644 --- a/tests/test_routine_telegram_report_source.py +++ b/tests/test_routine_telegram_report_source.py @@ -35,8 +35,7 @@ async def send_message(self, chat_id, text, **kwargs): @pytest.fixture def reports_dir(tmp_path, monkeypatch): - monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path) - monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports_index.json") + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path)) return tmp_path diff --git a/tests/test_share_report_path_containment.py b/tests/test_share_report_path_containment.py index 353bcaba8..58bfc2ef1 100644 --- a/tests/test_share_report_path_containment.py +++ b/tests/test_share_report_path_containment.py @@ -1,6 +1,6 @@ """SEC-268: Telegram's share button reads reports through the guarded helper. -``_share_report`` used to join ``CHARTS_DIR / entry["filename"]`` by hand and +``_share_report`` used to join the reports dir and ``entry["filename"]`` and ``open()`` the result, checking only ``.exists()``. A poisoned or hand-edited ``reports_index.json`` could therefore make the share button mail an arbitrary file to a Telegram chat. It now reads through @@ -14,7 +14,6 @@ import pytest -import condor.reports as rep import handlers.routines as hr CHAT_ID = 4242 @@ -83,8 +82,7 @@ def reports_dir(tmp_path, monkeypatch): ), encoding="utf-8", ) - monkeypatch.setattr(rep, "CHARTS_DIR", directory) - monkeypatch.setattr(rep, "INDEX_FILE", index) + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(directory)) return directory diff --git a/tests/test_suite_isolation.py b/tests/test_suite_isolation.py index 3fd197340..849c9ac60 100644 --- a/tests/test_suite_isolation.py +++ b/tests/test_suite_isolation.py @@ -9,15 +9,17 @@ bot may be running while the suite is), so what is pinned here is the mechanism instead: with the autouse fixture in ``conftest.py`` active, every path a writer can build resolves outside the repository. If someone reintroduces a -root that ignores ``CONDOR_RUNTIME_ROOT``, ``CONDOR_DATA_DIR`` or -``CONDOR_AGENTS_ROOT`` or ``CONDOR_STOCK_AGENTS_ROOT``, one of these fails. +root that ignores ``CONDOR_RUNTIME_ROOT``, ``CONDOR_DATA_DIR``, +``CONDOR_AGENTS_ROOT``, ``CONDOR_STOCK_AGENTS_ROOT`` or ``CONDOR_REPORTS_DIR``, +one of these fails. -All three roots are covered, because all three are durable and all three were +All four roots are covered, because all four are durable and all four were reachable from a test: ``.condor/`` (conversations, delegations, state, telemetry), ``data/`` (the bell, routine hooks, backtests, code runs) -- whose three cwd-relative constants READ-215 replaced with resolvers -- and ``agents/`` (every agent's per-user memory and skill library), the one READ-215 -did not reach and CORR-220 did. +did not reach and CORR-220 did -- and ``reports/``, which had a constant *and* +a cwd-relative twin until ARCH-605 gave it a resolver. """ from pathlib import Path @@ -25,6 +27,8 @@ from condor import backtest_store, code_runs, paths from condor.agents.delegate import DelegateTask, _record_dir from condor.memory import paths as memory_paths +from condor.reports import store as report_store +from handlers.bots import archived_report REPO = Path(__file__).resolve().parent.parent @@ -111,6 +115,23 @@ def test_the_operational_store_is_isolated_too(): assert _outside_the_repo(paths.code_runs_dir()) +def test_the_report_output_root_is_isolated_too(): + """``reports/`` is the fourth durable root (ARCH-605). + + It was a module constant until the resolver arrived, so a test that forgot + the monkeypatch pair wrote an HTML report -- and an index row pointing at it + -- into the developer's live ``reports/``, where the dashboard then listed + it. Both producers are asserted: the store and the archived-bot report + writer, which used to build the *cwd-relative* name instead. + """ + assert paths.reports_dir() != REPO / "reports" + assert _outside_the_repo(paths.reports_dir()) + assert _outside_the_repo(paths.reports_index_path()) + assert _outside_the_repo(report_store._charts_dir()) + assert _outside_the_repo(report_store._index_file()) + assert _outside_the_repo(archived_report.ensure_reports_dir()) + + def test_the_default_stores_land_outside_the_install(): """The writers' own defaults, not a reconstruction of them. diff --git a/tests/test_web_report_assets.py b/tests/test_web_report_assets.py index dcd1ef3db..ab40dc882 100644 --- a/tests/test_web_report_assets.py +++ b/tests/test_web_report_assets.py @@ -69,8 +69,7 @@ def reports_dir(tmp_path, monkeypatch): ), encoding="utf-8", ) - monkeypatch.setattr(reports, "CHARTS_DIR", directory) - monkeypatch.setattr(reports, "INDEX_FILE", index) + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(directory)) monkeypatch.setattr(reports_routes, "get_config_manager", lambda: _NoAdmins()) return directory diff --git a/tests/test_web_report_serving.py b/tests/test_web_report_serving.py index 70cd47787..9100ad372 100644 --- a/tests/test_web_report_serving.py +++ b/tests/test_web_report_serving.py @@ -85,8 +85,7 @@ def reports_dir(tmp_path, monkeypatch): ), encoding="utf-8", ) - monkeypatch.setattr(reports, "CHARTS_DIR", directory) - monkeypatch.setattr(reports, "INDEX_FILE", index) + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(directory)) # Ownership (SEC-196) is orthogonal here: USER owns every seeded entry and # is not an admin, so these tests keep exercising serving/containment. monkeypatch.setattr(reports_routes, "get_config_manager", lambda: _NoAdmins()) From 9e35f04606c4e0453b1c843bd6d418bb96ae6c5a Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 17:31:56 +0300 Subject: [PATCH 011/154] Build the portfolio LP table with the builder the module already imports The CLMM section of get_portfolio_overview hand-rolled its own header, rule, column padding, address truncation and price formatting, the last inside two bare except blocks that rendered a malformed price as its raw value. It also capped the table at an inline [:10], so an account with 30 LP positions was shown 10 and nothing named the limit. Move the rendering to format_lp_positions_table in formatters/portfolio.py, built on TableBuilder/ColumnDef and truncate_address, with the cap as a named limit parameter defaulting to DEFAULT_LP_POSITIONS_LIMIT. A price that will not parse now reads N/A instead of passing for a real one, and the '... and N more' footer still reports the true remainder. --- .../hummingbot_api/formatters/__init__.py | 3 +- .../hummingbot_api/formatters/portfolio.py | 70 ++++++++++++++++++- mcp_servers/hummingbot_api/tools/portfolio.py | 55 ++------------- 3 files changed, 76 insertions(+), 52 deletions(-) diff --git a/mcp_servers/hummingbot_api/formatters/__init__.py b/mcp_servers/hummingbot_api/formatters/__init__.py index 087bec01d..8453067a7 100644 --- a/mcp_servers/hummingbot_api/formatters/__init__.py +++ b/mcp_servers/hummingbot_api/formatters/__init__.py @@ -61,7 +61,7 @@ ) # Portfolio formatters -from .portfolio import format_portfolio_as_table +from .portfolio import format_lp_positions_table, format_portfolio_as_table # Table builder for creating consistent tables from .table_builder import ColumnDef, TableBuilder, create_simple_table @@ -105,6 +105,7 @@ "format_controller_state", # Portfolio formatters "format_portfolio_as_table", + "format_lp_positions_table", # Executor formatters "format_executor_types_table", "format_executors_table", diff --git a/mcp_servers/hummingbot_api/formatters/portfolio.py b/mcp_servers/hummingbot_api/formatters/portfolio.py index 34c6242f3..a655640c7 100644 --- a/mcp_servers/hummingbot_api/formatters/portfolio.py +++ b/mcp_servers/hummingbot_api/formatters/portfolio.py @@ -6,7 +6,15 @@ from typing import Any -from .base import format_number, format_table_separator, get_field +from .base import format_number, format_table_separator, get_field, truncate_address +from .table_builder import ColumnDef, TableBuilder + +# Default number of LP position rows rendered in the CLMM table. The fetch is +# not bounded per venue (get_positions_owned returns every position a venue +# holds), so the table is capped to keep an LP-heavy account from pushing +# hundreds of rows into the model's context. Callers that want more can raise +# it; the "... and N more" footer always reports the true remainder. +DEFAULT_LP_POSITIONS_LIMIT = 50 def format_portfolio_as_table(portfolio_data: dict[str, Any]) -> str: @@ -69,3 +77,63 @@ def format_portfolio_as_table(portfolio_data: dict[str, Any]) -> str: return "No portfolio balances found." return f"{header}\n{separator}\n" + "\n".join(rows) + + +def _format_lp_price(value: Any) -> str: + """ + Format an LP position price bound. + + Malformed values raise ValueError/TypeError, which ColumnDef turns into the + column default ("N/A") instead of silently rendering the raw value. + """ + if value is None or value == "N/A": + return "N/A" + return f"{float(value):.4f}" + + +LP_POSITION_COLUMNS = [ + ColumnDef(name="connector", key="connector", width=10), + ColumnDef(name="trading_pair", key="trading_pair", width=15), + ColumnDef( + name="lower_price", key="lower_price", width=11, formatter=_format_lp_price + ), + ColumnDef( + name="upper_price", key="upper_price", width=11, formatter=_format_lp_price + ), + ColumnDef( + name="position_address", + key="position_address", + width=17, + formatter=lambda address: truncate_address(str(address)), + ), +] + + +def format_lp_positions_table( + positions: list[dict[str, Any]], limit: int = DEFAULT_LP_POSITIONS_LIMIT +) -> str: + """ + Format open CLMM (LP) positions as a table string. + + Columns: connector | trading_pair | lower_price | upper_price | position_address + + Args: + positions: List of open LP position dictionaries + limit: Maximum number of rows to render (default: + DEFAULT_LP_POSITIONS_LIMIT). Positions beyond the limit are reported + by a "... and N more open positions" footer. + + Returns: + Formatted table string + """ + if not positions: + return "No active LP positions found" + + builder = TableBuilder(LP_POSITION_COLUMNS) + table = builder.build_with_title(positions[:limit], "Status: OPEN positions") + + remaining = len(positions) - limit + if remaining > 0: + table += f"\n... and {remaining} more open positions" + + return table diff --git a/mcp_servers/hummingbot_api/tools/portfolio.py b/mcp_servers/hummingbot_api/tools/portfolio.py index 73d46195d..f2ee09f48 100644 --- a/mcp_servers/hummingbot_api/tools/portfolio.py +++ b/mcp_servers/hummingbot_api/tools/portfolio.py @@ -12,7 +12,10 @@ from typing import Any, Literal from mcp_servers.hummingbot_api.exceptions import ToolError -from mcp_servers.hummingbot_api.formatters import format_portfolio_as_table +from mcp_servers.hummingbot_api.formatters import ( + format_lp_positions_table, + format_portfolio_as_table, +) from mcp_servers.hummingbot_api.hummingbot_client import HummingbotClient from mcp_servers.hummingbot_api.tools import trading as trading_tools @@ -343,55 +346,7 @@ async def get_active_orders(): open_positions = lp_positions # Format LP positions - show all open positions with real-time data - if open_positions: - lp_table_lines = ["Status: OPEN positions", ""] - lp_table_lines.append( - "connector | trading_pair | lower_price | upper_price | position_address" - ) - lp_table_lines.append("-" * 100) - - for pos in open_positions[:10]: # Show up to 10 open positions - connector = pos.get("connector", "N/A") - trading_pair = pos.get("trading_pair", "N/A") - lower_price = pos.get("lower_price", "N/A") - upper_price = pos.get("upper_price", "N/A") - position_address = pos.get("position_address", "N/A") - - # Format prices - if lower_price != "N/A" and isinstance( - lower_price, (int, float, str) - ): - try: - lower_price = f"{float(lower_price):.4f}" - except: - pass - - if upper_price != "N/A" and isinstance( - upper_price, (int, float, str) - ): - try: - upper_price = f"{float(upper_price):.4f}" - except: - pass - - # Truncate position address - if position_address != "N/A" and len(position_address) > 20: - position_address = ( - f"{position_address[:8]}...{position_address[-6:]}" - ) - - lp_table_lines.append( - f"{connector[:10]:10} | {trading_pair[:15]:15} | {str(lower_price)[:11]:11} | {str(upper_price)[:11]:11} | {position_address}" - ) - - if len(open_positions) > 10: - lp_table_lines.append( - f"... and {len(open_positions) - 10} more open positions" - ) - - lp_table = "\n".join(lp_table_lines) - else: - lp_table = "No active LP positions found" + lp_table = format_lp_positions_table(open_positions) sections.append( { From 1b02bcbafe5a52ca4189df1916c1379d964c3f9c Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 17:41:38 +0300 Subject: [PATCH 012/154] Let the condor server's tools carry their enums in the signature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TOOL_STYLE.md says "Literal for every enum" and "the signature is the schema", and the hummingbot server obeys it. The condor server had none of it: every one of its nine dispatch tools took a bare `action: str` and spelled the closed set out in prose instead, so a typo cost a full round trip that came back "Unknown action" instead of a client-side error naming the parameter. Same gap on `on_complete`, `parse_mode`, memory `type`. Each `action` is now the Literal of exactly the keys its impl dispatches on, so the prose that duplicated it goes. The three FEAT-068 tools keep their legacy `*_agent` spellings ON the Literal: FastMCP validates arguments server-side, so leaving them off would have pydantic reject `start_agent` before `_resolve_action` ever answers to it, and the danger gate and the mutation log both deliberately know that spelling. With the actions on the signature, `control_agent` stops being a foreign tool the gate test has to scrape dicts for — it reads its actions off the annotation like every other gated tool, and the apologetic docstring goes with it. `trading_agent_journal_read`'s `section` stays a `str`: it takes `run:N`, so its set is not closed and no Literal can spell it. --- mcp_servers/condor/server.py | 110 ++++++++++++++++----- tests/test_dangerous_gate_names_resolve.py | 38 ++++--- 2 files changed, 105 insertions(+), 43 deletions(-) diff --git a/mcp_servers/condor/server.py b/mcp_servers/condor/server.py index e32fb9135..407b62af2 100644 --- a/mcp_servers/condor/server.py +++ b/mcp_servers/condor/server.py @@ -5,7 +5,7 @@ """ from collections.abc import Iterable -from typing import Any +from typing import Any, Literal from mcp.server.fastmcp import FastMCP @@ -355,11 +355,11 @@ def _build_instructions() -> str: @handle_errors("delegate task") @telemetry_taps.tracked("delegate") async def delegate( - action: str, + action: Literal["start", "ask", "list", "get", "stop"], agent: str = "", task: str = "", task_id: str = "", - on_complete: str = "notify", + on_complete: Literal["notify", "resume"] = "notify", timeout_sec: int = 0, context: str = "", ) -> dict: @@ -410,7 +410,7 @@ async def delegate( of itself; the recursion stops at depth one. Args: - action: start | ask | list | get | stop. + action: What to do with the agent or the delegation. agent: Agent slug to reach (for start/ask). For "start" your own slug is allowed and means "a background session of me"; for "ask" it is refused, because asking yourself is a round trip through your own @@ -453,13 +453,13 @@ async def delegate( @telemetry_taps.tracked("send_notification") async def send_notification( text: str, - parse_mode: str = "Markdown", + parse_mode: Literal["Markdown", "HTML"] = "Markdown", ) -> dict: """Send a Telegram message to the user. Args: text: Message text to send. - parse_mode: Telegram parse mode ("Markdown" or "HTML"). Default: "Markdown". + parse_mode: Telegram parse mode. Default: "Markdown". Returns: {"sent": true} on success, {"error": "..."} on failure. @@ -470,7 +470,20 @@ async def send_notification( @handle_errors("manage routines") @telemetry_taps.tracked("manage_routines") async def manage_routines( - action: str, + action: Literal[ + "list", + "describe", + "run", + "run_async", + "get_instance", + "start", + "stop", + "list_instances", + "create_routine", + "read_routine", + "edit_routine", + "delete_routine", + ], name: str | None = None, config: dict | None = None, agent: str | None = None, @@ -543,7 +556,7 @@ async def manage_routines( @telemetry_taps.tracked("run_code") async def run_code( code: str | None = None, - action: str = "run", + action: Literal["run", "history", "get"] = "run", label: str = "", timeout: int | None = None, run_id: str | None = None, @@ -591,7 +604,7 @@ async def run_code( (requires run_id) Args: - action: run | history | get. + action: What to do — execute a snippet or read past ones. code: The Python snippet to execute (for "run"). label: Short purpose of the run ("returns of SOL 1h"), shown in history and used as the report source name. @@ -612,7 +625,7 @@ async def run_code( @handle_errors("manage servers") @telemetry_taps.tracked("manage_servers") async def manage_servers( - action: str, + action: Literal["list", "status"], name: str | None = None, ) -> dict: """Manage Hummingbot API servers — and answer where you are pointed, as whom. @@ -625,7 +638,7 @@ async def manage_servers( - "status": Check if a server is online (optional name, defaults to active server) Args: - action: The action to perform (list, status) + action: The action to perform. name: Server name (optional for status) Returns: @@ -693,7 +706,21 @@ async def get_available_models( @handle_errors("manage agents") @telemetry_taps.tracked("manage_agents") async def manage_agents( - action: str, + action: Literal[ + "list", + "create", + "get", + "update", + "delete", + "publish", + # Legacy funnel-era spellings this family still answers to. + "list_agent_definitions", + "create_agent", + "get_agent", + "update_agent", + "delete_agent", + "publish_agent", + ], agent_slug: str | None = None, name: str | None = None, description: str | None = None, @@ -733,7 +760,7 @@ async def manage_agents( present the moment an install pulls. Args: - action: One of list, create, get, update, delete, publish. + action: What to do with the agent definition. agent_slug: The agent to act on (get/update/delete). name: Agent name (create/update). description: Agent description (create/update). @@ -780,7 +807,19 @@ async def manage_agents( @handle_errors("manage strategies") @telemetry_taps.tracked("manage_strategies") async def manage_strategies( - action: str, + action: Literal[ + "list", + "get", + "create", + "update", + "delete", + # Legacy funnel-era spellings this family still answers to. + "list_strategies", + "get_strategy", + "create_strategy", + "update_strategy", + "delete_strategy", + ], strategy_id: str | None = None, agent_slug: str | None = None, name: str | None = None, @@ -807,7 +846,7 @@ async def manage_strategies( - "delete": Delete a strategy (requires strategy_id). Args: - action: One of list, get, create, update, delete. + action: What to do with the strategy. strategy_id: Strategy key "agent_slug.strategy_slug" (get/update/delete). agent_slug: The owning agent — required to create. name: Strategy name (create/update). @@ -838,7 +877,23 @@ async def manage_strategies( @handle_errors("control agent") @telemetry_taps.tracked("control_agent") async def control_agent( - action: str, + action: Literal[ + "list", + "start", + "stop", + "pause", + "resume", + "shutdown", + "get_state", + "set_state", + # Legacy funnel-era spellings this family still answers to. + "list_agents", + "start_agent", + "stop_agent", + "pause_agent", + "resume_agent", + "shutdown_agent", + ], agent_id: str | None = None, strategy_id: str | None = None, config: dict | None = None, @@ -876,7 +931,7 @@ async def control_agent( agent_id, so an instance only ever sees its own. Args: - action: One of list, start, stop, shutdown, pause, resume, get_state, set_state. + action: The lifecycle action to take on the loop. agent_id: The running instance (everything except list and start). strategy_id: Strategy key "agent_slug.strategy_slug", or a bare agent slug (start only). @@ -909,11 +964,11 @@ async def control_agent( @handle_errors("manage memory") @telemetry_taps.tracked("manage_memory") async def manage_memory( - action: str, + action: Literal["write", "read", "search", "list", "delete", "audit"], name: str | None = None, content: str | None = None, description: str | None = None, - type: str = "fact", + type: Literal["preference", "fact", "feedback", "reference"] = "fact", query: str | None = None, max_entries: int = 30, ) -> dict: @@ -943,11 +998,11 @@ async def manage_memory( - "audit": Recent write/delete events (who changed what). Args: - action: write | read | search | list | delete | audit + action: What to do with the memory store. name: Short kebab/snake name for the memory (e.g. "report-in-usd"). content: The full fact/body (required for write). description: One-line summary shown in the index (required for write). - type: preference | fact | feedback | reference (default "fact"). + type: What kind of memory this is (default "fact"). query: Search string (for search). max_entries: Cap for search/audit results (default 30). @@ -962,7 +1017,16 @@ async def manage_memory( @handle_errors("manage skill") @telemetry_taps.tracked("manage_skill") async def manage_skill( - action: str, + action: Literal[ + "list", + "read", + "search", + "create", + "edit", + "delete", + "read_file", + "write_file", + ], name: str | None = None, description: str | None = None, when_to_use: str | None = None, @@ -1027,7 +1091,7 @@ async def manage_skill( - "delete": Remove a skill (requires name). Args: - action: read | read_file | write_file | search | list | create | edit | delete + action: What to do with the skills library. name: Short kebab/snake name (e.g. "grid-en-band-walk"). description: One-line summary (create/edit). when_to_use: The trigger/condition for the playbook (create/edit). diff --git a/tests/test_dangerous_gate_names_resolve.py b/tests/test_dangerous_gate_names_resolve.py index c7b04dfe5..022cd7a5a 100644 --- a/tests/test_dangerous_gate_names_resolve.py +++ b/tests/test_dangerous_gate_names_resolve.py @@ -26,20 +26,25 @@ DANGEROUS_TOOLS, is_dangerous_tool_call, ) +from mcp_servers.condor import server as condor_mcp_server from mcp_servers.hummingbot_api import server as mcp_server -# Gate names that belong to a different MCP server than hummingbot_api. -# ``control_agent`` is on the condor orchestration server, and gets its own -# resolution test below rather than this file's Literal-reading one — its -# actions are a dict in the tool module, not an annotation. -_FOREIGN_TOOLS = {"place_order", "control_agent"} +# Gate names that no MCP server registers as a tool of its own. +# ``place_order`` is gated by name without a tool behind it. +_FOREIGN_TOOLS = {"place_order"} def _registered_tools() -> dict: - """Every function the hummingbot_api MCP server registers as a tool.""" + """Every function either MCP server registers as a tool. + + Both are read because the gate spans them: ``control_agent`` lives on the + condor orchestration server and the rest on hummingbot_api. The two share + no tool name, so one flat mapping resolves every gated name. + """ return { name: obj.fn if hasattr(obj, "fn") else obj - for name, obj in vars(mcp_server).items() + for server in (mcp_server, condor_mcp_server) + for name, obj in vars(server).items() if callable(obj) and not name.startswith("_") } @@ -302,20 +307,13 @@ def test_gateway_config_fails_closed_on_an_unreadable_resource(): def _control_actions() -> set[str]: """Every action string ``control_agent`` actually accepts. - Its actions are not a ``Literal`` on the signature — the tool takes a bare - ``str`` and resolves it through ``_resolve_action``, which accepts both the - short spelling (``start``) and the legacy internal one (``start_agent``). - Both reach the same lifecycle call, so the gate has to know both. + Read off the signature like every other gated tool (ARCH-568). The + ``Literal`` carries both the short spelling (``start``) and the legacy + internal one (``start_agent``), because ``_resolve_action`` still answers + to both and they reach the same lifecycle call — so the gate has to know + both, and the schema has to advertise both. """ - from mcp_servers.condor.tools import trading_agent - - accepted = set(trading_agent._CONTROL_ACTIONS) - accepted.update( - action - for action, (owner, _call) in trading_agent._ACTION_OWNER.items() - if owner == "control_agent" - ) - return accepted + return _action_literals("control_agent") def test_control_agent_is_registered_by_the_condor_server(): From c9702ccd84df9f1d62f3348bfc42558534db05bb Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 17:53:28 +0300 Subject: [PATCH 013/154] Send the bots WS frame with the same enrichment the REST route adds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ARCH-050 unified the bots-page transform into build_bots_page, but not its inputs: only the REST route supplied ctrl_configs / bot_runs / latest_perf, so every WS frame shipped controllers with config={}, no deployed_at, the raw performance key as the controller id and a connector/pair guessed by splitting that key on underscores. The frontend repaired each of those fields out of the previous REST payload, joined on a controller key the two paths computed differently — so the repair could miss and let a blank row win. The three fetches move out of the list_bots closure into condor.fetchers.bots.fetch_bots_enrichment, registered as ServerDataType.BOTS_ENRICHMENT with a 60s TTL. Both delivery paths render through one enriched_bots_page helper, so a frame and GET /servers/{name}/bots are the same page, and the 5s frame reads the warm cache instead of asking Hummingbot again. The compensating merge in shared-socket.ts is now a plain cache replace. --- condor/fetchers/__init__.py | 7 +- condor/fetchers/bots.py | 208 ++++++++++++++++++++++++++- condor/server_data_service.py | 11 ++ condor/web/routes/bots.py | 209 +++++----------------------- condor/web/streams/hummingbot_ws.py | 38 +++-- condor/web/ws_manager.py | 19 +-- frontend/src/lib/shared-socket.ts | 41 +----- tests/test_bots_page_transform.py | 14 +- tests/test_bots_ws_frame_parity.py | 171 +++++++++++++++++++++++ 9 files changed, 468 insertions(+), 250 deletions(-) create mode 100644 tests/test_bots_ws_frame_parity.py diff --git a/condor/fetchers/__init__.py b/condor/fetchers/__init__.py index b952e0478..2dea38184 100644 --- a/condor/fetchers/__init__.py +++ b/condor/fetchers/__init__.py @@ -34,7 +34,11 @@ façade consumer; they are not the package's public surface. """ -from condor.fetchers.bots import fetch_bot_runs, fetch_bots_status +from condor.fetchers.bots import ( + fetch_bot_runs, + fetch_bots_enrichment, + fetch_bots_status, +) from condor.fetchers.connectors import ( fetch_available_cex_connectors, fetch_connectors, @@ -101,6 +105,7 @@ "get_executor_fees", "extract_executors_list", "fetch_bots_status", + "fetch_bots_enrichment", "fetch_bot_runs", "fetch_current_price", "fetch_candles", diff --git a/condor/fetchers/bots.py b/condor/fetchers/bots.py index 99663bc63..f3fadce3f 100644 --- a/condor/fetchers/bots.py +++ b/condor/fetchers/bots.py @@ -1,10 +1,16 @@ """Fetch bot data from Hummingbot API.""" +import asyncio import logging -from typing import Any, Optional +from typing import Any, NamedTuple, Optional logger = logging.getLogger(__name__) +# Per-call budget for one of the optional enrichment fetches below. The bots +# page renders without them (no age / config / DB perf), so a slow server +# should cost us those columns, never the whole response. +ENRICHMENT_TIMEOUT = 15.0 + def extract_bots_list(result: Any) -> list[dict]: """Normalize the various API response formats into a list of bot dicts.""" @@ -66,8 +72,15 @@ def build_bots_page( """Transform raw BOTS_STATUS data into a BotsPageResponse-shaped dict. Single source of truth for the {controllers, bots, total_pnl, total_volume} - transform, shared by the REST route (with enrichment data) and the WS - broadcast path (without enrichment, so all kwargs degrade to empty maps). + transform, shared by the REST route and the WS broadcast path. Both feed it + the same enrichment, fetched once by :func:`fetch_bots_enrichment` and + cached per server, so a WS frame carries the same config, deployed_at, + controller_id and connector/trading_pair as the REST body. + + The kwargs still default to empty maps, but that is degradation, not a + supported mode: with no configs a controller reports ``config={}``, no + deploy age, its raw performance key as its id, and a connector/pair guessed + by splitting that key on underscores. Args: raw_status: Raw bot status API response (any of the shapes handled by @@ -218,3 +231,192 @@ async def fetch_bots_status(client, **_kw): async def fetch_bot_runs(client, **_kw): """Fetch bot run history.""" return await client.bot_orchestration.get_bot_runs() + + +# ── Bots-page enrichment ── + + +class BotsEnrichment(NamedTuple): + """The three optional inputs :func:`build_bots_page` enriches its output with.""" + + #: Controller configs keyed by config id *and* controller name. + ctrl_configs: dict[str, dict] + #: Deployed-at timestamps keyed by bot name. + bot_runs: dict[str, str] + #: Latest DB performance snapshots keyed by controller id. + latest_perf: dict[str, dict] + + @classmethod + def empty(cls) -> "BotsEnrichment": + """A fresh, empty enrichment — what a caller falls back to on failure.""" + return cls({}, {}, {}) + + +def _extract_perf_snapshots(result: Any) -> list[dict]: + """Normalize controller performance API response into a list of snapshot dicts.""" + if isinstance(result, list): + return [s for s in result if isinstance(s, dict)] + if isinstance(result, dict): + data = result.get("data", result.get("snapshots", result.get("records", []))) + if isinstance(data, list): + return [s for s in data if isinstance(s, dict)] + if isinstance(data, dict): + out = [] + for key, val in data.items(): + if isinstance(val, dict): + val.setdefault("controller_id", key) + out.append(val) + elif isinstance(val, list): + for item in val: + if isinstance(item, dict): + item.setdefault("controller_id", key) + out.append(item) + return out + return [] + + +def _collect_bot_runs(result: Any, runs: dict[str, str]) -> None: + """Merge a bot-runs API response into ``runs`` (bot_name -> deployed_at).""" + if not isinstance(result, dict): + return + runs_data = result.get("data", result) + if isinstance(runs_data, dict): + for bot_name, run_info in runs_data.items(): + if isinstance(run_info, dict): + deployed = run_info.get("deployed_at") or run_info.get("created_at") + if deployed: + runs[bot_name] = str(deployed) + elif isinstance(run_info, str): + runs[bot_name] = run_info + elif isinstance(runs_data, list): + for run in runs_data: + if isinstance(run, dict): + bn = run.get("bot_name", "") + deployed = run.get("deployed_at") or run.get("created_at") + if bn and deployed: + runs[bn] = str(deployed) + + +async def _with_enrichment_timeout(coro, label: str, default: Any) -> Any: + """Cap one enrichment call so a slow server degrades instead of hanging. + + Each fetcher gets its own budget: one slow endpoint must not cost us the + other two. + """ + try: + return await asyncio.wait_for(coro, timeout=ENRICHMENT_TIMEOUT) + except asyncio.TimeoutError: + logger.warning( + "Bots enrichment '%s' timed out after %.0fs", label, ENRICHMENT_TIMEOUT + ) + return default + + +async def _fetch_ctrl_configs(client, bot_names: list[str]) -> dict[str, dict]: + """Controller configs for the given bots, keyed by config id and by name.""" + configs_map: dict[str, dict] = {} + if not bot_names: + return configs_map + + async def _get_one(bn: str): + try: + configs = await client.controllers.get_bot_controller_configs(bn) + if isinstance(configs, list): + for cfg in configs: + cid = cfg.get("id") or cfg.get("controller_id", "") + if cid: + configs_map[cid] = cfg + cname = cfg.get("controller_name", "") + if cname and cname != cid: + configs_map[cname] = cfg + except Exception: + pass + + await asyncio.gather(*[_get_one(bn) for bn in bot_names]) + return configs_map + + +async def _fetch_deployed_runs(client, bot_names: list[str]) -> dict[str, str]: + """Deployed-at timestamps keyed by bot name, for the Age column. + + Filtered to DEPLOYED on purpose: the unfiltered listing (:func:`fetch_bot_runs`) + also returns ARCHIVED runs, each carrying a multi-KB ``final_status`` blob. + On a remote server that is a multi-MB, multi-minute response that stalls the + whole page (brigado: 4.5 MB / 274s unfiltered vs 121 KB / 5s filtered), + leaving every bot without an age. + """ + runs: dict[str, str] = {} + try: + _collect_bot_runs( + await client.bot_orchestration.get_bot_runs(deployment_status="DEPLOYED"), + runs, + ) + except Exception: + logger.debug("Bot runs not available") + + # Any active bot the filtered listing missed gets a targeted lookup + # (~1 KB each) rather than falling back to the unfiltered listing. + missing = [bn for bn in bot_names if bn not in runs] + if missing: + + async def _get_one_run(bn: str): + try: + _collect_bot_runs( + await client.bot_orchestration.get_bot_runs(bot_name=bn, limit=1), + runs, + ) + except Exception: + pass + + await asyncio.gather(*[_get_one_run(bn) for bn in missing]) + return runs + + +async def _fetch_latest_perf(client) -> dict[str, dict]: + """Latest controller performance snapshots from the DB, keyed by controller id.""" + perf_map: dict[str, dict] = {} + try: + perf_result = await client.bot_orchestration.get_latest_controller_performance() + for snap in _extract_perf_snapshots(perf_result): + cid = snap.get("controller_id", "") + if cid: + perf_map[cid] = snap + except Exception: + logger.debug("Latest controller performance not available") + return perf_map + + +async def fetch_bots_enrichment( + client, bots_list: Optional[list[dict]] = None, **_kw +) -> BotsEnrichment: + """Fetch the three enrichment maps :func:`build_bots_page` takes. + + Fetched as one unit so both delivery paths can share a single cached answer: + the REST route and every WS ``bots:`` frame read it through + ServerDataService, instead of the REST route enriching and the WS path + shipping a stripped page the client has to repair. + + ``bots_list`` names the bots to enrich. It is optional because the + ServerDataService fetch is handed only a client: with no list, the active + bots are looked up first. The three calls run concurrently, each under its + own :data:`ENRICHMENT_TIMEOUT`, and every failure degrades to an empty map. + """ + if bots_list is None: + try: + bots_list = extract_bots_list(await fetch_bots_status(client)) + except Exception as e: + logger.debug("Bot status not available for enrichment: %s", e) + bots_list = [] + + bot_names = [bn for b in bots_list if (bn := b.get("bot_name", ""))] + + ctrl_configs, bot_runs, latest_perf = await asyncio.gather( + _with_enrichment_timeout( + _fetch_ctrl_configs(client, bot_names), "controller configs", {} + ), + _with_enrichment_timeout( + _fetch_deployed_runs(client, bot_names), "bot runs", {} + ), + _with_enrichment_timeout(_fetch_latest_perf(client), "latest performance", {}), + ) + return BotsEnrichment(ctrl_configs, bot_runs, latest_perf) diff --git a/condor/server_data_service.py b/condor/server_data_service.py index 11c4bab38..3b41e7986 100644 --- a/condor/server_data_service.py +++ b/condor/server_data_service.py @@ -43,6 +43,10 @@ class ServerDataType(Enum): TRADING_RULES = "trading_rules" CONNECTORS = "connectors" BOTS_STATUS = "bots_status" + #: The controller configs, deploy timestamps and DB performance the + #: bots page is enriched with — fetched as one unit so the REST route + #: and every WS bots frame render from the same answer. + BOTS_ENRICHMENT = "bots_enrichment" EXECUTORS = "executors" BOT_RUNS = "bot_runs" CANDLE_CONNECTORS = "candle_connectors" @@ -110,6 +114,11 @@ def interval_for(self, params: Dict[str, str]) -> float: ServerDataType.TRADING_RULES: DataTypeDefaults(interval=300, ttl=600), ServerDataType.CONNECTORS: DataTypeDefaults(interval=300, ttl=600), ServerDataType.BOTS_STATUS: DataTypeDefaults(interval=5, ttl=30), + # Enrichment moves far slower than status: a controller config, a deploy + # timestamp and a DB performance snapshot do not change between two 5s + # frames, and the fetch costs one call per bot. A minute of staleness + # buys eleven of every twelve frames a free, warm read. + ServerDataType.BOTS_ENRICHMENT: DataTypeDefaults(interval=30, ttl=60), ServerDataType.EXECUTORS: DataTypeDefaults(interval=2, ttl=30), ServerDataType.BOT_RUNS: DataTypeDefaults(interval=30, ttl=120), ServerDataType.CANDLE_CONNECTORS: DataTypeDefaults(interval=300, ttl=600), @@ -887,6 +896,7 @@ def register_default_fetches() -> None: fetch_active_orders, fetch_available_cex_connectors, fetch_bot_runs, + fetch_bots_enrichment, fetch_bots_status, fetch_candle_connectors, fetch_connectors, @@ -924,6 +934,7 @@ def register_default_fetches() -> None: sds.register_fetch(ServerDataType.ALL_CONNECTORS, fetch_connectors) sds.register_fetch(ServerDataType.VENUES, partial(fetch_venues, strict=True)) sds.register_fetch(ServerDataType.BOTS_STATUS, fetch_bots_status) + sds.register_fetch(ServerDataType.BOTS_ENRICHMENT, fetch_bots_enrichment) sds.register_fetch(ServerDataType.EXECUTORS, fetch_executors) sds.register_fetch(ServerDataType.BOT_RUNS, fetch_bot_runs) sds.register_fetch(ServerDataType.CANDLE_CONNECTORS, fetch_candle_connectors) diff --git a/condor/web/routes/bots.py b/condor/web/routes/bots.py index 61ca416d3..1ac8c2a87 100644 --- a/condor/web/routes/bots.py +++ b/condor/web/routes/bots.py @@ -9,7 +9,7 @@ from fastapi import APIRouter, Depends, HTTPException from condor.controller_configs import clean_config_for_save -from condor.fetchers.bots import build_bots_page, extract_bots_list +from condor.fetchers.bots import BotsEnrichment, build_bots_page, extract_bots_list from condor.web.auth import require_server_access from condor.web.models import ( AvailableControllersResponse, @@ -31,11 +31,6 @@ router = APIRouter(tags=["bots"]) -# Per-call budget for the optional enrichment fetches on the bots page. The -# page renders without them (no age / config / DB perf), so a slow server -# should cost us those columns, never the whole response. -ENRICHMENT_TIMEOUT = 15.0 - # ── Transitional state store ── # Tracks bots/controllers that have been sent a stop command but haven't # finished shutting down yet. Auto-expires after TTL seconds. @@ -187,55 +182,39 @@ def _parse_bot(bot: dict) -> BotInfo: ) -def _extract_perf_snapshots(result: Any) -> list[dict]: - """Normalize controller performance API response into a list of snapshot dicts.""" - if isinstance(result, list): - return [s for s in result if isinstance(s, dict)] - if isinstance(result, dict): - data = result.get("data", result.get("snapshots", result.get("records", []))) - if isinstance(data, list): - return [s for s in data if isinstance(s, dict)] - if isinstance(data, dict): - out = [] - for key, val in data.items(): - if isinstance(val, dict): - val.setdefault("controller_id", key) - out.append(val) - elif isinstance(val, list): - for item in val: - if isinstance(item, dict): - item.setdefault("controller_id", key) - out.append(item) - return out - return [] - - -def _collect_bot_runs(result: Any, runs: dict[str, str]) -> None: - """Merge a bot-runs API response into ``runs`` (bot_name -> deployed_at).""" - if not isinstance(result, dict): - return - runs_data = result.get("data", result) - if isinstance(runs_data, dict): - for bot_name, run_info in runs_data.items(): - if isinstance(run_info, dict): - deployed = run_info.get("deployed_at") or run_info.get("created_at") - if deployed: - runs[bot_name] = str(deployed) - elif isinstance(run_info, str): - runs[bot_name] = run_info - elif isinstance(runs_data, list): - for run in runs_data: - if isinstance(run, dict): - bn = run.get("bot_name", "") - deployed = run.get("deployed_at") or run.get("created_at") - if bn and deployed: - runs[bn] = str(deployed) +async def enriched_bots_page(name: str, raw_status: Any) -> dict: + """Build the bots page for a server from raw status plus cached enrichment. + + The one place the two delivery paths meet: the REST route and every WS + ``bots:`` frame render through this, so a frame carries the same + ``config``, ``deployed_at``, ``controller_id`` and connector/trading pair as + the REST body for the same raw payload — no client-side repair needed. + + The enrichment is read through ServerDataService, which holds it for a + minute: the 5s bots frame costs an extra Hummingbot round-trip only when + that cached answer has gone stale. + """ + from condor.server_data_service import ServerDataType, get_server_data_service + + try: + enrichment = await get_server_data_service().get_or_fetch( + name, ServerDataType.BOTS_ENRICHMENT + ) + except Exception as e: + logger.debug("Bots enrichment unavailable for '%s': %s", name, e) + enrichment = None + + ctrl_configs, bot_runs, latest_perf = enrichment or BotsEnrichment.empty() + return build_bots_page( + raw_status, + ctrl_configs=ctrl_configs, + bot_runs=bot_runs, + latest_perf=latest_perf, + ) @router.get("/servers/{name}/bots", response_model=BotsPageResponse) async def list_bots(name: str, user: WebUser = Depends(require_server_access)): - cm = get_config_manager() - from condor.server_data_service import ServerDataType, get_server_data_service try: @@ -255,136 +234,10 @@ async def list_bots(name: str, user: WebUser = Depends(require_server_access)): error_hint="Unable to reach server", ) - # Get client for enrichment calls - try: - client = await cm.get_client(name) - except Exception: - client = None - bots_list = extract_bots_list(result) logger.info("Server '%s': found %d bot(s)", name, len(bots_list)) - # Pre-fetch controller configs, bot runs, AND latest controller performance concurrently - ctrl_configs: dict[str, dict] = {} - bot_runs: dict[str, str] = {} - latest_perf: dict[str, dict] = {} # keyed by controller_id - - if client is not None: - import asyncio - - async def _fetch_ctrl_configs() -> dict[str, dict]: - configs_map: dict[str, dict] = {} - bot_names = [b.get("bot_name", "") for b in bots_list if b.get("bot_name")] - if not bot_names: - return configs_map - - async def _get_one(bn: str): - try: - configs = await client.controllers.get_bot_controller_configs(bn) - if isinstance(configs, list): - for cfg in configs: - cid = cfg.get("id") or cfg.get("controller_id", "") - if cid: - configs_map[cid] = cfg - cname = cfg.get("controller_name", "") - if cname and cname != cid: - configs_map[cname] = cfg - except Exception: - pass - - await asyncio.gather(*[_get_one(bn) for bn in bot_names]) - return configs_map - - async def _fetch_bot_runs() -> dict[str, str]: - """Deployed-at timestamps keyed by bot name, for the Age column. - - Filtered to DEPLOYED on purpose: the unfiltered listing also returns - ARCHIVED runs, each carrying a multi-KB ``final_status`` blob. On a - remote server that is a multi-MB, multi-minute response that stalls - the whole page (brigado: 4.5 MB / 274s unfiltered vs 121 KB / 5s - filtered), leaving every bot without an age. - """ - runs: dict[str, str] = {} - try: - _collect_bot_runs( - await client.bot_orchestration.get_bot_runs( - deployment_status="DEPLOYED" - ), - runs, - ) - except Exception: - logger.debug("Bot runs not available for '%s'", name) - - # Any active bot the filtered listing missed gets a targeted lookup - # (~1 KB each) rather than falling back to the unfiltered listing. - missing = [ - bn - for b in bots_list - if (bn := b.get("bot_name", "")) and bn not in runs - ] - if missing: - - async def _get_one_run(bn: str): - try: - _collect_bot_runs( - await client.bot_orchestration.get_bot_runs( - bot_name=bn, limit=1 - ), - runs, - ) - except Exception: - pass - - await asyncio.gather(*[_get_one_run(bn) for bn in missing]) - return runs - - async def _fetch_latest_perf() -> dict[str, dict]: - """Fetch latest controller performance snapshots from DB.""" - perf_map: dict[str, dict] = {} - try: - perf_result = ( - await client.bot_orchestration.get_latest_controller_performance() - ) - snapshots = _extract_perf_snapshots(perf_result) - for snap in snapshots: - cid = snap.get("controller_id", "") - if cid: - perf_map[cid] = snap - except Exception: - logger.debug( - "Latest controller performance not available for '%s'", name - ) - return perf_map - - async def _with_timeout(coro, label: str, default: Any) -> Any: - """Cap one enrichment call so a slow server degrades instead of hanging. - - Each fetcher gets its own budget: one slow endpoint must not cost us - the other two. - """ - try: - return await asyncio.wait_for(coro, timeout=ENRICHMENT_TIMEOUT) - except asyncio.TimeoutError: - logger.warning( - "Enrichment '%s' timed out after %.0fs for server '%s'", - label, - ENRICHMENT_TIMEOUT, - name, - ) - return default - - ctrl_configs, bot_runs, latest_perf = await asyncio.gather( - _with_timeout(_fetch_ctrl_configs(), "controller configs", {}), - _with_timeout(_fetch_bot_runs(), "bot runs", {}), - _with_timeout(_fetch_latest_perf(), "latest performance", {}), - ) - - page = build_bots_page( - result, - ctrl_configs=ctrl_configs, - bot_runs=bot_runs, - latest_perf=latest_perf, - ) + page = await enriched_bots_page(name, result) # Overlay transitional "stopping" state overlay_stopping_state(name, page["controllers"], page["bots"]) diff --git a/condor/web/streams/hummingbot_ws.py b/condor/web/streams/hummingbot_ws.py index 2e1a9dbaf..0c0b46a86 100644 --- a/condor/web/streams/hummingbot_ws.py +++ b/condor/web/streams/hummingbot_ws.py @@ -65,11 +65,32 @@ def _transform_executors(raw_data: Any) -> list[dict]: return result @staticmethod - def _transform_bots(raw_data: Any) -> dict: - """Transform raw BOTS_STATUS data to BotsPageResponse-compatible dict for WS broadcast.""" - from condor.fetchers.bots import build_bots_page + async def _transform_bots(server_name: str, raw_data: Any) -> dict: + """Transform raw BOTS_STATUS data to a BotsPageResponse-compatible dict. - return build_bots_page(raw_data) + Shares the REST route's builder *and* its enrichment: a frame that + omitted the controller configs, deploy timestamps and DB performance + left the client patching every row back out of the last REST payload. + """ + from condor.web.routes.bots import enriched_bots_page + + return await enriched_bots_page(server_name, raw_data) + + async def _broadcast_bots_update( + self, channel: str, server_name: str, raw_data: Any + ) -> None: + """Enrich a raw BOTS_STATUS payload and broadcast it, if it changed. + + Enrichment is an await, so every bots frame is built inside a task — + including the ones triggered by the synchronous SDS cache listener. + """ + try: + data = await self._transform_bots(server_name, raw_data) + self._overlay_stopping_state(server_name, data) + except Exception as e: + logger.debug("Failed to transform bots data for WS: %s", e) + return + await self._broadcast_update(channel, data) @staticmethod def _transform_controller_perf(raw_data: Any) -> list[dict]: @@ -481,7 +502,7 @@ async def _bots_ws_stream(self, channel: str) -> None: cached = sds.get(server_name, ServerDataType.BOTS_STATUS) if cached is not None: try: - data = self._transform_bots(cached) + data = await self._transform_bots(server_name, cached) await self.broadcast(channel, data) except Exception as e: logger.debug("Failed to send initial bots snapshot: %s", e) @@ -502,12 +523,7 @@ async def on_message(msg: dict) -> None: get_server_data_service().put( server_name, ServerDataType.BOTS_STATUS, raw_data ) - try: - data = self._transform_bots(raw_data) - self._overlay_stopping_state(server_name, data) - await self._broadcast_update(channel, data) - except Exception as e: - logger.debug("Failed to transform bots WS data: %s", e) + await self._broadcast_bots_update(channel, server_name, raw_data) await self._run_ws_stream( channel, diff --git a/condor/web/ws_manager.py b/condor/web/ws_manager.py index cc4df17dd..d1b57a644 100644 --- a/condor/web/ws_manager.py +++ b/condor/web/ws_manager.py @@ -475,21 +475,16 @@ def _on_data_update(self, key: CacheKey, value: Any) -> None: if task and not task.done(): return - # Transform raw data to match REST endpoint response shapes + # Bots frames are enriched (an await) and carry the transitional + # "stopping" overlay, so they are built inside the broadcast task + # rather than here — this listener is synchronous. if dt_name == "BOTS_STATUS": - try: - value = self._transform_bots(value) - # Overlay transitional "stopping" state from Condor's in-memory store - self._overlay_stopping_state(server_name, value) - except Exception as e: - logger.debug("Failed to transform bots data for WS: %s", e) - return + coro = self._broadcast_bots_update(channel, server_name, value) + else: + coro = self._broadcast_update(channel, value) self._oneshot_tasks.track( - asyncio.create_task( - self._broadcast_update(channel, value), - name=f"broadcast:{channel}", - ) + asyncio.create_task(coro, name=f"broadcast:{channel}") ) async def _broadcast_update(self, channel: str, data: Any) -> None: diff --git a/frontend/src/lib/shared-socket.ts b/frontend/src/lib/shared-socket.ts index f268f863c..e3035cc0d 100644 --- a/frontend/src/lib/shared-socket.ts +++ b/frontend/src/lib/shared-socket.ts @@ -32,8 +32,6 @@ import { queryClient, } from "./queryClient"; import type { - BotsPageResponse, - ControllerInfo, ControllerPerformanceHistoryResponse, ControllerPerformanceSnapshot, } from "./api"; @@ -287,41 +285,10 @@ export function handleMessage(channel: string, data: unknown): void { if (prefix === "portfolio") { queryClient.setQueryData(["portfolio", server], data); } else if (prefix === "bots") { - queryClient.setQueryData(["bots", server], (old: BotsPageResponse | undefined) => { - const incoming = data as BotsPageResponse; - if (!incoming?.controllers) return old ?? data; - if (!old?.controllers?.length) return incoming; - - // Key by bot + controller_id (stable) — controller_name may differ - // between REST and WS, and the id alone is shared by every bot running - // the same controller config (CORR-241). - const oldMap = new Map(); - for (const c of old.controllers) { - oldMap.set(controllerKey(c), c); - } - const oldBotMap = new Map(old.bots.map((b) => [b.bot_name, b])); - - return { - ...incoming, - controllers: incoming.controllers.map((c) => { - const prev = oldMap.get(controllerKey(c)); - if (!prev) return c; - return { - ...c, - config: Object.keys(c.config || {}).length ? c.config : prev.config, - deployed_at: c.deployed_at ?? prev.deployed_at, - connector: c.connector || prev.connector, - trading_pair: c.trading_pair || prev.trading_pair, - controller_name: prev.controller_name || c.controller_name, - controller_id: prev.controller_id || c.controller_id, - }; - }), - bots: incoming.bots.map((b) => { - const prev = oldBotMap.get(b.bot_name); - return { ...b, deployed_at: b.deployed_at ?? prev?.deployed_at ?? null }; - }), - }; - }); + // A plain replace: the WS frame is enriched server-side (ARCH-586), so it + // carries the same config / deployed_at / ids / pair as the REST body and + // needs no field-by-field repair from the previous payload. + queryClient.setQueryData(["bots", server], data); } else if (prefix === "executors") { const unfiltered = executorsQuery(server); queryClient.setQueryData(unfiltered.queryKey, data); diff --git a/tests/test_bots_page_transform.py b/tests/test_bots_page_transform.py index 66794e520..ce3ffe46d 100644 --- a/tests/test_bots_page_transform.py +++ b/tests/test_bots_page_transform.py @@ -219,15 +219,13 @@ def test_rest_response_matches_pre_refactor_golden(): assert BotsPageResponse(**page).model_dump() == GOLDEN_REST -def test_ws_transform_matches_pre_refactor_golden(): - """build_bots_page without enrichment reproduces the old _transform_bots.""" - assert build_bots_page(SAMPLE_RAW) == GOLDEN_WS - - -def test_ws_manager_delegates_to_shared_transform(): - from condor.web.ws_manager import WebSocketManager +def test_unenriched_transform_matches_pre_refactor_golden(): + """The degraded shape, for when a server cannot answer the enrichment calls. - assert WebSocketManager._transform_bots(SAMPLE_RAW) == build_bots_page(SAMPLE_RAW) + Both delivery paths now feed the builder the same enrichment (ARCH-586); + what the empty maps still pin is how a page renders when those fetches fail. + """ + assert build_bots_page(SAMPLE_RAW) == GOLDEN_WS def test_live_zero_wins_over_stale_db_snapshot(): diff --git a/tests/test_bots_ws_frame_parity.py b/tests/test_bots_ws_frame_parity.py new file mode 100644 index 000000000..7b3ede074 --- /dev/null +++ b/tests/test_bots_ws_frame_parity.py @@ -0,0 +1,171 @@ +"""ARCH-586: a bots WS frame carries the same enrichment as the REST body. + +The WS path used to call ``build_bots_page`` with no enrichment at all, so a +frame arrived with ``config={}``, no ``deployed_at``, the raw performance key +as the controller id and a connector/pair guessed by splitting that key on +underscores. The frontend patched every one of those fields back out of the +previous REST payload. Both paths now read one cached enrichment, so the frame +and the REST body are the same page. +""" + +import asyncio + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +import condor.web.routes.bots as bots_route +from condor.fetchers.bots import fetch_bots_enrichment +from condor.server_data_service import ServerDataService, ServerDataType +from condor.web.auth import require_server_access +from condor.web.models import BotsPageResponse, WebUser + +SERVER = "srv" +USER = WebUser(id=7, username="u", first_name="U", role="user") + +RAW_STATUS = { + "status": "success", + "data": { + "epsilon": { + "status": "running", + "performance": { + "pmm_binance_BTC-USDT_1": { + "status": "running", + "performance": { + "realized_pnl_quote": 1.5, + "unrealized_pnl_quote": -0.5, + "volume_traded": 1234.5, + }, + }, + }, + "error_logs": [], + "general_logs": [], + }, + }, +} + +CONFIG = { + "id": "pmm_binance_BTC-USDT_1", + "controller_name": "pmm_simple", + "connector_name": "binance", + "trading_pair": "BTC-USDT", +} + +DEPLOYED_AT = "2026-07-01T00:00:00Z" + + +class _Client: + """Upstream stub that counts every enrichment call it serves.""" + + def __init__(self): + self.calls: list[str] = [] + + # Both namespaces are this object. + @property + def controllers(self): + return self + + @property + def bot_orchestration(self): + return self + + async def get_active_bots_status(self): + self.calls.append("status") + return RAW_STATUS + + async def get_bot_controller_configs(self, bot_name): + self.calls.append(f"configs:{bot_name}") + return [CONFIG] + + async def get_bot_runs(self, **kw): + self.calls.append("runs") + return {"data": {"epsilon": {"deployed_at": DEPLOYED_AT}}} + + async def get_latest_controller_performance(self): + self.calls.append("perf") + return { + "data": [ + { + "controller_id": "pmm_binance_BTC-USDT_1", + "performance": {"global_pnl_pct": 0.42}, + } + ] + } + + +@pytest.fixture +def sds(monkeypatch): + """A private ServerDataService, primed with the raw status, no real client.""" + import condor.server_data_service as sds_mod + + client = _Client() + service = ServerDataService() + service.register_fetch(ServerDataType.BOTS_ENRICHMENT, fetch_bots_enrichment) + + async def _get_client(_server): + return client + + monkeypatch.setattr(service, "_get_client", _get_client, raising=True) + monkeypatch.setattr(sds_mod, "_instance", service, raising=False) + service.put(SERVER, ServerDataType.BOTS_STATUS, RAW_STATUS) + return service, client + + +def _rest_body(monkeypatch) -> dict: + app = FastAPI() + app.include_router(bots_route.router) + app.dependency_overrides[require_server_access] = lambda: USER + with TestClient(app) as api: + r = api.get(f"/servers/{SERVER}/bots") + assert r.status_code == 200 + return r.json() + + +def test_ws_frame_matches_the_rest_body(sds, monkeypatch): + from condor.web.ws_manager import WebSocketManager + + body = _rest_body(monkeypatch) + frame = asyncio.run(WebSocketManager._transform_bots(SERVER, RAW_STATUS)) + + assert BotsPageResponse(**frame).model_dump(mode="json") == body + + # The fields the frontend used to repair are all really there. + ctrl = frame["controllers"][0] + assert ctrl["config"] == CONFIG + assert ctrl["deployed_at"] == DEPLOYED_AT + assert ctrl["controller_id"] == "pmm_binance_BTC-USDT_1" + assert ctrl["connector"] == "binance" + assert ctrl["trading_pair"] == "BTC-USDT" + + +def test_a_warm_enrichment_costs_the_5s_frame_no_round_trip(sds): + from condor.web.ws_manager import WebSocketManager + + _service, client = sds + + asyncio.run(WebSocketManager._transform_bots(SERVER, RAW_STATUS)) + assert client.calls, "the first frame has to fetch the enrichment" + + after_first = list(client.calls) + for _ in range(3): + asyncio.run(WebSocketManager._transform_bots(SERVER, RAW_STATUS)) + + assert client.calls == after_first + + +def test_a_failed_enrichment_still_renders_the_page(sds, monkeypatch): + """The maps are optional: a server that cannot answer costs those columns.""" + from condor.web.ws_manager import WebSocketManager + + service, _client = sds + + async def _boom(_server): + raise RuntimeError("no client") + + monkeypatch.setattr(service, "_get_client", _boom, raising=True) + + frame = asyncio.run(WebSocketManager._transform_bots(SERVER, RAW_STATUS)) + + assert frame["controllers"][0]["config"] == {} + assert frame["controllers"][0]["deployed_at"] is None + assert frame["total_volume"] == 1234.5 From 21ed03a8f1313c778c3b8208bb442a329fa1e8eb Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 18:00:32 +0300 Subject: [PATCH 014/154] Let the fetchers own the shapes they return, not the web layer Two fetchers imported their return types from condor.web.models, which inverted the layering the package docstring declares and made a real cycle: web.models calls into fetchers.executors, and only a function-local import inside ExecutorInfo.from_raw kept that from failing at import time. ArchivedBotPerformance, NormalizedExecutor, PnlPoint, BotRunInfo and ControllerInfo move verbatim into condor/fetchers/models.py. They stay pydantic models and condor.web.models re-exports all five, so every route signature, response_model and importer is untouched. With both edges now pointing the same way, the deferred import in from_raw is hoisted to module scope, and archived_run's docstring carve-out for "the one thing it borrows from condor.web" is gone along with the borrowing. --- condor/fetchers/archived_run.py | 14 +++- condor/fetchers/models.py | 144 ++++++++++++++++++++++++++++++++ condor/fetchers/run_history.py | 2 +- condor/web/models.py | 131 ++--------------------------- 4 files changed, 164 insertions(+), 127 deletions(-) create mode 100644 condor/fetchers/models.py diff --git a/condor/fetchers/archived_run.py b/condor/fetchers/archived_run.py index dd43d7c70..bd55bf68c 100644 --- a/condor/fetchers/archived_run.py +++ b/condor/fetchers/archived_run.py @@ -12,9 +12,11 @@ is now one of its callers, and a routine published to ``agents/_shared/routines`` must not reach into the web package to read an archive. -The one thing this module borrows from ``condor.web`` is ``models`` — the -pydantic wire shapes, which import nothing from condor and so introduce no cycle -the package's no-web rule exists to prevent. It raises +The shapes it returns live in :mod:`condor.fetchers.models` and are re-exported +by ``condor.web.models``, so the direction stays one-way: this module borrows +nothing from ``condor.web``. It used to import those shapes from there, which +was a real cycle — ``condor.web.models`` calls into ``condor.fetchers.executors`` +— masked only by a function-local import on the other side. It raises :class:`ArchivedRunUnavailable` rather than ``HTTPException``; mapping that to a status code is the route's job. """ @@ -28,7 +30,11 @@ from typing import Any from condor.fetchers.executors import normalize_executor_side -from condor.web.models import ArchivedBotPerformance, NormalizedExecutor, PnlPoint +from condor.fetchers.models import ( + ArchivedBotPerformance, + NormalizedExecutor, + PnlPoint, +) logger = logging.getLogger(__name__) diff --git a/condor/fetchers/models.py b/condor/fetchers/models.py new file mode 100644 index 000000000..26e6ecdc4 --- /dev/null +++ b/condor/fetchers/models.py @@ -0,0 +1,144 @@ +"""The shapes the fetchers return, owned by the data layer. + +These five pydantic models are what ``archived_run`` and ``run_history`` build +and hand back, and they are also the wire schema the web layer serializes — +``condor.web.models`` re-exports every one of them under its own name, so a +route's ``response_model`` and every ``from condor.web.models import +BotRunInfo`` importer are unaffected by their living here. + +They live in the data layer because the direction matters: a fetcher must not +import ``condor.web`` (see this package's docstring), and the reverse edge that +rule prevents is real — ``condor.web.models.ExecutorInfo.from_raw`` calls into +``condor.fetchers.executors``. With the shapes down here both edges point the +same way, and an agent routine that reads an archived run no longer reaches +through the web package to name its own return type. + +They stay ``BaseModel`` rather than becoming dataclasses precisely so the web +layer can keep using them as response models unchanged. +""" + +from __future__ import annotations + +from typing import Any, Optional + +from pydantic import BaseModel + + +class ControllerInfo(BaseModel): + controller_name: str + #: The coarse bucket upstream sorts every controller into (``generic``, + #: ``directional_trading``, ``market_making``) — a fallback class for a + #: terminated controller whose config lookup could not recover the specific + #: one (see ``fill_classes_from_config``). + controller_type: str = "" + controller_id: str = "" + bot_name: str + status: str = "unknown" + connector: str = "" + trading_pair: str = "" + realized_pnl_quote: float = 0.0 + unrealized_pnl_quote: float = 0.0 + global_pnl_quote: float = 0.0 + global_pnl_pct: float = 0.0 + volume_traded: float = 0.0 + close_type_counts: dict[str, int] = {} + positions_summary: list[dict[str, Any]] = [] + deployed_at: Optional[str] = None + config: dict[str, Any] = {} + + +class BotRunInfo(BaseModel): + bot_name: str + bot_run_id: Optional[int] = None + account_name: str = "" + strategy_type: str = "" + strategy_name: str = "" + run_status: str = "" + deployment_status: str = "" + created_at: Optional[str] = None + stopped_at: Optional[str] = None + realized_pnl_quote: float = 0.0 + unrealized_pnl_quote: float = 0.0 + global_pnl_quote: float = 0.0 + volume_traded: float = 0.0 + num_controllers: int = 0 + # Path to this run's archived sqlite database, when one survived the bot. + # Present iff the run has a deep history to open; the archived-bot routes take + # it as their ``db_path``. + archive_db_path: Optional[str] = None + # The controller config ids this run was *deployed with*, straight from its + # own ``deployment_config``. + # + # This is the authoritative run -> controller mapping, and it is the only one + # that exists for a run old enough to have no performance snapshots left: the + # deployment declared these ids before the bot ever traded. It is what lets a + # closed executor be attributed to the run that created it rather than to + # whichever live controller happens to share its config id (FEAT-089). + controller_ids: list[str] = [] + # Whether this run is the live fleet rather than history. + # + # Derived, because ``run_status`` cannot answer it: upstream never writes + # ``RUNNING``, and the eight bots trading right now on a real server all + # report the literal string ``CREATED``. A container that is deployed and has + # no stop time is what "still running" actually means here. + is_live: bool = False + + +class PnlPoint(BaseModel): + timestamp: float + pnl: float + + +class NormalizedExecutor(BaseModel): + id: str = "" + type: str = "" + connector: str = "" + trading_pair: str = "" + side: str = "" + status: str = "" + close_type: str = "" + pnl: float = 0.0 + volume: float = 0.0 + timestamp: float = 0.0 + close_timestamp: float = 0.0 + entry_price: float = 0.0 + current_price: float = 0.0 + cum_fees_quote: float = 0.0 + net_pnl_pct: float = 0.0 + controller_id: str = "" + custom_info: dict[str, Any] = {} + config: dict[str, Any] = {} + # USD value of one unit of this market's quote currency. `pnl`, `volume` and + # `cum_fees_quote` above stay quote-denominated so prices on the same row + # remain comparable to the market's candles; renderers multiply by this. + usd_rate: float = 1.0 + + +class ArchivedBotPerformance(BaseModel): + bot_name: str + db_path: str + total_pnl: float = 0.0 + total_fees: float = 0.0 + total_volume: float = 0.0 + trade_count: int = 0 + buy_count: int = 0 + sell_count: int = 0 + pnl_by_pair: dict[str, float] = {} + cumulative_pnl: list[PnlPoint] = [] + trading_pairs: list[str] = [] + exchanges: list[str] = [] + executors: list[NormalizedExecutor] = [] + primary_connector: str = "" + primary_trading_pair: str = "" + executor_count: int = 0 + # Quote currency of the primary market, for labelling a converted figure. + quote_currency: str = "" + # USD rate per quote currency seen in the run. + usd_rates: dict[str, float] = {} + # False when some quote had no path to USD and its figures are reported at + # face value in their own currency rather than silently passed off as USD. + converted: bool = True + # Which source the headline stats above were computed from. An archived + # database with an empty trades table falls back to executors, and the UI + # labels the count card accordingly instead of claiming zero trades. + stats_source: str = "trades" diff --git a/condor/fetchers/run_history.py b/condor/fetchers/run_history.py index 8e955b5cf..af2455f49 100644 --- a/condor/fetchers/run_history.py +++ b/condor/fetchers/run_history.py @@ -43,13 +43,13 @@ from condor.fetchers._pagination import collect_pages from condor.fetchers.bot_performance import extract_snapshots +from condor.fetchers.models import BotRunInfo, ControllerInfo from condor.run_history_store import ( RunHistoryEntry, get_run_history_store, is_settled, run_key, ) -from condor.web.models import BotRunInfo, ControllerInfo logger = logging.getLogger(__name__) diff --git a/condor/web/models.py b/condor/web/models.py index e73716854..6f3cea5dd 100644 --- a/condor/web/models.py +++ b/condor/web/models.py @@ -4,6 +4,15 @@ from pydantic import BaseModel +from condor.fetchers.executors import build_executor_row, get_executor_type +from condor.fetchers.models import ( + ArchivedBotPerformance, + BotRunInfo, + ControllerInfo, + NormalizedExecutor, + PnlPoint, +) + # ── Auth ── @@ -102,29 +111,6 @@ class BotDetailResponse(BaseModel): performance: dict[str, Any] = {} -class ControllerInfo(BaseModel): - controller_name: str - #: The coarse bucket upstream sorts every controller into (``generic``, - #: ``directional_trading``, ``market_making``) — a fallback class for a - #: terminated controller whose config lookup could not recover the specific - #: one (see ``fill_classes_from_config``). - controller_type: str = "" - controller_id: str = "" - bot_name: str - status: str = "unknown" - connector: str = "" - trading_pair: str = "" - realized_pnl_quote: float = 0.0 - unrealized_pnl_quote: float = 0.0 - global_pnl_quote: float = 0.0 - global_pnl_pct: float = 0.0 - volume_traded: float = 0.0 - close_type_counts: dict[str, int] = {} - positions_summary: list[dict[str, Any]] = [] - deployed_at: Optional[str] = None - config: dict[str, Any] = {} - - class BotSummary(BaseModel): bot_name: str status: str = "unknown" @@ -147,43 +133,6 @@ class BotsPageResponse(BaseModel): # ── Bot Runs ── -class BotRunInfo(BaseModel): - bot_name: str - bot_run_id: Optional[int] = None - account_name: str = "" - strategy_type: str = "" - strategy_name: str = "" - run_status: str = "" - deployment_status: str = "" - created_at: Optional[str] = None - stopped_at: Optional[str] = None - realized_pnl_quote: float = 0.0 - unrealized_pnl_quote: float = 0.0 - global_pnl_quote: float = 0.0 - volume_traded: float = 0.0 - num_controllers: int = 0 - # Path to this run's archived sqlite database, when one survived the bot. - # Present iff the run has a deep history to open; the archived-bot routes take - # it as their ``db_path``. - archive_db_path: Optional[str] = None - # The controller config ids this run was *deployed with*, straight from its - # own ``deployment_config``. - # - # This is the authoritative run -> controller mapping, and it is the only one - # that exists for a run old enough to have no performance snapshots left: the - # deployment declared these ids before the bot ever traded. It is what lets a - # closed executor be attributed to the run that created it rather than to - # whichever live controller happens to share its config id (FEAT-089). - controller_ids: list[str] = [] - # Whether this run is the live fleet rather than history. - # - # Derived, because ``run_status`` cannot answer it: upstream never writes - # ``RUNNING``, and the eight bots trading right now on a real server all - # report the literal string ``CREATED``. A container that is deployed and has - # no stop time is what "still running" actually means here. - is_live: bool = False - - class BotRunsResponse(BaseModel): runs: list[BotRunInfo] = [] total: int = 0 @@ -500,8 +449,6 @@ class name. if not isinstance(ex, dict): return None - from condor.fetchers.executors import build_executor_row, get_executor_type - row = build_executor_row(ex) return cls( id=row["id"], @@ -673,66 +620,6 @@ class ArchivedBotSummary(BaseModel): end_time: Optional[float] = None -class PnlPoint(BaseModel): - timestamp: float - pnl: float - - -class NormalizedExecutor(BaseModel): - id: str = "" - type: str = "" - connector: str = "" - trading_pair: str = "" - side: str = "" - status: str = "" - close_type: str = "" - pnl: float = 0.0 - volume: float = 0.0 - timestamp: float = 0.0 - close_timestamp: float = 0.0 - entry_price: float = 0.0 - current_price: float = 0.0 - cum_fees_quote: float = 0.0 - net_pnl_pct: float = 0.0 - controller_id: str = "" - custom_info: dict[str, Any] = {} - config: dict[str, Any] = {} - # USD value of one unit of this market's quote currency. `pnl`, `volume` and - # `cum_fees_quote` above stay quote-denominated so prices on the same row - # remain comparable to the market's candles; renderers multiply by this. - usd_rate: float = 1.0 - - -class ArchivedBotPerformance(BaseModel): - bot_name: str - db_path: str - total_pnl: float = 0.0 - total_fees: float = 0.0 - total_volume: float = 0.0 - trade_count: int = 0 - buy_count: int = 0 - sell_count: int = 0 - pnl_by_pair: dict[str, float] = {} - cumulative_pnl: list[PnlPoint] = [] - trading_pairs: list[str] = [] - exchanges: list[str] = [] - executors: list[NormalizedExecutor] = [] - primary_connector: str = "" - primary_trading_pair: str = "" - executor_count: int = 0 - # Quote currency of the primary market, for labelling a converted figure. - quote_currency: str = "" - # USD rate per quote currency seen in the run. - usd_rates: dict[str, float] = {} - # False when some quote had no path to USD and its figures are reported at - # face value in their own currency rather than silently passed off as USD. - converted: bool = True - # Which source the headline stats above were computed from. An archived - # database with an empty trades table falls back to executors, and the UI - # labels the count card accordingly instead of claiming zero trades. - stats_source: str = "trades" - - class ArchivedControllerRollup(BaseModel): """What one controller did inside an archived run. Money is USD. From 175080fc5bee10bb3cb9c1debb3571f7ffba1ec5 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 18:09:53 +0300 Subject: [PATCH 015/154] Write the single-flight idiom once, with every copy's guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "One in-flight task per key, every concurrent caller awaits it" was written out seven times across condor/, and no two copies agreed. Three of them joined the shared task bare, so one waiter going away — a wait_for timeout, a cancelled gather leg, a WS teardown — cancelled the fetch every other waiter was riding on and handed them all a CancelledError. Two carried a loop-identity guard the rest lacked; one guarded against joining a settled task, which would otherwise hand a fresh caller the previous run's failure. condor.asyncutil.SingleFlight is now the one implementation, carrying all three guards and the rationale that had been split across the copies, next to TaskSet which got the same treatment for fire-and-forget tasks. The seven call sites — gecko listings, archived runs, the whole-server snapshot, instance history, run history, the SDS cache and the candles route — all delegate to it; the SDS keeps its own instance so two services in a test never share one. The behavioural fix is the shield reaching the snapshot, history and SDS copies: new tests cancel one of two coalesced waiters and assert the survivor still gets its value, and both fail against the old bare await. --- condor/asyncutil.py | 97 +++++++++++++++++- condor/fetchers/archived_run.py | 22 ++--- condor/fetchers/bot_performance.py | 36 ++----- condor/fetchers/run_history.py | 42 +++----- condor/pool_data.py | 22 +---- condor/server_data_service.py | 14 +-- condor/web/routes/market.py | 54 ++++------ tests/test_archived_analyzer_detail.py | 3 +- tests/test_asyncutil_singleflight.py | 130 +++++++++++++++++++++++++ tests/test_bot_performance.py | 27 ++++- tests/test_candle_coalescing.py | 4 +- tests/test_sds_inflight_coalescing.py | 46 ++++++++- 12 files changed, 358 insertions(+), 139 deletions(-) create mode 100644 tests/test_asyncutil_singleflight.py diff --git a/condor/asyncutil.py b/condor/asyncutil.py index 481e7bed6..6d64d71e4 100644 --- a/condor/asyncutil.py +++ b/condor/asyncutil.py @@ -1,5 +1,10 @@ """Shared asyncio helpers. +Each class here is the one implementation of an idiom ``condor`` had hand-rolled +in several places, with the copies' guards folded together rather than left to +drift apart: ``TaskSet`` for fire-and-forget tasks, ``SingleFlight`` for +per-key request coalescing. + ``TaskSet`` is the one implementation of the fire-and-forget task tracker that ``condor`` had hand-rolled twice: once in ``condor/web/ws_manager.py`` (CORR-107, one-shot broadcasts and backfills) and once in @@ -21,13 +26,25 @@ Everything else (discard-then-check, cancellation is not a failure, snapshot before cancelling because ``cancel()`` re-enters the done-callback) was identical in both copies and is fixed here. + +``SingleFlight`` (ARCH-606) is the same story for "one in-flight task per key, +every concurrent caller awaits it", which had been written out seven times: in +``condor/pool_data.py`` (gecko listings), ``condor/fetchers/archived_run.py``, +``condor/fetchers/bot_performance.py`` twice (whole-server snapshot, instance +history), ``condor/fetchers/run_history.py``, ``condor/server_data_service.py`` +and ``condor/web/routes/market.py`` (candles). No two copies agreed: three of +the guards below were present in some and missing from others, so the same +idiom silently behaved differently depending on which module you were in. +``SingleFlight`` carries all three. """ from __future__ import annotations import asyncio import logging -from typing import Iterator +from typing import Any, Awaitable, Callable, Hashable, Iterator, TypeVar + +T = TypeVar("T") class TaskSet: @@ -90,3 +107,81 @@ def __contains__(self, task: object) -> bool: def __bool__(self) -> bool: return bool(self._tasks) + + +class SingleFlight: + """One in-flight task per key, shared by every concurrent caller of that key. + + ``run(key, factory)`` calls ``factory()`` only when no usable task is already + running for ``key``; everyone else joins the one that is. A TTL cache only + helps once an answer has *arrived*, so this is what collapses the burst of + identical requests that all miss a cold cache at the same instant. + + Three guards, each of which one of the hand-rolled copies had and the others + lacked: + + * **Shielded.** The work runs as a detached task and awaiters ``shield`` it, + so the very thing that causes the stampede — a viewer navigating away and + cancelling their request — cannot also cancel the fetch the remaining + viewers are waiting on. Awaiting a ``Task`` bare propagates the awaiter's + cancellation into it, handing every other waiter a ``CancelledError``. + * **Same loop.** Reuse an in-flight fetch only from the loop that created + it: a task is bound to its loop and awaiting it from another one raises. + * **Not done.** A finished task lingers in the map until its done-callback + runs on the next loop iteration; joining it would hand a fresh caller the + previous run's outcome, a failure included. So a settled task is never + reused and the next caller genuinely retries — a failure is shared by the + waiters that were already on it, never remembered for the ones after. + + The done-callback is identity-checked, so a callback that fires late can + only ever evict its own entry and never a newer task registered under the + same key. + """ + + def __init__(self) -> None: + self._inflight: dict[ + Hashable, tuple[asyncio.AbstractEventLoop, asyncio.Task] + ] = {} + + async def run(self, key: Hashable, factory: Callable[[], Awaitable[T]]) -> T: + """Run ``factory()`` once per key, sharing its outcome with every waiter.""" + loop = asyncio.get_running_loop() + entry = self._inflight.get(key) + task = ( + entry[1] + if entry is not None and entry[0] is loop and not entry[1].done() + else None + ) + if task is None: + task = asyncio.ensure_future(factory()) + self._inflight[key] = (loop, task) + + def _clear(finished: asyncio.Task, _key: Hashable = key) -> None: + current = self._inflight.get(_key) + if current is not None and current[1] is finished: + self._inflight.pop(_key, None) + + task.add_done_callback(_clear) + return await asyncio.shield(task) + + def clear(self) -> None: + """Forget every in-flight entry (tests, reconfiguration). + + The tasks themselves are left to finish: their waiters are still + shielded on them, and only the sharing is dropped. + """ + self._inflight.clear() + + # -- Introspection (tests and log lines; not part of the hot path) -- + + def __len__(self) -> int: + return len(self._inflight) + + def __bool__(self) -> bool: + return bool(self._inflight) + + def __contains__(self, key: object) -> bool: + return key in self._inflight + + def __iter__(self) -> Iterator[Any]: + return iter(self._inflight) diff --git a/condor/fetchers/archived_run.py b/condor/fetchers/archived_run.py index bd55bf68c..86542e79f 100644 --- a/condor/fetchers/archived_run.py +++ b/condor/fetchers/archived_run.py @@ -29,6 +29,7 @@ from collections import OrderedDict from typing import Any +from condor.asyncutil import SingleFlight from condor.fetchers.executors import normalize_executor_side from condor.fetchers.models import ( ArchivedBotPerformance, @@ -59,11 +60,8 @@ def __init__(self, detail: str, *, missing: bool = False): _performance_cache: OrderedDict[tuple[str, str], ArchivedBotPerformance] = OrderedDict() # In-flight fetches keyed like the cache, so concurrent cold-cache requests for -# the same archived bot share one backend walk (same idiom as -# condor.pool_data._single_flight). -_performance_inflight: dict[tuple[str, str], "asyncio.Task[ArchivedBotPerformance]"] = ( - {} -) +# the same archived bot share one backend walk. +_performance_inflight = SingleFlight() # Attempts per page of the archived trade walk. A run with tens of thousands of # trades needs dozens of round trips, and one transient failure must not decide @@ -236,17 +234,9 @@ async def fetch_archived_run( if cached is not None: return cached - task = _performance_inflight.get(cache_key) - if task is None or task.done(): - task = asyncio.ensure_future(_fetch_performance(client, name, db_path)) - _performance_inflight[cache_key] = task - - def _clear(finished: "asyncio.Task", _key: tuple[str, str] = cache_key) -> None: - if _performance_inflight.get(_key) is finished: - _performance_inflight.pop(_key, None) - - task.add_done_callback(_clear) - return await asyncio.shield(task) + return await _performance_inflight.run( + cache_key, lambda: _fetch_performance(client, name, db_path) + ) async def _fetch_performance( diff --git a/condor/fetchers/bot_performance.py b/condor/fetchers/bot_performance.py index 3a6481618..e82d9001e 100644 --- a/condor/fetchers/bot_performance.py +++ b/condor/fetchers/bot_performance.py @@ -24,6 +24,7 @@ from functools import partial from typing import Any, Iterable +from condor.asyncutil import SingleFlight from condor.fetchers._pagination import collect_pages from condor.fetchers.executors import normalize_executor_side @@ -190,7 +191,7 @@ def _aggregate_by_bot(snapshots: list[dict]) -> dict[str, dict]: # the agents route already tolerates above this call. _SNAPSHOT_TTL = 5.0 _snapshot_cache: dict[str, tuple[float, dict[str, dict]]] = {} -_snapshot_inflight: dict[str, tuple[Any, asyncio.Task]] = {} +_snapshot_inflight = SingleFlight() def _server_key(client: Any) -> str: @@ -237,17 +238,7 @@ async def fetch_all_bot_performance(client: Any) -> dict[str, dict]: if entry is not None and time.monotonic() - entry[0] <= _SNAPSHOT_TTL: return entry[1] - # Reuse an in-flight fetch only from the loop that created it: a task is - # bound to its loop and awaiting it from another one raises. - loop = asyncio.get_running_loop() - inflight = _snapshot_inflight.get(key) - task = inflight[1] if inflight is not None and inflight[0] is loop else None - if task is None: - task = asyncio.ensure_future(_fetch_and_aggregate(client)) - _snapshot_inflight[key] = (loop, task) - task.add_done_callback(lambda _t, k=key: _snapshot_inflight.pop(k, None)) - - agg = await task + agg = await _snapshot_inflight.run(key, lambda: _fetch_and_aggregate(client)) _snapshot_cache[key] = (time.monotonic(), agg) return agg @@ -505,7 +496,7 @@ def extract_history_rows(result: Any) -> list[dict]: _history_cache: OrderedDict[ tuple, tuple[float, list[tuple[float, float, float, float, float]]] ] = OrderedDict() -_history_inflight: dict[tuple, tuple[Any, asyncio.Task]] = {} +_history_inflight = SingleFlight() def clear_history_cache() -> None: @@ -585,20 +576,13 @@ async def fetch_instance_history( _history_cache.move_to_end(key) return entry[1] - # Reuse an in-flight walk only from the loop that created it: a task is - # bound to its loop and awaiting it from another one raises. - loop = asyncio.get_running_loop() - inflight = _history_inflight.get(key) - task = inflight[1] if inflight is not None and inflight[0] is loop else None - if task is None: - task = asyncio.ensure_future( - _walk_instance_history(client, instance_name, interval, limit, max_rows) - ) - _history_inflight[key] = (loop, task) - task.add_done_callback(lambda _t, k=key: _history_inflight.pop(k, None)) - try: - rows = await task + rows = await _history_inflight.run( + key, + lambda: _walk_instance_history( + client, instance_name, interval, limit, max_rows + ), + ) except Exception as e: logger.debug("fetch_instance_history(%s) failed: %s", instance_name, e) return [] diff --git a/condor/fetchers/run_history.py b/condor/fetchers/run_history.py index af2455f49..e306f3ea5 100644 --- a/condor/fetchers/run_history.py +++ b/condor/fetchers/run_history.py @@ -41,6 +41,7 @@ from datetime import datetime, timezone from typing import Any, Iterable +from condor.asyncutil import SingleFlight from condor.fetchers._pagination import collect_pages from condor.fetchers.bot_performance import extract_snapshots from condor.fetchers.models import BotRunInfo, ControllerInfo @@ -505,10 +506,8 @@ class RunHistory: # Single-flight, keyed like the store, so concurrent cold-cache readers of one -# run share a walk instead of each paying for their own. Same idiom as -# ``archived_run.py``: the fetch is a detached task and awaiters ``shield`` it, -# so one reader navigating away cannot cancel the walk the others are waiting on. -_inflight: dict[str, "asyncio.Task[RunHistory]"] = {} +# run share a walk instead of each paying for their own. +_inflight = SingleFlight() async def fetch_run_history( @@ -543,28 +542,19 @@ async def fetch_run_history( cached=True, ) - task = _inflight.get(key) - if task is None or task.done(): - task = asyncio.ensure_future( - _build( - client, - server, - key=key, - bot_name=bot_name, - deployed_at=deployed_at, - stopped_at=stopped_at, - controller_ids=list(controller_ids), - db_path=db_path, - ) - ) - _inflight[key] = task - - def _clear(finished: "asyncio.Task", _key: str = key) -> None: - if _inflight.get(_key) is finished: - _inflight.pop(_key, None) - - task.add_done_callback(_clear) - return await asyncio.shield(task) + return await _inflight.run( + key, + lambda: _build( + client, + server, + key=key, + bot_name=bot_name, + deployed_at=deployed_at, + stopped_at=stopped_at, + controller_ids=list(controller_ids), + db_path=db_path, + ), + ) async def _build( diff --git a/condor/pool_data.py b/condor/pool_data.py index cc371a46b..c6ec9aedd 100644 --- a/condor/pool_data.py +++ b/condor/pool_data.py @@ -28,6 +28,7 @@ import utils.config # noqa: F401 (imported for its load_dotenv() side effect) from condor import orca_api +from condor.asyncutil import SingleFlight logger = logging.getLogger(__name__) @@ -697,27 +698,12 @@ async def gecko_request(method: str, path: str, **kwargs) -> Any: # air, so every one of them misses the cache and opens its own gecko request — the # fastest way there is to spend the minute's budget. This collapses concurrent # callers of the same key onto one upstream request. -_gecko_inflight: Dict[Tuple, "asyncio.Task"] = {} +_gecko_inflight = SingleFlight() async def _single_flight(key: Tuple, factory) -> Any: - """Run ``factory()`` once per key, sharing its result with every concurrent caller. - - The work runs as a detached task and awaiters ``shield`` it, so the very thing - that causes the stampede — a viewer navigating away and cancelling their - request — cannot also cancel the fetch the remaining viewers are waiting on. - """ - task = _gecko_inflight.get(key) - if task is None or task.done(): - task = asyncio.ensure_future(factory()) - _gecko_inflight[key] = task - - def _clear(finished: "asyncio.Task", _key: Tuple = key) -> None: - if _gecko_inflight.get(_key) is finished: - _gecko_inflight.pop(_key, None) - - task.add_done_callback(_clear) - return await asyncio.shield(task) + """Run ``factory()`` once per key, sharing its result with every concurrent caller.""" + return await _gecko_inflight.run(key, factory) # ── Small TTL caches for token lookups ── diff --git a/condor/server_data_service.py b/condor/server_data_service.py index 3b41e7986..d085b2ec0 100644 --- a/condor/server_data_service.py +++ b/condor/server_data_service.py @@ -22,7 +22,7 @@ from functools import partial from typing import Any, Callable, Dict, FrozenSet, List, Optional, Tuple -from condor.asyncutil import TaskSet +from condor.asyncutil import SingleFlight, TaskSet logger = logging.getLogger(__name__) @@ -309,8 +309,9 @@ def __init__(self): self._rate_limiters: Dict[str, RateLimiter] = {} self._fetch_registry: Dict[ServerDataType, FetchSpec] = {} self._poll_task: Optional[asyncio.Task] = None - # In-flight fetches per key (single-flight coalescing) - self._inflight: Dict[CacheKey, asyncio.Task] = {} + # In-flight fetches per key (single-flight coalescing). Per-instance, so + # a second ServerDataService in a test never shares one. + self._inflight = SingleFlight() self._running = False self._last_cleanup = time.time() # Sync listeners (e.g. WebSocketManager broadcasts) @@ -746,12 +747,7 @@ async def _fetch_and_cache(self, key: CacheKey) -> Optional[Any]: starting a duplicate backend request. The in-flight entry is cleared when the fetch settles, so a failure never poisons subsequent fetches. """ - task = self._inflight.get(key) - if task is None: - task = asyncio.ensure_future(self._do_fetch_and_cache(key)) - self._inflight[key] = task - task.add_done_callback(lambda _t, k=key: self._inflight.pop(k, None)) - return await task + return await self._inflight.run(key, lambda: self._do_fetch_and_cache(key)) async def _do_fetch_and_cache(self, key: CacheKey) -> Optional[Any]: """Fetch data and update cache. Returns the fetched value.""" diff --git a/condor/web/routes/market.py b/condor/web/routes/market.py index bf1d342f3..a94e92dd1 100644 --- a/condor/web/routes/market.py +++ b/condor/web/routes/market.py @@ -1,12 +1,12 @@ from __future__ import annotations -import asyncio import logging import time from fastapi import APIRouter, Depends, HTTPException, Query from condor import dex_candles +from condor.asyncutil import SingleFlight from config_manager import get_config_manager logger = logging.getLogger(__name__) @@ -35,10 +35,8 @@ def _candle_cache_put(key: tuple, value: list, now: float) -> None: # grid of executor panels for the same pair and window misses a cold cache in # all of them at once, and each miss used to fire its own upstream call — N # duplicate GeckoTerminal requests against the process-wide rate budget, or N -# duplicate historical-candle calls to the API server. Same idiom as -# `_snapshot_inflight` in condor/fetchers/bot_performance.py: one task per key, -# popped by a done-callback so a failure is retried rather than remembered. -_candle_inflight: dict[tuple, tuple[asyncio.AbstractEventLoop, asyncio.Task]] = {} +# duplicate historical-candle calls to the API server. +_candle_inflight = SingleFlight() from condor.fetchers.market_data import fetch_historical_candles @@ -543,38 +541,22 @@ async def get_candles( return cached[1] # Cache miss: share one upstream fetch with every concurrent request on this - # key. Reuse an in-flight task only from the loop that created it — a task is - # bound to its loop and awaiting it from another one raises. - loop = asyncio.get_running_loop() - inflight = _candle_inflight.get(cache_key) - task = ( - inflight[1] - if inflight is not None and inflight[0] is loop and not inflight[1].done() - # A finished task lingers until its done-callback runs; joining it would - # hand a fresh request the previous fetch's outcome (a failure included). - else None + # key. SingleFlight shields the shared task, so a request that goes away + # cannot cancel the fetch every other waiter on the key is riding on. + candles = await _candle_inflight.run( + cache_key, + lambda: _fetch_candles_upstream( + cm, + name, + connector, + trading_pair, + interval, + limit, + start_time, + end_time, + pool_address, + ), ) - if task is None: - task = asyncio.ensure_future( - _fetch_candles_upstream( - cm, - name, - connector, - trading_pair, - interval, - limit, - start_time, - end_time, - pool_address, - ) - ) - _candle_inflight[cache_key] = (loop, task) - task.add_done_callback(lambda _t, k=cache_key: _candle_inflight.pop(k, None)) - - # Shielded: a browser that disconnects mid-request cancels this handler, and - # an unshielded `await task` would cancel the shared fetch out from under - # every other waiter on the same key. - candles = await asyncio.shield(task) _candle_cache_put(cache_key, candles, now) return candles diff --git a/tests/test_archived_analyzer_detail.py b/tests/test_archived_analyzer_detail.py index 76ef4f72c..9e27b53f5 100644 --- a/tests/test_archived_analyzer_detail.py +++ b/tests/test_archived_analyzer_detail.py @@ -16,6 +16,7 @@ import pytest from condor.archived_chart_series import _MAX_PNL_POINTS +from condor.asyncutil import SingleFlight from condor.fetchers import archived_run from condor.quote_conversion import QuoteRates from condor.reports import subjects @@ -86,7 +87,7 @@ async def _rates(server, quotes): monkeypatch.setattr("condor.quote_conversion.resolve_usd_rates", _rates) monkeypatch.setattr(archived_run, "_performance_cache", OrderedDict()) - monkeypatch.setattr(archived_run, "_performance_inflight", {}) + monkeypatch.setattr(archived_run, "_performance_inflight", SingleFlight()) def _detail(controller_id="", client=None): diff --git a/tests/test_asyncutil_singleflight.py b/tests/test_asyncutil_singleflight.py new file mode 100644 index 000000000..eeb3ac308 --- /dev/null +++ b/tests/test_asyncutil_singleflight.py @@ -0,0 +1,130 @@ +"""Tests for ARCH-606: ``condor.asyncutil.SingleFlight``. + +The idiom was hand-rolled seven times across ``condor/`` and no two copies +carried the same guards. These cover the three the class folds together — +shield, same-loop, not-done — plus the identity-checked done-callback, on the +class itself rather than through one of its call sites. +""" + +import asyncio + +import pytest + +from condor.asyncutil import SingleFlight + + +def test_concurrent_callers_of_one_key_share_a_single_run(): + calls = {"n": 0} + + async def _work(): + calls["n"] += 1 + await asyncio.sleep(0.02) + return calls["n"] + + async def _go(): + sf = SingleFlight() + results = await asyncio.gather(*[sf.run("k", _work) for _ in range(5)]) + return sf, results + + sf, results = asyncio.run(_go()) + assert calls["n"] == 1 + assert results == [1, 1, 1, 1, 1] + assert not sf, "the entry is dropped once the task settles" + + +def test_distinct_keys_never_share_a_run(): + async def _work(): + await asyncio.sleep(0.02) + return "done" + + async def _go(): + sf = SingleFlight() + pending = [asyncio.ensure_future(sf.run(k, _work)) for k in ("a", "b", "c")] + await asyncio.sleep(0.01) + assert len(sf) == 3 + assert "a" in sf and "z" not in sf + await asyncio.gather(*pending) + return sf + + assert not asyncio.run(_go()) + + +def test_a_cancelled_waiter_leaves_the_others_with_the_result(): + """The shield: the stampede's cause must not also be its kill switch.""" + calls = {"n": 0} + + async def _work(): + calls["n"] += 1 + await asyncio.sleep(0.05) + return "value" + + async def _go(): + sf = SingleFlight() + leaver = asyncio.ensure_future(sf.run("k", _work)) + stayer = asyncio.ensure_future(sf.run("k", _work)) + await asyncio.sleep(0.01) + leaver.cancel() + with pytest.raises(asyncio.CancelledError): + await leaver + return await stayer + + assert asyncio.run(_go()) == "value" + assert calls["n"] == 1 + + +def test_a_failure_is_shared_by_its_waiters_and_never_remembered(): + calls = {"n": 0} + + async def _flaky(): + calls["n"] += 1 + await asyncio.sleep(0.02) + if calls["n"] == 1: + raise RuntimeError("upstream down") + return "recovered" + + async def _go(): + sf = SingleFlight() + failed = await asyncio.gather( + *[sf.run("k", _flaky) for _ in range(2)], return_exceptions=True + ) + # The settled task is not reused: the next caller genuinely retries. + return failed, await sf.run("k", _flaky) + + failed, recovered = asyncio.run(_go()) + assert all(isinstance(e, RuntimeError) for e in failed) + assert recovered == "recovered" + assert calls["n"] == 2 + + +def test_an_entry_from_another_loop_is_never_awaited(): + """A task is bound to its loop; awaiting it from another one raises.""" + + async def _work(): + return "fresh" + + sf = SingleFlight() + # A stale entry left behind by a loop that is gone. Its "task" is a plain + # object, so reusing it instead of the guard firing would blow up loudly. + sf._inflight["k"] = (object(), object()) + + assert asyncio.run(sf.run("k", _work)) == "fresh" + + +def test_a_late_done_callback_cannot_evict_a_newer_task(): + """The callback is identity-checked, so it only ever drops its own entry.""" + + async def _work(): + return "done" + + async def _go(): + sf = SingleFlight() + first = await sf.run("k", _work) + # The first task is settled but its callback has already popped the key; + # re-running registers a fresh entry that the stale callback must not + # touch. Drive a full loop iteration so any late callback would fire. + pending = asyncio.ensure_future(sf.run("k", _work)) + await asyncio.sleep(0) + assert "k" in sf, "the newer task must still own the key" + return first, await pending + + assert asyncio.run(_go()) == ("done", "done") diff --git a/tests/test_bot_performance.py b/tests/test_bot_performance.py index 33287004e..2f14edebb 100644 --- a/tests/test_bot_performance.py +++ b/tests/test_bot_performance.py @@ -162,6 +162,31 @@ async def _go(): assert all(set(r) == {"river", "otherbot"} for r in results) +def test_a_cancelled_waiter_does_not_kill_the_shared_snapshot_fetch(): + """ARCH-606: the shared fetch is shielded, so one waiter going away is local. + + The whole-server snapshot is shared by the agents rollup's ``asyncio.gather`` + legs and by web handlers; before the extraction this joined the task bare, so + a single cancelled leg cancelled the fetch and handed every other waiter a + ``CancelledError``. + """ + client = _ServerClient("http://server-cancel:8000", delay=0.05) + + async def _go(): + leaver = asyncio.ensure_future(fetch_all_bot_performance(client)) + stayer = asyncio.ensure_future(fetch_all_bot_performance(client)) + # Both are parked on the one in-flight fetch before either can finish. + await asyncio.sleep(0.01) + leaver.cancel() + with pytest.raises(asyncio.CancelledError): + await leaver + return await stayer + + agg = asyncio.run(_go()) + assert client.calls == 1 + assert set(agg) == {"river", "otherbot"} + + def test_staggered_callers_within_ttl_reuse_the_snapshot(): # The rollup's per-strategy calls do not always overlap exactly; the TTL keeps # a staggered fan-out at one round-trip too. @@ -1183,7 +1208,7 @@ def test_history_inflight_from_another_loop_is_not_awaited(): # A stale in-flight entry left by a different loop: the guard must ignore it # (awaiting a foreign loop's task raises) and start its own walk. foreign = SimpleNamespace() # never awaited — would blow up if it were - bp._history_inflight[key] = (object(), foreign) + bp._history_inflight._inflight[key] = (object(), foreign) hist = asyncio.run(bp.fetch_instance_history(client, "bot-1")) assert client.bot_orchestration.calls == 1 diff --git a/tests/test_candle_coalescing.py b/tests/test_candle_coalescing.py index a642afc60..6958386fe 100644 --- a/tests/test_candle_coalescing.py +++ b/tests/test_candle_coalescing.py @@ -147,7 +147,7 @@ async def scenario(): # Nothing cached, nothing left in flight: the next request retries. cached = dict(market._candle_cache) - inflight = dict(market._candle_inflight) + inflight = len(market._candle_inflight) healthy = _CountingClient() _install(monkeypatch, healthy) @@ -159,7 +159,7 @@ async def scenario(): assert len(errors) == 3 assert all(isinstance(e, Exception) for e in errors) assert cached == {} - assert inflight == {} + assert inflight == 0 # The retry actually hit upstream and succeeded. assert ok_calls == 1 assert len(retried) == 1 diff --git a/tests/test_sds_inflight_coalescing.py b/tests/test_sds_inflight_coalescing.py index 447b114f7..29cd6506e 100644 --- a/tests/test_sds_inflight_coalescing.py +++ b/tests/test_sds_inflight_coalescing.py @@ -44,7 +44,7 @@ async def _drive(): assert calls["count"] == 1, "concurrent cold reads must coalesce to one fetch" assert results[0] == results[1] == {"value": 1} - assert sds._inflight == {}, "in-flight map must be cleared once settled" + assert not sds._inflight, "in-flight map must be cleared once settled" def test_fetch_failure_shared_by_waiters_and_does_not_poison_next_fetch(): @@ -69,13 +69,53 @@ async def _drive(): ) assert failed == [None, None] assert calls["count"] == 1 - assert sds._inflight == {} + assert not sds._inflight # Failure must not poison the key: a later fetch runs and succeeds fail["on"] = False recovered = await sds.get_or_fetch("srv", ServerDataType.PORTFOLIO) assert recovered == {"ok": True} assert calls["count"] == 2 - assert sds._inflight == {} + assert not sds._inflight asyncio.run(_drive()) + + +def test_cancelled_waiter_does_not_kill_the_shared_fetch(): + """ARCH-606: SDS joins the shared fetch through a shield, like its siblings. + + Two callers coalesce onto one cold-key fetch and the first is cancelled — a + ``wait_for`` timeout, a cancelled ``gather`` leg, a WS teardown. Before the + extraction SDS joined with a bare ``await task``, so that cancellation + propagated into the shared task and the survivor got a ``CancelledError`` + instead of the value. + """ + calls = {"count": 0} + + async def counting_fetcher(client, **params): + calls["count"] += 1 + await asyncio.sleep(0.05) + return {"value": calls["count"]} + + async def _drive(): + sds = _make_sds(counting_fetcher) + leaver = asyncio.ensure_future( + sds.get_or_fetch("srv", ServerDataType.PORTFOLIO) + ) + stayer = asyncio.ensure_future( + sds.get_or_fetch("srv", ServerDataType.PORTFOLIO) + ) + # Both are parked on the single in-flight fetch before it settles. + await asyncio.sleep(0.01) + leaver.cancel() + try: + await leaver + except asyncio.CancelledError: + pass + return sds, await stayer + + sds, survived = asyncio.run(_drive()) + + assert calls["count"] == 1, "the cancellation must not have started a second fetch" + assert survived == {"value": 1}, "the surviving waiter still gets the value" + assert not sds._inflight From edae12bdfb3e69fcf0dfd6186a10e68fa050d7c9 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 18:18:11 +0300 Subject: [PATCH 016/154] Write an executor overlay's shared tail once, not five times The five compute*Overlay builders each ended by hand-copying the same ten fields off the executor -- executorId, side, status, closeType, pnl, pnlPct, volume, fees, timeRange and config -- with only the type, the lines and markers, the prices and whether a segment or a grid box is drawn actually differing between them. The two-line lifetime clamp appeared verbatim five times as well. Adding a field to ExecutorOverlay was therefore five edits, and missing one of the optional ones drifted silently per executor type with no type error to catch it -- the shape this file already shipped once, when side was read differently in each builder. baseOverlay now returns the shared tail and every builder ends in it, passing only what it derives itself; lifetimeRange holds the single copy of the clamp, and takes part as a parameter so a builder that already sized a grid box or an order segment from the bounds draws both from the same one. A test builds one executor of each of the five types, each with distinct values, and asserts every shared field on the overlay against the source ExecutorInfo, so a builder that goes back to hand-copying and crosses or drops a field fails. --- .../src/lib/executor-overlays.base.test.ts | 181 ++++++++++++++++ frontend/src/lib/executor-overlays.ts | 204 +++++++++--------- 2 files changed, 285 insertions(+), 100 deletions(-) create mode 100644 frontend/src/lib/executor-overlays.base.test.ts diff --git a/frontend/src/lib/executor-overlays.base.test.ts b/frontend/src/lib/executor-overlays.base.test.ts new file mode 100644 index 000000000..e036085bf --- /dev/null +++ b/frontend/src/lib/executor-overlays.base.test.ts @@ -0,0 +1,181 @@ +/** + * @vitest-environment jsdom + * + * The fields every executor overlay shares (ARCH-332). + * + * The five `compute*Overlay` builders used to write out the same ten-field tail + * by hand, so a field added to `ExecutorOverlay` had to be added five times and + * an optional one missed in a builder drifted silently -- no type error, just a + * chart that lost a value for one executor type (the shape CORR-280 shipped + * once, with `side` read differently per builder). They now all return through + * `baseOverlay`, and this pins that: one executor of every type, each carrying + * distinct values, with every shared field asserted against the source + * `ExecutorInfo`. A builder that goes back to hand-copying and drops or crosses + * a field fails here. + */ + +import { describe, expect, it } from "vitest"; + +import type { ExecutorInfo } from "./api"; +import { computeMultiOverlays } from "./executor-overlays"; + +function executor(patch: Partial = {}): ExecutorInfo { + return { + id: "e1", + type: "position", + connector: "binance_perpetual", + trading_pair: "SOL-USDC", + side: "BUY", + status: "running", + close_type: "", + pnl: 5, + volume: 100, + timestamp: 1_700_000_000, + controller_id: "c1", + cum_fees_quote: 0.1, + net_pnl_pct: 0.01, + entry_price: 200, + current_price: 210, + close_timestamp: 0, + custom_info: {}, + config: {}, + ...patch, + }; +} + +/** One closed executor per builder, each field distinct so a swap is visible. */ +const cases: Array<{ label: string; overlayType: string; executor: ExecutorInfo }> = [ + { + label: "position", + overlayType: "position", + executor: executor({ + id: "pos-1", + type: "position", + side: "BUY", + status: "terminated", + close_type: "TAKE_PROFIT", + pnl: 11.5, + net_pnl_pct: 0.021, + volume: 1101, + cum_fees_quote: 1.11, + timestamp: 1_700_000_100, + close_timestamp: 1_700_003_100, + config: { stop_loss: 0.02, take_profit: 0.04 }, + }), + }, + { + label: "grid", + overlayType: "grid", + executor: executor({ + id: "grid-2", + type: "grid", + side: "SELL", + status: "terminated", + close_type: "EARLY_STOP", + pnl: -22.5, + net_pnl_pct: -0.032, + volume: 2202, + cum_fees_quote: 2.22, + timestamp: 1_700_000_200, + close_timestamp: 1_700_003_200, + config: { start_price: 190, end_price: 210, limit_price: 180 }, + }), + }, + { + label: "lp", + overlayType: "lp", + executor: executor({ + id: "lp-3", + type: "lp", + side: "", + status: "terminated", + close_type: "TIME_LIMIT", + pnl: 33.5, + net_pnl_pct: 0.043, + volume: 3303, + cum_fees_quote: 3.33, + timestamp: 1_700_000_300, + close_timestamp: 1_700_003_300, + custom_info: { lower_price: 180, upper_price: 220 }, + config: { lower_price: 181, upper_price: 219 }, + }), + }, + { + label: "order", + overlayType: "order", + executor: executor({ + id: "order-4", + type: "order", + side: "BUY", + status: "terminated", + close_type: "FAILED", + pnl: -4.5, + net_pnl_pct: -0.054, + volume: 4404, + cum_fees_quote: 4.44, + timestamp: 1_700_000_400, + close_timestamp: 1_700_003_400, + config: { price: 205, amount: 2, execution_strategy: "LIMIT_CHASER" }, + }), + }, + { + label: "generic fallback", + overlayType: "dca", + executor: executor({ + id: "dca-5", + type: "DCA", + side: "SELL", + status: "terminated", + close_type: "STOP_LOSS", + pnl: 55.5, + net_pnl_pct: 0.065, + volume: 5505, + cum_fees_quote: 5.55, + timestamp: 1_700_000_500, + close_timestamp: 1_700_003_500, + config: { dca_levels: 3 }, + }), + }, +]; + +describe("every overlay builder copies the same shared fields", () => { + it("covers all five builders", () => { + // computeExecutorOverlay switches on four named types plus the fallback. + expect(new Set(cases.map((c) => c.overlayType)).size).toBe(5); + }); + + it.each(cases)("$label carries the executor's own values through", ({ overlayType, executor: ex }) => { + const overlay = computeMultiOverlays([ex])[0]; + + expect(overlay.type).toBe(overlayType); + expect(overlay.executorId).toBe(ex.id); + expect(overlay.side).toBe(ex.side === "BUY" ? "buy" : "sell"); + expect(overlay.status).toBe(ex.status); + expect(overlay.closeType).toBe(ex.close_type); + expect(overlay.pnl).toBe(ex.pnl); + expect(overlay.pnlPct).toBe(ex.net_pnl_pct); + expect(overlay.volume).toBe(ex.volume); + expect(overlay.fees).toBe(ex.cum_fees_quote); + expect(overlay.config).toBe(ex.config); + expect(overlay.timeRange).toEqual({ start: ex.timestamp, end: ex.close_timestamp }); + }); +}); + +describe("the lifetime range", () => { + it("clamps a still-open executor's end to now", () => { + const now = Math.floor(Date.now() / 1000); + const overlay = computeMultiOverlays([executor({ close_timestamp: 0 })])[0]; + + expect(overlay.timeRange.start).toBe(1_700_000_000); + expect(overlay.timeRange.end).toBeGreaterThanOrEqual(now); + expect(overlay.timeRange.end).toBeLessThanOrEqual(now + 2); + }); + + it("clamps a timestamp-less executor's start to now rather than the epoch", () => { + const now = Math.floor(Date.now() / 1000); + const overlay = computeMultiOverlays([executor({ timestamp: 0, close_timestamp: 0 })])[0]; + + expect(overlay.timeRange.start).toBeGreaterThanOrEqual(now); + expect(overlay.timeRange.start).toBeLessThanOrEqual(now + 2); + }); +}); diff --git a/frontend/src/lib/executor-overlays.ts b/frontend/src/lib/executor-overlays.ts index 0a1e9aa00..dbad663ba 100644 --- a/frontend/src/lib/executor-overlays.ts +++ b/frontend/src/lib/executor-overlays.ts @@ -118,6 +118,60 @@ function isActiveStatus(status: string): boolean { return s === "running" || s === "active_position" || s === "active"; } +/** + * The executor's span on the chart: its own timestamps, clamped to now while a + * bound is still missing. An open executor has no `close_timestamp`, and a + * malformed one can arrive with neither, so both ends fall back to the present + * rather than to 1970 -- which would stretch every chart back to the epoch. + */ +function lifetimeRange(executor: ExecutorInfo): { start: number; end: number } { + const now = Math.floor(Date.now() / 1000); + return { + start: executor.timestamp > 0 ? executor.timestamp : now, + end: executor.close_timestamp > 0 ? executor.close_timestamp : now, + }; +} + +/** The parts of an overlay each executor type draws for itself. */ +type OverlayExtras = Pick & + Partial>; + +/** + * Every overlay field that is copied off the executor unchanged. + * + * The five `compute*Overlay` builders differ only in `type`, the lines, markers + * and prices they derive, and whether they draw a `segment` or a `gridBox`; the + * other ten fields were written out by hand five times, so adding one to + * `ExecutorOverlay` was five edits and forgetting one of them was a silent + * per-type drift for the optional fields (the shape CORR-280 shipped once, with + * `side` read differently per builder). Each builder now returns through here. + * + * `range` is a parameter only so a builder that already needed the bounds -- + * to size a grid box or an order's segment -- draws the box and the time range + * from the same clamp instead of two calls to `Date.now()`. + */ +function baseOverlay( + executor: ExecutorInfo, + type: string, + extras: OverlayExtras, + range: { start: number; end: number } = lifetimeRange(executor), +): ExecutorOverlay { + return { + executorId: executor.id, + type, + side: normSide(executor.side), + status: executor.status, + closeType: executor.close_type, + pnl: executor.pnl, + pnlPct: executor.net_pnl_pct, + volume: executor.volume, + fees: executor.cum_fees_quote, + timeRange: range, + config: executor.config, + ...extras, + }; +} + // ── Position Executor Overlay ── function computePositionOverlay(executor: ExecutorInfo): ExecutorOverlay { @@ -244,49 +298,33 @@ function computePositionOverlay(executor: ExecutorInfo): ExecutorOverlay { }); } - const start = executor.timestamp > 0 ? executor.timestamp : Math.floor(Date.now() / 1000); - const end = executor.close_timestamp > 0 ? executor.close_timestamp : Math.floor(Date.now() / 1000); - - return { - executorId: executor.id, - type: "position", - side, - status: executor.status, - closeType: executor.close_type, - pnl: executor.pnl, - pnlPct: executor.net_pnl_pct, - volume: executor.volume, - fees: executor.cum_fees_quote, + return baseOverlay(executor, "position", { priceLines: lines, markers, segment, - timeRange: { start, end }, - config: executor.config, entryPrice: entry, exitPrice: closePrice, - }; + }); } // ── Grid Executor Overlay ── function computeGridOverlay(executor: ExecutorInfo): ExecutorOverlay { - const side = normSide(executor.side); const config = executor.config || {}; const startPrice = Number(config.start_price); const endPrice = Number(config.end_price); const limitPrice = Number(config.limit_price); - const start = executor.timestamp > 0 ? executor.timestamp : Math.floor(Date.now() / 1000); - const end = executor.close_timestamp > 0 ? executor.close_timestamp : Math.floor(Date.now() / 1000); + const range = lifetimeRange(executor); // Grid box: rectangle from start_price to end_price over the executor lifetime let gridBox: GridBox | undefined; - if (startPrice > 0 && endPrice > 0 && start > 0) { + if (startPrice > 0 && endPrice > 0 && range.start > 0) { const profitable = executor.pnl >= 0; gridBox = { - startTime: start, - endTime: end, + startTime: range.start, + endTime: range.end, startPrice, endPrice, limitPrice: limitPrice > 0 ? limitPrice : undefined, @@ -294,24 +332,18 @@ function computeGridOverlay(executor: ExecutorInfo): ExecutorOverlay { }; } - return { - executorId: executor.id, - type: "grid", - side, - status: executor.status, - closeType: executor.close_type, - pnl: executor.pnl, - pnlPct: executor.net_pnl_pct, - volume: executor.volume, - fees: executor.cum_fees_quote, - priceLines: [], - markers: [], - gridBox, - timeRange: { start, end }, - config: executor.config, - entryPrice: startPrice, - exitPrice: endPrice, - }; + return baseOverlay( + executor, + "grid", + { + priceLines: [], + markers: [], + gridBox, + entryPrice: startPrice, + exitPrice: endPrice, + }, + range, + ); } // ── LP Executor Overlay ── @@ -333,7 +365,6 @@ function computeGridOverlay(executor: ExecutorInfo): ExecutorOverlay { function computeLpOverlay(executor: ExecutorInfo): ExecutorOverlay { const customInfo = executor.custom_info || {}; const config = executor.config || {}; - const side = normSide(executor.side); // custom_info wins: a CLMM position is snapped to the venue's bins, so the // on-chain bounds are not the requested ones, and the box has to show where the @@ -346,14 +377,13 @@ function computeLpOverlay(executor: ExecutorInfo): ExecutorOverlay { const lower = num(customInfo.lower_price ?? customInfo.price_lower ?? config.lower_price); const upper = num(customInfo.upper_price ?? customInfo.price_upper ?? config.upper_price); - const start = executor.timestamp > 0 ? executor.timestamp : Math.floor(Date.now() / 1000); - const end = executor.close_timestamp > 0 ? executor.close_timestamp : Math.floor(Date.now() / 1000); + const range = lifetimeRange(executor); let gridBox: GridBox | undefined; - if (lower > 0 && upper > 0 && start > 0) { + if (lower > 0 && upper > 0 && range.start > 0) { gridBox = { - startTime: start, - endTime: end, + startTime: range.start, + endTime: range.end, // startPrice is the box's dashed edge and endPrice its solid one; the grid // overlay puts start_price (the far bound) first, so upper goes first here. startPrice: upper, @@ -362,24 +392,18 @@ function computeLpOverlay(executor: ExecutorInfo): ExecutorOverlay { }; } - return { - executorId: executor.id, - type: "lp", - side, - status: executor.status, - closeType: executor.close_type, - pnl: executor.pnl, - pnlPct: executor.net_pnl_pct, - volume: executor.volume, - fees: executor.cum_fees_quote, - priceLines: [], - markers: [], - gridBox, - timeRange: { start, end }, - config: executor.config, - entryPrice: lower, - exitPrice: upper, - }; + return baseOverlay( + executor, + "lp", + { + priceLines: [], + markers: [], + gridBox, + entryPrice: lower, + exitPrice: upper, + }, + range, + ); } // ── Order Executor Overlay ── @@ -411,8 +435,8 @@ function computeOrderOverlay(executor: ExecutorInfo): ExecutorOverlay { const descriptiveLabel = `${sideLabel}${amountStr}${chaserSuffix}`; const active = isActiveStatus(executor.status); - const start = executor.timestamp > 0 ? executor.timestamp : Math.floor(Date.now() / 1000); - const end = executor.close_timestamp > 0 ? executor.close_timestamp : Math.floor(Date.now() / 1000); + const range = lifetimeRange(executor); + const start = range.start; let segment: ExecutorSegment | undefined; @@ -473,24 +497,18 @@ function computeOrderOverlay(executor: ExecutorInfo): ExecutorOverlay { } } - return { - executorId: executor.id, - type: "order", - side, - status: executor.status, - closeType: executor.close_type, - pnl: executor.pnl, - pnlPct: executor.net_pnl_pct, - volume: executor.volume, - fees: executor.cum_fees_quote, - priceLines: lines, - markers, - segment, - timeRange: { start, end }, - config: executor.config, - entryPrice: orderPrice, - exitPrice: closePrice, - }; + return baseOverlay( + executor, + "order", + { + priceLines: lines, + markers, + segment, + entryPrice: orderPrice, + exitPrice: closePrice, + }, + range, + ); } // ── Generic Executor Overlay (fallback) ── @@ -553,27 +571,13 @@ function computeGenericOverlay(executor: ExecutorInfo): ExecutorOverlay { }); } - const start = executor.timestamp > 0 ? executor.timestamp : Math.floor(Date.now() / 1000); - const end = executor.close_timestamp > 0 ? executor.close_timestamp : Math.floor(Date.now() / 1000); - - return { - executorId: executor.id, - type: executor.type?.toLowerCase() || "unknown", - side, - status: executor.status, - closeType: executor.close_type, - pnl: executor.pnl, - pnlPct: executor.net_pnl_pct, - volume: executor.volume, - fees: executor.cum_fees_quote, + return baseOverlay(executor, executor.type?.toLowerCase() || "unknown", { priceLines: lines, markers, segment, - timeRange: { start, end }, - config: executor.config, entryPrice: entryPrice, exitPrice: closePrice, - }; + }); } // ── Public API ── From 5c7d3b018063c97a6a3f33f204a102689bb74c7c Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 18:23:24 +0300 Subject: [PATCH 017/154] Ask lib/configYaml whether the text is a mapping, in all five editors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eight-line "parse YAML, reject a non-mapping" block was written out five times and had already drifted into three behaviours: two editors said "YAML must be a mapping (key: value)", two said "YAML must be a mapping", and YamlConfigEditor did not check the shape while typing at all — it accepted a YAML array and only rejected it inside the save mutation. lib/configYaml.ts already owns the config->YAML direction, so it now owns the way back: parseYamlMapping() returns the mapping or a message (never throws), and validateYamlMapping() is the validate-only view the editors call on every keystroke. Parse errors keep YamlConfigEditor's first-line trim, which is the human-readable half of a js-yaml message. UploadDialog keeps its extra `id` check on top, now on the already-parsed value. yaml.load survives only where a value is actually consumed: the upload, clone and controller-config save mutations. --- .../src/components/editor/EditorDialogs.tsx | 29 ++------- .../src/components/editor/EditorModal.tsx | 14 +--- .../src/components/perf/YamlConfigEditor.tsx | 9 +-- frontend/src/lib/configYaml.test.ts | 64 +++++++++++++++++++ frontend/src/lib/configYaml.ts | 37 +++++++++++ 5 files changed, 111 insertions(+), 42 deletions(-) create mode 100644 frontend/src/lib/configYaml.test.ts diff --git a/frontend/src/components/editor/EditorDialogs.tsx b/frontend/src/components/editor/EditorDialogs.tsx index 66798e89d..818126f71 100644 --- a/frontend/src/components/editor/EditorDialogs.tsx +++ b/frontend/src/components/editor/EditorDialogs.tsx @@ -6,7 +6,7 @@ import yaml from "js-yaml"; import { CodeEditor } from "@/components/editor/CodeEditor"; import { useEscapeKey } from "@/hooks/useEscapeKey"; import { api, type ControllerConfigSummary } from "@/lib/api"; -import { configToYaml } from "@/lib/configYaml"; +import { configToYaml, parseYamlMapping, validateYamlMapping } from "@/lib/configYaml"; // ── Delete Confirm Dialog ── @@ -155,18 +155,10 @@ export function UploadDialog({ const validateContent = useCallback( (val: string, m: UploadMode) => { if (m === "config") { - try { - const parsed = yaml.load(val); - if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { - return "YAML must be a mapping"; - } - if (!(parsed as Record).id) { - return "Config must have an 'id' field"; - } - return null; - } catch (e) { - return e instanceof Error ? e.message : "Invalid YAML"; - } + const result = parseYamlMapping(val); + if (!result.ok) return result.error; + if (!result.value.id) return "Config must have an 'id' field"; + return null; } // Controller: just needs non-empty Python if (!val.trim()) return "Paste or drop a Python file"; @@ -418,16 +410,7 @@ export function CloneConfigDialog({ const handleYamlChange = useCallback((val: string) => { setYamlContent(val); - try { - const parsed = yaml.load(val); - if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { - setYamlError("YAML must be a mapping"); - } else { - setYamlError(null); - } - } catch (e) { - setYamlError(e instanceof Error ? e.message : "Invalid YAML"); - } + setYamlError(validateYamlMapping(val)); }, []); const createMutation = useMutation({ diff --git a/frontend/src/components/editor/EditorModal.tsx b/frontend/src/components/editor/EditorModal.tsx index 597bbeb9c..c22c7f9de 100644 --- a/frontend/src/components/editor/EditorModal.tsx +++ b/frontend/src/components/editor/EditorModal.tsx @@ -18,7 +18,6 @@ import { X, } from "lucide-react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import yaml from "js-yaml"; import { NoServerCard } from "@/components/NoServerCard"; import { CodeEditor } from "@/components/editor/CodeEditor"; @@ -33,7 +32,7 @@ import { useDismissOnOutsideClick } from "@/hooks/useDismissOnOutsideClick"; import { useEscapeKey } from "@/hooks/useEscapeKey"; import { useServer } from "@/hooks/useServer"; import { api, type ControllerConfigSummary } from "@/lib/api"; -import { configToYaml } from "@/lib/configYaml"; +import { configToYaml, validateYamlMapping } from "@/lib/configYaml"; // ── Types ── @@ -306,16 +305,7 @@ function EditorPane({ (val: string) => { onContentChange(tab.file.id, val); if (tab.language === "yaml") { - try { - const parsed = yaml.load(val); - if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { - setYamlError("YAML must be a mapping (key: value)"); - } else { - setYamlError(null); - } - } catch (e) { - setYamlError(e instanceof Error ? e.message : "Invalid YAML"); - } + setYamlError(validateYamlMapping(val)); } }, [tab.file.id, tab.language, onContentChange], diff --git a/frontend/src/components/perf/YamlConfigEditor.tsx b/frontend/src/components/perf/YamlConfigEditor.tsx index b9fdcd003..816ef7fd9 100644 --- a/frontend/src/components/perf/YamlConfigEditor.tsx +++ b/frontend/src/components/perf/YamlConfigEditor.tsx @@ -5,7 +5,7 @@ import yamlLib from "js-yaml"; import { CodeEditor } from "@/components/editor/CodeEditor"; import { api } from "@/lib/api"; -import { configToYaml, CONTROLLER_HIDDEN_KEYS } from "@/lib/configYaml"; +import { configToYaml, CONTROLLER_HIDDEN_KEYS, validateYamlMapping } from "@/lib/configYaml"; /** * The right drawer's config column: a controller's config as editable YAML. @@ -57,12 +57,7 @@ export function YamlConfigEditor({ const handleChange = useCallback((value: string) => { setYamlContent(value); - try { - yamlLib.load(value); - setParseError(null); - } catch (e) { - setParseError((e as Error).message?.split("\n")[0] || "Invalid YAML"); - } + setParseError(validateYamlMapping(value)); }, []); const saveMutation = useMutation({ diff --git a/frontend/src/lib/configYaml.test.ts b/frontend/src/lib/configYaml.test.ts new file mode 100644 index 000000000..7035b38f4 --- /dev/null +++ b/frontend/src/lib/configYaml.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; + +import { + YAML_NOT_A_MAPPING, + parseYamlMapping, + validateYamlMapping, +} from "@/lib/configYaml"; + +describe("validateYamlMapping", () => { + it("accepts a mapping", () => { + expect(validateYamlMapping("id: my-config\nspread: 0.001\n")).toBeNull(); + }); + + it("accepts an empty mapping written as {}", () => { + expect(validateYamlMapping("{}")).toBeNull(); + }); + + it("rejects a YAML array", () => { + expect(validateYamlMapping("- one\n- two\n")).toBe(YAML_NOT_A_MAPPING); + }); + + it("rejects null", () => { + expect(validateYamlMapping("null")).toBe(YAML_NOT_A_MAPPING); + }); + + it("rejects empty input, which parses to undefined", () => { + expect(validateYamlMapping("")).toBe(YAML_NOT_A_MAPPING); + }); + + it("rejects a bare scalar", () => { + expect(validateYamlMapping("42")).toBe(YAML_NOT_A_MAPPING); + expect(validateYamlMapping("just a string")).toBe(YAML_NOT_A_MAPPING); + }); + + it("reports malformed YAML with the first line of the parser message", () => { + const error = validateYamlMapping("id: [1, 2\n"); + expect(error).toBeTruthy(); + expect(error).not.toBe(YAML_NOT_A_MAPPING); + expect(error).not.toContain("\n"); + }); + + it("gives every editor the same message for the same input", () => { + expect(validateYamlMapping("- one")).toBe(validateYamlMapping("null")); + }); +}); + +describe("parseYamlMapping", () => { + it("hands back the parsed mapping on success", () => { + const result = parseYamlMapping("id: my-config\nnested:\n a: 1\n"); + expect(result).toEqual({ ok: true, value: { id: "my-config", nested: { a: 1 } } }); + }); + + it("never throws on malformed input", () => { + const result = parseYamlMapping("id: [1, 2\n"); + expect(result.ok).toBe(false); + }); + + it("agrees with validateYamlMapping", () => { + for (const text of ["id: x", "- one", "null", "42", "id: [1, 2"]) { + const result = parseYamlMapping(text); + expect(validateYamlMapping(text)).toBe(result.ok ? null : result.error); + } + }); +}); diff --git a/frontend/src/lib/configYaml.ts b/frontend/src/lib/configYaml.ts index 4d5af119f..8a4c3ecf2 100644 --- a/frontend/src/lib/configYaml.ts +++ b/frontend/src/lib/configYaml.ts @@ -10,6 +10,43 @@ const ALWAYS_HIDDEN_KEYS = ["id"] as const; // read-only / partial-update controller browser opts into hiding them. export const CONTROLLER_HIDDEN_KEYS = ["controller_name", "controller_type"] as const; +/** The one message every editor shows for YAML that parses but isn't a mapping. */ +export const YAML_NOT_A_MAPPING = "YAML must be a mapping (key: value)"; + +export type YamlMappingResult = + | { ok: true; value: Record } + | { ok: false; error: string }; + +/** + * Parse YAML that is required to be a mapping (the shape every config editor + * round-trips). Never throws: a parse failure comes back as `ok: false` with the + * first line of the js-yaml message, which is the human-readable part — the rest + * is the source snippet the editor already shows. + */ +export function parseYamlMapping(text: string): YamlMappingResult { + let parsed: unknown; + try { + parsed = yaml.load(text); + } catch (e) { + const message = e instanceof Error ? e.message : ""; + return { ok: false, error: message.split("\n")[0] || "Invalid YAML" }; + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + return { ok: false, error: YAML_NOT_A_MAPPING }; + } + return { ok: true, value: parsed as Record }; +} + +/** + * Validate-only view of `parseYamlMapping`: `null` when `text` is a YAML + * mapping, otherwise the message to show the user. This is what the editors + * call on every keystroke. + */ +export function validateYamlMapping(text: string): string | null { + const result = parseYamlMapping(text); + return result.ok ? null : result.error; +} + export interface ConfigToYamlOptions { /** Extra keys to strip in addition to `id` (e.g. CONTROLLER_HIDDEN_KEYS). */ hiddenKeys?: readonly string[]; From a05163b093cf979a18859725d90bbc2ff55baed2 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 18:24:01 +0300 Subject: [PATCH 018/154] Ask lib/configYaml the same question in BotDetail too Left out of 5c7d3b01 by a concurrent index reset: BotDetail's YAML editor was the fifth copy of the hand-rolled mapping check. --- frontend/src/pages/BotDetail.tsx | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/frontend/src/pages/BotDetail.tsx b/frontend/src/pages/BotDetail.tsx index 858093bb6..643fb6444 100644 --- a/frontend/src/pages/BotDetail.tsx +++ b/frontend/src/pages/BotDetail.tsx @@ -11,13 +11,12 @@ import { } from "lucide-react"; import { useCallback, useMemo, useRef, useState } from "react"; import { Link, useParams } from "react-router-dom"; -import yaml from "js-yaml"; import { CodeEditor } from "@/components/editor/CodeEditor"; import { FallbackSpinner } from "@/components/ui/FallbackSpinner"; import { useServer } from "@/hooks/useServer"; import { api } from "@/lib/api"; -import { configToYaml } from "@/lib/configYaml"; +import { configToYaml, validateYamlMapping } from "@/lib/configYaml"; export function BotDetail() { const { id } = useParams<{ id: string }>(); @@ -60,16 +59,7 @@ export function BotDetail() { const handleYamlChange = useCallback((val: string) => { setYamlValue(val); - try { - const parsed = yaml.load(val); - if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { - setYamlError("YAML must be a mapping (key: value)"); - } else { - setYamlError(null); - } - } catch (e) { - setYamlError(e instanceof Error ? e.message : "Invalid YAML"); - } + setYamlError(validateYamlMapping(val)); }, []); const isDirty = yamlValue !== originalYaml; From 32ebad7cfd192ca5a027bee24e661826d4bfeb78 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 18:34:59 +0300 Subject: [PATCH 019/154] Both PnL charts read their activity pane from one hook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two charts each wired the same activity pane by hand: the measured width, the bucket and its label, the `volumeBarWidth(width - 2*AXIS_WIDTH - PANE_MARGIN_RIGHT, ...)` subtraction, a re-centring `` bar shape, and the two position-axis memos. Every helper underneath was already shared in lib/pnl-chart; only the wiring was copied — along with the recharts smallest-gap workaround, explained twice. The copies had started to drift. PnlEvolutionChart's shape forwards `radius` and `fillOpacity` from the ``; OwnerPnlChart's hardcoded them. The forwarding version is the one that survives, so OwnerPnlChart's `` now passes the two literals its shape used to hold, and the bars render identically because the same code draws them. `useActivityPane` sits beside the helpers it uses, in a `.tsx` sibling rather than in pnl-chart.ts, which is a plain module with no React in it. --- .../src/components/bots/PnlEvolutionChart.tsx | 80 ++------ .../src/components/perf/OwnerPnlChart.tsx | 60 ++---- frontend/src/lib/pnl-chart-pane.test.tsx | 188 ++++++++++++++++++ frontend/src/lib/pnl-chart-pane.tsx | 126 ++++++++++++ 4 files changed, 340 insertions(+), 114 deletions(-) create mode 100644 frontend/src/lib/pnl-chart-pane.test.tsx create mode 100644 frontend/src/lib/pnl-chart-pane.tsx diff --git a/frontend/src/components/bots/PnlEvolutionChart.tsx b/frontend/src/components/bots/PnlEvolutionChart.tsx index fce26624a..81c3c44dd 100644 --- a/frontend/src/components/bots/PnlEvolutionChart.tsx +++ b/frontend/src/components/bots/PnlEvolutionChart.tsx @@ -42,13 +42,11 @@ import { CartesianGrid, ComposedChart, Line, - Rectangle, ReferenceLine, ResponsiveContainer, Tooltip, XAxis, YAxis, - type BarShapeProps, } from "recharts"; import { formatAxisCurrency, formatAxisTime, formatCurrencyVolume, formatCurrencyPnl, pnlColor } from "@/lib/formatters"; @@ -63,22 +61,17 @@ import { PNL_SERIES_LABELS, PNL_SERIES_PANE, RANGE_PRESETS, - chartBucketMs, - formatBucketLabel, hiddenSeriesSnapshot, paneSeries, - positionAreaExtent, - positionAxisDomain, resolveTimeRange, setSeriesHidden, sliceToRange, subscribeToHiddenSeries, - volumeBarWidth, - zeroGradientOffset, type PnlChartPoint, type PnlSeriesKey, type TimeRange, } from "@/lib/pnl-chart"; +import { useActivityPane } from "@/lib/pnl-chart-pane"; import { getThemeColors } from "@/lib/theme-colors"; import { PnlEvolutionTooltip } from "./PnlChartTooltips"; import { PnlRangeStrip } from "./PnlRangeStrip"; @@ -506,67 +499,18 @@ export function PnlEvolutionChart({ data, title, pnlHeight, volumeHeight, curren [drawnPnl, drawnActivity], ); - // The position axis is pinned across zero rather than left to recharts, so - // the signed area always has its baseline on screen (READ-246). Memoised - // because recharts keeps the domain in its own store and a fresh array on - // every render would churn it. - const positionDomain = useMemo(() => positionAxisDomain(visible), [visible]); - // Measured against the area's own extent, not the padded domain: the fill's - // gradient is in objectBoundingBox units. See zeroGradientOffset. - const positionZeroOffset = useMemo(() => zeroGradientOffset(positionAreaExtent(visible)), [visible]); - - // ── Volume bars (READ-245) ── - // - // The bars are sized by us, not by recharts. On a numeric X axis recharts - // takes a bar's width from the *smallest* gap between two adjacent points and - // clamps any explicit `barSize` back under it — and this series always has - // one gap far smaller than the rest, because the fold ends it with a live - // "now" point a fraction of a bucket after the last snapshot. Left alone, - // every bar in the pane would be drawn at that fraction, thinning to a - // hairline and thickening again with each snapshot that lands. See - // `volumeBarWidth` and `chartBucketMs`. + // ── Volume bars and the position axis (READ-245, READ-246) ── // - // The measurement comes from the pane's own ResponsiveContainer, which is - // already observing its size, rather than from a second observer of ours. It - // is 0 until the first callback — and stays 0 where there is no layout at all - // — which `volumeBarWidth` answers with `undefined`, i.e. "leave it to - // recharts". - const [activityWidth, setActivityWidth] = useState(0); - const onActivityResize = useCallback((width: number) => setActivityWidth(width), []); - const bucketMs = useMemo(() => chartBucketMs(visible), [visible]); - // The bucket has to be named in the tooltip: "Volume" used to be a running - // total, which needs no qualifier, and is now one bucket's worth, which means - // nothing until you know how long a bucket is. - const bucketLabel = useMemo(() => formatBucketLabel(bucketMs), [bucketMs]); - const barWidth = volumeBarWidth( - // The plot area, not the card: both gutters and the right margin are - // outside the time domain the bars are placed in. - activityWidth - 2 * AXIS_WIDTH - PANE_MARGIN_RIGHT, - spanMs, - bucketMs, - ); - // Centred on its instant rather than starting there (recharts' own - // convention on a numeric axis), so a bar sits under the synced cursor and - // the tooltip that reports it, in both panes. - const volumeBar = useCallback( - (props: BarShapeProps) => { - const width = barWidth ?? props.width; - const x = props.x + props.width / 2 - width / 2; - return ( - - ); - }, - [barWidth], - ); + // The pane's whole geometry — the measured width the bars are sized from, + // the bucket, the re-centring bar shape, the zero-pinned position axis — + // comes from one hook, shared with OwnerPnlChart's activity pane (ARCH-341). + const { + onActivityResize, + bucketLabel, + volumeBar, + positionDomain, + positionZeroOffset, + } = useActivityPane(visible, spanMs); // What the bars on screen add up to — the flow the activity pane draws, as // opposed to `latest.volume`, the lifetime counter they were differenced from diff --git a/frontend/src/components/perf/OwnerPnlChart.tsx b/frontend/src/components/perf/OwnerPnlChart.tsx index 1f3b4286d..7a075fc9e 100644 --- a/frontend/src/components/perf/OwnerPnlChart.tsx +++ b/frontend/src/components/perf/OwnerPnlChart.tsx @@ -5,7 +5,6 @@ import { CartesianGrid, ComposedChart, Line, - Rectangle, ReferenceLine, ResponsiveContainer, Tooltip, @@ -40,17 +39,12 @@ import { PANE_MARGIN_RIGHT, PANE_PAD_X, PNL_SERIES_COLORS, - chartBucketMs, - formatBucketLabel, - positionAreaExtent, - positionAxisDomain, resolveTimeRange, sliceToRange, - volumeBarWidth, - zeroGradientOffset, type PnlChartPoint, type TimeRange, } from "@/lib/pnl-chart"; +import { useActivityPane } from "@/lib/pnl-chart-pane"; import { getThemeColors } from "@/lib/theme-colors"; /** @@ -216,46 +210,18 @@ export function OwnerPnlChart({ const fmtTimeAxis = useCallback((v: number) => formatAxisTime(v, spanMs), [spanMs]); // ── The activity pane (step 6) ── - const [activityWidth, setActivityWidth] = useState(0); - const onActivityResize = useCallback((width: number) => setActivityWidth(width), []); - const bucketMs = useMemo(() => chartBucketMs(visible as PnlChartPoint[]), [visible]); - const bucketLabel = useMemo(() => formatBucketLabel(bucketMs), [bucketMs]); - const barWidth = volumeBarWidth( - activityWidth - 2 * AXIS_WIDTH - PANE_MARGIN_RIGHT, - spanMs, - bucketMs, - ); - // recharts sizes a bar on a numeric axis from the *smallest* gap between two - // points, and this series always has one far smaller than the rest — the live - // "now" point lands a fraction of a bucket after the last snapshot. Left - // alone every bar thins to a hairline and thickens again as snapshots land. - const volumeBar = useCallback( - (props: { x: number; y: number; width: number; height: number; fill?: string }) => { - const width = barWidth ?? props.width; - return ( - - ); - }, - [barWidth], - ); + // + // Its geometry is the same geometry PnlEvolutionChart's activity pane draws + // with — the measured width, the bucket, the re-centring bar shape, the + // zero-pinned position axis — so both read it from one hook (ARCH-341). + const { + onActivityResize, + bucketLabel, + volumeBar, + positionDomain, + positionZeroOffset, + } = useActivityPane(visible as PnlChartPoint[], spanMs); const hasPosition = rows.some((row) => row.position !== 0); - const positionDomain = useMemo( - () => positionAxisDomain(visible as PnlChartPoint[]), - [visible], - ); - const positionZeroOffset = useMemo( - () => zeroGradientOffset(positionAreaExtent(visible as PnlChartPoint[])), - [visible], - ); // ── The stated gap between the chart and the strip ── // @@ -596,6 +562,8 @@ export function OwnerPnlChart({ dataKey="volumeDelta" name={bucketLabel ? `Traded / ${bucketLabel}` : "Traded"} fill={PNL_SERIES_COLORS.volume} + fillOpacity={0.45} + radius={[2, 2, 0, 0]} shape={volumeBar} isAnimationActive={false} /> diff --git a/frontend/src/lib/pnl-chart-pane.test.tsx b/frontend/src/lib/pnl-chart-pane.test.tsx new file mode 100644 index 000000000..c3a6f4985 --- /dev/null +++ b/frontend/src/lib/pnl-chart-pane.test.tsx @@ -0,0 +1,188 @@ +/** + * The activity pane's shared wiring (ARCH-341). + * + * The pure helpers are pinned in `pnl-chart.test.ts`; what this file pins is + * the part that used to be copied into both charts and could therefore drift: + * the *plot* width the bars are sized from — the container minus both axis + * gutters and the right margin, not the container itself — the bar shape's + * re-centring and its forwarding of the cosmetic props, and the position + * gradient's offset being measured against the area's own signed extent rather + * than the padded axis domain. + * + * Needs a DOM to hold the hook's state, so this file overrides vitest's default + * `node` environment. + * + * @vitest-environment jsdom + */ + +import { act, useEffect } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import type { BarShapeProps } from "recharts"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { + AXIS_WIDTH, + PANE_MARGIN_RIGHT, + chartBucketMs, + positionAreaExtent, + positionAxisDomain, + volumeBarWidth, + zeroGradientOffset, + type PnlChartPoint, +} from "./pnl-chart"; +import { useActivityPane, type ActivityPane } from "./pnl-chart-pane"; + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +const BUCKET = 5 * 60_000; + +/** A window of `count` points a bucket apart, with the given net positions. */ +function points(positions: number[]): PnlChartPoint[] { + return positions.map((position, index) => ({ + time: 1_700_000_000_000 + index * BUCKET, + realized: 0, + unrealized: 0, + total: 0, + volume: index * 100, + volumeDelta: 100, + position, + })); +} + +function spanOf(data: PnlChartPoint[]): number { + return data.length > 1 ? data[data.length - 1].time - data[0].time : 0; +} + +let container: HTMLDivElement; +let root: Root; + +/** + * Mount the hook over `data` and hand back its live result. + * + * The result is published from an effect rather than assigned during render: + * writing to a captured variable while rendering is a side effect the + * react-hooks rules reject, test harness or not. `act` flushes the effect, so + * the value is there by the time the caller reads it. + */ +function mount(data: PnlChartPoint[]): { pane: () => ActivityPane } { + let latest: ActivityPane | null = null; + const publish = (pane: ActivityPane) => { + latest = pane; + }; + function Harness() { + const pane = useActivityPane(data, spanOf(data)); + useEffect(() => { + publish(pane); + }); + return null; + } + act(() => { + root.render(); + }); + return { + pane: () => { + if (!latest) throw new Error("hook did not run"); + return latest; + }, + }; +} + +/** The rect the pane would draw for one bar, given recharts' own placement. */ +function drawnBar(pane: ActivityPane, from: Partial = {}) { + const props = { + x: 100, + y: 20, + width: 4, + height: 60, + fill: "#3b82f6", + fillOpacity: 0.45, + radius: [2, 2, 0, 0], + ...from, + } as unknown as BarShapeProps; + return pane.volumeBar(props).props as { + x: number; + width: number; + radius?: unknown; + fillOpacity?: number; + fill?: string; + }; +} + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); +}); + +describe("useActivityPane", () => { + it("sizes a bar from the plot area, not the container", () => { + // Enough buckets that the bar sits below VOLUME_BAR_MAX_PX at both widths, + // or the clamp would hide the difference the subtraction makes. + const data = points(new Array(40).fill(0)); + const { pane } = mount(data); + // Nothing is measured yet, so the width is recharts' own. + expect(drawnBar(pane()).width).toBe(4); + + const containerWidth = 640; + act(() => pane().onActivityResize(containerWidth)); + + const span = spanOf(data); + const bucketMs = chartBucketMs(data); + const expected = volumeBarWidth( + containerWidth - 2 * AXIS_WIDTH - PANE_MARGIN_RIGHT, + span, + bucketMs, + ); + expect(expected).toBeDefined(); + expect(drawnBar(pane()).width).toBeCloseTo(expected as number, 10); + // The subtraction is load-bearing: the whole container would be wider. + expect(expected).toBeLessThan(volumeBarWidth(containerWidth, span, bucketMs) as number); + }); + + it("centres the bar on its instant and forwards the cosmetic props", () => { + const data = points(new Array(40).fill(0)); + const { pane } = mount(data); + act(() => pane().onActivityResize(640)); + + const rect = drawnBar(pane(), { x: 100, width: 4 }); + // recharts starts a bar at its instant; the pane draws it centred there. + expect(rect.x).toBeCloseTo(100 + 4 / 2 - rect.width / 2, 10); + expect(rect.radius).toEqual([2, 2, 0, 0]); + expect(rect.fillOpacity).toBe(0.45); + + const bare = drawnBar(pane(), { radius: undefined, fillOpacity: undefined }); + expect(bare.radius).toBeUndefined(); + expect(bare.fillOpacity).toBeUndefined(); + }); + + it("puts the gradient's zero on the signed area's own extent", () => { + const data = points([0, 400, -100, 200]); + const { pane } = mount(data); + + const extent = positionAreaExtent(data); + expect(extent).toEqual([-100, 400]); + expect(pane().positionZeroOffset).toBeCloseTo(zeroGradientOffset(extent), 12); + expect(pane().positionZeroOffset).toBeGreaterThan(0); + expect(pane().positionZeroOffset).toBeLessThan(1); + // Not the padded domain: that would put the colour change off the baseline. + expect(pane().positionZeroOffset).not.toBeCloseTo( + zeroGradientOffset(positionAxisDomain(data)), + 12, + ); + expect(pane().positionDomain).toEqual(positionAxisDomain(data)); + }); + + it("names the window's bucket", () => { + const { pane } = mount(points([0, 0, 0, 0])); + expect(pane().bucketMs).toBe(BUCKET); + expect(pane().bucketLabel).toBe("5m"); + }); +}); diff --git a/frontend/src/lib/pnl-chart-pane.tsx b/frontend/src/lib/pnl-chart-pane.tsx new file mode 100644 index 000000000..4fec7d36a --- /dev/null +++ b/frontend/src/lib/pnl-chart-pane.tsx @@ -0,0 +1,126 @@ +// ── The activity pane's wiring, in one place (ARCH-341) ── +// +// `lib/pnl-chart` holds everything pure about the two PNL charts — the axis +// gutter, the pane insets, the bucket and bar geometry, the position axis +// rules. What used to sit outside it, copied into both `PnlEvolutionChart` and +// `OwnerPnlChart`, was the *wiring* that turns those helpers into an activity +// pane: the measured width, the bar shape that re-centres a rect on its +// instant, and the two position-axis memos. Two copies meant the recharts +// workaround below was explained twice and could be fixed once. +// +// It lives beside `pnl-chart.ts` rather than inside it because the bar shape is +// JSX and that module is a plain `.ts` with no React import. + +import { useCallback, useMemo, useState, type ReactElement } from "react"; +import { Rectangle, type BarShapeProps } from "recharts"; + +import { + AXIS_WIDTH, + PANE_MARGIN_RIGHT, + chartBucketMs, + formatBucketLabel, + positionAreaExtent, + positionAxisDomain, + volumeBarWidth, + zeroGradientOffset, + type PnlChartPoint, + type SamplingInterval, +} from "@/lib/pnl-chart"; + +export interface ActivityPane { + /** Hand to the pane's ``; sizes the bars. */ + onActivityResize: (width: number) => void; + /** The window's bucket, in ms — one bar's worth of volume. */ + bucketMs: number; + /** That bucket named for the tooltip and the caption, when it has a name. */ + bucketLabel: SamplingInterval | undefined; + /** The `` that draws a bar at our width, centred on its instant. */ + volumeBar: (props: BarShapeProps) => ReactElement; + /** The position axis' domain, pinned across zero. */ + positionDomain: [number, number]; + /** Where zero falls in the position area's own extent, for its gradient. */ + positionZeroOffset: number; +} + +/** + * Everything the activity pane needs that is not markup, derived from the + * window it is drawing (READ-245, READ-246). + * + * @param visible the points currently on screen + * @param spanMs the window's time span, `last.time - first.time` + */ +export function useActivityPane(visible: PnlChartPoint[], spanMs: number): ActivityPane { + // The measurement comes from the pane's own ResponsiveContainer, which is + // already observing its size, rather than from a second observer of ours. It + // is 0 until the first callback — and stays 0 where there is no layout at all + // — which `volumeBarWidth` answers with `undefined`, i.e. "leave it to + // recharts". + const [activityWidth, setActivityWidth] = useState(0); + const onActivityResize = useCallback((width: number) => setActivityWidth(width), []); + + const bucketMs = useMemo(() => chartBucketMs(visible), [visible]); + // The bucket has to be named in the tooltip: "Volume" used to be a running + // total, which needs no qualifier, and is now one bucket's worth, which means + // nothing until you know how long a bucket is. + const bucketLabel = useMemo(() => formatBucketLabel(bucketMs), [bucketMs]); + + // The bars are sized by us, not by recharts. On a numeric X axis recharts + // takes a bar's width from the *smallest* gap between two adjacent points and + // clamps any explicit `barSize` back under it — and this series always has + // one gap far smaller than the rest, because the fold ends it with a live + // "now" point a fraction of a bucket after the last snapshot. Left alone, + // every bar in the pane would be drawn at that fraction, thinning to a + // hairline and thickening again with each snapshot that lands. See + // `volumeBarWidth` and `chartBucketMs`. + const barWidth = volumeBarWidth( + // The plot area, not the card: both gutters and the right margin are + // outside the time domain the bars are placed in. + activityWidth - 2 * AXIS_WIDTH - PANE_MARGIN_RIGHT, + spanMs, + bucketMs, + ); + // Centred on its instant rather than starting there (recharts' own + // convention on a numeric axis), so a bar sits under the synced cursor and + // the tooltip that reports it, in both panes. Everything cosmetic is + // forwarded from the ``, so a pane sets its own radius and opacity. + const volumeBar = useCallback( + (props: BarShapeProps) => { + const width = barWidth ?? props.width; + const x = props.x + props.width / 2 - width / 2; + return ( + + ); + }, + [barWidth], + ); + + // The position axis is pinned across zero rather than left to recharts, so + // the signed area always has its baseline on screen (READ-246). Memoised + // because recharts keeps the domain in its own store and a fresh array on + // every render would churn it. + const positionDomain = useMemo(() => positionAxisDomain(visible), [visible]); + // Measured against the area's own extent, not the padded domain: the fill's + // gradient is in objectBoundingBox units. See zeroGradientOffset. + const positionZeroOffset = useMemo( + () => zeroGradientOffset(positionAreaExtent(visible)), + [visible], + ); + + return { + onActivityResize, + bucketMs, + bucketLabel, + volumeBar, + positionDomain, + positionZeroOffset, + }; +} From ca2c0ea96a46ea8eb468822d37e7778635b26dc5 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 18:43:23 +0300 Subject: [PATCH 020/154] Merge the panels' saved defaults in one place, not five MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each executor config module carried its own copy of the same eleven lines: read the key, parse, shallow-copy the defaults constant, copy the whitelisted keys that are present and not undefined, catch back to the defaults. Five copies is what let CORR-308 — a loaded object whose nested array was the exported constant's own, so resizing it rewrote the defaults — be fixed in the DCA copy alone and stay latent in the other four. loadPersistedDefaults / savePersistedDefaults now hold the merge, and the copy they hand back is structured rather than shallow, so no caller can reach the constant it was made from however deep it writes. Order, position and LP become one-liners; DCA keeps its prices reconciliation and the grid its last-market overlay as post-steps on the returned object. The exported names are unchanged, so every importer and the existing tests are untouched. --- .../src/components/executor/dca-config.ts | 33 ++--- frontend/src/components/executor/lp-config.ts | 20 +-- .../src/components/executor/order-config.ts | 20 +-- .../executor/persisted-defaults.test.ts | 115 ++++++++++++++++++ .../components/executor/persisted-defaults.ts | 60 +++++++++ .../components/executor/position-config.ts | 20 +-- frontend/src/lib/gridExecutor.ts | 40 ++---- 7 files changed, 206 insertions(+), 102 deletions(-) create mode 100644 frontend/src/components/executor/persisted-defaults.test.ts create mode 100644 frontend/src/components/executor/persisted-defaults.ts diff --git a/frontend/src/components/executor/dca-config.ts b/frontend/src/components/executor/dca-config.ts index 1a62b3d0b..113868922 100644 --- a/frontend/src/components/executor/dca-config.ts +++ b/frontend/src/components/executor/dca-config.ts @@ -13,6 +13,7 @@ import { barrierPct, barrierPrice, } from "./barriers"; +import { loadPersistedDefaults, savePersistedDefaults } from "./persisted-defaults"; import { getThemeColors } from "@/lib/theme-colors"; import { DCA_DEFAULTS_KEY } from "@/lib/sessionState"; @@ -73,32 +74,18 @@ const PERSISTED_FIELDS: (keyof DCAState)[] = [ ]; export function loadSavedDefaults(): DCAState { - try { - const raw = localStorage.getItem(STORAGE_KEY); - if (!raw) return DCA_DEFAULTS; - const saved = JSON.parse(raw); - // `prices` is not persisted, so the shallow copy would hand back the - // constant's own array — and the resize below would push/pop DCA_DEFAULTS - // itself, leaving every later load with more prices than amounts. - const merged = { ...DCA_DEFAULTS, prices: [...DCA_DEFAULTS.prices] }; - for (const key of PERSISTED_FIELDS) { - if (key in saved && saved[key] !== undefined) { - (merged as Record)[key] = saved[key]; - } - } - // Ensure prices array matches amounts length - while (merged.prices.length < merged.amounts_quote.length) merged.prices.push(0); - while (merged.prices.length > merged.amounts_quote.length) merged.prices.pop(); - return merged; - } catch { - return DCA_DEFAULTS; - } + const merged = loadPersistedDefaults(STORAGE_KEY, DCA_DEFAULTS, PERSISTED_FIELDS); + // `prices` is not persisted — it is per-trade — so it arrives at the default + // length while `amounts_quote` may not have. The two are read pairwise by + // index, so reconcile them here; the loader's copy is structured, so this + // resizes the loaded object's own array and never DCA_DEFAULTS' (CORR-308). + while (merged.prices.length < merged.amounts_quote.length) merged.prices.push(0); + while (merged.prices.length > merged.amounts_quote.length) merged.prices.pop(); + return merged; } function saveDefaults(state: DCAState) { - const toSave: Record = {}; - for (const key of PERSISTED_FIELDS) toSave[key] = state[key]; - localStorage.setItem(STORAGE_KEY, JSON.stringify(toSave)); + savePersistedDefaults(STORAGE_KEY, state, PERSISTED_FIELDS); } /** diff --git a/frontend/src/components/executor/lp-config.ts b/frontend/src/components/executor/lp-config.ts index 0537102b7..b386dc9b2 100644 --- a/frontend/src/components/executor/lp-config.ts +++ b/frontend/src/components/executor/lp-config.ts @@ -2,6 +2,7 @@ import { useQuery } from "@tanstack/react-query"; import { useEffect, useMemo, useReducer } from "react"; import type { ChartPriceMapping, ExecutorValidation, PickSlot } from "./types"; +import { loadPersistedDefaults, savePersistedDefaults } from "./persisted-defaults"; import { api, type DexPoolInfo } from "@/lib/api"; import { getThemeColors } from "@/lib/theme-colors"; import { LP_DEFAULTS_KEY } from "@/lib/sessionState"; @@ -95,26 +96,11 @@ const PERSISTED_FIELDS: (keyof LPState)[] = [ ]; function loadSavedDefaults(): LPState { - try { - const raw = localStorage.getItem(STORAGE_KEY); - if (!raw) return DEFAULTS; - const saved = JSON.parse(raw); - const merged = { ...DEFAULTS }; - for (const key of PERSISTED_FIELDS) { - if (key in saved && saved[key] !== undefined) { - (merged as Record)[key] = saved[key]; - } - } - return merged; - } catch { - return DEFAULTS; - } + return loadPersistedDefaults(STORAGE_KEY, DEFAULTS, PERSISTED_FIELDS); } function saveDefaults(state: LPState) { - const toSave: Record = {}; - for (const key of PERSISTED_FIELDS) toSave[key] = state[key]; - localStorage.setItem(STORAGE_KEY, JSON.stringify(toSave)); + savePersistedDefaults(STORAGE_KEY, state, PERSISTED_FIELDS); } /** diff --git a/frontend/src/components/executor/order-config.ts b/frontend/src/components/executor/order-config.ts index 976f1b939..baaa3899e 100644 --- a/frontend/src/components/executor/order-config.ts +++ b/frontend/src/components/executor/order-config.ts @@ -6,6 +6,7 @@ import { useMemo, useReducer } from "react"; import type { ChartPriceMapping, ExecutorValidation, PickSlot } from "./types"; +import { loadPersistedDefaults, savePersistedDefaults } from "./persisted-defaults"; import { ORDER_DEFAULTS_KEY } from "@/lib/sessionState"; // ── State ── @@ -51,26 +52,11 @@ const PERSISTED_FIELDS: (keyof OrderState)[] = [ ]; function loadSavedDefaults(): OrderState { - try { - const raw = localStorage.getItem(STORAGE_KEY); - if (!raw) return ORDER_DEFAULTS; - const saved = JSON.parse(raw); - const merged = { ...ORDER_DEFAULTS }; - for (const key of PERSISTED_FIELDS) { - if (key in saved && saved[key] !== undefined) { - (merged as Record)[key] = saved[key]; - } - } - return merged; - } catch { - return ORDER_DEFAULTS; - } + return loadPersistedDefaults(STORAGE_KEY, ORDER_DEFAULTS, PERSISTED_FIELDS); } function saveDefaults(state: OrderState) { - const toSave: Record = {}; - for (const key of PERSISTED_FIELDS) toSave[key] = state[key]; - localStorage.setItem(STORAGE_KEY, JSON.stringify(toSave)); + savePersistedDefaults(STORAGE_KEY, state, PERSISTED_FIELDS); } export function orderReducer(state: OrderState, action: OrderAction): OrderState { diff --git a/frontend/src/components/executor/persisted-defaults.test.ts b/frontend/src/components/executor/persisted-defaults.test.ts new file mode 100644 index 000000000..3a972c5bb --- /dev/null +++ b/frontend/src/components/executor/persisted-defaults.test.ts @@ -0,0 +1,115 @@ +/** + * @vitest-environment jsdom + * + * The one merge every executor panel's defaults come back through (ARCH-346). + * + * What it has to get right is what the five copies of it each had to: only the + * whitelisted fields are restored, a missing or unusable payload is not an + * error, and the object handed back shares nothing with the exported constant + * it was made from — which is the property CORR-308 was the absence of, and the + * one a caller cannot check for itself. + */ + +import { beforeEach, describe, expect, it } from "vitest"; + +import { loadPersistedDefaults, savePersistedDefaults } from "./persisted-defaults"; + +const KEY = "condor_test_defaults"; + +interface TestState { + side: number; + amount: number; + levels: number[]; + activePickField: string | null; +} + +const DEFAULTS: TestState = { + side: 1, + amount: 0, + levels: [0, 0, 0], + activePickField: null, +}; + +const FIELDS: (keyof TestState)[] = ["side", "amount"]; + +describe("loadPersistedDefaults", () => { + beforeEach(() => { + localStorage.clear(); + }); + + it("restores the whitelisted fields and ignores every other stored key", () => { + localStorage.setItem( + KEY, + JSON.stringify({ side: 2, amount: 500, activePickField: "price", nonsense: 7 }), + ); + + const loaded = loadPersistedDefaults(KEY, DEFAULTS, FIELDS); + + expect(loaded.side).toBe(2); + expect(loaded.amount).toBe(500); + // Not on the whitelist: the panel's own transient state never comes back. + expect(loaded.activePickField).toBeNull(); + expect(loaded).not.toHaveProperty("nonsense"); + }); + + it("falls back to the default for a whitelisted key stored as undefined", () => { + localStorage.setItem(KEY, JSON.stringify({ side: 2, amount: undefined })); + + const loaded = loadPersistedDefaults(KEY, DEFAULTS, FIELDS); + + expect(loaded.side).toBe(2); + expect(loaded.amount).toBe(DEFAULTS.amount); + }); + + it("returns the defaults when nothing is stored", () => { + expect(loadPersistedDefaults(KEY, DEFAULTS, FIELDS)).toEqual(DEFAULTS); + }); + + it("returns the defaults for a corrupt blob", () => { + localStorage.setItem(KEY, "{not json"); + + expect(loadPersistedDefaults(KEY, DEFAULTS, FIELDS)).toEqual(DEFAULTS); + }); + + it("returns the defaults for a payload that is not an object", () => { + localStorage.setItem(KEY, "null"); + + expect(loadPersistedDefaults(KEY, DEFAULTS, FIELDS)).toEqual(DEFAULTS); + }); + + it("hands back a copy whose nested values do not alias the defaults", () => { + localStorage.setItem(KEY, JSON.stringify({ side: 2 })); + + const loaded = loadPersistedDefaults(KEY, DEFAULTS, FIELDS); + loaded.levels.push(99); + + expect(DEFAULTS.levels).toEqual([0, 0, 0]); + }); + + it("copies the nested values on the empty and the corrupt paths too", () => { + loadPersistedDefaults(KEY, DEFAULTS, FIELDS).levels.push(1); + localStorage.setItem(KEY, "{not json"); + loadPersistedDefaults(KEY, DEFAULTS, FIELDS).levels.push(2); + + expect(DEFAULTS.levels).toEqual([0, 0, 0]); + }); +}); + +describe("savePersistedDefaults", () => { + beforeEach(() => { + localStorage.clear(); + }); + + it("writes only the whitelisted fields, and round-trips them", () => { + const state: TestState = { side: 2, amount: 250, levels: [1, 2], activePickField: "price" }; + + savePersistedDefaults(KEY, state, FIELDS); + + expect(JSON.parse(localStorage.getItem(KEY)!)).toEqual({ side: 2, amount: 250 }); + expect(loadPersistedDefaults(KEY, DEFAULTS, FIELDS)).toEqual({ + ...DEFAULTS, + side: 2, + amount: 250, + }); + }); +}); diff --git a/frontend/src/components/executor/persisted-defaults.ts b/frontend/src/components/executor/persisted-defaults.ts new file mode 100644 index 000000000..bfc90f03e --- /dev/null +++ b/frontend/src/components/executor/persisted-defaults.ts @@ -0,0 +1,60 @@ +// ── Panel defaults that outlive the session ── +// +// Every executor config module remembers the same thing across page loads: the +// handful of its state's fields that describe how this user trades — side, +// size, leverage, the barriers they habitually set. The rest of the state +// belongs to the panel that is open (a picked field, a resolved pool, a price, +// the anchoring flag), so what is restored is a *whitelist*, never the blob. +// +// That merge was written out five times over — order, position, DCA and LP +// panels plus the grid — and being five copies is what let CORR-308 (a loaded +// object whose nested array was the exported constant's own, so resizing it +// rewrote the defaults) be fixed in one of them and stay broken in four. Here +// it is once, and the copy it hands back is structured rather than shallow, so +// no caller can reach the constant it was made from however deep it writes. + +/** + * The `defaults`, with the whitelisted `fields` of a previous session merged + * over them, read from `storageKey`. + * + * Anything unreadable — storage denied, a corrupt blob, a payload that is not + * an object — yields the defaults untouched: a remembered preference is a + * convenience, and there is no state of storage worth failing a panel's first + * render over. A key absent from the payload, or present but `undefined`, keeps + * the default too, so a field added after the last save arrives with its new + * default rather than as a hole. + */ +export function loadPersistedDefaults( + storageKey: string, + defaults: T, + fields: readonly (keyof T)[], +): T { + try { + const merged = structuredClone(defaults); + const raw = localStorage.getItem(storageKey); + if (!raw) return merged; + const saved = JSON.parse(raw) as Record; + for (const key of fields) { + if (key in saved && saved[key as string] !== undefined) { + (merged as Record)[key as string] = saved[key as string]; + } + } + return merged; + } catch { + // A half-merged object is not the defaults, so this re-copies rather than + // returning whatever the loop had got to when the payload turned out to be + // something `in` could not be asked about. + return structuredClone(defaults); + } +} + +/** Write just the whitelisted `fields` of `state` to `storageKey`. */ +export function savePersistedDefaults( + storageKey: string, + state: T, + fields: readonly (keyof T)[], +) { + const toSave: Record = {}; + for (const key of fields) toSave[key as string] = state[key]; + localStorage.setItem(storageKey, JSON.stringify(toSave)); +} diff --git a/frontend/src/components/executor/position-config.ts b/frontend/src/components/executor/position-config.ts index d20e5019b..8811a64d1 100644 --- a/frontend/src/components/executor/position-config.ts +++ b/frontend/src/components/executor/position-config.ts @@ -14,6 +14,7 @@ import { barrierPct, barrierPrice, } from "./barriers"; +import { loadPersistedDefaults, savePersistedDefaults } from "./persisted-defaults"; import { getThemeColors } from "@/lib/theme-colors"; import { POSITION_DEFAULTS_KEY } from "@/lib/sessionState"; @@ -83,26 +84,11 @@ const PERSISTED_FIELDS: (keyof PositionState)[] = [ ]; function loadSavedDefaults(): PositionState { - try { - const raw = localStorage.getItem(STORAGE_KEY); - if (!raw) return POSITION_DEFAULTS; - const saved = JSON.parse(raw); - const merged = { ...POSITION_DEFAULTS }; - for (const key of PERSISTED_FIELDS) { - if (key in saved && saved[key] !== undefined) { - (merged as Record)[key] = saved[key]; - } - } - return merged; - } catch { - return POSITION_DEFAULTS; - } + return loadPersistedDefaults(STORAGE_KEY, POSITION_DEFAULTS, PERSISTED_FIELDS); } function saveDefaults(state: PositionState) { - const toSave: Record = {}; - for (const key of PERSISTED_FIELDS) toSave[key] = state[key]; - localStorage.setItem(STORAGE_KEY, JSON.stringify(toSave)); + savePersistedDefaults(STORAGE_KEY, state, PERSISTED_FIELDS); } export function positionReducer(state: PositionState, action: PositionAction): PositionState { diff --git a/frontend/src/lib/gridExecutor.ts b/frontend/src/lib/gridExecutor.ts index c104cf5ed..9a4f5a7f0 100644 --- a/frontend/src/lib/gridExecutor.ts +++ b/frontend/src/lib/gridExecutor.ts @@ -1,6 +1,7 @@ // ── Grid executor state machine (shared by CreateExecutor, GridConfigPanel and DexPool) ── import { roundToPricePrecision } from "@/lib/formatters"; +import { loadPersistedDefaults, savePersistedDefaults } from "@/components/executor/persisted-defaults"; import { GRID_STORAGE_KEY, LAST_MARKET_KEY } from "@/lib/sessionState"; export interface GridState { @@ -91,31 +92,18 @@ export const GRID_PERSISTED_FIELDS: (keyof GridState)[] = [ * CreateExecutor page so the connector/pair persists across executor types. */ export function loadGridDefaults(applyLastMarket = false): GridState { - try { - const raw = localStorage.getItem(GRID_STORAGE_KEY); - const merged = { ...GRID_DEFAULTS }; - if (raw) { - const saved = JSON.parse(raw); - for (const key of GRID_PERSISTED_FIELDS) { - if (key in saved && saved[key] !== undefined) { - (merged as Record)[key] = saved[key]; - } + const merged = loadPersistedDefaults(GRID_STORAGE_KEY, GRID_DEFAULTS, GRID_PERSISTED_FIELDS); + if (applyLastMarket) { + try { + const market = localStorage.getItem(LAST_MARKET_KEY); + if (market) { + const { connector, pair } = JSON.parse(market); + if (connector) merged.connector = connector; + if (pair) merged.pair = pair; } - } - if (applyLastMarket) { - try { - const market = localStorage.getItem(LAST_MARKET_KEY); - if (market) { - const { connector, pair } = JSON.parse(market); - if (connector) merged.connector = connector; - if (pair) merged.pair = pair; - } - } catch { /* ok */ } - } - return merged; - } catch { - return GRID_DEFAULTS; + } catch { /* ok */ } } + return merged; } /** @@ -139,11 +127,7 @@ export function hasRememberedMarket(): boolean { } export function saveGridDefaults(state: GridState) { - const toSave: Record = {}; - for (const key of GRID_PERSISTED_FIELDS) { - toSave[key] = state[key]; - } - localStorage.setItem(GRID_STORAGE_KEY, JSON.stringify(toSave)); + savePersistedDefaults(GRID_STORAGE_KEY, state, GRID_PERSISTED_FIELDS); } export function isSpotConnector(connector: string): boolean { From 81fbc6c321e008cf13da2f48b1815937f5590b53 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 18:53:22 +0300 Subject: [PATCH 021/154] Give the grid the same config hook the other four executors have Order, Position, DCA and LP each hand CreateExecutor one object -- {state, dispatch, validation, chartProps, buildPayload, save, handleChartPriceSet} -- and the grid handed it nothing. The page held the reducer itself, so all five of its switches over ExecutorType had four one-line arms and one hand-written block: a separately declared useGridValidation, an inline chart mapping, an inline isChartLineSlot + clampGridPrice dispatch, the only executor payload in the app not built by a buildPayload, and a bare saveGridDefaults call. Every executor-wide concern had to be implemented twice. useGridConfig now owns all of it, and each grid arm reads like its four siblings. The grid's state still carries the page's market (connector, pair, interval, lookback), so state and dispatch stay destructured under their own names -- that asymmetry is real and stays. handleChartPriceSet takes a third argument the others do not: the tick precision it clamps onto is read off trading rules queried for the market this very state holds, so it cannot be a hook argument and comes back from the page instead. useGridValidation moves here from GridConfigPanel, where nothing used it, next to its siblings' own validation hooks -- which also retires that panel's baselined react-refresh error. --- frontend/eslint-baseline.json | 1 - .../components/executor/grid-config.test.tsx | 157 ++++++++++++++++++ .../src/components/executor/grid-config.ts | 124 ++++++++++++++ .../src/components/grid/GridConfigPanel.tsx | 7 - frontend/src/pages/CreateExecutor.tsx | 95 +++-------- 5 files changed, 307 insertions(+), 77 deletions(-) create mode 100644 frontend/src/components/executor/grid-config.test.tsx create mode 100644 frontend/src/components/executor/grid-config.ts diff --git a/frontend/eslint-baseline.json b/frontend/eslint-baseline.json index 50afd4a46..fd44631f5 100644 --- a/frontend/eslint-baseline.json +++ b/frontend/eslint-baseline.json @@ -9,7 +9,6 @@ "src/components/executor/DCAConfigPanel.tsx | @typescript-eslint/no-unused-vars": 1, "src/components/executor/fields.tsx | react-hooks/set-state-in-effect": 3, "src/components/grid/GridConfigPanel.tsx | react-hooks/set-state-in-effect": 2, - "src/components/grid/GridConfigPanel.tsx | react-refresh/only-export-components": 1, "src/components/market/OrderBook.tsx | react-hooks/immutability": 1, "src/components/market/PriceTicker.tsx | react-hooks/refs": 3, "src/components/market/RecentTrades.tsx | react-hooks/set-state-in-effect": 1, diff --git a/frontend/src/components/executor/grid-config.test.tsx b/frontend/src/components/executor/grid-config.test.tsx new file mode 100644 index 000000000..43497ce96 --- /dev/null +++ b/frontend/src/components/executor/grid-config.test.tsx @@ -0,0 +1,157 @@ +/** + * `useGridConfig().buildPayload` emits exactly what CreateExecutor used to build + * by hand (ARCH-347). + * + * The grid was the one executor whose `grid_executor` payload was written out in + * the page's mutation rather than by a `buildPayload`, so moving it into the hook + * is a refactor with a wire format on the other side of it: these cases pin the + * key set, the key *order*, the nested `triple_barrier_config`, and the spot + * override that forces `leverage: 1`. + * + * Needs a DOM to run a hook, so this file overrides vitest's default `node` + * environment. + * + * @vitest-environment jsdom + */ + +import { act, useEffect } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { useGridConfig } from "./grid-config"; + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +type GridConfig = ReturnType; + +let container: HTMLDivElement; +let root: Root; +let latest: GridConfig; + +function Probe() { + const config = useGridConfig(); + // Published from an effect rather than assigned during render: the render of + // a component is not the place for a side effect, and `act` flushes effects + // before it returns either way. + useEffect(() => { + latest = config; + }); + return null; +} + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + localStorage.clear(); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + act(() => { + root.render(); + }); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + localStorage.clear(); +}); + +/** Apply one field edit through the reducer and settle the render. */ +function setField(field: string, value: unknown) { + act(() => { + latest.dispatch({ type: "SET_FIELD", field, value }); + }); +} + +describe("useGridConfig().buildPayload", () => { + it("emits the grid_executor config CreateExecutor used to build inline", () => { + setField("side", 2); + setField("start_price", 100); + setField("end_price", 110); + setField("limit_price", 99); + setField("total_amount_quote", 500); + setField("leverage", 7); + + const payload = latest.buildPayload("binance_perpetual", "SOL-USDC", false); + const state = latest.state; + + expect(payload.executor_type).toBe("grid_executor"); + // Byte-equal, key order included — this is what goes on the wire. + expect(JSON.stringify(payload.config)).toBe( + JSON.stringify({ + connector_name: "binance_perpetual", + trading_pair: "SOL-USDC", + side: 2, + start_price: 100, + end_price: 110, + limit_price: 99, + total_amount_quote: 500, + min_order_amount_quote: state.min_order_amount_quote, + min_spread_between_orders: state.min_spread_between_orders, + max_open_orders: state.max_open_orders, + max_orders_per_batch: state.max_orders_per_batch, + order_frequency: state.order_frequency, + leverage: 7, + activation_bounds: state.activation_bounds, + keep_position: state.keep_position, + coerce_tp_to_step: state.coerce_tp_to_step, + triple_barrier_config: { + take_profit: state.take_profit, + open_order_type: state.open_order_type, + take_profit_order_type: state.take_profit_order_type, + }, + }), + ); + }); + + it("forces leverage to 1 on a spot venue without touching the form's own value", () => { + setField("leverage", 7); + + const payload = latest.buildPayload("binance", "SOL-USDC", true); + + expect(payload.config.leverage).toBe(1); + // The state keeps what the user typed, so switching back to a perp venue + // does not silently lose their leverage. + expect(latest.state.leverage).toBe(7); + }); +}); + +describe("useGridConfig().handleChartPriceSet", () => { + it("writes the picked price into the slot's field and disarms the picker", () => { + setField("activePickField", "start"); + + act(() => { + latest.handleChartPriceSet("start", 123.45); + }); + + expect(latest.state.start_price).toBe(123.45); + expect(latest.state.activePickField).toBeNull(); + }); + + it("clamps the picked price against the prices already set", () => { + setField("end_price", 110); + + act(() => { + latest.handleChartPriceSet("start", 120); + }); + + // A lower bound above the upper one is not a range; the clamp lands it just + // below rather than letting the form go red under the click. + expect(latest.state.start_price).toBeLessThan(110); + }); + + it("ignores a slot the grid does not own", () => { + setField("activePickField", "start"); + + act(() => { + latest.handleChartPriceSet("take_profit", 500); + }); + + // Not the grid's line: nothing written, and the grid's own armed picker is + // left alone for the panel that does own the slot. + expect(latest.state.start_price).toBe(0); + expect(latest.state.activePickField).toBe("start"); + }); +}); diff --git a/frontend/src/components/executor/grid-config.ts b/frontend/src/components/executor/grid-config.ts new file mode 100644 index 000000000..6248b9e7d --- /dev/null +++ b/frontend/src/components/executor/grid-config.ts @@ -0,0 +1,124 @@ +// ── Grid executor config hook ── +// +// The fifth of the five. Order, Position, DCA and LP each hand the page one +// object — `{ state, dispatch, validation, chartProps, buildPayload, save, +// handleChartPriceSet }` — and the grid used to hand it nothing: the page held +// the reducer itself and wrote out, by hand, the validation it read, the chart +// mapping it drew, the clamp it applied on a chart click, the `grid_executor` +// payload it posted and the defaults it saved. Five switches over +// `ExecutorType` therefore had four one-line arms and one long one, and every +// executor-wide concern had to be implemented twice. +// +// The state machine itself stays in `lib/gridExecutor`, which `GridConfigPanel` +// and the DEX pool page also read; this is only the page-facing shape of it. + +import { useMemo, useReducer } from "react"; + +import type { ChartPriceMapping, ExecutorValidation, PickSlot } from "./types"; +import { isChartLineSlot } from "./types"; +import { + clampGridPrice, + gridConfigErrors, + gridLineLabels, + gridReducer, + loadGridDefaults, + saveGridDefaults, +} from "@/lib/gridExecutor"; +import type { GridState } from "@/lib/gridExecutor"; + +// ── Validation ── + +export function useGridValidation(state: GridState): ExecutorValidation { + return useMemo(() => { + const errors = gridConfigErrors(state); + return { valid: errors.length === 0, errors }; + }, [state]); +} + +// ── Hook ── + +export function useGridConfig() { + // `applyLastMarket`: the grid's connector/pair are the page's market, shared + // by every tab, so they come back from the last market rather than from the + // grid's own saved defaults. + const [state, dispatch] = useReducer(gridReducer, undefined, () => loadGridDefaults(true)); + const validation = useGridValidation(state); + + const chartProps: ChartPriceMapping = useMemo(() => ({ + startPrice: state.start_price, + endPrice: state.end_price, + limitPrice: state.limit_price, + side: state.side, + minSpread: state.min_spread_between_orders, + activePickField: state.activePickField, + lineLabels: gridLineLabels(state.side), + }), [ + state.start_price, + state.end_price, + state.limit_price, + state.side, + state.min_spread_between_orders, + state.activePickField, + ]); + + const buildPayload = (connector: string, pair: string, isSpot: boolean) => { + const config: Record = { + connector_name: connector, + trading_pair: pair, + side: state.side, + start_price: state.start_price, + end_price: state.end_price, + limit_price: state.limit_price, + total_amount_quote: state.total_amount_quote, + min_order_amount_quote: state.min_order_amount_quote, + min_spread_between_orders: state.min_spread_between_orders, + max_open_orders: state.max_open_orders, + max_orders_per_batch: state.max_orders_per_batch, + order_frequency: state.order_frequency, + leverage: isSpot ? 1 : state.leverage, + activation_bounds: state.activation_bounds, + keep_position: state.keep_position, + coerce_tp_to_step: state.coerce_tp_to_step, + triple_barrier_config: { + take_profit: state.take_profit, + open_order_type: state.open_order_type, + take_profit_order_type: state.take_profit_order_type, + }, + }; + + return { executor_type: "grid_executor" as const, config }; + }; + + const save = () => saveGridDefaults(state); + + /** + * Write a price the user picked off the chart into the field behind `field`. + * + * `pricePrecision` is the one thing this hook cannot hold itself: it is read + * off the venue's trading rules, which are queried for the connector and pair + * that live in *this* state — so the page computes it downstream of the hook + * and hands it back here. The other four panels clamp against nothing and + * take two arguments. + */ + const handleChartPriceSet = ( + field: PickSlot, + price: number, + pricePrecision?: number | null, + ) => { + // The grid owns exactly the chart's own three lines; any other slot belongs + // to a panel that draws its own and would name a grid field that does not + // exist. + if (!isChartLineSlot(field)) return; + // Bound the picked price against the two the user already set, so a click + // (and, later, a drag) cannot write a price the form will only reject + // afterwards. The chart stays ignorant of grid semantics. + dispatch({ + type: "SET_FIELD", + field: `${field}_price`, + value: clampGridPrice(field, price, state, pricePrecision), + }); + dispatch({ type: "SET_FIELD", field: "activePickField", value: null }); + }; + + return { state, dispatch, validation, chartProps, buildPayload, save, handleChartPriceSet }; +} diff --git a/frontend/src/components/grid/GridConfigPanel.tsx b/frontend/src/components/grid/GridConfigPanel.tsx index 3db37f3e8..ab0c03169 100644 --- a/frontend/src/components/grid/GridConfigPanel.tsx +++ b/frontend/src/components/grid/GridConfigPanel.tsx @@ -462,10 +462,3 @@ export function GridConfigPanel({ state, dispatch, currentPrice, isSpot = false, ); } - -export function useGridValidation(state: GridState) { - return useMemo(() => { - const errors = gridConfigErrors(state); - return { valid: errors.length === 0, errors }; - }, [state]); -} diff --git a/frontend/src/pages/CreateExecutor.tsx b/frontend/src/pages/CreateExecutor.tsx index 0b0d09619..9a9f1219b 100644 --- a/frontend/src/pages/CreateExecutor.tsx +++ b/frontend/src/pages/CreateExecutor.tsx @@ -23,7 +23,7 @@ import { MarketBrowser, type MarketPick } from "@/components/market/MarketBrowse import { FavoritesStrip } from "@/components/market/FavoritesStrip"; import { StarMarketButton } from "@/components/market/StarMarketButton"; import { TradeChart } from "@/components/trade/TradeChart"; -import { GridConfigPanel, useGridValidation } from "@/components/grid/GridConfigPanel"; +import { GridConfigPanel } from "@/components/grid/GridConfigPanel"; import { ErrorToast, ExecutorSuccessModal, @@ -36,6 +36,7 @@ import { DCAConfigPanel } from "@/components/executor/DCAConfigPanel"; import { useDCAConfig } from "@/components/executor/dca-config"; import { LPConfigPanel } from "@/components/executor/LPConfigPanel"; import { LP_SIDE_RANGE, useLpConfig } from "@/components/executor/lp-config"; +import { useGridConfig } from "@/components/executor/grid-config"; import { HintBubble } from "@/components/ui/HintBubble"; import { useOneTimeHint } from "@/hooks/useOneTimeHint"; import { TradeBottomPane } from "@/components/trade/TradeBottomPane"; @@ -51,16 +52,10 @@ import { candleStore } from "@/lib/candle-store"; import { connectorCapabilities, orderBookVenues } from "@/lib/connector-capabilities"; import { executorsQuery } from "@/lib/queryClient"; import { BROWSE_HINT_KEY } from "@/lib/sessionState"; -import { isChartLineSlot } from "@/components/executor/types"; import type { ChartPriceMapping, ExecutorType, PickSlot } from "@/components/executor/types"; import { - clampGridPrice, - gridLineLabels, - gridReducer, hasRememberedMarket, isSpotConnector, - loadGridDefaults, - saveGridDefaults, LAST_MARKET_KEY, INTERVALS, LOOKBACK_OPTIONS, @@ -105,11 +100,13 @@ export function CreateExecutor() { setSearchParams({ type }, { replace: true }); }; - // ── Grid state (always initialized for hooks rules) ── - const [gridState, gridDispatch] = React.useReducer(gridReducer, undefined, () => loadGridDefaults(true)); - const gridValidation = useGridValidation(gridState); - - // ── Other executor configs ── + // ── Executor configs (all initialized for hooks rules) ── + // + // The grid's is destructured as well: its state carries the page's market + // (connector, pair, interval, lookback), which every tab reads and several + // effects dispatch into, so those two stay in scope under their own names. + const gridConfig = useGridConfig(); + const { state: gridState, dispatch: gridDispatch } = gridConfig; const positionConfig = usePositionConfig(); const orderConfig = useOrderConfig(); const dcaConfig = useDCAConfig(); @@ -259,7 +256,10 @@ export function CreateExecutor() { if (!server) return; const newLookback = Math.ceil(Date.now() / 1000 - startTime) + 3600; // +1h padding gridDispatch({ type: "SET_FIELD", field: "lookbackSeconds", value: newLookback }); - }, [server]); + // `gridDispatch` is the reducer's own dispatch, stable for the page's life; + // it is named because it now arrives through `gridConfig` rather than + // straight out of a `useReducer` the compiler can see. + }, [server, gridDispatch]); // Persist last-used connector/pair to localStorage. The executor selection is // not cleared here -- it expires on its own, see `selection` above. @@ -376,7 +376,7 @@ export function CreateExecutor() { gridDispatch({ type: "SET_PAIR", value: market.pair }); setBrowserOpen(false); }, - [connector], + [connector, gridDispatch], ); // A shortcut nobody can see. The chip used to carry a bare `/`, @@ -414,13 +414,13 @@ export function CreateExecutor() { // ── Active config derived values ── const activeValidation = useMemo(() => { switch (executorType) { - case "grid": return gridValidation; + case "grid": return gridConfig.validation; case "position": return positionConfig.validation; case "order": return orderConfig.validation; case "dca": return dcaConfig.validation; case "lp": return lpConfig.validation; } - }, [executorType, gridValidation, positionConfig.validation, orderConfig.validation, dcaConfig.validation, lpConfig.validation]); + }, [executorType, gridConfig.validation, positionConfig.validation, orderConfig.validation, dcaConfig.validation, lpConfig.validation]); // What the form currently says, for the chat bubble (FEAT-060/FEAT-072). // @@ -502,22 +502,13 @@ export function CreateExecutor() { // Chart props depend on active type const chartProps = useMemo((): ChartPriceMapping => { switch (executorType) { - case "grid": - return { - startPrice: gridState.start_price, - endPrice: gridState.end_price, - limitPrice: gridState.limit_price, - side: gridState.side, - minSpread: gridState.min_spread_between_orders, - activePickField: gridState.activePickField, - lineLabels: gridLineLabels(gridState.side), - }; + case "grid": return gridConfig.chartProps; case "position": return positionConfig.chartProps; case "order": return orderConfig.chartProps; case "dca": return dcaConfig.chartProps; case "lp": return lpConfig.chartProps; } - }, [executorType, gridState, positionConfig.chartProps, orderConfig.chartProps, dcaConfig.chartProps, lpConfig.chartProps]); + }, [executorType, gridConfig.chartProps, positionConfig.chartProps, orderConfig.chartProps, dcaConfig.chartProps, lpConfig.chartProps]); // Chart price set handler. // @@ -527,9 +518,9 @@ export function CreateExecutor() { // bought that stability with a stale closure -- the callback kept calling // whichever `handleChartPriceSet` existed when the type last changed, not the // current one. A latest-value ref gives the stability without the staleness. - const priceSetTargets = useRef({ positionConfig, orderConfig, dcaConfig, lpConfig, gridState, pricePrecision }); + const priceSetTargets = useRef({ gridConfig, positionConfig, orderConfig, dcaConfig, lpConfig, pricePrecision }); useEffect(() => { - priceSetTargets.current = { positionConfig, orderConfig, dcaConfig, lpConfig, gridState, pricePrecision }; + priceSetTargets.current = { gridConfig, positionConfig, orderConfig, dcaConfig, lpConfig, pricePrecision }; }); const handlePriceSet = useCallback( @@ -537,19 +528,10 @@ export function CreateExecutor() { const targets = priceSetTargets.current; switch (executorType) { case "grid": - // The grid owns exactly the chart's own three lines; any other slot - // belongs to a panel that draws its own and would name a grid field - // that does not exist. - if (!isChartLineSlot(field)) break; - // Bound the picked price against the two the user already set, so a - // click (and, later, a drag) cannot write a price the form will only - // reject afterwards. The chart stays ignorant of grid semantics. - gridDispatch({ - type: "SET_FIELD", - field: `${field}_price`, - value: clampGridPrice(field, price, targets.gridState, targets.pricePrecision), - }); - gridDispatch({ type: "SET_FIELD", field: "activePickField", value: null }); + // The only arm that takes a third argument: the grid clamps the picked + // price onto the venue's tick grid, and the precision is read off + // trading rules queried for the market this very state holds. + targets.gridConfig.handleChartPriceSet(field, price, targets.pricePrecision); break; case "position": targets.positionConfig.handleChartPriceSet(field, price); @@ -582,32 +564,7 @@ export function CreateExecutor() { switch (executorType) { case "grid": - payload = { - executor_type: "grid_executor", - config: { - connector_name: connector, - trading_pair: pair, - side: gridState.side, - start_price: gridState.start_price, - end_price: gridState.end_price, - limit_price: gridState.limit_price, - total_amount_quote: gridState.total_amount_quote, - min_order_amount_quote: gridState.min_order_amount_quote, - min_spread_between_orders: gridState.min_spread_between_orders, - max_open_orders: gridState.max_open_orders, - max_orders_per_batch: gridState.max_orders_per_batch, - order_frequency: gridState.order_frequency, - leverage: isSpot ? 1 : gridState.leverage, - activation_bounds: gridState.activation_bounds, - keep_position: gridState.keep_position, - coerce_tp_to_step: gridState.coerce_tp_to_step, - triple_barrier_config: { - take_profit: gridState.take_profit, - open_order_type: gridState.open_order_type, - take_profit_order_type: gridState.take_profit_order_type, - }, - }, - }; + payload = gridConfig.buildPayload(connector, pair, isSpot); break; case "position": payload = positionConfig.buildPayload(connector, pair, isSpot); @@ -629,7 +586,7 @@ export function CreateExecutor() { onSuccess: (data) => { // Save defaults for the active type switch (executorType) { - case "grid": saveGridDefaults(gridState); break; + case "grid": gridConfig.save(); break; case "position": positionConfig.save(); break; case "order": orderConfig.save(); break; case "dca": dcaConfig.save(); break; From 4ca6fbad3ac91e86cb0bec3082aa87ba8de5929a Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 19:00:09 +0300 Subject: [PATCH 022/154] Drop the ACP event queue's None sentinel that nothing ever enqueues Every producer on _event_queue puts a concrete ACPEvent, so the `event is None` branch in prompt_stream invented a fourth way for a turn to end that a maintainer had to hunt down and disprove before touching the loop. The `| None` in the annotation is what made it look deliberate. The turn now ends on PromptDone, on the heartbeat timeout/liveness branch, or through the finally when the consumer walks away -- which is the contract _cancel_locally's docstring already states. --- condor/acp/client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/condor/acp/client.py b/condor/acp/client.py index a06ecb545..7e6376af9 100644 --- a/condor/acp/client.py +++ b/condor/acp/client.py @@ -488,7 +488,7 @@ def __init__( self.accepts_images = False self._read_task: asyncio.Task | None = None self._stderr_task: asyncio.Task | None = None - self._event_queue: asyncio.Queue[ACPEvent | None] = asyncio.Queue() + self._event_queue: asyncio.Queue[ACPEvent] = asyncio.Queue() self._current_req_id: int | None = None # tracks in-flight prompt request # A turn the agent has not settled and that nobody is streaming any # more: one that ignored ``session/cancel``, or one whose consumer @@ -1009,8 +1009,6 @@ def _on_response(fut: asyncio.Future) -> None: break yield Heartbeat(elapsed_seconds=elapsed) continue - if event is None: - break yield event if isinstance(event, PromptDone): break From d092bf2ad92afcc1f6317d0df22f41d094c88f70 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 19:06:24 +0300 Subject: [PATCH 023/154] Name specialist_slug, not agent_slug, in the condor tool docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven docstrings and comments in the memory, skills and routines tools said the assistant's store/library is selected by settings.agent_slug and called it empty for the chat, while every code path they document reads settings.specialist_slug. CHAT_SLUG is "condor", so for the chat seat agent_slug is the truthy string "condor" and the "empty -> chat condor" prose is true only of specialist_slug — which is exactly what that property's own docstring warns about ("Ask this, not agent_slug"). A developer copying these comments into a truthiness guard or an audit source would silently treat the chat as a specialist. Comments only; no code changed. delegate.py keeps settings.agent_slug: it compares a worker's own slug, which for a Condor worker really is "condor". --- mcp_servers/condor/tools/memory.py | 7 ++++--- mcp_servers/condor/tools/routines.py | 9 +++++---- mcp_servers/condor/tools/skills.py | 12 ++++++------ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/mcp_servers/condor/tools/memory.py b/mcp_servers/condor/tools/memory.py index 179877c6a..7a09c8c4c 100644 --- a/mcp_servers/condor/tools/memory.py +++ b/mcp_servers/condor/tools/memory.py @@ -1,8 +1,8 @@ """User memory tool — thin MCP wrapper over condor.memory.MemoryStore. Resolves the store by ``settings.user_id`` (already injected into the MCP -process) and derives the audit ``source`` from ``settings.agent_slug`` so the -LLM never has to report who is writing. +process) and derives the audit ``source`` from ``settings.specialist_slug`` so +the LLM never has to report who is writing. """ from condor.memory import MemoryStore @@ -14,7 +14,8 @@ def _source() -> str: def _store() -> MemoryStore: - # agent_slug selects this assistant's store (FEAT-003); empty -> chat condor. + # specialist_slug selects this assistant's store (FEAT-003); empty -> chat + # condor. return MemoryStore(settings.user_id, settings.specialist_slug or None) diff --git a/mcp_servers/condor/tools/routines.py b/mcp_servers/condor/tools/routines.py index c86042d95..3e42f3e5c 100644 --- a/mcp_servers/condor/tools/routines.py +++ b/mcp_servers/condor/tools/routines.py @@ -68,7 +68,7 @@ def _get_agent_routines_dir(target: str | None, shared: bool = False) -> Path | *owning agent's* dir (the strategy half is discarded — there is no per-strategy routines dir). Without a ``target``, the current assistant's own dir — the general library (root ``routines/``) for the chat, or the launched - Agent's (``.condor/agents//routines``, ``settings.agent_slug``). + Agent's (``.condor/agents//routines``, ``settings.specialist_slug``). ``shared=True`` targets the published library every assistant reads (:func:`condor.memory.paths.shared_routines_root`), and is honored **only** @@ -135,9 +135,10 @@ def _own_plus_shared(slug: str | None) -> dict: def _resolve_routine(name: str): """Look up a routine in the current assistant's scope. - A domain expert/trading agent (``settings.agent_slug`` set) resolves its own - routines plus the shared library, its own shadowing a shared name. The chat - ``condor`` resolves the general library (root ``routines/`` + shared). + A domain expert/trading agent (``settings.specialist_slug`` set) resolves + its own routines plus the shared library, its own shadowing a shared name. + The chat ``condor`` resolves the general library (root ``routines/`` + + shared). """ return _own_plus_shared(settings.specialist_slug).get(name) diff --git a/mcp_servers/condor/tools/skills.py b/mcp_servers/condor/tools/skills.py index 1ba4c82eb..a97f98dae 100644 --- a/mcp_servers/condor/tools/skills.py +++ b/mcp_servers/condor/tools/skills.py @@ -1,9 +1,9 @@ """Skill tool — thin MCP wrapper over condor.memory.SkillStore. Skills are general to the assistant (playbooks shared by everyone using it), not -per-user. The library is selected by ``settings.agent_slug`` (empty -> chat -condor) and is editable at runtime: read/search/list plus create/edit/delete. -Mirrors ``tools/memory.py``. +per-user. The library is selected by ``settings.specialist_slug`` (empty -> +chat condor) and is editable at runtime: read/search/list plus +create/edit/delete. Mirrors ``tools/memory.py``. """ from condor.memory import SkillStore @@ -15,12 +15,12 @@ def _resolve_agent_slug(target: str | None) -> tuple[str | None, bool]: Mirrors routines' ``_get_agent_routines_dir``. ``target`` lets the chat condor author/inspect a *specific* agent's local skills (the chat MCP has no - ``agent_slug`` of its own). A skill library is keyed by the **agent** alone, - so a bare agent slug is the canonical form; a composite strategy key + ``specialist_slug`` of its own). A skill library is keyed by the **agent** + alone, so a bare agent slug is the canonical form; a composite strategy key ``"agent_slug.strategy_slug"`` is still accepted and resolves to its *owning agent* (the strategy half is discarded — there is no per-strategy library). Without a ``target`` the current assistant is used — the launched agent - (``settings.agent_slug``) or the chat condor (``None``). + (``settings.specialist_slug``) or the chat condor (``None``). Returns ``(agent_slug, ok)``; ``ok`` is False only when a ``target`` was given but matched no agent or strategy, so the caller errors instead of From 375a5ff23d5c1c0dafcf51819be3068857037648 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 19:12:01 +0300 Subject: [PATCH 024/154] Drop the inert table spec format_orders_as_table never renders with format_orders_as_table declared an eight-column ColumnDef list and a TableBuilder, then hand-formatted every row anyway. The builder was never read again, and each ColumnDef was inert by construction: its key ("__time", "__pair", ...) matches no order field and its formatter returns "". A developer widening the amount column there would see the table unchanged, because the real widths are the f-string specs and the literal header spacing below. Delete the block and its now-unused import. The output is byte-identical, and the function now reads like format_positions_as_table directly below it. --- mcp_servers/hummingbot_api/formatters/trading.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/mcp_servers/hummingbot_api/formatters/trading.py b/mcp_servers/hummingbot_api/formatters/trading.py index ab0ddd685..f3803546c 100644 --- a/mcp_servers/hummingbot_api/formatters/trading.py +++ b/mcp_servers/hummingbot_api/formatters/trading.py @@ -8,7 +8,6 @@ from typing import Any from .base import format_number, get_field, get_timestamp_field -from .table_builder import ColumnDef, TableBuilder def format_orders_as_table(orders: list[dict[str, Any]]) -> str: @@ -57,19 +56,6 @@ def format_filled(item: dict) -> str: def format_status(item: dict) -> str: return str(get_field(item, "status", default="N/A"))[:8] - columns = [ - ColumnDef(name="time", key="__time", width=11, formatter=lambda _: ""), - ColumnDef(name="pair", key="__pair", width=13, formatter=lambda _: ""), - ColumnDef(name="side", key="__side", width=4, formatter=lambda _: ""), - ColumnDef(name="type", key="__type", width=6, formatter=lambda _: ""), - ColumnDef(name="amount", key="__amount", width=8, formatter=lambda _: ""), - ColumnDef(name="price", key="__price", width=8, formatter=lambda _: ""), - ColumnDef(name="filled", key="__filled", width=8, formatter=lambda _: ""), - ColumnDef(name="status", key="__status", width=8, formatter=lambda _: ""), - ] - - builder = TableBuilder(columns, empty_message="No orders found.") - # Build header header = "time | pair | side | type | amount | price | filled | status" separator = "-" * 120 From 6af776c0da4dc8f5be8ef59b9ef746b5dac9cbca Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 19:18:38 +0300 Subject: [PATCH 025/154] Stop advertising six formatter helpers nothing calls base.py's format_full_datetime, get_truncated, get_formatted_number, get_formatted_currency and get_formatted_percentage, plus table_builder.py's create_simple_table, are defined, doctested and re-exported in __all__ with no caller anywhere in the repo. __all__ is what a developer reads as the package's sanctioned API, so it was pointing at six helpers no live formatter uses while omitting format_amm_result and format_clmm_result, which the server actually calls. Delete the six, add the two missing names, and drop the unused dataclasses.field import. build_with_title survives: portfolio.py now calls it. Its conditional returned the same f-string from both arms, so it collapses to the one return. --- .../hummingbot_api/formatters/__init__.py | 13 +- mcp_servers/hummingbot_api/formatters/base.py | 126 ------------------ .../formatters/table_builder.py | 36 +---- 3 files changed, 4 insertions(+), 171 deletions(-) diff --git a/mcp_servers/hummingbot_api/formatters/__init__.py b/mcp_servers/hummingbot_api/formatters/__init__.py index 8453067a7..f906209c6 100644 --- a/mcp_servers/hummingbot_api/formatters/__init__.py +++ b/mcp_servers/hummingbot_api/formatters/__init__.py @@ -19,11 +19,7 @@ format_time_only, format_timestamp, get_field, - get_formatted_currency, - get_formatted_number, - get_formatted_percentage, get_timestamp_field, - get_truncated, truncate_address, truncate_string, ) @@ -64,7 +60,7 @@ from .portfolio import format_lp_positions_table, format_portfolio_as_table # Table builder for creating consistent tables -from .table_builder import ColumnDef, TableBuilder, create_simple_table +from .table_builder import ColumnDef, TableBuilder # Trading formatters from .trading import format_orders_as_table, format_positions_as_table @@ -75,6 +71,8 @@ "format_gateway_config_result", "format_gateway_swap_result", "format_gateway_clmm_pool_result", + "format_amm_result", + "format_clmm_result", # Base utilities "format_currency", "format_number", @@ -83,17 +81,12 @@ "format_time_only", "format_timestamp", "get_field", - "get_formatted_currency", - "get_formatted_number", - "get_formatted_percentage", "get_timestamp_field", - "get_truncated", "truncate_address", "truncate_string", # Table builder "ColumnDef", "TableBuilder", - "create_simple_table", # Trading formatters "format_orders_as_table", "format_positions_as_table", diff --git a/mcp_servers/hummingbot_api/formatters/base.py b/mcp_servers/hummingbot_api/formatters/base.py index 7a898033d..2795154d4 100644 --- a/mcp_servers/hummingbot_api/formatters/base.py +++ b/mcp_servers/hummingbot_api/formatters/base.py @@ -99,19 +99,6 @@ def format_time_only(ts: float) -> str: return format_timestamp(ts, "%H:%M:%S") -def format_full_datetime(ts: Any) -> str: - """ - Format a timestamp to full datetime (YYYY-MM-DD HH:MM:SS). - - Args: - ts: Unix timestamp or datetime string - - Returns: - Formatted datetime string - """ - return format_timestamp(ts, "%Y-%m-%d %H:%M:%S") - - def format_percentage(pct: Any, decimals: int = 2) -> str: """ Format a decimal percentage to percentage string. @@ -284,116 +271,3 @@ def get_timestamp_field(item: dict[str, Any], *keys: str) -> str: ts = get_field(item, *all_keys, default=0) return format_timestamp(ts) - - -def get_truncated( - item: dict[str, Any], key: str, max_len: int, default: str = "N/A" -) -> str: - """ - Get a string field and truncate it to a maximum length. - - Args: - item: Dictionary to extract value from - key: Key to extract - max_len: Maximum length for the result - default: Default value if key is not found (default: "N/A") - - Returns: - Truncated string value - - Examples: - >>> data = {"description": "This is a very long description"} - >>> get_truncated(data, "description", 10) # Returns "This is..." - """ - value = item.get(key) - if value is None: - return default[:max_len] if len(default) > max_len else default - - value_str = str(value) - return truncate_string(value_str, max_len) - - -def get_formatted_number( - item: dict[str, Any], - *keys: str, - decimals: int = 2, - compact: bool = True, - default: str = "N/A", -) -> str: - """ - Get a numeric field and format it. - - Args: - item: Dictionary to extract value from - *keys: One or more keys to try in order - decimals: Number of decimal places (default: 2) - compact: Use K/M notation for large numbers (default: True) - default: Default value if no key is found (default: "N/A") - - Returns: - Formatted number string - - Examples: - >>> data = {"amount": 1500.5, "volume": None} - >>> get_formatted_number(data, "amount", decimals=2) # Returns "1.50K" - >>> get_formatted_number(data, "volume", "amount") # Returns "1.50K" - """ - value = get_field(item, *keys, default=None) - if value is None: - return default - return format_number(value, decimals=decimals, compact=compact) - - -def get_formatted_currency( - item: dict[str, Any], - *keys: str, - symbol: str = "$", - decimals: int = 2, - default: str = "N/A", -) -> str: - """ - Get a numeric field and format it as currency. - - Args: - item: Dictionary to extract value from - *keys: One or more keys to try in order - symbol: Currency symbol (default: "$") - decimals: Number of decimal places (default: 2) - default: Default value if no key is found (default: "N/A") - - Returns: - Formatted currency string - - Examples: - >>> data = {"price": 1234.56} - >>> get_formatted_currency(data, "price") # Returns "$1,234.56" - """ - value = get_field(item, *keys, default=None) - if value is None: - return default - return format_currency(value, symbol=symbol, decimals=decimals) - - -def get_formatted_percentage( - item: dict[str, Any], *keys: str, decimals: int = 2, default: str = "N/A" -) -> str: - """ - Get a decimal field and format it as percentage. - - Args: - item: Dictionary to extract value from - *keys: One or more keys to try in order - decimals: Number of decimal places (default: 2) - default: Default value if no key is found (default: "N/A") - - Returns: - Formatted percentage string - - Examples: - >>> data = {"change_pct": 0.05} - >>> get_formatted_percentage(data, "change_pct") # Returns "5.00%" - """ - value = get_field(item, *keys, default=None) - if value is None: - return default - return format_percentage(value, decimals=decimals) diff --git a/mcp_servers/hummingbot_api/formatters/table_builder.py b/mcp_servers/hummingbot_api/formatters/table_builder.py index 493843048..fbab93a07 100644 --- a/mcp_servers/hummingbot_api/formatters/table_builder.py +++ b/mcp_servers/hummingbot_api/formatters/table_builder.py @@ -5,7 +5,7 @@ table creation, reducing code duplication across all formatters. """ -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any, Callable from .base import format_table_separator @@ -222,38 +222,4 @@ def build_with_title( Formatted table string with title """ table = self.build(data, empty_message) - if data: - return f"{title}\n\n{table}" return f"{title}\n\n{table}" - - -def create_simple_table( - data: list[dict[str, Any]], - column_config: list[tuple[str, str, int]], - empty_message: str = "No data found.", -) -> str: - """ - Convenience function to create a simple table without defining ColumnDef objects. - - Args: - data: List of dictionaries containing the data - column_config: List of tuples (name, key, width) for each column - empty_message: Message to return if data is empty - - Returns: - Formatted table string - - Example: - >>> data = [{"name": "Alice", "age": 30}, {"name": "Bob", "age": 25}] - >>> config = [("Name", "name", 10), ("Age", "age", 5)] - >>> print(create_simple_table(data, config)) - Name | Age - ------------------ - Alice | 30 - Bob | 25 - """ - columns = [ - ColumnDef(name=name, key=key, width=width) for name, key, width in column_config - ] - builder = TableBuilder(columns, empty_message=empty_message) - return builder.build(data) From 5b41984f2b63e84df16866477dc2dfb6579918d7 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 19:26:18 +0300 Subject: [PATCH 026/154] Delete the two dead v1 routine-execution endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /routines/servers/{server_name}/{routine_name}/run and .../schedule had no caller left in the product: the dashboard posts to /routines/run and /routines/schedule, the MCP runner to /routines/run and /routines/start, and the only references to the v1 path shape were the access tests written for it. The v2 pair exists because this path shape cannot address an agent routine (a name with a slash in it), so the two were never going to converge — and they had already drifted: only v2 carries agent, conversation_id, session_key and the CORR-287 on_complete bound. Keeping both meant reasoning about every future change to the run/schedule contract twice. Removes the routes with their RunRequest/ScheduleRequest models, and the require_server_access_by_server_name dependency they were the last users of. The SEC-045 gate is unaffected: the surviving endpoints take the server name in the request body and are pinned by the body-site tests. --- condor/web/auth.py | 8 ---- condor/web/routes/routines.py | 53 ---------------------- tests/test_routines_access.py | 52 ++------------------- tests/test_web_server_access_dependency.py | 25 +++------- 4 files changed, 11 insertions(+), 127 deletions(-) diff --git a/condor/web/auth.py b/condor/web/auth.py index 7666f9d5b..e1e9a2949 100644 --- a/condor/web/auth.py +++ b/condor/web/auth.py @@ -216,14 +216,6 @@ async def require_server_access( return user -async def require_server_access_by_server_name( - server_name: str, user: WebUser = Depends(get_current_user) -) -> WebUser: - """Same as :func:`require_server_access` for a ``{server_name}`` path param.""" - check_server_access(user.id, server_name) - return user - - async def require_server_access_query( server: str = Query(...), user: WebUser = Depends(get_current_user) ) -> WebUser: diff --git a/condor/web/routes/routines.py b/condor/web/routes/routines.py index dbfc2db4c..e1649654d 100644 --- a/condor/web/routes/routines.py +++ b/condor/web/routes/routines.py @@ -22,7 +22,6 @@ from condor.web.auth import ( check_server_access, get_current_user, - require_server_access_by_server_name, require_server_access_query, ) from condor.web.models import WebUser @@ -35,15 +34,6 @@ # ── Request / Response Models ── -class RunRequest(BaseModel): - config: dict = {} - - -class ScheduleRequest(BaseModel): - config: dict = {} - interval_sec: int = 300 - - class RunRequestV2(BaseModel): routine_name: str server_name: str @@ -175,49 +165,6 @@ async def get_instance_image( return Response(content=result.chart_image, media_type="image/png") -@router.post("/servers/{server_name}/{routine_name}/run") -async def run_routine( - server_name: str, - routine_name: str, - body: RunRequest, - user: WebUser = Depends(require_server_access_by_server_name), -): - """Execute a one-shot routine. Returns instance_id for polling.""" - store = get_routine_store() - try: - instance_id = await store.execute( - routine_name=routine_name, - config=body.config, - server_name=server_name, - user_id=user.id, - ) - except ValueError as e: - raise HTTPException(404, str(e)) - return {"instance_id": instance_id} - - -@router.post("/servers/{server_name}/{routine_name}/schedule") -async def schedule_routine( - server_name: str, - routine_name: str, - body: ScheduleRequest, - user: WebUser = Depends(require_server_access_by_server_name), -): - """Schedule a routine at an interval. Returns instance_id.""" - store = get_routine_store() - try: - instance_id = await store.schedule( - routine_name=routine_name, - config=body.config, - server_name=server_name, - interval_sec=body.interval_sec, - user_id=user.id, - ) - except ValueError as e: - raise HTTPException(404, str(e)) - return {"instance_id": instance_id} - - @router.post("/run") async def run_routine_v2( body: RunRequestV2, diff --git a/tests/test_routines_access.py b/tests/test_routines_access.py index eacd25f7a..904e629fd 100644 --- a/tests/test_routines_access.py +++ b/tests/test_routines_access.py @@ -1,10 +1,10 @@ """Tests for SEC-045: routine run/schedule endpoints must enforce server access. -The four execution endpoints (``/routines/servers/{server}/{routine}/run``, -``.../schedule``, ``/routines/run``, ``/routines/schedule``) accept an -arbitrary ``server_name``. Without a ``has_server_access`` gate, any approved -user could run/schedule routines against servers owned by other users, using -those servers' stored API credentials (cross-server IDOR). +The execution endpoints (``/routines/run``, ``/routines/start``, +``/routines/schedule``) take an arbitrary ``server_name`` in the request body. +Without a ``has_server_access`` gate, any approved user could run/schedule +routines against servers owned by other users, using those servers' stored API +credentials (cross-server IDOR). """ import pytest @@ -127,26 +127,6 @@ def client_and_cm(monkeypatch): # ── Denied: no access to the target server ── -def test_run_denied_on_foreign_server(client_and_store): - client, store = client_and_store - resp = client.post( - f"/routines/servers/{FOREIGN_SERVER}/some_routine/run", - json={"config": {}}, - ) - assert resp.status_code == 403 - assert store.execute_calls == [] - - -def test_schedule_denied_on_foreign_server(client_and_store): - client, store = client_and_store - resp = client.post( - f"/routines/servers/{FOREIGN_SERVER}/some_routine/schedule", - json={"config": {}, "interval_sec": 60}, - ) - assert resp.status_code == 403 - assert store.schedule_calls == [] - - def test_run_v2_denied_on_foreign_server(client_and_store): client, store = client_and_store resp = client.post( @@ -174,28 +154,6 @@ def test_schedule_v2_denied_on_foreign_server(client_and_store): # ── Allowed: user has access to the target server ── -def test_run_allowed_on_owned_server(client_and_store): - client, store = client_and_store - resp = client.post( - f"/routines/servers/{OWNED_SERVER}/some_routine/run", - json={"config": {}}, - ) - assert resp.status_code == 200 - assert resp.json() == {"instance_id": "inst-run"} - assert store.execute_calls == [("some_routine", OWNED_SERVER, USER.id)] - - -def test_schedule_allowed_on_owned_server(client_and_store): - client, store = client_and_store - resp = client.post( - f"/routines/servers/{OWNED_SERVER}/some_routine/schedule", - json={"config": {}, "interval_sec": 60}, - ) - assert resp.status_code == 200 - assert resp.json() == {"instance_id": "inst-sched"} - assert store.schedule_calls == [("some_routine", OWNED_SERVER, USER.id)] - - def test_run_v2_allowed_on_owned_server(client_and_store): client, store = client_and_store resp = client.post( diff --git a/tests/test_web_server_access_dependency.py b/tests/test_web_server_access_dependency.py index 34ae5fb3e..22d746285 100644 --- a/tests/test_web_server_access_dependency.py +++ b/tests/test_web_server_access_dependency.py @@ -8,16 +8,19 @@ route carries one of the ``require_server_access*`` dependencies, with a small explicit allowlist for the routes that deliberately answer differently. -Three shapes exist, and they are not interchangeable: +Two shapes exist, and they are not interchangeable: * ``/servers/{name}/...`` → ``require_server_access`` -* ``/servers/{server_name}/...`` → ``require_server_access_by_server_name`` * ``?server=...`` → ``require_server_access_query`` -and a fourth that no path/query dependency can cover — the server name arriving +and a third that no path/query dependency can cover — the server name arriving in the request **body** — which uses the shared ``check_server_access`` helper the dependencies are built on. Those sites are pinned by source inspection below, because there is no signature for FastAPI to reflect on. + +The sweep below also recognises a ``{server_name}`` path parameter, a shape no +route uses today (READ-595 deleted the last two): a route reintroducing it is +reported as unguarded rather than quietly skipped. """ from __future__ import annotations @@ -36,14 +39,12 @@ check_server_access, get_current_user, require_server_access, - require_server_access_by_server_name, require_server_access_query, ) from condor.web.models import WebUser GUARDS = { require_server_access, - require_server_access_by_server_name, require_server_access_query, } @@ -167,11 +168,6 @@ def test_each_shape_uses_the_dependency_that_matches_it(app): query_params = {p.name for p in sub.query_params} if sub.call is require_server_access and "name" not in path_params: wrong.append(f"{route.path}: require_server_access has no {{name}}") - if ( - sub.call is require_server_access_by_server_name - and "server_name" not in path_params - ): - wrong.append(f"{route.path}: no {{server_name}} path param") if sub.call is require_server_access_query and "server" not in query_params: wrong.append(f"{route.path}: no ?server= query param") assert not wrong, wrong @@ -263,13 +259,6 @@ def test_path_guard_returns_the_user_on_an_owned_server(deny): assert asyncio.run(require_server_access(OWNED, user=USER)) is USER -def test_server_name_alias_guard_refuses_a_foreign_server(deny): - with pytest.raises(HTTPException) as exc: - asyncio.run(require_server_access_by_server_name(FOREIGN, user=USER)) - assert exc.value.status_code == 403 - assert asyncio.run(require_server_access_by_server_name(OWNED, user=USER)) is USER - - def test_query_guard_refuses_a_foreign_server(deny): with pytest.raises(HTTPException) as exc: asyncio.run(require_server_access_query(FOREIGN, user=USER)) @@ -306,8 +295,6 @@ def client(app, deny): ("GET", f"/api/v1/settings/gateway/status?server={FOREIGN}"), ("GET", f"/api/v1/settings/credentials?server={FOREIGN}"), ("GET", f"/api/v1/routines/options/controller_configs?server={FOREIGN}"), - # {server_name} path param - ("POST", f"/api/v1/routines/servers/{FOREIGN}/some_routine/run"), ], ) def test_a_foreign_server_is_refused_over_http(client, method, url): From baab7a1fff1bccca7c32f7d699c5de6e1b9c9303 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 8 Sep 2026 20:15:32 +0300 Subject: [PATCH 027/154] Normalize a controller-performance response in one place bots.py carried a byte-identical copy of bot_performance.extract_snapshots, down to the setdefault("controller_id", key) fallback. Import the original instead, so the two bot data paths cannot drift in how they read a snapshot. --- condor/fetchers/bots.py | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/condor/fetchers/bots.py b/condor/fetchers/bots.py index f3fadce3f..119e369af 100644 --- a/condor/fetchers/bots.py +++ b/condor/fetchers/bots.py @@ -4,6 +4,8 @@ import logging from typing import Any, NamedTuple, Optional +from condor.fetchers.bot_performance import extract_snapshots as _extract_perf_snapshots + logger = logging.getLogger(__name__) # Per-call budget for one of the optional enrichment fetches below. The bots @@ -252,29 +254,6 @@ def empty(cls) -> "BotsEnrichment": return cls({}, {}, {}) -def _extract_perf_snapshots(result: Any) -> list[dict]: - """Normalize controller performance API response into a list of snapshot dicts.""" - if isinstance(result, list): - return [s for s in result if isinstance(s, dict)] - if isinstance(result, dict): - data = result.get("data", result.get("snapshots", result.get("records", []))) - if isinstance(data, list): - return [s for s in data if isinstance(s, dict)] - if isinstance(data, dict): - out = [] - for key, val in data.items(): - if isinstance(val, dict): - val.setdefault("controller_id", key) - out.append(val) - elif isinstance(val, list): - for item in val: - if isinstance(item, dict): - item.setdefault("controller_id", key) - out.append(item) - return out - return [] - - def _collect_bot_runs(result: Any, runs: dict[str, str]) -> None: """Merge a bot-runs API response into ``runs`` (bot_name -> deployed_at).""" if not isinstance(result, dict): From aefbcd7259f26c1f23e56f1c1bf9d6eb759e17f7 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 07:43:26 +0300 Subject: [PATCH 028/154] Let a reloaded page find the approval an agent is still waiting on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /confirmations documented itself as the durable path for an approval stranded by a page reload, and no client ever called it. The claim is now true rather than deleted: the dashboard reads it on every socket open and merges what comes back, so a reload mid-approval re-renders the prompt instead of leaving the agent to be auto-denied when its 120s TTL expires. Two things had to hold for that to be safe. Each entry carries the slot that asked, so a click cannot authorize a different conversation's tool call, and WebSocketChannel.deliver now addresses the request the way turn events are addressed instead of pinning it to the socket that raised it — with no socket open it is a no-op and the entry simply stays pending for the next page load. _slot_of moves to condor/runtime/keys.slot_of, since both paths need it. --- condor/runtime/__init__.py | 3 +- condor/runtime/keys.py | 13 ++ condor/web/routes/chat_ws.py | 25 +-- condor/web/routes/confirmations.py | 16 +- .../useChatSocket.confirmations.test.tsx | 203 ++++++++++++++++++ frontend/src/hooks/useChatSocket.ts | 43 +++- frontend/src/lib/api.ts | 23 ++ tests/test_confirmations_api.py | 92 ++++++++ 8 files changed, 399 insertions(+), 19 deletions(-) create mode 100644 frontend/src/hooks/useChatSocket.confirmations.test.tsx create mode 100644 tests/test_confirmations_api.py diff --git a/condor/runtime/__init__.py b/condor/runtime/__init__.py index 5aee8f25a..156edf0c5 100644 --- a/condor/runtime/__init__.py +++ b/condor/runtime/__init__.py @@ -15,7 +15,7 @@ """ from condor.runtime.events import EventType, RuntimeEvent -from condor.runtime.keys import MCP, TELEGRAM, WEB, SessionKey +from condor.runtime.keys import MCP, TELEGRAM, WEB, SessionKey, slot_of from condor.runtime.models import ( PromptImage, PromptRequest, @@ -33,5 +33,6 @@ "RuntimeEvent", "SessionInfo", "SessionKey", + "slot_of", "SessionSpec", ] diff --git a/condor/runtime/keys.py b/condor/runtime/keys.py index 3d71dcadb..8610d8abd 100644 --- a/condor/runtime/keys.py +++ b/condor/runtime/keys.py @@ -98,3 +98,16 @@ def telegram_chat_id(self) -> int | None: return int(self.owner) except ValueError: return None + + +def slot_of(session_key: str) -> str: + """The slot a session key belongs to, or "" if the key is not canonical. + + Never raises: a confirmation that cannot be attributed is still worth + delivering unaddressed, which is what the dashboard did for all of them + before this became a field. + """ + try: + return SessionKey.parse(session_key).slot + except ValueError: + return "" diff --git a/condor/web/routes/chat_ws.py b/condor/web/routes/chat_ws.py index 37afe40ff..3507f8cb8 100644 --- a/condor/web/routes/chat_ws.py +++ b/condor/web/routes/chat_ws.py @@ -29,6 +29,7 @@ from condor.runtime import ( conversations, secrets, + slot_of, ) from condor.runtime.binding import remember_model_choice from condor.runtime.confirmations import ( @@ -239,19 +240,6 @@ def _to_ws_message(event: RuntimeEvent, slot_id: str) -> dict | None: return None -def _slot_of(session_key: str) -> str: - """The slot a registry entry belongs to, or "" if the key is not canonical. - - Never raises: a confirmation that cannot be attributed is still worth - delivering unaddressed, which is what the dashboard did for all of them - before this became a field. - """ - try: - return SessionKey.parse(session_key).slot - except ValueError: - return "" - - async def _send(ws: WebSocket, event: dict) -> None: """Send a JSON event to the client, ignoring closed connections.""" try: @@ -288,14 +276,21 @@ class WebSocketChannel: now the registry's id rather than a locally-minted one, which is what lets the same request also be answered from Telegram or over HTTP after a page reload kills this socket. + + Addressed like a turn's events rather than pinned to the socket that asked: + the request outlives its connection, so when that socket is gone the prompt + goes to whichever tabs the user still has open. With none open it is a + no-op and the entry stays pending in the registry, which is what the next + page load reads back over ``GET /api/v1/confirmations``. """ def __init__(self, ws: WebSocket): self._ws = ws async def deliver(self, pending: PendingConfirmation) -> None: - await _send( + await _send_turn( self._ws, + pending.user_id, { "event": "permission_request", # Addressed like every other chat event (CORR-101). One socket @@ -303,7 +298,7 @@ async def deliver(self, pending: PendingConfirmation) -> None: # slot the dashboard can only render the approval in whichever # one is on screen — and a click meant for one agent, on one # trading server, authorizes a live tool call in another. - "slot_id": _slot_of(pending.session_key), + "slot_id": slot_of(pending.session_key), "request_id": pending.id, "summary": pending.summary, # Which agent, on which server, is asking. The slot addresses diff --git a/condor/web/routes/confirmations.py b/condor/web/routes/confirmations.py index 306559516..a6d3f9525 100644 --- a/condor/web/routes/confirmations.py +++ b/condor/web/routes/confirmations.py @@ -16,6 +16,7 @@ from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel +from condor.runtime import slot_of from condor.runtime.confirmations import get_registry from condor.web.auth import get_current_user from condor.web.models import WebUser @@ -32,8 +33,19 @@ class ResolveRequest(BaseModel): @router.get("") async def list_confirmations(user: WebUser = Depends(get_current_user)): - """Approvals this user still needs to answer.""" - return [p.to_wire() for p in get_registry().list_pending(user_id=user.id)] + """Approvals this user still needs to answer. + + Read by the dashboard on every socket open (``useChatSocket``), which is + what makes the reload case above real: the ``permission_request`` event was + pushed to a socket that no longer exists, and this is the only way the new + page learns the agent is still waiting. ``slot_id`` addresses each entry the + same way that event does, so a click meant for one conversation cannot + authorize another one's tool call. + """ + return [ + {**p.to_wire(), "slot_id": slot_of(p.session_key)} + for p in get_registry().list_pending(user_id=user.id) + ] @router.get("/{confirmation_id}") diff --git a/frontend/src/hooks/useChatSocket.confirmations.test.tsx b/frontend/src/hooks/useChatSocket.confirmations.test.tsx new file mode 100644 index 000000000..567dd0046 --- /dev/null +++ b/frontend/src/hooks/useChatSocket.confirmations.test.tsx @@ -0,0 +1,203 @@ +/** + * What a socket open re-reads about approvals (READ-597). + * + * `permission_request` is a fire-and-forget push. A reload mid-approval killed + * the socket it was addressed to and nothing re-sent it, so the agent sat + * waiting behind a page that showed no prompt until its TTL denied the tool + * call. These pin the recovery path: every open asks the registry what is + * still pending, files it under the conversation that asked, and never + * disturbs an approval already on screen. + * + * Needs a DOM, so this file overrides vitest's default `node` environment. + * + * @vitest-environment jsdom + */ + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, useEffect } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ServerContext } from "@/hooks/useServer"; + +const getPendingConfirmations = vi.fn(); + +vi.mock("@/lib/api", () => ({ + api: { + listConversations: () => Promise.resolve([]), + getSessionOptions: () => Promise.resolve({ default_agent: "claude-code" }), + getConversation: () => Promise.resolve({ meta: {}, turns: [] }), + getPendingConfirmations: () => getPendingConfirmations(), + }, +})); + +vi.mock("@/lib/auth", () => ({ + useAuth: () => ({ token: "jwt", user: { id: 7 } }), +})); + +const { useChatSocket } = await import("./useChatSocket"); + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +class FakeSocket { + static last: FakeSocket | null = null; + + static readonly CONNECTING = 0; + static readonly OPEN = 1; + static readonly CLOSING = 2; + static readonly CLOSED = 3; + readyState = FakeSocket.OPEN; + onopen: (() => void) | null = null; + onclose: (() => void) | null = null; + onmessage: ((ev: { data: string }) => void) | null = null; + + constructor() { + FakeSocket.last = this; + } + send() {} + close() { + this.readyState = FakeSocket.CLOSED; + } + deliver(frame: Record) { + this.onmessage?.({ data: JSON.stringify(frame) }); + } +} + +const sock = () => FakeSocket.last!; + +const holder: { current: ReturnType | null } = { + current: null, +}; +const chat = () => holder.current!; + +function Harness() { + const state = useChatSocket(); + useEffect(() => { + holder.current = state; + }); + return null; +} + +let container: HTMLDivElement; +let root: Root; + +async function settle() { + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + }); +} + +/** Open the page and bring the socket up, the way the browser would. */ +async function arrive() { + act(() => { + root.render( + + {} }}> + + + , + ); + }); + act(() => { + chat().connect(); + sock().onopen?.(); + }); + await settle(); +} + +/** One approval, as `GET /api/v1/confirmations` lists it. */ +const stranded = { + id: "abc123", + slot_id: "s1", + summary: "Place a 1 SOL order?", + origin: "brigado on moneymaker", + expires_at: 0, +}; + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + vi.stubGlobal("WebSocket", FakeSocket); + FakeSocket.last = null; + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + getPendingConfirmations.mockResolvedValue([]); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); + vi.clearAllMocks(); +}); + +describe("re-reading pending approvals on a socket open", () => { + it("re-renders a prompt the reload stranded, in the slot that asked", async () => { + getPendingConfirmations.mockResolvedValue([stranded]); + + await arrive(); + + expect(chat().permissionRequests.s1).toEqual({ + request_id: "abc123", + summary: "Place a 1 SOL order?", + origin: "brigado on moneymaker", + }); + }); + + it("files an approval with no slot where an unaddressed one goes", async () => { + getPendingConfirmations.mockResolvedValue([{ ...stranded, slot_id: "" }]); + + await arrive(); + + expect(chat().permissionRequests[""].request_id).toBe("abc123"); + }); + + it("leaves an approval this session is already showing alone", async () => { + await arrive(); + + // The live push arrives first and is what the user is looking at. + act(() => { + sock().deliver({ + event: "permission_request", + slot_id: "s1", + request_id: "live-1", + summary: "the one on screen", + origin: "", + }); + }); + + // A later open re-reads and finds the registry's own view of that slot. + getPendingConfirmations.mockResolvedValue([stranded]); + act(() => { + sock().readyState = FakeSocket.CLOSED; + sock().onclose?.(); + }); + act(() => { + chat().connect(); + sock().onopen?.(); + }); + await settle(); + + expect(chat().permissionRequests.s1.request_id).toBe("live-1"); + }); + + it("says nothing when there is nothing pending", async () => { + await arrive(); + + expect(getPendingConfirmations).toHaveBeenCalled(); + expect(chat().permissionRequests).toEqual({}); + }); + + it("keeps the chat working when the read fails", async () => { + getPendingConfirmations.mockRejectedValue(new Error("gateway is down")); + + await arrive(); + + expect(chat().permissionRequests).toEqual({}); + expect(chat().isConnected).toBe(true); + }); +}); diff --git a/frontend/src/hooks/useChatSocket.ts b/frontend/src/hooks/useChatSocket.ts index e93881503..64d1d3bdc 100644 --- a/frontend/src/hooks/useChatSocket.ts +++ b/frontend/src/hooks/useChatSocket.ts @@ -1025,6 +1025,44 @@ export function useChatSocket() { [send, updateSlotMessages], ); + /** + * Re-read the approvals this user has not answered yet (FEAT-010). + * + * `permission_request` is a fire-and-forget push: a reload mid-approval + * killed the socket it was addressed to, and nothing re-sent it, so the + * agent sat waiting behind a page that showed no prompt until its TTL denied + * the call two minutes later. The registry outlives the connection, so every + * socket open asks it what is still pending. + * + * Merged, never assigned: an approval this session already has on screen is + * left exactly as it is, and one answered between the read going out and + * coming back is simply absent from the reply. A failed read is silent — + * the socket is up and the live path still works. + */ + const replayPendingConfirmations = useCallback(async () => { + try { + const pending = await api.getPendingConfirmations(); + if (pending.length === 0) return; + setPermissionRequests((prev) => { + const next = { ...prev }; + let added = false; + for (const p of pending) { + const slot = p.slot_id || UNATTRIBUTED; + if (next[slot]) continue; + next[slot] = { + request_id: p.id, + summary: p.summary, + origin: p.origin || "", + }; + added = true; + } + return added ? next : prev; + }); + } catch { + /* the live path is unaffected; the next connect asks again */ + } + }, []); + // Drop the current socket without letting its asynchronous `onclose` speak // for a connection we already decided to abandon. const closeSocket = useCallback(() => { @@ -1075,6 +1113,9 @@ export function useChatSocket() { const queued = unsent.current; unsent.current = []; for (const msg of queued) ws.send(JSON.stringify(msg)); + // The roster that follows says which conversations are alive; it says + // nothing about which of them is holding a tool call waiting on a click. + void replayPendingConfirmations(); }; ws.onclose = () => { // A socket we replaced or closed on purpose still fires `onclose`, long @@ -1096,7 +1137,7 @@ export function useChatSocket() { /* ignore */ } }; - }, [token, closeSocket]); + }, [token, closeSocket, replayPendingConfirmations]); const disconnect = useCallback(() => { shouldConnect.current = false; diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 3a8fd114d..9a0b8bb4a 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -2264,6 +2264,16 @@ export async function fetchPerformanceHistoryAll( // ── API functions ── +/** One approval still waiting on this user, as `GET /api/v1/confirmations` lists it. */ +export interface PendingConfirmation { + id: string; + /** The conversation that asked, so the prompt renders where the click belongs. */ + slot_id: string; + summary: string; + origin: string; + expires_at: number; +} + export const api = { getServers: () => apiFetch("/api/v1/servers"), @@ -3919,6 +3929,19 @@ export const api = { { method: "DELETE" }, ), + // ── Pending approvals (FEAT-010) ── + + /** + * Approvals this user has not answered yet, scoped to the JWT server-side. + * + * The WS `permission_request` event is the low-latency path and dies with the + * socket that carried it; this is how a page that just reloaded mid-approval + * finds out an agent is still waiting on it. Answering still goes over the + * socket — the registry is the same one either way. + */ + getPendingConfirmations: () => + apiFetch("/api/v1/confirmations"), + // ── Notifications (FEAT-048) ── /** The bell's history. Scoped to the JWT server-side — there is no user param. */ diff --git a/tests/test_confirmations_api.py b/tests/test_confirmations_api.py new file mode 100644 index 000000000..b4fae4ec1 --- /dev/null +++ b/tests/test_confirmations_api.py @@ -0,0 +1,92 @@ +"""The REST view of a pending approval (FEAT-010, READ-597). + +The WS ``permission_request`` event is a fire-and-forget push, so a reload +mid-approval leaves the agent waiting behind a page that shows no prompt. This +module is the read that fixes that, and the dashboard calls it on every socket +open — which makes two things load-bearing here: it must list only *your* +pending approvals, and each one must say which conversation asked, so a click +cannot authorize a different agent's tool call. +""" + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +import condor.runtime.confirmations as confirmations_module +import condor.web.routes.confirmations as routes +from condor.runtime.confirmations import ConfirmationRegistry +from condor.web.auth import get_current_user +from condor.web.models import WebUser + +USER = WebUser(id=111, username="u", first_name="U", role="user") +OTHER_ID = 222 + + +@pytest.fixture +def registry(monkeypatch): + """A throwaway process-global registry for one test.""" + fresh = ConfirmationRegistry() + monkeypatch.setattr(confirmations_module, "_registry", fresh) + return fresh + + +def _client(user: WebUser = USER) -> TestClient: + app = FastAPI() + app.include_router(routes.router) + app.dependency_overrides[get_current_user] = lambda: user + return TestClient(app) + + +def _register(registry, user_id: int, summary: str, slot: str = "s1"): + return registry.register( + session_key=f"web:{user_id}:{slot}", + user_id=user_id, + summary=summary, + tool_call={"name": "place_order"}, + options=[], + ) + + +def test_list_carries_the_slot_that_asked(registry): + pending = _register(registry, USER.id, "Place a 1 SOL order?", slot="chat-2") + + body = _client().get("/confirmations").json() + + assert [p["id"] for p in body] == [pending.id] + assert body[0]["slot_id"] == "chat-2" + assert body[0]["summary"] == "Place a 1 SOL order?" + + +def test_list_is_scoped_to_the_asking_user(registry): + mine = _register(registry, USER.id, "mine") + _register(registry, OTHER_ID, "not mine") + + body = _client().get("/confirmations").json() + + assert [p["id"] for p in body] == [mine.id] + + +def test_a_non_canonical_session_key_is_listed_unaddressed(registry): + """A Telegram-raised approval has no slot; it is still answerable.""" + registry.register( + session_key="not-a-canonical-key", + user_id=USER.id, + summary="from somewhere else", + tool_call={}, + options=[], + ) + + body = _client().get("/confirmations").json() + + assert body[0]["slot_id"] == "" + + +def test_an_answered_approval_leaves_the_list(registry): + pending = _register(registry, USER.id, "mine") + client = _client() + + assert len(client.get("/confirmations").json()) == 1 + + client.post(f"/confirmations/{pending.id}/resolve", json={"approved": True}) + + assert client.get("/confirmations").json() == [] From 8cc496161f0f36117e4169edaec7c56604de1013 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 07:49:24 +0300 Subject: [PATCH 029/154] Express the login-token TTL in one place create_login_token inlined the same expiry sweep that _gc_expired_login_tokens implements two functions later, and redeem_login_token re-checked the TTL after that helper had already swept the store with the same clock, so the guard could never fire. Move the helper above its first caller, call it from both, and drop the dead re-check. --- condor/web/auth.py | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/condor/web/auth.py b/condor/web/auth.py index e1e9a2949..14f80ed86 100644 --- a/condor/web/auth.py +++ b/condor/web/auth.py @@ -227,10 +227,8 @@ async def require_server_access_query( # ── One-time login tokens (generated from Telegram /web command) ── -def create_login_token(user_id: int, username: str = "", first_name: str = "") -> str: - """Create a one-time login token for a Telegram user.""" - # Clean up expired tokens - now = time.time() +def _gc_expired_login_tokens(now: float) -> None: + """Remove expired one-time login tokens from the in-memory store.""" expired = [ k for k, v in _pending_login_tokens.items() @@ -239,6 +237,12 @@ def create_login_token(user_id: int, username: str = "", first_name: str = "") - for k in expired: _pending_login_tokens.pop(k, None) + +def create_login_token(user_id: int, username: str = "", first_name: str = "") -> str: + """Create a one-time login token for a Telegram user.""" + now = time.time() + _gc_expired_login_tokens(now) + token = secrets.token_urlsafe(32) _pending_login_tokens[token] = { "user_id": user_id, @@ -249,17 +253,6 @@ def create_login_token(user_id: int, username: str = "", first_name: str = "") - return token -def _gc_expired_login_tokens(now: float) -> None: - """Remove expired one-time login tokens from the in-memory store.""" - expired = [ - k - for k, v in _pending_login_tokens.items() - if now - v["created_at"] > _LOGIN_TOKEN_TTL - ] - for k in expired: - _pending_login_tokens.pop(k, None) - - def redeem_login_token(token: str) -> Optional[dict]: """Redeem a one-time login token. Returns user info or None if invalid/expired. @@ -278,8 +271,4 @@ def redeem_login_token(token: str) -> Optional[dict]: if info is None: return None - # Reject expired tokens (already popped above). - if now - info["created_at"] > _LOGIN_TOKEN_TTL: - return None - return info From d361e460b672ec686d38f911054141d108ac7e75 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 07:55:33 +0300 Subject: [PATCH 030/154] Describe the caching and web-import rules the fetchers package actually follows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The package docstring is the contract every new fetcher is judged against, and two of its statements had drifted from the code below it. It claimed one sanctioned cache. There are six, each well argued where it lives, so the "find exactly one" rule was unenforceable and read as already broken. State the line that is actually enforceable instead — a fetcher may hold a process-wide cache only when the answer is identical for every caller, keyed so one server or Gateway network cannot be served another's, with a rationale comment, and listed here — then list all six with their module, constant, TTL or bound, and clear hook. The web rule is true again since the shared wire shapes moved into condor.fetchers.models, so it stays as written; it now also says where those shapes live and that a fetcher raises its own exception rather than HTTPException. The deep-import list was missing four modules added since it was written: archived_run, run_history, gateway_tokens and models. --- condor/fetchers/__init__.py | 49 +++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/condor/fetchers/__init__.py b/condor/fetchers/__init__.py index 2dea38184..f483d8142 100644 --- a/condor/fetchers/__init__.py +++ b/condor/fetchers/__init__.py @@ -13,21 +13,50 @@ Rules: - Functions receive an API *client* and return data. No UI. - - No per-caller result caching — that belongs to condor.server_data_service. - One sanctioned exception: condor.fetchers.bot_performance keeps a short-TTL, - in-flight-coalescing cache of the whole-server controller-performance - snapshot, because that call returns the same payload for every caller and - the agents rollup fans out N of them at once (see the rationale comment - above ``_SNAPSHOT_TTL``). The aggregate ``fetch_all_bot_performance`` - returns is therefore shared between callers and must be treated as - read-only. - - No handlers/ or condor.web imports (prevents circular deps). + - No *per-caller* result caching — that belongs to condor.server_data_service. + A fetcher may keep a **process-wide** cache only when the answer is the same + for every caller: the upstream call is whole-server (so its payload does not + depend on who asks) or its subject is immutable. Every such cache must be + keyed so one server's — or one Gateway network's — answer can never be + served for another, carry a comment saying why the caller cannot hold it + instead, and be listed here. There are six today: + + * ``bot_performance._snapshot_cache`` — whole-server controller + performance, ``_SNAPSHOT_TTL`` 5s, in-flight coalesced, + ``clear_snapshot_cache()``. + * ``bot_performance._archived_cache`` — the archived-database listing, + ``_ARCHIVED_TTL`` 60s, ``clear_archived_cache()``. + * ``bot_performance._history_cache`` — per-instance history pages, + ``_HISTORY_TTL`` 20s, LRU-capped at ``_HISTORY_CACHE_MAX`` (256), + in-flight coalesced, ``clear_history_cache()``. + * ``archived_run._performance_cache`` — whole archived runs. No TTL: an + archived sqlite file is immutable. LRU-capped at + ``_PERFORMANCE_CACHE_MAX`` (32) because each entry is large; tests + clear it directly, there is no ``clear_*`` hook. + * ``run_history._CLASS_CACHE`` — a terminated controller's class, held + for the process lifetime (a stored config is immutable, and one that + is gone does not come back), ``clear_controller_class_cache()``. + * ``gateway_tokens._listed`` — addresses confirmed present on a Gateway + token list. Confirmations only, never failures; flushed wholesale past + ``_MAX_MEMO`` (4000). ``reset_listed_memo()`` / + ``forget_listed()``. + + Everything one of these caches hands back is shared between callers and must + be treated as read-only — including the aggregate ``fetch_all_bot_performance`` + returns. + - No handlers/ or condor.web imports (prevents circular deps). Wire shapes + shared with the web layer live in ``condor.fetchers.models`` and are + re-exported by ``condor.web.models``, so the edge points outward only; a + fetcher raises its own exception type (e.g. + ``archived_run.ArchivedRunUnavailable``) rather than ``HTTPException``, and + mapping that to a status code is the route's job. - Keep thin: call client method, light transform, return. Importing: Deep module imports are the convention — ``from condor.fetchers.bot_performance import fetch_all_bot_performance``. Every consumer except one does that, and - modules with no re-export below (``bot_performance``, ``bots.build_bots_page`` / + modules with no re-export below (``bot_performance``, ``archived_run``, + ``run_history``, ``gateway_tokens``, ``models``, ``bots.build_bots_page`` / ``extract_bots_list``, ``portfolio.fetch_portfolio_refreshed`` / ``fetch_cex_balances``) are reached that way only. The names re-exported here exist for condor.server_data_service.register_default_fetches(), the sole From 3ad758c6af2f21af2526ab11f0e0020007cbda3f Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 08:01:06 +0300 Subject: [PATCH 031/154] Say that a failed SDS fetch hands back the old value, at any age get_or_fetch's docstring promised "cached data if fresh, otherwise fetch", but a fetch that raises returns the previous value silently, at whatever age it has. The staleness is unbounded for exactly the keys the dashboard cares about: _cleanup_stale skips every key with a live subscriber, so with the API server down the portfolio, bot status and executor entries keep serving the last good snapshot forever. Spell out the three outcomes on get_or_fetch and point callers at get_entry for fetched_at / consecutive_errors / last_error_at, which is what they must read when the age is load-bearing. Note the subscriber exemption on _cleanup_stale, say on get that its None means "nothing usable in cache" rather than "the server is fine", and have get_entry point back at get_or_fetch. Docstrings only; no behaviour changes. --- condor/server_data_service.py | 44 +++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/condor/server_data_service.py b/condor/server_data_service.py index d085b2ec0..2ccceb52e 100644 --- a/condor/server_data_service.py +++ b/condor/server_data_service.py @@ -421,7 +421,14 @@ def unsubscribe_all(self, subscriber_id: str) -> None: # ------ Read API ------ def get(self, server: str, data_type: ServerDataType, **params) -> Optional[Any]: - """Read from cache only (hot path). Returns None if not cached or expired.""" + """Read from cache only (hot path). Returns None if not cached or expired. + + ``None`` means "nothing usable in cache", never "the server is fine": a + failing fetch leaves the previous value in place (see + :meth:`get_or_fetch`) and the entry only disappears once it expires and + :meth:`_cleanup_stale` is allowed to evict it. Use :meth:`get_entry` for + the error/age metadata behind a ``None``. + """ key = CacheKey.make(server, data_type, **params) entry = self._cache.get(key) if entry is None: @@ -436,7 +443,24 @@ def get(self, server: str, data_type: ServerDataType, **params) -> Optional[Any] async def get_or_fetch( self, server: str, data_type: ServerDataType, **params ) -> Optional[Any]: - """Return cached data if fresh, otherwise fetch. For REST/one-shot reads.""" + """Return cached data if fresh, otherwise fetch. For REST/one-shot reads. + + Three outcomes, indistinguishable from the return value alone: + + 1. a cache hit within the key's TTL; + 2. a successful fetch, just written to the cache; + 3. a *failed* fetch, which returns the previous value at whatever age it + has — or ``None`` if there never was one. + + The third case is silent and unbounded: nothing here caps how old the + returned value may be, and a key with a live subscriber is never evicted + by :meth:`_cleanup_stale`, so with the API server down this keeps handing + out the last good snapshot for as long as the subscriber stays attached. + + The value carries no age of its own. When freshness is load-bearing, + read :meth:`get_entry` alongside it for ``fetched_at``, + ``consecutive_errors`` and ``last_error_at``. + """ key = CacheKey.make(server, data_type, **params) # Check cache @@ -450,7 +474,13 @@ async def get_or_fetch( def get_entry( self, server: str, data_type: ServerDataType, **params ) -> Optional[CacheEntry]: - """Get the full cache entry (for age/metadata checks).""" + """Get the full cache entry (for age/metadata checks). + + This is the sanctioned way to age-check a :meth:`get_or_fetch` result: + that call can return a value of any age when the fetch failed, and only + the entry's ``fetched_at`` / ``consecutive_errors`` / ``last_error_at`` + say so. + """ key = CacheKey.make(server, data_type, **params) return self._cache.get(key) @@ -826,7 +856,13 @@ def _fire_callbacks(self, key: CacheKey, old_value: Any, new_value: Any) -> None self._callback_tasks.track(task, sub.subscriber_id) def _cleanup_stale(self) -> None: - """Remove cache entries with no subscribers and expired TTL.""" + """Remove cache entries with no subscribers and expired TTL. + + A key with at least one live subscriber is exempt: it is kept no matter + how old it is. That is what makes :meth:`get_or_fetch`'s stale-on-error + window unbounded for the subscribed keys (portfolio, bot status, + executors) while a subscriber is attached. + """ now = time.time() stale = [] for key, entry in self._cache.items(): From 97e0a82cbaf9096e353d27d64d67694cda210e87 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 08:07:00 +0300 Subject: [PATCH 032/154] Name the two interval pickers for what each one picks Two public `pick_interval` functions sat in the same charting domain, one taking seconds and returning an `Interval` NamedTuple off a ladder that starts at 1m, the other taking milliseconds and returning a bare string off a ladder that starts at 5m. Mixing them up was silent: milliseconds into the first always yield 4h, seconds into the second always yield 5m. Rename them to `pick_candle_interval` and `pick_sampling_interval`, and have each docstring name the other along with its unit. No thresholds, ladders or return values change. --- condor/archived_chart_series.py | 8 ++++++-- condor/fetchers/run_history.py | 10 +++++++--- condor/run_history_store.py | 8 ++++---- tests/test_archived_chart_series.py | 12 ++++++------ tests/test_run_history_fetch.py | 9 +++++---- 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/condor/archived_chart_series.py b/condor/archived_chart_series.py index 4ea801ee0..207c7e2b4 100644 --- a/condor/archived_chart_series.py +++ b/condor/archived_chart_series.py @@ -38,12 +38,16 @@ class Interval(NamedTuple): seconds: int -def pick_interval(duration_sec: float) -> Interval: +def pick_candle_interval(duration_sec: float) -> Interval: """Candle interval proportionate to how long the run actually lasted. Mirrors the thresholds the chart component used client-side. It lives server-side now because the choice depends on the *true* activity range, which only the full executor set knows. + + Not to be confused with :func:`condor.fetchers.run_history. + pick_sampling_interval`, which takes **milliseconds** and returns a bare + interval string off a coarser ladder that starts at ``5m``. """ if duration_sec < 2 * 3600: return Interval("1m", 60) @@ -253,7 +257,7 @@ def build_chart_series( start, end = activity_range(group) if start <= 0: continue - interval = pick_interval(max(end - start, 0)) + interval = pick_candle_interval(max(end - start, 0)) rate = rates.for_pair(getattr(group[0], "trading_pair", "")) if rates else 1.0 series[key] = { "interval": interval.name, diff --git a/condor/fetchers/run_history.py b/condor/fetchers/run_history.py index e306f3ea5..a31bad9b0 100644 --- a/condor/fetchers/run_history.py +++ b/condor/fetchers/run_history.py @@ -370,7 +370,7 @@ def __init__(self, detail: str, *, missing: bool = False): } -def pick_interval(span_ms: float, budget: int = HISTORY_POINT_BUDGET) -> str: +def pick_sampling_interval(span_ms: float, budget: int = HISTORY_POINT_BUDGET) -> str: """The finest interval whose point count over ``span_ms`` fits the budget. The same ladder the client uses (``pickSamplingInterval``), and it means the @@ -383,6 +383,10 @@ def pick_interval(span_ms: float, budget: int = HISTORY_POINT_BUDGET) -> str: Upstream validates the parameter against exactly this set and answers 422 for anything else, so a value outside it turns a chart into an error rather than a coarser chart. + + Not to be confused with :func:`condor.archived_chart_series. + pick_candle_interval`, which takes **seconds** and returns an ``Interval`` + NamedTuple whose ladder starts at ``1m`` — a rung this endpoint rejects. """ if not span_ms or span_ms <= 0: return _ORDER[0] @@ -571,7 +575,7 @@ async def _build( start_ms = _to_ms(deployed_at) or 0.0 end_ms = _to_ms(stopped_at) if stopped_at else None span = (end_ms or time.time() * 1000) - start_ms - interval = pick_interval(span) + interval = pick_sampling_interval(span) # The window is widened by one bucket at each end. A run's first dump lands # a moment after its deploy row is written and its last a moment after the @@ -591,7 +595,7 @@ async def _build( # so it asks at the finest rung and accepts the cost. # # Bound to a name rather than passed as a literal because the interval - # is also *recorded*: this path asks finer than ``pick_interval`` chose + # is also *recorded*: this path asks finer than ``pick_sampling_interval`` chose # for the span, and an entry that claims the coarser rung is a lie # frozen into a cache that is never rewritten. walked_interval = "5m" diff --git a/condor/run_history_store.py b/condor/run_history_store.py index 8d713b08d..0c455323b 100644 --- a/condor/run_history_store.py +++ b/condor/run_history_store.py @@ -103,10 +103,10 @@ class RunHistoryEntry: #: Total points across every controller. points: int = 0 #: The upstream sampling interval the rows were actually **fetched** at — - #: provenance, not shape. It is ``pick_interval`` of the run's span (the - #: per-controller walk deliberately goes coarse for a long run; see the - #: note in :mod:`condor.fetchers.run_history`), or ``5m`` when the run - #: declared no controller ids and the walk had no id to bind. + #: provenance, not shape. It is ``pick_sampling_interval`` of the run's + #: span (the per-controller walk deliberately goes coarse for a long run; + #: see the note in :mod:`condor.fetchers.run_history`), or ``5m`` when the + #: run declared no controller ids and the walk had no id to bind. #: #: It is *not* the spacing of the points below it: those are thinned to #: ``HISTORY_POINT_BUDGET`` by time bucket, which lands on no rung of the diff --git a/tests/test_archived_chart_series.py b/tests/test_archived_chart_series.py index 04964d468..f2fb23904 100644 --- a/tests/test_archived_chart_series.py +++ b/tests/test_archived_chart_series.py @@ -5,7 +5,7 @@ from condor.archived_chart_series import ( activity_range, build_chart_series, - pick_interval, + pick_candle_interval, ) from condor.archived_pnl import calculate_pnl_from_executors @@ -41,11 +41,11 @@ def _ex( def test_interval_matches_run_length_not_archive_lag(): """A 28-minute run charts at 1m, however long ago it was archived.""" - assert pick_interval(28 * 60).name == "1m" - assert pick_interval(6 * 3600).name == "5m" - assert pick_interval(2 * 86400).name == "15m" - assert pick_interval(7 * 86400).name == "1h" - assert pick_interval(30 * 86400).name == "4h" + assert pick_candle_interval(28 * 60).name == "1m" + assert pick_candle_interval(6 * 3600).name == "5m" + assert pick_candle_interval(2 * 86400).name == "15m" + assert pick_candle_interval(7 * 86400).name == "1h" + assert pick_candle_interval(30 * 86400).name == "4h" def test_activity_range_spans_first_open_to_last_close(): diff --git a/tests/test_run_history_fetch.py b/tests/test_run_history_fetch.py index c3300057b..b7e5a9e92 100644 --- a/tests/test_run_history_fetch.py +++ b/tests/test_run_history_fetch.py @@ -157,7 +157,8 @@ def test_a_run_that_declared_nothing_records_the_interval_it_actually_asked_for( client = FakeClient(TWO) history = asyncio.run(_fetch(client, controller_ids=())) - assert rh.pick_interval(90 * 3_600_000) == "15m" # what the span alone says + # what the span alone says + assert rh.pick_sampling_interval(90 * 3_600_000) == "15m" assert {c["interval"] for c in client.calls} == {"5m"} assert history.interval == "5m" @@ -369,12 +370,12 @@ def test_the_interval_ladder_only_ever_offers_values_upstream_accepts(): a value outside it turns a chart into an error rather than a coarser chart.""" accepted = {"5m", "15m", "30m", "1h", "4h", "12h", "1d"} for hours in (0, 1, 24, 24 * 30, 24 * 365, 24 * 3650): - assert rh.pick_interval(hours * 3_600_000) in accepted + assert rh.pick_sampling_interval(hours * 3_600_000) in accepted def test_an_unknown_span_falls_back_to_the_finest_interval(): - assert rh.pick_interval(0) == "5m" - assert rh.pick_interval(-1) == "5m" + assert rh.pick_sampling_interval(0) == "5m" + assert rh.pick_sampling_interval(-1) == "5m" # ── The archive fallback, for a run older than the snapshot table ── From 2245acf4cb1c7e87da3dc702b342a0faa121deba Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 08:19:26 +0300 Subject: [PATCH 033/154] Let condor/web declare its dependency on the data facade at the top of the file The same deferred import of condor.server_data_service was repeated inside function bodies 24 times across condor/web, with no comment saying what cycle it was breaking -- because there is none. The SDS's only module-level first-party import is condor.asyncutil, so importing it at module scope from the web layer cannot close a loop. The cost was that grep '^from condor' on condor/web/streams/hummingbot_ws.py returned nothing at all: a reader had to walk every function body to learn the module is an SDS consumer, and the genuine circular-import deferrals elsewhere were indistinguishable from copy-paste. Hoist all of them, plus the two condor.fetchers.portfolio copies in ws_manager. ws_manager's TYPE_CHECKING import of CacheKey goes with them: the runtime import now covers the annotation. Tests that faked the service were patching get_server_data_service on the module that defines it, which a module-level `from ... import` no longer routes through. They now patch the consumer that binds the name -- the route module, ws_manager, or the streams mixin that owns the executor pre-fetch -- which is also the more honest target, since that is the binding the code under test actually reads. No behaviour change. --- condor/web/routes/bots.py | 5 +--- condor/web/routes/executors.py | 6 ++--- condor/web/routes/market.py | 10 +------- condor/web/routes/portfolio.py | 5 +--- condor/web/routes/positions.py | 3 +-- condor/web/routes/settings.py | 7 +----- condor/web/streams/hummingbot_ws.py | 19 ++------------- condor/web/ws_manager.py | 27 +++------------------ tests/test_executor_read_errors.py | 2 +- tests/test_executors_poll_bound.py | 2 +- tests/test_fetcher_identifier_validation.py | 8 ++---- tests/test_fetcher_venues.py | 3 +-- tests/test_pagination_cursor_progress.py | 4 +-- tests/test_portfolio_history_sds.py | 5 ++-- tests/test_portfolio_unified_dedupe.py | 2 +- tests/test_web_credentials_owner_only.py | 2 +- tests/test_ws_executor_prefetch_linear.py | 4 +-- 17 files changed, 25 insertions(+), 89 deletions(-) diff --git a/condor/web/routes/bots.py b/condor/web/routes/bots.py index 1ac8c2a87..25196fe5d 100644 --- a/condor/web/routes/bots.py +++ b/condor/web/routes/bots.py @@ -10,6 +10,7 @@ from condor.controller_configs import clean_config_for_save from condor.fetchers.bots import BotsEnrichment, build_bots_page, extract_bots_list +from condor.server_data_service import ServerDataType, get_server_data_service from condor.web.auth import require_server_access from condor.web.models import ( AvailableControllersResponse, @@ -194,8 +195,6 @@ async def enriched_bots_page(name: str, raw_status: Any) -> dict: minute: the 5s bots frame costs an extra Hummingbot round-trip only when that cached answer has gone stale. """ - from condor.server_data_service import ServerDataType, get_server_data_service - try: enrichment = await get_server_data_service().get_or_fetch( name, ServerDataType.BOTS_ENRICHMENT @@ -215,8 +214,6 @@ async def enriched_bots_page(name: str, raw_status: Any) -> dict: @router.get("/servers/{name}/bots", response_model=BotsPageResponse) async def list_bots(name: str, user: WebUser = Depends(require_server_access)): - from condor.server_data_service import ServerDataType, get_server_data_service - try: result = await get_server_data_service().get_or_fetch( name, ServerDataType.BOTS_STATUS diff --git a/condor/web/routes/executors.py b/condor/web/routes/executors.py index c9011fd99..120ca7339 100644 --- a/condor/web/routes/executors.py +++ b/condor/web/routes/executors.py @@ -5,6 +5,8 @@ from fastapi import APIRouter, Depends, HTTPException, Query +from condor.server_data_service import ServerDataType, get_server_data_service + logger = logging.getLogger(__name__) @@ -60,8 +62,6 @@ async def list_executors( ): cm = get_config_manager() - from condor.server_data_service import ServerDataType, get_server_data_service - # For filtered queries or when a custom limit is requested, go direct to API. # For unfiltered default requests, use the SDS cache. if executor_type or trading_pair or status or controller_id or limit: @@ -154,8 +154,6 @@ async def list_executors_page( offset = int(cursor[len(_SDS_OFFSET_PREFIX) :] or 0) if offset is not None: - from condor.server_data_service import ServerDataType, get_server_data_service - cached = get_server_data_service().get(name, ServerDataType.EXECUTORS) cached_rows = _extract_executors_list(cached) if cached is not None else [] # The poll caps its walk at EXECUTORS_POLL_MAX, so a cache of exactly diff --git a/condor/web/routes/market.py b/condor/web/routes/market.py index a94e92dd1..6452ca2fb 100644 --- a/condor/web/routes/market.py +++ b/condor/web/routes/market.py @@ -7,6 +7,7 @@ from condor import dex_candles from condor.asyncutil import SingleFlight +from condor.server_data_service import ServerDataType, get_server_data_service from config_manager import get_config_manager logger = logging.getLogger(__name__) @@ -83,8 +84,6 @@ async def _fetch_dex_candles( @router.get("/servers/{name}/market/connectors") async def get_connectors(name: str, user: WebUser = Depends(require_server_access)): - from condor.server_data_service import ServerDataType, get_server_data_service - try: result = await get_server_data_service().get_or_fetch( name, ServerDataType.CANDLE_CONNECTORS @@ -101,8 +100,6 @@ async def get_connected_exchanges( ): """Get connectors that have credentials configured (accounts connected).""" - from condor.server_data_service import ServerDataType, get_server_data_service - try: result = await get_server_data_service().get_or_fetch( name, ServerDataType.CONNECTORS @@ -140,8 +137,6 @@ async def get_venues(name: str, user: WebUser = Depends(require_server_access)): inside the fetcher and still reports the venues the other sources supplied. """ - from condor.server_data_service import ServerDataType, get_server_data_service - try: result = await get_server_data_service().get_or_fetch( name, ServerDataType.VENUES @@ -224,8 +219,6 @@ async def get_price( user: WebUser = Depends(require_server_access), ): - from condor.server_data_service import ServerDataType, get_server_data_service - try: result = await get_server_data_service().get_or_fetch( name, @@ -288,7 +281,6 @@ async def get_trading_rules( ): from condor.fetchers._identifiers import IdentifierError, validate_identifier - from condor.server_data_service import ServerDataType, get_server_data_service # Rejected here, not in the fetcher: SDS records a failed fetch as an # error-only cache entry, so a bad connector would still mint a key. diff --git a/condor/web/routes/portfolio.py b/condor/web/routes/portfolio.py index 6d0b3fc27..7a9e6bd99 100644 --- a/condor/web/routes/portfolio.py +++ b/condor/web/routes/portfolio.py @@ -12,6 +12,7 @@ balance_value, dedupe_unified_accounts, ) +from condor.server_data_service import ServerDataType, get_server_data_service from condor.web.auth import require_server_access from condor.web.models import ( BalanceItem, @@ -36,8 +37,6 @@ async def get_portfolio( ): cm = get_config_manager() - from condor.server_data_service import ServerDataType, get_server_data_service - try: if refresh: # Bypass SDS cache — force exchange re-fetch via Hummingbot @@ -149,8 +148,6 @@ async def get_portfolio_history( user: WebUser = Depends(require_server_access), ): - from condor.server_data_service import ServerDataType, get_server_data_service - # The raw snapshots are the same regardless of *breakdown* (parsing happens # below), so the cache key is the range alone. SDS coalesces concurrent # reads of the same expired key into one backend fetch. diff --git a/condor/web/routes/positions.py b/condor/web/routes/positions.py index 34002489f..4704692fa 100644 --- a/condor/web/routes/positions.py +++ b/condor/web/routes/positions.py @@ -6,6 +6,7 @@ from fastapi import APIRouter, Depends from condor.fetchers.market_data import fetch_current_price +from condor.server_data_service import ServerDataType, get_server_data_service from condor.web.auth import require_server_access from condor.web.models import WebUser from config_manager import get_config_manager @@ -60,8 +61,6 @@ async def get_consolidated_positions( ): cm = get_config_manager() - from condor.server_data_service import ServerDataType, get_server_data_service - # Fetch executor positions and bot data in parallel async def fetch_executor_positions(): try: diff --git a/condor/web/routes/settings.py b/condor/web/routes/settings.py index 6031513a6..d61b48574 100644 --- a/condor/web/routes/settings.py +++ b/condor/web/routes/settings.py @@ -6,6 +6,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query +from condor.server_data_service import ServerDataType, get_server_data_service from condor.web.auth import ( get_current_user, require_owner, @@ -626,8 +627,6 @@ async def list_connectors( user: WebUser = Depends(require_server_access_query), ): - from condor.server_data_service import ServerDataType, get_server_data_service - sds = get_server_data_service() raw = await sds.get_or_fetch(server, ServerDataType.ALL_CONNECTORS) if raw is None: @@ -692,8 +691,6 @@ async def add_credential( credentials=req.credentials, ) # Invalidate configured connectors cache - from condor.server_data_service import ServerDataType, get_server_data_service - get_server_data_service().invalidate(server, ServerDataType.CONNECTORS) return {"added": True, "result": result} except Exception as e: @@ -719,8 +716,6 @@ async def delete_credential( connector_name=connector, ) # Invalidate configured connectors + portfolio caches so the removed key disappears immediately - from condor.server_data_service import ServerDataType, get_server_data_service - sds = get_server_data_service() sds.invalidate(server, ServerDataType.CONNECTORS) sds.invalidate(server, ServerDataType.PORTFOLIO) diff --git a/condor/web/streams/hummingbot_ws.py b/condor/web/streams/hummingbot_ws.py index 0c0b46a86..3135c62af 100644 --- a/condor/web/streams/hummingbot_ws.py +++ b/condor/web/streams/hummingbot_ws.py @@ -18,6 +18,8 @@ import aiohttp +from condor.server_data_service import ServerDataType, get_server_data_service + logger = logging.getLogger(__name__) # What a stream should do about the error that broke it. @@ -372,8 +374,6 @@ async def _executor_stream(self, channel: str) -> None: cm = get_config_manager() # Try SDS cache first (pre-warmed by auto_subscribe_servers or REST prefetch) - from condor.server_data_service import ServerDataType, get_server_data_service - if channel not in self._last_data: sds = get_server_data_service() cached = sds.get(server_name, ServerDataType.EXECUTORS) @@ -493,11 +493,6 @@ async def _bots_ws_stream(self, channel: str) -> None: # Send SDS-cached bots data as initial snapshot if channel not in self._last_data: - from condor.server_data_service import ( - ServerDataType, - get_server_data_service, - ) - sds = get_server_data_service() cached = sds.get(server_name, ServerDataType.BOTS_STATUS) if cached is not None: @@ -515,11 +510,6 @@ async def on_message(msg: dict) -> None: return raw_data = msg.get("data", {}) # Update SDS cache so REST and Telegram benefit - from condor.server_data_service import ( - ServerDataType, - get_server_data_service, - ) - get_server_data_service().put( server_name, ServerDataType.BOTS_STATUS, raw_data ) @@ -551,11 +541,6 @@ async def on_message(msg: dict) -> None: return raw_data = msg.get("data", []) # Update SDS cache - from condor.server_data_service import ( - ServerDataType, - get_server_data_service, - ) - get_server_data_service().put( server_name, ServerDataType.POSITIONS, raw_data ) diff --git a/condor/web/ws_manager.py b/condor/web/ws_manager.py index d1b57a644..bb5684c35 100644 --- a/condor/web/ws_manager.py +++ b/condor/web/ws_manager.py @@ -13,7 +13,7 @@ import json import logging import time -from typing import TYPE_CHECKING, Any, Optional +from typing import Any, Optional from fastapi import WebSocket @@ -21,6 +21,8 @@ # (and patch ``ws_manager.dex_candles``) through this module. from condor import dex_candles # noqa: F401 from condor.asyncutil import TaskSet +from condor.fetchers.portfolio import PORTFOLIO_HISTORY_RANGES +from condor.server_data_service import CacheKey, ServerDataType, get_server_data_service from condor.web.auth import decode_jwt from condor.web.streams.candles import ( # noqa: F401 CandleStreamsMixin, @@ -30,9 +32,6 @@ ) from condor.web.streams.hummingbot_ws import HummingbotStreamsMixin -if TYPE_CHECKING: - from condor.server_data_service import CacheKey - logger = logging.getLogger(__name__) # Mapping from WS channel prefix to ServerDataType @@ -157,8 +156,6 @@ def _server_from_channel(channel: str) -> str | None: def start(self) -> None: if self._sds_listener_registered: return - from condor.server_data_service import get_server_data_service - sds = get_server_data_service() sds.add_listener(self._on_data_update) self._sds_listener_registered = True @@ -169,8 +166,6 @@ def start(self) -> None: def stop(self) -> None: if self._sds_listener_registered: - from condor.server_data_service import get_server_data_service - sds = get_server_data_service() sds.remove_listener(self._on_data_update) self._sds_listener_registered = False @@ -205,8 +200,6 @@ def stop(self) -> None: def _cleanup_sds_subscriptions(self) -> None: """Remove all SDS subscriptions.""" - from condor.server_data_service import get_server_data_service - sds = get_server_data_service() sds.unsubscribe_all("ws_manager") self._sds_subscriptions.clear() @@ -257,8 +250,6 @@ def _maybe_unsub_sds(self, channel: str) -> None: self._last_data.pop(channel, None) if channel in self._sds_subscriptions: - from condor.server_data_service import get_server_data_service - sds = get_server_data_service() cache_key = self._sds_subscriptions.pop(channel) sds.unsubscribe(cache_key, "ws_manager") @@ -350,8 +341,6 @@ async def _subscribe_sds(self, channel: str) -> None: if not sdt_name: return - from condor.server_data_service import ServerDataType, get_server_data_service - sds = get_server_data_service() data_type = ServerDataType[sdt_name] @@ -402,9 +391,6 @@ async def _subscribe_portfolio_history( while the priming is in flight, unsubscribe again — otherwise the poll would outlive its last subscriber. """ - from condor.fetchers.portfolio import PORTFOLIO_HISTORY_RANGES - from condor.server_data_service import ServerDataType, get_server_data_service - sds = get_server_data_service() async def _sub(range_key: str) -> None: @@ -434,13 +420,6 @@ def _unsubscribe_portfolio_history(self, server_name: str) -> None: SDS stops polling a key once it has no subscribers left, so this is what ends the history refresh. """ - from condor.fetchers.portfolio import PORTFOLIO_HISTORY_RANGES - from condor.server_data_service import ( - CacheKey, - ServerDataType, - get_server_data_service, - ) - sds = get_server_data_service() for range_key in PORTFOLIO_HISTORY_RANGES: sds.unsubscribe( diff --git a/tests/test_executor_read_errors.py b/tests/test_executor_read_errors.py index 4221ef37a..3ac8d19b4 100644 --- a/tests/test_executor_read_errors.py +++ b/tests/test_executor_read_errors.py @@ -115,7 +115,7 @@ def _bind(exc): lambda: _FakeCM(FakeClient(exc)), ) monkeypatch.setattr( - "condor.server_data_service.get_server_data_service", + "condor.web.routes.executors.get_server_data_service", lambda: _FakeSDS(exc), ) diff --git a/tests/test_executors_poll_bound.py b/tests/test_executors_poll_bound.py index cb5628fcc..b518273d2 100644 --- a/tests/test_executors_poll_bound.py +++ b/tests/test_executors_poll_bound.py @@ -125,7 +125,7 @@ def page_env(monkeypatch): """Wire ``list_executors_page`` to a fresh SDS and a fake API client.""" sds = ServerDataService() monkeypatch.setattr( - "condor.server_data_service.get_server_data_service", lambda: sds + "condor.web.routes.executors.get_server_data_service", lambda: sds ) def _bind(client): diff --git a/tests/test_fetcher_identifier_validation.py b/tests/test_fetcher_identifier_validation.py index af7f03ae7..224622c93 100644 --- a/tests/test_fetcher_identifier_validation.py +++ b/tests/test_fetcher_identifier_validation.py @@ -209,9 +209,7 @@ async def _fake_get_client(server_name): sds._get_client = _fake_get_client sds.register_fetch(ServerDataType.TRADING_RULES, _spy_fetch) - monkeypatch.setattr( - "condor.server_data_service.get_server_data_service", lambda: sds - ) + monkeypatch.setattr("condor.web.routes.market.get_server_data_service", lambda: sds) resp = _client(monkeypatch).get( "/servers/srv/market/trading-rules", params={"connector": payload} @@ -235,9 +233,7 @@ async def _fake_get_client(server_name): sds._get_client = _fake_get_client sds.register_fetch(ServerDataType.TRADING_RULES, _fetch) - monkeypatch.setattr( - "condor.server_data_service.get_server_data_service", lambda: sds - ) + monkeypatch.setattr("condor.web.routes.market.get_server_data_service", lambda: sds) resp = _client(monkeypatch).get( "/servers/srv/market/trading-rules", params={"connector": "binance"} diff --git a/tests/test_fetcher_venues.py b/tests/test_fetcher_venues.py index 92bcd05f5..1981bc34c 100644 --- a/tests/test_fetcher_venues.py +++ b/tests/test_fetcher_venues.py @@ -586,12 +586,11 @@ async def get_or_fetch(self, name, data_type, **params): def _call_route(monkeypatch, sds): - from condor import server_data_service from condor.web.models import WebUser from condor.web.routes.market import get_venues monkeypatch.setattr( - server_data_service, "get_server_data_service", lambda: sds, raising=True + "condor.web.routes.market.get_server_data_service", lambda: sds, raising=True ) class _Cm: diff --git a/tests/test_pagination_cursor_progress.py b/tests/test_pagination_cursor_progress.py index 020124595..62f033869 100644 --- a/tests/test_pagination_cursor_progress.py +++ b/tests/test_pagination_cursor_progress.py @@ -77,7 +77,7 @@ def put(self, server_name, data_type, value): def test_ws_executor_prefetch_stops_when_the_cursor_does_not_advance(monkeypatch): """The pre-fetch must not re-page, or duplicates land in the SDS cache.""" - import condor.server_data_service as sds_module + import condor.web.streams.hummingbot_ws as executor_stream_module import config_manager as cm_module client = EchoingCursorClient( @@ -90,7 +90,7 @@ class _FakeCM: async def get_client(self, server_name): return client - monkeypatch.setattr(sds_module, "get_server_data_service", lambda: sds) + monkeypatch.setattr(executor_stream_module, "get_server_data_service", lambda: sds) monkeypatch.setattr(cm_module, "get_config_manager", lambda: _FakeCM()) manager = WebSocketManager() diff --git a/tests/test_portfolio_history_sds.py b/tests/test_portfolio_history_sds.py index 65a3229a8..708b9da65 100644 --- a/tests/test_portfolio_history_sds.py +++ b/tests/test_portfolio_history_sds.py @@ -111,9 +111,8 @@ def env(monkeypatch): def _bind(delay: float = 0.0): client = FakeClient(delay=delay) sds = _make_sds(client) - monkeypatch.setattr( - "condor.server_data_service.get_server_data_service", lambda: sds - ) + for module in ("condor.web.routes.portfolio", "condor.web.ws_manager"): + monkeypatch.setattr(f"{module}.get_server_data_service", lambda: sds) monkeypatch.setattr( "condor.web.routes.portfolio.get_config_manager", lambda: _FakeCM() ) diff --git a/tests/test_portfolio_unified_dedupe.py b/tests/test_portfolio_unified_dedupe.py index 9358f90d9..82968df12 100644 --- a/tests/test_portfolio_unified_dedupe.py +++ b/tests/test_portfolio_unified_dedupe.py @@ -138,7 +138,7 @@ async def get_or_fetch(self, name, data_type, **kwargs): def web_portfolio(monkeypatch): async def _call(state): monkeypatch.setattr( - "condor.server_data_service.get_server_data_service", + "condor.web.routes.portfolio.get_server_data_service", lambda: _FakeSDS(state), ) return await get_portfolio(SERVER, refresh=False, user=_USER) diff --git a/tests/test_web_credentials_owner_only.py b/tests/test_web_credentials_owner_only.py index 3890c706c..ec4240483 100644 --- a/tests/test_web_credentials_owner_only.py +++ b/tests/test_web_credentials_owner_only.py @@ -89,7 +89,7 @@ def env(monkeypatch): monkeypatch.setattr(settings_routes, "get_config_manager", lambda: cm) monkeypatch.setattr("condor.web.auth.get_config_manager", lambda: cm) monkeypatch.setattr( - "condor.server_data_service.get_server_data_service", lambda: sds + "condor.web.routes.settings.get_server_data_service", lambda: sds ) app = FastAPI() app.include_router(settings_routes.router) diff --git a/tests/test_ws_executor_prefetch_linear.py b/tests/test_ws_executor_prefetch_linear.py index 47cc9509a..51ed25a3d 100644 --- a/tests/test_ws_executor_prefetch_linear.py +++ b/tests/test_ws_executor_prefetch_linear.py @@ -18,7 +18,7 @@ import pytest -import condor.server_data_service as sds_module +import condor.web.streams.hummingbot_ws as executor_stream_module import config_manager as cm_module from condor.web.models import ExecutorInfo from condor.web.ws_manager import WebSocketManager @@ -93,7 +93,7 @@ class _FakeCM: async def get_client(self, server_name): return client - monkeypatch.setattr(sds_module, "get_server_data_service", lambda: sds) + monkeypatch.setattr(executor_stream_module, "get_server_data_service", lambda: sds) monkeypatch.setattr(cm_module, "get_config_manager", lambda: _FakeCM()) # Count every raw row pushed through the pydantic transform. This is the From a983557bb484103bcb4c5e8998283025f39708f6 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 08:25:30 +0300 Subject: [PATCH 034/154] Delete the tool filter mode that never filtered a tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `tool_filter_mode` was computed by a 70-line size-and-family heuristic, logged confidently ("Auto-detected 4.0B model -> tool_filter_mode=essential"), threaded through preferences, sessions, the agent engine and the client factory — and then never read. The only access to `self.tool_filter_mode` in the repo was the assignment that created it. That made it worse than useless: `preferences.py` defaulted every agent to "essential", `llm_client` documented a precedence order over a `PYDANTIC_AI_TOOL_FILTER` env var, and a reader reasonably concluded small models were being handed a reduced toolset. They were not. The module's one real tool-scoping seam is `allowed_tools` via `_prepare_tools`, which stays. Removing it changes no behaviour: there was none to change. --- condor/acp/pydantic_ai_client.py | 76 -------------------------------- condor/agents/agent.py | 2 +- condor/agents/engine.py | 6 +-- condor/preferences.py | 2 - condor/runtime/llm_client.py | 8 ---- condor/runtime/sessions.py | 1 - tests/test_custom_provider.py | 9 ---- 7 files changed, 3 insertions(+), 101 deletions(-) diff --git a/condor/acp/pydantic_ai_client.py b/condor/acp/pydantic_ai_client.py index 95e2a978d..2430c3100 100644 --- a/condor/acp/pydantic_ai_client.py +++ b/condor/acp/pydantic_ai_client.py @@ -34,77 +34,6 @@ log = logging.getLogger(__name__) -def _infer_tool_filter_mode(model_name: str) -> str: - """Automatically detect the best tool filter mode based on model name. - - Analyzes model size and family to determine capability: - - Small models (≤8B): essential (minimal tools) - - Medium models (9B-32B): moderate (common operations) - - Large models (>32B) or cloud APIs: full (all tools) - - Args: - model_name: Model identifier like "ollama:llama3.1:8b" or "lmstudio:qwen-14b" - - Returns: - "essential", "moderate", or "full" - """ - import re - - model_lower = model_name.lower() - - # Cloud providers always get full access (they're powerful enough). - # Custom endpoints are deliberately absent: "custom:" says nothing about - # the model behind it — it's just as likely a 4B model on a local vLLM as - # a frontier model on Together — so those fall through to the size - # heuristics below like any other unknown backend. - if any( - provider in model_lower - for provider in [ - "openai:", - "anthropic:", - "groq:", - "google:", - "openrouter:", - ] - ): - log.info("Auto-detected cloud provider → tool_filter_mode=full") - return "full" - - # Extract parameter count (e.g., "7b", "14b", "72b", "32b") - # Matches patterns like: 7b, 8b, 14b, 32b, 72b, 1.5b, 2.7b, etc. - size_match = re.search(r"(\d+(?:\.\d+)?)\s*[bB](?![a-z])", model_lower) - - if size_match: - size = float(size_match.group(1)) - - if size <= 8.0: - mode = "essential" - log.info(f"Auto-detected {size}B model → tool_filter_mode=essential") - elif size <= 32.0: - mode = "moderate" - log.info(f"Auto-detected {size}B model → tool_filter_mode=moderate") - else: - mode = "full" - log.info(f"Auto-detected {size}B model → tool_filter_mode=full") - - return mode - - # Model name-based heuristics (if no size found) - # Small models - if any(name in model_lower for name in ["gemma", "phi", "tiny", "mini", "small"]): - log.info(f"Auto-detected small model family → tool_filter_mode=essential") - return "essential" - - # Large models - if any(name in model_lower for name in ["deepseek", "mixtral", "command-r", "gpt"]): - log.info(f"Auto-detected large model family → tool_filter_mode=full") - return "full" - - # Default to moderate for unknown models - log.info(f"Unknown model size, defaulting → tool_filter_mode=moderate") - return "moderate" - - # Model prefix → pydantic-ai model string mapping # Users set agent_key like "ollama:llama3.1:70b" or "openai:gpt-4o" # which maps directly to pydantic-ai model identifiers. @@ -456,9 +385,6 @@ def __init__( extra_env: dict[str, str] | None = None, base_url: str | None = None, api_key: str | None = None, - tool_filter_mode: ( - str | None - ) = None, # "essential", "moderate", "full", or None for auto-detect allowed_tools: ( list[str] | None ) = None, # restrict the agent to these tool names @@ -478,8 +404,6 @@ def __init__( # When set, the agent only sees tools whose name is in this allowlist # (used by delegated domain agents to scope an agent to one domain). self.allowed_tools = set(allowed_tools) if allowed_tools else None - # Auto-detect filter mode based on model if not explicitly set - self.tool_filter_mode = tool_filter_mode or _infer_tool_filter_mode(model) self._mcp_servers: list[Any] = [] self._agent: Any = None # Carries each tool call's permission decision from prompt_stream (where diff --git a/condor/agents/agent.py b/condor/agents/agent.py index 88ab6bcbf..5a5485431 100644 --- a/condor/agents/agent.py +++ b/condor/agents/agent.py @@ -85,7 +85,7 @@ class Agent: agent_key: str = "" # default model (pydantic-ai or ACP, e.g. "claude-code") # Tool-name allowlist (pydantic-ai only), enforced on BOTH delegate and loop. # Names match full (``mcp__condor__manage_skill``) or short (``manage_skill``). - # Empty => UNRESTRICTED (all discovered tools, subject to tool_filter_mode). + # Empty => UNRESTRICTED (all discovered tools). tools: list[str] = field(default_factory=list) # Optional one-line routing hint ("when should condor pick this agent?"). # NOT a capability switch — see consult_hint and the module docstring. diff --git a/condor/agents/engine.py b/condor/agents/engine.py index b8f3dc886..1b95cd48d 100644 --- a/condor/agents/engine.py +++ b/condor/agents/engine.py @@ -1040,9 +1040,8 @@ async def _create_client( ) # Shared factory (ARCH-192). Engine specifics: an explicit model_base_url - # in the run config still wins over the owner's saved custom endpoint, - # and the run config's tool_filter_mode beats the env fallback. Same - # allowlist the agent gets when delegated to; empty => unrestricted. + # in the run config still wins over the owner's saved custom endpoint. + # Same allowlist the agent gets when delegated to; empty => unrestricted. from condor.runtime.llm_client import build_llm_client return build_llm_client( @@ -1052,7 +1051,6 @@ async def _create_client( allowed_tools=self.agent.tools or None, user_id=self.user_id, base_url_override=self.config.get("model_base_url") or None, - tool_filter_mode=self.config.get("tool_filter_mode"), ) # ------------------------------------------------------------------ diff --git a/condor/preferences.py b/condor/preferences.py index 0acb391df..1a740cf6d 100644 --- a/condor/preferences.py +++ b/condor/preferences.py @@ -298,7 +298,6 @@ class ChatBindingPrefs(TypedDict, total=False): class AgentPrefs(TypedDict, total=False): default_agent: str # "claude-code", "gemini", "codex", "copilot" show_tool_calls: bool # Show tool call indicators (default True) - tool_filter_mode: str # "essential", "moderate", or "full" for PydanticAI models custom_providers: List[CustomProviderPrefs] # OpenAI-compatible endpoints chat_binding: ChatBindingPrefs # who this chat talks to across respawns # Mirror of the live chat selection (user_data["agent_llm"]). Kept here so @@ -1030,7 +1029,6 @@ def get_agent_prefs(user_data: Dict) -> "AgentPrefs": { "default_agent": "claude-code", "show_tool_calls": True, - "tool_filter_mode": "essential", }, ) ) diff --git a/condor/runtime/llm_client.py b/condor/runtime/llm_client.py index d826148e8..af6ff594f 100644 --- a/condor/runtime/llm_client.py +++ b/condor/runtime/llm_client.py @@ -21,7 +21,6 @@ from __future__ import annotations -import os from typing import Any from condor.acp import client as acp_client @@ -42,7 +41,6 @@ def build_llm_client( user_id: int | None = None, base_url_override: str | None = None, default_base_url: str | None = None, - tool_filter_mode: str | None = None, strict_custom_endpoint: bool = False, ) -> acp_client.ACPClient | pydantic_ai.PydanticAIClient: """Build (but do not start) the right client for ``agent_key``. @@ -59,9 +57,6 @@ def build_llm_client( beats ``default_base_url`` (a generic preference, e.g. the saved LM Studio URL — deliberately last so it cannot shadow a named custom endpoint). - ``tool_filter_mode`` resolves as: explicit value (a user/config pref) > - ``PYDANTIC_AI_TOOL_FILTER`` env > ``None`` (auto-detect by model size). - ``extra_env``, ``system_prompt`` and ``allowed_tools`` are forwarded to whichever client understands them. Both clients take the env and the system prompt — each over its own system-level channel (``_meta.systemPrompt`` for @@ -82,9 +77,6 @@ def build_llm_client( extra_env=extra_env, base_url=base_url_override or custom_url or default_base_url or None, api_key=api_key, - tool_filter_mode=( - tool_filter_mode or os.environ.get("PYDANTIC_AI_TOOL_FILTER") or None - ), allowed_tools=allowed_tools, system_prompt=system_prompt, ) diff --git a/condor/runtime/sessions.py b/condor/runtime/sessions.py index bcca61288..19c76d788 100644 --- a/condor/runtime/sessions.py +++ b/condor/runtime/sessions.py @@ -744,7 +744,6 @@ async def _spawn_session( default_base_url=( agent_prefs.get("base_url") or os.environ.get("LMSTUDIO_BASE_URL") or None ), - tool_filter_mode=agent_prefs.get("tool_filter_mode"), strict_custom_endpoint=True, ) diff --git a/tests/test_custom_provider.py b/tests/test_custom_provider.py index 40ab31ee2..83857f95f 100644 --- a/tests/test_custom_provider.py +++ b/tests/test_custom_provider.py @@ -13,7 +13,6 @@ from condor.acp.pydantic_ai_client import ( PydanticAIClient, - _infer_tool_filter_mode, is_pydantic_ai_model, model_prefix, ) @@ -67,14 +66,6 @@ def test_model_prefix_strips_endpoint_name(): assert model_prefix("claude-code") == "" -def test_custom_tool_filter_follows_the_model_not_the_prefix(): - # "custom:" says nothing about capability — a 4B model behind a local vLLM - # would choke on the full toolset, so the size heuristics still apply. - assert _infer_tool_filter_mode("custom:qwen3-4b") == "essential" - assert _infer_tool_filter_mode("custom@local:qwen3-14b") == "moderate" - assert _infer_tool_filter_mode("custom@together:llama-3.3-70b") == "full" - - # -- agent key composition -- From f4d52f5872bb124be47605329d9c01f60e5ebd39 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 08:30:15 +0300 Subject: [PATCH 035/154] Decide once how an optional query parameter is skipped and stringified MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit api.ts already had a generic query-string builder inside fetchPerformanceHistoryPage, and five other endpoints re-enumerated their own parameter names by hand two hundred lines away from it. The cost was never the line count: each name was written twice, once in the TypeScript param type and once in a qs.set line, so adding a field to the type and forgetting the setter compiled cleanly and dropped the parameter from the request — a filter that looks applied and is not. Promote that loop to a module-level query() that returns the leading "?" or an empty string, and route fetchControllerPerformanceHistoryPage, getBotRuns, getExecutors, getExecutorsPage and getReports through it. getRunHistory and getDexPools build required parameters through the URLSearchParams constructor and stay as they are. api.query.test.ts pins the emitted URL for each of them against a mixed set of present, empty-string and undefined parameters, and against the bare path with no "?" when nothing is set. --- frontend/src/lib/api.query.test.ts | 156 +++++++++++++++++++++++++++++ frontend/src/lib/api.ts | 78 ++++++--------- 2 files changed, 185 insertions(+), 49 deletions(-) create mode 100644 frontend/src/lib/api.query.test.ts diff --git a/frontend/src/lib/api.query.test.ts b/frontend/src/lib/api.query.test.ts new file mode 100644 index 000000000..320b4dfd9 --- /dev/null +++ b/frontend/src/lib/api.query.test.ts @@ -0,0 +1,156 @@ +/** + * @vitest-environment jsdom + * + * The URLs the optional-parameter endpoints actually put on the wire (READ-333). + * + * Each of these used to spell its parameter names twice — once in its + * TypeScript type and once in a `qs.set` line — so a field added to the type + * and forgotten in the setter compiled cleanly and was silently dropped from + * the request: a filter that looks applied and is not. They now share one + * `query()` builder, and this file pins what that builder emits for a mixed set + * of present, empty-string and undefined parameters, plus the bare path with no + * `?` when nothing is set. + */ + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { api } from "./api"; + +const SERVER = "prod"; + +/** Answer every request with `body` and record the URLs requested. */ +function serve(body: unknown = {}) { + const urls: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string) => { + urls.push(url); + return { ok: true, json: async () => body } as unknown as Response; + }), + ); + return urls; +} + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe("getBotRuns", () => { + it("keeps set parameters and drops undefined and empty ones", async () => { + const urls = serve({ bot_runs: [], total: 0 }); + await api.getBotRuns(SERVER, { + bot_name: "bot-a", + run_status: "", + deployment_status: undefined, + limit: 50, + }); + expect(urls[0]).toBe( + "/api/v1/servers/prod/bot-runs?bot_name=bot-a&limit=50", + ); + }); + + it("omits the ? entirely when nothing is set", async () => { + const urls = serve({ bot_runs: [], total: 0 }); + await api.getBotRuns(SERVER); + expect(urls[0]).toBe("/api/v1/servers/prod/bot-runs"); + }); +}); + +describe("getExecutors", () => { + it("keeps set parameters and drops undefined and empty ones", async () => { + const urls = serve([]); + await api.getExecutors(SERVER, { + executor_type: "position_executor", + trading_pair: "", + status: undefined, + controller_id: "ctrl-1", + limit: 200, + }); + expect(urls[0]).toBe( + "/api/v1/servers/prod/executors?executor_type=position_executor&controller_id=ctrl-1&limit=200", + ); + }); + + it("omits the ? entirely when called with no parameters", async () => { + const urls = serve([]); + await api.getExecutors(SERVER); + expect(urls[0]).toBe("/api/v1/servers/prod/executors"); + }); +}); + +describe("getExecutorsPage", () => { + it("always carries a limit, defaulting to 50", async () => { + const urls = serve({ executors: [], next_cursor: null }); + await api.getExecutorsPage(SERVER, { status: "", trading_pair: "SOL-USDC" }); + expect(urls[0]).toBe( + "/api/v1/servers/prod/executors/page?trading_pair=SOL-USDC&limit=50", + ); + }); + + it("carries the cursor and an explicit limit when given them", async () => { + const urls = serve({ executors: [], next_cursor: null }); + await api.getExecutorsPage(SERVER, { + cursor: "cur-1", + limit: 200, + controller_id: undefined, + }); + expect(urls[0]).toBe( + "/api/v1/servers/prod/executors/page?cursor=cur-1&limit=200", + ); + }); +}); + +describe("getReports", () => { + it("keeps set parameters and drops undefined and empty ones", async () => { + const urls = serve({ reports: [], total: 0 }); + await api.getReports({ + source_type: "routine", + tag: "", + search: undefined, + offset: 20, + }); + expect(urls[0]).toBe("/api/v1/reports?source_type=routine&offset=20"); + }); + + it("omits the ? entirely when called with no parameters", async () => { + const urls = serve({ reports: [], total: 0 }); + await api.getReports(); + expect(urls[0]).toBe("/api/v1/reports"); + }); +}); + +describe("controller performance history page", () => { + it("builds its query from the walk's parameters, page size included", async () => { + const urls = serve({ snapshots: [], next_cursor: null, interval: "1h" }); + await api.getControllerPerformanceHistoryAll( + SERVER, + { + bot_name: "bot-a", + controller_id: "", + start_time: "2026-07-01T00:00:00Z", + interval: "1h", + }, + { pageSize: 3, maxRows: 100 }, + ); + expect(urls[0]).toBe( + "/api/v1/servers/prod/controller-performance/history" + + "?bot_name=bot-a&start_time=2026-07-01T00%3A00%3A00Z&interval=1h&limit=3", + ); + }); +}); + +describe("performance history page", () => { + it("builds its query from the walk's parameters, page size included", async () => { + const urls = serve({ snapshots: [], next_cursor: null, interval: "1h" }); + await api.getPerformanceHistory( + SERVER, + { subject: "controller", bot_name: "bot-a", executor_id: "", interval: "1h" }, + { pageSize: 3, maxRows: 100 }, + ); + expect(urls[0]).toBe( + "/api/v1/servers/prod/performance/history" + + "?subject=controller&bot_name=bot-a&interval=1h&limit=3", + ); + }); +}); diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 9a0b8bb4a..ec308ea24 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -23,6 +23,27 @@ async function apiFetch(path: string, init?: RequestInit): Promise { return res.json(); } +/** + * The query string for a set of optional parameters, `?` included — or `""` + * when none of them is set. + * + * Every optional query parameter in this file is skipped and stringified here, + * so a field added to an endpoint's parameter type is carried without a second + * edit. Spelling each name a second time in a `qs.set` line is what let a + * filter compile cleanly and never reach the server (READ-333). + * + * `undefined`, `null` and `""` are omitted; everything else is `String()`d, so + * a deliberate `0` is sent rather than dropped. + */ +function query(params: object): string { + const qs = new URLSearchParams(); + for (const [key, value] of Object.entries(params)) { + if (value !== undefined && value !== null && value !== "") qs.set(key, String(value)); + } + const q = qs.toString(); + return q ? `?${q}` : ""; +} + /** A finished background task, addressed to the user rather than to a chat. */ export interface AppNotification { id: string; @@ -1983,17 +2004,8 @@ function fetchControllerPerformanceHistoryPage( } = {}, init?: RequestInit, ) { - const qs = new URLSearchParams(); - if (params.bot_name) qs.set("bot_name", params.bot_name); - if (params.controller_id) qs.set("controller_id", params.controller_id); - if (params.start_time) qs.set("start_time", params.start_time); - if (params.end_time) qs.set("end_time", params.end_time); - if (params.interval) qs.set("interval", params.interval); - if (params.limit) qs.set("limit", String(params.limit)); - if (params.cursor) qs.set("cursor", params.cursor); - const q = qs.toString(); return apiFetch( - `/api/v1/servers/${encodeURIComponent(server)}/controller-performance/history${q ? `?${q}` : ""}`, + `/api/v1/servers/${encodeURIComponent(server)}/controller-performance/history${query(params)}`, init, ); } @@ -2192,12 +2204,8 @@ function fetchPerformanceHistoryPage( params: PerformanceHistoryQuery & { limit?: number; cursor?: string }, init?: RequestInit, ) { - const qs = new URLSearchParams(); - for (const [key, value] of Object.entries(params)) { - if (value !== undefined && value !== null && value !== "") qs.set(key, String(value)); - } return apiFetch( - `/api/v1/servers/${encodeURIComponent(server)}/performance/history?${qs.toString()}`, + `/api/v1/servers/${encodeURIComponent(server)}/performance/history${query(params)}`, init, ); } @@ -2494,16 +2502,8 @@ export const api = { offset?: number; } = {}, ) => { - const qs = new URLSearchParams(); - if (params.bot_name) qs.set("bot_name", params.bot_name); - if (params.run_status) qs.set("run_status", params.run_status); - if (params.deployment_status) - qs.set("deployment_status", params.deployment_status); - if (params.limit) qs.set("limit", String(params.limit)); - if (params.offset) qs.set("offset", String(params.offset)); - const q = qs.toString(); return apiFetch( - `/api/v1/servers/${encodeURIComponent(server)}/bot-runs${q ? `?${q}` : ""}`, + `/api/v1/servers/${encodeURIComponent(server)}/bot-runs${query(params)}`, ); }, @@ -2559,15 +2559,8 @@ export const api = { limit?: number; }, ) => { - const qs = new URLSearchParams(); - if (params?.executor_type) qs.set("executor_type", params.executor_type); - if (params?.trading_pair) qs.set("trading_pair", params.trading_pair); - if (params?.status) qs.set("status", params.status); - if (params?.controller_id) qs.set("controller_id", params.controller_id); - if (params?.limit) qs.set("limit", String(params.limit)); - const q = qs.toString(); return apiFetch( - `/api/v1/servers/${encodeURIComponent(server)}/executors${q ? `?${q}` : ""}`, + `/api/v1/servers/${encodeURIComponent(server)}/executors${query(params ?? {})}`, ); }, @@ -2587,15 +2580,8 @@ export const api = { controller_id?: string; } = {}, ) => { - const qs = new URLSearchParams(); - if (params.cursor) qs.set("cursor", params.cursor); - qs.set("limit", String(params.limit ?? 50)); - if (params.executor_type) qs.set("executor_type", params.executor_type); - if (params.trading_pair) qs.set("trading_pair", params.trading_pair); - if (params.status) qs.set("status", params.status); - if (params.controller_id) qs.set("controller_id", params.controller_id); return apiFetch<{ executors: ExecutorInfo[]; next_cursor: string | null }>( - `/api/v1/servers/${encodeURIComponent(server)}/executors/page?${qs.toString()}`, + `/api/v1/servers/${encodeURIComponent(server)}/executors/page${query({ ...params, limit: params.limit ?? 50 })}`, ); }, @@ -3367,15 +3353,9 @@ export const api = { limit?: number; offset?: number; }) => { - const qs = new URLSearchParams(); - if (params?.source_type) qs.set("source_type", params.source_type); - if (params?.tag) qs.set("tag", params.tag); - if (params?.search) qs.set("search", params.search); - if (params?.agent) qs.set("agent", params.agent); - if (params?.limit) qs.set("limit", String(params.limit)); - if (params?.offset) qs.set("offset", String(params.offset)); - const q = qs.toString(); - return apiFetch(`/api/v1/reports${q ? `?${q}` : ""}`); + return apiFetch( + `/api/v1/reports${query(params ?? {})}`, + ); }, getReport: (id: string) => From 5a481d1123e29fe29fc0f8091d86737d18872db4 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 08:34:25 +0300 Subject: [PATCH 036/154] Mint the chat's system notes in one place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `appendSystemNote` was documented as the one place a system divider is built, but the same literal — id, `role: "system"`, an empty tool list, a timestamp — was re-typed by hand at four other sites, and the copies had already drifted in key order. A `systemNote(text, kind?)` factory beside `nextMsgId`/`nowTs` now owns the shape: `role: "system"` appears exactly once in the file, and the upload-failure path, the `error` frame and both switch dividers go through it. The upload failure is a pure append, so it calls `appendSystemNote` outright. switchBrain and switchServer were the same forty-line `setSlots(prev => prev.map(...))` skeleton — merge fields into `info`, then conditionally append a divider — so they share one `applySwitch` helper. It takes a function of the previous info rather than a finished pair, because both the merge and the suppression rule are read off it: a server switch is only a switch when the name actually changed. --- frontend/src/hooks/useChatSocket.ts | 192 ++++++++++++++-------------- 1 file changed, 93 insertions(+), 99 deletions(-) diff --git a/frontend/src/hooks/useChatSocket.ts b/frontend/src/hooks/useChatSocket.ts index 64d1d3bdc..de66155dd 100644 --- a/frontend/src/hooks/useChatSocket.ts +++ b/frontend/src/hooks/useChatSocket.ts @@ -343,6 +343,26 @@ function nowTs(): number { return Date.now() / 1000; } +/** + * One system entry, ready to append — a reload, a routine's note, an error, a + * handover divider. + * + * Every one of them is the same shape, so the shape lives here alone: callers + * say the words and the kind, and the id and the timestamp are minted at the + * moment the note is made. Anything `ChatMessage` later grows for system + * entries is added once, here, instead of being hunted through the file. + */ +function systemNote(text: string, kind?: string): ChatMessage { + return { + id: nextMsgId(), + role: "system", + text, + kind, + toolCalls: [], + ts: nowTs(), + }; +} + let clientRefCounter = 0; /** Local handle for a tab that has no conversation id yet. Echoed by the * backend on `session_started`, which is how the two are reconciled. */ @@ -704,17 +724,7 @@ export function useChatSocket() { */ const appendSystemNote = useCallback( (slotId: string, text: string, kind?: string) => { - updateSlotMessages(slotId, (msgs) => [ - ...msgs, - { - id: nextMsgId(), - role: "system" as const, - text, - kind, - toolCalls: [], - ts: nowTs(), - }, - ]); + updateSlotMessages(slotId, (msgs) => [...msgs, systemNote(text, kind)]); }, [updateSlotMessages], ); @@ -1000,17 +1010,11 @@ export function useChatSocket() { ); ids = stored.map((a) => a.id); } catch (e) { - updateSlotMessages(slotId, (msgs) => [ - ...msgs, - { - id: nextMsgId(), - role: "system" as const, - kind: "error", - text: e instanceof Error ? e.message : "Could not attach that image", - toolCalls: [], - ts: nowTs(), - }, - ]); + appendSystemNote( + slotId, + e instanceof Error ? e.message : "Could not attach that image", + "error", + ); return; } } @@ -1022,7 +1026,7 @@ export function useChatSocket() { ...(ids.length ? { attachments: ids } : {}), }); }, - [send, updateSlotMessages], + [appendSystemNote, send], ); /** @@ -1789,21 +1793,10 @@ export function useChatSocket() { setSlots((prev) => prev.map((s) => { if (s.info.slot_id !== errSlotId) return s; - const id = nextMsgId(); return { ...s, pending: false, - messages: [ - ...s.messages, - { - id, - role: "system" as const, - kind: "error", - text: errMsg, - toolCalls: [], - ts: nowTs(), - }, - ], + messages: [...s.messages, systemNote(errMsg, "error")], }; }), ); @@ -1931,6 +1924,37 @@ export function useChatSocket() { [flushChunks, updateSlotMessages, uploadAndSend], ); + /** + * Repoint one slot, and mark the scrollback if the move is worth marking. + * + * A brain switch and a server switch are the same edit — merge the session's + * new fields into the slot's `info`, then append a divider — and differ only + * in which fields move and in when the move earns a divider at all. Both + * answers are read off the *previous* info (a server switch is suppressed by + * comparing the old server name against the new one), so the caller hands in + * a function of it rather than a finished pair. + */ + const applySwitch = useCallback( + ( + slotId: string, + compute: (prev: SlotInfo) => { info: SlotInfo; divider?: string }, + ) => { + setSlots((prev) => + prev.map((s) => { + if (s.info.slot_id !== slotId) return s; + const { info, divider } = compute(s.info); + if (!divider) return { ...s, info }; + return { + ...s, + info, + messages: [...s.messages, systemNote(divider, "switch")], + }; + }), + ); + }, + [], + ); + /** * Rebind the chat to a different brain, mid-conversation. * @@ -1949,42 +1973,27 @@ export function useChatSocket() { // The outgoing brain's last words belong above the divider that retires // it, not in a new bubble underneath it. flushChunks(); - setSlots((prev) => - prev.map((s) => { - if (s.info.slot_id !== slotId) return s; - const info: SlotInfo = { - ...s.info, - agent_key: session.agent_key, - // A brain switch can move the server too: binding to an Agent that - // pins one overrides the chat's ambient choice, and unbinding - // hands it back. Both are read off the respawned session. - server_name: session.server_name || undefined, - server_pinned: session.server_pinned, - agent_slug: session.agent_slug, - label: session.label, - }; - // Only a change of *who* divides the scrollback; a model swap under - // the same identity is not a handover the reader needs marked. - if (selection.agentSlug === undefined) return { ...s, info }; - return { - ...s, - info, - messages: [ - ...s.messages, - { - id: nextMsgId(), - role: "system" as const, - text: `Switched to ${session.label}`, - kind: "switch", - toolCalls: [], - ts: nowTs(), - }, - ], - }; - }), - ); + applySwitch(slotId, (prev) => ({ + info: { + ...prev, + agent_key: session.agent_key, + // A brain switch can move the server too: binding to an Agent that + // pins one overrides the chat's ambient choice, and unbinding + // hands it back. Both are read off the respawned session. + server_name: session.server_name || undefined, + server_pinned: session.server_pinned, + agent_slug: session.agent_slug, + label: session.label, + }, + // Only a change of *who* divides the scrollback; a model swap under + // the same identity is not a handover the reader needs marked. + divider: + selection.agentSlug === undefined + ? undefined + : `Switched to ${session.label}`, + })); }, - [flushChunks, user], + [applySwitch, flushChunks, user], ); /** @@ -2004,36 +2013,21 @@ export function useChatSocket() { // Same ordering rule as the brain switch: buffered text first, divider // after it. flushChunks(); - setSlots((prev) => - prev.map((s) => { - if (s.info.slot_id !== slotId) return s; - const info: SlotInfo = { - ...s.info, - server_name: session.server_name || undefined, - server_pinned: session.server_pinned, - }; - // Only an actual move divides the scrollback. A pinned Agent ignores - // the request, and a divider claiming otherwise would be a lie. - if (session.server_name === s.info.server_name) return { ...s, info }; - return { - ...s, - info, - messages: [ - ...s.messages, - { - id: nextMsgId(), - role: "system" as const, - text: `Now using server ${session.server_name}`, - kind: "switch", - toolCalls: [], - ts: nowTs(), - }, - ], - }; - }), - ); + applySwitch(slotId, (prev) => ({ + info: { + ...prev, + server_name: session.server_name || undefined, + server_pinned: session.server_pinned, + }, + // Only an actual move divides the scrollback. A pinned Agent ignores + // the request, and a divider claiming otherwise would be a lie. + divider: + session.server_name === prev.server_name + ? undefined + : `Now using server ${session.server_name}`, + })); }, - [flushChunks, user], + [applySwitch, flushChunks, user], ); const destroySession = useCallback( From 1b59f4a3a36ebef67388c65343b15f0759e745a6 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 08:39:27 +0300 Subject: [PATCH 037/154] The grid panel uses the shared field kit instead of its own copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GridConfigPanel imported LeverageField/SelectField/ToggleField from components/executor/fields but kept private copies of PriceField, NumberField and SectionHeader that the same module already exports. SectionHeader was byte-identical; the NumberField copy had drifted and was strictly worse, still scaling a percent with the naive `value * 100` the shared one replaced with a .toPrecision(12) round — so a take profit of 0.0212 read 2.12 in the Position, DCA and LP panels and 2.1199999999999997 in the grid. The three copies are gone. The shared PriceField writes the field it is named after rather than suffixing `_price`, so the panel now names the fields in full and GridState.activePickField holds a field name; the chart still speaks in slots, and useGridConfig maps across the two the way lp-config already does. The grid copy's min-w-0/truncate classes, which keep a suffix inside the narrow sidebar, move into the shared NumberField, where they are inert wherever there is room. Closes READ-345. --- frontend/eslint-baseline.json | 1 - frontend/src/components/executor/fields.tsx | 14 +- .../components/executor/grid-config.test.tsx | 6 +- .../src/components/executor/grid-config.ts | 11 +- .../src/components/grid/GridConfigPanel.tsx | 175 ++---------------- frontend/src/lib/gridExecutor.ts | 8 +- 6 files changed, 42 insertions(+), 173 deletions(-) diff --git a/frontend/eslint-baseline.json b/frontend/eslint-baseline.json index fd44631f5..935e5dd98 100644 --- a/frontend/eslint-baseline.json +++ b/frontend/eslint-baseline.json @@ -8,7 +8,6 @@ "src/components/editor/EditorModal.tsx | react-hooks/refs": 6, "src/components/executor/DCAConfigPanel.tsx | @typescript-eslint/no-unused-vars": 1, "src/components/executor/fields.tsx | react-hooks/set-state-in-effect": 3, - "src/components/grid/GridConfigPanel.tsx | react-hooks/set-state-in-effect": 2, "src/components/market/OrderBook.tsx | react-hooks/immutability": 1, "src/components/market/PriceTicker.tsx | react-hooks/refs": 3, "src/components/market/RecentTrades.tsx | react-hooks/set-state-in-effect": 1, diff --git a/frontend/src/components/executor/fields.tsx b/frontend/src/components/executor/fields.tsx index 3d3d6c5b8..1ceabc974 100644 --- a/frontend/src/components/executor/fields.tsx +++ b/frontend/src/components/executor/fields.tsx @@ -188,10 +188,14 @@ export function NumberField({ } }, [displayValue]); + // `min-w-0`/`truncate`/`shrink-0`: a flex item's default minimum width is its + // content, so in a narrow sidebar (the grid panel, and its two-column advanced + // row) an unconstrained input pushes the suffix out of the panel instead of + // shrinking. Harmless where there is room. return ( -
- -
+
+ +
setLocalValue(displayValue === 0 ? "" : String(displayValue))} placeholder="0" - className="flex-1 rounded border border-[var(--color-border)] bg-[var(--color-bg)] px-2.5 py-1.5 font-mono text-xs text-[var(--color-text)] placeholder:text-[var(--color-text-muted)]/40 focus:border-[var(--color-primary)] focus:outline-none" + className="w-full min-w-0 flex-1 rounded border border-[var(--color-border)] bg-[var(--color-bg)] px-2.5 py-1.5 font-mono text-xs text-[var(--color-text)] placeholder:text-[var(--color-text-muted)]/40 focus:border-[var(--color-primary)] focus:outline-none" /> {suffix && ( - {suffix} + {suffix} )}
diff --git a/frontend/src/components/executor/grid-config.test.tsx b/frontend/src/components/executor/grid-config.test.tsx index 43497ce96..b1b21c2f6 100644 --- a/frontend/src/components/executor/grid-config.test.tsx +++ b/frontend/src/components/executor/grid-config.test.tsx @@ -120,7 +120,7 @@ describe("useGridConfig().buildPayload", () => { describe("useGridConfig().handleChartPriceSet", () => { it("writes the picked price into the slot's field and disarms the picker", () => { - setField("activePickField", "start"); + setField("activePickField", "start_price"); act(() => { latest.handleChartPriceSet("start", 123.45); @@ -143,7 +143,7 @@ describe("useGridConfig().handleChartPriceSet", () => { }); it("ignores a slot the grid does not own", () => { - setField("activePickField", "start"); + setField("activePickField", "start_price"); act(() => { latest.handleChartPriceSet("take_profit", 500); @@ -152,6 +152,6 @@ describe("useGridConfig().handleChartPriceSet", () => { // Not the grid's line: nothing written, and the grid's own armed picker is // left alone for the panel that does own the slot. expect(latest.state.start_price).toBe(0); - expect(latest.state.activePickField).toBe("start"); + expect(latest.state.activePickField).toBe("start_price"); }); }); diff --git a/frontend/src/components/executor/grid-config.ts b/frontend/src/components/executor/grid-config.ts index 6248b9e7d..9e2f20f7a 100644 --- a/frontend/src/components/executor/grid-config.ts +++ b/frontend/src/components/executor/grid-config.ts @@ -26,6 +26,15 @@ import { } from "@/lib/gridExecutor"; import type { GridState } from "@/lib/gridExecutor"; +// ── Chart pick slots ── +// The panel arms the picker by field name; the chart draws its three lines under +// its own slot names. One map across, the way `lp-config` does it. +const PICK_SLOT: Record = { + start_price: "start", + end_price: "end", + limit_price: "limit", +}; + // ── Validation ── export function useGridValidation(state: GridState): ExecutorValidation { @@ -50,7 +59,7 @@ export function useGridConfig() { limitPrice: state.limit_price, side: state.side, minSpread: state.min_spread_between_orders, - activePickField: state.activePickField, + activePickField: PICK_SLOT[state.activePickField ?? ""] ?? null, lineLabels: gridLineLabels(state.side), }), [ state.start_price, diff --git a/frontend/src/components/grid/GridConfigPanel.tsx b/frontend/src/components/grid/GridConfigPanel.tsx index ab0c03169..ab12bb1ed 100644 --- a/frontend/src/components/grid/GridConfigPanel.tsx +++ b/frontend/src/components/grid/GridConfigPanel.tsx @@ -1,14 +1,14 @@ -import { useEffect, useId, useMemo, useRef, useState } from "react"; -import { - AlertTriangle, - Check, - ChevronDown, - ChevronUp, - Crosshair, - Sparkles, -} from "lucide-react"; +import { useEffect, useMemo } from "react"; +import { AlertTriangle, ChevronDown, ChevronUp, Sparkles } from "lucide-react"; -import { LeverageField, SelectField, ToggleField } from "@/components/executor/fields"; +import { + LeverageField, + NumberField, + PriceField, + SectionHeader, + SelectField, + ToggleField, +} from "@/components/executor/fields"; import { ORDER_TYPE_OPTIONS } from "@/components/executor/field-options"; import { autoFillGridPrices, gridConfigErrors, gridPriceFieldValid } from "@/lib/gridExecutor"; import type { GridState, GridAction } from "@/lib/gridExecutor"; @@ -21,155 +21,6 @@ interface GridConfigPanelProps { quoteCurrency?: string; } -function PriceField({ - label, - value, - field, - activePickField, - dispatch, - valid, - hint, -}: { - label: string; - value: number; - field: "start" | "end" | "limit"; - activePickField: "start" | "end" | "limit" | null; - dispatch: React.Dispatch; - valid: boolean; - hint?: string; -}) { - const isActive = activePickField === field; - const id = useId(); - const inputRef = useRef(null); - const [localValue, setLocalValue] = useState(value === 0 ? "" : String(value)); - - // Sync from parent when value changes externally (e.g. auto-fill, chart pick) - useEffect(() => { - if (document.activeElement !== inputRef.current) { - setLocalValue(value === 0 ? "" : String(value)); - } - }, [value]); - - return ( -
- -
- { - setLocalValue(e.target.value); - const num = parseFloat(e.target.value); - dispatch({ type: "SET_FIELD", field: `${field}_price`, value: isNaN(num) ? 0 : num }); - }} - onBlur={() => setLocalValue(value === 0 ? "" : String(value))} - placeholder="0.00" - className={`flex-1 rounded border bg-[var(--color-bg)] px-2.5 py-1.5 font-mono text-xs text-[var(--color-text)] placeholder:text-[var(--color-text-muted)]/40 focus:outline-none ${ - isActive - ? "border-[var(--color-primary)] ring-1 ring-[var(--color-primary)]" - : "border-[var(--color-border)] focus:border-[var(--color-primary)]" - }`} - /> - -
- {hint &&

{hint}

} -
- ); -} - -function NumberField({ - label, - value, - field, - dispatch, - step = 1, - min, - suffix, - isPercent = false, -}: { - label: string; - value: number; - field: string; - dispatch: React.Dispatch; - step?: number; - min?: number; - suffix?: string; - isPercent?: boolean; -}) { - const displayValue = isPercent ? value * 100 : value; - const id = useId(); - const inputRef = useRef(null); - const [localValue, setLocalValue] = useState(displayValue === 0 ? "" : String(displayValue)); - - useEffect(() => { - if (document.activeElement !== inputRef.current) { - setLocalValue(displayValue === 0 ? "" : String(displayValue)); - } - }, [displayValue]); - - return ( -
- -
- { - setLocalValue(e.target.value); - const raw = parseFloat(e.target.value); - dispatch({ type: "SET_FIELD", field, value: isPercent ? (isNaN(raw) ? 0 : raw / 100) : (isNaN(raw) ? 0 : raw) }); - }} - onBlur={() => setLocalValue(displayValue === 0 ? "" : String(displayValue))} - placeholder="0" - className="w-full min-w-0 flex-1 rounded border border-[var(--color-border)] bg-[var(--color-bg)] px-2.5 py-1.5 font-mono text-xs text-[var(--color-text)] placeholder:text-[var(--color-text-muted)]/40 focus:border-[var(--color-primary)] focus:outline-none" - /> - {suffix && ( - {suffix} - )} -
-
- ); -} - - -function SectionHeader({ children }: { children: React.ReactNode }) { - return ( - - {children} - - ); -} - export function GridConfigPanel({ state, dispatch, currentPrice, isSpot = false, quoteCurrency = "USDT" }: GridConfigPanelProps) { const validation = useMemo(() => { const errors = gridConfigErrors(state); @@ -270,7 +121,7 @@ export function GridConfigPanel({ state, dispatch, currentPrice, isSpot = false, Date: Wed, 9 Sep 2026 08:45:28 +0300 Subject: [PATCH 038/154] Write the inline "click trash, then confirm" control once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six delete controls were the same idea typed out six times: a useState holding the id awaiting confirmation, and a ternary swapping a trash button for a confirm/cancel pair. Being copies, they had drifted in every dimension that shows — Check/X here, "Yes"/"No" or "Forget" there; a pending spinner in ServersSettings and in none of the others, so a slow delete elsewhere looked like a click that did nothing; and accessible names on some of the buttons but not all. Add components/ui/InlineConfirm, beside the other shared primitives, and use it at all six: the two in ApiKeysSettings, ServersSettings, CustomProvidersSettings, and the byte-for-byte pair in ReportBrowser and ReportViewer. The caller keeps owning the confirming state, so a list still holds the id of the single row awaiting confirmation rather than growing a boolean per row. GatewaySettings' Stop is deliberately left alone. It is the odd member — a Square trigger with a text "Stop"/"Confirm Stop"/"Cancel" pair, not a trash icon — and folding it in would need an icon, a variant and a label prop, which costs more readability than the copy does. --- .../src/components/routines/ReportBrowser.tsx | 40 ++--- .../src/components/routines/ReportViewer.tsx | 40 ++--- .../components/settings/ApiKeysSettings.tsx | 87 +++-------- .../settings/CustomProvidersSettings.tsx | 39 ++--- .../components/settings/ServersSettings.tsx | 44 ++---- .../src/components/ui/InlineConfirm.test.tsx | 147 ++++++++++++++++++ frontend/src/components/ui/InlineConfirm.tsx | 88 +++++++++++ 7 files changed, 310 insertions(+), 175 deletions(-) create mode 100644 frontend/src/components/ui/InlineConfirm.test.tsx create mode 100644 frontend/src/components/ui/InlineConfirm.tsx diff --git a/frontend/src/components/routines/ReportBrowser.tsx b/frontend/src/components/routines/ReportBrowser.tsx index 362282c66..4ff22ecc9 100644 --- a/frontend/src/components/routines/ReportBrowser.tsx +++ b/frontend/src/components/routines/ReportBrowser.tsx @@ -19,6 +19,7 @@ import { } from "lucide-react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { InlineConfirm } from "@/components/ui/InlineConfirm"; import { type RoutineInstance, api } from "@/lib/api"; import { toMs } from "@/lib/formatters"; import { @@ -828,31 +829,20 @@ export function ReportBrowser({ )} {/* Delete */} {view === "report" && selectedReport && ( - confirmDelete ? ( -
- Delete? - - -
- ) : ( - - ) + setConfirmDelete(true)} + onConfirm={() => { + deleteMutation.mutate(selectedReport.id); + setConfirmDelete(false); + }} + onCancel={() => setConfirmDelete(false)} + pending={deleteMutation.isPending} + triggerLabel="Delete report" + confirmLabel="Confirm delete report" + cancelLabel="Cancel delete report" + size="md" + /> )} {/* Close — the sheet's header has one when hosted, and the page is a nav destination rather than something you dismiss. */} diff --git a/frontend/src/components/routines/ReportViewer.tsx b/frontend/src/components/routines/ReportViewer.tsx index 38db97a0b..b7f7b63b6 100644 --- a/frontend/src/components/routines/ReportViewer.tsx +++ b/frontend/src/components/routines/ReportViewer.tsx @@ -4,12 +4,12 @@ import { Layers, Maximize2, Minimize2, - Trash2, X, } from "lucide-react"; import { useCallback, useEffect, useState } from "react"; import { ReportFrame } from "@/components/routines/ReportFrame"; +import { InlineConfirm } from "@/components/ui/InlineConfirm"; import { type ReportSummary } from "@/lib/api"; interface ReportViewerProps { @@ -133,31 +133,19 @@ export function ReportViewer({ )} {onDelete && ( - confirmDelete ? ( -
- Delete? - - -
- ) : ( - - ) + setConfirmDelete(true)} + onConfirm={() => { + onDelete(report.id); + setConfirmDelete(false); + }} + onCancel={() => setConfirmDelete(false)} + triggerLabel="Delete report" + confirmLabel="Confirm delete report" + cancelLabel="Cancel delete report" + size="md" + /> )} {fullscreen && onClose && (
- {confirmDelete === c.connector_name ? ( -
- - -
- ) : ( - - )} + setConfirmDelete(c.connector_name)} + onConfirm={() => deleteMut.mutate(c.connector_name)} + onCancel={() => setConfirmDelete(null)} + pending={deleteMut.isPending} + disabled={!isOwner} + triggerLabel={isOwner ? "Delete credential" : OWNER_ONLY_HINT} + />
))} @@ -669,37 +648,21 @@ export function ApiKeysSettings() { )} - {confirmDeleteWallet === walletKey ? ( - <> - - - - ) : ( - - )} + setConfirmDeleteWallet(walletKey)} + onConfirm={() => + deleteWalletMut.mutate({ chain: w.chain, address: w.address }) + } + onCancel={() => setConfirmDeleteWallet(null)} + pending={deleteWalletMut.isPending} + disabled={!isOwner} + triggerLabel={ + isOwner ? "Remove wallet from Gateway" : OWNER_ONLY_HINT + } + confirmLabel="Confirm remove" + cancelLabel="Cancel remove" + /> ); diff --git a/frontend/src/components/settings/CustomProvidersSettings.tsx b/frontend/src/components/settings/CustomProvidersSettings.tsx index 3001028a2..dc2be63be 100644 --- a/frontend/src/components/settings/CustomProvidersSettings.tsx +++ b/frontend/src/components/settings/CustomProvidersSettings.tsx @@ -1,7 +1,8 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Check, Loader2, Plus, Server, Trash2, X } from "lucide-react"; +import { Check, Loader2, Plus, Server } from "lucide-react"; import { useState } from "react"; +import { InlineConfirm } from "@/components/ui/InlineConfirm"; import { api } from "@/lib/api"; import type { CustomProvider } from "@/lib/api"; @@ -139,32 +140,16 @@ function ProviderRow({ - {confirming ? ( -
- - -
- ) : ( - - )} + ); } diff --git a/frontend/src/components/settings/ServersSettings.tsx b/frontend/src/components/settings/ServersSettings.tsx index 97a8b6a69..ca12c0c43 100644 --- a/frontend/src/components/settings/ServersSettings.tsx +++ b/frontend/src/components/settings/ServersSettings.tsx @@ -1,16 +1,14 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { - Check, Edit2, Loader2, Plus, Star, - Trash2, - X, } from "lucide-react"; import { useState } from "react"; import { useSearchParams } from "react-router-dom"; +import { InlineConfirm } from "@/components/ui/InlineConfirm"; import { useIsAdmin } from "@/hooks/useIsAdmin"; import { useServer } from "@/hooks/useServer"; import { type ServerInfo, api } from "@/lib/api"; @@ -308,38 +306,14 @@ export function ServersSettings() { > - {confirmDelete === s.name ? ( -
- - -
- ) : ( - - )} + setConfirmDelete(s.name)} + onConfirm={() => deleteMut.mutate(s.name)} + onCancel={() => setConfirmDelete(null)} + pending={deleteMut.isPending} + triggerLabel="Delete" + /> )} diff --git a/frontend/src/components/ui/InlineConfirm.test.tsx b/frontend/src/components/ui/InlineConfirm.test.tsx new file mode 100644 index 000000000..1940f311a --- /dev/null +++ b/frontend/src/components/ui/InlineConfirm.test.tsx @@ -0,0 +1,147 @@ +/** + * The affordance the six former hand-rolled copies now share (READ-352). + * + * The copies had drifted apart — a pending spinner in one, "Yes"/"No" in + * another, an accessible name on some buttons and not others — so the point of + * folding them into one primitive is that these guarantees hold everywhere at + * once. A regression here is a regression in all six delete controls. + * + * Needs a DOM, so this file overrides vitest's default `node` environment. + * + * @vitest-environment jsdom + */ + +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { InlineConfirm } from "./InlineConfirm"; + +declare global { + // eslint-disable-next-line no-var + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +let container: HTMLDivElement; +let root: Root; + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); +}); + +const click = (el: Element | null) => + act(() => { + el?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + +const byLabel = (name: string) => + container.querySelector(`[aria-label="${name}"]`) as HTMLButtonElement | null; + +describe("InlineConfirm", () => { + it("takes two clicks: the trigger asks, the confirm acts", () => { + const onRequest = vi.fn(); + const onConfirm = vi.fn(); + + act(() => { + root.render( + , + ); + }); + + // Closed: only the trigger, and it never deletes on its own. + expect(byLabel("Delete server")).not.toBeNull(); + expect(byLabel("Confirm delete")).toBeNull(); + click(byLabel("Delete server")); + expect(onRequest).toHaveBeenCalledTimes(1); + expect(onConfirm).not.toHaveBeenCalled(); + + // The caller owns the state, so confirming arrives as a prop. + act(() => { + root.render( + , + ); + }); + + click(byLabel("Confirm delete")); + expect(onConfirm).toHaveBeenCalledTimes(1); + }); + + it("gives the confirm and cancel buttons accessible names", () => { + act(() => { + root.render( + , + ); + }); + + expect(byLabel("Confirm remove")).not.toBeNull(); + expect(byLabel("Cancel remove")).not.toBeNull(); + }); + + it("blocks a second submission while the mutation is pending", () => { + const onConfirm = vi.fn(); + + act(() => { + root.render( + , + ); + }); + + const confirm = byLabel("Confirm delete"); + expect(confirm?.disabled).toBe(true); + expect(confirm?.querySelector(".animate-spin")).not.toBeNull(); + }); + + it("refuses the first click when the trigger is disabled", () => { + const onRequest = vi.fn(); + + act(() => { + root.render( + , + ); + }); + + expect(byLabel("Owner access required")?.disabled).toBe(true); + }); +}); diff --git a/frontend/src/components/ui/InlineConfirm.tsx b/frontend/src/components/ui/InlineConfirm.tsx new file mode 100644 index 000000000..af6e1d7f6 --- /dev/null +++ b/frontend/src/components/ui/InlineConfirm.tsx @@ -0,0 +1,88 @@ +import { Check, Loader2, Trash2, X } from "lucide-react"; + +/** + * The two-click delete affordance for a row or a toolbar: a trash trigger that + * swaps itself for a confirm/cancel pair. + * + * It was written by hand six times over (credentials, Gateway wallets, servers, + * LLM endpoints and the two report headers) and had drifted in every dimension + * — Check/X here, "Yes"/"No" or "Forget" there, a pending spinner in one copy + * and none in the rest. A modal (`components/agent/ConfirmDialog`) is the wrong + * control inside a table row, which is why they were hand-rolled; this is that + * control, once. + * + * The caller keeps owning the `confirming` state, so a list can hold the id of + * the single row awaiting confirmation rather than one boolean per row. + */ +export interface InlineConfirmProps { + /** True while this control is the one awaiting a confirming click. */ + confirming: boolean; + onRequest: () => void; + onConfirm: () => void; + onCancel: () => void; + /** Runs a spinner in the confirm button and blocks a second submission. */ + pending?: boolean; + /** Title and accessible name of the trash trigger, e.g. "Delete credential". */ + triggerLabel: string; + /** Blocks the first click (owner-only rows); `triggerLabel` should say why. */ + disabled?: boolean; + confirmLabel?: string; + cancelLabel?: string; + /** Icon scale: `sm` for dense settings rows, `md` for report toolbars. */ + size?: "sm" | "md"; +} + +export function InlineConfirm({ + confirming, + onRequest, + onConfirm, + onCancel, + pending = false, + triggerLabel, + disabled = false, + confirmLabel = "Confirm delete", + cancelLabel = "Cancel delete", + size = "sm", +}: InlineConfirmProps) { + const icon = size === "sm" ? "h-3.5 w-3.5" : "h-4 w-4"; + + if (!confirming) { + return ( + + ); + } + + return ( +
+ + +
+ ); +} From 0867e12750636fe20471f583981e1455441698a8 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 08:52:42 +0300 Subject: [PATCH 039/154] (fix) one bad line no longer deafens the ACP connection The read loop treated any per-line failure as fatal for the whole connection: a non-UTF-8 byte in a tool result, a line over the 10MB stream limit, or a JSON-RPC `error` member that is a string rather than an object all ended the loop for good. The subprocess kept running, so `alive` still said True and the session cache kept handing back a client that could no longer hear a word -- every later prompt only collected 30s heartbeats until the overall deadline. Isolate the failure at the line: decode with errors="replace" (the same policy _drain_stderr already used), skip an oversized line (readline() drops it from the buffer before raising, so the next one still parses), and log-and-continue past a bad dispatch. A non-dict `error` now settles its future instead of raising. And once the read loop is over, `alive` is False whatever the subprocess is doing, so the session layer respawns instead of reusing a deaf client. --- condor/acp/client.py | 41 +++++- condor/acp/jsonrpc.py | 5 + .../runtime/test_acp_read_loop_resilience.py | 117 ++++++++++++++++++ 3 files changed, 158 insertions(+), 5 deletions(-) create mode 100644 tests/runtime/test_acp_read_loop_resilience.py diff --git a/condor/acp/client.py b/condor/acp/client.py index 7e6376af9..64e856df9 100644 --- a/condor/acp/client.py +++ b/condor/acp/client.py @@ -488,6 +488,9 @@ def __init__( self.accepts_images = False self._read_task: asyncio.Task | None = None self._stderr_task: asyncio.Task | None = None + # Set once _read_loop is over: the process may still be up, but + # nothing it says will ever reach us again (see :attr:`alive`). + self._read_loop_ended = False self._event_queue: asyncio.Queue[ACPEvent] = asyncio.Queue() self._current_req_id: int | None = None # tracks in-flight prompt request # A turn the agent has not settled and that nobody is streaming any @@ -552,6 +555,7 @@ async def start(self) -> None: limit=10 * 1024 * 1024, start_new_session=True, # Own process group so we can kill all children ) + self._read_loop_ended = False self._read_task = asyncio.create_task(self._read_loop()) self._stderr_task = asyncio.create_task(self._drain_stderr()) @@ -694,8 +698,18 @@ async def stop(self) -> None: @property def alive(self) -> bool: - """Check if the subprocess is still running.""" - return self._process is not None and self._process.returncode is None + """Check if the subprocess can still answer us. + + A running subprocess is not enough: once the read loop is over nothing + it writes will ever be read again, so the client is deaf even though + the process is up. Saying True there would have the session cache hand + the client another prompt (CORR-328). + """ + return ( + self._process is not None + and self._process.returncode is None + and not self._read_loop_ended + ) # --- Read loop --- @@ -703,16 +717,33 @@ async def _read_loop(self) -> None: assert self._process and self._process.stdout try: while True: - line = await self._process.stdout.readline() + try: + line = await self._process.stdout.readline() + except ValueError: + # Line longer than the stream limit. readline() drops it + # from the buffer before raising, so the next one still + # parses -- one oversized line must not deafen us. + log.warning("ACP line over the stream limit; skipped") + continue if not line: break - await self._peer.handle_line(line.decode(), self._process.stdin) + try: + # errors="replace", like _drain_stderr: one non-UTF-8 byte + # in a tool result is not a reason to lose the connection. + await self._peer.handle_line( + line.decode(errors="replace"), self._process.stdin + ) + except Exception: + # Isolate the failure at the line, not at the connection. + log.exception("ACP dropped a bad line") except asyncio.CancelledError: return # Intentional shutdown via stop() -- skip sentinel except Exception: log.exception("ACP read loop error") - # Subprocess died or stream ended -- unblock any consumer waiting on _event_queue + # Subprocess died or stream ended -- unblock any consumer waiting on + # _event_queue, and stop claiming to be alive: we can no longer hear. + self._read_loop_ended = True self._peer.cancel_all() self._event_queue.put_nowait(PromptDone(stop_reason="disconnected")) diff --git a/condor/acp/jsonrpc.py b/condor/acp/jsonrpc.py index c1563553d..71e57caec 100644 --- a/condor/acp/jsonrpc.py +++ b/condor/acp/jsonrpc.py @@ -92,6 +92,11 @@ async def handle_line(self, line: str, writer: asyncio.StreamWriter) -> None: if future and not future.done(): if "error" in data: err = data["error"] + if not isinstance(err, dict): + # The spec says an object; a peer that sends a bare + # string still has to settle the future, not blow up + # the caller's read loop (CORR-328). + err = {"message": str(err)} future.set_exception( JSONRPCError( err.get("code", -1), err.get("message", ""), err.get("data") diff --git a/tests/runtime/test_acp_read_loop_resilience.py b/tests/runtime/test_acp_read_loop_resilience.py new file mode 100644 index 000000000..6bd10802f --- /dev/null +++ b/tests/runtime/test_acp_read_loop_resilience.py @@ -0,0 +1,117 @@ +"""One bad line must not deafen the ACP connection (CORR-328). + +The read loop used to treat any per-line failure as fatal for the whole +connection: a non-UTF-8 byte, an oversized line or a malformed JSON-RPC error +member ended the loop for good while the subprocess kept running -- so +``alive`` still said True and the session cache kept handing back a client +that could no longer hear a word. +""" + +import asyncio +import json + +import pytest + +from condor.acp.client import ACPClient + + +class _FakeStdin: + """Subprocess stdin: records whatever the peer writes back.""" + + def __init__(self) -> None: + self.written: list[dict] = [] + + def write(self, data: bytes) -> None: + self.written.append(json.loads(data.decode())) + + async def drain(self) -> None: + pass + + +class _FakeProcess: + def __init__(self, stdout: asyncio.StreamReader) -> None: + self.stdout = stdout + self.stdin = _FakeStdin() + self.returncode = None # still running, exactly as in the bug report + + +def _client(stdout: asyncio.StreamReader) -> ACPClient: + client = ACPClient(command="true") + client._process = _FakeProcess(stdout) # type: ignore[assignment] + return client + + +async def _run_loop(client: ACPClient) -> None: + await asyncio.wait_for(client._read_loop(), timeout=5) + + +@pytest.mark.asyncio +async def test_invalid_utf8_line_is_skipped_and_the_next_one_dispatched(): + seen: list[dict] = [] + stdout = asyncio.StreamReader() + client = _client(stdout) + client._peer.register_handler("probe", lambda **kw: seen.append(kw)) + + # A tool result carrying one raw non-UTF-8 byte, then a good line. + stdout.feed_data(b'{"jsonrpc":"2.0","method":"probe","params":{"v":"\xff"}}\n') + stdout.feed_data(b'{"jsonrpc":"2.0","method":"probe","params":{"v":"ok"}}\n') + stdout.feed_eof() + + await _run_loop(client) + + # Both lines land: the bad byte is replaced, not fatal. + assert [s["v"] for s in seen] == ["�", "ok"] + + +@pytest.mark.asyncio +async def test_string_error_member_settles_the_future_without_killing_the_loop(): + stdout = asyncio.StreamReader() + client = _client(stdout) + future: asyncio.Future = asyncio.get_event_loop().create_future() + client._peer._pending[1] = future + seen: list[dict] = [] + client._peer.register_handler("probe", lambda **kw: seen.append(kw)) + + # `error` as a bare string instead of the object the spec mandates. + stdout.feed_data(b'{"jsonrpc":"2.0","id":1,"error":"boom"}\n') + stdout.feed_data(b'{"jsonrpc":"2.0","method":"probe","params":{"v":"ok"}}\n') + stdout.feed_eof() + + await _run_loop(client) + + assert "boom" in str(future.exception()) + assert [s["v"] for s in seen] == ["ok"] + + +@pytest.mark.asyncio +async def test_oversized_line_is_skipped_and_the_next_one_dispatched(): + seen: list[dict] = [] + stdout = asyncio.StreamReader(limit=64) + client = _client(stdout) + client._peer.register_handler("probe", lambda **kw: seen.append(kw)) + + huge = json.dumps({"jsonrpc": "2.0", "method": "probe", "params": {"v": "x" * 500}}) + stdout.feed_data(huge.encode() + b"\n") + stdout.feed_data(b'{"jsonrpc":"2.0","method":"probe","params":{"v":"ok"}}\n') + stdout.feed_eof() + + await _run_loop(client) + + assert [s["v"] for s in seen] == ["ok"] + + +@pytest.mark.asyncio +async def test_client_is_not_alive_once_the_read_loop_is_over(): + stdout = asyncio.StreamReader() + client = _client(stdout) + + assert client.alive is True + + # Stream ends while the subprocess is still up (returncode is None). + stdout.feed_eof() + await _run_loop(client) + + # Nothing the process writes can reach us any more, so the session layer + # must not be told the client is usable. + assert client._process is not None and client._process.returncode is None + assert client.alive is False From 8a7dcb439ce19613e922953250f35db9017064aa Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 09:00:07 +0300 Subject: [PATCH 040/154] (fix) apply the ACP prompt ceiling while events are still arriving MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit prompt_stream's wall-clock hard stop was only compared inside the idle branch, reached after 30s of silence on the event queue. An agent stuck in a tool-call loop, or a model that keeps narrating, never goes idle — so the ceiling was never read and the turn ran without bound. Chat survived it because sessions.py re-checks its own deadline per event and the tick engine wraps the stream in its own timeout; the delegate path (run_agent_to_completion, reflection, the eager initial-context prompt) enforces nothing of its own and inherited the leak, kept relaying, and kept auto-approving tool calls long after the policy's deadline. Evaluate elapsed on the event path too, and end the turn at the agent with abort_prompt before returning — breaking out only stops us relaying, as CORR-140 established for the session-level budget. The ceiling still comes from TIMEOUTS.prompt_hard_stop; the idle path (heartbeat, disconnected) is untouched. --- condor/acp/client.py | 27 ++++- tests/runtime/test_prompt_hard_ceiling.py | 115 ++++++++++++++++++++++ 2 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 tests/runtime/test_prompt_hard_ceiling.py diff --git a/condor/acp/client.py b/condor/acp/client.py index 64e856df9..69ecdac0e 100644 --- a/condor/acp/client.py +++ b/condor/acp/client.py @@ -1025,6 +1025,22 @@ def _on_response(fut: asyncio.Future) -> None: # CONDOR_TIMEOUT_PROMPT_OVERALL is not silently cut short here. max_duration = TIMEOUTS.prompt_hard_stop + async def _hard_stop(elapsed: float) -> None: + """End the turn at the agent, not merely here. + + Breaking out of the loop only stops us *relaying*: the turn would + keep generating and keep running tools against a permission + callback nobody is watching, and the next prompt would overlap it + at the subprocess. Same reasoning — and the same bounded + ``abort_prompt`` — as the session-level budget in ``sessions.py`` + (CORR-140). + """ + log.warning("Prompt hard timeout after %.0fs", elapsed) + try: + await self.abort_prompt() + except Exception: # noqa: BLE001 - never mask the timeout + log.warning("Could not cancel timed-out prompt", exc_info=True) + try: while True: try: @@ -1035,7 +1051,7 @@ def _on_response(fut: asyncio.Future) -> None: yield PromptDone(stop_reason="disconnected") break if elapsed > max_duration: - log.warning("Prompt hard timeout after %.0fs", elapsed) + await _hard_stop(elapsed) yield PromptDone(stop_reason="timeout") break yield Heartbeat(elapsed_seconds=elapsed) @@ -1043,6 +1059,15 @@ def _on_response(fut: asyncio.Future) -> None: yield event if isinstance(event, PromptDone): break + # The ceiling is wall-clock, so it has to be evaluated on the + # event path too: an agent stuck in a tool-call loop, or a + # model that keeps narrating, never leaves the queue idle for + # 30s and used to run forever past this budget. + elapsed = loop.time() - start_time + if elapsed > max_duration: + await _hard_stop(elapsed) + yield PromptDone(stop_reason="timeout") + break finally: # Reached on every way out, including the one that used to leak: # the consumer walking away mid-answer (a WS drop, a page reload, a diff --git a/tests/runtime/test_prompt_hard_ceiling.py b/tests/runtime/test_prompt_hard_ceiling.py new file mode 100644 index 000000000..1976dfa07 --- /dev/null +++ b/tests/runtime/test_prompt_hard_ceiling.py @@ -0,0 +1,115 @@ +"""The ACP stream's wall-clock ceiling applies while events keep arriving (CORR-331). + +``prompt_stream``'s hard stop used to be evaluated only in the idle branch — +the one reached after 30s of silence on the event queue. An agent stuck in a +tool-call loop, or a model that keeps narrating, never goes idle, so the +ceiling was never compared and the turn ran forever. Callers that enforce no +budget of their own (``run_agent_to_completion``, reflection, the eager +initial-context prompt) inherited that: they kept relaying, and the agent kept +auto-approving tool calls, long past the policy's deadline. +""" + +import asyncio +import json + +from condor.acp.client import ACPClient, Heartbeat, PromptDone, TextChunk +from condor.runtime import timeouts + + +class _FastCeiling(timeouts.TimeoutPolicy): + """Real policy, sub-second stream ceiling. + + ``prompt_hard_stop`` is derived (``prompt_overall + 60``), so it cannot be + lowered into test range with ``dataclasses.replace``; overriding the + property keeps every other deadline exactly as shipped. + """ + + @property + def prompt_hard_stop(self) -> float: + return 0.3 + + +class _FakeStdin: + """Subprocess stdin that settles session/cancel like a conforming agent.""" + + def __init__(self): + self.sent: list[dict] = [] + self.prompt_id: int | None = None + self.peer = None + self._tasks: list[asyncio.Task] = [] + + def write(self, data: bytes) -> None: + msg = json.loads(data.decode()) + self.sent.append(msg) + if msg.get("method") == "session/prompt": + self.prompt_id = msg.get("id") + elif msg.get("method") == "session/cancel": + self._tasks.append(asyncio.create_task(self._reply_cancelled())) + + async def drain(self) -> None: + pass + + async def _reply_cancelled(self) -> None: + line = json.dumps( + { + "jsonrpc": "2.0", + "id": self.prompt_id, + "result": {"stopReason": "cancelled"}, + } + ) + await self.peer.handle_line(line, self) + + def methods(self) -> list: + return [m.get("method") for m in self.sent] + + +class _FakeProcess: + def __init__(self, stdin: _FakeStdin): + self.stdin = stdin + self.returncode = None + + +def _client() -> ACPClient: + client = ACPClient(command="fake-agent") + stdin = _FakeStdin() + stdin.peer = client._peer + client._process = _FakeProcess(stdin) + client._session_id = "sess-1" + return client + + +def test_ceiling_stops_a_stream_that_never_goes_idle(monkeypatch): + """A chatty agent is cut off at the ceiling, and cancelled at the agent.""" + monkeypatch.setattr(timeouts, "TIMEOUTS", _FastCeiling(prompt_cancel=0.2)) + client = _client() + + async def scenario(): + events: list = [] + + async def chatter(): + """The stuck agent: an event every 20ms, so the queue never empties.""" + while True: + client._event_queue.put_nowait(TextChunk(text=".")) + await asyncio.sleep(0.02) + + noise = asyncio.create_task(chatter()) + try: + async for event in client.prompt_stream("loop forever"): + events.append(event) + finally: + noise.cancel() + return events + + # Without the fix this never returns: the ceiling is only read after 30s of + # silence, which a stream emitting every 20ms never produces. + events = asyncio.run(asyncio.wait_for(scenario(), timeout=10)) + + assert any(isinstance(e, TextChunk) for e in events), "events were relayed" + # The queue never went idle, so no heartbeat: this is the event path. + assert not any(isinstance(e, Heartbeat) for e in events) + + done = events[-1] + assert isinstance(done, PromptDone) + assert done.stop_reason == "timeout" + # Cancelled at the agent, not merely abandoned (CORR-140's reasoning). + assert "session/cancel" in client._process.stdin.methods() From 34bafddf9d73e517fe61f0fb7f686acb9f5a4ea3 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 09:07:33 +0300 Subject: [PATCH 041/154] (fix) an MCP server that dies mid-session no longer looks healthy The lifecycle task recorded a failure only when it fired before the ready event. An MCP stdio server that died later -- subprocess crash, host restart, OOM -- raised on context exit into a branch that was skipped, so the exception went nowhere and nothing was logged. The agent stayed set, `alive` kept reporting True, and the session layer went on handing prompts to a client with no tools left: an agent that mysteriously forgot how to do anything, with empty logs. Now a post-ready collapse is logged with its traceback and drops the agent, so `alive` is False and the session layer builds a fresh client. Cancelling the lifecycle task is propagated instead of being turned into a clean return, so a genuine crash and an orderly shutdown no longer look identical; stop() distinguishes the task's own cancellation from the caller's. Startup failures still reach start() through _startup_error unchanged. --- condor/acp/pydantic_ai_client.py | 44 ++++++- tests/test_pydantic_ai_mcp_lifecycle.py | 145 ++++++++++++++++++++++++ 2 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 tests/test_pydantic_ai_mcp_lifecycle.py diff --git a/condor/acp/pydantic_ai_client.py b/condor/acp/pydantic_ai_client.py index 2430c3100..4e6bcc766 100644 --- a/condor/acp/pydantic_ai_client.py +++ b/condor/acp/pydantic_ai_client.py @@ -421,6 +421,9 @@ def __init__( self._ready_event: asyncio.Event | None = None self._shutdown_event: asyncio.Event | None = None self._startup_error: BaseException | None = None + # Set when the MCP context collapses *after* startup; the agent is + # dropped alongside it so ``alive`` reports False (CORR-332). + self._lifecycle_error: BaseException | None = None # Accumulated turn history — grows with each prompt_stream() call so # the model sees prior turns. A fresh client is created per session/tick, # so history is reset by recreating the client rather than in-place. @@ -691,6 +694,7 @@ async def start(self) -> None: self._ready_event = asyncio.Event() self._shutdown_event = asyncio.Event() self._startup_error = None + self._lifecycle_error = None self._mcp_task = asyncio.create_task(self._run_mcp_lifecycle()) await self._ready_event.wait() @@ -744,15 +748,46 @@ def _ok(name: str) -> bool: return kept async def _run_mcp_lifecycle(self) -> None: - """Background task that holds the MCP server context open.""" + """Background task that holds the MCP server context open. + + A failure *before* ready is handed to ``start()`` through + ``_startup_error``. A failure *after* ready means an MCP server died + under us (subprocess crash, host restart, OOM): log it and tear the + agent down so ``alive`` reports False and the session layer builds a + fresh client, instead of quietly handing prompts to a toolless one + (CORR-332). ``CancelledError`` is never converted into a normal return. + """ try: async with self._agent.run_mcp_servers(): self._ready_event.set() await self._shutdown_event.wait() + except asyncio.CancelledError as exc: + # Unblock start() if we were cancelled before ready, then stay + # visibly cancelled rather than completing "successfully". + if not self._ready_event.is_set(): + self._startup_error = exc + self._ready_event.set() + else: + self._lifecycle_error = exc + self._teardown_after_lifecycle_failure() + raise except BaseException as exc: if not self._ready_event.is_set(): self._startup_error = exc self._ready_event.set() + return + log.exception( + "MCP server lifecycle failed after startup (model=%s); " + "marking client dead so a new one is built", + self.model_name, + ) + self._lifecycle_error = exc + self._teardown_after_lifecycle_failure() + + def _teardown_after_lifecycle_failure(self) -> None: + """Drop the agent so ``alive`` cannot report a toolless client healthy.""" + self._agent = None + self._mcp_servers.clear() async def stop(self) -> None: """Signal the MCP lifecycle task to shut down and wait for it.""" @@ -760,6 +795,13 @@ async def stop(self) -> None: self._shutdown_event.set() try: await asyncio.wait_for(self._mcp_task, timeout=10) + except asyncio.CancelledError: + # The lifecycle task now propagates its own cancellation + # (CORR-332). That is its shutdown, not ours -- only re-raise + # when it is *this* task being cancelled. + if not self._mcp_task.cancelled(): + raise + log.warning("MCP server task was cancelled during shutdown") except Exception: log.exception("Error stopping MCP server task") self._mcp_task.cancel() diff --git a/tests/test_pydantic_ai_mcp_lifecycle.py b/tests/test_pydantic_ai_mcp_lifecycle.py new file mode 100644 index 000000000..4a27b48db --- /dev/null +++ b/tests/test_pydantic_ai_mcp_lifecycle.py @@ -0,0 +1,145 @@ +"""A post-startup MCP collapse must be loud and must kill the client (CORR-332). + +``_run_mcp_lifecycle`` used to record an exception only when it fired *before* +the ready event. If an MCP stdio server died later — subprocess crash, host +restart, tool server OOM — ``run_mcp_servers()`` raised on exit, the exception +was discarded with nothing logged, and ``self._agent`` stayed set. ``alive`` +therefore kept reporting True and the session layer kept handing prompts to an +agent that had no tools left. + +The same clause swallowed ``CancelledError``, so a cancelled lifecycle task +completed "successfully" and ``stop()``'s ``wait_for`` reported no problem. +""" + +import asyncio +import contextlib +import logging + +from condor.acp.pydantic_ai_client import PydanticAIClient + + +class _FakeAgent: + """Stands in for the pydantic-ai ``Agent`` the lifecycle task holds open.""" + + def __init__(self, fail: BaseException | None = None, on_enter: bool = False): + self._fail = fail + self._on_enter = on_enter + self.entered = False + + def run_mcp_servers(self): + @contextlib.asynccontextmanager + async def _ctx(): + if self._fail is not None and self._on_enter: + raise self._fail + self.entered = True + yield + if self._fail is not None: + raise self._fail + + return _ctx() + + +def _client(agent: _FakeAgent) -> PydanticAIClient: + """A client wired up exactly as ``start()`` leaves it, minus the real Agent.""" + client = PydanticAIClient(model="ollama:llama3.1") + client._agent = agent + client._mcp_servers = [object()] + client._ready_event = asyncio.Event() + client._shutdown_event = asyncio.Event() + client._startup_error = None + client._lifecycle_error = None + return client + + +def test_post_startup_mcp_failure_is_logged_and_kills_the_client(caplog): + """The MCP context blows up after ready: log it, and stop claiming alive.""" + boom = RuntimeError("mcp stdio server died") + client = _client(_FakeAgent(fail=boom)) + + async def run(): + client._mcp_task = asyncio.create_task(client._run_mcp_lifecycle()) + await client._ready_event.wait() + assert client.alive is True # healthy while the servers are up + client._shutdown_event.set() # unblocks the wait; __aexit__ then raises + await client._mcp_task + + with caplog.at_level(logging.ERROR, logger="condor.acp.pydantic_ai_client"): + asyncio.run(run()) + + assert client.alive is False, "a toolless client must not report alive" + assert client._lifecycle_error is boom + assert client._mcp_servers == [] + assert any( + rec.levelno >= logging.ERROR and rec.exc_info for rec in caplog.records + ), "the post-startup failure must be logged with its traceback" + # start()'s contract is untouched: this was never a startup failure. + assert client._startup_error is None + + +def test_cancelled_lifecycle_task_is_visibly_cancelled(): + """A cancelled lifecycle task must not complete like a clean shutdown.""" + client = _client(_FakeAgent()) + + async def run(): + client._mcp_task = asyncio.create_task(client._run_mcp_lifecycle()) + await client._ready_event.wait() + client._mcp_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await client._mcp_task + return client._mcp_task + + task = asyncio.run(run()) + + assert task.cancelled(), "cancellation must not be converted into a normal return" + assert client.alive is False + + +def test_startup_failure_still_reaches_start_unchanged(): + """A failure before ready is still handed to ``start()`` via _startup_error.""" + boom = RuntimeError("mcp server never came up") + agent = _FakeAgent(fail=boom, on_enter=True) + client = _client(agent) + + async def run(): + client._mcp_task = asyncio.create_task(client._run_mcp_lifecycle()) + await client._ready_event.wait() + await client._mcp_task # returns normally; start() raises the error + + asyncio.run(run()) + + assert client._startup_error is boom + assert agent.entered is False + + +def test_clean_shutdown_leaves_no_lifecycle_error(): + """The ordinary stop() path stays silent and clears the agent itself.""" + client = _client(_FakeAgent()) + + async def run(): + client._mcp_task = asyncio.create_task(client._run_mcp_lifecycle()) + await client._ready_event.wait() + await client.stop() + + asyncio.run(run()) + + assert client._lifecycle_error is None + assert client._startup_error is None + assert client.alive is False # stop() clears the agent + assert client._mcp_task is None + + +def test_stop_survives_a_lifecycle_task_cancelled_from_outside(): + """stop() must not turn the task's own cancellation into the caller's.""" + client = _client(_FakeAgent()) + + async def run(): + client._mcp_task = asyncio.create_task(client._run_mcp_lifecycle()) + await client._ready_event.wait() + client._mcp_task.cancel() + await asyncio.sleep(0) + await client.stop() # must not raise CancelledError at us + + asyncio.run(run()) + + assert client.alive is False + assert client._mcp_task is None From 58bb881318c65c80f9adccd04fa55af49a0d6597 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 09:16:46 +0300 Subject: [PATCH 042/154] Give bringing an agent up a deadline The ACP handshake awaited initialize and then session/new with no bound, and JSONRPCPeer.send_request ended in a bare `return await future` with no timeout anywhere in the peer. A child that spawns but never answers -- an npx fetch that stalls, a CLI waiting on an interactive prompt it will never get, a bridge blocked on auth -- parked the caller forever: every call site opens its own budget only after start() returns, and _spawn_session waits holding the per-key creation lock, so one mute child blocked every later session for that key. send_request now takes an optional timeout that drops its _pending entry on expiry, start() spends ONE TIMEOUTS.agent_handshake budget across both steps and kills the subprocess tree with the error it already raises, and the pydantic-ai client's equally unbounded `await self._ready_event.wait()` is held to the same deadline, cancelling the MCP lifecycle task so nothing is left running. The deadline is a policy field (CONDOR_TIMEOUT_AGENT_HANDSHAKE), not a literal: 120s, generous enough for a cold npx fetch and still finite. --- condor/acp/client.py | 19 ++++ condor/acp/jsonrpc.py | 23 +++- condor/acp/pydantic_ai_client.py | 42 ++++++- condor/runtime/timeouts.py | 7 ++ tests/runtime/test_acp_handshake_deadline.py | 113 +++++++++++++++++++ 5 files changed, 195 insertions(+), 9 deletions(-) create mode 100644 tests/runtime/test_acp_handshake_deadline.py diff --git a/condor/acp/client.py b/condor/acp/client.py index 69ecdac0e..50950d088 100644 --- a/condor/acp/client.py +++ b/condor/acp/client.py @@ -559,6 +559,16 @@ async def start(self) -> None: self._read_task = asyncio.create_task(self._read_loop()) self._stderr_task = asyncio.create_task(self._drain_stderr()) + # Deferred like the other TIMEOUTS uses in this file: importing the + # runtime package at module scope closes an import cycle. + from condor.runtime.timeouts import TIMEOUTS + + # ONE deadline across both steps, not one each: what the caller is + # owed is a bounded ``start()``. A child that spawns but never answers + # -- an npx fetch that stalls, a CLI waiting on an interactive prompt, + # a bridge blocked on auth -- used to park it forever, and every call + # site opens its own budget only after we return (CORR-333). + deadline = time.monotonic() + TIMEOUTS.agent_handshake try: handshake = await self._peer.send_request( "initialize", @@ -568,12 +578,21 @@ async def start(self) -> None: "clientInfo": {"name": "condor", "version": "0.1.0"}, }, self._process.stdin, + timeout=max(0.0, deadline - time.monotonic()), ) result = await self._peer.send_request( "session/new", self._session_new_params(), self._process.stdin, + timeout=max(0.0, deadline - time.monotonic()), ) + except asyncio.TimeoutError: + await self.stop() + raise TimeoutError( + f"The agent did not complete the ACP handshake within " + f"{TIMEOUTS.agent_handshake}s and was killed (cmd={self.command}). " + f"Check that the command runs and speaks ACP on stdio." + ) from None except Exception: # Handshake failed -- kill the subprocess to prevent orphan await self.stop() diff --git a/condor/acp/jsonrpc.py b/condor/acp/jsonrpc.py index 71e57caec..a75ffbe17 100644 --- a/condor/acp/jsonrpc.py +++ b/condor/acp/jsonrpc.py @@ -48,9 +48,20 @@ def register_handler(self, method: str, handler: Callable) -> None: self._handlers[method] = handler async def send_request( - self, method: str, params: dict[str, Any], writer: asyncio.StreamWriter + self, + method: str, + params: dict[str, Any], + writer: asyncio.StreamWriter, + timeout: float | None = None, ) -> Any: - """Send a JSON-RPC request and wait for the response.""" + """Send a JSON-RPC request and wait for the response. + + ``timeout`` bounds that wait: nothing else in the peer does, so a child + that reads our line and never answers parks the caller forever + (CORR-333). On expiry the pending entry is dropped -- an abandoned + request must not leak a future that only ``cancel_all`` would ever + clear -- and :class:`asyncio.TimeoutError` propagates to the caller. + """ req_id = self._next_id self._next_id += 1 @@ -62,7 +73,13 @@ async def send_request( future: asyncio.Future[Any] = asyncio.get_event_loop().create_future() self._pending[req_id] = future - return await future + if timeout is None: + return await future + try: + return await asyncio.wait_for(future, timeout) + except (asyncio.TimeoutError, asyncio.CancelledError): + self._pending.pop(req_id, None) + raise async def send_notification( self, method: str, params: dict[str, Any], writer: asyncio.StreamWriter diff --git a/condor/acp/pydantic_ai_client.py b/condor/acp/pydantic_ai_client.py index 4e6bcc766..9ffc40c37 100644 --- a/condor/acp/pydantic_ai_client.py +++ b/condor/acp/pydantic_ai_client.py @@ -697,12 +697,7 @@ async def start(self) -> None: self._lifecycle_error = None self._mcp_task = asyncio.create_task(self._run_mcp_lifecycle()) - await self._ready_event.wait() - if self._startup_error is not None: - self._mcp_task = None - self._mcp_servers.clear() - self._agent = None - raise self._startup_error + await self._await_ready() log.info( "PydanticAI client ready: model=%s, mcp_servers=%d", @@ -710,6 +705,41 @@ async def start(self) -> None: len(self._mcp_servers), ) + async def _await_ready(self) -> None: + """Wait for the MCP lifecycle task to come up, under a deadline. + + The wait used to be a bare ``self._ready_event.wait()`` -- the same + unbounded shape as the ACP handshake, with the same failure: an MCP + stdio server that spawns and never finishes its own init parks + ``start()`` forever, and with it the per-key session-creation lock the + caller holds (CORR-333). On expiry the lifecycle task is cancelled so + no MCP subprocess is left behind, and the client is left visibly dead. + """ + from condor.runtime.timeouts import TIMEOUTS + + try: + await asyncio.wait_for( + self._ready_event.wait(), timeout=TIMEOUTS.agent_handshake + ) + except asyncio.TimeoutError: + task, self._mcp_task = self._mcp_task, None + if task is not None: + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + self._mcp_servers.clear() + self._agent = None + raise TimeoutError( + f"MCP servers for {self.model_name} did not become ready within " + f"{TIMEOUTS.agent_handshake}s; the agent was not started." + ) from None + + if self._startup_error is not None: + self._mcp_task = None + self._mcp_servers.clear() + self._agent = None + raise self._startup_error + def _gate_toolsets(self, toolsets: list) -> list: """Wrap toolsets so a denied call never reaches the tool (SEC-080). diff --git a/condor/runtime/timeouts.py b/condor/runtime/timeouts.py index 61919fe22..4d9f0baa9 100644 --- a/condor/runtime/timeouts.py +++ b/condor/runtime/timeouts.py @@ -50,6 +50,13 @@ class TimeoutPolicy: sse_stream: int = 1800 # Budget for one MCP tool call. mcp_call: float = 15.0 + # How long an agent client may take to become usable: the ACP + # ``initialize`` + ``session/new`` handshake, and the pydantic-ai wait for + # its MCP servers to come up. Generous because a cold start can include an + # ``npx`` fetch of the bridge, but bounded: a child that spawns and never + # answers used to park the caller forever, holding a per-user session slot + # and the session-creation lock behind it (CORR-333). + agent_handshake: int = 120 # Wall-clock budget for one agent session: a strategy tick's LLM turn, and # the shutdown cleanup pass that runs under the same ceiling. 10 minutes. tick_default: int = 600 diff --git a/tests/runtime/test_acp_handshake_deadline.py b/tests/runtime/test_acp_handshake_deadline.py new file mode 100644 index 000000000..18031d502 --- /dev/null +++ b/tests/runtime/test_acp_handshake_deadline.py @@ -0,0 +1,113 @@ +"""Bringing an agent up has a deadline (CORR-333). + +``ACPClient.start()`` awaited ``initialize`` and then ``session/new`` with no +bound, and ``JSONRPCPeer.send_request`` ended in a bare ``return await future`` +with no timeout anywhere in the peer. A child that spawns but never answers -- +an ``npx`` fetch that stalls, a CLI waiting on an interactive prompt it will +never get, a bridge blocked on auth -- therefore parked the caller forever. +Every call site opens its own budget only *after* ``start()`` returns, and +``_spawn_session`` holds the per-key creation lock while it waits, so one mute +child blocked every later session for that key. + +The pydantic-ai client's ``await self._ready_event.wait()`` had the same shape +and is bounded by the same policy field. +""" + +import asyncio +import contextlib +import dataclasses + +import pytest + +from condor.acp.client import ACPClient +from condor.acp.jsonrpc import JSONRPCPeer +from condor.acp.pydantic_ai_client import PydanticAIClient +from condor.runtime import timeouts + + +def _fast(seconds: float): + """The real policy with only the handshake deadline pulled into test range.""" + return dataclasses.replace(timeouts.TIMEOUTS, agent_handshake=seconds) + + +class _FakeWriter: + """Subprocess stdin that swallows everything and never answers.""" + + def __init__(self): + self.written: list[bytes] = [] + + def write(self, data: bytes) -> None: + self.written.append(data) + + async def drain(self) -> None: + pass + + +def test_send_request_timeout_drops_the_pending_entry(): + """An abandoned request must not leak the future that only shutdown clears.""" + peer = JSONRPCPeer() + + async def scenario(): + with pytest.raises(asyncio.TimeoutError): + await peer.send_request("initialize", {}, _FakeWriter(), timeout=0.05) + + asyncio.run(scenario()) + assert peer._pending == {}, "the expired request left an entry behind" + + +def test_start_gives_up_on_an_agent_that_never_answers(monkeypatch): + """A spawned-but-mute agent fails start() at the deadline, killed.""" + monkeypatch.setattr(timeouts, "TIMEOUTS", _fast(0.3)) + # A real subprocess that holds the pipes open and says nothing: exactly the + # shape of a bridge stuck fetching itself. + client = ACPClient(command="sleep 30") + + async def scenario(): + with pytest.raises(TimeoutError) as excinfo: + # Without the deadline this never returns -- the outer wait_for is + # what keeps the unfixed code from hanging the suite. + await client.start() + return excinfo.value + + error = asyncio.run(asyncio.wait_for(scenario(), timeout=10)) + + assert "handshake" in str(error), f"unhelpful error: {error}" + assert client.alive is False, "the mute subprocess was left running" + assert client._process is None, "stop() did not run on the timeout path" + assert client._peer._pending == {}, "the abandoned handshake leaked a future" + + +class _NeverReadyAgent: + """A pydantic-ai agent whose MCP servers never finish coming up.""" + + @contextlib.asynccontextmanager + async def _ctx(self): + await asyncio.Event().wait() # never entered + yield # pragma: no cover + + def run_mcp_servers(self): + return self._ctx() + + +def test_pydantic_ai_start_gives_up_on_mcp_servers_that_never_come_up(monkeypatch): + """The same deadline bounds the pydantic-ai ready wait.""" + monkeypatch.setattr(timeouts, "TIMEOUTS", _fast(0.3)) + client = PydanticAIClient(model="ollama:llama3.1") + client._agent = _NeverReadyAgent() + client._mcp_servers = [object()] + client._ready_event = asyncio.Event() + client._shutdown_event = asyncio.Event() + client._startup_error = None + client._lifecycle_error = None + + async def scenario(): + task = asyncio.create_task(client._run_mcp_lifecycle()) + client._mcp_task = task + with pytest.raises(TimeoutError): + await client._await_ready() + return task + + task = asyncio.run(asyncio.wait_for(scenario(), timeout=10)) + + assert task.cancelled(), "the MCP lifecycle task was left running" + assert client.alive is False, "a client that never came up must not look alive" From c967c6ecedcb7d94d3b78e7157799f85dd999471 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 09:24:41 +0300 Subject: [PATCH 043/154] (fix) an agent that cannot launch reports an error, not a cancellation When the agent command cannot run, stdout hits EOF and the read loop swept every pending future with cancel(). The handshake in start() was parked on one of them, so it raised CancelledError -- a BaseException that the guard right below it does not catch. The subprocess that guard promises to reap was left running on exactly the path it was written for, and every `except Exception` between there and the user read the failure as "the user pressed Stop": a chat whose bridge is simply missing died silently. The read loop now fails those futures with a ConnectionError naming the command instead. cancel_all() stays for the shutdown we initiate ourselves, where a cancellation is the truth. The failure is sticky on the peer so a request that races the EOF -- registered just after the sweep -- fails with the same real error rather than waiting out the handshake deadline for an answer that can never come. --- condor/acp/client.py | 9 +- condor/acp/jsonrpc.py | 41 +++++++- tests/runtime/test_acp_child_death_error.py | 103 ++++++++++++++++++++ 3 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 tests/runtime/test_acp_child_death_error.py diff --git a/condor/acp/client.py b/condor/acp/client.py index 50950d088..29def7b71 100644 --- a/condor/acp/client.py +++ b/condor/acp/client.py @@ -763,7 +763,14 @@ async def _read_loop(self) -> None: # Subprocess died or stream ended -- unblock any consumer waiting on # _event_queue, and stop claiming to be alive: we can no longer hear. self._read_loop_ended = True - self._peer.cancel_all() + # Fail the pending futures rather than cancelling them: the handshake + # in start() and any in-flight turn are parked on them, and a + # CancelledError there is a BaseException that start()'s + # `except Exception` guard cannot catch -- so a command that cannot run + # left its subprocess orphaned on exactly the path that guard was + # written for, and the caller saw a cancellation instead of a broken + # agent (CORR-329). + self._peer.fail_all(ConnectionError(f"ACP agent exited: {self.command}")) self._event_queue.put_nowait(PromptDone(stop_reason="disconnected")) async def _drain_stderr(self) -> None: diff --git a/condor/acp/jsonrpc.py b/condor/acp/jsonrpc.py index a75ffbe17..416c2e477 100644 --- a/condor/acp/jsonrpc.py +++ b/condor/acp/jsonrpc.py @@ -43,6 +43,11 @@ def __init__(self): self._next_id = 1 self._pending: dict[int, asyncio.Future] = {} self._handlers: dict[str, Callable] = {} + # Set by :meth:`fail_all` when the connection dies for good. Sticky, so + # a request that races the EOF -- registered a moment *after* the read + # loop swept the pending table -- fails with the same real error + # instead of parking on a future nobody will ever settle (CORR-329). + self._failure: BaseException | None = None def register_handler(self, method: str, handler: Callable) -> None: self._handlers[method] = handler @@ -62,6 +67,9 @@ async def send_request( request must not leak a future that only ``cancel_all`` would ever clear -- and :class:`asyncio.TimeoutError` propagates to the caller. """ + if self._failure is not None: + raise self._failure + req_id = self._next_id self._next_id += 1 @@ -72,6 +80,11 @@ async def send_request( log.debug("-> %s (id=%d)", method, req_id) future: asyncio.Future[Any] = asyncio.get_event_loop().create_future() + # Checked again after the drain above: the peer can die while we are + # writing, and a future registered after that sweep would wait out the + # whole timeout for an answer that can never come. + if self._failure is not None: + raise self._failure self._pending[req_id] = future if timeout is None: return await future @@ -169,8 +182,34 @@ async def handle_line(self, line: str, writer: asyncio.StreamWriter) -> None: await writer.drain() def cancel_all(self) -> None: - """Cancel all pending futures (used during shutdown).""" + """Cancel all pending futures (used during our own shutdown).""" for future in self._pending.values(): if not future.done(): future.cancel() self._pending.clear() + + def fail_all(self, exc: BaseException) -> None: + """Settle every pending future with ``exc``: the connection is gone. + + Not :meth:`cancel_all`. A cancelled future raises ``CancelledError`` + into whoever awaits it, and that is a ``BaseException`` that every + ``except Exception`` between here and the user walks straight past -- + asyncio and the callers alike read it as "this task was cancelled" + rather than "the agent died", so a launch that failed surfaced as a + silent cancellation and the caller's cleanup never ran (CORR-329). + An exception says what happened and is catchable. + + Use it when the peer stopped being able to answer (EOF on stdout); + ``cancel_all`` stays for the shutdown *we* initiate, where a + cancellation is the truth. + """ + self._failure = exc + for future in self._pending.values(): + if not future.done(): + future.set_exception(exc) + # Retrieve it here so a future nobody awaits any more -- a + # stale prompt settled only by its done-callback -- does not + # log "exception was never retrieved" when it is collected. + # A real awaiter still gets it raised. + future.exception() + self._pending.clear() diff --git a/tests/runtime/test_acp_child_death_error.py b/tests/runtime/test_acp_child_death_error.py new file mode 100644 index 000000000..72b7de5eb --- /dev/null +++ b/tests/runtime/test_acp_child_death_error.py @@ -0,0 +1,103 @@ +"""A dead ACP child must surface as an error, not as a cancellation (CORR-329). + +When the agent command cannot run -- bridge not installed, wrong node, a shell +printing "command not found" -- stdout hits EOF and the read loop sweeps the +pending futures on its way out. Sweeping them with ``cancel()`` raised +``CancelledError`` into the handshake, which is a ``BaseException``: the +``except Exception`` guard in ``start()`` did not catch it, so the subprocess it +promises to reap was left running, and every caller between here and the user +read the failure as "the user pressed Stop". +""" + +import asyncio +import json + +import pytest + +from condor.acp.client import ACPClient +from condor.acp.jsonrpc import JSONRPCPeer + +_MISSING = "condor-definitely-not-a-real-acp-binary-xyz" + + +class _FakeStdin: + def __init__(self) -> None: + self.written: list[dict] = [] + + def write(self, data: bytes) -> None: + self.written.append(json.loads(data.decode())) + + async def drain(self) -> None: + pass + + +class _FakeProcess: + def __init__(self, stdout: asyncio.StreamReader) -> None: + self.stdout = stdout + self.stdin = _FakeStdin() + self.returncode = None + + +def _client(stdout: asyncio.StreamReader) -> ACPClient: + client = ACPClient(command="true") + client._process = _FakeProcess(stdout) # type: ignore[assignment] + return client + + +@pytest.mark.asyncio +async def test_a_command_that_cannot_run_fails_start_with_a_catchable_error(): + client = ACPClient(command=_MISSING) + + with pytest.raises(Exception) as excinfo: # noqa: B017 - the point is the type + await asyncio.wait_for(client.start(), timeout=30) + + # Not a CancelledError: `except Exception` has to be able to see this. + assert not isinstance(excinfo.value, asyncio.CancelledError) + assert isinstance(excinfo.value, ConnectionError) + assert _MISSING in str(excinfo.value) + # ...and because it was catchable, start()'s guard reaped the subprocess. + assert client._process is None + + +@pytest.mark.asyncio +async def test_the_read_loop_hands_a_pending_turn_the_real_error(): + stdout = asyncio.StreamReader() + client = _client(stdout) + future: asyncio.Future = asyncio.get_event_loop().create_future() + client._peer._pending[1] = future + + stdout.feed_eof() # the child died mid-turn + await asyncio.wait_for(client._read_loop(), timeout=5) + + assert not future.cancelled() + assert isinstance(future.exception(), ConnectionError) + assert client.command in str(future.exception()) + # The consumer of prompt_stream still gets its terminal event. + assert client._event_queue.get_nowait().stop_reason == "disconnected" + + +@pytest.mark.asyncio +async def test_a_request_that_races_the_eof_fails_instead_of_parking(): + """The sweep can land between the write and the future's registration.""" + peer = JSONRPCPeer() + peer.fail_all(ConnectionError("ACP agent exited: nope")) + + with pytest.raises(ConnectionError): + await asyncio.wait_for( + peer.send_request("initialize", {}, _FakeStdin(), timeout=5), # type: ignore[arg-type] + timeout=5, + ) + + +@pytest.mark.asyncio +async def test_our_own_shutdown_still_cancels_pending_futures(): + """``stop()`` is a cancellation, and must stay one -- no error logged.""" + stdout = asyncio.StreamReader() + client = _client(stdout) + client._process.returncode = 0 # already exited: nothing to reap + future: asyncio.Future = asyncio.get_event_loop().create_future() + client._peer._pending[1] = future + + await client.stop() + + assert future.cancelled() From aebe597fda989440239bb22bdae46a057d80d199 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 09:33:25 +0300 Subject: [PATCH 044/154] (fix) stopping a turn mid-confirmation no longer widens backend concurrency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-server request slot changes hands twice per confirmation: it is handed back while a human decides, then taken again afterwards. Web Stop cancels the prompt task outright, so a cancellation lands in that window by construction — and the two independent context managers guarding it could not stay balanced across one. Teardown parked in `await sem.acquire()` behind another session's inference, dragging the session lock with it; and a second cancellation during that re-acquire raised with the slot NOT held while the outer `async with sem` still released, permanently adding a permit to the process-global, never-rebuilt `_SERVER_SEMAPHORES`. Each occurrence silently undid the serialization that keeps LM Studio/Ollama from ConnectTimeout-ing against themselves. Ownership is now explicit: `_slot_held` is the single source of truth, the outer guard releases only what it holds, and a cancelled confirmation simply does not queue for a slot it would hand straight back. Cloud providers (semaphore None) stay a no-op through both paths. --- condor/acp/pydantic_ai_client.py | 55 ++++++++++++++++- tests/test_agents.py | 103 +++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 3 deletions(-) diff --git a/condor/acp/pydantic_ai_client.py b/condor/acp/pydantic_ai_client.py index 9ffc40c37..64833fac3 100644 --- a/condor/acp/pydantic_ai_client.py +++ b/condor/acp/pydantic_ai_client.py @@ -414,6 +414,12 @@ def __init__( # are serialized. Stays None for natively-resolved cloud providers # (anthropic/groq/default openai/google), which handle concurrency fine. self._request_semaphore: asyncio.Semaphore | None = None + # Whether *this* client currently owns a permit of that semaphore. The + # slot changes hands twice per confirmation (released for the human + # wait, re-acquired after), and a cancellation can land in either gap — + # so ownership is tracked explicitly rather than inferred from nesting, + # and only the owner ever releases (CORR-330). + self._slot_held = False # Background task that owns the MCP server cancel scopes. # anyio requires cancel scopes to be entered/exited in the same task, # so we can't close them from an arbitrary caller task. @@ -843,6 +849,32 @@ async def stop(self) -> None: def alive(self) -> bool: return self._agent is not None + @contextlib.asynccontextmanager + async def _hold_request_slot(self) -> AsyncIterator[None]: + """Hold the per-server request slot for the duration of one turn. + + Deliberately not ``async with sem``: the slot is handed back and taken + again mid-turn by :meth:`_release_request_slot`, so a context manager + that releases unconditionally on exit would hand back a permit this + client no longer owns whenever the turn ends inside that window — + permanently widening concurrency against the shared, process-global + semaphore. ``_slot_held`` is the single source of truth (CORR-330). + + No-op for cloud providers, whose semaphore is None (PERF-038). + """ + sem = self._request_semaphore + if sem is None: + yield + return + await sem.acquire() + self._slot_held = True + try: + yield + finally: + if self._slot_held: + self._slot_held = False + sem.release() + @contextlib.asynccontextmanager async def _release_request_slot(self) -> AsyncIterator[None]: """Temporarily release the per-server request slot for a blocking wait. @@ -857,16 +889,33 @@ async def _release_request_slot(self) -> AsyncIterator[None]: Releases the slot on entry and re-acquires it before returning, so model HTTP work stays serialized. No-op for cloud providers, whose semaphore is None (PERF-038). + + When the wait is *cancelled* the slot is deliberately not re-acquired + (CORR-330). Web Stop cancels the prompt task outright, which lands in + this window by construction; queueing for a slot we would immediately + hand back would park the turn's teardown — and the session lock it + carries — behind another session's inference. ``_slot_held`` stays + False so :meth:`_hold_request_slot` skips a release it does not own. """ sem = self._request_semaphore if sem is None: yield return + self._slot_held = False sem.release() + cancelled = False try: yield + except asyncio.CancelledError: + cancelled = True + raise finally: - await sem.acquire() + if not cancelled: + # A cancellation landing here instead raises out of acquire() + # without taking a permit, leaving _slot_held False — which is + # exactly the state the outer guard needs to stay balanced. + await sem.acquire() + self._slot_held = True async def prompt(self, text: str) -> str: """One-shot prompt: send text, return response.""" @@ -910,8 +959,8 @@ async def prompt_stream( # one request at a time. Without this, concurrent ticks race to connect # and the losing ticks ConnectTimeout against a busy server. Cloud # providers leave the semaphore None (see start()) so concurrent prompts - # run in parallel; nullcontext() makes the guard a no-op for them. - async with self._request_semaphore or contextlib.nullcontext(): + # run in parallel; the guard is a no-op for them. + async with self._hold_request_slot(): start_time = time.monotonic() self._abort_requested = False aborted = False diff --git a/tests/test_agents.py b/tests/test_agents.py index 47ea667dd..a2251d0fd 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -468,6 +468,109 @@ async def _run(): asyncio.run(_run()) +# ── cancelling a turn mid-confirmation (CORR-330) ── + + +async def _count_permits(sem): + """How many concurrent requests the backend could receive right now.""" + taken = 0 + while True: + try: + await asyncio.wait_for(sem.acquire(), timeout=0.05) + except asyncio.TimeoutError: + break + taken += 1 + for _ in range(taken): + sem.release() + return taken + + +def _confirming_turn(client, started): + """A turn holding the slot, parked on a human confirmation.""" + + async def _turn(): + async with client._hold_request_slot(): + async with client._release_request_slot(): + started.set() + await asyncio.sleep(3600) # the human is deciding + + return _turn + + +def test_cancel_during_confirmation_does_not_park_teardown(): + # CORR-330 (a): Web Stop cancels the prompt task, which lands inside the + # confirmation window by construction. Teardown must not queue for a slot + # another session is holding — that parks the session lock the teardown + # carries, and the user's next message waits behind foreign inference. + async def _run(): + sem = asyncio.Semaphore(1) + client = PydanticAIClient(model="ollama:x") + client._request_semaphore = sem + started = asyncio.Event() + + task = asyncio.create_task(_confirming_turn(client, started)()) + await started.wait() + # A concurrent session takes the slot freed for the confirmation. + await asyncio.wait_for(sem.acquire(), timeout=0.5) + + task.cancel() + done, _ = await asyncio.wait([task], timeout=0.5) + assert done, "teardown parked waiting for a slot another session holds" + assert task.cancelled() + + sem.release() # the concurrent session finishes + assert await _count_permits(sem) == 1 + + asyncio.run(_run()) + + +def test_cancel_during_confirmation_preserves_semaphore_permits(): + # CORR-330 (b): a second cancellation landing while the slot is being taken + # back must not leave the turn's guard releasing a permit it does not hold. + # _SERVER_SEMAPHORES is a process-global cache that is never rebuilt, so a + # single leaked permit permanently widens concurrency against the backend. + async def _run(): + sem = asyncio.Semaphore(1) + client = PydanticAIClient(model="ollama:x") + client._request_semaphore = sem + started = asyncio.Event() + + for _ in range(3): + started.clear() + task = asyncio.create_task(_confirming_turn(client, started)()) + await started.wait() + await asyncio.wait_for(sem.acquire(), timeout=0.5) # foreign session + + task.cancel() # lands in the confirmation wait + for _ in range(5): + if task.done(): + break + await asyncio.sleep(0) + task.cancel() # and again, in the re-acquire + done, _ = await asyncio.wait([task], timeout=0.5) + assert done, "teardown parked waiting for a slot another session holds" + assert task.cancelled() + + sem.release() # the foreign session finishes + assert not client._slot_held + assert await _count_permits(sem) == 1 + + asyncio.run(_run()) + + +def test_hold_request_slot_noop_for_cloud_providers(): + # CORR-330: the outer guard is explicit acquire/release now; cloud providers + # (semaphore None) must still pass straight through it. + async def _run(): + client = PydanticAIClient(model="anthropic:claude-sonnet-4-6") + client._request_semaphore = None + async with client._hold_request_slot(): + pass + assert not client._slot_held + + asyncio.run(_run()) + + def test_resolve_base_url_distinguishes_cloud_from_local_backends(): # PERF-038: only backends with a resolved base URL get serialized. Cloud # providers pydantic-ai resolves natively return None (no semaphore). From 348bcc70bf81eaced6c84eee541c75826bc1e7d5 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 09:40:47 +0300 Subject: [PATCH 045/154] (fix) a partial barrier no longer wipes the barriers the user saved MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Executor defaults were merged one level: `{**defaults, **user_config}`. A key that is a dict on both sides was replaced wholesale, so the grid tool — which always sends `triple_barrier_config`, even empty, because the backend schema requires it — wiped the block the shipped preferences write on first run. A bare `create_grid_executor(...)` therefore deployed a grid with no take-profit and taker exits, a different strategy from the one on file and nothing said so. The same replace dropped a saved stop-loss from a funded position whenever the call named only a take-profit, and `update_defaults` truncated a stored block on save the same way. Both merges now descend one level: where a key is a dict on both sides the two are merged, with the call still winning key by key. Scalars and lists keep replace semantics. The grid lifetime tests read the real preferences file; with a nested merge a developer's own saved barrier would leak into them, so they now stub the defaults away — what they pin is the wiring, not the machine they run on. --- .../hummingbot_api/executor_preferences.py | 34 ++++- tests/test_grid_executor_time_limit.py | 13 ++ tests/test_typed_executor_tools.py | 117 ++++++++++++++++++ 3 files changed, 159 insertions(+), 5 deletions(-) diff --git a/mcp_servers/hummingbot_api/executor_preferences.py b/mcp_servers/hummingbot_api/executor_preferences.py index 1d639dd55..7cc2e15c3 100644 --- a/mcp_servers/hummingbot_api/executor_preferences.py +++ b/mcp_servers/hummingbot_api/executor_preferences.py @@ -124,6 +124,28 @@ """ +def _merge_one_level( + defaults: dict[str, Any], overrides: dict[str, Any] +) -> dict[str, Any]: + """Merge ``overrides`` over ``defaults``, descending one level into nested blocks. + + A plain ``{**defaults, **overrides}`` replaces a nested block wholesale, so a + create that sends ``triple_barrier_config={}`` (the grid tool always sends the + block, because the backend schema requires it) or only ``{"take_profit": ...}`` + silently wiped every other barrier the user had saved — deploying a different + strategy from the one on file, with no message saying so. Where a key is a dict + on BOTH sides the two are merged instead, with the override still winning + key-by-key. Scalars and lists keep replace semantics: a saved list is a whole + value, not something to append to. + """ + merged = {**defaults, **overrides} + for key, default_value in defaults.items(): + override_value = overrides.get(key) + if isinstance(default_value, dict) and isinstance(override_value, dict): + merged[key] = {**default_value, **override_value} + return merged + + class ExecutorPreferencesManager: """Manager for executor preferences stored in markdown format.""" @@ -217,7 +239,8 @@ def update_defaults(self, executor_type: str, config: dict[str, Any]) -> None: """Update default configuration for an executor type. Merges new config with existing defaults so that only the provided - keys are updated while previously saved keys are preserved. + keys are updated while previously saved keys are preserved — a partial + nested block updates that block rather than truncating it. Args: executor_type: The executor type to update @@ -227,7 +250,7 @@ def update_defaults(self, executor_type: str, config: dict[str, Any]) -> None: # Merge with existing defaults so we don't lose previously saved keys existing_defaults = self.get_defaults(executor_type) - merged_config = {**existing_defaults, **config} + merged_config = _merge_one_level(existing_defaults, config) # Create the new YAML block new_yaml = yaml.dump( @@ -285,7 +308,9 @@ def merge_with_defaults( ) -> dict[str, Any]: """Merge user configuration with stored defaults. - User-provided values take precedence over defaults. + User-provided values take precedence over defaults, key by key — including + inside a nested block such as ``triple_barrier_config``, which is merged + rather than replaced (see :func:`_merge_one_level`). Args: executor_type: The executor type @@ -295,8 +320,7 @@ def merge_with_defaults( Merged configuration with defaults filled in """ defaults = self.get_defaults(executor_type) - merged = {**defaults, **user_config} - return merged + return _merge_one_level(defaults, user_config) def get_raw_content(self) -> str: """Get the raw markdown content of the preferences file. diff --git a/tests/test_grid_executor_time_limit.py b/tests/test_grid_executor_time_limit.py index d69bebc32..d9e2dede1 100644 --- a/tests/test_grid_executor_time_limit.py +++ b/tests/test_grid_executor_time_limit.py @@ -51,6 +51,19 @@ def _fresh_cache(): trading_rules_cache.clear() +@pytest.fixture(autouse=True) +def _no_saved_defaults(monkeypatch): + """What is pinned here is the wiring, not the developer's own preferences file. + + Saved defaults merge underneath a create — a nested barrier block included — so + without this the assertions below would read whichever barrier the machine + running the tests happens to have on disk. + """ + monkeypatch.setattr( + executor_create.executor_preferences, "get_defaults", lambda executor_type: {} + ) + + def _grid(client, **overrides): kwargs = { "connector_name": "binance_perpetual", diff --git a/tests/test_typed_executor_tools.py b/tests/test_typed_executor_tools.py index 2f40432de..8ad8203f5 100644 --- a/tests/test_typed_executor_tools.py +++ b/tests/test_typed_executor_tools.py @@ -397,3 +397,120 @@ def test_a_read_tool_is_never_gated(): ) ) assert result["outcome"]["outcome"] == "selected", f"{name} was gated" + + +# --------------------------------------------------------------------------- +# A nested block merges too — it is not replaced wholesale (CORR-560) +# --------------------------------------------------------------------------- + + +@pytest.fixture +def shipped_preferences(tmp_path, monkeypatch): + """The preferences a fresh install actually gets, on a throwaway path. + + Not a stub: the shipped template is the thing under test here, since it writes + an ACTIVE grid ``triple_barrier_config`` on first run. + """ + from mcp_servers.hummingbot_api.executor_preferences import ( + ExecutorPreferencesManager, + ) + + manager = ExecutorPreferencesManager(tmp_path / "executor_preferences.md") + monkeypatch.setattr(executor_create, "executor_preferences", manager) + return manager + + +def test_the_shipped_grid_barrier_survives_a_create_that_names_no_barrier( + shipped_preferences, +): + """The grid tool always sends the block (the backend schema requires it). + + An empty one used to replace the saved block, deploying a grid with no + take-profit and taker exits — a different strategy from the one on file. + """ + client = _RecordingClient() + + asyncio.run( + executor_create.create_grid_executor( + client, + connector_name="binance_perpetual", + trading_pair="SOL-USDT", + side=1, + start_price=140, + end_price=150, + limit_price=138, + total_amount_quote=500, + ) + ) + + barrier = client.calls[0]["config"]["triple_barrier_config"] + assert barrier["take_profit"] == 0.0002 + assert barrier["open_order_type"] == 3 + assert barrier["take_profit_order_type"] == 3 + + +def test_an_explicit_barrier_wins_key_by_key_and_the_rest_fills_in( + shipped_preferences, +): + client = _RecordingClient() + + asyncio.run( + executor_create.create_grid_executor( + client, + connector_name="binance_perpetual", + trading_pair="SOL-USDT", + side=1, + start_price=140, + end_price=150, + limit_price=138, + total_amount_quote=500, + take_profit=0.002, + ) + ) + + barrier = client.calls[0]["config"]["triple_barrier_config"] + assert barrier["take_profit"] == 0.002, "the explicit argument must win" + assert barrier["open_order_type"] == 3, "the saved sibling must still fill in" + + +def test_a_saved_stop_loss_survives_a_create_that_passes_only_a_take_profit( + monkeypatch, +): + """The funded-position case: a partial barrier must not drop the other legs.""" + client = _RecordingClient() + monkeypatch.setattr( + executor_create.executor_preferences, + "get_defaults", + lambda executor_type: { + "triple_barrier_config": {"stop_loss": 0.01, "open_order_type": 3} + }, + ) + + asyncio.run( + executor_create.create_position_executor( + client, + connector_name="binance_perpetual", + trading_pair="BTC-USDT", + side=1, + amount=0.01, + take_profit=0.02, + ) + ) + + barrier = client.calls[0]["config"]["triple_barrier_config"] + assert barrier["stop_loss"] == 0.01, "the saved stop-loss must not be dropped" + assert barrier["take_profit"] == 0.02 + assert barrier["open_order_type"] == 3 + + +def test_saving_a_partial_barrier_updates_the_stored_block_rather_than_truncating_it( + shipped_preferences, +): + """``save_as_default`` writes through the same merge, so a save cannot truncate.""" + shipped_preferences.update_defaults( + "grid_executor", {"triple_barrier_config": {"take_profit": 0.005}} + ) + + stored = shipped_preferences.get_defaults("grid_executor")["triple_barrier_config"] + assert stored["take_profit"] == 0.005 + assert stored["open_order_type"] == 3, "the untouched key must survive the save" From 9a371e4559115773ea87565c5c0b2e65d688af97 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 09:47:34 +0300 Subject: [PATCH 046/154] Wire up the Gateway container tool's two halves so it stops failing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit manage_gateway_container mounts on the admin ring, but its body called two names server.py never bound: manage_gateway_container_impl and format_gateway_container_result. Registration succeeded regardless — the resolver only needs the wrapper, and the missing names are looked up at call time — so every one of the five actions came back through @handle_errors as "Failed to manage Gateway container: name 'manage_gateway_container_impl' is not defined". That is also the tool GATEWAY_LOG_HINT points the model at whenever a swap or LP action fails opaquely, so the documented escape hatch from a Gateway failure answered with a second, more confusing error. Import the impl beside its manage_gateway_config sibling and the formatter in the formatters block, and drop the three formatter names server.py imported but never used — format_active_bots_as_table, format_bot_logs_as_table and format_portfolio_as_table, whose real consumers import them from the package directly — since a block of unused names is what let the missing pair hide. The existing tests only pinned the tool's name and its profile ring, never its body, which is why a dangling call survived; the two added here drive get_status and get_logs against a stub client and fail with the NameError without the imports. --- mcp_servers/hummingbot_api/server.py | 7 ++- tests/test_hummingbot_mcp_tools.py | 88 ++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/mcp_servers/hummingbot_api/server.py b/mcp_servers/hummingbot_api/server.py index 772907636..6d688fa86 100644 --- a/mcp_servers/hummingbot_api/server.py +++ b/mcp_servers/hummingbot_api/server.py @@ -15,14 +15,12 @@ from mcp_servers._profiles import register_tools as _register_tools from mcp_servers._profiles import resolve_profiles from mcp_servers.hummingbot_api.formatters import ( - format_active_bots_as_table, format_amm_result, - format_bot_logs_as_table, format_clmm_result, format_gateway_clmm_pool_result, format_gateway_config_result, + format_gateway_container_result, format_gateway_swap_result, - format_portfolio_as_table, ) from mcp_servers.hummingbot_api.hummingbot_client import hummingbot_client from mcp_servers.hummingbot_api.middleware import GATEWAY_LOG_HINT, handle_errors @@ -47,6 +45,9 @@ from mcp_servers.hummingbot_api.tools.gateway import ( manage_gateway_config as manage_gateway_config_impl, ) +from mcp_servers.hummingbot_api.tools.gateway import ( + manage_gateway_container as manage_gateway_container_impl, +) from mcp_servers.hummingbot_api.tools.gateway_amm import manage_amm_impl from mcp_servers.hummingbot_api.tools.gateway_clmm import ( explore_gateway_clmm_pools as explore_gateway_clmm_pools_impl, diff --git a/tests/test_hummingbot_mcp_tools.py b/tests/test_hummingbot_mcp_tools.py index 0e9ac3b6c..6953e7996 100644 --- a/tests/test_hummingbot_mcp_tools.py +++ b/tests/test_hummingbot_mcp_tools.py @@ -242,5 +242,93 @@ def test_lp_branch_is_skipped_when_not_requested(): assert not any(s["title"] == "LP Positions (CLMM)" for s in result["sections"]) +class FakeGatewayContainerApi: + """The three ``client.gateway`` calls the container tool's branches make.""" + + def __init__(self): + self.calls = [] + + async def get_status(self): + self.calls.append("get_status") + return { + "running": True, + "container_id": "abc123def4567890", + "image": "hummingbot/gateway:latest", + "port": 15888, + "created_at": "2026-09-08T12:00:00.000000Z", + } + + async def get_logs(self, tail): + self.calls.append(("get_logs", tail)) + return "gateway | ERROR the swap reverted" + + async def restart(self, config): + self.calls.append(("restart", config)) + return {"ok": True} + + +class FakeGatewayContainerClient: + def __init__(self, gateway): + self.gateway = gateway + + +class FakeClientHolder: + """Stands in for the ``hummingbot_client`` singleton server.py awaits.""" + + def __init__(self, client): + self._client = client + + async def get_client(self): + return self._client + + +def _stub_gateway_container(monkeypatch): + from mcp_servers.hummingbot_api import server as hb_server + + gateway = FakeGatewayContainerApi() + monkeypatch.setattr( + hb_server, + "hummingbot_client", + FakeClientHolder(FakeGatewayContainerClient(gateway)), + ) + return hb_server, gateway + + +def test_manage_gateway_container_get_status_reaches_the_impl_and_formatter(): + """CORR-561: the wrapper's body called two names server.py never imported. + + Registration succeeded (the tool is in ADMIN_TOOLS and the resolver only + needs the wrapper), so nothing caught it until a call ran the body and + @handle_errors reformatted the NameError into "Failed to manage Gateway + container: name 'manage_gateway_container_impl' is not defined". + """ + monkeypatch = pytest.MonkeyPatch() + try: + hb_server, gateway = _stub_gateway_container(monkeypatch) + output = asyncio.run(hb_server.manage_gateway_container(action="get_status")) + finally: + monkeypatch.undo() + + assert gateway.calls == ["get_status"], "the impl branch never ran" + assert "Gateway Container Status" in output + assert "Running" in output + assert "abc123def456" in output + + +def test_manage_gateway_container_get_logs_is_a_working_escape_hatch(): + """GATEWAY_LOG_HINT points every opaque swap/LP failure at this action.""" + monkeypatch = pytest.MonkeyPatch() + try: + hb_server, gateway = _stub_gateway_container(monkeypatch) + output = asyncio.run( + hb_server.manage_gateway_container(action="get_logs", tail=25) + ) + finally: + monkeypatch.undo() + + assert gateway.calls == [("get_logs", 25)] + assert "the swap reverted" in output + + if __name__ == "__main__": pytest.main([__file__, "-v"]) From 076e07b0ca9fd13ceaf999daf0ea5e8b03f2dc4f Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 09:53:29 +0300 Subject: [PATCH 047/154] (fix) deleting a controller config no longer reports a failure it did not have The config-delete branch of modify_controllers called bot_orchestration.deploy_v2_controllers() with no arguments against a signature with three required ones, so it raised TypeError at binding time -- after the DELETE had already succeeded. The tool mutated and then handed back an opaque Python signature error, so the model retried (a 404 by then) and told the user the delete had failed when it had not. Nothing needs a redeploy after a config delete: the delete endpoint stands alone, and every other call site passes the full argument set for a real deployment. The line has never once executed successfully since it arrived. The regression test drives the delete branch with a client whose bot_orchestration attribute raises on any access, so the branch has to run to completion without reaching for it. --- .../hummingbot_api/tools/controllers.py | 1 - tests/test_hummingbot_mcp_tools.py | 47 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/mcp_servers/hummingbot_api/tools/controllers.py b/mcp_servers/hummingbot_api/tools/controllers.py index 903408b80..85826c0ec 100644 --- a/mcp_servers/hummingbot_api/tools/controllers.py +++ b/mcp_servers/hummingbot_api/tools/controllers.py @@ -472,7 +472,6 @@ async def modify_controllers( raise ValueError("config_name is required for config delete") result = await client.controllers.delete_controller_config(config_name) - await client.bot_orchestration.deploy_v2_controllers() return { "action": "delete", diff --git a/tests/test_hummingbot_mcp_tools.py b/tests/test_hummingbot_mcp_tools.py index 6953e7996..4b83117f2 100644 --- a/tests/test_hummingbot_mcp_tools.py +++ b/tests/test_hummingbot_mcp_tools.py @@ -46,6 +46,53 @@ def __init__(self): self.controllers = FakeControllers() +class FakeConfigControllers: + """Only the config endpoints a delete legitimately needs.""" + + def __init__(self): + self.deleted = [] + + async def delete_controller_config(self, config_name): + self.deleted.append(config_name) + return {"message": f"Config {config_name} deleted"} + + +class NoOrchestrationClient: + """A config delete must never reach bot_orchestration at all.""" + + def __init__(self): + self.controllers = FakeConfigControllers() + + @property + def bot_orchestration(self): + raise AssertionError( + "deleting a controller config must not touch bot_orchestration" + ) + + +def test_deleting_a_config_does_not_redeploy_controllers(): + """The delete endpoint stands alone; a redeploy after it only ever raised. + + The stray ``deploy_v2_controllers()`` here was called with zero arguments + against a three-required-argument signature, so it raised TypeError *after* + the config was already deleted: the tool mutated and then reported failure. + """ + client = NoOrchestrationClient() + result = asyncio.run( + modify_controllers( + client, + action="delete", + target="config", + config_name="ema_trend_v1_sol", + ) + ) + + assert client.controllers.deleted == ["ema_trend_v1_sol"] + assert result["action"] == "delete" + assert result["config_name"] == "ema_trend_v1_sol" + assert result["message"].startswith("Config deleted:") + + def test_controller_upload_sends_a_controller_object_not_a_bare_string(): """POST /controllers/{type}/{name} takes {"content": ...}; a raw string is a 422.""" client = FakeControllerClient() From e6de962a9cd99f0e4d7ed1486a42f58452969d6f Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 10:01:01 +0300 Subject: [PATCH 048/154] Refuse the search_history filters the perp branch cannot honour MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit search_history advertised one filter set for all three data types, but the perp_positions branch forwards only account_names/connector_names/limit to POST /trading/positions — the backend has no closed-position history endpoint at all. So search_history("perp_positions", status="CLOSED", start_time=...) returned today's OPEN book under a "Perpetual Positions History" header, with no signal that the window had been dropped: the worst possible shape for a PnL or tax report. Raise a ToolError naming the offending parameter instead, before any request goes out, and correct the docstrings: perp_positions reads the current open book, the filter list is per data type rather than "common to all", and the example that could never work is gone. The orders branch's offset pagination hint is left alone; that is CORR-569's. --- mcp_servers/hummingbot_api/server.py | 31 ++-- mcp_servers/hummingbot_api/tools/history.py | 84 +++++++++-- tests/test_mcp_search_history_filters.py | 156 ++++++++++++++++++++ 3 files changed, 248 insertions(+), 23 deletions(-) create mode 100644 tests/test_mcp_search_history_filters.py diff --git a/mcp_servers/hummingbot_api/server.py b/mcp_servers/hummingbot_api/server.py index 6d688fa86..9e591756c 100644 --- a/mcp_servers/hummingbot_api/server.py +++ b/mcp_servers/hummingbot_api/server.py @@ -226,7 +226,9 @@ async def get_portfolio_overview( - Includes real-time fees and token amounts 4. Active Orders - Currently open orders across all exchanges - NOTE: This only shows ACTIVE/OPEN positions. For historical data, use search_history() instead. + NOTE: This only shows ACTIVE/OPEN positions. For historical data, use + search_history(data_type="orders") — note that search_history's perp_positions + reads this same open book, not a closed-position history. Args: account_names: List of account names to filter by (optional). If empty, returns all accounts. @@ -324,18 +326,22 @@ async def search_history( Data Types: - orders: Historical order data (filled, cancelled, failed) - - perp_positions: Perpetual positions (both open and closed) + - perp_positions: The CURRENT open perpetual book, NOT a history. The backend + has no closed-position endpoint, so there is no closed perp history to search; + get_portfolio_overview() returns the same positions. For a time-windowed + record of perp activity use data_type="orders". - clmm_positions: CLMM LP positions (both open and closed) - Common Filters (apply to all data types): - account_names: Filter by account names (optional) - connector_names: Filter by connector names (optional) - trading_pairs: Filter by trading pairs (optional) - status: Filter by status (optional, e.g., 'OPEN', 'CLOSED', 'FILLED', 'CANCELED') - start_time: Start timestamp in seconds (optional) - end_time: End timestamp in seconds (optional) - limit: Maximum number of results (default: 50, max: 1000) - offset: Pagination offset (default: 0) + Filters (only the ones listed for a data type are honoured; passing any other + filter raises an error instead of silently ignoring it): + account_names: All data types (optional) + connector_names: All data types (optional) + limit: All data types (default: 50, max: 1000) + trading_pairs: orders, clmm_positions (optional) + status: orders, clmm_positions (optional, e.g., 'FILLED', 'CANCELED') + start_time: orders only, timestamp in seconds (optional) + end_time: orders only, timestamp in seconds (optional) + offset: clmm_positions only, pagination offset (default: 0) CLMM-Specific Filters: network: Network filter for CLMM positions (optional) @@ -344,7 +350,8 @@ async def search_history( Examples: - Search filled orders: search_history("orders", status="FILLED", limit=100) - - Search closed perp positions: search_history("perp_positions", status="CLOSED") + - Orders in a time window: search_history("orders", start_time=..., end_time=...) + - Current perp book: search_history("perp_positions", account_names=["master"]) - Search all CLMM positions: search_history("clmm_positions", limit=100) """ client = await hummingbot_client.get_client() diff --git a/mcp_servers/hummingbot_api/tools/history.py b/mcp_servers/hummingbot_api/tools/history.py index 6fde5af54..cc8276212 100644 --- a/mcp_servers/hummingbot_api/tools/history.py +++ b/mcp_servers/hummingbot_api/tools/history.py @@ -10,6 +10,7 @@ import logging from typing import Any, Literal +from mcp_servers.hummingbot_api.exceptions import ToolError from mcp_servers.hummingbot_api.hummingbot_client import HummingbotClient from . import gateway_clmm as gateway_clmm_tools @@ -18,6 +19,45 @@ logger = logging.getLogger("hummingbot-mcp") +# Filters that each data_type can actually forward to the backend. Anything else +# in the shared signature is refused instead of being silently dropped. +_UNSUPPORTED_FILTERS: dict[str, tuple[str, ...]] = { + # The trading router has no closed-position history endpoint: get_positions + # POSTs /trading/positions with only account_names/connector_names/limit. + # orders and clmm_positions forward their filters, so only the perp branch + # needs a guard. (The orders branch's offset pagination hint is CORR-569's.) + "perp_positions": ("trading_pairs", "status", "start_time", "end_time", "offset"), +} + +_FILTER_ALTERNATIVES: dict[str, str] = { + "perp_positions": ( + "perp_positions returns the CURRENT open book (the backend has no closed " + 'position history endpoint). Use data_type="orders" for a time-windowed ' + "history, or get_portfolio_overview() for the same open positions." + ), +} + + +def _reject_unsupported_filters(data_type: str, **filters: Any) -> None: + """Raise ToolError naming any filter the given data_type cannot honour. + + ``offset`` is only a filter when it is non-zero, since it defaults to 0. + """ + unsupported = _UNSUPPORTED_FILTERS.get(data_type, ()) + # A falsy value (None, [], the default offset=0) means "not supplied". + supplied = [name for name in unsupported if filters.get(name)] + if not supplied: + return + + names = ", ".join(supplied) + plural = "s" if len(supplied) > 1 else "" + raise ToolError( + f"search_history(data_type={data_type!r}) cannot filter by {names}: " + f"the parameter{plural} would be silently ignored. " + f"{_FILTER_ALTERNATIVES.get(data_type, '')}".strip() + ) + + async def search_history( client: HummingbotClient, data_type: Literal["orders", "perp_positions", "clmm_positions"], @@ -44,27 +84,45 @@ async def search_history( Data Types: - orders: Historical order data (filled, cancelled, failed) - - perp_positions: Perpetual positions (both open and closed) + - perp_positions: The CURRENT open perpetual book. The backend has no closed + position history endpoint, so this cannot be filtered by pair, status or + time; use get_portfolio_overview() for the same data, or data_type="orders" + for a time-windowed history. - clmm_positions: CLMM LP positions (both open and closed) Args: client: Hummingbot client instance data_type: Type of historical data to search - account_names: Filter by account names (optional) - connector_names: Filter by connector names (optional) - trading_pairs: Filter by trading pairs (optional) - status: Filter by status (optional, e.g., 'OPEN', 'CLOSED', 'FILLED', 'CANCELED') - start_time: Start timestamp in seconds (optional) - end_time: End timestamp in seconds (optional) - limit: Maximum number of results (default: 50, max: 1000) - offset: Pagination offset (default: 0) + account_names: Filter by account names (all data types, optional) + connector_names: Filter by connector names (all data types, optional) + trading_pairs: Filter by trading pairs (orders, clmm_positions; optional) + status: Filter by status (orders, clmm_positions; optional, e.g., 'FILLED') + start_time: Start timestamp in seconds (orders only, optional) + end_time: End timestamp in seconds (orders only, optional) + limit: Maximum number of results (all data types, default: 50, max: 1000) + offset: Pagination offset (clmm_positions only, default: 0) network: Network filter for CLMM positions (optional) wallet_address: Wallet address filter for CLMM positions (optional) position_addresses: Specific position addresses for CLMM (optional) Returns: Dictionary containing search results with formatted output + + Raises: + ToolError: If a filter is supplied that the chosen data_type cannot honour """ + # Fail loudly rather than silently dropping filters the branch cannot apply. + # Raised before the try/except below, which would flatten it into a generic + # Exception, and before any request reaches the client. + _reject_unsupported_filters( + data_type, + trading_pairs=trading_pairs, + status=status, + start_time=start_time, + end_time=end_time, + offset=offset, + ) + try: # ============================================ # ORDERS - Historical order data @@ -101,7 +159,8 @@ async def search_history( # PERP POSITIONS - Perpetual positions # ============================================ elif data_type == "perp_positions": - # Use existing trading_tools.get_positions function + # The backend exposes no closed-position history: this is the current + # open book. Unsupported filters were already refused above. result = await trading_tools.get_positions( client=client, account_names=account_names, @@ -109,7 +168,10 @@ async def search_history( limit=min(limit, 1000), ) - formatted_output = f"Perpetual Positions History\n{'=' * 100}\n\n{result['positions_table']}" + formatted_output = ( + f"Perpetual Positions (current open book)\n{'=' * 100}\n\n" + f"{result['positions_table']}" + ) return { "data_type": "perp_positions", diff --git a/tests/test_mcp_search_history_filters.py b/tests/test_mcp_search_history_filters.py new file mode 100644 index 000000000..a27cab8e2 --- /dev/null +++ b/tests/test_mcp_search_history_filters.py @@ -0,0 +1,156 @@ +"""search_history must honour, or refuse, the filters its signature promises (CORR-563). + +The tool advertises one filter set for three data types, but the perp branch calls +``client.trading.get_positions``, whose backend route (POST /trading/positions) takes +only account_names/connector_names/limit. So +``search_history("perp_positions", status="CLOSED", start_time=...)`` used to return +today's OPEN book under a "Perpetual Positions History" header, with no signal that +the time window had been dropped — the worst shape for a PnL or tax report. + +These tests drive the real branch through the server-level tool (decorator and the +bare ``except Exception`` rewrap included) and assert on the request that actually +reaches the client, not on a helper's return value. + +The repo has no async test setup, so the coroutines are driven with asyncio.run(). +""" + +import asyncio + +import pytest + +from mcp_servers.hummingbot_api import server as hb_server +from mcp_servers.hummingbot_api.exceptions import ToolError +from mcp_servers.hummingbot_api.tools import history as history_tools + + +class RecordingTrading: + """Records every outgoing request instead of hitting the backend.""" + + def __init__(self): + self.position_calls = [] + self.order_calls = [] + + async def get_positions(self, **kwargs): + self.position_calls.append(kwargs) + return { + "data": [ + { + "account_name": "master", + "connector_name": "binance_perpetual", + "trading_pair": "SOL-USDT", + "side": "LONG", + "amount": 10, + "entry_price": 100, + "unrealized_pnl": 5, + } + ] + } + + async def search_orders(self, **kwargs): + self.order_calls.append(kwargs) + return {"data": [], "pagination": {"has_more": False}} + + +class RecordingClient: + def __init__(self): + self.trading = RecordingTrading() + + +@pytest.fixture +def client_calls(monkeypatch): + """Drive the server-level tool against a recording client.""" + client = RecordingClient() + + async def fake_get_client(): + return client + + monkeypatch.setattr(hb_server.hummingbot_client, "get_client", fake_get_client) + return client.trading + + +@pytest.mark.parametrize( + "filters, expected_names", + [ + ({"status": "CLOSED"}, ["status"]), + ( + {"start_time": 1757000000, "end_time": 1757600000}, + ["start_time", "end_time"], + ), + ({"trading_pairs": ["SOL-USDT"]}, ["trading_pairs"]), + ({"offset": 50}, ["offset"]), + ], +) +def test_perp_positions_refuses_filters_it_cannot_honour( + client_calls, filters, expected_names +): + """The tool raises naming the parameter, and no request is sent.""" + with pytest.raises(ToolError) as excinfo: + asyncio.run(hb_server.search_history(data_type="perp_positions", **filters)) + + message = str(excinfo.value) + for name in expected_names: + assert name in message, f"{name} not named in refusal: {message}" + # The refusal must survive history.py's bare `except Exception` rewrap and the + # handle_errors decorator with its parameter names intact, not be flattened + # into "Failed to search history: ...". + assert "silently ignored" in message + + # Acceptance criterion: the perp branch never reaches the positions endpoint. + assert client_calls.position_calls == [] + + +def test_perp_positions_still_works_with_supported_filters(client_calls): + """Supported filters reach the client, and the header no longer says "History".""" + output = asyncio.run( + hb_server.search_history( + data_type="perp_positions", + account_names=["master"], + connector_names=["binance_perpetual"], + limit=25, + ) + ) + + assert client_calls.position_calls == [ + { + "account_names": ["master"], + "connector_names": ["binance_perpetual"], + "limit": 25, + } + ] + assert "History" not in output + assert "current open book" in output + + +def test_orders_branch_still_forwards_every_filter(client_calls): + """The guard is perp-only: orders genuinely sends its filters to the backend.""" + asyncio.run( + hb_server.search_history( + data_type="orders", + account_names=["master"], + connector_names=["binance"], + trading_pairs=["SOL-USDC"], + status="FILLED", + start_time=1757000000, + end_time=1757600000, + limit=100, + ) + ) + + assert len(client_calls.order_calls) == 1 + sent = client_calls.order_calls[0] + assert sent["trading_pairs"] == ["SOL-USDC"] + assert sent["status"] == "FILLED" + assert sent["start_time"] == 1757000000 + assert sent["end_time"] == 1757600000 + assert sent["limit"] == 100 + + +def test_docstrings_no_longer_promise_common_filters(): + """The signature's promise and the perp reality have to match.""" + doc = hb_server.search_history.__doc__ + assert "Common Filters (apply to all data types)" not in doc + assert 'search_history("perp_positions", status="CLOSED")' not in doc + assert "both open and closed" not in doc.split("clmm_positions")[0] + + tool_doc = history_tools.search_history.__doc__ + assert "perp_positions: Perpetual positions (both open and closed)" not in tool_doc From c2f77ebee97ea9412c4c2dd83594c6fe0fb9e1c5 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 10:08:14 +0300 Subject: [PATCH 049/154] A second candle subscriber no longer evicts the history the first one is drawing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Candle buffers are shared per channel, so every tab charting the same pair and interval draws from one `_CandleBuffer`. The subscribe path called `set_duration` unconditionally, and `set_duration` evicts down to the new max — so a second tab, or the same tab's automatic re-subscribe after a WS reconnect (which sends no duration and lands on the 3-day default), physically deleted the candles a 30-day window had already backfilled. Nothing refilled them: the backfill only fires when the buffer grew. `_handle_candle_duration_change` always enforced the opposite invariant — grow only, the frontend manages its own display window. That guard now lives on the buffer as `grow_to`, and both paths share it. `set_duration` stays the unconditional primitive `__init__` uses. --- condor/web/streams/candles.py | 26 +++- tests/test_candle_subscribe_grow_only.py | 153 +++++++++++++++++++++++ 2 files changed, 173 insertions(+), 6 deletions(-) create mode 100644 tests/test_candle_subscribe_grow_only.py diff --git a/condor/web/streams/candles.py b/condor/web/streams/candles.py index 6ae41b096..a37a9e818 100644 --- a/condor/web/streams/candles.py +++ b/condor/web/streams/candles.py @@ -157,6 +157,21 @@ def set_duration(self, duration_seconds: int) -> int: ) return new_max + def grow_to(self, duration_seconds: int) -> int: + """Resize for ``duration_seconds`` but never below the current size. + + Candle buffers are shared per channel, so a second subscriber asking for + a shorter window must not evict history the first subscriber's snapshot + still carries. The frontend manages its own display window; the backend + only guarantees *enough* history is buffered. Returns the max size, + unchanged when the request would shrink the buffer. + """ + interval_sec = _INTERVAL_SECONDS.get(self.interval, 60) + needed = max(math.ceil(duration_seconds / interval_sec), 200) + if needed <= self._max_size: + return self._max_size + return self.set_duration(duration_seconds) + def upsert(self, candle: dict) -> None: self._data[candle["timestamp"]] = candle self._evict() @@ -261,10 +276,12 @@ async def _handle_candle_subscribe( buf = _CandleBuffer(interval, dur) self._candle_buffers[channel] = buf else: - # Expand buffer if this client needs more + # Expand buffer if this client needs more — never shrink it, or a + # second tab (or a reconnect, which re-subscribes with the 3-day + # default) would evict history the first tab is still drawing. if dur > 0: old_max = buf.max_size - buf.set_duration(dur) + buf.grow_to(dur) if buf.max_size > old_max and buf.needs_backfill: self._oneshot_tasks.track( asyncio.create_task( @@ -294,11 +311,8 @@ async def _handle_candle_duration_change( return old_max = buf.max_size # Only expand — skip if requested duration would shrink the buffer - interval_sec = _INTERVAL_SECONDS.get(buf.interval, 60) - needed = max(math.ceil(duration / interval_sec), 200) - if needed <= old_max: + if buf.grow_to(duration) <= old_max: return - buf.set_duration(duration) if buf.needs_backfill: await self._backfill_candles(channel) # Broadcast updated snapshot to ALL subscribers on this channel diff --git a/tests/test_candle_subscribe_grow_only.py b/tests/test_candle_subscribe_grow_only.py new file mode 100644 index 000000000..21116011d --- /dev/null +++ b/tests/test_candle_subscribe_grow_only.py @@ -0,0 +1,153 @@ +"""Subscribing to a shared candle channel must never shrink its buffer (CORR-581). + +``_candle_buffers`` is keyed by channel, so every tab charting +``candles:srv:binance:SOL-USDC:1m`` shares one buffer. The subscribe path used +to call ``set_duration`` unconditionally, and ``set_duration`` evicts down to +the new max — so a second tab (or the same tab's automatic re-subscribe after a +WS reconnect, which sends no duration and lands on the 3-day default) physically +deleted the history a 30-day window had already backfilled, with nothing to +refill it: the backfill only fires when the buffer *grew*. + +``_handle_candle_duration_change`` always documented and enforced the opposite +invariant. Both paths now share it through ``_CandleBuffer.grow_to``. +""" + +import asyncio + +import pytest + +from condor.web.streams.candles import _CandleBuffer +from condor.web.ws_manager import WebSocketManager + +CHANNEL = "candles:srv:binance:SOL-USDC:1m" +MINUTE = 60 +THIRTY_DAYS = 30 * 86400 +ONE_HOUR = 3600 + + +def run(coro): + return asyncio.run(coro) + + +def _candles(n: int) -> list[dict]: + return [{"timestamp": float(i * MINUTE), "close": 1.0 + i} for i in range(n)] + + +@pytest.fixture +def mgr(monkeypatch): + """A manager whose subscribe path is fully driven but has no I/O.""" + manager = WebSocketManager() + sent: list[list[dict]] = [] + + async def fake_send(self, conn, channel, data): + if data.get("type") == "candles": + sent.append(data["data"]) + + monkeypatch.setattr(WebSocketManager, "_send", fake_send) + monkeypatch.setattr(WebSocketManager, "_ensure_stream", lambda self, p, c: None) + manager.sent_snapshots = sent + return manager + + +# ── the buffer primitive ── + + +def test_grow_to_refuses_to_shrink(): + buf = _CandleBuffer("1m", THIRTY_DAYS) + buf.upsert_many(_candles(5000)) + assert buf.grow_to(ONE_HOUR) == 30 * 1440 + assert buf.size == 5000 + + +def test_grow_to_still_grows(): + buf = _CandleBuffer("1m", ONE_HOUR) + assert buf.max_size == 200 # the floor + assert buf.grow_to(THIRTY_DAYS) == 30 * 1440 + + +def test_set_duration_remains_the_unconditional_primitive(): + """``__init__`` and nothing else relies on set_duration shrinking.""" + buf = _CandleBuffer("1m", THIRTY_DAYS) + buf.upsert_many(_candles(5000)) + assert buf.set_duration(ONE_HOUR) == 200 + assert buf.size == 200 + + +# ── the subscribe path, end to end ── + + +def test_second_subscribe_with_a_smaller_window_keeps_the_history(mgr): + """Tab A charts 30 days; tab B subscribes with the 3-day default.""" + run(mgr._handle_candle_subscribe(object(), CHANNEL, THIRTY_DAYS)) + buf = mgr._candle_buffers[CHANNEL] + buf.upsert_many(_candles(5000)) + + run(mgr._handle_candle_subscribe(object(), CHANNEL, None)) + + assert buf.max_size == 30 * 1440 + assert buf.size == 5000, "a second subscriber evicted the shared history" + assert buf.get_sorted()[0]["timestamp"] == 0.0 + # Tab B's opening snapshot carries the whole buffer, not a truncated tail. + assert len(mgr.sent_snapshots[-1]) == 5000 + + +def test_second_subscribe_with_a_larger_window_still_grows_and_backfills(mgr): + backfilled: list[str] = [] + + async def fake_backfill(channel): + backfilled.append(channel) + + async def scenario(): + await mgr._handle_candle_subscribe(object(), CHANNEL, ONE_HOUR) + mgr._backfill_candles = fake_backfill + await mgr._handle_candle_subscribe(object(), CHANNEL, THIRTY_DAYS) + await asyncio.sleep(0) # let the tracked backfill task run + + run(scenario()) + + assert mgr._candle_buffers[CHANNEL].max_size == 30 * 1440 + assert backfilled == [CHANNEL] + + +# ── the sibling path is unchanged ── + + +def test_duration_change_shrink_is_still_a_no_op(mgr): + run(mgr._handle_candle_subscribe(object(), CHANNEL, THIRTY_DAYS)) + buf = mgr._candle_buffers[CHANNEL] + buf.upsert_many(_candles(5000)) + broadcast: list = [] + + async def fake_broadcast(channel, data): + broadcast.append(data) + + mgr.broadcast = fake_broadcast + + run(mgr._handle_candle_duration_change(object(), CHANNEL, ONE_HOUR)) + + assert buf.max_size == 30 * 1440 + assert buf.size == 5000 + assert broadcast == [], "a shrink must not re-broadcast" + + +def test_duration_change_grow_resizes_backfills_and_rebroadcasts(mgr): + run(mgr._handle_candle_subscribe(object(), CHANNEL, ONE_HOUR)) + buf = mgr._candle_buffers[CHANNEL] + buf.upsert_many(_candles(100)) + backfilled: list[str] = [] + broadcast: list = [] + + async def fake_backfill(channel): + backfilled.append(channel) + + async def fake_broadcast(channel, data): + broadcast.append(data["data"]) + + mgr._backfill_candles = fake_backfill + mgr.broadcast = fake_broadcast + + run(mgr._handle_candle_duration_change(object(), CHANNEL, THIRTY_DAYS)) + + assert buf.max_size == 30 * 1440 + assert backfilled == [CHANNEL] + assert len(broadcast) == 1 and len(broadcast[0]) == 100 From af9a9e1cd07003ddb141ab486b3881fe0b5dd40b Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 10:15:12 +0300 Subject: [PATCH 050/154] (fix) a failed controller stop no longer shows as stopping for five minutes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stop_controllers_endpoint marks controllers as stopping before the upstream call so the UI reacts instantly, but its except block only logged and re-raised. Nothing was stopped, so manual_kill_switch never flips — and that flag is the sole condition under which overlay_stopping_state clears a controller mark. The mark therefore survived the full 300s transitional TTL on both the REST /bots body and every bots WS frame, telling the operator a stop was in flight when none was, and disabling the button they would have used to retry. Mirror the sibling stop_bot_endpoint, which has always cleared its own mark before raising. --- condor/web/routes/bots.py | 5 + ...t_stop_controllers_failure_clears_marks.py | 189 ++++++++++++++++++ 2 files changed, 194 insertions(+) create mode 100644 tests/test_stop_controllers_failure_clears_marks.py diff --git a/condor/web/routes/bots.py b/condor/web/routes/bots.py index 25196fe5d..3f84e5a67 100644 --- a/condor/web/routes/bots.py +++ b/condor/web/routes/bots.py @@ -773,6 +773,11 @@ async def stop_controllers_endpoint( controller_names=body.controller_names, ) except Exception as e: + # Nothing was stopped, so the kill switch will never flip and the overlay + # would show these controllers as "stopping" until the TTL expires — + # blocking a retry from the UI. Mirror stop_bot_endpoint and undo the mark. + for controller_id in body.controller_names: + clear_controller_stopping(name, bot_name, controller_id) logger.exception( "Failed to stop controllers on bot '%s' of '%s'", bot_name, name ) diff --git a/tests/test_stop_controllers_failure_clears_marks.py b/tests/test_stop_controllers_failure_clears_marks.py new file mode 100644 index 000000000..14366a9d9 --- /dev/null +++ b/tests/test_stop_controllers_failure_clears_marks.py @@ -0,0 +1,189 @@ +"""CORR-582: a failed stop_controllers must not leave a stale "stopping" mark. + +``stop_controllers_endpoint`` marks the controllers as stopping *before* the +upstream call so the UI reacts immediately. When the call fails nothing was +stopped, so ``manual_kill_switch`` never flips — and the only other clearing +path in ``overlay_stopping_state`` is gated on exactly that flag. The mark +therefore survived for the whole 300s transitional TTL, painting the +controllers as "stopping" (a state the upstream payload cannot report: its +``status`` is a hardcoded "running", which is why the overlay exists at all) +and disabling the very button an operator would use to retry. + +``stop_bot_endpoint`` has always cleared its own mark on failure; these tests +pin the same symmetry for controllers, and pin that a *successful* stop still +leaves the mark in place so the overlay can show "stopping" until the kill +switch is observed. +""" + +import asyncio + +import pytest +from fastapi import HTTPException + +import condor.web.routes.bots as bots_module +from condor.web.models import WebUser +from condor.web.routes.bots import ( + ControllerActionRequest, + get_stopping_bots, + get_stopping_controllers, +) + +_USER = WebUser(id=1, role="admin") + +SERVER = "srv" +BOT = "bot-1" +CONTROLLERS = ["pmm_sol", "pmm_eth"] + + +class _Controllers: + """The controllers sub-API ``_set_kill_switches`` actually drives.""" + + def __init__(self, update_error: Exception | None): + self._update_error = update_error + self.updated: list[str] = [] + + async def get_bot_controller_configs(self, _bot_name): + return [ + {"id": cid, "_config_name": cid, "manual_kill_switch": cid in self.updated} + for cid in CONTROLLERS + ] + + async def update_bot_controller_config(self, _bot_name, config_name, _update): + if self._update_error is not None: + raise self._update_error + self.updated.append(config_name) + return {"updated": True} + + +class _BotOrchestration: + def __init__(self, stop_error: Exception | None): + self._stop_error = stop_error + + async def stop_and_archive_bot(self, _bot_name): + if self._stop_error is not None: + raise self._stop_error + return {"stopped": True} + + +class _FakeClient: + def __init__(self, error: Exception | None = None): + self.controllers = _Controllers(error) + self.bot_orchestration = _BotOrchestration(error) + + +class _FakeCM: + def __init__(self, client): + self._client = client + + def has_server_access(self, *_args, **_kwargs): + return True + + async def get_client(self, _name): + return self._client + + +@pytest.fixture +def bind_client(monkeypatch): + """Point the route module at a client, and start from a clean state store.""" + + def _bind(client): + monkeypatch.setattr(bots_module, "get_config_manager", lambda: _FakeCM(client)) + return client + + for cid in CONTROLLERS: + bots_module.clear_controller_stopping(SERVER, BOT, cid) + bots_module.clear_bot_stopping(SERVER, BOT) + yield _bind + for cid in CONTROLLERS: + bots_module.clear_controller_stopping(SERVER, BOT, cid) + bots_module.clear_bot_stopping(SERVER, BOT) + + +def _stop_controllers(): + return asyncio.run( + bots_module.stop_controllers_endpoint( + name=SERVER, + bot_name=BOT, + body=ControllerActionRequest(controller_names=list(CONTROLLERS)), + user=_USER, + ) + ) + + +def test_a_failed_stop_clears_the_stopping_marks(bind_client): + """The real failure path: every update rejected → ValueError → marks gone.""" + bind_client(_FakeClient(RuntimeError("backend rejected the config write"))) + + with pytest.raises(HTTPException) as caught: + _stop_controllers() + + # The caller still gets the same upstream failure response. + assert caught.value.status_code in (400, 502) + + stopping = get_stopping_controllers(SERVER) + assert stopping == set(), ( + "a stop that did not happen must not leave controllers painted as " + f"stopping for the transitional TTL; still marked: {stopping}" + ) + + +def test_the_overlay_no_longer_paints_a_failed_stop_as_stopping(bind_client): + """The observable surface: the REST/WS overlay both feed from this state.""" + bind_client(_FakeClient(RuntimeError("backend rejected the config write"))) + + with pytest.raises(HTTPException): + _stop_controllers() + + # The upstream payload: ``status`` is the hardcoded "running" the backend + # always reports, and the kill switch is still off because nothing stopped. + controllers = [ + { + "bot_name": BOT, + "controller_id": cid, + "status": "running", + "config": {"manual_kill_switch": False}, + } + for cid in CONTROLLERS + ] + bots_module.overlay_stopping_state(SERVER, controllers, []) + + assert [c["status"] for c in controllers] == ["running", "running"] + + +def test_a_successful_stop_keeps_the_marks(bind_client): + """The mark must survive until the kill switch is actually observed.""" + client = bind_client(_FakeClient()) + + result = _stop_controllers() + + assert sorted(result["succeeded"]) == sorted(CONTROLLERS) + assert get_stopping_controllers(SERVER) == {f"{BOT}:{cid}" for cid in CONTROLLERS} + + # And the overlay renders them as stopping while the flag is still unseen. + controllers = [ + { + "bot_name": BOT, + "controller_id": cid, + "status": "running", + "config": {"manual_kill_switch": False}, + } + for cid in CONTROLLERS + ] + bots_module.overlay_stopping_state(SERVER, controllers, []) + assert [c["status"] for c in controllers] == ["stopping", "stopping"] + assert client.controllers.updated == CONTROLLERS + + +def test_the_bot_and_controller_failure_paths_agree(bind_client): + """Symmetry with ``stop_bot_endpoint``, whose failure path already cleared.""" + bind_client(_FakeClient(RuntimeError("backend down"))) + + with pytest.raises(HTTPException): + asyncio.run( + bots_module.stop_bot_endpoint(name=SERVER, bot_name=BOT, user=_USER) + ) + assert get_stopping_bots(SERVER) == set() + + with pytest.raises(HTTPException): + _stop_controllers() + assert get_stopping_controllers(SERVER) == set() From fcc6e0e1ad49cc44d73c9ba0c0ed0610fcdf40ee Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 10:22:50 +0300 Subject: [PATCH 051/154] (fix) a crashing chat-WS handler says so instead of vanishing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every action on /ws/chat is dispatched fire-and-forget, and the tracker attached only `discard`: nobody read the task's exception, so a handler that raised — a disk error minting the conversation, a dead ACP subprocess on abort — reached the operator only as asyncio's GC-time "Task exception was never retrieved", on no logger and naming no action. Track them with the shared TaskSet, which is the repo's one answer to exactly this (ws_manager, SDS, candles) and was written for it. The action name rides along as the task's label, so the line names the handler that failed and the user it failed for. Cancellation is not a failure and stays silent, which is what a disconnect does to every task here that is not a turn. --- condor/web/routes/chat_ws.py | 30 ++++-- tests/test_chat_ws_task_failures.py | 160 ++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+), 11 deletions(-) create mode 100644 tests/test_chat_ws_task_failures.py diff --git a/condor/web/routes/chat_ws.py b/condor/web/routes/chat_ws.py index 3507f8cb8..1356f01fe 100644 --- a/condor/web/routes/chat_ws.py +++ b/condor/web/routes/chat_ws.py @@ -13,6 +13,7 @@ from fastapi import APIRouter, Depends, Query, WebSocket, WebSocketDisconnect +from condor.asyncutil import TaskSet from condor.llm.openrouter_models import fetch_models from condor.llm.options import DEFAULT_AGENT from condor.notifications import Notification, register_push_sink @@ -436,17 +437,24 @@ async def chat_websocket(ws: WebSocket, token: str | None = Query(default=None)) sessions = await _get_user_sessions(user_id) await _send(ws, {"event": "sessions_list", "sessions": sessions}) - # Background tasks so long-running operations don't block the receive loop - bg_tasks: set[asyncio.Task] = set() + # Background tasks so long-running operations don't block the receive loop. + # + # Tracked by the shared helper rather than by a hand-rolled set (CORR-583): + # the old tracker attached only ``discard``, so nobody ever read the task's + # exception and a handler that raised — a disk error minting a conversation, + # a dead subprocess on abort — surfaced only as asyncio's GC-time "Task + # exception was never retrieved", on no logger and naming no action. TaskSet + # logs it here, named by the action that failed. Cancellation is not a + # failure and stays silent, which is what the disconnect below does to every + # task that is not a turn. + bg_tasks = TaskSet(log, f"Chat WS handler %s failed for user {user_id}: %s") # The subset of those that are a *turn*. A turn is the one piece of work # here that belongs to the conversation rather than to this connection, so # it is the one thing a disconnect must not cancel. turn_tasks: set[asyncio.Task] = set() - def _run_bg(coro, *, is_turn: bool = False): - task = asyncio.create_task(coro) - bg_tasks.add(task) - task.add_done_callback(bg_tasks.discard) + def _run_bg(coro, action: str, *, is_turn: bool = False): + task = bg_tasks.track(asyncio.create_task(coro), action) if is_turn: turn_tasks.add(task) task.add_done_callback(turn_tasks.discard) @@ -463,20 +471,20 @@ def _run_bg(coro, *, is_turn: bool = False): action = msg.get("action") if action == "start_session": - _run_bg(_handle_start_session(ws, user_id, msg)) + _run_bg(_handle_start_session(ws, user_id, msg), action) elif action == "resume_conversation": - _run_bg(_handle_resume_conversation(ws, user_id, msg)) + _run_bg(_handle_resume_conversation(ws, user_id, msg), action) elif action == "send_message": - _run_bg(_handle_send_message(ws, user_id, msg), is_turn=True) + _run_bg(_handle_send_message(ws, user_id, msg), action, is_turn=True) elif action == "destroy_session": - _run_bg(_handle_destroy_session(ws, user_id, msg)) + _run_bg(_handle_destroy_session(ws, user_id, msg), action) elif action == "list_sessions": sessions = await _get_user_sessions(user_id) await _send(ws, {"event": "sessions_list", "sessions": sessions}) elif action == "resolve_permission": await _handle_resolve_permission(user_id, msg) elif action == "abort_prompt": - _run_bg(_handle_abort_prompt(ws, user_id, msg)) + _run_bg(_handle_abort_prompt(ws, user_id, msg), action) else: await _send( ws, {"event": "error", "message": f"Unknown action: {action}"} diff --git a/tests/test_chat_ws_task_failures.py b/tests/test_chat_ws_task_failures.py new file mode 100644 index 000000000..20dbc0bd9 --- /dev/null +++ b/tests/test_chat_ws_task_failures.py @@ -0,0 +1,160 @@ +"""A background chat-WS handler that crashes must say so (CORR-583). + +Every action on ``/ws/chat`` is dispatched as a fire-and-forget task. The +tracker attached only ``discard``, so nobody ever read the task's exception: +a handler that raised — a disk error minting the conversation, a dead ACP +subprocess on abort — reached the operator only as asyncio's GC-time "Task +exception was never retrieved", on no module's logger and naming no action, +while the tab that asked sat there forever. + +These tests drive the real endpoint and assert on the log record it emits, and +on the silence that a plain disconnect still has to keep. +""" + +import asyncio +import json +import logging + +import pytest +from fastapi import WebSocketDisconnect + +from condor.web.routes import chat_ws + +USER = 909 +LOGGER = "condor.web.routes.chat_ws" + + +class _FakeWS: + """A socket the test feeds frames into and can hang up on.""" + + def __init__(self): + self.sent: list[dict] = [] + self._inbox: asyncio.Queue = asyncio.Queue() + self._closed = False + + # -- endpoint side ---------------------------------------------------- + async def accept(self, subprotocol=None) -> None: + return None + + async def close(self, code=1000, reason="") -> None: + self._closed = True + + async def send_text(self, raw: str) -> None: + if self._closed: + raise RuntimeError("socket is closed") + self.sent.append(json.loads(raw)) + + async def receive_text(self) -> str: + raw = await self._inbox.get() + if raw is None: + raise WebSocketDisconnect(code=1001) + return raw + + # -- test side -------------------------------------------------------- + def feed(self, frame: dict) -> None: + self._inbox.put_nowait(json.dumps(frame)) + + def hang_up(self) -> None: + self._inbox.put_nowait(None) + self._closed = True + + def events(self, name: str) -> list[dict]: + return [e for e in self.sent if e.get("event") == name] + + async def wait_for(self, name: str) -> dict: + for _ in range(400): + found = self.events(name) + if found: + return found[0] + await asyncio.sleep(0.005) + raise AssertionError(f"no {name} frame arrived") + + +@pytest.fixture +def ws_env(monkeypatch): + """Just enough of the world to open the socket; no session machinery.""" + from config_manager import UserRole, get_config_manager + + monkeypatch.setattr(chat_ws, "_attached_sockets", {}) + monkeypatch.setattr(chat_ws, "_orphaned_turns", set()) + monkeypatch.setattr(chat_ws, "extract_ws_token", lambda ws, token: ("t", None)) + monkeypatch.setattr(chat_ws, "decode_jwt", lambda token: {"sub": USER}) + monkeypatch.setattr( + type(get_config_manager()), + "get_user_role", + lambda self, uid: UserRole.ADMIN, + ) + + async def _no_sessions(*args, **kwargs): + return [] + + monkeypatch.setattr(chat_ws.runtime, "list_sessions", _no_sessions) + return chat_ws + + +async def _connect(ws: _FakeWS) -> asyncio.Task: + conn = asyncio.create_task(chat_ws.chat_websocket(ws, token="t")) + await ws.wait_for("sessions_list") + return conn + + +def _failures(caplog) -> list[logging.LogRecord]: + return [r for r in caplog.records if r.name == LOGGER] + + +def test_a_handler_that_raises_is_logged_with_its_action_and_user( + ws_env, monkeypatch, caplog +): + async def boom(ws, user_id, msg): + raise OSError("no space left on device") + + monkeypatch.setattr(chat_ws, "_handle_start_session", boom) + + async def scenario(): + ws = _FakeWS() + conn = await _connect(ws) + ws.feed({"action": "start_session", "agent_key": "claude-code"}) + for _ in range(400): + if _failures(caplog): + break + await asyncio.sleep(0.005) + ws.hang_up() + await asyncio.wait_for(conn, timeout=5) + + with caplog.at_level(logging.ERROR, logger=LOGGER): + asyncio.run(scenario()) + + records = _failures(caplog) + assert len(records) == 1, "the crash was swallowed by the background tracker" + message = records[0].getMessage() + assert "start_session" in message, message + assert str(USER) in message, message + # ...with the traceback attached, so the log names the disk error itself. + assert records[0].exc_info is not None + assert isinstance(records[0].exc_info[1], OSError) + + +def test_a_disconnect_cancelling_a_handler_logs_nothing(ws_env, monkeypatch, caplog): + """Cancellation is not a failure: closing a tab must stay silent.""" + entered = asyncio.Event() + + async def hang(ws, user_id, msg): + entered.set() + await asyncio.Event().wait() + + monkeypatch.setattr(chat_ws, "_handle_destroy_session", hang) + + async def scenario(): + ws = _FakeWS() + conn = await _connect(ws) + ws.feed({"action": "destroy_session", "slot_id": "slot-1"}) + await asyncio.wait_for(entered.wait(), timeout=5) + # The disconnect path cancels every non-turn task it started. + ws.hang_up() + await asyncio.wait_for(conn, timeout=5) + await asyncio.sleep(0) + + with caplog.at_level(logging.ERROR, logger=LOGGER): + asyncio.run(scenario()) + + assert _failures(caplog) == [] From e89e9f6e3d7d31e3b554f91623e4f9457c56278a Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 10:29:43 +0300 Subject: [PATCH 052/154] Stamp the REST candle cache when the fetch lands, not when it started GET /market/candles captured time.monotonic() before the cache lookup and reused it as the insert stamp after the upstream fetch, so every entry was born aged by the whole fetch duration against a 30s TTL. A GeckoTerminal pool chart under the shared rate limiter can outrun that TTL, leaving an entry stale the instant it was written and re-firing the upstream call on the very next request -- in exactly the rate-limited path the cache exists to protect. The same under-counted clock also let genuinely expired entries survive the eviction sweep. _candle_cache_put now takes its own clock at insert time, which is also the right clock for its sweep. Coalescing and shielding are untouched. --- condor/web/routes/market.py | 12 ++- tests/test_candle_cache.py | 55 +++++++++--- tests/test_candle_cache_slow_fetch.py | 117 ++++++++++++++++++++++++++ 3 files changed, 169 insertions(+), 15 deletions(-) create mode 100644 tests/test_candle_cache_slow_fetch.py diff --git a/condor/web/routes/market.py b/condor/web/routes/market.py index 6452ca2fb..2b0f52640 100644 --- a/condor/web/routes/market.py +++ b/condor/web/routes/market.py @@ -18,8 +18,14 @@ _CANDLE_CACHE_MAX = 50 # hard cap on entries (keys rotate every minute per chart) -def _candle_cache_put(key: tuple, value: list, now: float) -> None: - """Insert into the candle cache, evicting expired entries and capping size.""" +def _candle_cache_put(key: tuple, value: list) -> None: + """Insert into the candle cache, evicting expired entries and capping size. + + The stamp is taken here, at insert time, not at the start of the request: + a slow upstream fetch would otherwise write an entry already aged by its + own duration, and a fetch longer than the TTL would cache nothing at all. + """ + now = time.monotonic() expired = [ k for k, (ts, _) in _candle_cache.items() if now - ts >= _CANDLE_CACHE_TTL ] @@ -549,7 +555,7 @@ async def get_candles( pool_address, ), ) - _candle_cache_put(cache_key, candles, now) + _candle_cache_put(cache_key, candles) return candles diff --git a/tests/test_candle_cache.py b/tests/test_candle_cache.py index 17e792742..6e1ef9d85 100644 --- a/tests/test_candle_cache.py +++ b/tests/test_candle_cache.py @@ -5,6 +5,22 @@ import condor.web.routes.market as market +class _Clock: + """Stand-in for the `time` module inside condor.web.routes.market.""" + + def __init__(self, start=1000.0): + self.now = start + + def monotonic(self): + return self.now + + def time(self): + return 1_700_000_000.0 + + def advance(self, seconds): + self.now += seconds + + @pytest.fixture(autouse=True) def _clean_cache(): market._candle_cache.clear() @@ -12,37 +28,52 @@ def _clean_cache(): market._candle_cache.clear() -def test_cache_bounded_under_advancing_bucketed_starts(): +@pytest.fixture +def clock(monkeypatch): + c = _Clock() + monkeypatch.setattr(market, "time", c, raising=True) + return c + + +def test_cache_bounded_under_advancing_bucketed_starts(clock): """Simulate a chart minting a new bucketed_start key every request.""" - now = 1000.0 for i in range(500): key = ("srv", "binance", "BTC-USDT", "1m", 1000, i * 60, None) - market._candle_cache_put(key, [i], now + i) + market._candle_cache_put(key, [i]) + clock.advance(1) assert len(market._candle_cache) <= market._CANDLE_CACHE_MAX -def test_cache_bounded_under_burst_at_same_timestamp(): +def test_cache_bounded_under_burst_at_same_timestamp(clock): """Even with no time advancing (nothing expires), the size cap holds.""" - now = 1000.0 for i in range(500): key = ("srv", "binance", "BTC-USDT", "1m", 1000, i * 60, None) - market._candle_cache_put(key, [i], now) + market._candle_cache_put(key, [i]) assert len(market._candle_cache) <= market._CANDLE_CACHE_MAX -def test_fresh_entry_still_hits_within_ttl(): +def test_fresh_entry_still_hits_within_ttl(clock): """Repeated identical requests within the TTL keep hitting the cache.""" key = ("srv", "binance", "BTC-USDT", "1m", 1000, 0, None) - market._candle_cache_put(key, ["candles"], 100.0) + market._candle_cache_put(key, ["candles"]) # Another key inserted within the TTL must not evict the fresh entry - market._candle_cache_put(("other",), ["x"], 100.0 + market._CANDLE_CACHE_TTL - 1) + clock.advance(market._CANDLE_CACHE_TTL - 1) + market._candle_cache_put(("other",), ["x"]) cached = market._candle_cache.get(key) assert cached is not None assert cached[1] == ["candles"] -def test_expired_entries_swept_on_write(): - market._candle_cache_put(("old",), ["x"], 100.0) - market._candle_cache_put(("new",), ["y"], 100.0 + market._CANDLE_CACHE_TTL) +def test_expired_entries_swept_on_write(clock): + market._candle_cache_put(("old",), ["x"]) + clock.advance(market._CANDLE_CACHE_TTL) + market._candle_cache_put(("new",), ["y"]) assert ("old",) not in market._candle_cache assert ("new",) in market._candle_cache + + +def test_entry_is_stamped_at_insert_time_not_before_the_fetch(clock): + """The stamp is the clock at write time (CORR-584).""" + clock.advance(45.0) # a fetch that outran the TTL + market._candle_cache_put(("k",), ["candles"]) + assert market._candle_cache[("k",)][0] == clock.now diff --git a/tests/test_candle_cache_slow_fetch.py b/tests/test_candle_cache_slow_fetch.py new file mode 100644 index 000000000..cb1be186d --- /dev/null +++ b/tests/test_candle_cache_slow_fetch.py @@ -0,0 +1,117 @@ +"""A slow upstream fetch must still leave a usable cache entry (CORR-584). + +GET /market/candles used to stamp the cache with the timestamp captured +*before* the fetch, so an entry was born aged by the whole fetch duration. A +GeckoTerminal pool chart under the shared rate limiter can outrun the 30s TTL, +which made the entry stale the instant it was written and re-fired the upstream +call on the very next request — exactly the path the cache exists to protect. +""" + +import asyncio + +import pytest + +import condor.web.routes.market as market +from condor.web.models import WebUser + + +class _Clock: + """Stand-in for the `time` module inside condor.web.routes.market.""" + + def __init__(self, start=1000.0): + self.now = start + + def monotonic(self): + return self.now + + def time(self): + return 1_700_000_000.0 + + def advance(self, seconds): + self.now += seconds + + +class _Cm: + def __init__(self, client): + self._client = client + + def has_server_access(self, user_id, name): + return True + + async def get_client(self, name): + return self._client + + +class _SlowClient: + """Counts upstream calls; each one burns `duration` seconds of the clock.""" + + def __init__(self, clock, duration): + self.clock = clock + self.duration = duration + self.calls = 0 + + @property + def market_data(self): + return self + + async def get_historical_candles(self, *a, **kw): + self.calls += 1 + self.clock.advance(self.duration) + return [ + { + "timestamp": 1000.0, + "open": 1.0, + "high": 2.0, + "low": 0.5, + "close": 1.5, + "volume": 10.0, + } + ] + + async def get_candles(self, *a, **kw): + return [] + + +@pytest.fixture(autouse=True) +def _clean_state(): + market._candle_cache.clear() + market._candle_inflight.clear() + yield + market._candle_cache.clear() + market._candle_inflight.clear() + + +def _call(): + return market.get_candles( + "srv", + connector="binance", + trading_pair="BTC-USDT", + interval="1m", + limit=100, + start_time=1_700_000_000.0, + end_time=1_700_003_600.0, + pool_address=None, + user=WebUser(id=1, role="user"), + ) + + +def test_fetch_longer_than_the_ttl_still_caches(monkeypatch): + clock = _Clock() + client = _SlowClient(clock, duration=market._CANDLE_CACHE_TTL + 15.0) + monkeypatch.setattr(market, "time", clock, raising=True) + monkeypatch.setattr(market, "get_config_manager", lambda: _Cm(client), raising=True) + + async def scenario(): + first = await _call() + assert len(market._candle_cache) == 1 + # A request arriving immediately afterwards is served from the cache. + second = await _call() + return first, second + + first, second = asyncio.run(scenario()) + assert client.calls == 1, "the slow fetch's entry was already stale on write" + assert second is first + # The entry is stamped with the clock as of insert time, not request start. + ((stamp, cached_value),) = market._candle_cache.values() + assert stamp == clock.now + assert cached_value is first From acca075cf013d2e5d01d9034c2fa7281c1c27d9b Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 10:37:42 +0300 Subject: [PATCH 053/154] The routine source viewer stops 403ing every agent's routines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovery deliberately returns the general library, the shared one and every agent's own routines under the prefixed name slug/name — but the source route confined its read to a single, cwd-relative "routines". Agent routines live under agents//routines and .condor/agents//routines, so "View source" refused roughly two thirds of what the list above it offers, and a process started outside the repo lost the root library too. Widen the confinement to exactly the directories discovery imports from — routine_source_roots() — and compare with Path.is_relative_to against a resolved path, the traversal idiom the SPA guard already uses. The agent homes themselves stay out, so a journal or a memory store beside a routines/ dir is no more readable than before; ".." and symlinks are collapsed before the check, and a sibling like routines_backup/ no longer slips through on a string prefix. --- condor/routine_store.py | 33 ++++- condor/web/routes/routines.py | 8 +- routines/base.py | 12 +- tests/test_routine_source_confinement.py | 179 +++++++++++++++++++++++ 4 files changed, 227 insertions(+), 5 deletions(-) create mode 100644 tests/test_routine_source_confinement.py diff --git a/condor/routine_store.py b/condor/routine_store.py index 9a142b6b4..d664cc4e6 100644 --- a/condor/routine_store.py +++ b/condor/routine_store.py @@ -17,13 +17,18 @@ import condor.reports as reports from condor import primitives, routine_hooks -from condor.memory.paths import agent_home_layers, iter_agent_slugs +from condor.memory.paths import ( + agent_home_layers, + iter_agent_slugs, + shared_routines_roots, +) from condor.telemetry import taps as telemetry_taps from routines.base import ( RoutineResult, _merged_from, discover_routines, get_routine, + library_dir, normalize_result, ) @@ -35,6 +40,32 @@ def _agent_routine_dirs(slug: str) -> tuple: return tuple(home / "routines" for home in agent_home_layers(slug)) +def routine_source_roots() -> tuple[Path, ...]: + """Every directory :meth:`RoutineStore._discover_all` imports routine files from. + + The allowlist a reader of routine source confines itself to (CORR-585): the + general library, the shared library in both layers, and each agent's own + ``routines/`` dir in both layers — the agent *homes* themselves stay out, so + a journal or a memory store next door is never in scope. ``_shared`` is + named explicitly because ``iter_agent_slugs`` skips the ``_``-prefixed + library dirs. + + Every root comes back resolved, so a caller comparing with + :meth:`pathlib.Path.is_relative_to` against an equally resolved path admits + neither ``..`` nor a symlink out of one, and (unlike a string prefix) never + mistakes a sibling like ``routines_backup/`` for the library. + """ + roots = [library_dir(), *shared_routines_roots()] + for slug in iter_agent_slugs(): + roots.extend(_agent_routine_dirs(slug)) + resolved: list[Path] = [] + for root in roots: + candidate = root.resolve() + if candidate not in resolved: + resolved.append(candidate) + return tuple(resolved) + + class _HttpBot: """Fallback bot that sends Telegram messages via HTTP when no real bot is available.""" diff --git a/condor/web/routes/routines.py b/condor/web/routes/routines.py index e1649654d..a19e6c782 100644 --- a/condor/web/routes/routines.py +++ b/condor/web/routes/routines.py @@ -12,7 +12,7 @@ from condor import routine_hooks from condor.reports import list_reports -from condor.routine_store import get_routine_store +from condor.routine_store import get_routine_store, routine_source_roots from condor.runtime import client, wake from condor.runtime.wake import ( ON_COMPLETE_CHOICES, @@ -358,8 +358,10 @@ async def get_routine_source( try: source_file = inspect.getfile(routine.run_fn) source_path = Path(source_file).resolve() - routines_dir = Path("routines").resolve() - if not str(source_path).startswith(str(routines_dir)): + # CORR-585: confine to the roots discovery actually reads — the general + # library, the shared one and each agent's own routines/ — not just a + # cwd-relative "routines", which 403'd every agent routine above. + if not any(source_path.is_relative_to(r) for r in routine_source_roots()): raise HTTPException(403, "Source not available") source = source_path.read_text() return {"filename": source_path.name, "source": source} diff --git a/routines/base.py b/routines/base.py index 9f1f6c19e..1bfcd1781 100644 --- a/routines/base.py +++ b/routines/base.py @@ -27,6 +27,16 @@ _PROJECT_ROOT = Path(__file__).resolve().parent.parent +def library_dir() -> Path: + """The general routine library on disk: the directory this module lives in. + + Anchored at the module, never at the working directory: a process started + outside the repo still resolves the same ``routines/`` discovery imports + from, which a cwd-relative ``Path("routines")`` does not. + """ + return Path(__file__).resolve().parent + + def assistant_routines_dir(agent_slug: str | None) -> Path: """The **writable** routines dir of an assistant — the one it owns. @@ -210,7 +220,7 @@ def discover_routines(force_reload: bool = False) -> dict[str, RoutineInfo]: prev_routines = {} if fresh_start else _routines_cache prev_mtimes = {} if fresh_start else _routines_mtimes - routines_dir = Path(__file__).parent + routines_dir = library_dir() routines = {} scanned_mtimes: dict[str, float | None] = {} diff --git a/tests/test_routine_source_confinement.py b/tests/test_routine_source_confinement.py new file mode 100644 index 000000000..f748b732f --- /dev/null +++ b/tests/test_routine_source_confinement.py @@ -0,0 +1,179 @@ +"""CORR-585: the source viewer's allowlist must cover every root discovery reads. + +``GET /routines/{name}/source`` confines the file it reads to an allowlist, and +that allowlist used to hold a single cwd-relative ``routines``. Discovery, +though, deliberately also returns the shared library and every agent's own +routines under the prefixed name ``slug/name`` — none of which live under the +root library, so "View source" 403'd for every one of them. + +The allowlist stays a confinement check: it now names the roots +``RoutineStore._discover_all`` actually imports from, and nothing wider. +""" + +from pathlib import Path + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +import condor.web.routes.routines as routines_module +import routines.base +from condor.memory.paths import ( + local_agents_root, + shared_routines_roots, + stock_agents_root, +) +from condor.web.auth import get_current_user +from condor.web.models import WebUser + +USER = WebUser(id=111, username="u", first_name="U", role="user") + +SOURCE = "async def run(config, context):\n return 'ok'\n" + + +class _Routine: + """The one attribute the source route reads off a discovered routine.""" + + def __init__(self, run_fn): + self.run_fn = run_fn + + +def _routine_at(path: Path): + """Write a routine file and return it as a discovered routine. + + Compiled with the file as its name, so ``inspect.getfile(run_fn)`` reports + that path exactly as it would for a really-imported routine module. + """ + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(SOURCE) + namespace: dict = {} + exec(compile(SOURCE, str(path), "exec"), namespace) + return _Routine(namespace["run"]) + + +class FakeStore: + def __init__(self, routines): + self._routines = routines + + def _discover_all(self): + return self._routines + + +@pytest.fixture +def client_for(monkeypatch): + """Serve the routes with a store that returns exactly the given routines.""" + + def _build(routines): + monkeypatch.setattr( + routines_module, "get_routine_store", lambda: FakeStore(routines) + ) + app = FastAPI() + app.include_router(routines_module.router) + app.dependency_overrides[get_current_user] = lambda: USER + return TestClient(app) + + return _build + + +# ── Admitted: the roots discovery reads ── + + +def test_a_shipped_agents_routine_serves_its_source(client_for): + path = stock_agents_root() / "brigado" / "routines" / "bot_report.py" + client = client_for({"brigado/bot_report": _routine_at(path)}) + + resp = client.get("/routines/brigado/bot_report/source") + + assert resp.status_code == 200, resp.text + assert resp.json() == {"filename": "bot_report.py", "source": SOURCE} + + +def test_a_locally_authored_agent_routine_serves_its_source(client_for): + path = local_agents_root() / "mine" / "routines" / "scratch.py" + client = client_for({"mine/scratch": _routine_at(path)}) + + assert client.get("/routines/mine/scratch/source").status_code == 200 + + +def test_a_shared_library_routine_serves_its_source(client_for): + # ``_shared`` is skipped by iter_agent_slugs, so it has to be allowlisted + # by name rather than inherited from the agents root. + for root in shared_routines_roots(): + client = client_for({"shared_one": _routine_at(root / "shared_one.py")}) + assert client.get("/routines/shared_one/source").status_code == 200 + + +def test_the_general_library_still_serves_its_source(client_for, tmp_path, monkeypatch): + library = Path(routines.base.__file__).resolve().parent + existing = next( + p for p in sorted(library.glob("*.py")) if p.stem not in ("__init__", "base") + ) + client = client_for({existing.stem: _Routine(_fn_named(existing))}) + # A cwd outside the repo used to 403 even this: the old root was relative. + monkeypatch.chdir(tmp_path) + + resp = client.get(f"/routines/{existing.stem}/source") + + assert resp.status_code == 200, resp.text + assert resp.json()["filename"] == existing.name + + +def _fn_named(path: Path): + """A callable that ``inspect.getfile`` reports as living at ``path``.""" + namespace: dict = {} + exec(compile(SOURCE, str(path), "exec"), namespace) + return namespace["run"] + + +# ── Refused: everything else ── + + +def test_a_file_outside_every_root_is_refused(client_for, tmp_path): + path = tmp_path / "elsewhere" / "secrets.py" + client = client_for({"sneaky": _routine_at(path)}) + + resp = client.get("/routines/sneaky/source") + + assert resp.status_code == 403 + assert "secret" not in resp.text.lower() + + +def test_traversal_out_of_a_root_is_refused(client_for): + # Resolved before the check, so ``..`` never leaves the path it walks out of. + path = stock_agents_root() / "brigado" / "routines" / ".." / ".." / ".." / "keys.py" + client = client_for({"brigado/escape": _routine_at(path)}) + + assert client.get("/routines/brigado/escape/source").status_code == 403 + + +def test_a_symlink_pointing_out_of_a_root_is_refused(client_for, tmp_path): + outside = tmp_path / "outside" + outside.mkdir(parents=True, exist_ok=True) + (outside / "keys.py").write_text(SOURCE) + routines_dir = stock_agents_root() / "brigado" / "routines" + routines_dir.mkdir(parents=True, exist_ok=True) + link = routines_dir / "linked.py" + link.symlink_to(outside / "keys.py") + client = client_for({"brigado/linked": _Routine(_fn_named(link))}) + + assert client.get("/routines/brigado/linked/source").status_code == 403 + + +def test_an_agent_home_next_to_its_routines_is_not_readable(client_for): + # The allowlist admits ``/routines``, not the home itself: a + # journal or a memory store beside it stays out of reach. + home = local_agents_root() / "mine" + (home / "routines").mkdir(parents=True, exist_ok=True) + client = client_for({"mine/journal": _routine_at(home / "store" / "journal.py")}) + + assert client.get("/routines/mine/journal/source").status_code == 403 + + +def test_a_sibling_sharing_a_root_s_prefix_is_refused(client_for): + # ``startswith`` admitted ``…/routines_backup``; is_relative_to does not. + routines_dir = stock_agents_root() / "brigado" / "routines" + routines_dir.mkdir(parents=True, exist_ok=True) + sibling = routines_dir.with_name("routines_backup") / "leak.py" + client = client_for({"brigado/leak": _routine_at(sibling)}) + + assert client.get("/routines/brigado/leak/source").status_code == 403 From 6acf5ea9a3d3fec81aea4bd6ceddd05cedb47cd4 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 10:47:13 +0300 Subject: [PATCH 054/154] Stop one stray cancellation from silently retiring the SDS poller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_rate_limited_fetch` guarded its fetch with `except Exception`, which cannot catch `CancelledError` — a `BaseException` since 3.8. A shared single-flight fetch cancelled by one of its own dependencies therefore raised straight through `_poll_tick`'s gather into `_poll_loop`, which broke on cancellation unconditionally while `_running` stayed True. `start()` returns early on `_running`, so the loop was gone for the lifetime of the process, with nothing logged, and every surface served the last cached snapshot until Condor was restarted. Both sites now draw the distinction between "I was cancelled" and "what I awaited was cancelled", by asking the running task whether a cancellation was actually requested of it. Ours propagates — the loop re-raises rather than breaking, so the task genuinely ends cancelled and a shutdown can never be swallowed; somebody else's is logged and the next tick carries on. Closes CORR-601 (steps 1-3 and 5 were already delivered by ARCH-606). --- condor/server_data_service.py | 37 ++++++- tests/test_sds_poll_loop_cancellation.py | 128 +++++++++++++++++++++++ 2 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 tests/test_sds_poll_loop_cancellation.py diff --git a/condor/server_data_service.py b/condor/server_data_service.py index 2ccceb52e..34d304f6e 100644 --- a/condor/server_data_service.py +++ b/condor/server_data_service.py @@ -721,7 +721,26 @@ async def _poll_loop(self) -> None: self._cleanup_stale() self._last_cleanup = now except asyncio.CancelledError: - break + # Two very different events arrive here as the same exception: + # *this* task being cancelled (stop(), shutdown), and something + # this tick awaited being cancelled — a shared single-flight + # fetch killed by one of its own dependencies. Only the first + # ends the loop, and it is re-raised rather than swallowed so + # the task genuinely finishes cancelled. + # + # The second must not end it. ``break`` here left ``_running`` + # True, and start() returns early on that, so one stray + # cancellation silently retired the poller for the lifetime of + # the process and every surface served the last cached value + # until Condor was restarted. + task = asyncio.current_task() + if not self._running or (task is not None and task.cancelling()): + raise + logger.error( + "SDS poll loop absorbed a cancellation it did not request; " + "continuing to poll" + ) + continue except Exception as e: logger.error("SDS poll loop error: %s", e, exc_info=True) await asyncio.sleep(5) @@ -765,6 +784,22 @@ async def _rate_limited_fetch(key: CacheKey): return try: await self._fetch_and_cache(key) + except asyncio.CancelledError: + # ``except Exception`` never caught this: CancelledError is a + # BaseException. Ours — the poll task was cancelled and gather + # cancelled this child with it — must propagate so stop() really + # stops. A cancellation that came out of the shared fetch is not + # ours, and must not travel up through gather into _poll_loop, + # where it is indistinguishable from a shutdown. + task = asyncio.current_task() + if task is not None and task.cancelling(): + raise + logger.warning( + "SDS: fetch for %s:%s was cancelled by something it awaited; " + "skipping it this tick", + key.server, + key.data_type.value, + ) except Exception: pass # Error already recorded in _fetch_and_cache diff --git a/tests/test_sds_poll_loop_cancellation.py b/tests/test_sds_poll_loop_cancellation.py new file mode 100644 index 000000000..707deb513 --- /dev/null +++ b/tests/test_sds_poll_loop_cancellation.py @@ -0,0 +1,128 @@ +"""CORR-601: a stray cancellation must not permanently retire the SDS poller. + +``_rate_limited_fetch`` guarded its fetch with ``except Exception``, which +cannot catch ``CancelledError`` (a ``BaseException`` since 3.8). So a shared +single-flight fetch cancelled by one of *its* dependencies raised straight +through ``_poll_tick``'s gather into ``_poll_loop``, which ``break``\\ -ed on +cancellation unconditionally — while ``_running`` stayed True. ``start()`` +returns early on ``_running``, so the poll loop was gone for the lifetime of +the process, silently, and every surface (dashboard WS, REST, Telegram) served +the last cached snapshot until Condor was restarted. + +The distinction the fix draws is the one CORR-332 drew: "I was cancelled" +(propagate — a shutdown must never be swallowed) versus "what I awaited was +cancelled" (log and carry on). +""" + +import asyncio + +from condor import server_data_service as sds_module +from condor.server_data_service import ServerDataService, ServerDataType + + +def _make_sds(fetch_func): + """Fresh (non-singleton) SDS with a fake client and a registered fetcher.""" + sds = ServerDataService() + + async def _fake_get_client(server_name): + return object() + + sds._get_client = _fake_get_client + sds.register_fetch(ServerDataType.PORTFOLIO, fetch_func) + return sds + + +async def _until(predicate, timeout=3.0): + """Wait for ``predicate()`` to hold, yielding to the poll loop meanwhile.""" + deadline = asyncio.get_running_loop().time() + timeout + while asyncio.get_running_loop().time() < deadline: + if predicate(): + return True + await asyncio.sleep(0.01) + return predicate() + + +def test_poll_loop_survives_a_fetch_cancelled_by_its_dependency(monkeypatch): + """A cancelled shared fetch must not take the poll loop down with it.""" + monkeypatch.setattr(sds_module, "_POLL_TICK", 0.01) + + calls = {"count": 0} + armed = {"on": False} + + async def _drive(): + # Something the fetch awaits and does not own: another task, cancelled + # by whoever does own it while this fetch is parked on it. + dependency = asyncio.ensure_future(asyncio.sleep(3600)) + + async def fetcher(client, **params): + calls["count"] += 1 + if armed["on"]: + armed["on"] = False + await dependency # cancelled out from under us + return {"tick": calls["count"]} + + sds = _make_sds(fetcher) + key = await sds.subscribe("srv", ServerDataType.PORTFOLIO, "sub", interval=0.02) + assert sds._cache[key].value == {"tick": 1}, "priming fetch cached a value" + + armed["on"] = True + sds.start() + try: + # Let the poll loop reach the armed fetch and park on the dependency. + assert await _until(lambda: calls["count"] >= 2), "poll loop never ticked" + assert await _until(lambda: bool(sds._inflight)), "fetch never in flight" + + dependency.cancel() + + # The loop must still be alive and still refreshing the cache. + refreshed = await _until(lambda: sds._cache[key].value != {"tick": 1}) + poll_task = sds._poll_task + assert poll_task is not None + assert ( + not poll_task.done() + ), "the poll task died on a cancellation it did not request" + assert sds._running is True + assert refreshed, "the poll loop stopped refreshing the cache" + assert sds._cache[key].value["tick"] >= 3 + finally: + sds.stop() + await asyncio.sleep(0.02) + + asyncio.run(_drive()) + + +def test_stop_still_stops_the_poll_loop_mid_fetch(monkeypatch): + """The absorbed cancellation must never absorb a real shutdown. + + ``stop()`` while a fetch is in flight cancels the poll task, which cancels + the gather child parked on that fetch; that cancellation is *ours* and has + to propagate all the way out, leaving the task cancelled and ``_running`` + False. + """ + monkeypatch.setattr(sds_module, "_POLL_TICK", 0.01) + + calls = {"count": 0} + armed = {"on": False} + + async def _drive(): + async def fetcher(client, **params): + calls["count"] += 1 + if armed["on"]: + await asyncio.sleep(3600) # never settles + return {"tick": calls["count"]} + + sds = _make_sds(fetcher) + await sds.subscribe("srv", ServerDataType.PORTFOLIO, "sub", interval=0.02) + + armed["on"] = True + sds.start() + assert await _until(lambda: calls["count"] >= 2), "poll loop never ticked" + assert await _until(lambda: bool(sds._inflight)), "fetch never in flight" + + sds.stop() + poll_task = sds._poll_task + assert await _until(lambda: poll_task.done()), "stop() did not stop the loop" + assert poll_task.cancelled(), "the shutdown cancellation was swallowed" + assert sds._running is False + + asyncio.run(_drive()) From e1b27a7eafd56a469efd1a743a7c5602dd352548 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 10:53:42 +0300 Subject: [PATCH 055/154] Price the executors KPI strip off the shared USD rate resolver The strip resolved quote->USD itself: it asked the ticker pool for `{QUOTE}-USDT` and, on a miss, kept the figure at face value and flipped `converted` to false. That is the job `condor.quote_conversion.resolve_usd_rates` already does for the archived-run path -- whose own docstring names this module as the convention it follows, i.e. an acknowledged second copy. The copies disagreed on stablecoins. `resolve_usd_rates` short-circuits every quote in USD_QUOTES to 1.0, precisely so a quote with no market at all still prices correctly; the strip had no such short-circuit, so on a server whose pool lists no DAI/FDUSD/PYUSD market the same executor history read as "approximate" in the KPI tiles and as converted everywhere else. The dollars move too, slightly: a USDC-quoted total is now exactly its own value instead of being scaled by the live USDC/USDT market. Deletes the local resolution and the try/except around it (the helper catches, logs and degrades internally). The rate-outage test moves to a BRL quote, since a stablecoin total needs no lookup to flag. --- condor/web/routes/executors.py | 30 +++++++------- tests/test_executors_period_summary.py | 56 +++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 17 deletions(-) diff --git a/condor/web/routes/executors.py b/condor/web/routes/executors.py index 120ca7339..b63096702 100644 --- a/condor/web/routes/executors.py +++ b/condor/web/routes/executors.py @@ -224,33 +224,33 @@ async def _usd_summary( ticker pool. A quote with no path to USD is added at face value and flips ``converted`` — the same fallback the strip's client-side ``convert()`` made, but reported instead of silent. + + The resolution itself belongs to :func:`condor.quote_conversion.resolve_usd_rates`, + which the archived-run path already uses (CORR-602). The copy that used to live + here asked the pool for ``DAI-USDT`` and called the total "approximate" on a + server with no such market, while the same history read as converted elsewhere — + the shared helper short-circuits every stablecoin quote to 1.0 instead, so the + two surfaces agree on both the dollars and the confidence flag. """ - from condor.market_rates import get_rates + from condor.quote_conversion import resolve_usd_rates - rates: dict[str, float | None] = {} - if by_quote: - try: - rates = await get_rates(server, [f"{q}-USDT" for q in by_quote]) - except Exception as e: - logger.warning( - "Rates unavailable while summarizing executors for %s: %s", server, e - ) + quote_rates = await resolve_usd_rates(server, set(by_quote)) pnl = 0.0 volume = 0.0 count = 0 - converted = True for quote, totals in by_quote.items(): - rate = rates.get(f"{quote}-USDT") - if not rate or rate <= 0: - converted = False - rate = 1.0 + rate = quote_rates.rates.get(quote, 1.0) pnl += totals["pnl"] * rate volume += totals["volume"] * rate count += int(totals["count"]) return ExecutorPeriodSummary( - period=period, pnl=pnl, volume=volume, count=count, converted=converted + period=period, + pnl=pnl, + volume=volume, + count=count, + converted=quote_rates.converted, ) diff --git a/tests/test_executors_period_summary.py b/tests/test_executors_period_summary.py index 7883ae477..6bc0093dd 100644 --- a/tests/test_executors_period_summary.py +++ b/tests/test_executors_period_summary.py @@ -171,8 +171,45 @@ def test_an_unpriceable_quote_is_reported_not_hidden(summary_env): def test_a_rate_lookup_failure_still_returns_the_totals(summary_env, monkeypatch): """Rates down is not a reason to blank the tile; it is a reason to flag it.""" - rows = [_executor(0, pnl=6.0)] - summary_env(rows, {"USDT-USDT": 1.0}) + rows = [_executor(0, pair="BTC-BRL", pnl=6.0)] + summary_env(rows, {"BRL-USDT": 0.2}) + + async def _boom(server, pairs, connector=None): + raise RuntimeError("ticker pool unreachable") + + monkeypatch.setattr("condor.market_rates.get_rates", _boom) + + result = _summary("1D") + + assert result.pnl == pytest.approx(6.0) + assert result.converted is False + + +# ── CORR-602: stablecoin quotes price off the shared helper, not a market ── + + +def test_a_stablecoin_quote_prices_without_a_market(summary_env): + """DAI is a dollar even on a server whose pool lists no DAI market. + + The KPI strip used to resolve rates itself and ask the pool for ``DAI-USDT``; + a server without that market got ``converted: false`` here while the + archived-run path — which has always gone through ``resolve_usd_rates`` — + called the same history converted. Both now short-circuit the stablecoin. + """ + rows = [_executor(0, pair="ETH-DAI", pnl=4.0, volume=9.0)] + summary_env(rows, {}) # no DAI-USDT market anywhere in the pool + + result = _summary("1D") + + assert result.pnl == pytest.approx(4.0), "a DAI dollar is a dollar" + assert result.volume == pytest.approx(9.0) + assert result.converted is True, "a stablecoin total is exact, not approximate" + + +def test_a_stablecoin_total_survives_a_rate_outage(summary_env, monkeypatch): + """No quote needs a lookup, so an unreachable pool cannot make it approximate.""" + rows = [_executor(0, pair="SOL-USDC", pnl=5.0, volume=11.0)] + summary_env(rows, {}) async def _boom(server, pairs, connector=None): raise RuntimeError("ticker pool unreachable") @@ -181,6 +218,21 @@ async def _boom(server, pairs, connector=None): result = _summary("1D") + assert result.pnl == pytest.approx(5.0) + assert result.volume == pytest.approx(11.0) + assert result.converted is True + + +def test_a_mixed_total_flags_only_the_quote_that_failed(summary_env): + """A resolvable stable plus an unpriceable quote: dollars right, flag honest.""" + rows = [ + _executor(0, pair="ETH-DAI", pnl=4.0, volume=9.0), + _executor(1, pair="FOO-XYZ", pnl=2.0, volume=3.0), + ] + summary_env(rows, {}) + + result = _summary("1D") + assert result.pnl == pytest.approx(6.0) assert result.converted is False From c6350196ebef8f5f02d52a88bd37c2f3e6e18c99 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 11:01:42 +0300 Subject: [PATCH 056/154] Paginate search_history orders by the cursor the backend actually uses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /trading/orders/search is cursor-paginated and has no offset parameter at all, but the orders branch passed `cursor=None, # We use offset instead for pagination` and then told the caller `... and more (use offset=N to see more)`. A model that followed that hint re-issued the identical request forever and, because the rows came back identical while the tool kept saying "more", read page one over and over as though it were walking history. For the documented use case — reporting and tax — that silently truncated every result set at the first page. Thread the real cursor through instead: search_history takes `cursor`, forwards it to search_orders, and prints the backend's own next-cursor (read with the shared fetchers._pagination.next_cursor, which already knows all four spellings) so the hint is something the API can act on. When the backend says has_more but returns no cursor, say that rather than inventing one. The three branches paginate three different ways, so each now refuses the spelling it would drop, through the _UNSUPPORTED_FILTERS dict CORR-563 added: orders refuses `offset`, and clmm_positions and perp_positions refuse `cursor`. The perp entry matters most — the trading router accepts a cursor but our own get_positions wrapper neither takes nor forwards one, so accepting it here would have reintroduced exactly the silent drop CORR-563 just closed. CORR-569 --- mcp_servers/hummingbot_api/server.py | 6 + mcp_servers/hummingbot_api/tools/history.py | 50 +++++- tests/test_mcp_search_history_pagination.py | 164 ++++++++++++++++++++ 3 files changed, 213 insertions(+), 7 deletions(-) create mode 100644 tests/test_mcp_search_history_pagination.py diff --git a/mcp_servers/hummingbot_api/server.py b/mcp_servers/hummingbot_api/server.py index 9e591756c..2d40f4d4f 100644 --- a/mcp_servers/hummingbot_api/server.py +++ b/mcp_servers/hummingbot_api/server.py @@ -315,6 +315,7 @@ async def search_history( end_time: int | None = None, limit: int = 50, offset: int = 0, + cursor: str | None = None, network: str | None = None, wallet_address: str | None = None, position_addresses: list[str] | None = None, @@ -342,6 +343,9 @@ async def search_history( start_time: orders only, timestamp in seconds (optional) end_time: orders only, timestamp in seconds (optional) offset: clmm_positions only, pagination offset (default: 0) + cursor: orders only, the cursor printed at the end of the previous page + (optional). orders is cursor-paginated and has no offset at all; + clmm_positions is offset-paginated and takes no cursor. CLMM-Specific Filters: network: Network filter for CLMM positions (optional) @@ -351,6 +355,7 @@ async def search_history( Examples: - Search filled orders: search_history("orders", status="FILLED", limit=100) - Orders in a time window: search_history("orders", start_time=..., end_time=...) + - Next page of orders: search_history("orders", cursor="") - Current perp book: search_history("perp_positions", account_names=["master"]) - Search all CLMM positions: search_history("clmm_positions", limit=100) """ @@ -367,6 +372,7 @@ async def search_history( end_time=end_time, limit=limit, offset=offset, + cursor=cursor, network=network, wallet_address=wallet_address, position_addresses=position_addresses, diff --git a/mcp_servers/hummingbot_api/tools/history.py b/mcp_servers/hummingbot_api/tools/history.py index cc8276212..9d4bf98a4 100644 --- a/mcp_servers/hummingbot_api/tools/history.py +++ b/mcp_servers/hummingbot_api/tools/history.py @@ -10,6 +10,7 @@ import logging from typing import Any, Literal +from condor.fetchers._pagination import next_cursor from mcp_servers.hummingbot_api.exceptions import ToolError from mcp_servers.hummingbot_api.hummingbot_client import HummingbotClient @@ -22,19 +23,41 @@ # Filters that each data_type can actually forward to the backend. Anything else # in the shared signature is refused instead of being silently dropped. _UNSUPPORTED_FILTERS: dict[str, tuple[str, ...]] = { + # The three branches paginate three different ways, and the shared signature + # offers both spellings, so each one has to refuse the spelling it drops + # (CORR-569). POST /trading/orders/search is cursor-only — it has no offset + # parameter at all — while gateway_clmm.search_positions is offset-only. + "orders": ("offset",), # The trading router has no closed-position history endpoint: get_positions - # POSTs /trading/positions with only account_names/connector_names/limit. - # orders and clmm_positions forward their filters, so only the perp branch - # needs a guard. (The orders branch's offset pagination hint is CORR-569's.) - "perp_positions": ("trading_pairs", "status", "start_time", "end_time", "offset"), + # POSTs /trading/positions with only account_names/connector_names/limit. The + # route accepts a cursor but our wrapper (tools/trading.py:get_positions) + # neither takes nor forwards one, so cursor is refused here too rather than + # accepted and dropped. + "perp_positions": ( + "trading_pairs", + "status", + "start_time", + "end_time", + "offset", + "cursor", + ), + "clmm_positions": ("cursor",), } _FILTER_ALTERNATIVES: dict[str, str] = { + "orders": ( + "orders is cursor-paginated: pass the cursor printed at the end of the " + "previous page back as cursor=, rather than an offset." + ), "perp_positions": ( "perp_positions returns the CURRENT open book (the backend has no closed " 'position history endpoint). Use data_type="orders" for a time-windowed ' "history, or get_portfolio_overview() for the same open positions." ), + "clmm_positions": ( + "clmm_positions is offset-paginated: use offset= (the value printed at " + "the end of the previous page) rather than a cursor." + ), } @@ -71,6 +94,7 @@ async def search_history( # Pagination limit: int = 50, offset: int = 0, + cursor: str | None = None, # CLMM-specific filters network: str | None = None, wallet_address: str | None = None, @@ -101,6 +125,7 @@ async def search_history( end_time: End timestamp in seconds (orders only, optional) limit: Maximum number of results (all data types, default: 50, max: 1000) offset: Pagination offset (clmm_positions only, default: 0) + cursor: Pagination cursor from the previous page (orders only, optional) network: Network filter for CLMM positions (optional) wallet_address: Wallet address filter for CLMM positions (optional) position_addresses: Specific position addresses for CLMM (optional) @@ -121,6 +146,7 @@ async def search_history( start_time=start_time, end_time=end_time, offset=offset, + cursor=cursor, ) try: @@ -138,14 +164,24 @@ async def search_history( start_time=start_time, end_time=end_time, limit=min(limit, 1000), - cursor=None, # We use offset instead for pagination + cursor=cursor, ) formatted_output = f"Order History\n{'=' * 100}\n\n{result['orders_table']}" - if result["pagination"].get("has_more"): + # The backend paginates this route by opaque cursor. The hint used to + # print `use offset=N`, which search_orders has no parameter for, so a + # model that followed it re-fetched page one forever and read the + # identical rows as fresh history (CORR-569). + following = next_cursor(result) + if following: + formatted_output += ( + f'\n\n... and more (use cursor="{following}" to see the next page)' + ) + elif result["pagination"].get("has_more"): formatted_output += ( - f"\n\n... and more (use offset={offset + limit} to see more)" + "\n\n... and more, but the backend returned no next cursor: " + "narrow the search with start_time/end_time to reach the rest." ) return { diff --git a/tests/test_mcp_search_history_pagination.py b/tests/test_mcp_search_history_pagination.py new file mode 100644 index 000000000..f02608e18 --- /dev/null +++ b/tests/test_mcp_search_history_pagination.py @@ -0,0 +1,164 @@ +"""search_history must paginate orders by the cursor the API actually uses (CORR-569). + +POST /trading/orders/search is cursor-paginated and has no ``offset`` parameter at +all, but the orders branch passed ``cursor=None, # We use offset instead`` and then +told the caller ``... and more (use offset=N to see more)``. A model that followed +that hint re-issued the identical request forever and — because the rows came back +identical while the tool said "more" — read page one over and over as if it were +walking history. For the stated use case (reporting, tax) that silently truncated +every result set at the first page. + +These tests drive the real branch through the server-level tool and assert on what +actually crosses the wire (the cursor in the outgoing request) and on what the model +actually reads back (the next-cursor in the formatted output) — not on a helper's +return value. + +The repo has no async test setup, so the coroutines are driven with asyncio.run(). +""" + +import asyncio +import re + +import pytest + +from mcp_servers.hummingbot_api import server as hb_server +from mcp_servers.hummingbot_api.exceptions import ToolError + +PAGE_ONE = [ + { + "trading_pair": "SOL-USDC", + "trade_type": "BUY", + "order_type": "LIMIT", + "amount": 10, + "price": 200, + "status": "FILLED", + } +] +PAGE_TWO = [ + { + "trading_pair": "ETH-USDC", + "trade_type": "SELL", + "order_type": "LIMIT", + "amount": 2, + "price": 3000, + "status": "FILLED", + } +] + +NEXT_CURSOR = "eyJvZmZzZXQiOjF9" + + +class PaginatingTrading: + """A cursor-paginated /trading/orders/search, as the backend really behaves. + + The two pages hold different pairs, so a caller that re-fetches page one is + visibly distinguishable from one that genuinely advanced. + """ + + def __init__(self): + self.order_calls = [] + self.position_calls = [] + + async def search_orders(self, **kwargs): + self.order_calls.append(kwargs) + cursor = kwargs.get("cursor") + if cursor is None: + return { + "data": PAGE_ONE, + "pagination": {"has_more": True, "next_cursor": NEXT_CURSOR}, + } + if cursor == NEXT_CURSOR: + return { + "data": PAGE_TWO, + "pagination": {"has_more": False, "next_cursor": None}, + } + raise AssertionError(f"unknown cursor sent to the backend: {cursor!r}") + + async def get_positions(self, **kwargs): + self.position_calls.append(kwargs) + return {"data": []} + + +class PaginatingClient: + def __init__(self): + self.trading = PaginatingTrading() + + +@pytest.fixture +def client_calls(monkeypatch): + """Drive the server-level tool against a cursor-paginated recording client.""" + client = PaginatingClient() + + async def fake_get_client(): + return client + + monkeypatch.setattr(hb_server.hummingbot_client, "get_client", fake_get_client) + return client.trading + + +def test_first_page_of_orders_surfaces_the_backend_cursor_not_an_offset(client_calls): + """The hint the model reads has to be something the backend can act on.""" + output = asyncio.run(hb_server.search_history(data_type="orders", limit=1)) + + assert client_calls.order_calls[0]["cursor"] is None + assert "SOL-USDC" in output + assert NEXT_CURSOR in output, f"next cursor not surfaced to the model: {output}" + # The old hint was `use offset=1`, which search_orders has no parameter for. + assert "offset=" not in output + + +def test_passing_the_cursor_back_returns_a_different_page(client_calls): + """The acceptance criterion: page two is genuinely page two, not page one again.""" + first = asyncio.run(hb_server.search_history(data_type="orders", limit=1)) + + # Take the cursor the way a model would: read it out of the rendered hint. + match = re.search(r'cursor="([^"]+)"', first) + assert match, f"no reusable cursor in the output: {first}" + + second = asyncio.run( + hb_server.search_history(data_type="orders", limit=1, cursor=match.group(1)) + ) + + assert client_calls.order_calls[1]["cursor"] == NEXT_CURSOR + assert "ETH-USDC" in second, "the second page repeated page one" + assert "SOL-USDC" not in second + # Last page: no cursor came back, so no hint is invented. + assert "cursor=" not in second + + +def test_orders_refuses_an_offset_it_would_silently_drop(client_calls): + """search_orders has no offset parameter; accepting one is the CORR-563 bug.""" + with pytest.raises(ToolError) as excinfo: + asyncio.run(hb_server.search_history(data_type="orders", offset=50)) + + message = str(excinfo.value) + assert "offset" in message + assert "silently ignored" in message + assert "cursor" in message, "the refusal should name the pagination that works" + assert client_calls.order_calls == [], "no request may go out" + + +@pytest.mark.parametrize("data_type", ["perp_positions", "clmm_positions"]) +def test_branches_without_cursor_support_refuse_a_cursor(client_calls, data_type): + """tools/trading.py:get_positions and gateway_clmm.search_positions take no cursor. + + The trading router does accept one, but our wrapper neither takes nor forwards + it, so an accepted-but-ignored cursor here would be exactly the silent drop + CORR-563 closed. + """ + with pytest.raises(ToolError) as excinfo: + asyncio.run(hb_server.search_history(data_type=data_type, cursor="abc")) + + message = str(excinfo.value) + assert "cursor" in message + assert "silently ignored" in message + assert client_calls.position_calls == [], "no request may go out" + + +def test_the_tool_signature_offers_a_cursor(): + """The signature is the schema: a model cannot pass what is not declared.""" + import inspect + + params = inspect.signature(hb_server.search_history).parameters + assert "cursor" in params + assert params["cursor"].default is None From 98186fe4cf0c9987ed2f0372ecf59609f72ecb79 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 11:17:28 +0300 Subject: [PATCH 057/154] Give the "a stored server name is not a capability" rule one home MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit check_server_access enforces reach, not existence, and has_server_access answers True for an admin on any arbitrary string. So every caller that turns a *stored* server name — written earlier, read now by someone else, possibly since revoked — into credentials needs existence AND reach, and five of them had each re-derived the same four lines: the strategy routes, the conversation deployments route, the tick engine, the session resolver and the toolset builder. SEC-164, SEC-178, SEC-333 and SEC-334 each landed one of those copies, because the previous fix left nothing to import. may_use_stored_server() now says it once. It lives in config_manager beside has_server_access rather than in web/auth.py beside check_server_access: three of the five callers sit below the web layer and cannot import from it without inverting the layering. It takes the manager as its first argument, matching require_owner — every caller already holds the one it means. The strategy-start gate keeps its own plain existence check. It runs check_server_access first, deliberately, so the route answers "No access" rather than becoming an oracle for which server names exist; routing it through this predicate would re-run the access test and say less. No behaviour changes — all five copies agreed. This is drift prevention on a security-relevant predicate. --- condor/agents/engine.py | 10 +++++---- condor/runtime/sessions.py | 10 +++++---- condor/runtime/toolsets.py | 5 ++--- condor/web/routes/agents.py | 26 +++++++-------------- condor/web/routes/conversations.py | 6 ++--- config_manager.py | 36 ++++++++++++++++++++++++++++++ 6 files changed, 60 insertions(+), 33 deletions(-) diff --git a/condor/agents/engine.py b/condor/agents/engine.py index 1b95cd48d..fa0173195 100644 --- a/condor/agents/engine.py +++ b/condor/agents/engine.py @@ -1115,14 +1115,16 @@ def _resolve_server(self) -> tuple[str | None, dict | None]: Falls back to the subject's accessible servers, as before, so a run with no server configured still works. """ - from config_manager import get_config_manager, get_effective_server + from config_manager import ( + get_config_manager, + get_effective_server, + may_use_stored_server, + ) cm = get_config_manager() def usable(name: str | None) -> bool: - return bool(name and cm.get_server(name)) and cm.has_server_access( - self.user_id, name - ) + return may_use_stored_server(cm, self.user_id, name) server_name = self.config.get("server_name") if not usable(server_name): diff --git a/condor/runtime/sessions.py b/condor/runtime/sessions.py index 19c76d788..3b06e928d 100644 --- a/condor/runtime/sessions.py +++ b/condor/runtime/sessions.py @@ -775,15 +775,17 @@ async def _spawn_session( # *and* reach, subjected on the run's own principal — the same id the # MCP toolset is built for, so the label and the credentials downstream # can never disagree about who this session belongs to. - from config_manager import get_config_manager, get_effective_server + from config_manager import ( + get_config_manager, + get_effective_server, + may_use_stored_server, + ) cm = get_config_manager() subject_id, _ = spec.effective_ids() def usable(name: str | None) -> bool: - return bool(name and cm.get_server(name)) and cm.has_server_access( - subject_id, name - ) + return may_use_stored_server(cm, subject_id, name) resolved_server = spec.server_name if not usable(resolved_server): diff --git a/condor/runtime/toolsets.py b/condor/runtime/toolsets.py index 5d60df21c..41c8f7277 100644 --- a/condor/runtime/toolsets.py +++ b/condor/runtime/toolsets.py @@ -311,6 +311,7 @@ def build_mcp_servers_for_session( ServerPermission, get_config_manager, get_effective_server, + may_use_stored_server, ) cm = get_config_manager() @@ -330,9 +331,7 @@ def build_mcp_servers_for_session( # is always ``user_id``, the authenticated owner of the run; the same # predicate guards TickEngine._resolve_server. def usable(name: str | None) -> bool: - return bool(name and cm.get_server(name)) and cm.has_server_access( - user_id, name - ) + return may_use_stored_server(cm, user_id, name) def candidates(): # Lazy on purpose: resolving the chat default writes back into diff --git a/condor/web/routes/agents.py b/condor/web/routes/agents.py index 2020dfaea..498ea06ab 100644 --- a/condor/web/routes/agents.py +++ b/condor/web/routes/agents.py @@ -931,26 +931,16 @@ def _strategy_principal(strategy, user: WebUser) -> int: def _may_use_strategy_server(server_name: str, principal: int) -> bool: """Whether ``principal`` may turn ``server_name`` into credentials. - Existence *and* reach, in that order of importance: a stored name is not a - capability. It was written by whoever created the strategy, it is read now - by somebody else, and a share can be withdrawn long after either happened - (SEC-334). The level is the TRADER floor ``check_server_access`` applies to - every server-scoped web call. - - The existence check is not redundant with the access one: - ``has_server_access`` answers True for an admin on an arbitrary string, so - without it a name that resolves to nothing today would be honoured the - moment a server is created under it — the same reasoning spelled out for - SEC-164 in ``_start``. + The stored-name predicate (SEC-334), which lives on the ConfigManager + beside ``has_server_access`` because four other callers — the tick engine, + session and toolset resolvers, and ``conversations.py`` — need the same + rule and cannot import the web layer (ARCH-587). The level it applies is + the TRADER floor ``check_server_access`` applies to every server-scoped web + call. """ - from config_manager import ServerPermission, get_config_manager + from config_manager import get_config_manager, may_use_stored_server - if not server_name: - return False - cm = get_config_manager() - return bool(cm.get_server(server_name)) and cm.has_server_access( - principal, server_name, ServerPermission.TRADER - ) + return may_use_stored_server(get_config_manager(), principal, server_name) async def _get_client_for_strategy( diff --git a/condor/web/routes/conversations.py b/condor/web/routes/conversations.py index f39a7f115..5d728458b 100644 --- a/condor/web/routes/conversations.py +++ b/condor/web/routes/conversations.py @@ -30,7 +30,7 @@ from condor.web.auth import get_current_user from condor.web.models import WebUser from condor.web.routes.agents import DeploymentRow -from config_manager import get_config_manager +from config_manager import get_config_manager, may_use_stored_server log = logging.getLogger(__name__) @@ -391,9 +391,7 @@ async def _client_for(meta: ConversationMeta, subject_id: int): if not meta.server_name: return None cm = get_config_manager() - if not cm.get_server(meta.server_name) or not cm.has_server_access( - subject_id, meta.server_name - ): + if not may_use_stored_server(cm, subject_id, meta.server_name): log.warning( "deployments: %s cannot reach server %s; listing without money", subject_id, diff --git a/config_manager.py b/config_manager.py index 2bd6ce6be..2f7b4751b 100644 --- a/config_manager.py +++ b/config_manager.py @@ -1396,6 +1396,42 @@ def get_config_manager() -> ConfigManager: return ConfigManager.instance() +def may_use_stored_server(cm, user_id: int, server_name: Optional[str]) -> bool: + """Whether ``user_id`` may turn a **stored** ``server_name`` into credentials. + + Existence *and* reach, in that order of importance: a stored name is not a + capability. It was written earlier — by whoever created the strategy, + conversation, agent or session, often from an unvalidated request body — it + is read now by somebody else, and a share can be withdrawn long after + either happened. + + The existence half is not redundant with the access half: + ``has_server_access`` answers True for an admin on an arbitrary string, so + without it a name that resolves to nothing today would be honoured the + moment a server is created under it. That is the SEC-164 shape, and it is + why SEC-178, SEC-333 and SEC-334 each landed another hand-written copy of + these four lines — five in all, until ARCH-587 gave them one home. + + The level is the TRADER floor ``check_server_access`` applies to every + server-scoped web call; a caller needing OWNER layers ``require_owner`` on + top, as it already does over that floor. + + Lives here rather than in ``condor/web/auth.py`` beside + ``check_server_access`` because three of the five callers — the tick + engine, the session resolver and the toolset builder — sit below the web + layer and cannot import from it without inverting the layering. ``cm`` is + passed in rather than resolved, matching ``require_owner``: every caller + already holds the manager it means. + + ``check_server_access`` remains the reach-only line for a name the *caller* + just supplied and a route is about to act on. This is the line for a name + the caller merely inherited. + """ + if not server_name or not cm.get_server(server_name): + return False + return cm.has_server_access(user_id, server_name) + + def get_effective_server(chat_id: int, user_data: dict = None) -> str | None: """Get the effective default server for a chat, checking both user_data and config.yml. From e2079c7b6f5b95a0acd76a93d7ad22fba2262cbc Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 11:24:58 +0300 Subject: [PATCH 058/154] Move the run ledger out of the route module that renders it build_deployments joins OwnedBot rows, tick_for and a performance object into ledger rows. It touches no request, no response and no HTTP -- its own docstring says "Pure" -- yet it lived in the /agents router, so conversations.py imported another route module's domain logic and two test modules dragged a 3,700-line FastAPI router in to unit-test it. It moves to condor/agents/attribution.py, which already owns this exact genre (session_ownership, current_owner_bases, owner_windows), together with its two private helpers and the DeploymentRow shape. The row type stays in the agents layer rather than condor/web/models.py so the dependency keeps pointing web -> agents; both routes import it from there. --- condor/agents/attribution.py | 176 +++++++++++++++++++++++++++++ condor/web/routes/agents.py | 167 +-------------------------- condor/web/routes/conversations.py | 3 +- tests/test_deeds.py | 2 +- tests/test_session_deployments.py | 2 +- 5 files changed, 181 insertions(+), 169 deletions(-) diff --git a/condor/agents/attribution.py b/condor/agents/attribution.py index 39925ca7c..d7ee2c0bc 100644 --- a/condor/agents/attribution.py +++ b/condor/agents/attribution.py @@ -33,6 +33,8 @@ from pathlib import Path from typing import Any +from pydantic import BaseModel + from condor.agents.ownership import OwnedBot, read_owned from condor.agents.sessions_index import find_session_dir @@ -308,3 +310,177 @@ async def apply_bot_mode_pnl( operator.open_count += sum(1 for r in b_rows if r["status"] == "RUNNING") operator.executors = list(operator.executors) + b_rows operator.total_pnl = operator.realized_pnl + operator.unrealized_pnl + + +# ── The run ledger ── +# What a run put into the world, as rows. The same genre as everything above — +# a pure join over ``OwnedBot`` and a performance object — and the reason it +# lives here rather than in the route module that renders it: two routes serve +# it (a session's executors and a conversation's deployments) and its tests +# unit-test a pure function, none of which should have to drag a FastAPI +# router in ([[ARCH-588]]). + + +class DeploymentRow(BaseModel): + """One thing a run put into the world (FEAT-100). + + A bot it deployed, a controller one of those bots ran, or a standalone + executor it created — the three kinds together are the answer to *what did + this run actually do out there*, which until now could only be assembled by + leaving the agent for the fleet browser and reading a strategy's whole + lifetime instead of the one run. + """ + + #: ``bot`` | ``controller`` | ``executor``. + kind: str + #: The base name, the controller id, or ``"grid SOL-USDC"``. + label: str + #: Origin for a bot, connector·pair for a controller, connector otherwise. + detail: str = "" + #: The tick whose creating call most likely produced this — ``None`` when the + #: join found nothing, which is every run predating the actions log. Never + #: guessed; see :func:`condor.agents.actions.tick_for`. + created_tick: int | None = None + started_at: float = 0.0 + #: When this run stopped owning it, or ``None`` while it still does. + ended_at: float | None = None + #: Whether this run still holds it. Read off ownership, never off ``status`` + #: — an archived instance's performance snapshot still says "running". + live: bool = False + pnl: float = 0.0 + volume: float = 0.0 + #: The fleet address this row links to (``bot:``/``ctrl:``/``exec:``). + scope: str = "" + + +def _instance_for_base(base: str, live: list[str], instances: list[str]) -> str: + """The deploy a bot row should link to: the live one, else the newest.""" + from condor.agents.ownership import strip_deploy_suffix + + for name in live: + if strip_deploy_suffix(name) == base: + return name + mine = [n for n in instances if strip_deploy_suffix(n) == base] + return mine[-1] if mine else base + + +def build_deployments( + owned: list[Any], + bot_bases: list[str], + perf: Any, + actions: list[Any], + agent_id: str, +) -> list[DeploymentRow]: + """Everything one run put into the world, from values it already has (FEAT-100). + + Pure — every input is already on ``get_session_executors``'s stack, which is + the whole reason the ledger is a field on that response rather than a second + endpoint that would have to redo ``session_ownership`` *and* re-fetch the + session's performance to fill the same PnL column. + + Three joins, none of them clever: + + - a **bot** is an :class:`~condor.agents.ownership.OwnedBot`, and it is live + iff this session is still the base's current owner (``bot_bases``) — not + iff its snapshot says "running", which an archived instance also does; + - a **controller** belongs to the bot whose deploy it ran under, so it + inherits that bot's window and its tick; + - an **executor** is this run's own iff it is tagged with the session's + ``agent_id``, the same join the fleet browser performs. + + The tick column is the one heuristic, and it is allowed to say nothing: the + actions log records arguments and never results, so there is no id to join + on and a record is credited to the nearest preceding create of its kind. Runs + written before that log exists get ``None`` everywhere, and the ledger still + renders — the bots and the executors are all there. + """ + from condor.agents.actions import tick_for + from condor.agents.ownership import strip_deploy_suffix + + live_instances = list(getattr(perf, "bot_names", None) or []) + all_instances = list(getattr(perf, "bot_instances", None) or []) + controllers = list(getattr(perf, "controllers", None) or []) + executors = list(getattr(perf, "executors", None) or []) + owned_by_base = {b.base: b for b in owned} + + rows: list[DeploymentRow] = [] + for bot in sorted(owned, key=lambda b: (b.since, b.base)): + mine = [ + c + for c in controllers + if strip_deploy_suffix(str(c.get("bot_name") or "")) == bot.base + ] + rows.append( + DeploymentRow( + kind="bot", + label=bot.base, + detail=bot.origin, + created_tick=tick_for(actions, "bot", bot.since), + started_at=bot.since, + ended_at=bot.until or None, + live=bot.base in bot_bases, + pnl=sum(_controller_pnl(c) for c in mine), + volume=sum(float(c.get("volume_traded") or 0.0) for c in mine), + scope=f"bot:{_instance_for_base(bot.base, live_instances, all_instances)}", + ) + ) + + live_set = set(live_instances) + for c in controllers: + instance = str(c.get("bot_name") or "") + base = strip_deploy_suffix(instance) + parent = owned_by_base.get(base) + cid = str(c.get("controller_id") or "") + detail = " · ".join( + p + for p in (str(c.get("connector") or ""), str(c.get("trading_pair") or "")) + if p + ) + rows.append( + DeploymentRow( + kind="controller", + label=cid or str(c.get("controller_name") or "controller"), + detail=detail, + # A controller has no creating call of its own: it came into the + # world with the deploy that carried it. + created_tick=( + tick_for(actions, "bot", parent.since) if parent else None + ), + started_at=parent.since if parent else 0.0, + ended_at=(parent.until or None) if parent else None, + live=instance in live_set, + pnl=_controller_pnl(c), + volume=float(c.get("volume_traded") or 0.0), + scope=f"ctrl:{instance}:{cid}" if instance and cid else "", + ) + ) + + for ex in executors: + if str(ex.get("controller_id") or "") != agent_id: + continue + started = float(ex.get("timestamp") or 0.0) + closed = float(ex.get("close_timestamp") or 0.0) + kind_name = str(ex.get("type") or "").replace("_executor", "") + pair = str(ex.get("pair") or "") + rows.append( + DeploymentRow( + kind="executor", + label=" ".join(p for p in (kind_name, pair) if p) or str(ex.get("id")), + detail=str(ex.get("connector") or ""), + created_tick=tick_for(actions, "executor", started), + started_at=started, + ended_at=closed or None, + live=closed <= 0, + pnl=float(ex.get("pnl") or 0.0), + volume=float(ex.get("volume") or 0.0), + scope=f"exec:{ex.get('id')}" if ex.get("id") else "", + ) + ) + return rows + + +def _controller_pnl(c: dict[str, Any]) -> float: + """What a controller has made, on the same basis as the KPI strip's total.""" + return float(c.get("realized_pnl_quote") or 0.0) + float( + c.get("unrealized_pnl_quote") or 0.0 + ) diff --git a/condor/web/routes/agents.py b/condor/web/routes/agents.py index 498ea06ab..10ea4e6cc 100644 --- a/condor/web/routes/agents.py +++ b/condor/web/routes/agents.py @@ -30,7 +30,9 @@ from pydantic import BaseModel, Field from condor.agents.attribution import ( + DeploymentRow, apply_bot_mode_pnl, + build_deployments, current_owner_bases, session_ownership, ) @@ -243,38 +245,6 @@ class AgentPerformanceModel(BaseModel): fees_known: bool = True -class DeploymentRow(BaseModel): - """One thing a run put into the world (FEAT-100). - - A bot it deployed, a controller one of those bots ran, or a standalone - executor it created — the three kinds together are the answer to *what did - this run actually do out there*, which until now could only be assembled by - leaving the agent for the fleet browser and reading a strategy's whole - lifetime instead of the one run. - """ - - #: ``bot`` | ``controller`` | ``executor``. - kind: str - #: The base name, the controller id, or ``"grid SOL-USDC"``. - label: str - #: Origin for a bot, connector·pair for a controller, connector otherwise. - detail: str = "" - #: The tick whose creating call most likely produced this — ``None`` when the - #: join found nothing, which is every run predating the actions log. Never - #: guessed; see :func:`condor.agents.actions.tick_for`. - created_tick: int | None = None - started_at: float = 0.0 - #: When this run stopped owning it, or ``None`` while it still does. - ended_at: float | None = None - #: Whether this run still holds it. Read off ownership, never off ``status`` - #: — an archived instance's performance snapshot still says "running". - live: bool = False - pnl: float = 0.0 - volume: float = 0.0 - #: The fleet address this row links to (``bot:``/``ctrl:``/``exec:``). - scope: str = "" - - class StrategyPerformanceResponse(BaseModel): slug: str sessions: list[AgentPerformanceModel] = [] @@ -2680,139 +2650,6 @@ async def get_strategy_performance( return StrategyPerformanceResponse(slug=sslug, sessions=sessions, totals=totals) -def _instance_for_base(base: str, live: list[str], instances: list[str]) -> str: - """The deploy a bot row should link to: the live one, else the newest.""" - from condor.agents.ownership import strip_deploy_suffix - - for name in live: - if strip_deploy_suffix(name) == base: - return name - mine = [n for n in instances if strip_deploy_suffix(n) == base] - return mine[-1] if mine else base - - -def build_deployments( - owned: list[Any], - bot_bases: list[str], - perf: Any, - actions: list[Any], - agent_id: str, -) -> list[DeploymentRow]: - """Everything one run put into the world, from values it already has (FEAT-100). - - Pure — every input is already on ``get_session_executors``'s stack, which is - the whole reason the ledger is a field on that response rather than a second - endpoint that would have to redo ``session_ownership`` *and* re-fetch the - session's performance to fill the same PnL column. - - Three joins, none of them clever: - - - a **bot** is an :class:`~condor.agents.ownership.OwnedBot`, and it is live - iff this session is still the base's current owner (``bot_bases``) — not - iff its snapshot says "running", which an archived instance also does; - - a **controller** belongs to the bot whose deploy it ran under, so it - inherits that bot's window and its tick; - - an **executor** is this run's own iff it is tagged with the session's - ``agent_id``, the same join the fleet browser performs. - - The tick column is the one heuristic, and it is allowed to say nothing: the - actions log records arguments and never results, so there is no id to join - on and a record is credited to the nearest preceding create of its kind. Runs - written before that log exists get ``None`` everywhere, and the ledger still - renders — the bots and the executors are all there. - """ - from condor.agents.actions import tick_for - from condor.agents.ownership import strip_deploy_suffix - - live_instances = list(getattr(perf, "bot_names", None) or []) - all_instances = list(getattr(perf, "bot_instances", None) or []) - controllers = list(getattr(perf, "controllers", None) or []) - executors = list(getattr(perf, "executors", None) or []) - owned_by_base = {b.base: b for b in owned} - - rows: list[DeploymentRow] = [] - for bot in sorted(owned, key=lambda b: (b.since, b.base)): - mine = [ - c - for c in controllers - if strip_deploy_suffix(str(c.get("bot_name") or "")) == bot.base - ] - rows.append( - DeploymentRow( - kind="bot", - label=bot.base, - detail=bot.origin, - created_tick=tick_for(actions, "bot", bot.since), - started_at=bot.since, - ended_at=bot.until or None, - live=bot.base in bot_bases, - pnl=sum(_controller_pnl(c) for c in mine), - volume=sum(float(c.get("volume_traded") or 0.0) for c in mine), - scope=f"bot:{_instance_for_base(bot.base, live_instances, all_instances)}", - ) - ) - - live_set = set(live_instances) - for c in controllers: - instance = str(c.get("bot_name") or "") - base = strip_deploy_suffix(instance) - parent = owned_by_base.get(base) - cid = str(c.get("controller_id") or "") - detail = " · ".join( - p - for p in (str(c.get("connector") or ""), str(c.get("trading_pair") or "")) - if p - ) - rows.append( - DeploymentRow( - kind="controller", - label=cid or str(c.get("controller_name") or "controller"), - detail=detail, - # A controller has no creating call of its own: it came into the - # world with the deploy that carried it. - created_tick=( - tick_for(actions, "bot", parent.since) if parent else None - ), - started_at=parent.since if parent else 0.0, - ended_at=(parent.until or None) if parent else None, - live=instance in live_set, - pnl=_controller_pnl(c), - volume=float(c.get("volume_traded") or 0.0), - scope=f"ctrl:{instance}:{cid}" if instance and cid else "", - ) - ) - - for ex in executors: - if str(ex.get("controller_id") or "") != agent_id: - continue - started = float(ex.get("timestamp") or 0.0) - closed = float(ex.get("close_timestamp") or 0.0) - kind_name = str(ex.get("type") or "").replace("_executor", "") - pair = str(ex.get("pair") or "") - rows.append( - DeploymentRow( - kind="executor", - label=" ".join(p for p in (kind_name, pair) if p) or str(ex.get("id")), - detail=str(ex.get("connector") or ""), - created_tick=tick_for(actions, "executor", started), - started_at=started, - ended_at=closed or None, - live=closed <= 0, - pnl=float(ex.get("pnl") or 0.0), - volume=float(ex.get("volume") or 0.0), - scope=f"exec:{ex.get('id')}" if ex.get("id") else "", - ) - ) - return rows - - -def _controller_pnl(c: dict[str, Any]) -> float: - """What a controller has made, on the same basis as the KPI strip's total.""" - return float(c.get("realized_pnl_quote") or 0.0) + float( - c.get("unrealized_pnl_quote") or 0.0 - ) - - @router.get("/{slug}/strategies/{sslug}/sessions/{session_num}/executors") async def get_session_executors( slug: str, diff --git a/condor/web/routes/conversations.py b/condor/web/routes/conversations.py index 5d728458b..617d3fdb2 100644 --- a/condor/web/routes/conversations.py +++ b/condor/web/routes/conversations.py @@ -23,13 +23,13 @@ from pydantic import BaseModel from condor import paths +from condor.agents.attribution import DeploymentRow, build_deployments from condor.runtime import attachments from condor.runtime import client as runtime from condor.runtime import conversations from condor.runtime.conversations import ConversationIdError, ConversationMeta from condor.web.auth import get_current_user from condor.web.models import WebUser -from condor.web.routes.agents import DeploymentRow from config_manager import get_config_manager, may_use_stored_server log = logging.getLogger(__name__) @@ -316,7 +316,6 @@ async def get_conversation_deployments( from condor.agents.deeds import attribution_tag, for_conversation from condor.agents.ownership import read_owned from condor.agents.performance import AgentPerformance, fetch_agent_performance - from condor.web.routes.agents import build_deployments owner_id = _owner(user, user_id) meta = _meta_or_404(owner_id, conversation_id) diff --git a/tests/test_deeds.py b/tests/test_deeds.py index f0a37e16e..f72f43669 100644 --- a/tests/test_deeds.py +++ b/tests/test_deeds.py @@ -366,7 +366,7 @@ def test_the_existing_readers_work_on_a_conversation_with_no_changes(): """ from types import SimpleNamespace - from condor.web.routes.agents import build_deployments + from condor.agents.attribution import build_deployments owner = deeds.for_conversation(USER, "conv1") deeds.record_deeds(owner, _deploy_calls("pmm-king-btcbrl")) diff --git a/tests/test_session_deployments.py b/tests/test_session_deployments.py index 1ea7fe6c7..986aae23f 100644 --- a/tests/test_session_deployments.py +++ b/tests/test_session_deployments.py @@ -8,8 +8,8 @@ """ from condor.agents.actions import AgentAction +from condor.agents.attribution import build_deployments from condor.agents.ownership import OwnedBot -from condor.web.routes.agents import build_deployments AGENT_ID = "brigado.brl_mm_3" From cc0331f8a621cf0b52e4d90883b613de97053f41 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 11:32:27 +0300 Subject: [PATCH 059/154] Let the peer own every request, session/prompt included ACPClient reached into JSONRPCPeer's id counter and pending table from ten sites, and prompt_stream hand-rolled the frame the peer already knows how to write -- so session/prompt, the hottest request, was the one that never passed through the peer's write path or its debug log. begin_request is the seam it actually needed: allocate the id, register the future, write and drain, and hand back both. send_request becomes a thin await on top of it, and pending()/discard() replace the direct dict access in _cancel_locally, _settle_previous_turn, abort_prompt and prompt_stream's finally. Registering the future before the write also closes a real race: drain is a yield point, so a child that answered fast used to have its response hit an empty table and be dropped. --- condor/acp/client.py | 39 +++---- condor/acp/jsonrpc.py | 71 +++++++++--- tests/runtime/test_acp_peer_encapsulation.py | 111 +++++++++++++++++++ 3 files changed, 182 insertions(+), 39 deletions(-) create mode 100644 tests/runtime/test_acp_peer_encapsulation.py diff --git a/condor/acp/client.py b/condor/acp/client.py index 29def7b71..26c40754a 100644 --- a/condor/acp/client.py +++ b/condor/acp/client.py @@ -9,7 +9,6 @@ import asyncio import base64 import hashlib -import json import logging import os import signal @@ -817,11 +816,11 @@ def _cancel_locally(self, req_id: int) -> None: live = req_id == self._current_req_id if live: self._current_req_id = None - future = self._peer._pending.get(req_id) + future = self._peer.pending(req_id) if future is not None and not future.done(): self._unsettled_req = req_id else: - self._peer._pending.pop(req_id, None) + self._peer.discard(req_id) if not live: return self._drain_events() @@ -868,11 +867,11 @@ async def _settle_previous_turn(self) -> None: return self._current_req_id = None - future = self._peer._pending.get(req_id) + future = self._peer.pending(req_id) if future is None or future.done() or not self.alive: # Nothing still generating: a dead subprocess emits nothing, and # the read loop cancels every pending future on its way out. - self._peer._pending.pop(req_id, None) + self._peer.discard(req_id) self._unsettled_req = None return @@ -902,7 +901,7 @@ async def _settle_previous_turn(self) -> None: pass self._unsettled_req = None - self._peer._pending.pop(req_id, None) + self._peer.discard(req_id) async def abort_prompt(self) -> None: """Cancel the in-flight prompt at the agent, not just locally. @@ -922,7 +921,7 @@ async def abort_prompt(self) -> None: if req_id is None: return - future = self._peer._pending.get(req_id) + future = self._peer.pending(req_id) if future is None or future.done() or not self.alive: self._cancel_locally(req_id) return @@ -992,14 +991,12 @@ async def prompt_stream( # produced by the turn below. self._drain_events() - # Send request without awaiting so read loop can dispatch notifications - req_id = self._peer._next_id - self._peer._next_id += 1 - self._current_req_id = req_id - msg = { - "jsonrpc": "2.0", - "method": "session/prompt", - "params": { + # Sent through the peer like every other request, but with the future + # handed back instead of awaited: this turn's answer only arrives after + # a stream of notifications, which the loop below is what reads. + req_id, future = await self._peer.begin_request( + "session/prompt", + { "sessionId": self._session_id, "prompt": [ *( @@ -1013,13 +1010,9 @@ async def prompt_stream( {"type": "text", "text": text}, ], }, - "id": req_id, - } - self._process.stdin.write((json.dumps(msg) + "\n").encode()) - await self._process.stdin.drain() - - future: asyncio.Future[Any] = asyncio.get_event_loop().create_future() - self._peer._pending[req_id] = future + self._process.stdin, + ) + self._current_req_id = req_id def _on_response(fut: asyncio.Future) -> None: # Only enqueue PromptDone if this is still the current prompt @@ -1101,7 +1094,7 @@ async def _hard_stop(elapsed: float) -> None: # still generating and nothing ever telling it to stop. if self._current_req_id == req_id: self._current_req_id = None - unfinished = self._peer._pending.get(req_id) + unfinished = self._peer.pending(req_id) if unfinished is not None and not unfinished.done(): self._unsettled_req = req_id # Not awaited: under GeneratorExit there may be no one left diff --git a/condor/acp/jsonrpc.py b/condor/acp/jsonrpc.py index 416c2e477..62356dc6a 100644 --- a/condor/acp/jsonrpc.py +++ b/condor/acp/jsonrpc.py @@ -52,20 +52,35 @@ def __init__(self): def register_handler(self, method: str, handler: Callable) -> None: self._handlers[method] = handler - async def send_request( + def pending(self, req_id: int) -> asyncio.Future | None: + """The still-unsettled future for ``req_id``, or None. + + The peer owns the pending table; callers that need to know whether a + request is still in flight ask here rather than reading the dict + (ARCH-332). + """ + return self._pending.get(req_id) + + def discard(self, req_id: int) -> None: + """Forget ``req_id``: nobody is waiting on its answer any more.""" + self._pending.pop(req_id, None) + + async def begin_request( self, method: str, params: dict[str, Any], writer: asyncio.StreamWriter, - timeout: float | None = None, - ) -> Any: - """Send a JSON-RPC request and wait for the response. + ) -> tuple[int, asyncio.Future]: + """Send a request and hand back its id and its unsettled future. - ``timeout`` bounds that wait: nothing else in the peer does, so a child - that reads our line and never answers parks the caller forever - (CORR-333). On expiry the pending entry is dropped -- an abandoned - request must not leak a future that only ``cancel_all`` would ever - clear -- and :class:`asyncio.TimeoutError` propagates to the caller. + The seam for callers that cannot simply await the answer inline -- + ``session/prompt`` streams notifications for the whole turn and only + settles at the end -- so that every request still goes out through the + peer's own framing, id allocation and logging (ARCH-332). + + The future is registered *before* the write, not after: ``drain`` is a + yield point, so a child that answers fast used to have its response hit + an empty pending table and be dropped on the floor. """ if self._failure is not None: raise self._failure @@ -73,25 +88,49 @@ async def send_request( req_id = self._next_id self._next_id += 1 + future: asyncio.Future[Any] = asyncio.get_event_loop().create_future() + self._pending[req_id] = future + msg = {"jsonrpc": "2.0", "method": method, "params": params, "id": req_id} line = json.dumps(msg) + "\n" - writer.write(line.encode()) - await writer.drain() + try: + writer.write(line.encode()) + await writer.drain() + except BaseException: + self.discard(req_id) + raise log.debug("-> %s (id=%d)", method, req_id) - future: asyncio.Future[Any] = asyncio.get_event_loop().create_future() # Checked again after the drain above: the peer can die while we are - # writing, and a future registered after that sweep would wait out the - # whole timeout for an answer that can never come. + # writing, and a caller left holding a future the death sweep already + # settled should see the real error now rather than at its own timeout. if self._failure is not None: + self.discard(req_id) raise self._failure - self._pending[req_id] = future + return req_id, future + + async def send_request( + self, + method: str, + params: dict[str, Any], + writer: asyncio.StreamWriter, + timeout: float | None = None, + ) -> Any: + """Send a JSON-RPC request and wait for the response. + + ``timeout`` bounds that wait: nothing else in the peer does, so a child + that reads our line and never answers parks the caller forever + (CORR-333). On expiry the pending entry is dropped -- an abandoned + request must not leak a future that only ``cancel_all`` would ever + clear -- and :class:`asyncio.TimeoutError` propagates to the caller. + """ + req_id, future = await self.begin_request(method, params, writer) if timeout is None: return await future try: return await asyncio.wait_for(future, timeout) except (asyncio.TimeoutError, asyncio.CancelledError): - self._pending.pop(req_id, None) + self.discard(req_id) raise async def send_notification( diff --git a/tests/runtime/test_acp_peer_encapsulation.py b/tests/runtime/test_acp_peer_encapsulation.py new file mode 100644 index 000000000..2d5239648 --- /dev/null +++ b/tests/runtime/test_acp_peer_encapsulation.py @@ -0,0 +1,111 @@ +"""The peer owns id allocation, framing and the pending table (ARCH-332). + +``session/prompt`` used to be the one request the client framed and sent by +hand, because it needs the future rather than the answer. ``begin_request`` is +the seam that gives it the future *and* keeps it on the peer's write path, so +the hottest request is logged and id-allocated like every other one — and, +because the future is now registered before the write rather than after the +drain, a child that answers during that drain is no longer dropped on the floor. +""" + +import asyncio +import contextlib +import json +import logging + +import pytest + +from condor.acp.client import ACPClient +from condor.acp.jsonrpc import JSONRPCPeer + + +class _EagerStdin: + """A child that answers inside our own ``drain()``. + + Not a contrivance: ``drain`` yields to the event loop, and that is exactly + where the read loop gets to dispatch a response to a request whose write + already went out. + """ + + def __init__(self, peer: JSONRPCPeer, result): + self.peer = peer + self.result = result + self.sent: list[dict] = [] + + def write(self, data: bytes) -> None: + self.sent.append(json.loads(data.decode())) + + async def drain(self) -> None: + msg = self.sent[-1] + if "id" in msg: + await self.peer.handle_line( + json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": self.result}), + self, + ) + + +class _SilentStdin: + def __init__(self): + self.sent: list[dict] = [] + + def write(self, data: bytes) -> None: + self.sent.append(json.loads(data.decode())) + + async def drain(self) -> None: + pass + + +def test_a_response_that_lands_during_the_drain_is_not_dropped(): + """The future is registered before the write, so no answer arrives too early.""" + peer = JSONRPCPeer() + stdin = _EagerStdin(peer, {"ok": True}) + + async def scenario(): + return await asyncio.wait_for( + peer.send_request("initialize", {}, stdin, timeout=1), timeout=2 + ) + + assert asyncio.run(scenario()) == {"ok": True} + assert peer._pending == {} + + +def test_a_failed_write_does_not_leak_a_pending_future(): + """Registering early must not turn a broken pipe into a stuck entry.""" + peer = JSONRPCPeer() + + class _BrokenStdin(_SilentStdin): + async def drain(self) -> None: + raise BrokenPipeError("gone") + + async def scenario(): + with pytest.raises(BrokenPipeError): + await peer.send_request("initialize", {}, _BrokenStdin(), timeout=1) + + asyncio.run(scenario()) + assert peer._pending == {} + + +def test_session_prompt_is_framed_and_logged_by_the_peer(caplog): + """The hottest request goes out through the same write path as the rest.""" + client = ACPClient(command="fake-agent") + stdin = _SilentStdin() + client._process = type("_P", (), {"stdin": stdin, "returncode": None})() + client._session_id = "sess-1" + + async def scenario(): + agen = client.prompt_stream("hello") + started = asyncio.ensure_future(agen.__anext__()) + await asyncio.sleep(0.05) + started.cancel() + with contextlib.suppress(asyncio.CancelledError): + await started + await agen.aclose() + + with caplog.at_level(logging.DEBUG, logger="condor.acp.jsonrpc"): + asyncio.run(scenario()) + + prompt = next(m for m in stdin.sent if m.get("method") == "session/prompt") + assert prompt["jsonrpc"] == "2.0" + assert prompt["id"] == 1 # allocated by the peer, from its own counter + assert prompt["params"]["prompt"] == [{"type": "text", "text": "hello"}] + assert "-> session/prompt (id=1)" in caplog.text From 267e39cd9bbe164d0ba29cc223cf9ce6e72d872f Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 11:37:25 +0300 Subject: [PATCH 060/154] Say when the next tick is due in one place, not five MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rule for "seconds until the next tick" was exported and tested as dueInSec, and then written out again inline by the loop banner, the agent card, the loop bar and the pulse. The copies had already drifted: none of them carried the helper's frequency_sec <= 0 guard, so a loop with no cadence computed last_tick_at - now, a large negative, and those four surfaces printed a permanent "overdue 4h" for a loop that was not overdue at all — while the dock, the one call site on the helper, correctly said nothing. All four now call dueInSec, keeping only what is genuinely theirs at the call site: the banner's paused short-circuit, the card's ms clock, the pulse's elapsed-based progress bar. The words go with it — tickCountdownLabel sits beside dueInSec so "next in 40s" / "overdue 40s" cannot be worded three ways for the same state. The card's countdown stays bare on purpose: its tile is already labelled "Next tick", and "Next tick — next in 40s" says it twice. --- frontend/src/components/agent/EntityCard.tsx | 4 ++-- frontend/src/components/agent/LoopBanner.tsx | 16 ++++++++-------- frontend/src/components/agent/LoopPulse.tsx | 14 ++++++++------ .../src/components/agent/workspace/LoopBar.tsx | 13 ++++++------- .../src/components/agent/workspace/fleet.test.ts | 16 ++++++++++++++++ frontend/src/components/agent/workspace/fleet.ts | 11 ++++++++++- 6 files changed, 50 insertions(+), 24 deletions(-) diff --git a/frontend/src/components/agent/EntityCard.tsx b/frontend/src/components/agent/EntityCard.tsx index 3af044341..584444b0b 100644 --- a/frontend/src/components/agent/EntityCard.tsx +++ b/frontend/src/components/agent/EntityCard.tsx @@ -2,6 +2,7 @@ import { ChevronRight, FlaskConical, Trash2, type LucideIcon } from "lucide-reac import { deriveAgentStatus } from "@/components/agent/agentStatus"; import { StatusBadge } from "@/components/agent/StatusBadge"; +import { dueInSec } from "@/components/agent/workspace/fleet"; import { useSeconds } from "@/hooks/useSeconds"; import { countdown } from "@/lib/agent-attribution"; import type { RunningInstance } from "@/lib/api"; @@ -65,8 +66,7 @@ export function EntityCard({ const ticks = live?.tick_count ?? entity.tick_count ?? 0; const lastTickAt = live?.last_tick_at ?? 0; - const dueIn = - live && lastTickAt > 0 ? lastTickAt + live.frequency_sec - now / 1000 : null; + const dueIn = dueInSec(live, now / 1000); return ( diff --git a/frontend/src/components/agent/LoopPulse.tsx b/frontend/src/components/agent/LoopPulse.tsx index cba1b32ab..a42e546c0 100644 --- a/frontend/src/components/agent/LoopPulse.tsx +++ b/frontend/src/components/agent/LoopPulse.tsx @@ -1,6 +1,10 @@ import { AlertTriangle, Power, Repeat, Zap } from "lucide-react"; import { ModeBadge } from "@/components/agent/ModeBadge"; +import { + dueInSec, + tickCountdownLabel, +} from "@/components/agent/workspace/fleet"; import { useSeconds } from "@/hooks/useSeconds"; import { countdown } from "@/lib/agent-attribution"; import type { RunningInstance } from "@/lib/api"; @@ -91,7 +95,9 @@ export function LoopPulse({ // after a first tick: `0 + frequency` is 1970, and a bar filled from 1970 is // a bar pinned at 100% that means nothing. const elapsed = running && lastTickAt > 0 ? now / 1000 - lastTickAt : null; - const dueIn = elapsed === null ? null : frequency - elapsed; + // The countdown itself is the shared rule, which is also what keeps a loop + // with no cadence from reading as permanently overdue here. + const dueIn = dueInSec(running ? instance : null, now / 1000); const progress = elapsed === null ? 0 : Math.max(0, Math.min(1, elapsed / Math.max(1, frequency))); const overdue = dueIn !== null && dueIn <= 0; @@ -196,11 +202,7 @@ export function LoopPulse({ /> - {dueIn === null - ? "—" - : overdue - ? `overdue ${countdown(-dueIn)}` - : `next in ${countdown(dueIn)}`} + {dueIn === null ? "—" : tickCountdownLabel(dueIn)} )} diff --git a/frontend/src/components/agent/workspace/LoopBar.tsx b/frontend/src/components/agent/workspace/LoopBar.tsx index 465f97121..a43245f91 100644 --- a/frontend/src/components/agent/workspace/LoopBar.tsx +++ b/frontend/src/components/agent/workspace/LoopBar.tsx @@ -7,6 +7,10 @@ import { runDurationSec, runLabel, } from "@/components/agent/lab/runs"; +import { + dueInSec, + tickCountdownLabel, +} from "@/components/agent/workspace/fleet"; import { useSeconds } from "@/hooks/useSeconds"; import { countdown } from "@/lib/agent-attribution"; import type { @@ -70,10 +74,7 @@ export function LoopBar({ const nowSec = now / 1000; const duration = run ? formatDuration(runDurationSec(run, nowSec)) : ""; - const dueIn = - instance && instance.last_tick_at > 0 - ? instance.last_tick_at + instance.frequency_sec - nowSec - : null; + const dueIn = dueInSec(instance, nowSec); return ( <> @@ -141,9 +142,7 @@ export function LoopBar({ dueIn > 0 ? "text-[var(--color-text-muted)]" : "text-amber-400" }`} > - {dueIn > 0 - ? `next in ${countdown(dueIn)}` - : `overdue ${countdown(-dueIn)}`} + {tickCountdownLabel(dueIn)} )} diff --git a/frontend/src/components/agent/workspace/fleet.test.ts b/frontend/src/components/agent/workspace/fleet.test.ts index d1f086b00..95b90fb7c 100644 --- a/frontend/src/components/agent/workspace/fleet.test.ts +++ b/frontend/src/components/agent/workspace/fleet.test.ts @@ -24,6 +24,7 @@ import { rowHref, scopeStrategy, strategylessAgents, + tickCountdownLabel, } from "./fleet"; function instance(over: Partial = {}): RunningInstance { @@ -187,6 +188,21 @@ describe("the next tick", () => { expect(dueInSec(instance({ last_tick_at: 0 }), 1_000)).toBeNull(); expect(dueInSec(null, 1_000)).toBeNull(); }); + + it("is unknowable for a loop with no cadence, rather than forever overdue", () => { + // Nothing constrains `frequency_sec` to be positive, and `last_tick_at - + // now` on a 0-cadence loop is a large negative that reads as "overdue 4h" + // on every surface that skips this guard. + expect(dueInSec(instance({ frequency_sec: 0 }), 1_000)).toBeNull(); + expect(dueInSec(instance({ frequency_sec: -30 }), 1_000)).toBeNull(); + }); + + it("is worded the same wherever it is printed", () => { + expect(tickCountdownLabel(40)).toBe("next in 40s"); + expect(tickCountdownLabel(-40)).toBe("overdue 40s"); + // Due exactly now has already slipped, so it is overdue, not "next in 0s". + expect(tickCountdownLabel(0)).toBe("overdue 0s"); + }); }); describe("the order of the rows", () => { diff --git a/frontend/src/components/agent/workspace/fleet.ts b/frontend/src/components/agent/workspace/fleet.ts index 6c5ade5c8..523b56153 100644 --- a/frontend/src/components/agent/workspace/fleet.ts +++ b/frontend/src/components/agent/workspace/fleet.ts @@ -22,7 +22,7 @@ import { alertsFor, type WorkspaceAlert, } from "@/components/agent/workspace/views"; -import type { AgentActionRow } from "@/lib/agent-attribution"; +import { countdown, type AgentActionRow } from "@/lib/agent-attribution"; import type { AgentSummary, RunningInstance, @@ -200,6 +200,15 @@ export function dueInSec( return live.last_tick_at + live.frequency_sec - nowSec; } +/** + * The words that go with `dueInSec`, so the four surfaces that print a + * countdown cannot word the same state differently. A tick due exactly now is + * overdue, not "next in 0s" — the beat has already slipped. + */ +export function tickCountdownLabel(due: number): string { + return due > 0 ? `next in ${countdown(due)}` : `overdue ${countdown(-due)}`; +} + /** Running, then paused, then everything idle — the sort's first key. */ function loopRank(live: RunningInstance | null): number { if (!live) return 2; From 9bccd5aee309cf7e88ab0a0aea6b8ed3aa9d4947 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 11:46:06 +0300 Subject: [PATCH 061/154] (fix) tell the user what the ACP child wrote to stderr before it died _drain_stderr existed only to keep the pipe from filling and threw the content away at DEBUG, a level nothing enables in a normal deployment. Every diagnosable launch failure writes its reason there -- "command not found", a node version error, "Claude Code cannot be launched inside another Claude Code session" -- and the exception string is forwarded verbatim to the browser, so the user was handed a failure that named no cause while the cause sat in a pipe this process had already read. Keep a bounded tail (20 lines, 500 chars each) and fold it into both errors that report a dead or mute child: the ConnectionError the read loop hands a racing request (CORR-329) and whatever start() re-raises on a failed handshake. The handshake path rewrites the message in place so the exception type stays load-bearing, and waits a bounded moment for the drain task to reach EOF first -- a dying child's stdout and stderr hit EOF in the same breath and the read loop can get there first. --- condor/acp/client.py | 64 ++++++++++++++++++++-- tests/runtime/test_acp_stderr_in_errors.py | 59 ++++++++++++++++++++ 2 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 tests/runtime/test_acp_stderr_in_errors.py diff --git a/condor/acp/client.py b/condor/acp/client.py index 26c40754a..c7e58024a 100644 --- a/condor/acp/client.py +++ b/condor/acp/client.py @@ -14,6 +14,7 @@ import signal import subprocess import time +from collections import deque from dataclasses import dataclass from typing import Any, AsyncIterator, Awaitable, Callable @@ -28,6 +29,21 @@ #: only after a JSON-quoted scalar has been unwrapped. _ABSENT_TOOL_NAMES = frozenset({"undefined", "null", "none"}) +#: How much of the child's stderr is kept for the error message that reports +#: its death. Stderr is the only place a failed launch says *why* -- "command +#: not found", a node version error, "Claude Code cannot be launched inside +#: another Claude Code session" -- and it is the last lines that carry the +#: cause, so a short bounded tail is all that is worth holding (READ-337). +_STDERR_TAIL_LINES = 20 +#: Each kept line is truncated to this: the read limit on the pipe is 10MB, so +#: a chatty agent must not be able to turn a 20-line tail into 200MB. +_STDERR_LINE_CHARS = 500 +#: How long the death path waits for the drain task to reach EOF before +#: quoting the tail. A dying child's stdout and stderr hit EOF in the same +#: breath and the read loop can get there first, so the lines that explain the +#: death may still be in the pipe when we are asked for them. +_STDERR_SETTLE_TIMEOUT = 0.5 + def normalize_tool_title(value: Any) -> str: """A tool title stripped of encoding artefacts, or ``""`` when it says nothing. @@ -487,6 +503,10 @@ def __init__( self.accepts_images = False self._read_task: asyncio.Task | None = None self._stderr_task: asyncio.Task | None = None + # The child's last words. Kept because stderr is the only text that + # explains a failed launch, and the DEBUG line _drain_stderr writes is + # off in every normal deployment (READ-337). + self._stderr_tail: deque[str] = deque(maxlen=_STDERR_TAIL_LINES) # Set once _read_loop is over: the process may still be up, but # nothing it says will ever reach us again (see :attr:`alive`). self._read_loop_ended = False @@ -555,6 +575,7 @@ async def start(self) -> None: start_new_session=True, # Own process group so we can kill all children ) self._read_loop_ended = False + self._stderr_tail.clear() self._read_task = asyncio.create_task(self._read_loop()) self._stderr_task = asyncio.create_task(self._drain_stderr()) @@ -586,15 +607,24 @@ async def start(self) -> None: timeout=max(0.0, deadline - time.monotonic()), ) except asyncio.TimeoutError: + detail = await self._stderr_detail() await self.stop() raise TimeoutError( f"The agent did not complete the ACP handshake within " f"{TIMEOUTS.agent_handshake}s and was killed (cmd={self.command}). " - f"Check that the command runs and speaks ACP on stdio." + f"Check that the command runs and speaks ACP on stdio.{detail}" ) from None - except Exception: + except Exception as exc: # Handshake failed -- kill the subprocess to prevent orphan + detail = await self._stderr_detail() await self.stop() + if detail and detail not in str(exc): + # Rewritten in place rather than re-raised as a new class: the + # type is load-bearing -- a dead child is a ConnectionError + # (CORR-329) and a bridge that answered with an error is a + # JSONRPCError -- while the message is what actually reaches + # the user, verbatim, in the chat (READ-337). + exc.args = (f"{exc}{detail}", *exc.args[1:]) raise self._session_id = result["sessionId"] @@ -769,11 +799,22 @@ async def _read_loop(self) -> None: # left its subprocess orphaned on exactly the path that guard was # written for, and the caller saw a cancellation instead of a broken # agent (CORR-329). - self._peer.fail_all(ConnectionError(f"ACP agent exited: {self.command}")) + # The stderr tail rides along: this error is what a racing request and + # an in-flight turn are handed, and for a child that could not run at + # all it is the ONLY evidence of why (READ-337). + detail = await self._stderr_detail() + self._peer.fail_all( + ConnectionError(f"ACP agent exited: {self.command}{detail}") + ) self._event_queue.put_nowait(PromptDone(stop_reason="disconnected")) async def _drain_stderr(self) -> None: - """Read and log stderr to prevent pipe buffer from filling up and blocking the subprocess.""" + """Read stderr to keep the pipe from filling up and blocking the subprocess. + + What it reads is also kept, bounded, in :attr:`_stderr_tail`: the DEBUG + line below is off in every normal deployment, and stderr is where every + diagnosable launch failure writes its reason (READ-337). + """ assert self._process and self._process.stderr try: while True: @@ -783,11 +824,26 @@ async def _drain_stderr(self) -> None: text = line.decode(errors="replace").rstrip() if text: log.debug("ACP stderr: %s", text) + self._stderr_tail.append(text[:_STDERR_LINE_CHARS]) except asyncio.CancelledError: return except Exception: log.exception("ACP stderr drain error") + async def _stderr_detail(self) -> str: + """The child's stderr tail, formatted for an exception message. + + Empty when it said nothing -- a healthy session pays for none of this. + """ + task = self._stderr_task + if task is not None and not task.done(): + # Bounded, and never re-raises what the drain task raised: this is + # already an error path and must not be turned into another one. + await asyncio.wait({task}, timeout=_STDERR_SETTLE_TIMEOUT) + if not self._stderr_tail: + return "" + return "\nAgent stderr:\n" + "\n".join(self._stderr_tail) + # --- Prompt --- def _drain_events(self) -> None: diff --git a/tests/runtime/test_acp_stderr_in_errors.py b/tests/runtime/test_acp_stderr_in_errors.py new file mode 100644 index 000000000..afb916360 --- /dev/null +++ b/tests/runtime/test_acp_stderr_in_errors.py @@ -0,0 +1,59 @@ +"""The ACP child's stderr must reach the error that reports its death (READ-337). + +``_drain_stderr`` existed only to keep the pipe from filling and threw the +content away at DEBUG -- a level nothing enables in a normal deployment. Every +diagnosable launch failure writes its reason there ("command not found", a node +version error, "Claude Code cannot be launched inside another Claude Code +session"), and the exception string is forwarded verbatim to the browser, so +the user was handed a failure that named no cause while the cause sat in a pipe +this process had already read. +""" + +import asyncio + +import pytest + +from condor.acp.client import _STDERR_TAIL_LINES, ACPClient + +_MARKER = "condor-read337-marker: command not found" + + +@pytest.mark.asyncio +async def test_a_child_that_dies_talking_to_stderr_says_so_in_the_error(): + client = ACPClient(command=f"echo '{_MARKER}' >&2; exit 127") + + with pytest.raises(ConnectionError) as excinfo: + await asyncio.wait_for(client.start(), timeout=30) + + # The type stays the one CORR-329 established; only the message grew. + assert _MARKER in str(excinfo.value) + assert client.command in str(excinfo.value) + + +@pytest.mark.asyncio +async def test_a_silent_child_adds_nothing_to_the_error(): + """A healthy-but-dead child pays nothing: no empty stderr section.""" + client = ACPClient(command="exit 1") + + with pytest.raises(ConnectionError) as excinfo: + await asyncio.wait_for(client.start(), timeout=30) + + assert "Agent stderr" not in str(excinfo.value) + + +@pytest.mark.asyncio +async def test_the_kept_tail_is_bounded_in_lines_and_in_width(): + """A chatty agent cannot grow the buffer without limit.""" + client = ACPClient(command="true") + stderr = asyncio.StreamReader() + client._process = type("_P", (), {"stderr": stderr})() # type: ignore[assignment] + + for i in range(_STDERR_TAIL_LINES * 5): + stderr.feed_data(f"line {i} ".encode() + b"x" * 5000 + b"\n") + stderr.feed_eof() + await asyncio.wait_for(client._drain_stderr(), timeout=5) + + assert len(client._stderr_tail) == _STDERR_TAIL_LINES + assert all(len(line) <= 500 for line in client._stderr_tail) + # It is the *last* lines that carry the cause of a death. + assert client._stderr_tail[-1].startswith(f"line {_STDERR_TAIL_LINES * 5 - 1} ") From 367d7cf3fa45d939a3376753dc14a9881e9dae03 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 11:54:40 +0300 Subject: [PATCH 062/154] Ask before a model takes the Gateway container up or down MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit manage_gateway_container was the only registered tool controlling a container lifecycle that reached no gate at all: DANGEROUS_TOOLS did not name it, so is_dangerous_tool_call fell through to False and is_mutating_tool_call's terminal name set excluded it too — start, stop and restart were neither confirmed nor written to the action log. The gate weighs what a call does, not whether it signs. stop/restart take Gateway down underneath a live CLMM/LP executor that needs it to manage or close a position, and start passes the caller's config dict — Docker image included — straight through to the API host that holds the exchange API keys and the Gateway wallet keys, from a model whose input includes untrusted tool output. So: DANGEROUS_CONTAINER_ACTIONS = {start, stop, restart}, its mutating twin, and READ_ONLY_CONTAINER_ACTIONS = {get_status, get_logs}. The two reads stay on the fast path — get_logs is the escape hatch every opaque Gateway failure points at — and an unreadable or missing action still fails closed. The confirmation line names the image rather than the word "start", since that is what the human is actually approving. SEC-565 --- condor/runtime/danger.py | 53 +++++++++++++++ handlers/agents/_shared.py | 1 + tests/test_acp_permission_gate.py | 79 ++++++++++++++++++++-- tests/test_dangerous_gate_names_resolve.py | 22 +++++- 4 files changed, 147 insertions(+), 8 deletions(-) diff --git a/condor/runtime/danger.py b/condor/runtime/danger.py index 59db2e22a..fd81e7162 100644 --- a/condor/runtime/danger.py +++ b/condor/runtime/danger.py @@ -49,6 +49,7 @@ "manage_clmm", # every action that moves liquidity "manage_amm", # every action that moves liquidity "manage_gateway_config", # no resource of it is gated today; see below + "manage_gateway_container", # only the lifecycle actions; see below "control_agent", # only `start`, which launches an unattended trading loop # The executor family is gated by NAME (FEAT-062), the same way the swap family # is: a create and a stop each have their own tool, so there is no `action` to @@ -125,6 +126,29 @@ # so the gate has to know both or `start_agent` walks straight past it. DANGEROUS_CONTROL_ACTIONS = {"start", "start_agent"} +# Actions within manage_gateway_container that require confirmation (SEC-565). +# This is the only registered tool that controls a container lifecycle, and the +# gate weighs what a call does rather than whether it signs: `manage_gateway_config` +# is gated while signing nothing, and toolsets.py already calls container control an +# owner-only operator action. +# +# `stop` and `restart` take Gateway down underneath any live CLMM/LP executor that +# needs it to manage or close a position. `start` is the wider one: the impl passes +# the caller's `config` dict straight to the API host (tools/gateway.py), and that +# dict names the Docker `image` — so an ungated `start` lets a model whose input +# includes untrusted tool output (pool names, bot logs, GeckoTerminal rows) choose +# what runs on the host holding the exchange API keys and the Gateway wallet keys. +# +# `get_status` and `get_logs` stay on the fast path. `get_logs` in particular is the +# escape hatch every opaque Gateway failure points at (middleware.GATEWAY_LOG_HINT), +# and putting a prompt in front of reading a log would put one in front of diagnosis. +# +# Unlike control_agent there is no second spelling to cover: the action is a plain +# five-member Literal validated by pydantic (schemas.GatewayContainerRequest) and the +# impl dispatches on it directly, with no alias resolver. An unreadable or missing +# action still fails closed. +DANGEROUS_CONTAINER_ACTIONS = {"start", "stop", "restart"} + # Resource types within manage_gateway_config that require confirmation. This tool # is gated on `resource_type`, not `action`, because what it edits matters and how # it edits does not. The set is empty and the gate stays: `wallets` used to be in it @@ -174,6 +198,10 @@ "set_state", } +#: `manage_gateway_container`'s writes. Identical to its confirmation set: the +#: three lifecycle actions are everything it can change, and the other two read. +MUTATING_CONTAINER_ACTIONS = DANGEROUS_CONTAINER_ACTIONS + # The reads of each dispatch tool, named explicitly. They are what keeps the # fail-open rule below from recording a `manage_bots(action="status")`: an # action in neither set is one this module has not heard of, and *that* is what @@ -188,6 +216,9 @@ "quote_liquidity", } READ_ONLY_CONTROL_ACTIONS = {"list", "list_agents", "get_state"} +#: `manage_gateway_container`'s reads, named so the fail-open rule below does not +#: record a status poll or a log tail as a change to the world. +READ_ONLY_CONTAINER_ACTIONS = {"get_status", "get_logs"} #: `manage_controllers`' writes and reads. The tool is outside the *gate* #: entirely and stays there (it writes controller templates and saved configs, #: never a running bot), but a fleet is *built* out of these calls: the twelve @@ -361,6 +392,9 @@ def is_dangerous_tool_call(tool_call: dict[str, Any]) -> bool: if tool_name == "manage_gateway_config": return _has_dangerous_resource(tool_call, DANGEROUS_CONFIG_RESOURCES) + if tool_name == "manage_gateway_container": + return _has_dangerous_action(tool_call, DANGEROUS_CONTAINER_ACTIONS) + if tool_name == "control_agent": return _has_dangerous_action(tool_call, DANGEROUS_CONTROL_ACTIONS) @@ -430,6 +464,11 @@ def is_mutating_tool_call(tool_call: dict[str, Any]) -> bool: tool_call, MUTATING_CONTROL_ACTIONS, READ_ONLY_CONTROL_ACTIONS ) + if tool_name == "manage_gateway_container": + return _is_mutating_action( + tool_call, MUTATING_CONTAINER_ACTIONS, READ_ONLY_CONTAINER_ACTIONS + ) + if tool_name == "manage_gateway_config": # Recorded unless it is one of the two reads. The resource type is read # only to stay a superset of the gate, which fails closed on a missing @@ -586,6 +625,20 @@ def format_tool_summary(tool_call: dict[str, Any]) -> str: action = input_data.get("action", "?") return f"Gateway config: {action} {resource}" + if tool_name == "manage_gateway_container": + # `start` and `restart` hand a caller-chosen Docker image to the API host, + # so the line names the image: the human is approving what will run on the + # box that holds the exchange and wallet keys, not the word "start". + action = input_data.get("action", "?") + config = input_data.get("config") + image = config.get("image") if isinstance(config, dict) else None + if action in ("start", "restart"): + what = "Start" if action == "start" else "Restart" + return f"{what} the Gateway container with image {image or 'default'}" + if action == "stop": + return "Stop the Gateway container (live LP positions lose their manager)" + return f"Gateway container: {action}" + if tool_name in ("manage_clmm", "manage_amm"): action = input_data.get("action", "?") kind = "CLMM" if tool_name == "manage_clmm" else "AMM" diff --git a/handlers/agents/_shared.py b/handlers/agents/_shared.py index bdafdc3a3..ebb2fe9bc 100644 --- a/handlers/agents/_shared.py +++ b/handlers/agents/_shared.py @@ -37,6 +37,7 @@ DANGEROUS_BOT_ACTIONS, DANGEROUS_CLMM_ACTIONS, DANGEROUS_CONFIG_RESOURCES, + DANGEROUS_CONTAINER_ACTIONS, DANGEROUS_CONTROL_ACTIONS, DANGEROUS_TOOLS, is_dangerous_tool_call, diff --git a/tests/test_acp_permission_gate.py b/tests/test_acp_permission_gate.py index 861ac6fa7..b8cf13c88 100644 --- a/tests/test_acp_permission_gate.py +++ b/tests/test_acp_permission_gate.py @@ -388,6 +388,11 @@ def test_dry_run_cancels_a_swap_but_not_a_quote(): "manage_bots", "manage_clmm", "manage_gateway_config", # the wallets resource takes a private key + # start/stop/restart of the Gateway container (SEC-565). It signs nothing, + # which is not the question: stopping it strands live CLMM/LP executors, and + # starting it hands a caller-chosen Docker image to the host that holds the + # exchange API keys and the Gateway wallet keys. + "manage_gateway_container", } #: Tools that read, or that only write config the trading loop must be told to @@ -395,7 +400,6 @@ def test_dry_run_cancels_a_swap_but_not_a_quote(): #: ``test_every_action_gated_tool_is_classified`` below. NON_FUND_MOVING_TOOLS = { "manage_controllers", # writes controller templates, never a running bot - "manage_gateway_container", # starts and stops Gateway; signs nothing "executor_defaults", # edits a local preferences file; creates nothing "explore_dex_pools", "explore_geckoterminal", @@ -471,12 +475,15 @@ def test_every_mutating_action_of_a_fund_moving_tool_is_dangerous(): f"{tool_name}({action}) mutates but is auto-approved; " "add it to the matching DANGEROUS_* set in condor/runtime/danger.py" ) - # 3 AMM + 3 CLMM + 5 bot today: a floor, so a signature refactor that silently - # stops yielding actions fails instead of passing vacuously. Neither the swap - # nor the executor family is counted: they have no `action` since FEAT-064 and - # FEAT-062 and are gated by name instead (see test_swap_signing_action_is_dangerous - # and tests/test_dangerous_gate_names_resolve.py). - assert checked >= 11, f"only {checked} mutating actions found — enumeration broke" + # 3 AMM + 3 CLMM + 5 bot + container `stop` today: a floor, so a signature + # refactor that silently stops yielding actions fails instead of passing + # vacuously. Only `stop` of the container tool is counted here — `start` and + # `restart` match no MUTATING_PREFIXES entry ("start_" has the underscore), which + # is why test_stopping_or_restarting_gateway_asks_a_human below names all three. + # Neither the swap nor the executor family is counted: they have no `action` since + # FEAT-064 and FEAT-062 and are gated by name instead (see + # test_swap_signing_action_is_dangerous and tests/test_dangerous_gate_names_resolve.py). + assert checked >= 12, f"only {checked} mutating actions found — enumeration broke" # --------------------------------------------------------------------------- @@ -526,6 +533,64 @@ def test_control_agent_with_unreadable_arguments_fails_closed(): assert is_dangerous_tool_call(call), f"{raw!r} slipped past the gate" +# --------------------------------------------------------------------------- +# manage_gateway_container's lifecycle actions must ask first (SEC-565) +# --------------------------------------------------------------------------- + +CONTAINER = "mcp__mcp-hummingbot__manage_gateway_container" + + +def test_stopping_or_restarting_gateway_asks_a_human(): + """A seat that is not authorized by a human cannot take Gateway down. + + Before SEC-565 this tool reached no gate at all: ``is_dangerous_tool_call`` + fell through to ``return False`` and the ACP callback auto-approved, so a + stop under a live CLMM position was neither confirmed nor logged. + """ + for action in ("stop", "restart", "start"): + channel = _CapturingChannel(answer=False) + result = _drive_acp(_acp_request(CONTAINER, {"action": action}), channel) + assert ( + len(channel.delivered) == 1 + ), f"manage_gateway_container({action}) ran with no confirmation" + assert ( + result["outcome"]["outcome"] == "cancelled" + ), f"manage_gateway_container({action}) proceeded after a refusal" + + +def test_the_gateway_start_prompt_names_the_image(): + """`start` hands a caller-chosen image to the host holding the keys.""" + channel = _CapturingChannel(answer=False) + _drive_acp( + _acp_request( + CONTAINER, + {"action": "start", "config": {"image": "evil/gateway:latest"}}, + ), + channel, + ) + + assert channel.delivered[0].summary == ( + "Start the Gateway container with image evil/gateway:latest" + ) + + +def test_reading_gateway_status_or_logs_never_asks(): + """`get_logs` is the escape hatch every opaque Gateway failure points at.""" + for action in ("get_status", "get_logs"): + channel = _CapturingChannel(answer=True) + result = _drive_acp(_acp_request(CONTAINER, {"action": action}), channel) + assert ( + not channel.delivered + ), f"manage_gateway_container({action}) raised a confirmation" + assert result["outcome"]["outcome"] == "selected" + + +def test_gateway_container_with_unreadable_arguments_fails_closed(): + for raw in (None, "not json", ["stop"], {}, {"action": 7}): + call = normalize_tool_call(_acp_request(CONTAINER, raw)) + assert is_dangerous_tool_call(call), f"{raw!r} slipped past the gate" + + # --------------------------------------------------------------------------- # A summary that raises must not become a silent "no" (CORR-294) # --------------------------------------------------------------------------- diff --git a/tests/test_dangerous_gate_names_resolve.py b/tests/test_dangerous_gate_names_resolve.py index 022cd7a5a..8a69d312f 100644 --- a/tests/test_dangerous_gate_names_resolve.py +++ b/tests/test_dangerous_gate_names_resolve.py @@ -22,6 +22,7 @@ DANGEROUS_BOT_ACTIONS, DANGEROUS_CLMM_ACTIONS, DANGEROUS_CONFIG_RESOURCES, + DANGEROUS_CONTAINER_ACTIONS, DANGEROUS_CONTROL_ACTIONS, DANGEROUS_TOOLS, is_dangerous_tool_call, @@ -72,6 +73,7 @@ def test_gated_actions_exist_on_their_tools(): ("manage_clmm", DANGEROUS_CLMM_ACTIONS), ("manage_amm", DANGEROUS_AMM_ACTIONS), ("manage_bots", DANGEROUS_BOT_ACTIONS), + ("manage_gateway_container", DANGEROUS_CONTAINER_ACTIONS), ): available = _action_literals(tool_name) unknown = actions - available @@ -610,6 +612,18 @@ def test_an_ungated_config_edit_is_recorded(): ) +def test_a_gateway_container_lifecycle_call_is_gated_and_recorded(): + """SEC-565: the three lifecycle actions; the two reads stay off both lists.""" + for action in ("start", "stop", "restart"): + call = _call("manage_gateway_container", action=action) + assert is_dangerous_tool_call(call), f"{action} is auto-approved" + assert is_mutating_tool_call(call), f"{action} leaves no row" + for action in ("get_status", "get_logs"): + call = _call("manage_gateway_container", action=action) + assert not is_dangerous_tool_call(call), f"{action} raised a confirmation" + assert not is_mutating_tool_call(call), f"{action} was recorded as a write" + + def test_the_log_fails_open_where_the_gate_fails_closed(): """An action nobody has heard of is recorded, not dropped.""" assert is_mutating_tool_call({"tool": "manage_bots", "input": None}) @@ -634,7 +648,13 @@ def _every_plausible_call() -> list[dict]: action added to a gated tool lands in the subset assertion below on its own. """ calls: list[dict] = [] - for tool in ("manage_bots", "manage_clmm", "manage_amm", "manage_gateway_config"): + for tool in ( + "manage_bots", + "manage_clmm", + "manage_amm", + "manage_gateway_config", + "manage_gateway_container", + ): for action in _action_literals(tool): calls.append(_call(tool, action=action, resource_type="tokens")) calls.append(_call(tool, action=action)) From 1428383f7a88b61ab1ea70b51bda25b723694313 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 12:04:32 +0300 Subject: [PATCH 063/154] Refuse a routine name that is not a name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit create_routine checked its name and read/edit/delete_routine did not, though all four join the model-supplied string onto a directory. So delete_routine(name="../../_shared/routines/x") reached unlink() outside the caller's writable library, edit_routine wrote outside it before reverting, and read_routine returned the source of any .py file it could walk to. One predicate now spells the rule for all four. It admits a bare module stem and nothing else, anchored with \Z rather than $ so a trailing newline is not a name either: no separator, dot, NUL or colon can appear, so dir / f"{name}.py" is provably a direct child of dir. read_routine returns file contents, so it gets a second layer on top: every candidate is checked against routine_source_roots() resolved, which also refuses a symlink pointing out of a library and a prefix sibling like routines_backup/. The write paths keep the name rule alone — an allowlist built by enumerating existing dirs would refuse a first create into an agent home nobody has written to yet. The guard stays out of the manage_routines dispatcher on purpose: name carries an instance_id for stop/get_instance, which is not a routine name. --- mcp_servers/condor/tools/routines.py | 81 +++++++++-- tests/test_condor_tool_surface.py | 198 +++++++++++++++++++++++++++ 2 files changed, 269 insertions(+), 10 deletions(-) diff --git a/mcp_servers/condor/tools/routines.py b/mcp_servers/condor/tools/routines.py index 3e42f3e5c..1ff161876 100644 --- a/mcp_servers/condor/tools/routines.py +++ b/mcp_servers/condor/tools/routines.py @@ -16,6 +16,7 @@ import asyncio import logging +import re import shutil import time from pathlib import Path @@ -37,6 +38,61 @@ "server selector in the dashboard) and try again." ) +# A routine name is a bare Python module stem and nothing else. Anchored with +# ``\Z`` rather than ``$`` because ``$`` also matches before a trailing newline, +# which would admit "foo\n". The class carries no ".", no "/" or "\", no NUL and +# no ":", and cannot start with one either, so ``routines_dir / f"{name}.py"`` is +# provably a direct child of ``routines_dir``: there is no spelling of ``name`` +# — traversal, absolute, encoded, or Windows-style — that leaves the directory. +_ROUTINE_NAME = re.compile(r"[a-z][a-z0-9_]*\Z") + + +def _bad_name(name: str | None) -> dict | None: + """The invalid-name error, or ``None`` when ``name`` may be joined onto a dir. + + One spelling of the rule for all four CRUD actions (SEC-577): ``create_routine`` + checked it and read/edit/delete did not, so a ``name`` of "../../_shared/routines/x" + reached ``unlink()`` outside the caller's writable library. Deliberately *not* + hoisted into the ``manage_routines`` dispatcher — ``name`` carries an + instance_id for ``stop``/``get_instance``, which is not a routine name. + """ + if not name or not _ROUTINE_NAME.match(name): + return { + "error": "name must be lowercase alphanumeric with underscores (e.g. 'my_scanner')" + } + return None + + +def _confined(path: Path, base: Path | None = None) -> bool: + """Is ``path`` really inside a directory routine source may be read from? + + Defense in depth on the read path only (CORR-585's ``routine_source_roots``): + ``_bad_name`` already makes traversal via ``name`` unreachable, but + ``read_routine`` returns *file contents*, and its last fallback joins onto a + cwd-relative ``Path("routines")`` that need not be this install's library at + all. Everything is compared **resolved** and with ``is_relative_to``, so + neither a symlink pointing out of a library nor a prefix sibling such as + ``routines_backup/`` is mistaken for it — which a string prefix would be. + + ``base`` is the directory this very call derived from an anchored resolver + (``_get_agent_routines_dir``, ``_shared_roots``, ``_stock_twin``); it is + trusted as a root of its own because ``routine_source_roots`` enumerates + *existing* dirs, so a library that is legitimate but not yet enumerated must + not read as an escape. It is never caller-controlled, and admitting it still + rejects a symlinked file inside it. + + Not used on the write paths: there the name rule is already total, and an + allowlist built by enumeration would refuse a first ``create_routine`` into + an agent home that has yet to be written. + """ + from condor.routine_store import routine_source_roots + + roots = list(routine_source_roots()) + if base is not None: + roots.append(base.resolve()) + resolved = path.resolve() + return any(resolved.is_relative_to(root) for root in roots) + def _shared_roots() -> tuple[Path, ...]: """Both shared routine libraries in read order: this install's, then shipped.""" @@ -662,12 +718,8 @@ def create_routine( ``shared=True`` publishes it to every assistant — chat only, see :func:`_get_agent_routines_dir`. """ - import re - - if not name or not re.match(r"^[a-z][a-z0-9_]*$", name): - return { - "error": "name must be lowercase alphanumeric with underscores (e.g. 'my_scanner')" - } + if bad := _bad_name(name): + return bad if not code: return {"error": "code is required"} @@ -716,15 +768,18 @@ def create_routine( def read_routine(name: str, target: str | None, shared: bool = False) -> dict: """Read the source code of a routine.""" + if bad := _bad_name(name): + return bad + routines_dir = _get_agent_routines_dir(target, shared) if routines_dir: file_path = routines_dir / f"{name}.py" - if file_path.exists(): + if file_path.exists() and _confined(file_path, routines_dir): return {"name": name, "code": file_path.read_text(), "scope": "agent"} # ...and the shipped one under it, which an agent can read and edit (the # edit forks it down) but never delete. twin = _stock_twin(routines_dir, name) - if twin is not None: + if twin is not None and _confined(twin, twin.parent): return {"name": name, "code": twin.read_text(), "scope": "agent"} # An assistant can read the source of anything it can run, so the shared @@ -732,11 +787,11 @@ def read_routine(name: str, target: str | None, shared: bool = False) -> dict: # `scope` says (writes go through _get_agent_routines_dir and never land here). for shared_path in _shared_roots(): candidate = shared_path / f"{name}.py" - if candidate.exists(): + if candidate.exists() and _confined(candidate, shared_path): return {"name": name, "code": candidate.read_text(), "scope": "shared"} global_path = Path("routines") / f"{name}.py" - if global_path.exists(): + if global_path.exists() and _confined(global_path): return {"name": name, "code": global_path.read_text(), "scope": "global"} return {"error": f"Routine '{name}' not found"} @@ -746,6 +801,9 @@ def edit_routine( name: str, code: str, target: str | None, shared: bool = False ) -> dict: """Update the source code of a routine in the caller's writable library.""" + if bad := _bad_name(name): + return bad + routines_dir = _get_agent_routines_dir(target, shared) if not routines_dir: return { @@ -792,6 +850,9 @@ def edit_routine( def delete_routine(name: str, target: str | None, shared: bool = False) -> dict: """Delete a routine from the caller's writable library.""" + if bad := _bad_name(name): + return bad + routines_dir = _get_agent_routines_dir(target, shared) if not routines_dir: return { diff --git a/tests/test_condor_tool_surface.py b/tests/test_condor_tool_surface.py index dbf66d7b5..c7a0d6ed5 100644 --- a/tests/test_condor_tool_surface.py +++ b/tests/test_condor_tool_surface.py @@ -357,3 +357,201 @@ def test_no_condor_skill_names_a_funnel_era_action_as_a_tool(): "these are actions on manage_agents/manage_strategies/control_agent, " "not tools: " + "; ".join(offenders) ) + + +# ── SEC-577: a routine name is joined onto a directory, so it must be a name ── + +# Every shape of "not a bare name" that has ever been used to leave a directory: +# relative traversal, traversal buried mid-path, an absolute path, the +# URL-encoded and double-encoded spellings (the name reaches the tool over an +# MCP/HTTP hop, so a decoding layer must not be able to hand back a separator), +# a NUL truncator, Windows separators, a prefix-sibling of the library, and the +# trailing newline that a ``$``-anchored regex would have let through. +ESCAPING_NAMES = [ + "../../_shared/routines/some_playbook", + "../main", + "..", + ".", + "sub/../../main", + "/etc/passwd", + "/tmp/evil", + "%2e%2e%2fmain", + "%252e%252e%252fmain", + "..%2fmain", + "main\x00.py", + "..\\..\\main", + "..\\outside\\main", + "../library_backup/x", + "main\n", + "Main", + "main.py", + "main routine", + "_private", + "1main", + "", +] + +_ROUTINE_SRC = '''"""A routine used by the tests.""" + +from pydantic import BaseModel, Field + + +class Config(BaseModel): + """Test routine""" + + pair: str = Field(default="SOL-USDC", description="pair") + + +async def run(config, context) -> str: + return "ok" +''' + + +@pytest.fixture +def sandboxed_library(tmp_path, monkeypatch): + """A writable routines dir with a victim file just outside it. + + ``library_backup`` is a *prefix sibling*: a confinement check written as a + string ``startswith`` would admit it. + """ + from mcp_servers.condor.tools import routines as routines_tool + + library = tmp_path / "library" + library.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (outside / "main.py").write_text("# the victim\n") + sibling = tmp_path / "library_backup" + sibling.mkdir() + (sibling / "x.py").write_text("# the sibling victim\n") + + monkeypatch.setattr( + routines_tool, "_get_agent_routines_dir", lambda *a, **k: library + ) + monkeypatch.chdir(tmp_path) + return library, outside + + +@pytest.mark.parametrize("name", ESCAPING_NAMES) +def test_routine_crud_refuses_every_name_that_is_not_a_bare_name( + name, sandboxed_library +): + """read/edit/delete validate the name create_routine always did (SEC-577). + + Before the guard, ``delete_routine("../main")`` reached ``file_path.unlink()`` + after nothing but an ``exists()`` check and removed a file outside the + caller's writable library; ``edit_routine`` wrote outside it and only then + reverted, and ``read_routine`` returned the source of any reachable ``.py``. + """ + from mcp_servers.condor.tools import routines as routines_tool + + victim = sandboxed_library[1] / "main.py" + before = victim.read_text() + + for result in ( + routines_tool.read_routine(name, None), + routines_tool.edit_routine(name, _ROUTINE_SRC, None), + routines_tool.delete_routine(name, None), + routines_tool.create_routine(name, _ROUTINE_SRC, None), + ): + assert "lowercase alphanumeric" in result.get("error", ""), (name, result) + assert "code" not in result + + # Nothing outside the library was read, written, or unlinked. + assert victim.exists() and victim.read_text() == before + assert (sandboxed_library[1].parent / "library_backup" / "x.py").exists() + assert not list(sandboxed_library[0].iterdir()) + + +def test_an_ordinary_routine_name_still_round_trips(sandboxed_library): + """The guard must not cost a legitimate create → read → edit → delete.""" + from mcp_servers.condor.tools import routines as routines_tool + + library = sandboxed_library[0] + + created = routines_tool.create_routine("my_scanner", _ROUTINE_SRC, None) + assert created.get("created") is True + assert (library / "my_scanner.py").is_file() + + read = routines_tool.read_routine("my_scanner", None) + assert read.get("code") == _ROUTINE_SRC + + edited = routines_tool.edit_routine( + "my_scanner", _ROUTINE_SRC.replace("SOL-USDC", "BTC-USDT"), None + ) + assert edited.get("updated") is True + assert "BTC-USDT" in (library / "my_scanner.py").read_text() + + deleted = routines_tool.delete_routine("my_scanner", None) + assert deleted.get("deleted") is True + assert not (library / "my_scanner.py").exists() + + +def test_instance_actions_still_accept_an_instance_id_in_name(monkeypatch): + """``name`` is overloaded: for stop/get_instance it carries an instance_id. + + That is why the guard lives in the three CRUD functions and not in the + ``manage_routines`` dispatcher — an instance_id is not a routine name. + """ + from mcp_servers.condor.tools import routines as routines_tool + + seen = [] + + async def _fake_call(method, path, *args, **kwargs): + seen.append(path) + return {"status": "completed", "routine_name": "x", "result_text": "done"} + + monkeypatch.setattr(routines_tool, "call_main_api", _fake_call) + + instance_id = "3f7A-21b0_ID" + got = asyncio.run(routines_tool.manage_routines("get_instance", name=instance_id)) + assert "lowercase alphanumeric" not in got.get("error", "") + stopped = asyncio.run(routines_tool.manage_routines("stop", name=instance_id)) + assert stopped.get("stopped") is True + assert all(instance_id in path for path in seen) + + +def test_reading_routine_source_stays_inside_the_discovery_roots(tmp_path): + """The second layer: a resolved path outside every source root is not read. + + ``_bad_name`` already makes traversal via ``name`` unreachable, so this pins + the two escapes a name rule cannot see — a symlink pointing out of the + library, and a prefix sibling of it. + """ + from condor.routine_store import routine_source_roots + from mcp_servers.condor.tools import routines as routines_tool + from routines.base import library_dir + + library = library_dir() + assert library.resolve() in routine_source_roots() + assert routines_tool._confined(library / "some_routine.py") + + assert not routines_tool._confined(tmp_path / "elsewhere.py") + assert not routines_tool._confined(library.parent / "routines_backup" / "x.py") + + secret = tmp_path / "secret.py" + secret.write_text("# not a routine\n") + link = library / "_sec577_link.py" + link.symlink_to(secret) + try: + assert not routines_tool._confined(link) + finally: + link.unlink() + + +def test_read_routine_refuses_a_routine_file_that_symlinks_out(sandboxed_library): + """A valid name over a symlinked file still does not leak what it points at. + + ``_confined`` trusts the library the call derived, so this pins that the + trust is of the *directory* and not of whatever a file inside it resolves to. + """ + from mcp_servers.condor.tools import routines as routines_tool + + library, outside = sandboxed_library + (outside / "secret.py").write_text("# a secret\n") + (library / "leak.py").symlink_to(outside / "secret.py") + + result = routines_tool.read_routine("leak", None) + + assert "a secret" not in str(result) + assert result.get("error") == "Routine 'leak' not found" From 34124ecb1b57f9730c618d72cc7e02808b1db34b Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 13:02:31 +0300 Subject: [PATCH 064/154] Three more routes fail without naming the backend str(e) on an aiohttp client error carries the backend's host and port, so a trader on a shared server could read the internal address off any blip they could provoke. SEC-116/126/130 swept this pattern through five modules; the archived, portfolio and dex routes were never on those lists, and fetchers/archived_run.py built the same string one call deeper and handed it to the route as an ArchivedRunUnavailable.detail the route re-raises verbatim. All four now go through the existing upstream_error helper, and each logs the exception first: the address belongs in the server log, where an operator needs it, not in the response. The static guard is why this survived three audits. It matched only detail=str(e), and all three modules used detail=f"...{e}"; it now rejects both spellings and covers the three added modules. --- condor/fetchers/archived_run.py | 15 +++- condor/web/routes/archived.py | 4 +- condor/web/routes/dex.py | 5 +- condor/web/routes/portfolio.py | 5 +- tests/test_route_upstream_errors.py | 114 +++++++++++++++++++++++++++- 5 files changed, 134 insertions(+), 9 deletions(-) diff --git a/condor/fetchers/archived_run.py b/condor/fetchers/archived_run.py index 86542e79f..caf8019a1 100644 --- a/condor/fetchers/archived_run.py +++ b/condor/fetchers/archived_run.py @@ -30,7 +30,10 @@ from typing import Any from condor.asyncutil import SingleFlight -from condor.fetchers.executors import normalize_executor_side +from condor.fetchers.executors import ( + describe_executor_error, + normalize_executor_side, +) from condor.fetchers.models import ( ArchivedBotPerformance, NormalizedExecutor, @@ -249,7 +252,15 @@ async def _fetch_performance( try: summary = await client.archived_bots.get_database_summary(db_path) except Exception as e: - raise ArchivedRunUnavailable(f"Failed to fetch summary: {e}") + # ``str(e)`` on an aiohttp client error carries the backend's host and + # port, and this detail is handed straight to the browser by the route. + # The operator keeps the address in the log; the caller gets the API's + # own reason (SEC-590). + logger.exception( + "Failed to fetch archived summary for %s on '%s'", db_path, name + ) + _status, message = describe_executor_error(e) + raise ArchivedRunUnavailable(f"Failed to fetch summary: {message}") if not summary or not isinstance(summary, dict): raise ArchivedRunUnavailable("Database not found", missing=True) diff --git a/condor/web/routes/archived.py b/condor/web/routes/archived.py index f3f5fee2e..e39eead2f 100644 --- a/condor/web/routes/archived.py +++ b/condor/web/routes/archived.py @@ -26,6 +26,7 @@ PaginatedExecutors, WebUser, ) +from condor.web.routes._errors import upstream_error from config_manager import get_config_manager logger = logging.getLogger(__name__) @@ -72,7 +73,8 @@ async def list_archived_bots(name: str, user: WebUser = Depends(require_server_a try: databases = await client.archived_bots.list_databases() except Exception as e: - raise HTTPException(status_code=502, detail=f"Failed to list databases: {e}") + logger.exception("Failed to list archived databases on '%s'", name) + raise upstream_error("Failed to list databases", e) if not databases or not isinstance(databases, list): return {"bots": []} diff --git a/condor/web/routes/dex.py b/condor/web/routes/dex.py index 40279bdfb..1417496f8 100644 --- a/condor/web/routes/dex.py +++ b/condor/web/routes/dex.py @@ -21,6 +21,7 @@ from condor import dex_candles from condor.web.auth import require_owner, require_server_access from condor.web.models import WebUser +from condor.web.routes._errors import upstream_error from config_manager import get_config_manager logger = logging.getLogger(__name__) @@ -507,8 +508,8 @@ async def register() -> str: else None ) except Exception as e: - logger.warning("add token %s on %s failed: %s", address, gateway_network, e) - raise HTTPException(status_code=502, detail=f"Gateway refused: {e}") from e + logger.exception("add token %s on %s failed", address, gateway_network) + raise upstream_error("Gateway refused", e) from e # `ensure_tokens_listed` folds its own upstream errors into this verdict so # the automatic path stays quiet; a clicked button must not. diff --git a/condor/web/routes/portfolio.py b/condor/web/routes/portfolio.py index 7a9e6bd99..71d0f9785 100644 --- a/condor/web/routes/portfolio.py +++ b/condor/web/routes/portfolio.py @@ -22,6 +22,7 @@ PortfolioResponse, WebUser, ) +from condor.web.routes._errors import upstream_error from config_manager import get_config_manager logger = logging.getLogger(__name__) @@ -51,8 +52,8 @@ async def get_portfolio( name, ServerDataType.PORTFOLIO ) except Exception as e: - logger.warning("Portfolio fetch exception for %s: %s", name, e) - raise HTTPException(status_code=502, detail=f"Failed to get portfolio: {e}") + logger.exception("Portfolio fetch failed for '%s'", name) + raise upstream_error("Failed to get portfolio", e) if state is None: # Check if fetch is registered diff --git a/tests/test_route_upstream_errors.py b/tests/test_route_upstream_errors.py index b879773b7..c2e3285db 100644 --- a/tests/test_route_upstream_errors.py +++ b/tests/test_route_upstream_errors.py @@ -12,6 +12,13 @@ the contract: the client loses the address, and the operator does not — the full exception still reaches the server log, because a redaction that also destroys the diagnostic is not a fix. + +SEC-590 extended the sweep to the three modules none of those items listed — +``archived``, ``portfolio``, ``dex`` — and to ``condor/fetchers/archived_run.py``, +which built the same leak one call deeper and handed it to the route as an +``ArchivedRunUnavailable.detail``. The static guard below now also rejects the +f-string form (``detail=f"...{e}"``) those three used, which is why three audits +in a row could re-discover the same pattern: the guard only knew ``detail=str(e)``. """ import asyncio @@ -19,6 +26,7 @@ import re from pathlib import Path from types import SimpleNamespace +from unittest import mock import pytest from aiohttp import ClientConnectorError, ClientResponseError, RequestInfo @@ -26,9 +34,14 @@ from multidict import CIMultiDict from yarl import URL +import condor.fetchers.archived_run as archived_run_module +import condor.fetchers.gateway_tokens as gateway_tokens_module +import condor.web.routes.archived as archived_module import condor.web.routes.bots as bots_module import condor.web.routes.controller_performance as cperf_module +import condor.web.routes.dex as dex_module import condor.web.routes.market as market_module +import condor.web.routes.portfolio as portfolio_module import condor.web.routes.settings as settings_module from condor.web.models import WebUser @@ -123,11 +136,65 @@ def _cperf_delete_run(): ) +def _archived_list_databases(): + return asyncio.run(archived_module.list_archived_bots(name="srv", user=_USER)) + + +def _portfolio_refresh(): + return asyncio.run( + portfolio_module.get_portfolio(name="srv", refresh=True, user=_USER) + ) + + +# Wrapped SOL, purely as a syntactically valid mint for the address parser. +_SOL_MINT = "So11111111111111111111111111111111111111112" + + +def _dex_add_token(): + """The token-registration route, entered at the branch that can actually raise. + + ``ensure_tokens_listed`` folds its own upstream errors into a ``failed`` + verdict, so the handler's catch-all is never reached through it. What does + reach it is the second question the collision branch asks: Gateway refuses + the ticker, and the follow-up lookup naming the current holder is the call + that blips. The stand-in for that lookup reaches through the client it is + handed, so it raises whatever the fixture bound rather than a second, made-up + error. + """ + + async def _symbol_taken(_client, _network, addresses, **_kwargs): + return {address: "symbol_taken" for address in addresses} + + async def _holder_lookup_fails(client, *_args, **_kwargs): + return await client.gateway.get_tokens() + + with ( + mock.patch.object(gateway_tokens_module, "ensure_tokens_listed", _symbol_taken), + mock.patch.object( + gateway_tokens_module, "find_symbol_holder", _holder_lookup_fails + ), + ): + return asyncio.run( + dex_module.add_dex_token( + name="srv", + body=dex_module.AddTokenRequest( + network="solana-mainnet-beta", + address=_SOL_MINT, + symbol="WSOL", + ), + user=_USER, + ) + ) + + ENDPOINTS = [ pytest.param(bots_module, _bots_status, id="bots-get-bot-status"), pytest.param(settings_module, _settings_pull_status, id="settings-pull-status"), pytest.param(market_module, _market_order_book, id="market-order-book"), pytest.param(cperf_module, _cperf_delete_run, id="cperf-delete-bot-run"), + pytest.param(archived_module, _archived_list_databases, id="archived-list-dbs"), + pytest.param(portfolio_module, _portfolio_refresh, id="portfolio-refresh"), + pytest.param(dex_module, _dex_add_token, id="dex-add-token"), ] @@ -205,6 +272,38 @@ def test_the_full_exception_still_reaches_the_server_log( ), "the traceback is what makes the log entry actionable" +# --- The fetcher one call deeper, whose detail the route re-raises verbatim --- + + +def test_the_archived_run_fetcher_does_not_hand_the_address_to_the_route( + failing_backend, caplog +): + """``ArchivedRunUnavailable.detail`` becomes the 502 body unchanged. + + ``archived.py::_load_run`` re-raises it as-is, so a detail built with + ``str(e)`` leaks the backend exactly as if the route had interpolated it + itself — the redaction has to happen where the string is built. + """ + failing_backend(archived_module, _transport_error()) + + with caplog.at_level(logging.ERROR, logger=archived_run_module.__name__): + with pytest.raises(HTTPException) as caught: + asyncio.run( + archived_module.get_archived_performance( + name="srv", + db_path="/data/leak-probe.sqlite", + include_executors=False, + user=_USER, + ) + ) + + assert caught.value.status_code == 502, "a reachable-but-broken backend is a 502" + assert BACKEND_HOST not in caught.value.detail + assert BACKEND_PORT not in caught.value.detail + assert BACKEND_URL not in caught.value.detail + assert BACKEND_HOST in caplog.text, "the operator still gets the address" + + # --- The settings helper every settings endpoint funnels through --- @@ -248,10 +347,21 @@ async def get_client(self, name): # --- The pattern must not creep back in --- -_LEAK = re.compile(r"detail=str\((e|exc)\)") +# Both shapes of the same mistake. Only the first was pinned until SEC-590, and +# the three modules that item found all used the second — which is how the rule +# survived three audits without the guard ever noticing. +_LEAK = re.compile(r"""detail=(?:str\((?:e|exc)\)|f["'][^"']*\{\s*(?:e|exc)\b)""") _BARE_EXCEPT = re.compile(r"^\s*except (Exception|BaseException) as (e|exc):") -CONVERTED_MODULES = [bots_module, settings_module, market_module, cperf_module] +CONVERTED_MODULES = [ + bots_module, + settings_module, + market_module, + cperf_module, + archived_module, + portfolio_module, + dex_module, +] @pytest.mark.parametrize( From ec4fafb14b904192fd3cb8e5c37851b375945aa4 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 13:15:19 +0300 Subject: [PATCH 065/154] Refuse a db_path that is not a database path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The archived routes take the database to read as a query parameter and hand it to a client that interpolates it raw into the upstream URL — f"/archived-bots/{db_path}/summary" — where yarl parses rather than escapes. Unlike a path parameter, a query parameter still carries "/", "..", "?" and "#", so "../accounts/master_account/credentials?" did not 404: it resolved to a different, real endpoint and the shared session attached the server's BasicAuth to it. Anyone holding TRADER on one server could make Condor issue arbitrary authenticated GETs against that server's backend, and read the answer's shape back through the error the route re-raises. validate_db_path is the multi-segment sibling of validate_identifier: an optional leading "/", then "/"-joined segments of letters, digits, dot, dash and underscore, no "." or ".." segment, ending in .sqlite or .db. That leaves no way to spell "%" (so no percent- or double-encoded separator for the backend to decode), "?", "#", "\", ":", whitespace, NUL or an empty segment, and \Z rather than $ so a trailing newline is a rejection. Every admitted value therefore names something strictly below /archived-bots/ — not its parent, not a sibling, not the prefix-sibling /archived-bots-private. The check is lexical on purpose: this path names a file on the backend host, never one Condor opens, so resolving it against a local root would follow this machine's symlinks to answer a question about another machine's filesystem. It runs at the four archived endpoints and at /terminated/history before a client is built, so a rejected value is a 400 that was never sent anywhere. --- condor/fetchers/_identifiers.py | 60 +++++ condor/web/routes/archived.py | 25 ++ condor/web/routes/controller_performance.py | 12 + tests/test_db_path_validation.py | 283 ++++++++++++++++++++ 4 files changed, 380 insertions(+) create mode 100644 tests/test_db_path_validation.py diff --git a/condor/fetchers/_identifiers.py b/condor/fetchers/_identifiers.py index 88a37ae66..8cfa72735 100644 --- a/condor/fetchers/_identifiers.py +++ b/condor/fetchers/_identifiers.py @@ -41,3 +41,63 @@ def validate_identifier(value: str, kind: str = "identifier") -> str: "use letters, digits, dot, dash or underscore." ) return value + + +# A database path is a *multi-segment* path, so it cannot use the single-segment +# charset above — but it is still only ever interpolated into a URL path, and it +# is only ever read by the backend's own archive reader. Admitted shape, and +# nothing else: +# +# [/] segment ( "/" segment )* segment = [A-Za-z0-9._-]+ +# no segment is "." or ".." +# the last segment ends in ".sqlite" or ".db" +# +# What that forbids, and why each one matters here: "%" (so a percent-encoded +# "..%2f" — or a double-encoded "..%252f" — can never be handed to the backend +# for it to decode into a separator), "?" and "#" (which would end the path and +# make the rest of the URL a query or fragment, the exact SEC-115 pivot), ".." +# and "." segments (traversal above /archived-bots/), "\" and ":" (Windows +# separators and drive letters), whitespace and NUL (\Z rather than $, so a +# trailing "\n" is a rejection rather than a match), and empty segments. What +# remains cannot change which endpoint the URL addresses: the value is appended +# after a literal "/archived-bots/", every segment moves strictly downwards, so +# neither a parent, a sibling, nor a prefix-sibling of that directory +# ("/archived-botsX") is nameable. +_SAFE_DB_PATH = re.compile(r"\A/?[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*\Z") + +# Both suffixes the archive reader itself accepts (``fetchers/bot_performance``). +_DB_SUFFIXES = (".sqlite", ".db") + + +def validate_db_path(value: str, kind: str = "database path") -> str: + """Return ``value`` if it is safe to interpolate into a URL path. + + A leading ``/`` is allowed — the backend reports its archives by absolute + path — because it cannot escape anything: it only doubles the separator + already written before it. + + The resolve-then-``is_relative_to`` check used for local roots + (``routine_store.routine_source_roots``) deliberately is *not* used here: + this path is never opened by Condor, it names a file on the backend host, so + there is no local directory to resolve it against and a local ``resolve()`` + would follow this machine's symlinks to answer a question about another + machine's filesystem. The guard is therefore purely lexical, and strict + enough that no normalization is needed for it to hold. + + Raises: + IdentifierError: for anything outside the shape documented above. + """ + if not isinstance(value, str) or not _SAFE_DB_PATH.match(value): + raise IdentifierError( + f"Invalid {kind} {value!r}: use letters, digits, dot, dash, " + "underscore and forward slash." + ) + if any(segment in (".", "..") for segment in value.split("/")): + raise IdentifierError( + f"Invalid {kind} {value!r}: a path segment may not be '.' or '..'." + ) + if not value.endswith(_DB_SUFFIXES): + raise IdentifierError( + f"Invalid {kind} {value!r}: must name a .sqlite or .db database." + ) + return value diff --git a/condor/web/routes/archived.py b/condor/web/routes/archived.py index e39eead2f..805496080 100644 --- a/condor/web/routes/archived.py +++ b/condor/web/routes/archived.py @@ -8,6 +8,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query from condor.archived_controllers import group_by_controller +from condor.fetchers._identifiers import IdentifierError, validate_db_path from condor.fetchers.archived_run import ( ArchivedRunUnavailable, cached_run, @@ -34,6 +35,22 @@ router = APIRouter(tags=["archived"]) +def _checked_db_path(db_path: str) -> str: + """The database to read, or a 400 for a value that is not a database path. + + ``db_path`` arrives as a *query* parameter — so unlike a path parameter it + still carries ``/``, ``..``, ``?`` and ``#`` — and ends up interpolated raw + into the upstream URL (``f"/archived-bots/{db_path}/summary"``), where yarl + parses rather than escapes it. Refused here, before a client is built, so a + rejected value never becomes an authenticated GET against some other backend + endpoint (SEC-591, the SEC-115 class). + """ + try: + return validate_db_path(db_path) + except IdentifierError as e: + raise HTTPException(status_code=400, detail=str(e)) + + async def _load_run(client: Any, name: str, db_path: str) -> ArchivedBotPerformance: """The run's performance, or the HTTP answer for why it cannot be read.""" try: @@ -114,6 +131,8 @@ async def get_archived_performance( ), user: WebUser = Depends(require_server_access), ): + db_path = _checked_db_path(db_path) + cm = get_config_manager() client = await cm.get_client(name) @@ -135,6 +154,8 @@ async def get_archived_executors( limit: int = Query(50, ge=1, le=200), user: WebUser = Depends(require_server_access), ): + db_path = _checked_db_path(db_path) + cm = get_config_manager() # Page out of the cached performance entry; on a miss, trigger the full @@ -166,6 +187,8 @@ async def get_archived_controllers( Reads the same cached performance object the run's header and executor pages come from, so expanding a row costs nothing once the run is warm. """ + db_path = _checked_db_path(db_path) + perf = cached_run(name, db_path) if perf is None: client = await get_config_manager().get_client(name) @@ -196,6 +219,8 @@ async def get_archived_report( report index has since been pruned past it — so it answers 200 with a null id rather than a 404. """ + db_path = _checked_db_path(db_path) + entries, _ = list_reports( subject=subjects.bot_run(name, db_path, controller_id), owner_id=user.id, diff --git a/condor/web/routes/controller_performance.py b/condor/web/routes/controller_performance.py index 4dfb80b59..6bd88b55a 100644 --- a/condor/web/routes/controller_performance.py +++ b/condor/web/routes/controller_performance.py @@ -671,6 +671,18 @@ async def get_run_history( of this run" is a true statement about a run that really happened — which is a better thing to draw than a fabricated single step. """ + # The same guard the archived routes apply to the same value, for the same + # reason: it is interpolated raw into an upstream URL path, and a query + # parameter (unlike a path one) still carries "/", "..", "?" and "#". A bad + # value is refused here, before a client exists (SEC-591). + from condor.fetchers._identifiers import IdentifierError, validate_db_path + + if db_path is not None: + try: + validate_db_path(db_path) + except IdentifierError as e: + raise HTTPException(status_code=400, detail=str(e)) + cm = get_config_manager() client = await cm.get_client(name) diff --git a/tests/test_db_path_validation.py b/tests/test_db_path_validation.py new file mode 100644 index 000000000..8bb970a34 --- /dev/null +++ b/tests/test_db_path_validation.py @@ -0,0 +1,283 @@ +"""Tests for SEC-591: ``db_path`` is a URL path segment, not free text. + +The archived routes take the database to read as a *query* parameter, and hand +it to a client that interpolates it raw: +``f"/archived-bots/{db_path}/summary"``. yarl then *parses* the result rather +than escaping it, so a value like ``"../accounts/master_account/credentials?"`` +does not 404 — it resolves to a different, real endpoint, and the shared session +attaches the server's BasicAuth to it. Unlike a FastAPI *path* parameter (a +single non-slash segment by the time Starlette has routed it), a query parameter +carries ``/``, ``..``, ``?`` and ``#`` through untouched, so every one of these +routes was an authenticated-GET pivot for anyone holding TRADER on one server. + +This is the SEC-115 class at the sites SEC-115 never covered; ``validate_db_path`` +is the multi-segment sibling of its ``validate_identifier``, and these tests pin +both halves: the guard admits only what a real archive path looks like, and the +routes refuse a bad one *before* a client is ever asked for. +""" + +import asyncio + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +import condor.web.routes.archived as archived_routes +import condor.web.routes.controller_performance as cperf_routes +from condor.fetchers._identifiers import IdentifierError, validate_db_path +from condor.web.auth import get_current_user +from condor.web.models import WebUser + +USER = WebUser(id=111, username="u", first_name="U", role="user") + +# The pivots, one per escape technique. Every one of these reaches a *different* +# upstream endpoint (or a differently-parsed URL) if it is passed through. +PIVOTS = [ + # Plain traversal — verified against this repo's yarl to resolve to + # /accounts/master_account/credentials. + "../accounts/master_account/credentials?", + "../../admin", + # ".." embedded mid-path, not just as a prefix. + "bots/archived/../../accounts/master_account/credentials.sqlite", + "a/./b.sqlite", + # Query and fragment: everything after them stops being the path. + "run.sqlite?x=1", + "run.sqlite#frag", + # Percent-encoded and double-encoded separators, for the decoder upstream. + "..%2Faccounts%2Fmaster_account.sqlite", + "..%252Faccounts%252Fmaster_account.sqlite", + "%2e%2e/admin.sqlite", + # NUL and whitespace. + "run\x00.sqlite", + "run .sqlite", + "run.sqlite\n", + "\trun.sqlite", + # Windows separators and drive letters. + "..\\accounts\\master.sqlite", + "C:\\bots\\run.sqlite", + # Empty segments and bare separators. + "", + "/", + "//accounts/master.sqlite", + "bots//run.sqlite", + "bots/run.sqlite/", + # A prefix-sibling of the archive directory, reached by climbing out first. + "../archived-bots-private/run.sqlite", + # Not a database at all — the upstream reader only ever opens .sqlite/.db. + "accounts/master_account/credentials", + "run.sqlite.bak", +] + +# What the backend actually reports, absolute and relative, both suffixes. +LEGIT_PATHS = [ + "bots/archived/y/data/broken.sqlite", + "bots/archived/x/data/other-20260101-000000.db", + "/data/sec591-probe.sqlite", + "/archived/ancient-sec591.sqlite", + "/a-sec591.sqlite", + "bots/archived/hummingbot-v2-1.5/data/v2_with_controllers.sqlite", +] + +# The archived-run LRU is module level and keyed by (server, db_path), so the +# happy-path tests below would otherwise leave a warm entry that a later test +# reading the same run silently hits instead of its own stub backend. +SERVER = "sec591-srv" + + +@pytest.fixture(autouse=True) +def _no_cache_bleed(): + from condor.fetchers.archived_run import _performance_cache + + _performance_cache.clear() + yield + _performance_cache.clear() + + +# ── The guard itself ── + + +@pytest.mark.parametrize("payload", PIVOTS) +def test_the_guard_refuses_every_escape(payload): + with pytest.raises(IdentifierError): + validate_db_path(payload) + + +@pytest.mark.parametrize("path", LEGIT_PATHS) +def test_the_guard_admits_a_real_archive_path(path): + assert validate_db_path(path) == path + + +def test_a_non_string_is_refused_rather_than_crashing(): + for value in (None, 12, ["run.sqlite"]): + with pytest.raises(IdentifierError): + validate_db_path(value) + + +def test_no_admitted_path_can_leave_the_archive_directory(): + """The property the charset exists for, asserted on the built URL. + + Whatever is admitted, the URL the client builds still addresses something + *under* ``/archived-bots/`` — not its parent, not a sibling, and not the + prefix-sibling ``/archived-bots-private``. + """ + from yarl import URL + + for path in LEGIT_PATHS: + url = URL(f"http://api:8000/archived-bots/{validate_db_path(path)}/summary") + assert url.path.replace("//", "/").startswith("/archived-bots/") + assert url.query_string == "" + assert ".." not in url.path + + +# ── The routes refuse before a client is ever built ── + + +class SpyClient: + """Records every upstream call, so a pivot that got through is visible.""" + + def __init__(self): + self.calls = [] + self.archived_bots = self._ArchivedBots(self) + self.bot_orchestration = self._Anything(self, "bot_orchestration") + + class _ArchivedBots: + def __init__(self, outer): + self._outer = outer + + async def get_database_summary(self, db_path): + self._outer.calls.append(("get_database_summary", db_path)) + return {"bot_name": "b", "total_trades": 0} + + async def get_database_trades(self, db_path, **kw): + self._outer.calls.append(("get_database_trades", db_path)) + return {"trades": []} + + async def get_database_executors(self, db_path): + self._outer.calls.append(("get_database_executors", db_path)) + return {"executors": []} + + class _Anything: + def __init__(self, outer, name): + self._outer = outer + self._name = name + + def __getattr__(self, method): + async def _call(*args, **kwargs): + self._outer.calls.append((f"{self._name}.{method}", args, kwargs)) + return [] + + return _call + + +class FakeConfigManager: + """Access is granted — the point is that the *value* is refused anyway.""" + + def __init__(self, client): + self._client = client + + def has_server_access(self, user_id, name, *a, **kw): + return True + + async def get_client(self, name): + return self._client + + +def _client(monkeypatch, spy) -> TestClient: + cm = FakeConfigManager(spy) + for target in ( + archived_routes, + cperf_routes, + ): + monkeypatch.setattr(target, "get_config_manager", lambda: cm) + monkeypatch.setattr("condor.web.auth.get_config_manager", lambda: cm) + + app = FastAPI() + app.include_router(archived_routes.router) + app.include_router(cperf_routes.router) + app.dependency_overrides[get_current_user] = lambda: USER + return TestClient(app, raise_server_exceptions=False) + + +GUARDED_ROUTES = [ + "/servers/sec591-srv/archived/performance", + "/servers/sec591-srv/archived/executors", + "/servers/sec591-srv/archived/controllers", + "/servers/sec591-srv/archived/report", +] + + +@pytest.mark.parametrize("route", GUARDED_ROUTES) +@pytest.mark.parametrize("payload", PIVOTS) +def test_route_refuses_the_pivot_and_makes_no_upstream_call( + monkeypatch, route, payload +): + spy = SpyClient() + + resp = _client(monkeypatch, spy).get(route, params={"db_path": payload}) + + assert resp.status_code == 400, resp.text + assert spy.calls == [], f"{route} reached the backend with {payload!r}" + + +@pytest.mark.parametrize("payload", PIVOTS) +def test_run_history_refuses_the_pivot_too(monkeypatch, payload): + """``/terminated/history`` takes the same value on the same access check.""" + spy = SpyClient() + + resp = _client(monkeypatch, spy).get( + "/servers/sec591-srv/terminated/history", + params={"bot_name": "b", "deployed_at": "2026-01-01", "db_path": payload}, + ) + + assert resp.status_code == 400, resp.text + assert spy.calls == [], f"run history reached the backend with {payload!r}" + + +def test_run_history_without_a_db_path_is_still_allowed(monkeypatch): + """The parameter is optional; omitting it must not become a 400.""" + spy = SpyClient() + + resp = _client(monkeypatch, spy).get( + "/servers/sec591-srv/terminated/history", + params={"bot_name": "b", "deployed_at": "2026-01-01"}, + ) + + assert resp.status_code != 400, resp.text + + +@pytest.mark.parametrize("path", LEGIT_PATHS) +def test_a_real_path_still_reaches_the_backend(monkeypatch, path): + """The guard must not break the ordinary read.""" + spy = SpyClient() + + resp = _client(monkeypatch, spy).get( + "/servers/sec591-srv/archived/performance", params={"db_path": path} + ) + + assert resp.status_code == 200, resp.text + assert ("get_database_summary", path) in spy.calls + + +def test_the_rejection_is_not_swallowed_by_the_route_try_block(monkeypatch): + """The guard runs before the try that turns upstream failures into 404/502. + + If it moved inside, a pivot would come back as one of those instead of a + 400, and — worse — would already have been sent upstream. + """ + spy = SpyClient() + + resp = _client(monkeypatch, spy).get( + "/servers/sec591-srv/archived/performance", + params={"db_path": "../accounts/master_account/credentials?"}, + ) + + assert resp.status_code == 400 + assert spy.calls == [] + + +def test_the_guard_is_reachable_from_the_fetcher_module_too(): + """Sanity: the helper lives beside ``validate_identifier``, one shape.""" + from condor.fetchers import _identifiers + + assert hasattr(_identifiers, "validate_identifier") + assert asyncio.iscoroutinefunction(validate_db_path) is False From 5f9f33965cb103a4efe6fcfa241d1ae31f989616 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 13:29:38 +0300 Subject: [PATCH 066/154] Revoking a share now takes effect on the open socket, not at the next reload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A dashboard WebSocket was authorized once, at subscribe time, and `broadcast` then fanned out purely on channel membership. The socket is long-lived and auto-reconnecting, so when an owner revoked a trader's share — or an admin blocked the user — that tab kept receiving `portfolio:`, `bots_ws:` and `executors:` frames for the rest of the day. Every REST route re-checks per request; the socket was the one surface where a permission change did not land. `broadcast` now resolves the channel's server once and filters the subscriber walk it already does on both halves of the gate the connection passed on the way in: the role `connect` checked (a blocked user keeps their `shared_with` entry, so this half is not redundant) and the per-server share `handle_message` checked. The decision is memoised per user for the frame, so six tabs of one user cost one lookup, and `ConfigManager` is in memory — no IO on the hot path, and the payload is still encoded once per broadcast. A revoked subscriber is unsubscribed, not disconnected: its other channels are still legitimate. Dropping it runs the same teardown as an explicit unsubscribe — now one `_drop_subscription` shared by all three ways a subscription ends — so it stops holding the upstream stream open. --- condor/web/ws_manager.py | 89 +++++++-- tests/conftest.py | 24 +++ tests/test_portfolio_history_sds.py | 4 + tests/test_sds_listener_typed_keys.py | 6 + tests/test_ws_broadcast_access_revocation.py | 180 +++++++++++++++++++ tests/test_ws_broadcast_single_encode.py | 4 + tests/test_ws_manager_oneshot_tasks.py | 6 + 7 files changed, 300 insertions(+), 13 deletions(-) create mode 100644 tests/test_ws_broadcast_access_revocation.py diff --git a/condor/web/ws_manager.py b/condor/web/ws_manager.py index bb5684c35..96ee9df0b 100644 --- a/condor/web/ws_manager.py +++ b/condor/web/ws_manager.py @@ -234,11 +234,22 @@ def disconnect(self, conn: _Connection) -> None: self._connections.remove(conn) logger.info("WS disconnected: user %s", conn.user_id) for channel in list(conn.channels): - prefix = channel.split(":", 1)[0] - if prefix in self._stream_registry(): - self._maybe_stop_stream(prefix, channel) - else: - self._maybe_unsub_sds(channel) + self._drop_subscription(conn, channel) + + def _drop_subscription(self, conn: _Connection, channel: str) -> None: + """Unsubscribe one connection from one channel and tear the stream down. + + The three ways a subscription ends — the client unsubscribes, the socket + goes away, or the user's access to the server is revoked (SEC-592) — + must all release the upstream stream, or a channel nobody is listening + to any more keeps its poller alive. + """ + conn.channels.discard(channel) + prefix = channel.split(":", 1)[0] + if prefix in self._stream_registry(): + self._maybe_stop_stream(prefix, channel) + else: + self._maybe_unsub_sds(channel) def _maybe_unsub_sds(self, channel: str) -> None: """Unsubscribe from SDS if no WS clients remain for this channel.""" @@ -311,12 +322,7 @@ async def handle_message(self, conn: _Connection, raw: str) -> None: await self._subscribe_sds(channel) elif action == "unsubscribe" and channel: - conn.channels.discard(channel) - prefix = channel.split(":", 1)[0] - if prefix in self._stream_registry(): - self._maybe_stop_stream(prefix, channel) - else: - self._maybe_unsub_sds(channel) + self._drop_subscription(conn, channel) elif action == "set_candle_duration" and channel: # Frontend changed duration without re-subscribing @@ -473,11 +479,68 @@ async def _broadcast_update(self, channel: str, data: Any) -> None: # -- Broadcasting -- - async def broadcast(self, channel: str, data: Any) -> None: - self._last_data[channel] = data + def _authorized_subscribers(self, channel: str) -> list[_Connection]: + """Subscribers of ``channel`` whose access to its server still holds. + + ``handle_message`` gates a subscription once, at subscribe time, but a + dashboard socket is long-lived and auto-reconnecting: an owner can + revoke a share — or an admin block the user — hours after the tab was + opened, and until SEC-592 that connection kept receiving the server's + frames until the tab reloaded. Every REST route re-checks per request + (``check_server_access``); this is the socket's equivalent, applied to + the subscriber walk ``broadcast`` already does. + + A revoked subscriber is *unsubscribed*, not disconnected: its other + channels are still legitimate. Dropping it runs the same teardown as an + explicit unsubscribe, so it stops holding the upstream stream open. + + Cost: ``ConfigManager`` is in memory, so the check is a handful of dict + lookups, and it is memoised per user for the duration of the frame — + one lookup per *distinct* user, not per connection, and no extra pass + over the connection list. + """ + from config_manager import UserRole, get_config_manager + subscribers = [ conn for conn in list(self._connections) if channel in conn.channels ] + server_name = self._server_from_channel(channel) + if server_name is None or not subscribers: + # Nothing to authorize against: a channel with no server segment is + # rejected at subscribe time and cannot have subscribers anyway. + return subscribers + + cm = get_config_manager() + allowed: dict[int, bool] = {} + live: list[_Connection] = [] + revoked: list[_Connection] = [] + for conn in subscribers: + ok = allowed.get(conn.user_id) + if ok is None: + # Both halves of the gate the connection passed on the way in: + # the role ``connect`` checked, and the per-server share + # ``handle_message`` checked. A blocked user keeps their + # ``shared_with`` entry, so the role half is not redundant. + ok = cm.get_user_role(conn.user_id) in ( + UserRole.USER, + UserRole.ADMIN, + ) and cm.has_server_access(conn.user_id, server_name) + allowed[conn.user_id] = ok + (live if ok else revoked).append(conn) + + for conn in revoked: + logger.warning( + "WS subscription revoked: user=%s channel=%s server=%s (access lost)", + conn.user_id, + channel, + server_name, + ) + self._drop_subscription(conn, channel) + return live + + async def broadcast(self, channel: str, data: Any) -> None: + self._last_data[channel] = data + subscribers = self._authorized_subscribers(channel) if not subscribers: return # The frame is identical for every subscriber (no per-connection state diff --git a/tests/conftest.py b/tests/conftest.py index 807e787d3..54b57b42b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -97,3 +97,27 @@ def _isolated_runtime_root(tmp_path, monkeypatch): monkeypatch.setenv(paths.AGENTS_ROOT_ENV, str(tmp_path / "agents")) monkeypatch.setenv(paths.STOCK_AGENTS_ROOT_ENV, str(tmp_path / "stock-agents")) monkeypatch.setenv(paths.REPORTS_DIR_ENV, str(tmp_path / "reports")) + + +@pytest.fixture +def ws_access_granted(monkeypatch): + """Let every WS connection reach every server, for the duration of a test. + + Since SEC-592 ``WebSocketManager.broadcast`` re-reads the subscriber's + server access on every frame, so that revoking a share takes effect on an + open socket instead of at the next tab reload. A module whose + ``_Connection`` is a bare stand-in (``user_id=1``, no entry in any config) + would otherwise have its subscription revoked mid-test. These modules + exercise the fan-out and the stream lifecycle, not the gate — the gate has + its own module, ``test_ws_broadcast_access_revocation``. + """ + import config_manager + + class _PermissiveCM: + def get_user_role(self, user_id): + return config_manager.UserRole.USER + + def has_server_access(self, *_args, **_kwargs): + return True + + monkeypatch.setattr(config_manager, "get_config_manager", lambda: _PermissiveCM()) diff --git a/tests/test_portfolio_history_sds.py b/tests/test_portfolio_history_sds.py index 708b9da65..130f40150 100644 --- a/tests/test_portfolio_history_sds.py +++ b/tests/test_portfolio_history_sds.py @@ -30,6 +30,10 @@ from condor.web.routes.portfolio import get_portfolio_history from condor.web.ws_manager import WebSocketManager, _Connection +# These connections are stand-ins with no config entry; `broadcast` re-reads +# server access per frame since SEC-592 and would revoke them mid-test. +pytestmark = pytest.mark.usefixtures("ws_access_granted") + _USER = WebUser(id=1, role="admin") # Two snapshots of one account: BTC 100 -> 120, ETH 50 -> 40. diff --git a/tests/test_sds_listener_typed_keys.py b/tests/test_sds_listener_typed_keys.py index 46f322f99..32f926806 100644 --- a/tests/test_sds_listener_typed_keys.py +++ b/tests/test_sds_listener_typed_keys.py @@ -18,6 +18,8 @@ import inspect import json +import pytest + from condor.server_data_service import ( CacheKey, ServerDataService, @@ -26,6 +28,10 @@ ) from condor.web.ws_manager import WebSocketManager, _Connection, channel_for_key +# These connections are stand-ins with no config entry; `broadcast` re-reads +# server access per frame since SEC-592 and would revoke them mid-test. +pytestmark = pytest.mark.usefixtures("ws_access_granted") + class _FakeWS: def __init__(self): diff --git a/tests/test_ws_broadcast_access_revocation.py b/tests/test_ws_broadcast_access_revocation.py new file mode 100644 index 000000000..2e5c90e3f --- /dev/null +++ b/tests/test_ws_broadcast_access_revocation.py @@ -0,0 +1,180 @@ +"""A dashboard socket loses a server's frames the moment access is revoked (SEC-592). + +`handle_message` gates a subscription once, at subscribe time (the SEC-019 fix), +and `broadcast` used to fan out purely on channel membership. A dashboard socket +is long-lived and auto-reconnecting, so an owner revoking a trader's share — or +an admin blocking the user — did not take effect until the tab reloaded: the +connection kept receiving `portfolio:` / `bots_ws:` / +`executors:` payloads, possibly for the rest of the day. Every REST route +re-checks per request; this pins the same behaviour on the socket. +""" + +import asyncio +import json + +import pytest + +from condor.web import ws_manager as ws_manager_module +from condor.web.ws_manager import WebSocketManager, _Connection + + +class _FakeWS: + def __init__(self): + self.sent: list[str] = [] + self.closed = False + + async def send_text(self, raw: str) -> None: + self.sent.append(raw) + + async def close(self, **kwargs) -> None: # pragma: no cover - must not run + self.closed = True + + def channels(self) -> list[str]: + return [json.loads(raw)["channel"] for raw in self.sent] + + +class _FakeCM: + """The two dicts the real `ConfigManager` consults, and nothing else.""" + + def __init__(self, shares: dict[int, set[str]], roles: dict[int, object]): + self._shares = shares + self._roles = roles + + def get_user_role(self, user_id: int): + return self._roles.get(user_id) + + def has_server_access(self, user_id: int, server_name: str, *_a, **_kw) -> bool: + return server_name in self._shares.get(user_id, set()) + + +@pytest.fixture +def cm(monkeypatch): + import config_manager + + manager = _FakeCM( + shares={1: {"srv"}, 2: {"srv"}}, + roles={1: config_manager.UserRole.USER, 2: config_manager.UserRole.USER}, + ) + monkeypatch.setattr(config_manager, "get_config_manager", lambda: manager) + return manager + + +def _manager_with(channel: str, *conns: _Connection) -> WebSocketManager: + manager = WebSocketManager() + for conn in conns: + conn.channels.add(channel) + manager._connections.append(conn) + return manager + + +def _conn(user_id: int) -> _Connection: + return _Connection(_FakeWS(), user_id=user_id) + + +def test_revoked_share_stops_the_frames_without_a_reconnect(cm): + """The regression: the same open socket, before and after the revoke.""" + keeps, loses = _conn(1), _conn(2) + manager = _manager_with("portfolio:srv", keeps, loses) + + asyncio.run(manager.broadcast("portfolio:srv", {"total": 1})) + assert len(loses.ws.sent) == 1, "subscribe-time access should deliver the frame" + + cm._shares[2] = set() # owner revokes the share; the socket stays open + + asyncio.run(manager.broadcast("portfolio:srv", {"total": 2})) + + assert len(loses.ws.sent) == 1, "revoked subscriber still receiving broadcasts" + assert len(keeps.ws.sent) == 2, "a subscriber that kept access lost its stream" + + +def test_blocking_the_user_stops_the_frames_too(cm): + """A blocked user keeps their `shared_with` entry, so the role half matters.""" + import config_manager + + blocked = _conn(2) + manager = _manager_with("portfolio:srv", blocked) + + asyncio.run(manager.broadcast("portfolio:srv", {"total": 1})) + cm._roles[2] = config_manager.UserRole.BLOCKED + + asyncio.run(manager.broadcast("portfolio:srv", {"total": 2})) + + assert len(blocked.ws.sent) == 1, "blocked user still receiving broadcasts" + + +def test_revoked_subscription_is_dropped_and_the_stream_stops(cm): + """Dropping the last subscriber must release the upstream stream too.""" + + async def scenario(): + loses = _conn(2) + manager = _manager_with("executors:srv", loses) + + async def forever(): + await asyncio.Event().wait() + + task = asyncio.create_task(forever()) + manager._executor_tasks["executors:srv"] = task + + cm._shares[2] = set() + await manager.broadcast("executors:srv", [{"id": "e1"}]) + + assert "executors:srv" not in loses.channels + assert "executors:srv" not in manager._executor_tasks + await asyncio.sleep(0) + assert task.cancelled() or task.done() + assert not loses.ws.closed, "the socket itself must stay open" + + asyncio.run(scenario()) + + +def test_the_socket_keeps_its_other_channels(cm): + """Revocation is per server: it unsubscribes a channel, it does not disconnect.""" + conn = _conn(2) + cm._shares[2] = {"srv", "other"} + manager = _manager_with("portfolio:srv", conn) + conn.channels.add("portfolio:other") + + cm._shares[2] = {"other"} + asyncio.run(manager.broadcast("portfolio:srv", {"total": 2})) + asyncio.run(manager.broadcast("portfolio:other", {"total": 3})) + + assert conn.ws.channels() == ["portfolio:other"] + assert conn in manager._connections + assert conn.channels == {"portfolio:other"} + + +def test_the_frame_is_still_encoded_once_for_the_survivors(cm, monkeypatch): + """PERF-210 preserved: the re-check must not push encoding per connection.""" + conns = [_conn(1), _conn(1), _conn(2)] + manager = _manager_with("portfolio:srv", *conns) + cm._shares[2] = set() + + calls = [] + real_dumps = json.dumps + + def counting_dumps(obj, **kwargs): + calls.append(obj) + return real_dumps(obj, **kwargs) + + monkeypatch.setattr(ws_manager_module.json, "dumps", counting_dumps) + asyncio.run(manager.broadcast("portfolio:srv", {"total": 1})) + + assert len(calls) == 1, f"payload encoded {len(calls)} times" + assert [len(c.ws.sent) for c in conns] == [1, 1, 0] + + +def test_the_check_is_memoised_per_user_not_per_connection(cm): + """Six tabs of the same user cost one permission lookup per frame, not six.""" + lookups: list[int] = [] + inner = cm.has_server_access + + def counting(user_id, server_name, *a, **kw): + lookups.append(user_id) + return inner(user_id, server_name, *a, **kw) + + cm.has_server_access = counting + manager = _manager_with("portfolio:srv", *[_conn(1) for _ in range(6)]) + + asyncio.run(manager.broadcast("portfolio:srv", {"total": 1})) + + assert lookups == [1], f"one frame cost {len(lookups)} permission lookups" diff --git a/tests/test_ws_broadcast_single_encode.py b/tests/test_ws_broadcast_single_encode.py index e205840ec..fad1eb55d 100644 --- a/tests/test_ws_broadcast_single_encode.py +++ b/tests/test_ws_broadcast_single_encode.py @@ -14,6 +14,10 @@ from condor.web import ws_manager as ws_manager_module from condor.web.ws_manager import WebSocketManager, _Connection +# These connections are stand-ins with no config entry; `broadcast` re-reads +# server access per frame since SEC-592 and would revoke them mid-test. +pytestmark = pytest.mark.usefixtures("ws_access_granted") + class _FakeWS: def __init__(self): diff --git a/tests/test_ws_manager_oneshot_tasks.py b/tests/test_ws_manager_oneshot_tasks.py index fd553b390..e5ebeec19 100644 --- a/tests/test_ws_manager_oneshot_tasks.py +++ b/tests/test_ws_manager_oneshot_tasks.py @@ -13,9 +13,15 @@ import json import logging +import pytest + from condor.server_data_service import CacheKey, ServerDataType from condor.web.ws_manager import WebSocketManager, _Connection +# These connections are stand-ins with no config entry; `broadcast` re-reads +# server access per frame since SEC-592 and would revoke them mid-test. +pytestmark = pytest.mark.usefixtures("ws_access_granted") + WS_LOGGER = "condor.web.ws_manager" From 5f6e35c15290a6c68592403f5542ee69e413f6b5 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 13:38:38 +0300 Subject: [PATCH 067/154] (sec) scope every report listing to its owner, not just /reports SEC-196 made the reports API owner-scoped but left the listings that find reports by the *name* of the thing that produced them: the per-routine list, and the per-strategy and per-session lists on the agents router. A routine or strategy name is not a secret and was never ownership-checked, so any approved user could spell a colleague's routine and read the index entries of its runs - id, title, tags, subject and owner - and the routines page's report_count answered the same question as a number. All four now pass the caller's owner filter into the store call, ahead of the name match, so no spelling of the name widens the scope. The filter itself moves from routes/reports.py to condor/web/auth.py, beside require_admin, whose docstring already records what happens when a gate is imported privately across route modules. Admins keep the whole index; ownerless legacy entries stay admin-only. --- condor/routine_store.py | 16 +++++--- condor/web/auth.py | 17 ++++++++ condor/web/routes/agents.py | 22 +++++++++-- condor/web/routes/reports.py | 18 ++------- condor/web/routes/routines.py | 10 ++++- tests/test_reports_ownership.py | 69 +++++++++++++++++++++++++++++++++ 6 files changed, 126 insertions(+), 26 deletions(-) diff --git a/condor/routine_store.py b/condor/routine_store.py index d664cc4e6..bcf3eaf63 100644 --- a/condor/routine_store.py +++ b/condor/routine_store.py @@ -326,12 +326,18 @@ def _discover_all(self) -> dict[str, "RoutineInfo"]: return all_routines - def _get_report_counts(self) -> dict[str, int]: - """Get report count per routine source_name.""" + def _get_report_counts(self, owner_id: int | None = None) -> dict[str, int]: + """Get report count per routine source_name. + + ``owner_id`` scopes the tally the way ``list_reports`` does (SEC-593): + an unscoped count is still a read on another user's reports — it says + how many they ran — so the web callers pass the caller's filter and + only an admin (or an internal, already-per-user caller) gets ``None``. + """ try: from condor.reports import list_reports - reports, _ = list_reports(limit=1000) + reports, _ = list_reports(limit=1000, owner_id=owner_id) counts: dict[str, int] = {} for r in reports: sn = r.get("source_name", "") @@ -341,9 +347,9 @@ def _get_report_counts(self) -> dict[str, int]: except Exception: return {} - def list_routines(self) -> list[dict]: + def list_routines(self, owner_id: int | None = None) -> list[dict]: all_routines = self._discover_all() - report_counts = self._get_report_counts() + report_counts = self._get_report_counts(owner_id) out = [] for name, info in all_routines.items(): out.append( diff --git a/condor/web/auth.py b/condor/web/auth.py index 14f80ed86..78ede7366 100644 --- a/condor/web/auth.py +++ b/condor/web/auth.py @@ -152,6 +152,23 @@ def require_admin(user: WebUser) -> None: ) +def report_owner_filter(user: WebUser) -> int | None: + """Whose reports a listing may show: everyone's for admins, own otherwise. + + ``None`` disables the store's owner filter — the admin override the other + server-data surfaces already grant (``_owner`` in conversations, + ``_require_ownership`` in sessions). Anyone else is scoped to entries + stamped with their own id; legacy entries with no owner are dropped for + them (fail closed, SEC-196). + + Lives here rather than in ``routes/reports.py`` (SEC-593) because the + per-routine, per-strategy and per-session report listings need the same + line, and ``require_admin`` above records what happens when a gate is + imported privately across route modules instead: it grows copies. + """ + return None if get_config_manager().is_admin(user.id) else user.id + + # ── Server-scoped access (SEC-147) ── diff --git a/condor/web/routes/agents.py b/condor/web/routes/agents.py index 10ea4e6cc..92384b728 100644 --- a/condor/web/routes/agents.py +++ b/condor/web/routes/agents.py @@ -50,7 +50,11 @@ ) from condor.fsutil import atomic_write_text from condor.layering import fork_if_stock -from condor.web.auth import check_server_access, get_current_user +from condor.web.auth import ( + check_server_access, + get_current_user, + report_owner_filter, +) from condor.web.models import ReportSummary, WebUser # ── Simple in-memory TTL cache for performance data ── @@ -3429,7 +3433,12 @@ async def get_session_report( run_key = _runkey(slug, sslug) source = f"{run_key}/session_{session_num}" - reports, _total = list_reports(source_type="routine", search=run_key, limit=100) + reports, _total = list_reports( + source_type="routine", + search=run_key, + limit=100, + owner_id=report_owner_filter(user), + ) matched = [r for r in reports if r.get("source_name", "") == source] return {"report": ReportSummary(**matched[0]).model_dump() if matched else None} @@ -3545,7 +3554,7 @@ async def get_strategy_routines( from condor.routine_store import get_routine_store store = get_routine_store() - all_routines = store.list_routines() + all_routines = store.list_routines(owner_id=report_owner_filter(user)) prefix = f"{slug}/" return [r for r in all_routines if r.get("name", "").startswith(prefix)] @@ -3563,7 +3572,12 @@ async def get_strategy_reports( run_key = _runkey(slug, sslug) prefix = f"{run_key}/" - reports, _total = list_reports(source_type="routine", search=run_key, limit=limit) + reports, _total = list_reports( + source_type="routine", + search=run_key, + limit=limit, + owner_id=report_owner_filter(user), + ) matched = [r for r in reports if r.get("source_name", "").startswith(prefix)] return { "reports": [ReportSummary(**r).model_dump() for r in matched], diff --git a/condor/web/routes/reports.py b/condor/web/routes/reports.py index 44d15d666..1290dc0f5 100644 --- a/condor/web/routes/reports.py +++ b/condor/web/routes/reports.py @@ -14,25 +14,13 @@ list_reports_grouped, resolve_report_asset, ) -from condor.web.auth import get_current_user +from condor.web.auth import get_current_user, report_owner_filter from condor.web.models import ReportsListResponse, ReportSummary, WebUser from config_manager import get_config_manager router = APIRouter(prefix="/reports", tags=["reports"]) -def _owner_filter(user: WebUser) -> int | None: - """Whose reports a listing may show: everyone's for admins, own otherwise. - - ``None`` disables the store's owner filter — the admin override the other - server-data surfaces already grant (``_owner`` in conversations, - ``_require_ownership`` in sessions). Anyone else is scoped to entries - stamped with their own id; legacy entries with no owner are dropped for - them (fail closed, SEC-196). - """ - return None if get_config_manager().is_admin(user.id) else user.id - - def _authorized_entry(report_id: str, user: WebUser) -> dict: """Fetch an index entry, 404 if absent and 403 if it belongs to someone else. @@ -69,7 +57,7 @@ async def get_reports( agent=agent, limit=limit, offset=offset, - owner_id=_owner_filter(user), + owner_id=report_owner_filter(user), ) return ReportsListResponse( reports=[ReportSummary(**e) for e in entries], @@ -79,7 +67,7 @@ async def get_reports( @router.get("/latest-by-source") async def get_reports_grouped(user: WebUser = Depends(get_current_user)): - return list_reports_grouped(owner_id=_owner_filter(user)) + return list_reports_grouped(owner_id=report_owner_filter(user)) @router.get("/assets/{filename}") diff --git a/condor/web/routes/routines.py b/condor/web/routes/routines.py index a19e6c782..2def4f9aa 100644 --- a/condor/web/routes/routines.py +++ b/condor/web/routes/routines.py @@ -22,6 +22,7 @@ from condor.web.auth import ( check_server_access, get_current_user, + report_owner_filter, require_server_access_query, ) from condor.web.models import WebUser @@ -134,7 +135,7 @@ def _authorized_instance(instance_id: str, user: WebUser) -> dict: async def list_routines(user: WebUser = Depends(get_current_user)): """List all discovered routines with their fields.""" store = get_routine_store() - return store.list_routines() + return store.list_routines(owner_id=report_owner_filter(user)) @router.get("/instances") @@ -379,7 +380,12 @@ async def get_routine_reports( # Agent routines are prefixed (e.g. "agent_slug/routine_name") but reports # may be saved with just the base name. Match both. base_name = routine_name.split("/")[-1] if "/" in routine_name else routine_name - reports, total = list_reports(search=base_name, limit=limit) + # SEC-593: scope to the caller before matching. The routine name is a free + # string anyone may spell, so this listing is only as private as its owner + # filter — the same one ``GET /reports`` applies. + reports, total = list_reports( + search=base_name, limit=limit, owner_id=report_owner_filter(user) + ) # Filter to exact source_name match (full prefixed or base name) exact = [r for r in reports if r.get("source_name") in (routine_name, base_name)] return {"reports": exact, "total": len(exact)} diff --git a/tests/test_reports_ownership.py b/tests/test_reports_ownership.py index 0f122a14f..2c5354860 100644 --- a/tests/test_reports_ownership.py +++ b/tests/test_reports_ownership.py @@ -16,6 +16,7 @@ from starlette.testclient import TestClient import condor.reports as rep +import condor.web.auth as web_auth import condor.web.routes.reports as routes from condor.web.app import create_app from condor.web.auth import get_current_user @@ -72,6 +73,10 @@ def reports_dir(tmp_path, monkeypatch): ) monkeypatch.setenv("CONDOR_REPORTS_DIR", str(directory)) monkeypatch.setattr(routes, "get_config_manager", lambda: FakeConfigManager()) + # The listing gate moved to condor.web.auth (SEC-593), where the routine and + # strategy listings reach it too; the id-addressed reads still resolve theirs + # in routes/reports.py, so both namespaces are faked. + monkeypatch.setattr(web_auth, "get_config_manager", lambda: FakeConfigManager()) return directory @@ -110,6 +115,70 @@ def test_grouped_listing_is_scoped_the_same_way(reports_dir): assert sorted(g["source_name"] for g in groups) == ["legacy", "mine", "theirs"] +# ── Per-routine listing (SEC-593) ── + + +def test_routine_reports_listing_hides_another_users_reports(reports_dir): + """The one reports surface SEC-196 left unfiltered. + + ``GET /routines/{name}/reports`` matched on the routine name alone, so any + approved user could spell a colleague's routine and read the index entries + of its runs — id, title, tags, subject and owner. The name is not a secret + and never was ownership-checked, so the owner filter is the whole gate. + """ + with _client(USER) as client: + payload = client.get("/api/v1/routines/theirs/reports").json() + assert payload == {"reports": [], "total": 0} + + +def test_routine_reports_listing_still_returns_your_own(reports_dir): + with _client(USER) as client: + payload = client.get("/api/v1/routines/mine/reports").json() + assert [r["id"] for r in payload["reports"]] == ["mine01"] + + +def test_routine_reports_listing_hides_ownerless_entries_from_non_admins(reports_dir): + """Fail closed, exactly as the id-addressed reads do.""" + with _client(USER) as client: + payload = client.get("/api/v1/routines/legacy/reports").json() + assert payload == {"reports": [], "total": 0} + + +def test_an_agent_prefixed_name_does_not_widen_the_scope(reports_dir): + """The handler also matches the base name after the last ``/``. + + That fallback is what makes the route enumerable — ``x/theirs`` reaches the + same entries as ``theirs`` — so the filter has to sit in the store call, + ahead of the name match, rather than in the caller's spelling of the name. + """ + with _client(USER) as client: + payload = client.get("/api/v1/routines/someagent/theirs/reports").json() + assert payload == {"reports": [], "total": 0} + + +def test_admin_still_sees_every_owner_on_the_routine_listing(reports_dir): + with _client(ADMIN) as client: + theirs = client.get("/api/v1/routines/theirs/reports").json() + legacy = client.get("/api/v1/routines/legacy/reports").json() + assert [r["id"] for r in theirs["reports"]] == ["their1"] + assert [r["id"] for r in legacy["reports"]] == ["legacy"] + + +def test_report_counts_do_not_tally_another_users_runs(reports_dir): + """``report_count`` on the routines list was the same leak as a number. + + An unfiltered tally still answers "how many times did they run this", so the + store takes the caller's filter and the web routes pass it. + """ + from condor.routine_store import RoutineStore + + store = RoutineStore() + assert store._get_report_counts(owner_id=USER.id) == {"mine": 1} + assert store._get_report_counts(owner_id=OTHER.id) == {"theirs": 1} + # An admin (owner_id None) keeps the whole tally. + assert store._get_report_counts() == {"mine": 1, "theirs": 1, "legacy": 1} + + # ── Id-addressed reads and deletes ── From 07822d1c5f481432c40f9f21fb3660cf464d6aa0 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 13:45:46 +0300 Subject: [PATCH 068/154] Gate the server pin when an Agent is created, not only when it is edited MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /agents passed a caller-supplied server_name straight into AgentStore.create, while the very next route, PATCH /agents/{slug}/config, gated the identical field. So the same value was owner-checked when edited and unchecked when created. Credentials were never actually at risk — every resolution site holds a candidate server to existence and reach — but condor/runtime/binding.py returns the stored pin verbatim as SessionBinding.server_name, and that is what the chat header, the approval line and AgentSummary render. An ungated create therefore produced an Agent that names a foreign account while its tools trade on the caller's own: a mislabel on the one field that says which account is at risk. Both doors to the pin now call the same check_server_access floor, and the test asserts they answer identically so they cannot drift apart again. --- condor/web/routes/agents.py | 10 ++ tests/test_agent_create_server_pin_access.py | 106 +++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 tests/test_agent_create_server_pin_access.py diff --git a/condor/web/routes/agents.py b/condor/web/routes/agents.py index 92384b728..06f5cb925 100644 --- a/condor/web/routes/agents.py +++ b/condor/web/routes/agents.py @@ -2027,6 +2027,16 @@ async def create_agent( """Create a new Agent (identity + brain; strategies are added separately).""" from condor.preferences import get_active_agent_key + # The pin is the same field ``update_agent_config`` gates below, so it is + # gated identically here: creating an Agent already pinned to a server the + # caller cannot reach is the edit they are not allowed to make afterwards. + # Credentials never actually leak — every resolution site re-checks reach — + # but ``SessionBinding.server_name`` reports the stored pin verbatim, so an + # ungated create leaves an Agent naming a foreign account in the chat header + # and in ``AgentSummary`` (SEC-594). An empty pin needs no access at all. + if req.server_name: + check_server_access(user.id, req.server_name) + # Same rule as the Telegram/MCP path: an unspecified model inherits the # creator's active one rather than defaulting to a guess. agent = _agent_store().create( diff --git a/tests/test_agent_create_server_pin_access.py b/tests/test_agent_create_server_pin_access.py new file mode 100644 index 000000000..39ecd6001 --- /dev/null +++ b/tests/test_agent_create_server_pin_access.py @@ -0,0 +1,106 @@ +"""Creating an Agent cannot pin it to a server the caller has no access to (SEC-594). + +``POST /agents`` passed ``req.server_name`` straight into ``AgentStore.create`` +while the very next route, ``PATCH /agents/{slug}/config``, gated the identical +field. Credentials were never at risk — every resolution site re-checks +existence *and* reach — but ``condor/runtime/binding.py`` reports the stored pin +verbatim as ``SessionBinding.server_name``, so an ungated create produced an +Agent that names a foreign account in the chat header and in ``AgentSummary`` +while its tools trade on the caller's own. + +The tests assert the rule on both writes of the same field, so the two routes +cannot drift apart again. +""" + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +from condor.agents.agent import AgentStore +from condor.web.auth import get_current_user +from condor.web.models import WebUser +from condor.web.routes import agents as routes + +USER = WebUser(id=555, username="u", first_name="U", role="user") + +MINE = "mine-prod" +THEIRS = "someone-else" + + +class FakeConfigManager: + def is_admin(self, user_id): + return False + + def get_server(self, server_name): + return {"name": server_name} if server_name in (MINE, THEIRS) else None + + def has_server_access(self, user_id, server_name, *a, **k): + return user_id == USER.id and server_name == MINE + + +@pytest.fixture +def env(tmp_path, monkeypatch): + monkeypatch.setenv("CONDOR_AGENTS_ROOT", str(tmp_path)) + monkeypatch.setattr( + "config_manager.get_config_manager", lambda: FakeConfigManager() + ) + # The guard lives in condor.web.auth (SEC-147), which binds + # get_config_manager at import time — patch it there too. + monkeypatch.setattr( + "condor.web.auth.get_config_manager", lambda: FakeConfigManager() + ) + monkeypatch.setattr("condor.preferences.get_active_agent_key", lambda uid: "") + return tmp_path + + +def _client() -> TestClient: + app = FastAPI() + app.include_router(routes.router) + app.dependency_overrides[get_current_user] = lambda: USER + return TestClient(app) + + +def test_creating_an_agent_pinned_to_someone_elses_server_is_refused(env): + res = _client().post("/agents", json={"name": "Borrowed", "server_name": THEIRS}) + + assert res.status_code == 403 + assert res.json()["detail"] == "No access" + # And nothing was written: the gate runs before AgentStore.create. + assert AgentStore().get("borrowed") is None + assert not (env / "borrowed").exists() + + +def test_creating_an_agent_on_my_own_server_still_works(env): + res = _client().post("/agents", json={"name": "Mine", "server_name": MINE}) + + assert res.status_code == 200 + assert AgentStore().get("mine").server_name == MINE + + +def test_creating_an_agent_with_no_pin_is_untouched(env): + res = _client().post("/agents", json={"name": "Unpinned"}) + + assert res.status_code == 200 + assert AgentStore().get("unpinned") is not None + + +def test_create_and_patch_config_enforce_the_same_rule_on_the_same_field(env): + """The pin is one field; both doors to it must answer identically.""" + AgentStore().create(name="Existing", description="d") + client = _client() + + assert client.post( + "/agents", json={"name": "New", "server_name": THEIRS} + ).status_code == ( + client.patch( + "/agents/existing/config", json={"server_name": THEIRS} + ).status_code + ) + assert ( + client.patch( + "/agents/existing/config", json={"server_name": THEIRS} + ).status_code + == 403 + ) + # The foreign pin never reached storage through either door. + assert AgentStore().get("existing").server_name in (None, "") From 2c2cd3a02bdc971b31028c59c01e166ac69831e2 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 13:55:15 +0300 Subject: [PATCH 069/154] Ask a human before repointing the RPC every transaction goes through MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `manage_gateway_config` was in DANGEROUS_TOOLS with an empty resource set, justified by a comment claiming everything the tool touches — tokens, pools, connectors, networks — is Gateway's own symbol/address mapping that moves no funds. That is true of tokens and pools and false of the other two: a network config carries `nodeURL`, the RPC endpoint every transaction from this server is signed against and broadcast through, and a connector config carries the slippage every later swap inherits. The dashboard already knows this — its route calls the same write "a server-wide change" and demands OWNER — while the MCP path let a model repoint it with no human in the loop. Gate `networks` and `connectors` on resource *and* action, so a write asks and a read does not: finding out which RPC a chain is on is how an opaque swap failure gets diagnosed, and a prompt in front of that buys nothing. The gate fails closed twice — an unreadable resource is dangerous whatever the action claims, and on a gated resource an unreadable action is a write — so a call cannot slip through on the half of it we can parse. The confirmation line now names the network and the keys being set, because "update networks" is not what a human is being asked to approve. Tokens, pools, chains and wallets keep today's ungated behaviour; the log already recorded all of them and still does. --- condor/runtime/danger.py | 89 +++++++++++++++----- tests/test_acp_permission_gate.py | 98 +++++++++++++++++++++- tests/test_dangerous_gate_names_resolve.py | 91 ++++++++++++++++---- 3 files changed, 240 insertions(+), 38 deletions(-) diff --git a/condor/runtime/danger.py b/condor/runtime/danger.py index fd81e7162..a83ed7714 100644 --- a/condor/runtime/danger.py +++ b/condor/runtime/danger.py @@ -48,7 +48,7 @@ "execute_swap", # every call signs; quote/status/search are separate tools "manage_clmm", # every action that moves liquidity "manage_amm", # every action that moves liquidity - "manage_gateway_config", # no resource of it is gated today; see below + "manage_gateway_config", # only writes to networks/connectors; see below "manage_gateway_container", # only the lifecycle actions; see below "control_agent", # only `start`, which launches an unattended trading loop # The executor family is gated by NAME (FEAT-062), the same way the swap family @@ -149,24 +149,40 @@ # action still fails closed. DANGEROUS_CONTAINER_ACTIONS = {"start", "stop", "restart"} -# Resource types within manage_gateway_config that require confirmation. This tool -# is gated on `resource_type`, not `action`, because what it edits matters and how -# it edits does not. The set is empty and the gate stays: `wallets` used to be in it -# because `add` took a PRIVATE KEY, but that path no longer exists over MCP (wallets -# are read-only there, added and removed in the dashboard), so nothing this tool can -# reach is worth a human. Everything it still touches — tokens, pools, connectors, -# networks — is Gateway's own symbol/address mapping. Deleting a token there moves no -# funds and changes nothing on-chain, so gating it would put a human in front of a -# config edit while the trades that edit enables stay where they are. An unreadable -# `resource_type` still fails closed. -DANGEROUS_CONFIG_RESOURCES: set[str] = set() +# Resource types within manage_gateway_config whose *writes* require confirmation +# (SEC-566). This set used to be empty, justified by "everything it touches is +# Gateway's own symbol/address mapping". That is true of two of the four resources +# and false of the other two, which is the correction: +# +# - `tokens` and `pools` stay ungated. Adding or deleting one edits a symbol → +# address mapping. It moves no funds and changes nothing on-chain, so gating it +# would put a human in front of a config edit while the trades that edit enables +# stay where they are. `chains` and `wallets` stay ungated too — both are +# read-only over MCP since FEAT-065 (a wallet is imported in the dashboard, and +# `add` no longer takes a private key anywhere the model can reach). +# - `networks` and `connectors` are gated, because an `update` there is not a +# mapping: a network config carries `nodeURL`, the RPC endpoint every transaction +# from this server is signed against and broadcast through, and a connector config +# carries settings such as allowed slippage that every later swap inherits. The +# dashboard already treats exactly this write as privileged — the web route calls +# it "a server-wide change" and demands OWNER (condor/web/routes/settings.py) — +# while the MCP path let a model repoint it with no human in the loop. Tool output +# is untrusted input, so "the prompt says don't call this" is not a control +# (SEC-253). +# +# The gate is resource *and* action: `list`/`get` on a gated resource stay on the +# fast path, because reading which RPC a chain is on is how a model diagnoses a +# failed swap, and a prompt in front of a read buys nothing. An unreadable +# `resource_type` — and, on a gated resource, an unreadable `action` — still fails +# closed. See :func:`_is_dangerous_config_call`. +DANGEROUS_CONFIG_RESOURCES: set[str] = {"networks", "connectors"} # ── What changed the world (FEAT-097) ── # # The sets above answer "should a human approve this". The log asks a different # question — "did this change anything" — and the two deliberately differ: -# `manage_gateway_config` is gated on an intentionally empty resource set, and +# `manage_gateway_config` gates only its two funds-path resources (SEC-566), and # the brakes (`stop`, `pause`, `resume`, `shutdown`) are ungated on purpose. A # log built on the confirmation predicate would therefore be silent about every # config edit and every brake, which is the exact silence the log exists to end. @@ -358,14 +374,22 @@ def _executor_amount(tool_name: str, input_data: dict[str, Any]) -> str: return f" of {amount}" if amount is not None else "" -def _has_dangerous_resource( - tool_call: dict[str, Any], dangerous_resources: set[str] -) -> bool: - """Whether a resource-gated tool call selects one of its dangerous resources. +def _is_dangerous_config_call(tool_call: dict[str, Any]) -> bool: + """Whether a ``manage_gateway_config`` call writes a funds-path resource (SEC-566). - The resource-typed twin of :func:`_has_dangerous_action`, and it fails closed the - same way (SEC-093): unreadable arguments, or a missing/non-string ``resource_type``, - count as dangerous. + Resource *and* action, because neither half alone is the right gate. Resource + alone would prompt on `get networks`, the read a model does to diagnose a + failed swap; action alone would prompt on `add tokens`, a symbol → address + mapping that moves nothing. What needs a human is a *write* to `networks` or + `connectors`: the RPC every transaction is broadcast through, and the slippage + every later swap inherits. + + Fails closed twice over (SEC-093). Unreadable arguments, or a missing or + non-string ``resource_type``, are dangerous whatever the action claims to be — + a call we cannot classify is never let through on the strength of the half of + it we can read. On a resource we *can* read and that is gated, a missing or + non-string ``action`` is dangerous too, so a write cannot hide behind an + unparseable action string. """ input_data = tool_call_input(tool_call) if input_data is None: @@ -373,7 +397,12 @@ def _has_dangerous_resource( resource = input_data.get("resource_type") if not isinstance(resource, str) or not resource: return True - return resource in dangerous_resources + if resource not in DANGEROUS_CONFIG_RESOURCES: + return False + action = input_data.get("action") + if not isinstance(action, str) or not action: + return True + return action not in READ_ONLY_CONFIG_ACTIONS def is_dangerous_tool_call(tool_call: dict[str, Any]) -> bool: @@ -390,7 +419,7 @@ def is_dangerous_tool_call(tool_call: dict[str, Any]) -> bool: return _has_dangerous_action(tool_call, DANGEROUS_AMM_ACTIONS) if tool_name == "manage_gateway_config": - return _has_dangerous_resource(tool_call, DANGEROUS_CONFIG_RESOURCES) + return _is_dangerous_config_call(tool_call) if tool_name == "manage_gateway_container": return _has_dangerous_action(tool_call, DANGEROUS_CONTAINER_ACTIONS) @@ -623,6 +652,22 @@ def format_tool_summary(tool_call: dict[str, Any]) -> str: # accepting a private key at all (FEAT-065); wallets are read-only now. resource = input_data.get("resource_type", "?") action = input_data.get("action", "?") + if resource in DANGEROUS_CONFIG_RESOURCES and action not in ( + READ_ONLY_CONFIG_ACTIONS + ): + # The gated half (SEC-566). "update networks" is not what the human is + # approving — the target and the keys are, because one of those keys is + # `nodeURL`, the RPC every later transaction is broadcast through. + target = ( + input_data.get("network_id") or input_data.get("connector_name") or "?" + ) + updates = input_data.get("config_updates") + keys = ( + ", ".join(str(key) for key in updates) + if isinstance(updates, dict) and updates + else "?" + ) + return f"Gateway config: {action} {resource} '{target}', setting {keys}" return f"Gateway config: {action} {resource}" if tool_name == "manage_gateway_container": diff --git a/tests/test_acp_permission_gate.py b/tests/test_acp_permission_gate.py index b8cf13c88..47b672077 100644 --- a/tests/test_acp_permission_gate.py +++ b/tests/test_acp_permission_gate.py @@ -387,7 +387,10 @@ def test_dry_run_cancels_a_swap_but_not_a_quote(): "manage_amm", "manage_bots", "manage_clmm", - "manage_gateway_config", # the wallets resource takes a private key + # a write to `networks` repoints `nodeURL`, the RPC every transaction is + # broadcast through, and one to `connectors` sets the slippage every later + # swap inherits (SEC-566) + "manage_gateway_config", # start/stop/restart of the Gateway container (SEC-565). It signs nothing, # which is not the question: stopping it strands live CLMM/LP executors, and # starting it hands a caller-chosen Docker image to the host that holds the @@ -591,6 +594,99 @@ def test_gateway_container_with_unreadable_arguments_fails_closed(): assert is_dangerous_tool_call(call), f"{raw!r} slipped past the gate" +# --------------------------------------------------------------------------- +# manage_gateway_config's network/connector writes must ask first (SEC-566) +# --------------------------------------------------------------------------- + +CONFIG = "mcp__mcp-hummingbot__manage_gateway_config" + + +def test_repointing_the_rpc_endpoint_asks_a_human_and_is_refused(): + """A seat that is not authorized by a human cannot move the funds path. + + Before SEC-566 ``DANGEROUS_CONFIG_RESOURCES`` was empty, so this call was + auto-approved: a model could repoint `nodeURL` — the RPC every transaction + from this server is signed against and broadcast through — with no prompt, + while the dashboard demanded OWNER for the identical write. + """ + for resource, target in ( + ("networks", {"network_id": "solana-mainnet-beta"}), + ("connectors", {"connector_name": "jupiter"}), + ): + channel = _CapturingChannel(answer=False) + result = _drive_acp( + _acp_request( + CONFIG, + { + "resource_type": resource, + "action": "update", + **target, + "config_updates": {"nodeURL": "https://evil.example/rpc"}, + }, + ), + channel, + ) + assert ( + len(channel.delivered) == 1 + ), f"manage_gateway_config(update {resource}) ran with no confirmation" + assert ( + result["outcome"]["outcome"] == "cancelled" + ), f"manage_gateway_config(update {resource}) proceeded after a refusal" + + +def test_the_network_update_prompt_names_the_network_and_the_keys(): + """The human approves a `nodeURL` change, not the words "update networks".""" + channel = _CapturingChannel(answer=False) + _drive_acp( + _acp_request( + CONFIG, + { + "resource_type": "networks", + "action": "update", + "network_id": "solana-mainnet-beta", + "config_updates": {"nodeURL": "https://evil.example/rpc"}, + }, + ), + channel, + ) + + assert channel.delivered[0].summary == ( + "Gateway config: update networks 'solana-mainnet-beta', setting nodeURL" + ) + + +def test_reading_a_network_config_or_editing_a_token_never_asks(): + """Reads stay silent, and a token edit is a symbol → address mapping.""" + for args in ( + {"resource_type": "networks", "action": "get", "network_id": "solana-mainnet"}, + {"resource_type": "connectors", "action": "list"}, + {"resource_type": "tokens", "action": "add", "token_symbol": "WIF"}, + {"resource_type": "pools", "action": "delete"}, + ): + channel = _CapturingChannel(answer=True) + result = _drive_acp(_acp_request(CONFIG, args), channel) + assert not channel.delivered, f"manage_gateway_config({args}) raised a prompt" + assert result["outcome"]["outcome"] == "selected" + + +def test_gateway_config_with_an_unreadable_resource_or_action_fails_closed(): + """SEC-093/SEC-566: neither half of the gate can be defeated by junk.""" + for raw in ( + None, + "not json", + ["networks"], + {}, + {"resource_type": 7}, + {"action": "update"}, + # A gated resource whose action cannot be read is a write. + {"resource_type": "networks"}, + {"resource_type": "networks", "action": 7}, + {"resource_type": "connectors", "action": ""}, + ): + call = normalize_tool_call(_acp_request(CONFIG, raw)) + assert is_dangerous_tool_call(call), f"{raw!r} slipped past the gate" + + # --------------------------------------------------------------------------- # A summary that raises must not become a silent "no" (CORR-294) # --------------------------------------------------------------------------- diff --git a/tests/test_dangerous_gate_names_resolve.py b/tests/test_dangerous_gate_names_resolve.py index 8a69d312f..16b7b40ca 100644 --- a/tests/test_dangerous_gate_names_resolve.py +++ b/tests/test_dangerous_gate_names_resolve.py @@ -15,7 +15,7 @@ import inspect import typing -from condor.runtime.danger import is_mutating_tool_call +from condor.runtime.danger import READ_ONLY_CONFIG_ACTIONS, is_mutating_tool_call from handlers.agents._shared import ( CREATE_EXECUTOR_TOOLS, DANGEROUS_AMM_ACTIONS, @@ -74,6 +74,11 @@ def test_gated_actions_exist_on_their_tools(): ("manage_amm", DANGEROUS_AMM_ACTIONS), ("manage_bots", DANGEROUS_BOT_ACTIONS), ("manage_gateway_container", DANGEROUS_CONTAINER_ACTIONS), + # SEC-566 gates manage_gateway_config on a resource *and* an action: the + # exemption is spelled out as the read-only actions, so a rename of `get` + # would silently start prompting on every read rather than silently stop + # gating — but both halves have to keep resolving, so both are pinned. + ("manage_gateway_config", READ_ONLY_CONFIG_ACTIONS), ): available = _action_literals(tool_name) unknown = actions - available @@ -262,31 +267,49 @@ def test_gated_calls_render_a_specific_confirmation_summary(): assert summary != tool_name -def test_gateway_config_gates_nothing_now_that_wallets_are_read_only(): - """manage_gateway_config is gated on resource_type, not action — and gates nothing. - - `wallets` was the one gated resource because `add` took a private key; that path - is gone (wallets are read-only over MCP, FEAT-065). Everything the tool still - edits is Gateway's own symbol/address mapping — deleting a token moves no funds - and changes nothing on-chain, so gating it would stop a config edit while leaving - the trades it enables ungated. - """ +def _config_resource_literals() -> set[str]: + """Every ``resource_type`` ``manage_gateway_config`` actually accepts.""" fn = _registered_tools()["manage_gateway_config"] - resources = { + return { str(v) for v in typing.get_args( inspect.signature(fn).parameters["resource_type"].annotation ) if isinstance(v, str) } + + +def test_gateway_config_gates_the_funds_path_resources_and_nothing_else(): + """SEC-566: a write to `networks`/`connectors` asks; a token or pool edit does not. + + The gate used to be an empty resource set, justified by "everything this tool + touches is Gateway's own symbol/address mapping". Two of the four resources are + not: a network config carries `nodeURL`, the RPC every transaction is broadcast + through, and a connector config carries the slippage every later swap inherits. + The dashboard already demands OWNER for exactly that write; the MCP path asked + nobody. `tokens` and `pools` really are a mapping and stay ungated, so a human is + not put in front of a config edit while the trades it enables run unattended. + """ + resources = _config_resource_literals() assert DANGEROUS_CONFIG_RESOURCES <= resources, ( f"gated resource(s) the tool has no such value for: " f"{sorted(DANGEROUS_CONFIG_RESOURCES - resources)}" ) - assert DANGEROUS_CONFIG_RESOURCES == set() + assert DANGEROUS_CONFIG_RESOURCES == {"networks", "connectors"} + + for resource in DANGEROUS_CONFIG_RESOURCES: + for action in _action_literals("manage_gateway_config") - ( + READ_ONLY_CONFIG_ACTIONS + ): + assert is_dangerous_tool_call( + { + "tool": "manage_gateway_config", + "input": {"resource_type": resource, "action": action}, + } + ), f"{resource}/{action} repoints the funds path with no confirmation" for resource in resources - DANGEROUS_CONFIG_RESOURCES: - for action in ("list", "add", "delete"): + for action in _action_literals("manage_gateway_config"): assert not is_dangerous_tool_call( { "tool": "manage_gateway_config", @@ -295,6 +318,38 @@ def test_gateway_config_gates_nothing_now_that_wallets_are_read_only(): ), f"{resource}/{action} should not need confirmation" +def test_gateway_config_reads_stay_on_the_fast_path(): + """Reading a gated resource is how a failed swap gets diagnosed (SEC-566). + + The gate is resource *and* action for this reason alone: a prompt in front of + `get networks` would put one in front of finding out which RPC a chain is on. + """ + for resource in DANGEROUS_CONFIG_RESOURCES: + for action in READ_ONLY_CONFIG_ACTIONS: + assert not is_dangerous_tool_call( + { + "tool": "manage_gateway_config", + "input": {"resource_type": resource, "action": action}, + } + ), f"{resource}/{action} raised a confirmation for a read" + + +def test_gateway_config_fails_closed_on_an_unreadable_action(): + """SEC-566: on a gated resource, an action we cannot read is a write. + + The resource half is not enough on its own — once the gate started reading an + action, an unparseable one would otherwise fall through the read-only test and + be waved past. + """ + for bad in ({}, {"action": None}, {"action": 7}, {"action": ""}): + for resource in DANGEROUS_CONFIG_RESOURCES: + call = { + "tool": "manage_gateway_config", + "input": {"resource_type": resource, **bad}, + } + assert is_dangerous_tool_call(call), f"{resource}/{bad} slipped past" + + def test_gateway_config_fails_closed_on_an_unreadable_resource(): """SEC-093: a call whose resource_type cannot be read is treated as dangerous.""" for bad in ({}, {"resource_type": None}, {"resource_type": 7}, {"action": "add"}): @@ -601,7 +656,7 @@ def test_the_ungated_brakes_are_recorded(): def test_an_ungated_config_edit_is_recorded(): - """``DANGEROUS_CONFIG_RESOURCES`` is empty on purpose; the log is not.""" + """A token edit is ungated on purpose (SEC-566); the log keeps it anyway.""" for action in ("add", "delete", "update", "save"): call = _call("manage_gateway_config", action=action, resource_type="tokens") assert not is_dangerous_tool_call(call) @@ -658,8 +713,14 @@ def _every_plausible_call() -> list[dict]: for action in _action_literals(tool): calls.append(_call(tool, action=action, resource_type="tokens")) calls.append(_call(tool, action=action)) - for resource in _action_literals("manage_gateway_config"): + for resource in _config_resource_literals(): calls.append(_call("manage_gateway_config", resource_type=resource)) + # Every real (resource, action) pair, so the SEC-566 gate's own combinations + # — not just the fail-closed ones — are held to `dangerous ⊆ mutating`. + for action in _action_literals("manage_gateway_config"): + calls.append( + _call("manage_gateway_config", resource_type=resource, action=action) + ) for action in _control_actions(): calls.append(_call("control_agent", action=action, agent_id="a.b_1")) for tool in sorted(DANGEROUS_TOOLS | {"manage_bots"}): From 097dfb0518a73bccd904a2b0c871089027a433b2 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 14:23:10 +0300 Subject: [PATCH 070/154] Stop the whole bot freezing while a bare ollama: key finds its model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bare "ollama:" / "lmstudio:" key is a first-class menu choice, and it leaves the model id to us: session start asks the local backend what it serves. That probe was a synchronous urllib urlopen running on the one event loop that also carries Telegram polling, the dashboard, every WebSocket and every other agent session — so a backend that was down or hung froze all of it for the full timeout, twice over for ollama. Probe with the async client the same file already uses for its healthcheck, from one shared _probe_json helper, so the duplicate GET /models is gone too. The budget comes from TimeoutPolicy rather than a literal, and _build_model is now async up to start(). --- condor/acp/pydantic_ai_client.py | 67 ++++++---- condor/runtime/timeouts.py | 5 + tests/test_agent_identity.py | 5 +- tests/test_custom_provider.py | 10 +- tests/test_local_model_probe_nonblocking.py | 135 ++++++++++++++++++++ 5 files changed, 189 insertions(+), 33 deletions(-) create mode 100644 tests/test_local_model_probe_nonblocking.py diff --git a/condor/acp/pydantic_ai_client.py b/condor/acp/pydantic_ai_client.py index 64833fac3..34d489bdd 100644 --- a/condor/acp/pydantic_ai_client.py +++ b/condor/acp/pydantic_ai_client.py @@ -18,7 +18,6 @@ import uuid from typing import Any, AsyncIterator, Iterator from urllib.parse import urlparse -from urllib.request import Request, urlopen from .client import ( ACPEvent, @@ -439,9 +438,14 @@ def __init__( # so cancelling the run *is* the cancel. self._abort_requested = False - def _build_model(self) -> Any: + async def _build_model(self) -> Any: """Build the pydantic-ai model object with sensible defaults. + Async because a bare local key ("ollama:" / "lmstudio:") has to ask the + local backend which model it serves, and that probe must not park the + one event loop that also runs Telegram polling, the dashboard and every + other session. + All local providers (ollama, lmstudio) are routed through OpenAI-compatible endpoints so we control the base_url explicitly. This avoids requiring environment variables like OLLAMA_BASE_URL. @@ -532,7 +536,7 @@ def _build_model(self) -> Any: if prefix in DEFAULT_BASE_URLS: base_url = base_url or DEFAULT_BASE_URLS[prefix] if not model_id: - model_id = self._resolve_default_local_model( + model_id = await self._resolve_default_local_model( prefix=prefix, base_url=base_url ) openai_client = AsyncOpenAI( @@ -560,7 +564,7 @@ def _build_model(self) -> Any: return infer_model(self.model_name) - def _resolve_default_local_model(self, prefix: str, base_url: str) -> str: + async def _resolve_default_local_model(self, prefix: str, base_url: str) -> str: """Resolve a usable default model for local providers. For ollama/lmstudio with model strings like "ollama:" (no explicit model), @@ -572,12 +576,12 @@ def _resolve_default_local_model(self, prefix: str, base_url: str) -> str: if env_override: return env_override - model_id = self._fetch_openai_compatible_model(base_url) + model_id = await self._fetch_openai_compatible_model(base_url) if model_id: return model_id if prefix == "ollama": - model_id = self._fetch_ollama_native_model(base_url) + model_id = await self._fetch_ollama_native_model(base_url) if model_id: return model_id @@ -587,20 +591,36 @@ def _resolve_default_local_model(self, prefix: str, base_url: str) -> str: "or set CONDOR_DEFAULT_LOCAL_MODEL." ) - def _fetch_openai_compatible_model(self, base_url: str) -> str | None: - """Try GET {base_url}/models and return the first model id.""" - url = f"{base_url.rstrip('/')}/models" - try: - req = Request(url, method="GET") - with urlopen(req, timeout=2) as resp: - if resp.status != 200: - return None - import json + async def _probe_json(self, url: str) -> Any: + """GET ``url`` off the event loop and return the decoded JSON, or None. + + Uses the same async client ``healthcheck_local_backend`` uses. The old + stdlib ``urlopen`` here was synchronous, so a local backend that was + down or hung froze the single loop that also runs Telegram polling, the + dashboard and every other session for the whole timeout (PERF-331). + """ + import httpx + + from condor.runtime.timeouts import TIMEOUTS - payload = json.loads(resp.read().decode("utf-8")) + budget = TIMEOUTS.local_model_probe + timeout = httpx.Timeout(connect=budget, read=budget, write=budget, pool=budget) + try: + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.get(url) + if resp.status_code != 200: + return None + return resp.json() except Exception: return None + async def _fetch_openai_compatible_model(self, base_url: str) -> str | None: + """Try GET {base_url}/models and return the first model id.""" + url = f"{base_url.rstrip('/')}/models" + payload = await self._probe_json(url) + if not isinstance(payload, dict): + return None + data = payload.get("data") if isinstance(data, list) and data: first = data[0] @@ -610,21 +630,14 @@ def _fetch_openai_compatible_model(self, base_url: str) -> str | None: return model_id.strip() return None - def _fetch_ollama_native_model(self, base_url: str) -> str | None: + async def _fetch_ollama_native_model(self, base_url: str) -> str | None: """Try GET /api/tags from the Ollama host and return first model name.""" parsed = urlparse(base_url) if not parsed.scheme or not parsed.netloc: return None native_url = f"{parsed.scheme}://{parsed.netloc}/api/tags" - try: - req = Request(native_url, method="GET") - with urlopen(req, timeout=2) as resp: - if resp.status != 200: - return None - import json - - payload = json.loads(resp.read().decode("utf-8")) - except Exception: + payload = await self._probe_json(native_url) + if not isinstance(payload, dict): return None models = payload.get("models") @@ -673,7 +686,7 @@ async def start(self) -> None: toolsets.append(mcp_server) self._mcp_servers.append(mcp_server) - model = self._build_model() + model = await self._build_model() prepare = self._prepare_tools if self.allowed_tools else None self._agent = Agent( model, diff --git a/condor/runtime/timeouts.py b/condor/runtime/timeouts.py index 4d9f0baa9..55e020720 100644 --- a/condor/runtime/timeouts.py +++ b/condor/runtime/timeouts.py @@ -50,6 +50,11 @@ class TimeoutPolicy: sse_stream: int = 1800 # Budget for one MCP tool call. mcp_call: float = 15.0 + # How long to probe a local inference server for the model it serves, when + # a bare "ollama:" / "lmstudio:" key leaves the model id to us. Short: it + # is a localhost request on the session-start path, and a backend that is + # down should fall through to the explicit-key error quickly. + local_model_probe: float = 2.0 # How long an agent client may take to become usable: the ACP # ``initialize`` + ``session/new`` handshake, and the pydantic-ai wait for # its MCP servers to come up. Generous because a cold start can include an diff --git a/tests/test_agent_identity.py b/tests/test_agent_identity.py index 9824f8d99..da84d75e3 100644 --- a/tests/test_agent_identity.py +++ b/tests/test_agent_identity.py @@ -226,7 +226,10 @@ def respond(messages, info): seen["instructions"] = messages[0].instructions return ModelResponse(parts=[TextPart("ok")]) - client._build_model = lambda: FunctionModel(respond) + async def _stub_model(): + return FunctionModel(respond) + + client._build_model = _stub_model await client.start() try: await client._agent.run("who are you?") diff --git a/tests/test_custom_provider.py b/tests/test_custom_provider.py index 83857f95f..259e24674 100644 --- a/tests/test_custom_provider.py +++ b/tests/test_custom_provider.py @@ -488,13 +488,13 @@ def test_build_model_requires_base_url(monkeypatch): monkeypatch.delenv("CUSTOM_LLM_BASE_URL", raising=False) client = PydanticAIClient(model="custom:foo") with pytest.raises(RuntimeError, match="base URL"): - client._build_model() + asyncio.run(client._build_model()) def test_build_model_requires_model_id(): client = PydanticAIClient(model="custom:", base_url="https://api.example.com/v1") with pytest.raises(RuntimeError, match="model id"): - client._build_model() + asyncio.run(client._build_model()) def test_build_model_uses_base_url_and_key(): @@ -503,7 +503,7 @@ def test_build_model_uses_base_url_and_key(): base_url="https://api.venice.ai/api/v1", api_key="sk-test", ) - model = client._build_model() + model = asyncio.run(client._build_model()) assert model.model_name == "llama-3.3-70b" assert str(model.client.base_url).rstrip("/") == "https://api.venice.ai/api/v1" assert model.client.api_key == "sk-test" @@ -515,7 +515,7 @@ def test_build_model_strips_endpoint_name_from_model_id(): base_url="https://api.venice.ai/api/v1", api_key="sk-test", ) - model = client._build_model() + model = asyncio.run(client._build_model()) assert model.model_name == "meta-llama/Llama-3.3-70B" @@ -523,7 +523,7 @@ def test_build_model_env_fallbacks(monkeypatch): monkeypatch.setenv("CUSTOM_LLM_BASE_URL", "https://env.example/v1") monkeypatch.setenv("CUSTOM_LLM_API_KEY", "sk-env") client = PydanticAIClient(model="custom:m") - model = client._build_model() + model = asyncio.run(client._build_model()) assert str(model.client.base_url).rstrip("/") == "https://env.example/v1" assert model.client.api_key == "sk-env" diff --git a/tests/test_local_model_probe_nonblocking.py b/tests/test_local_model_probe_nonblocking.py new file mode 100644 index 000000000..9ed4a62b1 --- /dev/null +++ b/tests/test_local_model_probe_nonblocking.py @@ -0,0 +1,135 @@ +"""Resolving a bare ``ollama:`` / ``lmstudio:`` key must not park the loop. + +``PydanticAIClient.start()`` builds its model on the one event loop that also +runs Telegram polling, the dashboard, every WebSocket and every other agent +session. When the model id is left to us ("ollama:"), building it means asking +the local backend what it serves — which used to be a synchronous +``urllib.request.urlopen``, freezing everything else for the whole timeout +whenever the backend was down or hung (PERF-331). +""" + +import asyncio +import dataclasses + +import pytest +from aiohttp import web + +import condor.runtime.timeouts as timeouts_mod +from condor.acp.pydantic_ai_client import PydanticAIClient + + +def _short_probe(monkeypatch, budget: float = 0.25) -> None: + """Shrink the probe budget so a hung backend fails in test time.""" + monkeypatch.setattr( + timeouts_mod, + "TIMEOUTS", + dataclasses.replace(timeouts_mod.TIMEOUTS, local_model_probe=budget), + ) + + +async def _ticker(stop: asyncio.Event, counter: list) -> None: + """Stand in for every other coroutine sharing the loop.""" + while not stop.is_set(): + await asyncio.sleep(0.01) + counter.append(1) + + +def test_probe_against_a_hung_backend_leaves_the_loop_responsive(monkeypatch): + _short_probe(monkeypatch) + + async def scenario(): + # A socket that accepts the connection and then never answers: the + # worst case for the caller, because it fails on read, not connect. + release = asyncio.Event() + + async def hang(reader, writer): + await release.wait() + writer.close() + + server = await asyncio.start_server(hang, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + + stop = asyncio.Event() + ticks: list = [] + beat = asyncio.create_task(_ticker(stop, ticks)) + try: + client = PydanticAIClient(model="ollama:") + with pytest.raises(RuntimeError, match="No local model found"): + await client._resolve_default_local_model( + prefix="ollama", base_url=f"http://127.0.0.1:{port}/v1" + ) + finally: + stop.set() + await beat + release.set() + server.close() + await server.wait_closed() + + # The probe was in flight for at least one budget; a blocked loop + # produces zero ticks in that window. + assert ( + len(ticks) >= 10 + ), f"loop was starved during the probe: {len(ticks)} ticks" + + asyncio.run(scenario()) + + +def test_resolution_order_is_unchanged(monkeypatch): + """env override → /v1/models first id → Ollama /api/tags first name.""" + monkeypatch.delenv("CONDOR_DEFAULT_LOCAL_MODEL", raising=False) + monkeypatch.delenv("OLLAMA_MODEL", raising=False) + _short_probe(monkeypatch, budget=2.0) + + async def scenario(): + serve_v1 = {"on": True} + + async def models(_request): + if not serve_v1["on"]: + return web.json_response({"error": "nope"}, status=404) + return web.json_response({"data": [{"id": "openai-compat-model"}]}) + + async def tags(_request): + return web.json_response({"models": [{"name": "native-model"}]}) + + app = web.Application() + app.router.add_get("/v1/models", models) + app.router.add_get("/api/tags", tags) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "127.0.0.1", 0) + await site.start() + port = site._server.sockets[0].getsockname()[1] + base = f"http://127.0.0.1:{port}/v1" + client = PydanticAIClient(model="ollama:") + try: + assert ( + await client._resolve_default_local_model( + prefix="ollama", base_url=base + ) + == "openai-compat-model" + ) + + serve_v1["on"] = False + assert ( + await client._resolve_default_local_model( + prefix="ollama", base_url=base + ) + == "native-model" + ) + # lmstudio never falls back to the Ollama-native endpoint. + with pytest.raises(RuntimeError, match="No local model found"): + await client._resolve_default_local_model( + prefix="lmstudio", base_url=base + ) + + monkeypatch.setenv("CONDOR_DEFAULT_LOCAL_MODEL", "env-wins") + assert ( + await client._resolve_default_local_model( + prefix="ollama", base_url=base + ) + == "env-wins" + ) + finally: + await runner.cleanup() + + asyncio.run(scenario()) From ac4059792dd7d11f7902b792f3faf3f8c6ca82e9 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 14:30:48 +0300 Subject: [PATCH 071/154] (perf) Stop buffering a turn that nobody is streaming any more MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An abandoned answer — a dropped socket, a page reload, a cancelled prompt task — leaves the agent generating into an event queue with no consumer. Every chunk and every tool output of the rest of that turn was kept in RAM, and none of it could ever be delivered: the next prompt drains the queue before it reads anything, as does the local cancel. If no next prompt came, it was held until the idle sweep detached the session an hour later, which one big tool result turns into megabytes per session. Session notifications are now relayed only while a turn is actually being streamed. Terminal events are unaffected: they are put on the queue directly by the read loop, the local cancel and the response callback, so a parked consumer still gets its PromptDone. --- condor/acp/client.py | 13 +++++++ tests/runtime/test_acp_tool_title.py | 1 + tests/runtime/test_prompt_cancel.py | 55 ++++++++++++++++++++++++++++ tests/test_acp_permission_gate.py | 1 + tests/test_acp_tool_call_input.py | 3 ++ 5 files changed, 73 insertions(+) diff --git a/condor/acp/client.py b/condor/acp/client.py index c7e58024a..d526418a4 100644 --- a/condor/acp/client.py +++ b/condor/acp/client.py @@ -1172,6 +1172,19 @@ def _on_session_update( _meta: dict | None = None, **kw: Any, ) -> None: + # Only a turn someone is streaming owns the queue. With no + # ``_current_req_id`` there is no consumer these notifications could + # ever reach: an abandoned turn (a WS drop, a page reload, a cancelled + # prompt) keeps generating, and the next prompt drains the queue before + # it reads a single event — as does ``_cancel_locally``. Buffering it + # would only park the tail of a dead answer, tool outputs and all, in + # RAM until the idle sweep detaches the session an hour later + # (PERF-332). Terminal events never come through here, so a parked + # consumer is still unblocked: the read loop, ``_cancel_locally`` and + # ``_on_response`` put their ``PromptDone`` on the queue directly. + if self._current_req_id is None: + return + kind = update.get("sessionUpdate") if kind == "agent_message_chunk": content = update.get("content", {}) diff --git a/tests/runtime/test_acp_tool_title.py b/tests/runtime/test_acp_tool_title.py index f6fbdf07f..fc22c7942 100644 --- a/tests/runtime/test_acp_tool_title.py +++ b/tests/runtime/test_acp_tool_title.py @@ -68,6 +68,7 @@ def test_a_json_quoted_name_is_unwrapped_not_discarded(): def _drive(update: dict): client = ACPClient(command="true") + client._current_req_id = 1 # a turn is being streamed (PERF-332) client._on_session_update("s", update) return client._event_queue.get_nowait() diff --git a/tests/runtime/test_prompt_cancel.py b/tests/runtime/test_prompt_cancel.py index 827cc776f..86f8b51d1 100644 --- a/tests/runtime/test_prompt_cancel.py +++ b/tests/runtime/test_prompt_cancel.py @@ -500,3 +500,58 @@ async def consume(): assert client._peer._pending == {} assert session.is_busy is False assert session._lock.locked() is False + + +def test_an_abandoned_turn_stops_buffering_into_the_event_queue(): + """PERF-332: what nobody can receive is dropped, not parked in RAM. + + The turn above is abandoned mid-answer and the agent keeps generating. Its + remaining chunks and tool outputs are *provably* undeliverable — the next + prompt drains the queue before it reads anything — so the queue used to + grow with the length of the abandoned turn and hold it until the idle sweep + detached the session an hour later. One big tool result parked megabytes. + """ + client = _client(answers_cancel=True) + + async def scenario(): + events: list = [] + agen = client.prompt_stream("read me the whole log") + pending = asyncio.ensure_future(agen.__anext__()) + await asyncio.sleep(0.05) + client._on_session_update("sess-1", _chunk("half an answer")) + events.append(await asyncio.wait_for(pending, timeout=5)) + + await agen.aclose() # the socket dropped: nobody is reading any more + await asyncio.sleep(0.05) + + # The agent runs on for another hundred chunks and dumps a fat tool + # result nobody asked for. + for i in range(100): + client._on_session_update("sess-1", _chunk(f"tail {i} ")) + client._on_session_update( + "sess-1", + { + "sessionUpdate": "tool_call_update", + "toolCallId": "1", + "status": "completed", + "output": "x" * 100_000, + }, + ) + abandoned = client._event_queue.qsize() + + # ...and the next turn still reads its own words off an empty queue. + after: list = [] + task = asyncio.create_task(_drive(client.prompt_stream("still there?"), after)) + await asyncio.sleep(0.05) + client._on_session_update("sess-1", _chunk("yes")) + await _finish(client) + await asyncio.wait_for(task, timeout=5) + return events, abandoned, after + + events, abandoned, after = asyncio.run(scenario()) + + assert _said(events) == "half an answer" + # Flat, not 101 events deep: the queue never grew with the abandoned turn. + assert abandoned == 0 + assert _said(after) == "yes" + assert after[-1].stop_reason == "end_turn" diff --git a/tests/test_acp_permission_gate.py b/tests/test_acp_permission_gate.py index 47b672077..2dd2e823c 100644 --- a/tests/test_acp_permission_gate.py +++ b/tests/test_acp_permission_gate.py @@ -146,6 +146,7 @@ def test_read_only_acp_call_still_takes_the_fast_path(): def test_session_update_records_the_arguments(): """Transcripts recorded ``"input": null`` for all 123 ACP tool calls.""" client = ACPClient(command="true") + client._current_req_id = 1 # a turn is being streamed (PERF-332) client._on_session_update( "s", { diff --git a/tests/test_acp_tool_call_input.py b/tests/test_acp_tool_call_input.py index 902865e32..83ffb21cb 100644 --- a/tests/test_acp_tool_call_input.py +++ b/tests/test_acp_tool_call_input.py @@ -33,6 +33,9 @@ def _client(): client = ACPClient.__new__(ACPClient) client._event_queue = asyncio.Queue() + # A turn is being streamed: notifications are only relayed for one that is + # (PERF-332). + client._current_req_id = 1 return client From c32ce57ca58920eb775884f4033ee5cc6f1664b7 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 14:38:48 +0300 Subject: [PATCH 072/154] Reap leaked ACP trees from the snapshot already in hand (PERF-333) reap_stale_acp_trees took a full ps snapshot, built parent_of from it, then threw it away: every root re-forked `ps -eo pid=,ppid=` over the whole process table, so N leaked roots cost N extra scans on the boot path, before the event loop exists. Worse than the latency, the later snapshots disagreed with the first: `targets` came from them while the `_protected` guard looks argv up in `args_of` from the FIRST one, so a pid only the later scans saw resolved to "" and was SIGTERM/SIGKILLed without its --dangerously-skip-permissions / claude-code-acp guard ever being read. Invert parent_of into a children map once and walk that. The descent is factored into _descendants_in() so _descendant_pids() -- whose two stop() callers legitimately need a fresh snapshot to see what survived SIGTERM -- keeps taking its own and both share one traversal. The reaper test now fakes `ps` itself instead of monkeypatching _descendant_pids with a hand-rolled walk (that patch existed only to work around this redundancy), so the real snapshot-and-walk code is exercised and every fork is counted. --- condor/acp/client.py | 40 +++++++++++++++----- tests/test_mcp_argv_secrets.py | 69 +++++++++++++++++++++++++++------- 2 files changed, 85 insertions(+), 24 deletions(-) diff --git a/condor/acp/client.py b/condor/acp/client.py index d526418a4..9a1d3c893 100644 --- a/condor/acp/client.py +++ b/condor/acp/client.py @@ -105,13 +105,32 @@ def normalize_tool_call(payload: dict[str, Any]) -> dict[str, Any]: return normalized +def _descendants_in(root: int, children: dict[int, list[int]]) -> set[int]: + """Every transitive child of ``root`` in an already-built ``children`` map. + + The traversal is factored out so a caller that already holds a process-table + snapshot (the startup reaper) walks *that* one instead of forking another + ``ps`` per root — which also kept the walked snapshot from disagreeing with + the one the rest of the caller reasons about (PERF-333). + """ + found: set[int] = set() + stack = [root] + while stack: + for child in children.get(stack.pop(), []): + if child not in found: + found.add(child) + stack.append(child) + return found + + def _descendant_pids(root: int) -> set[int]: """Every transitive child PID of ``root``, from a single ``ps`` snapshot. Used at teardown to find MCP server subprocesses that ``claude`` spawns in their OWN process groups (so ``killpg`` of our group misses them). Must be called BEFORE the parent dies — once it exits the children reparent to init - and the ppid links that identify them are gone. + and the ppid links that identify them are gone. Each call deliberately takes + a FRESH snapshot: ``stop()`` re-scans after SIGTERM to see what survived. """ try: out = subprocess.run( @@ -129,14 +148,7 @@ def _descendant_pids(root: int) -> set[int]: except ValueError: continue children.setdefault(ppid, []).append(pid) - found: set[int] = set() - stack = [root] - while stack: - for child in children.get(stack.pop(), []): - if child not in found: - found.add(child) - stack.append(child) - return found + return _descendants_in(root, children) def _alive(pid: int) -> bool: @@ -252,9 +264,17 @@ def _acp_ish(a: str) -> bool: root = cur = p roots.add(root) + # Walk the snapshot already in hand rather than forking a fresh ``ps`` per + # root: N roots used to mean N extra full process-table scans on the boot + # path, and a pid seen only by one of those later scans had no entry in + # ``args_of`` — so it slipped past the ``_protected`` filter below unread. + children: dict[int, list[int]] = {} + for pid, ppid in parent_of.items(): + children.setdefault(ppid, []).append(pid) + targets: set[int] = set() for root in roots: - targets |= _descendant_pids(root) + targets |= _descendants_in(root, children) targets.add(root) targets = {p for p in targets if not _protected(args_of.get(p, ""))} if not targets: diff --git a/tests/test_mcp_argv_secrets.py b/tests/test_mcp_argv_secrets.py index d9fa9a412..7f2978383 100644 --- a/tests/test_mcp_argv_secrets.py +++ b/tests/test_mcp_argv_secrets.py @@ -7,6 +7,7 @@ still finds them through the non-secret marker that replaced it. """ +import subprocess import sys import pytest @@ -110,29 +111,40 @@ def test_marker_is_per_bot_and_absent_without_a_token(monkeypatch): # ── the reaper still finds our trees without the token on argv ── -def _reap_with_ps(monkeypatch, rows, token=BOT_TOKEN): - """Run the reaper against a fake ``ps`` snapshot; return the pids signalled.""" +def _run_reaper(monkeypatch, rows, token=BOT_TOKEN): + """Run the reaper against a fake process table. + + ``ps`` itself is faked (rather than ``_ps_rows``/``_descendant_pids``) so the + real snapshot-and-walk code runs, and every fork the reaper makes is counted. + Returns ``(pids signalled, argv of each ``ps`` invocation)``. + """ from condor.acp import client as acp_client signalled: list[int] = [] - monkeypatch.setattr(acp_client, "_ps_rows", lambda: rows) - - def _descendants(root: int) -> set[int]: - """Same walk the real helper does, over the fake snapshot.""" - out, frontier = set(), {root} - while frontier: - frontier = {p for p, ppid, _ in rows if ppid in frontier and p not in out} - out |= frontier - return out - - monkeypatch.setattr(acp_client, "_descendant_pids", _descendants) + ps_calls: list[list[str]] = [] + + def _fake_run(cmd, **_kwargs): + ps_calls.append(list(cmd)) + fmt = cmd[-1] + if fmt.endswith("args="): + text = "".join(f"{p} {ppid} {args}\n" for p, ppid, args in rows) + else: + text = "".join(f"{p} {ppid}\n" for p, ppid, _ in rows) + return subprocess.CompletedProcess(cmd, 0, stdout=text, stderr="") + + monkeypatch.setattr(acp_client.subprocess, "run", _fake_run) monkeypatch.setattr( acp_client, "_signal_all", lambda pids, _pg, _sig: signalled.extend(pids) ) monkeypatch.setattr(acp_client, "_alive", lambda _p: False) acp_client.reap_stale_acp_trees(token, wait_s=0) - return set(signalled) + return set(signalled), ps_calls + + +def _reap_with_ps(monkeypatch, rows, token=BOT_TOKEN): + """Run the reaper against a fake ``ps`` snapshot; return the pids signalled.""" + return _run_reaper(monkeypatch, rows, token)[0] def test_reaper_kills_the_tree_seeded_by_the_marker(monkeypatch): @@ -170,6 +182,35 @@ def test_reaper_ignores_another_bots_tree(monkeypatch): assert _reap_with_ps(monkeypatch, rows) == set() +def test_reaper_takes_one_ps_snapshot_however_many_roots(monkeypatch): + """One ``ps`` per reap, not one per root (PERF-333). + + Three independent leaked trees used to mean the boot-path snapshot plus one + full ``ps -eo pid=,ppid=`` fork per root — and a pid seen only by one of + those later snapshots had no argv in the first one, so it slipped past the + ``_protected`` guard unread. Both go away by walking the snapshot in hand. + """ + from condor.acp.client import bot_process_marker + + marker = f"--bot-id {bot_process_marker(BOT_TOKEN)}" + rows = [(999, 1, "some unrelated process")] + expected: set[int] = set() + for root in (100, 200, 300): # three separate acp trees, each nested deep + rows += [ + (root, 1, "node claude-agent-acp"), + (root + 1, root, "claude"), + (root + 2, root + 1, f"uv run python -m mcp_servers.condor {marker}"), + (root + 3, root + 2, "uv run python -m mcp_servers.hummingbot_api"), + ] + expected |= {root, root + 1, root + 2, root + 3} + + signalled, ps_calls = _run_reaper(monkeypatch, rows) + + assert signalled == expected + assert len(ps_calls) == 1, f"{len(ps_calls)} ps forks for 3 roots: {ps_calls}" + assert ps_calls[0] == ["ps", "-eo", "pid=,ppid=,args="] + + # ── the MCP servers still read what the spawner now sends ── From 21faa14a80c7d3e65ec3874a2dcb37f824826525 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 14:47:06 +0300 Subject: [PATCH 073/154] Render search_swaps as a table instead of a dict dump (PERF-564) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit format_gateway_swap_result ended its search branch with `f"Swaps: {swaps}"`, handing the model a Python repr of up to `limit` raw swap dicts — quoted keys repeated on every row, full 88-char transaction hashes, wallet addresses and timestamps. search_swaps defaults to limit=50 and is mounted on every seat including ticks, so a default call spent thousands of tokens on punctuation. Render the page with TableBuilder/ColumnDef instead: time, connector, network, pair, side, in, out, price, status and a truncated tx hash, reusing format_number / format_timestamp / truncate_address. An empty page now says "No swaps found." The header keeps limit/offset and the filters so pagination stays visible, and its count is relabelled "Swaps Returned" — it was the page count, never a total. --- .../hummingbot_api/formatters/gateway.py | 61 +++++++++++++++++-- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/mcp_servers/hummingbot_api/formatters/gateway.py b/mcp_servers/hummingbot_api/formatters/gateway.py index 9c565bfa6..6f54c53fa 100644 --- a/mcp_servers/hummingbot_api/formatters/gateway.py +++ b/mcp_servers/hummingbot_api/formatters/gateway.py @@ -4,6 +4,9 @@ from typing import Any +from .base import format_number, format_timestamp, truncate_address +from .table_builder import ColumnDef, TableBuilder + def format_gateway_container_result(result: dict[str, Any]) -> str: """Format gateway container action results into a human-readable string.""" @@ -111,20 +114,68 @@ def format_gateway_config_result(result: dict[str, Any]) -> str: return f"Gateway Configuration Result: {result}" +def _format_swap_amount(value: Any) -> str: + """Format a swap amount or price without K/M compaction.""" + return format_number(value, decimals=4, compact=False) + + +SWAP_SEARCH_COLUMNS = [ + ColumnDef( + name="time", + key=["timestamp", "created_at"], + width=11, + formatter=format_timestamp, + ), + ColumnDef(name="connector", key="connector", width=14), + ColumnDef(name="network", key="network", width=19), + ColumnDef(name="pair", key="trading_pair", width=13), + ColumnDef(name="side", key="side", width=4), + ColumnDef( + name="in", + key="input_amount", + width=10, + align="right", + formatter=_format_swap_amount, + ), + ColumnDef( + name="out", + key="output_amount", + width=10, + align="right", + formatter=_format_swap_amount, + ), + ColumnDef( + name="price", + key="price", + width=10, + align="right", + formatter=_format_swap_amount, + ), + ColumnDef(name="status", key="status", width=9), + ColumnDef( + name="tx", + key="transaction_hash", + width=17, + formatter=lambda tx_hash: truncate_address(str(tx_hash)), + ), +] + + def format_gateway_swap_result(action: str, result: dict[str, Any]) -> str: """Format gateway swap action results into a human-readable string.""" if action == "search" and isinstance(result, dict): filters = result.get("filters", {}) pagination = result.get("pagination", {}) - swaps = result.get("result", {}).get("data", []) + swaps = result.get("result", {}).get("data", []) or [] - return ( + header = ( f"Gateway Swaps Search Result:\n" - f"Total Swaps Found: {len(swaps)}\n" + f"Swaps Returned: {len(swaps)}\n" f"Limit: {pagination.get('limit', 'N/A')}, Offset: {pagination.get('offset', 'N/A')}\n" - f"Filters: {filters if filters else 'None'}\n\n" - f"Swaps: {swaps}" + f"Filters: {filters if filters else 'None'}" ) + builder = TableBuilder(SWAP_SEARCH_COLUMNS, empty_message="No swaps found.") + return builder.build_with_title(swaps, header) return f"Gateway Swap Result: {result}" From 518225f34c2f5bc50e673273013bd1a5de2a5ba6 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 14:54:37 +0300 Subject: [PATCH 074/154] Stop the Condor MCP server dragging telegram into every spawn (PERF-571) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit available_models reached condor.llm.readiness / .openrouter_models through the handlers.agents alias shims (ARCH-190). Same module objects either way, but the shim route runs the handlers package __init__ first — telegram, condor.acp, utils.auth — on a server that touches none of it, on every MCP subprocess spawn (per chat session, agent instance and loop start), the tick profile included, whose ring does not even mount get_available_models. Import them from condor.llm directly. acp_bridges / local_servers stay module attributes, so the readiness test's monkeypatches are untouched. import mcp_servers.condor.server: 698 ms -> 471 ms, 1869 -> 1559 modules. --- mcp_servers/condor/tools/available_models.py | 12 ++++++---- tests/test_condor_tool_surface.py | 25 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/mcp_servers/condor/tools/available_models.py b/mcp_servers/condor/tools/available_models.py index 5275d597b..5deaf1136 100644 --- a/mcp_servers/condor/tools/available_models.py +++ b/mcp_servers/condor/tools/available_models.py @@ -12,17 +12,21 @@ key's env var is set. An unreachable local server is a normal state (the user simply isn't running it), reported as ``reachable: false``, never raised. -The detection itself lives in :mod:`handlers.agents.readiness`, shared with the +The detection itself lives in :mod:`condor.llm.readiness`, shared with the setup wizard (``condor/setup_llm.py``) so the two can never disagree about what -this machine can run. +this machine can run. It is imported from ``condor.llm`` directly rather than +through the ``handlers.agents`` alias shims (ARCH-190): both are the same module +object, but the shim route drags the whole ``handlers`` package __init__ +(telegram, condor.acp, utils.auth) into every Condor MCP subprocess spawn for +nothing. """ from __future__ import annotations import os -from handlers.agents.openrouter_models import fetch_models -from handlers.agents.readiness import CLOUD_KEY_ENVS, acp_bridges, local_servers +from condor.llm.openrouter_models import fetch_models +from condor.llm.readiness import CLOUD_KEY_ENVS, acp_bridges, local_servers async def _openrouter(query: str, limit: int) -> dict: diff --git a/tests/test_condor_tool_surface.py b/tests/test_condor_tool_surface.py index c7a0d6ed5..e7538c94e 100644 --- a/tests/test_condor_tool_surface.py +++ b/tests/test_condor_tool_surface.py @@ -12,6 +12,7 @@ import asyncio import re import subprocess +import sys from pathlib import Path import pytest @@ -555,3 +556,27 @@ def test_read_routine_refuses_a_routine_file_that_symlinks_out(sandboxed_library assert "a secret" not in str(result) assert result.get("error") == "Routine 'leak' not found" + + +def test_importing_the_server_does_not_drag_in_handlers_or_telegram(): + """PERF-571: every MCP subprocess spawn pays for this import graph. + + ``tools/available_models`` used to reach ``condor.llm.readiness`` through the + ``handlers.agents`` alias shims (ARCH-190), which forced the whole + ``handlers`` package __init__ — telegram, condor.acp, utils.auth — into a + server that never touches any of it (~230 ms, ~310 modules). Asserted in a + fresh interpreter because this module already imports the server in-process. + """ + probe = ( + "import sys, mcp_servers.condor.server;" + "print(int(any(m == 'handlers' or m.startswith('handlers.') " + "or m == 'telegram' or m.startswith('telegram.') for m in sys.modules)))" + ) + done = subprocess.run( + [sys.executable, "-c", probe], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ) + assert done.stdout.strip() == "0", "handlers/telegram imported by the MCP server" From 0e399e78e157d5ec46759e28dc5ce6b5346d66de Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 15:02:07 +0300 Subject: [PATCH 075/154] Stop forcing a full routine re-import on every chat MCP call (PERF-572) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MCP chat seat passed force_reload=True to routine discovery on every manage_routines call — list, run and describe alike — which blanked the mtime cache and so re-imported all 11 modules in routines/ and re-executed every file in both shared roots, rebuilding each pydantic Config and rebinding each run function. The flag was never what made the chat's own edits visible: discovery is mtime-keyed, so it re-imports an edited file, loads a new one and drops a deleted one on the next call regardless. Hot reload is unchanged; the warm call now imports nothing. --- mcp_servers/condor/tools/routines.py | 11 +- tests/test_mcp_chat_routine_cache.py | 153 +++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 tests/test_mcp_chat_routine_cache.py diff --git a/mcp_servers/condor/tools/routines.py b/mcp_servers/condor/tools/routines.py index 1ff161876..5497671f2 100644 --- a/mcp_servers/condor/tools/routines.py +++ b/mcp_servers/condor/tools/routines.py @@ -179,13 +179,14 @@ def _own_plus_shared(slug: str | None) -> dict: One call for both seats (FEAT-038): a domain expert/trading agent gets ``agents//routines`` over ``agents/_shared/routines``, the chat gets the general library — which ``discover_routines`` already merges the shared - root into. The chat re-scans on every call because it is the library's - author and its edits must be visible immediately; an agent rides the mtime - cache. + root into. Both seats ride the mtime cache: discovery re-imports an edited + file, loads a new one and drops a deleted one on every call (PERF-572), so + the chat's own edits to its library are visible immediately without paying + a full re-import of every routine per tool call. """ from routines.base import assistant_routines - return assistant_routines(slug, force_reload=not slug) + return assistant_routines(slug) def _resolve_routine(name: str): @@ -223,7 +224,7 @@ def list_routines(target: str | None = None) -> dict: return {"routines": result} # Chat condor: the general library (root routines/). - for name, routine in sorted(discover_routines(force_reload=True).items()): + for name, routine in sorted(discover_routines().items()): result.append( { "name": name, diff --git a/tests/test_mcp_chat_routine_cache.py b/tests/test_mcp_chat_routine_cache.py new file mode 100644 index 000000000..b06e0d71f --- /dev/null +++ b/tests/test_mcp_chat_routine_cache.py @@ -0,0 +1,153 @@ +"""PERF-572: the MCP chat seat rides the mtime cache instead of forcing a reload. + +``_own_plus_shared`` and ``list_routines``' chat branch used to pass +``force_reload=True``, so every ``manage_routines`` call — ``list``, ``run``, +``describe`` — re-imported every module in ``routines/`` and re-executed every +file in the shared roots. The mtime cache already gives the chat what the flag +was there for: an edited file is re-imported, a new one is loaded and a deleted +one is dropped on the next call, with no MCP subprocess restart. +""" + +import pytest + +import condor.memory.paths as paths_mod +import routines.base as base +from condor.memory.paths import CHAT_SLUG + +ROUTINE_TEMPLATE = ''' +from pydantic import BaseModel + +with open({sentinel!r}, "a") as f: + f.write("exec\\n") + + +class Config(BaseModel): + """{desc}""" + value: int = 1 + + +async def run(config, context): + return "ok" +''' + + +def _write(dir_path, name, sentinel, desc="a routine"): + dir_path.mkdir(parents=True, exist_ok=True) + path = dir_path / f"{name}.py" + path.write_text(ROUTINE_TEMPLATE.format(sentinel=str(sentinel), desc=desc)) + return path + + +def _bump_mtime(path): + import os + + stat = path.stat() + os.utime(path, (stat.st_atime, stat.st_mtime + 10)) + + +def _execs(sentinel): + return sentinel.read_text().count("exec") if sentinel.exists() else 0 + + +@pytest.fixture +def chat_seat(tmp_path, monkeypatch): + """The MCP chat seat, with the shared root redirected under ``tmp_path``. + + The chat's *own* library stays the repo's real ``routines/`` — its modules + are imported as the ``routines`` package and cannot be relocated — so the + per-file assertions below are made on a shared routine, which + ``discover_routines`` merges into the very same general library. + """ + from mcp_servers.condor.tools import routines as mcp_routines + + monkeypatch.setattr(base, "_PROJECT_ROOT", tmp_path) + monkeypatch.setattr(base, "_routines_cache", None) + monkeypatch.setattr(base, "_routines_mtimes", {}) + monkeypatch.setattr(base, "_path_caches", {}) + monkeypatch.setattr(mcp_routines.settings, "agent_slug", CHAT_SLUG) + return mcp_routines + + +def _names(listed): + return {r["name"] for r in listed["routines"]} + + +class TestWarmCallsDoNotReimport: + def test_second_list_reimports_nothing(self, chat_seat, tmp_path, monkeypatch): + sentinel = tmp_path / "execs.txt" + _write(paths_mod.shared_routines_root(), "published", sentinel) + + chat_seat.list_routines() # warm + warm = _execs(sentinel) + assert warm >= 1 + + calls = [] + monkeypatch.setattr( + base.importlib, "reload", lambda m: calls.append(m.__name__) + ) + listed = chat_seat.list_routines() + + # Zero re-imports of the chat's own library, zero re-executions of the + # shared one — and the catalog is unchanged. + assert calls == [] + assert _execs(sentinel) == warm + assert "published" in _names(listed) + + def test_resolving_a_routine_to_run_reimports_nothing( + self, chat_seat, tmp_path, monkeypatch + ): + """``run``/``describe`` funnel through ``_resolve_routine`` too.""" + sentinel = tmp_path / "execs.txt" + _write(paths_mod.shared_routines_root(), "published", sentinel) + + assert chat_seat._resolve_routine("published") is not None # warm + warm = _execs(sentinel) + + calls = [] + monkeypatch.setattr( + base.importlib, "reload", lambda m: calls.append(m.__name__) + ) + assert chat_seat._resolve_routine("published") is not None + + assert calls == [] + assert _execs(sentinel) == warm + + +class TestEditsAreStillVisibleWithoutARestart: + def test_edited_file_is_reflected_on_the_next_call(self, chat_seat, tmp_path): + sentinel = tmp_path / "execs.txt" + path = _write( + paths_mod.shared_routines_root(), "published", sentinel, desc="before" + ) + + chat_seat.list_routines() + assert chat_seat.describe_routine("published")["description"] == "before" + + path.write_text(ROUTINE_TEMPLATE.format(sentinel=str(sentinel), desc="after")) + _bump_mtime(path) + + assert chat_seat.describe_routine("published")["description"] == "after" + listed = {r["name"]: r for r in chat_seat.list_routines()["routines"]} + assert listed["published"]["description"] == "after" + + def test_new_file_is_reflected_on_the_next_call(self, chat_seat, tmp_path): + sentinel = tmp_path / "execs.txt" + _write(paths_mod.shared_routines_root(), "first", sentinel) + + assert "second" not in _names(chat_seat.list_routines()) + + _write(paths_mod.shared_routines_root(), "second", sentinel) + + assert "second" in _names(chat_seat.list_routines()) + assert chat_seat._resolve_routine("second") is not None + + def test_deleted_file_is_dropped_on_the_next_call(self, chat_seat, tmp_path): + sentinel = tmp_path / "execs.txt" + path = _write(paths_mod.shared_routines_root(), "published", sentinel) + + assert "published" in _names(chat_seat.list_routines()) + + path.unlink() + + assert "published" not in _names(chat_seat.list_routines()) + assert chat_seat._resolve_routine("published") is None From aa9839134840e7ab9e0a94f4311b7a1f2a61a230 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 15:11:24 +0300 Subject: [PATCH 076/154] (perf) describe a controller in two round trips instead of five explore_controllers opened with list_controllers() then list_controller_configs() awaited one after the other, and the describe branch then chained get_controller_config(), get_controller_config_template() and get_controller() the same way -- five serial round trips to an API host that is usually remote, where only two of them depend on anything at all. The calls now go out in the two waves the dependency graph actually has: both lists plus the named config (which only feeds controller_name resolution) together, then the template plus the source, which both need the controller type and neither of which needs the other. The configs list is indexed by controller_name once instead of being rescanned per controller. _gather_calls uses return_exceptions=True rather than gather's default: the default propagates the first failure while its siblings keep running unawaited, so a second failure surfaces later as "Task exception was never retrieved" with nothing naming it. Every leg settles, then the first error in call order is re-raised -- the same error the sequential code produced. A leg handed back as CancelledError was cancelled by someone else, so it is reported as a plain failure instead of telling the caller it is being cancelled; our own cancellation still arrives through the await. The new tests record which calls are in flight together, so they fail against the sequential version rather than passing on both. --- .../hummingbot_api/tools/controllers.py | 92 +++++-- tests/test_explore_controllers_concurrency.py | 226 ++++++++++++++++++ 2 files changed, 299 insertions(+), 19 deletions(-) create mode 100644 tests/test_explore_controllers_concurrency.py diff --git a/mcp_servers/hummingbot_api/tools/controllers.py b/mcp_servers/hummingbot_api/tools/controllers.py index 85826c0ec..8343d3d23 100644 --- a/mcp_servers/hummingbot_api/tools/controllers.py +++ b/mcp_servers/hummingbot_api/tools/controllers.py @@ -5,6 +5,7 @@ configurations, including exploration, modification, and bot deployment. """ +import asyncio from typing import Any, Literal # Internal/auto-managed fields that should be skipped during schema validation @@ -17,6 +18,36 @@ } +async def _gather_calls(*awaitables: Any) -> list[Any]: + """Await independent API calls concurrently, preserving sequential failure. + + ``return_exceptions=True`` rather than gather's default: with the default, + the first failure propagates immediately while its siblings keep running + unawaited, so a second failure surfaces later as asyncio's "Task exception + was never retrieved" with nothing naming it. Here every leg settles first + and the first exception *in call order* is re-raised, which is exactly the + error the sequential code produced. + + A ``CancelledError`` handed back as a *result* means some other party + cancelled that leg -- not that this task is shutting down. Re-raising it + verbatim would make our caller read a leg's cancellation as its own + (CORR-332), so it is only propagated when this task is genuinely being + cancelled (CORR-601's ``cancelling()`` check); otherwise it is reported as + the plain failure it is. Our own cancellation still arrives the normal way: + ``gather`` cancels the legs and re-raises out of the ``await`` below. + """ + results = await asyncio.gather(*awaitables, return_exceptions=True) + for result in results: + if isinstance(result, asyncio.CancelledError): + current = asyncio.current_task() + if current is not None and current.cancelling(): + raise result + raise RuntimeError("Controller API call was cancelled") from result + if isinstance(result, BaseException): + raise result + return list(results) + + def _validate_config_against_template( config_data: dict[str, Any], template: dict[str, Any], @@ -147,9 +178,27 @@ async def explore_controllers( Returns: Dictionary containing exploration results and formatted output """ - # List all controllers and their configs - controllers = await client.controllers.list_controllers() - configs = await client.controllers.list_controller_configs() + # Wave 1: everything that depends on nothing. Both lists are independent + # of each other, and the named config only feeds controller_name resolution + # in the describe branch below, so all three travel together instead of + # costing three serial round trips to a usually remote API host. + wants_config = action == "describe" and bool(config_name) + wave_1: list[Any] = [ + client.controllers.list_controllers(), + client.controllers.list_controller_configs(), + ] + if wants_config: + wave_1.append(client.controllers.get_controller_config(config_name)) + + wave_1_results = await _gather_calls(*wave_1) + controllers = wave_1_results[0] + configs = wave_1_results[1] + config = wave_1_results[2] if wants_config else None + + # One pass over the configs instead of one rescan per controller below. + configs_by_controller: dict[Any, list[dict[str, Any]]] = {} + for cfg in configs: + configs_by_controller.setdefault(cfg.get("controller_name"), []).append(cfg) if action == "list": result = "Available Controllers:\n\n" @@ -158,9 +207,7 @@ async def explore_controllers( continue result += f"Controller Type: {c_type}\n" for controller in controller_list: - controller_configs = [ - c for c in configs if c.get("controller_name") == controller - ] + controller_configs = configs_by_controller.get(controller, []) result += f"- {controller} ({len(controller_configs)} configs)\n" if len(controller_configs) > 0: for config in controller_configs: @@ -175,11 +222,9 @@ async def explore_controllers( elif action == "describe": result = "" - config = None - # Get config if specified — show config details directly + # Config details (fetched in wave 1) — show them directly if config_name: - config = await client.controllers.get_controller_config(config_name) if config: if controller_name and controller_name != config.get("controller_name"): controller_name = config.get("controller_name") @@ -212,22 +257,31 @@ async def explore_controllers( "formatted_output": f"Controller '{controller_name}' not found.", } - # Get config template (lightweight — just parameter schema) + # Wave 2: the template (lightweight — just the parameter schema) and, + # only when explicitly requested, the full source. Both need + # found_controller_type, and neither needs the other. + wave_2: list[Any] = [ + client.controllers.get_controller_config_template( + found_controller_type, controller_name + ) + ] + if include_code: + wave_2.append( + client.controllers.get_controller( + found_controller_type, controller_name + ) + ) + wave_2_results = await _gather_calls(*wave_2) + template = wave_2_results[0] + controller_code_content = wave_2_results[1] if include_code else None + controller_configs = [ - c.get("id") for c in configs if c.get("controller_name") == controller_name + c.get("id") for c in configs_by_controller.get(controller_name, []) ] - template = await client.controllers.get_controller_config_template( - found_controller_type, controller_name - ) result += f"Controller: {controller_name} ({found_controller_type})\n\n" - # Only fetch and include full source code when explicitly requested - controller_code_content = None if include_code: - controller_code_content = await client.controllers.get_controller( - found_controller_type, controller_name - ) result += f"Controller Code:\n{controller_code_content}\n\n" # Format config template parameters as table diff --git a/tests/test_explore_controllers_concurrency.py b/tests/test_explore_controllers_concurrency.py new file mode 100644 index 000000000..8b62d0fd6 --- /dev/null +++ b/tests/test_explore_controllers_concurrency.py @@ -0,0 +1,226 @@ +"""``explore_controllers`` fans its independent API calls out, not one by one. + +A single ``manage_controllers(action="describe", config_name=..., include_code=True)`` +used to cost five serial round trips to the Hummingbot API — usually a remote +host — even though only two of them depend on anything: the config template and +the source both need the controller *type*, and nothing else needs anything. +PERF-573 collapsed that into two waves. + +The assertions here are about overlap, not just about the answer: the fake +client records which calls are in flight simultaneously, so the tests fail +against the sequential implementation (every observed in-flight set is a +singleton) rather than passing on both. + +The repo has no async test setup, so the coroutines are driven with +asyncio.run() instead of a pytest-asyncio marker. +""" + +import asyncio +import time + +import pytest + +from mcp_servers.hummingbot_api.tools.controllers import explore_controllers + +CONTROLLERS = { + "market_making": ["pmm_simple", "pmm_dynamic"], + "directional_trading": ["macd_bb_v1"], +} + +CONFIGS = [ + {"id": "pmm_simple_sol", "controller_name": "pmm_simple"}, + {"id": "pmm_simple_btc", "controller_name": "pmm_simple"}, + {"id": "macd_eth", "controller_name": "macd_bb_v1"}, +] + +TEMPLATE = { + "id": {"type": "str", "default": None}, + "spread": {"type": "float", "default": 0.001}, +} + + +class RecordingControllers: + """Records, for every call, the set of calls in flight when it started.""" + + def __init__(self, delay: float = 0.05, fail: dict | None = None): + self.delay = delay + self.fail = fail or {} + self._inflight: set[str] = set() + self.overlaps: list[frozenset[str]] = [] + self.calls: list[str] = [] + self.finished: list[str] = [] + + async def _call(self, name: str, result): + self.calls.append(name) + self._inflight.add(name) + self.overlaps.append(frozenset(self._inflight)) + try: + await asyncio.sleep(self.delay) + if name in self.fail: + raise self.fail[name] + return result + finally: + self._inflight.discard(name) + self.finished.append(name) + + async def list_controllers(self): + return await self._call("list_controllers", CONTROLLERS) + + async def list_controller_configs(self): + return await self._call("list_controller_configs", CONFIGS) + + async def get_controller_config(self, config_name): + return await self._call( + "get_controller_config", + {"id": config_name, "controller_name": "pmm_simple", "spread": 0.002}, + ) + + async def get_controller_config_template(self, controller_type, controller_name): + return await self._call("get_controller_config_template", TEMPLATE) + + async def get_controller(self, controller_type, controller_name): + return await self._call("get_controller", "class PMMSimple: ...") + + +class FakeClient: + def __init__(self, controllers): + self.controllers = controllers + + +def _run(controllers, **kwargs): + return asyncio.run(explore_controllers(client=FakeClient(controllers), **kwargs)) + + +def test_describe_issues_two_waves_not_five_serial_calls(): + """The five calls of a full describe overlap into two waves.""" + fake = RecordingControllers() + started = time.monotonic() + result = _run( + fake, + action="describe", + config_name="pmm_simple_sol", + include_code=True, + ) + elapsed = time.monotonic() - started + + assert len(fake.calls) == 5 + # Wave 1: both lists and the named config were in flight together. + assert ( + frozenset( + {"list_controllers", "list_controller_configs", "get_controller_config"} + ) + in fake.overlaps + ), f"wave 1 never overlapped: {fake.overlaps}" + # Wave 2: template and source were in flight together. + assert ( + frozenset({"get_controller_config_template", "get_controller"}) in fake.overlaps + ), f"wave 2 never overlapped: {fake.overlaps}" + # Two waves of 50ms, not five: allow plenty of slack for a loaded CI box. + assert elapsed < 5 * fake.delay, f"{elapsed:.3f}s looks sequential" + + assert result["controller_name"] == "pmm_simple" + assert result["controller_type"] == "market_making" + assert result["template"] == TEMPLATE + assert result["configs"] == ["pmm_simple_sol", "pmm_simple_btc"] + assert result["controller_code"] == "class PMMSimple: ..." + + +def test_list_still_issues_only_the_two_lists_and_overlaps_them(): + """The list branch never fetches a config, and its two lists overlap.""" + fake = RecordingControllers() + result = _run(fake, action="list") + + assert fake.calls == ["list_controllers", "list_controller_configs"] + assert ( + frozenset({"list_controllers", "list_controller_configs"}) in fake.overlaps + ), f"the two lists never overlapped: {fake.overlaps}" + # Configs are indexed once and read per controller, with the same grouping. + assert "- pmm_simple (2 configs)" in result["formatted_output"] + assert " - pmm_simple_sol\n" in result["formatted_output"] + assert "- pmm_dynamic (0 configs)" in result["formatted_output"] + assert "- macd_bb_v1 (1 configs)" in result["formatted_output"] + + +def test_describe_without_code_keeps_the_source_call_out_of_the_wave(): + fake = RecordingControllers() + result = _run(fake, action="describe", controller_name="pmm_simple") + + assert "get_controller" not in fake.calls + assert "get_controller_config" not in fake.calls + assert "controller_code" not in result + assert result["config_details"] is None + assert "Tip: Set include_code=True" in result["formatted_output"] + + +def test_describe_output_orders_the_sections_as_before(): + fake = RecordingControllers(delay=0) + out = _run( + fake, + action="describe", + config_name="pmm_simple_sol", + include_code=True, + )["formatted_output"] + + assert out.index("Config 'pmm_simple_sol' Details:") < out.index("Controller: ") + assert out.index("Controller: ") < out.index("Controller Code:") + assert out.index("Controller Code:") < out.index("Configuration Parameters:") + assert out.index("Configuration Parameters:") < out.index("Total Configs: 2") + + +def test_a_failing_leg_still_lets_its_siblings_settle(): + """One failure loses the whole answer, as before — but leaves no orphan. + + ``gather``'s default would propagate the first exception while the other + legs kept running unawaited; here every leg finishes before the error is + raised, and the error raised is the first *in call order*, which is the one + the sequential code produced. + """ + boom = RuntimeError("controllers listing is down") + fake = RecordingControllers( + fail={"list_controllers": boom, "list_controller_configs": ValueError("later")} + ) + + with pytest.raises(RuntimeError) as excinfo: + _run(fake, action="describe", config_name="pmm_simple_sol") + + assert excinfo.value is boom + assert set(fake.finished) == { + "list_controllers", + "list_controller_configs", + "get_controller_config", + } + + +def test_a_cancelled_leg_is_not_reported_as_our_own_cancellation(): + """A leg cancelled by someone else must not read as this task shutting down. + + ``gather(return_exceptions=True)`` hands a cancelled child back as a + ``CancelledError`` *result*. Re-raising it verbatim would tell the caller it + is being cancelled (CORR-332), so it surfaces as a plain failure instead. + """ + fake = RecordingControllers( + fail={"get_controller_config": asyncio.CancelledError()} + ) + + with pytest.raises(RuntimeError) as excinfo: + _run(fake, action="describe", config_name="pmm_simple_sol") + + assert not isinstance(excinfo.value, asyncio.CancelledError) + assert isinstance(excinfo.value.__cause__, asyncio.CancelledError) + + +def test_our_own_cancellation_still_propagates(): + """Cancelling the caller cancels the legs and raises CancelledError.""" + + async def scenario(): + fake = RecordingControllers(delay=5) + task = asyncio.ensure_future( + explore_controllers(client=FakeClient(fake), action="list") + ) + await asyncio.sleep(0.05) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert task.cancelled() + + asyncio.run(scenario()) From 6e620d88ae54653b590cf93b5275e678416ddec7 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 15:21:29 +0300 Subject: [PATCH 077/154] Ask the server for its controller performance once, not four times MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_latest_controller_performance() is a whole-server call: the latest row for every controller of every bot the API ever orchestrated, identical no matter who asks. condor/fetchers/bot_performance.py already cached and coalesced it, but four callers went around that wrapper and issued the same request themselves — the bots-page enrichment (every 30s poll, per open tab), the /controller-performance/latest route, the terminated-controllers route and the 30s WS poller — because the cache only held the aggregate-by-bot rollup and they all want the raw rows. So the cache is now two layers over the one round-trip: fetch_latest_snapshots() holds the rows, and fetch_all_bot_performance() memoises its aggregate on top, which keeps the rollup from re-aggregating on every hit. Both stamp the entry when the value is inserted, never before the fetch starts, so a round-trip slower than the 5s TTL cannot write an entry that is already stale. A fetch that raises is cached by neither layer: the exception reaches every waiter and the next caller retries, so nothing here hands back a previous value to paper over a failure. The enrichment call runs under a wait_for, where joining a shared in-flight task bare would cancel it for every other waiter — SingleFlight shields it, so a timeout there abandons only its own caller. /controller-performance/latest?bot_name=X is the one caller whose request is not whole-server, and it keeps its own filtered round-trip; the shared cache is never sliced to answer it. --- condor/fetchers/__init__.py | 12 +- condor/fetchers/bot_performance.py | 58 +++- condor/fetchers/bots.py | 14 +- condor/web/routes/controller_performance.py | 26 +- condor/web/streams/hummingbot_ws.py | 9 +- .../test_controller_perf_snapshot_sharing.py | 255 ++++++++++++++++++ 6 files changed, 353 insertions(+), 21 deletions(-) create mode 100644 tests/test_controller_perf_snapshot_sharing.py diff --git a/condor/fetchers/__init__.py b/condor/fetchers/__init__.py index f483d8142..1554e7075 100644 --- a/condor/fetchers/__init__.py +++ b/condor/fetchers/__init__.py @@ -19,10 +19,16 @@ depend on who asks) or its subject is immutable. Every such cache must be keyed so one server's — or one Gateway network's — answer can never be served for another, carry a comment saying why the caller cannot hold it - instead, and be listed here. There are six today: + instead, and be listed here. There are seven today: - * ``bot_performance._snapshot_cache`` — whole-server controller - performance, ``_SNAPSHOT_TTL`` 5s, in-flight coalesced, + * ``bot_performance._raw_snapshot_cache`` — the whole-server + controller-performance rows as fetched (``fetch_latest_snapshots``), + ``_SNAPSHOT_TTL`` 5s, in-flight coalesced, ``clear_snapshot_cache()``. + Shared by the bots-page enrichment, both controller-performance routes + and the WS poller, which used to issue the same request four times over. + * ``bot_performance._snapshot_cache`` — those same rows aggregated by bot + (``fetch_all_bot_performance``), memoised on top of the raw layer so a + hit re-runs neither the round-trip nor the aggregation. Same TTL and ``clear_snapshot_cache()``. * ``bot_performance._archived_cache`` — the archived-database listing, ``_ARCHIVED_TTL`` 60s, ``clear_archived_cache()``. diff --git a/condor/fetchers/bot_performance.py b/condor/fetchers/bot_performance.py index e82d9001e..af10f42ce 100644 --- a/condor/fetchers/bot_performance.py +++ b/condor/fetchers/bot_performance.py @@ -189,9 +189,20 @@ def _aggregate_by_bot(snapshots: list[dict]) -> dict[str, dict]: # in-flight coalescing collapses that burst into one round-trip and one # aggregation shared by every caller, while staying far fresher than the 30s TTL # the agents route already tolerates above this call. +# +# Two layers over the one round-trip, because the callers want two different +# shapes of the same payload: the web routes and streams want the raw snapshot +# rows, the agents rollup wants them aggregated by bot. Caching only the raw +# rows would re-run the aggregation on every ``fetch_all_bot_performance`` hit, +# and caching only the aggregate is what left the raw callers issuing their own +# byte-identical whole-server request (PERF-579). Both layers are stamped when +# the value is *inserted*, never before the fetch starts, so a round-trip slower +# than the TTL cannot write an entry that is already stale (CORR-584). _SNAPSHOT_TTL = 5.0 _snapshot_cache: dict[str, tuple[float, dict[str, dict]]] = {} _snapshot_inflight = SingleFlight() +_raw_snapshot_cache: dict[str, tuple[float, list[dict]]] = {} +_raw_snapshot_inflight = SingleFlight() def _server_key(client: Any) -> str: @@ -206,14 +217,55 @@ def _server_key(client: Any) -> str: def clear_snapshot_cache() -> None: - """Drop every cached whole-server snapshot (tests, server reconfiguration).""" + """Drop every cached whole-server snapshot (tests, server reconfiguration). + + Empties both layers — the raw rows and the aggregate built from them — so a + test that clears the cache cannot have one layer serve the other's stale + answer. + """ _snapshot_cache.clear() _snapshot_inflight.clear() + _raw_snapshot_cache.clear() + _raw_snapshot_inflight.clear() -async def _fetch_and_aggregate(client: Any) -> dict[str, dict]: +async def _fetch_snapshots(client: Any) -> list[dict]: result = await client.bot_orchestration.get_latest_controller_performance() - return _aggregate_by_bot(extract_snapshots(result)) + return extract_snapshots(result) + + +async def fetch_latest_snapshots(client: Any) -> list[dict]: + """Return the latest controller-performance snapshot rows for the whole server. + + One row per controller of every bot the API has ever orchestrated — the + finished ones included, since the rows outlive the bot. No filter argument: + this is the whole-server call whose payload is identical for every caller, + which is exactly what makes it cacheable. A caller that needs one bot's rows + must issue its own filtered request rather than take this cache. + + Cached per server for ``_SNAPSHOT_TTL`` seconds and coalesced while in + flight, so the bots-page enrichment, the ``/controller-performance/latest`` + route, the terminated-controllers route and the WS poller share one + round-trip instead of issuing four. A fetch that raises is never cached: the + exception propagates to every waiter and the next call retries — this cache + never hands back a previous value to paper over a failure. The returned list + is shared between callers and must be treated as read-only. + """ + key = _server_key(client) + if not key: + return await _fetch_snapshots(client) + + entry = _raw_snapshot_cache.get(key) + if entry is not None and time.monotonic() - entry[0] <= _SNAPSHOT_TTL: + return entry[1] + + snapshots = await _raw_snapshot_inflight.run(key, lambda: _fetch_snapshots(client)) + _raw_snapshot_cache[key] = (time.monotonic(), snapshots) + return snapshots + + +async def _fetch_and_aggregate(client: Any) -> dict[str, dict]: + return _aggregate_by_bot(await fetch_latest_snapshots(client)) async def fetch_all_bot_performance(client: Any) -> dict[str, dict]: diff --git a/condor/fetchers/bots.py b/condor/fetchers/bots.py index 119e369af..4d1393c8a 100644 --- a/condor/fetchers/bots.py +++ b/condor/fetchers/bots.py @@ -4,7 +4,7 @@ import logging from typing import Any, NamedTuple, Optional -from condor.fetchers.bot_performance import extract_snapshots as _extract_perf_snapshots +from condor.fetchers.bot_performance import fetch_latest_snapshots logger = logging.getLogger(__name__) @@ -352,11 +352,17 @@ async def _get_one_run(bn: str): async def _fetch_latest_perf(client) -> dict[str, dict]: - """Latest controller performance snapshots from the DB, keyed by controller id.""" + """Latest controller performance snapshots from the DB, keyed by controller id. + + Read through the shared whole-server cache: this runs on every 30s bots poll + of every open tab, against the same payload the ``/controller-performance`` + routes and the WS poller ask for. The join is safe under the enrichment + ``wait_for`` because the shared fetch is shielded — a timeout here abandons + this caller, never the round-trip the other waiters are on. + """ perf_map: dict[str, dict] = {} try: - perf_result = await client.bot_orchestration.get_latest_controller_performance() - for snap in _extract_perf_snapshots(perf_result): + for snap in await fetch_latest_snapshots(client): cid = snap.get("controller_id", "") if cid: perf_map[cid] = snap diff --git a/condor/web/routes/controller_performance.py b/condor/web/routes/controller_performance.py index 6bd88b55a..4739b3188 100644 --- a/condor/web/routes/controller_performance.py +++ b/condor/web/routes/controller_performance.py @@ -11,6 +11,7 @@ from condor.fetchers.bot_performance import ( fetch_all_bot_performance, fetch_archived_paths, + fetch_latest_snapshots, ) from condor.fetchers.performance_history import ( PerformanceHistoryUnsupported, @@ -277,15 +278,26 @@ async def get_latest_controller_performance( bot_name: Optional[str] = Query(None), user: WebUser = Depends(require_server_access), ): - """Get the most recent performance snapshot for each bot/controller.""" + """Get the most recent performance snapshot for each bot/controller. + + Unfiltered, this is the whole-server call every other controller-performance + caller also makes, so it is served from the shared 5s cache. A ``bot_name`` + filter is a different, narrower request and keeps its own round-trip — the + whole-server cache is never sliced to answer it. + """ cm = get_config_manager() client = await cm.get_client(name) try: - result = await client.bot_orchestration.get_latest_controller_performance( - bot_name=bot_name, - ) + if bot_name is None: + snapshots = await fetch_latest_snapshots(client) + else: + snapshots = _extract_snapshots( + await client.bot_orchestration.get_latest_controller_performance( + bot_name=bot_name, + ) + ) except Exception as e: logger.warning( "Failed to fetch latest controller performance from '%s': %s", name, e @@ -295,8 +307,6 @@ async def get_latest_controller_performance( error_hint=f"Connection error: {e}", ) - snapshots = _extract_snapshots(result) - return ControllerPerformanceLatestResponse( snapshots=[ControllerPerformanceSnapshot.from_raw(s) for s in snapshots], ) @@ -585,7 +595,7 @@ async def get_terminated_controllers( client = await cm.get_client(name) async def _fetch_latest(): - return await client.bot_orchestration.get_latest_controller_performance() + return await fetch_latest_snapshots(client) async def _fetch_runs(): return await client.bot_orchestration.get_bot_runs(limit=limit) @@ -600,7 +610,7 @@ async def _fetch_runs(): ) runs = [_parse_bot_run(r) for r in _extract_runs_list(runs_raw)] - controllers, runs_seen = terminated_controllers(_extract_snapshots(latest), runs) + controllers, runs_seen = terminated_controllers(latest, runs) # A run older than the snapshot table's retention floor has rows for none of # its controllers. Its deployment still named them, and a run with no leaf diff --git a/condor/web/streams/hummingbot_ws.py b/condor/web/streams/hummingbot_ws.py index 3135c62af..812b85843 100644 --- a/condor/web/streams/hummingbot_ws.py +++ b/condor/web/streams/hummingbot_ws.py @@ -590,6 +590,7 @@ async def _controller_perf_stream(self, channel: str) -> None: return server_name = parts[1] + from condor.fetchers.bot_performance import fetch_latest_snapshots from config_manager import get_config_manager cm = get_config_manager() @@ -606,9 +607,11 @@ async def _controller_perf_stream(self, channel: str) -> None: return client = await cm.get_client(server_name) - result = ( - await client.bot_orchestration.get_latest_controller_performance() - ) + # Shared whole-server cache: this 30s poll asks for the very + # payload the bots-page enrichment and the controller-performance + # routes ask for, so a poll that coincides with one of them costs + # no round-trip at all. + result = await fetch_latest_snapshots(client) snapshots = self._transform_controller_perf(result) diff --git a/tests/test_controller_perf_snapshot_sharing.py b/tests/test_controller_perf_snapshot_sharing.py new file mode 100644 index 000000000..ad0b0774f --- /dev/null +++ b/tests/test_controller_perf_snapshot_sharing.py @@ -0,0 +1,255 @@ +"""PERF-579: every whole-server controller-performance caller shares one fetch. + +``get_latest_controller_performance()`` is a whole-server call — the latest row +for every controller of every bot the API ever orchestrated — so its payload is +identical no matter who asks. Four call sites used to issue it independently: +the bots-page enrichment (every 30s poll, per open tab), the +``/controller-performance/latest`` route, the terminated-controllers route and +the 30s WS poller. This file pins that they now go through the one cached, +coalesced fetcher, and that the one caller whose request is *not* whole-server — +``?bot_name=X`` — still makes its own filtered round-trip. +""" + +from __future__ import annotations + +import asyncio + +import pytest + +import condor.web.routes.controller_performance as cp +from condor.fetchers.bot_performance import ( + clear_snapshot_cache, + fetch_all_bot_performance, + fetch_latest_snapshots, +) +from condor.fetchers.bots import _fetch_latest_perf +from condor.web.streams.hummingbot_ws import HummingbotStreamsMixin + +SNAPSHOTS = [ + { + "bot_name": "gan", + "controller_id": "c1", + "performance": { + "realized_pnl_quote": 10.0, + "unrealized_pnl_quote": 1.0, + "global_pnl_quote": 11.0, + "volume_traded": 500.0, + "close_type_counts": {}, + }, + }, +] + +RUNS = [ + { + "bot_name": "gan", + "deployed_at": "2026-08-21T18:05:02+00:00", + "stopped_at": "2026-08-25T05:46:25+00:00", + "deployment_status": "ARCHIVED", + "run_status": "STOPPED", + "deployment_config": '{"controllers_config": ["c1"]}', + }, +] + + +class CountingClient: + """A client that records every whole-server and every filtered request.""" + + base_url = "http://perf-579" + + def __init__(self): + self.whole_server_calls = 0 + self.filtered_calls = 0 + self.bot_orchestration = self + + async def get_latest_controller_performance(self, bot_name=None, **_kw): + if bot_name is None: + self.whole_server_calls += 1 + else: + self.filtered_calls += 1 + return SNAPSHOTS + + async def get_bot_runs(self, **_kw): + return RUNS + + +@pytest.fixture(autouse=True) +def _clean_caches(): + """Both the shared snapshot cache and the route's own 60s cache.""" + clear_snapshot_cache() + cp._terminated_cache.clear() + yield + clear_snapshot_cache() + cp._terminated_cache.clear() + + +def _with_fake_cm(monkeypatch, client): + class FakeCM: + async def get_client(self, _name): + return client + + monkeypatch.setattr(cp, "get_config_manager", lambda: FakeCM()) + + +class _Streams(HummingbotStreamsMixin): + """The WS poller with its surroundings stubbed to one tick.""" + + def __init__(self, client): + self._client = client + self._controller_perf_tasks = {} + self.broadcast = None + + def _has_subscribers(self, _channel): + return True + + async def _broadcast_update(self, _channel, data): + self.broadcast = data + # One tick is all this test needs; the stream's own handler exits on it. + raise asyncio.CancelledError + + +async def _poll_once(client, monkeypatch): + import config_manager + + class FakeCM: + async def get_client(self, _name): + return client + + monkeypatch.setattr(config_manager, "get_config_manager", lambda: FakeCM()) + streams = _Streams(client) + await streams._controller_perf_stream("controller_perf:srv") + return streams.broadcast + + +def test_the_four_whole_server_callers_share_one_upstream_call(monkeypatch): + """The observable claim: four callers within the TTL, one round-trip. + + Against the unfixed code this is 4 — the bots enrichment, both routes and + the WS poller each issued their own byte-identical whole-server request. + """ + client = CountingClient() + _with_fake_cm(monkeypatch, client) + + async def _all_four(): + perf_map = await _fetch_latest_perf(client) + latest = await cp.get_latest_controller_performance( + name="srv", bot_name=None, user=object() + ) + terminated = await cp.get_terminated_controllers( + name="srv", limit=200, user=object() + ) + broadcast = await _poll_once(client, monkeypatch) + return perf_map, latest, terminated, broadcast + + perf_map, latest, terminated, broadcast = asyncio.run(_all_four()) + + assert client.whole_server_calls == 1 + + # …and every caller still got its own answer out of that one payload. + assert perf_map["c1"]["bot_name"] == "gan" + assert [s.controller_id for s in latest.snapshots] == ["c1"] + assert terminated.server_online is True + assert [c.controller_id for c in terminated.controllers] == ["c1"] + assert [s["controller_id"] for s in broadcast["snapshots"]] == ["c1"] + + +def test_the_agents_rollup_shares_that_same_call(monkeypatch): + """``fetch_all_bot_performance`` is the fifth caller of the same payload.""" + client = CountingClient() + _with_fake_cm(monkeypatch, client) + + async def _both(): + agg = await fetch_all_bot_performance(client) + await cp.get_latest_controller_performance( + name="srv", bot_name=None, user=object() + ) + return agg + + agg = asyncio.run(_both()) + + assert client.whole_server_calls == 1 + assert agg["gan"]["global_pnl_quote"] == 11.0 + + +def test_a_bot_filter_is_not_served_from_the_whole_server_cache(monkeypatch): + """``?bot_name=X`` is a narrower request, so it keeps its own round-trip.""" + client = CountingClient() + _with_fake_cm(monkeypatch, client) + + async def _warm_then_filter(): + await cp.get_latest_controller_performance( + name="srv", bot_name=None, user=object() + ) + return await cp.get_latest_controller_performance( + name="srv", bot_name="gan", user=object() + ) + + out = asyncio.run(_warm_then_filter()) + + assert client.whole_server_calls == 1 + assert client.filtered_calls == 1 + assert [s.controller_id for s in out.snapshots] == ["c1"] + + +def test_clear_snapshot_cache_empties_the_raw_layer_too(monkeypatch): + """Both layers, or a cleared aggregate would be rebuilt from stale rows.""" + client = CountingClient() + + async def _twice_around_a_clear(): + await fetch_latest_snapshots(client) + await fetch_all_bot_performance(client) + clear_snapshot_cache() + await fetch_latest_snapshots(client) + await fetch_all_bot_performance(client) + + asyncio.run(_twice_around_a_clear()) + + assert client.whole_server_calls == 2 + + +def test_a_failed_fetch_is_never_cached(monkeypatch): + """No previous value papers over a failure, and the next caller retries.""" + + class Flaky(CountingClient): + def __init__(self): + super().__init__() + self.fail = False + + async def get_latest_controller_performance(self, bot_name=None, **_kw): + if self.fail: + self.whole_server_calls += 1 + raise ConnectionError("boom") + return await super().get_latest_controller_performance(bot_name, **_kw) + + client = Flaky() + + async def _fetch_fail_fetch(): + await fetch_latest_snapshots(client) + client.fail = True + clear_snapshot_cache() + with pytest.raises(ConnectionError): + await fetch_latest_snapshots(client) + # The failure left nothing behind: the next call goes upstream again. + client.fail = False + return await fetch_latest_snapshots(client) + + got = asyncio.run(_fetch_fail_fetch()) + + assert client.whole_server_calls == 3 + assert [s["controller_id"] for s in got] == ["c1"] + + +def test_an_unidentifiable_client_is_not_cached_at_all(): + """A client with no ``base_url`` cannot be told apart from any other.""" + + class Anonymous(CountingClient): + base_url = "" + + client = Anonymous() + + async def _twice(): + await fetch_latest_snapshots(client) + await fetch_latest_snapshots(client) + + asyncio.run(_twice()) + + assert client.whole_server_calls == 2 From d8a2b5a8576556859613e59b781aec38f57ee69a Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 15:30:46 +0300 Subject: [PATCH 078/154] Walk a server's executor history once for all three KPI windows The KPI strip's period total walks the whole executor history with fetch_all_executors -- up to ten sequential cursor pages the SDS rate limiter never sees. That walk was run once per period and once per concurrent caller: switching the strip from 1D to 1W re-read a history the 1D total had already read in full, and two open tabs missing the same cold cache each fired their own walk. The walk does not depend on the window. summarize_executors_by_quote filters the rows on the executor's start timestamp client-side, so the 1M window's rows are a strict superset of 1W's and 1D's. One walk now totals all three and stamps each into the existing per-period cache, which stays the read gate: 1D still re-walks after 60s, and the 1W total that walk also refreshed is still served for its own 300s. Concurrent callers share the walk through condor.asyncutil.SingleFlight rather than a sixth hand-rolled in-flight map. A walk that raises is shared by the waiters already on it and remembered for nobody: the entry is never cached and the next request retries. No reported figure moves. Each window folds the same rows in the same order, and each still resolves its own quote assets to USD, so pnl, volume, count and the converted flag are float-for-float what three separate walks produced. --- condor/web/routes/executors.py | 56 ++++++++- tests/test_executors_period_summary.py | 167 +++++++++++++++++++++++-- 2 files changed, 209 insertions(+), 14 deletions(-) diff --git a/condor/web/routes/executors.py b/condor/web/routes/executors.py index b63096702..7f26e19c7 100644 --- a/condor/web/routes/executors.py +++ b/condor/web/routes/executors.py @@ -10,6 +10,7 @@ logger = logging.getLogger(__name__) +from condor.asyncutil import SingleFlight from condor.fetchers.executors import EXECUTORS_POLL_MAX, MAX_EXECUTORS_FETCH from condor.fetchers.executors import extract_executors_list as _extract_executors_list from condor.fetchers.executors import fetch_all_executors, summarize_executors_by_quote @@ -44,6 +45,15 @@ # (server, period) -> (computed_at, summary). Bounded by servers x periods. _summary_cache: dict[tuple[str, str], tuple[float, ExecutorPeriodSummary]] = {} +# One executor walk per server at a time, shared by every concurrent caller +# (PERF-580). The TTL cache above only helps a request that arrives *after* an +# answer has landed; the KPI strip refetches on a 60s interval against a 60s TTL +# for 1D, so two open tabs used to fire two concurrent walks of up to +# MAX_EXECUTORS_FETCH / EXECUTORS_PAGE_SIZE sequential cursor pages each. +# Keyed by server, not by (server, period): the walk is identical for all three +# windows, which are filtered out of the same rows client-side. +_summary_walks = SingleFlight() + @router.get("/servers/{name}/executors", response_model=list[ExecutorInfo]) async def list_executors( @@ -254,6 +264,37 @@ async def _usd_summary( ) +async def _walk_and_summarize(server: str, client) -> dict[str, ExecutorPeriodSummary]: + """Walk a server's executor history once and total *every* window from it. + + The walk is the expensive part — up to ``MAX_EXECUTORS_FETCH / + EXECUTORS_PAGE_SIZE`` sequential cursor pages the SDS rate limiter never + sees — and it does not depend on the period: ``summarize_executors_by_quote`` + filters on the executor's start timestamp client-side, so the 1M window's + rows are a strict superset of 1W's and 1D's. Computing the three totals in + one pass is therefore exactly the arithmetic the three separate requests + performed, over the same rows in the same order, and costs one walk instead + of three (PERF-580). + + Each total is stamped into ``_summary_cache`` here, so the per-period TTLs + stay the read gate: a 1D request re-walks after 60s, but the 1W total it + also refreshed is still served from cache for its own 300s. + """ + now = time.time() + executors = await fetch_all_executors(client) + + summaries: dict[str, ExecutorPeriodSummary] = {} + for window_period, window in _PERIOD_SECONDS.items(): + summary = await _usd_summary( + server, + window_period, + summarize_executors_by_quote(executors, now - window), + ) + summaries[window_period] = summary + _summary_cache[(server, window_period)] = (now, summary) + return summaries + + @router.get("/servers/{name}/executors/summary", response_model=ExecutorPeriodSummary) async def executors_summary( name: str, @@ -271,6 +312,11 @@ async def executors_summary( A period total belongs here, over the full history: this walks it with ``fetch_all_executors``, on demand and cached per period, leaving the 2s poll at its one request per tick. + + The walk itself is shared (PERF-580). It is single-flighted per server, so + two tabs missing a cold cache at the same instant wait on one walk rather + than racing two, and one walk totals all three windows — switching the + strip's period no longer re-reads a history the previous period already had. """ cm = get_config_manager() @@ -290,16 +336,14 @@ async def executors_summary( client = await cm.get_client(name) try: - executors = await fetch_all_executors(client) + summaries = await _summary_walks.run( + name, lambda: _walk_and_summarize(name, client) + ) except Exception as e: logger.exception("Failed to summarize executors for server %s", name) raise upstream_error("Failed to fetch executors", e) - summary = await _usd_summary( - name, period, summarize_executors_by_quote(executors, now - window) - ) - _summary_cache[(name, period)] = (now, summary) - return summary + return summaries[period] async def _ensure_dex_tokens_listed(client, config: dict) -> None: diff --git a/tests/test_executors_period_summary.py b/tests/test_executors_period_summary.py index 6bc0093dd..a7f3a9a8f 100644 --- a/tests/test_executors_period_summary.py +++ b/tests/test_executors_period_summary.py @@ -19,7 +19,11 @@ from condor.fetchers.executors import EXECUTORS_PAGE_SIZE, summarize_executors_by_quote from condor.web.models import WebUser -from condor.web.routes.executors import _summary_cache, executors_summary +from condor.web.routes.executors import ( + _summary_cache, + _summary_walks, + executors_summary, +) _USER = WebUser(id=1, role="admin") _DAY = 86400 @@ -43,10 +47,22 @@ def _executor(i, *, age_days=0.0, pair="BTC-USDT", pnl=1.0, volume=10.0): class FakeClient: """Client serving ``rows`` one page at a time, recording every request.""" - def __init__(self, rows): + def __init__(self, rows, fail=False): self.calls = [] self.executors = self._Executors(self) self._rows = rows + self.fail = fail + + @property + def walks(self): + """How many full-history walks were started. + + A walk is one ``fetch_all_executors`` pass: it opens with a cursor-less + page and then follows ``next_cursor``. Counting pages hides the thing + PERF-580 is about — a second walk over a history short enough to fit in + one page adds exactly one page — so the tests below count openings. + """ + return sum(1 for kw in self.calls if not kw.get("cursor")) class _Executors: def __init__(self, outer): @@ -55,6 +71,13 @@ def __init__(self, outer): async def search_executors(self, **kwargs): outer = self._outer outer.calls.append(kwargs) + # A real page is I/O and yields to the loop. Without a yield here a + # "concurrent" test is not concurrent at all: the first caller runs + # to completion before the second is scheduled, and even uncoalesced + # walks look like one. + await asyncio.sleep(0) + if outer.fail: + raise RuntimeError("backend unreachable") start = int(kwargs.get("cursor") or 0) page = outer._rows[start : start + kwargs["limit"]] end = start + len(page) @@ -79,16 +102,18 @@ async def get_client(self, name): def clean_cache(): """The summary cache is module state — no test may inherit another's.""" _summary_cache.clear() + _summary_walks.clear() yield _summary_cache.clear() + _summary_walks.clear() @pytest.fixture def summary_env(monkeypatch): """Bind the endpoint to a fake client and a fixed rate table.""" - def _bind(rows, rates=None): - client = FakeClient(rows) + def _bind(rows, rates=None, fail=False): + client = FakeClient(rows, fail=fail) monkeypatch.setattr( "condor.web.routes.executors.get_config_manager", lambda: _FakeCM(client) ) @@ -241,18 +266,144 @@ def test_a_mixed_total_flags_only_the_quote_that_failed(summary_env): def test_the_aggregate_is_cached_per_period(summary_env): - """A second reader inside the TTL re-walks nothing; another period does.""" + """A second reader inside the TTL re-walks nothing.""" rows = [_executor(i, age_days=0.5) for i in range(10)] client = summary_env(rows, {"USDT-USDT": 1.0}) _summary("1D") - first_walk = len(client.calls) _summary("1D") - assert len(client.calls) == first_walk, "the cached period must not re-walk" + assert client.walks == 1, "the cached period must not re-walk" + + +# ── PERF-580: one walk feeds every window, and concurrent callers share it ── + +def test_every_period_is_totalled_from_a_single_walk(summary_env): + """Switching the strip's period re-reads nothing. + + The walk does not depend on the window — ``summarize_executors_by_quote`` + filters the same rows on their start timestamp — so the three windows are + three folds over one history, not three histories. This test used to assert + the opposite ("a different period is a different total"), which pinned the + defect: it counted the walk growing on every period switch. + """ + rows = [_executor(i, age_days=0.5) for i in range(10)] + client = summary_env(rows, {"USDT-USDT": 1.0}) + + _summary("1D") _summary("1W") - assert len(client.calls) > first_walk, "a different period is a different total" + _summary("1M") + + assert client.walks == 1, "each period switch re-walked the whole history" + assert set(_summary_cache) == {("srv", p) for p in ("1D", "1W", "1M")} + + +def test_the_windows_computed_together_match_the_windows_computed_apart(summary_env): + """The coalesced totals are the same dollars, to the last float. + + Same rows, same order, same cutoff arithmetic — folding three windows out of + one pass may not move a single PnL, volume or count, nor the ``converted`` + flag, which each window still resolves over its own quote assets. + """ + rows = [ + _executor(0, age_days=0.5, pair="ETH-BTC", pnl=0.1, volume=0.3), + _executor(1, age_days=0.5, pair="SOL-USDT", pnl=2.7, volume=9.1), + _executor(2, age_days=3, pair="SOL-USDT", pnl=-1.3, volume=4.4), + _executor(3, age_days=20, pair="ETH-BTC", pnl=0.02, volume=0.05), + ] + rates = {"BTC-USDT": 50_000.0, "USDT-USDT": 1.0} + summary_env(rows, rates) + + together = {p: _summary(p) for p in ("1D", "1W", "1M")} + + apart = {} + for period in ("1D", "1W", "1M"): + _summary_cache.clear() + _summary_walks.clear() + summary_env(rows, rates) + apart[period] = _summary(period) + + for period in ("1D", "1W", "1M"): + assert together[period].pnl == apart[period].pnl + assert together[period].volume == apart[period].volume + assert together[period].count == apart[period].count + assert together[period].converted == apart[period].converted + + +def test_concurrent_readers_of_a_cold_cache_share_one_walk(summary_env): + """Two tabs opening at once are one walk, not two. + + A TTL cache only helps a request that arrives after an answer has landed. + The strip refetches every 60s against a 60s TTL for 1D, so a second tab + reliably misses the same cold cache the first one is already filling. + """ + rows = [_executor(i, age_days=0.5) for i in range(10)] + client = summary_env(rows, {"USDT-USDT": 1.0}) + + async def _both(): + return await asyncio.gather( + executors_summary(name="srv", period="1D", user=_USER), + executors_summary(name="srv", period="1D", user=_USER), + ) + + first, second = asyncio.run(_both()) + + assert client.walks == 1, "concurrent readers each ran their own walk" + assert first.pnl == second.pnl == pytest.approx(10.0) + assert first.count == second.count == 10 + + +def test_concurrent_readers_of_different_periods_share_one_walk(summary_env): + """The walk is keyed by server, so 1D and 1W in flight together is one walk.""" + rows = [_executor(0, age_days=0.5), _executor(1, age_days=3)] + client = summary_env(rows, {"USDT-USDT": 1.0}) + + async def _both(): + return await asyncio.gather( + executors_summary(name="srv", period="1D", user=_USER), + executors_summary(name="srv", period="1W", user=_USER), + ) + + day, week = asyncio.run(_both()) + + assert client.walks == 1 + assert day.count == 1 and day.period == "1D" + assert week.count == 2 and week.period == "1W" + + +def test_a_failed_walk_is_not_cached_and_is_retried(summary_env): + """A walk that raises leaves no entry behind, and the shape stays upstream_error.""" + client = summary_env([_executor(0, age_days=0.5)], {"USDT-USDT": 1.0}, fail=True) + + with pytest.raises(HTTPException) as exc: + _summary("1D") + + assert exc.value.status_code == 502 + assert "Failed to fetch executors" in exc.value.detail + assert _summary_cache == {}, "a failure must not be served as a total" + assert not _summary_walks, "a settled walk must not be joined by the next caller" + + client.fail = False + assert _summary("1D").count == 1, "the next request must retry the walk" + + +def test_concurrent_readers_of_a_failing_walk_all_see_the_error(summary_env): + """Sharing a walk shares its failure — nobody gets a silent zero.""" + summary_env([_executor(0, age_days=0.5)], {"USDT-USDT": 1.0}, fail=True) + + async def _both(): + return await asyncio.gather( + executors_summary(name="srv", period="1D", user=_USER), + executors_summary(name="srv", period="1W", user=_USER), + return_exceptions=True, + ) + + first, second = asyncio.run(_both()) + + assert isinstance(first, HTTPException) and first.status_code == 502 + assert isinstance(second, HTTPException) and second.status_code == 502 + assert _summary_cache == {} def test_an_expired_entry_is_recomputed(summary_env): From ac515c56b5071bfd4d954f5a7d1404b2b934ce05 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 15:40:16 +0300 Subject: [PATCH 079/154] Open a pool you can already see without asking gecko again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same normalized pool dict is built by three paths and was cached in three unrelated places: the page listing and the favourites batch stored theirs in list-shaped caches, while opening a pool read a by-address cache only it ever wrote. So the primary navigation path — click a row in the listing, the pool workspace fetches it by address — always missed, and spent a fresh single-pool request against the shared GeckoTerminal budget seconds after the identical row had arrived. The listings now index each row they normalize under (gecko network, address), which is the key the by-address read already uses, at the same POOL_LIST_TTL. Only found pools are remembered: an address merely absent from a page is not the known-missing pool that fetch_pool_by_address caches as {}, so a pool outside the listing is still fetched on demand. A copy is stored because a listing hands its rows to callers uncopied and those callers must not reach into the cache through them. --- condor/pool_data.py | 25 +++++++++++ tests/test_dex_pool_discovery.py | 72 ++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/condor/pool_data.py b/condor/pool_data.py index c6ec9aedd..73d3b622d 100644 --- a/condor/pool_data.py +++ b/condor/pool_data.py @@ -1915,6 +1915,28 @@ def extract_pair_from_name(name: str) -> Tuple[str, str]: _multi_pool_cache: Dict[Tuple[str, str], Tuple[float, List[Dict[str, Any]]]] = {} _pool_by_address_cache: Dict[Tuple[str, str], Tuple[float, Dict[str, Any]]] = {} + +def _remember_pool(gnet: str, pool: Dict[str, Any]) -> None: + """Index one already-normalized listing row by its address. + + The listings above cache *lists* — a page, a favourites batch — but the + primary navigation path is clicking a row you can already see, which asks + ``fetch_pool_by_address`` for that one pool. Without this it always missed and + spent a fresh single-pool request on the shared GeckoTerminal budget, seconds + after the identical row arrived. Every path builds the row with + ``_normalize_gecko_pool``, so a listing row and a by-address row are the same + dict, and one entry serves both. + + A copy is stored because a listing hands its rows to callers uncopied, and + those callers must not be able to reach into this cache through them. Only + *found* pools are remembered: an address merely absent from a page is not the + known-missing pool that ``fetch_pool_by_address`` caches as ``{}``. + """ + address = str(pool.get("address") or "") + if address: + _ttl_put(_pool_by_address_cache, (gnet, address), dict(pool), POOL_LIST_TTL) + + # A pool list is a browser page, not a report: cap what an upstream can be asked # for so one query cannot drag a thousand rows through normalization. _POOL_LIST_MAX = 100 @@ -2144,6 +2166,7 @@ def _normalized_gecko_rows( rows: List[Dict[str, Any]], network: str, dexes: set ) -> List[Dict[str, Any]]: """Raw upstream rows → decorated pools, keeping only the venues asked for.""" + gnet = get_gecko_network(network) pools: List[Dict[str, Any]] = [] for row in rows: try: @@ -2156,6 +2179,7 @@ def _normalized_gecko_rows( if dexes and str(pool.get("dex_id") or "").strip().lower() not in dexes: continue pools.append(pool) + _remember_pool(gnet, pool) return pools @@ -2403,6 +2427,7 @@ async def fetch_pools_by_addresses( continue if pool.get("address"): batch_pools.append(pool) + _remember_pool(gnet, pool) _stale_put(_multi_pool_cache, batch_key, batch_pools) pools.extend(dict(row) for row in batch_pools) diff --git a/tests/test_dex_pool_discovery.py b/tests/test_dex_pool_discovery.py index 12a15db08..dba84b592 100644 --- a/tests/test_dex_pool_discovery.py +++ b/tests/test_dex_pool_discovery.py @@ -581,6 +581,78 @@ def test_pool_by_address_failure_is_none_and_not_cached(fake_gecko): assert run(pool_data.fetch_pool_by_address("solana", POOL)) is not None +# ── a listing seeds the by-address cache (PERF-600) ── +# +# Clicking a row you can already see is the primary navigation path, and it used +# to cost a fresh single-pool request against the shared budget seconds after the +# identical row had arrived in the listing. + + +def test_opening_a_listed_pool_costs_no_upstream_request(fake_gecko): + client = fake_gecko(pd.DataFrame([gecko_row()])) + run(pool_data.list_gecko_pools("solana-mainnet-beta")) + listing_calls = len(client.calls) + + pool = run(pool_data.fetch_pool_by_address("solana-mainnet-beta", POOL)) + + assert pool["address"] == POOL + assert len(client.calls) == listing_calls + assert not [c for c in client.calls if c[0] == "address"] + + +def test_opening_a_favourite_costs_no_upstream_request(fake_gecko): + client = fake_gecko(pd.DataFrame([gecko_row()])) + run(pool_data.fetch_pools_by_addresses("solana-mainnet-beta", [POOL])) + batch_calls = len(client.calls) + + pool = run(pool_data.fetch_pool_by_address("solana-mainnet-beta", POOL)) + + assert pool["address"] == POOL + assert len(client.calls) == batch_calls + assert not [c for c in client.calls if c[0] == "address"] + + +def test_a_listing_seeded_pool_equals_the_one_fetched_by_address(fake_gecko): + fake_gecko(pd.DataFrame([gecko_row()])) + run(pool_data.fetch_pools_by_addresses("solana-mainnet-beta", [POOL])) + seeded = run(pool_data.fetch_pool_by_address("solana-mainnet-beta", POOL)) + + pool_data._pool_by_address_cache.clear() + pool_data._multi_pool_cache.clear() + fetched = run(pool_data.fetch_pool_by_address("solana-mainnet-beta", POOL)) + + assert seeded == fetched + + +def test_a_listing_row_a_caller_mutates_cannot_poison_the_pool_cache(fake_gecko): + fake_gecko(pd.DataFrame([gecko_row()])) + rows = run(pool_data.list_gecko_pools("solana-mainnet-beta")) + rows[0]["address"] = "mutated" + rows[0]["volume_24h"] = -1 + + pool = run(pool_data.fetch_pool_by_address("solana-mainnet-beta", POOL)) + assert pool["address"] == POOL + assert pool["volume_24h"] == 4_200_000.0 + # And the copy this call handed out is not the cached one either. + pool["address"] = "mutated again" + assert ( + run(pool_data.fetch_pool_by_address("solana-mainnet-beta", POOL))["address"] + == POOL + ) + + +def test_a_pool_absent_from_a_listing_is_still_fetched_on_demand(fake_gecko): + """A listing must never write the negative entry: absent is not known-missing.""" + listed, other = addr(1), addr(2) + client = fake_gecko(pd.DataFrame([gecko_row(address=listed)])) + run(pool_data.list_gecko_pools("solana-mainnet-beta")) + + assert run(pool_data.fetch_pool_by_address("solana-mainnet-beta", other)) + assert [c for c in client.calls if c[0] == "address"] == [ + ("address", "solana", other) + ] + + # ── the route (condor/web/routes/dex.py) ── From edd1b2bd0f50a00f53457e5f9eb1553288615044 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 15:49:59 +0300 Subject: [PATCH 080/154] Keep reading the agent while a permission dialog is open `session/request_permission` waits on a human, and the ACP read loop awaited that handler inline: for the whole dialog nothing at all was read from the child's stdout. A second dangerous call in the same turn could not raise its prompt until the first was answered, notifications behind it never reached the stream, `session/cancel` could not be answered so Stop always degraded to a local cancel, and the pipe eventually backpressured the agent itself. Async handlers now run as their own tracked task and answer through a new `JSONRPCPeer.send_response` seam; JSON-RPC correlates by id, so replying out of order is legal. Sync handlers stay inline, which keeps `session/update` ordering into the event queue exactly as it was. A handler that raises fails its own request, and `cancel_all` cancels the in-flight ones at teardown. --- condor/acp/jsonrpc.py | 126 ++++++++-- ...acp_permission_does_not_block_read_loop.py | 226 ++++++++++++++++++ 2 files changed, 328 insertions(+), 24 deletions(-) create mode 100644 tests/runtime/test_acp_permission_does_not_block_read_loop.py diff --git a/condor/acp/jsonrpc.py b/condor/acp/jsonrpc.py index 62356dc6a..ee317c834 100644 --- a/condor/acp/jsonrpc.py +++ b/condor/acp/jsonrpc.py @@ -7,6 +7,8 @@ import logging from typing import Any, Callable +from condor.asyncutil import TaskSet + log = logging.getLogger(__name__) @@ -48,6 +50,10 @@ def __init__(self): # loop swept the pending table -- fails with the same real error # instead of parking on a future nobody will ever settle (CORR-329). self._failure: BaseException | None = None + # Reverse-RPC handlers that suspend (``session/request_permission`` + # waits on a human) run here rather than inline in the caller's read + # loop, which must get back to ``readline()`` immediately (PERF-330). + self._handler_tasks = TaskSet(log, "Reverse-RPC handler %s crashed: %s") def register_handler(self, method: str, handler: Callable) -> None: self._handlers[method] = handler @@ -143,6 +149,27 @@ async def send_notification( await writer.drain() log.debug("-> %s (notification)", method) + async def send_response( + self, + msg_id: Any, + body: dict[str, Any], + writer: asyncio.StreamWriter, + ) -> None: + """Write one JSON-RPC response frame for ``msg_id``. + + ``body`` is the ``{"result": …}`` or ``{"error": …}`` member. The + response seam, sibling of :meth:`begin_request`: a handler that answers + from its own task (see :meth:`handle_line`) no longer has the read + loop's inlined write to fall back on, and all three answer sites — a + result, a handler failure, an unknown method — frame it here. + + One ``write`` per frame, so two answers racing on the same writer + interleave whole lines and never halves of one. + """ + resp = {"jsonrpc": "2.0", **body, "id": msg_id} + writer.write((json.dumps(resp) + "\n").encode()) + await writer.drain() + async def handle_line(self, line: str, writer: asyncio.StreamWriter) -> None: """Process one line of JSON from the subprocess stdout.""" try: @@ -184,41 +211,88 @@ async def handle_line(self, line: str, writer: asyncio.StreamWriter) -> None: if handler is None: log.warning("No handler for reverse-RPC method: %s", method) if msg_id is not None: - resp = { - "jsonrpc": "2.0", - "error": { - "code": METHOD_NOT_FOUND, - "message": f"Method not found: {method}", + await self.send_response( + msg_id, + { + "error": { + "code": METHOD_NOT_FOUND, + "message": f"Method not found: {method}", + } }, - "id": msg_id, - } - writer.write((json.dumps(resp) + "\n").encode()) - await writer.drain() + writer, + ) return - try: - result = ( - handler(**params) - if not asyncio.iscoroutinefunction(handler) - else await handler(**params) + # An async handler is dispatched as its own task and we return to the + # caller's ``readline()`` at once (PERF-330). Awaiting it here made the + # whole stdout stream hostage to the slowest handler: + # ``session/request_permission`` waits on a human, so for the two + # minutes a confirmation dialog was open nothing at all was read from + # the child -- a second dangerous call in the same turn could not even + # raise its prompt, notifications behind it went unseen, the answer to + # our own ``session/cancel`` could not arrive, and the pipe eventually + # backpressured the agent process itself. JSON-RPC correlates answers + # by ``id``, so replying out of order is legal. + # + # Sync handlers stay inline: ``session/update`` is one, and the order + # it feeds the event queue in is the order of the turn's own chunks. + if asyncio.iscoroutinefunction(handler): + self._handler_tasks.track( + asyncio.get_event_loop().create_task( + self._run_handler(method, handler, params, msg_id, writer) + ), + label=method, ) + return + + try: + result = handler(**params) except Exception as e: log.exception("Handler error for %s", method) if msg_id is not None: - resp = { - "jsonrpc": "2.0", - "error": {"code": INTERNAL_ERROR, "message": str(e)}, - "id": msg_id, - } - writer.write((json.dumps(resp) + "\n").encode()) - await writer.drain() + await self.send_response( + msg_id, + {"error": {"code": INTERNAL_ERROR, "message": str(e)}}, + writer, + ) return # Send response only for requests (not notifications) if msg_id is not None: - resp = {"jsonrpc": "2.0", "result": result, "id": msg_id} - writer.write((json.dumps(resp) + "\n").encode()) - await writer.drain() + await self.send_response(msg_id, {"result": result}, writer) + + async def _run_handler( + self, + method: str, + handler: Callable, + params: dict[str, Any], + msg_id: Any, + writer: asyncio.StreamWriter, + ) -> None: + """Await one async reverse-RPC handler and answer it, off the read loop. + + A handler that raises fails *its own* request and nothing else: the + error response goes out here, and the read loop that dispatched it is + long gone back to reading. Cancellation (teardown, see + :meth:`cancel_all`) is not a failure and sends nothing -- the writer is + on its way out with us. + """ + try: + result = await handler(**params) + except asyncio.CancelledError: + raise + except Exception as e: + log.exception("Handler error for %s", method) + if msg_id is not None: + await self.send_response( + msg_id, + {"error": {"code": INTERNAL_ERROR, "message": str(e)}}, + writer, + ) + return + + if msg_id is not None: + await self.send_response(msg_id, {"result": result}, writer) def cancel_all(self) -> None: """Cancel all pending futures (used during our own shutdown).""" @@ -226,6 +300,10 @@ def cancel_all(self) -> None: if not future.done(): future.cancel() self._pending.clear() + # In-flight reverse-RPC handlers go too: a confirmation still waiting + # on a human outlives the subprocess otherwise, and asyncio would + # report it at GC time as "Task was destroyed but it is pending". + self._handler_tasks.cancel_all() def fail_all(self, exc: BaseException) -> None: """Settle every pending future with ``exc``: the connection is gone. diff --git a/tests/runtime/test_acp_permission_does_not_block_read_loop.py b/tests/runtime/test_acp_permission_does_not_block_read_loop.py new file mode 100644 index 000000000..f8a5fe152 --- /dev/null +++ b/tests/runtime/test_acp_permission_does_not_block_read_loop.py @@ -0,0 +1,226 @@ +"""A pending permission dialog must not freeze the ACP read loop (PERF-330). + +``session/request_permission`` waits on a human -- up to the confirmation TTL. +While the read loop awaited that handler inline, nothing at all was read from +the child's stdout for the whole dialog: a second dangerous tool call in the +same assistant turn could not even raise its prompt, every notification behind +it was invisible, and the answer to our own ``session/cancel`` could not arrive. +""" + +import asyncio +import json + +import pytest + +from condor.acp.client import ACPClient, TextChunk +from condor.acp.jsonrpc import JSONRPCPeer + + +class _FakeStdin: + """Subprocess stdin: records whatever the peer writes back.""" + + def __init__(self) -> None: + self.written: list[dict] = [] + + def write(self, data: bytes) -> None: + self.written.append(json.loads(data.decode())) + + async def drain(self) -> None: + pass + + +class _FakeProcess: + def __init__(self, stdout: asyncio.StreamReader) -> None: + self.stdout = stdout + self.stdin = _FakeStdin() + self.returncode = None + + +def _client(stdout: asyncio.StreamReader) -> ACPClient: + client = ACPClient(command="true") + client._process = _FakeProcess(stdout) # type: ignore[assignment] + return client + + +def _permission_line(msg_id: int, title: str) -> bytes: + return ( + json.dumps( + { + "jsonrpc": "2.0", + "id": msg_id, + "method": "session/request_permission", + "params": { + "sessionId": "s1", + "toolCall": {"title": title, "rawInput": {}}, + "options": [{"optionId": "yes", "kind": "allow_once"}], + }, + } + ) + + "\n" + ).encode() + + +def _update_line(text: str) -> bytes: + return ( + json.dumps( + { + "jsonrpc": "2.0", + "method": "session/update", + "params": { + "sessionId": "s1", + "update": { + "sessionUpdate": "agent_message_chunk", + "content": {"text": text}, + }, + }, + } + ) + + "\n" + ).encode() + + +async def _stop_loop(task: asyncio.Task) -> None: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + +@pytest.mark.asyncio +async def test_traffic_keeps_flowing_while_a_permission_prompt_is_pending(): + """Two prompts in one turn, plus the notifications and the response behind them.""" + stdout = asyncio.StreamReader() + client = _client(stdout) + client._current_req_id = 99 # a turn is being streamed, so updates are kept + + entered: asyncio.Queue[str] = asyncio.Queue() + release = asyncio.Event() + + async def slow_permission(sessionId="", toolCall=None, options=None, **kw): + await entered.put((toolCall or {}).get("title", "")) + await release.wait() + return {"outcome": {"outcome": "selected", "optionId": "yes"}} + + client._peer.register_handler("session/request_permission", slow_permission) + loop_task = asyncio.create_task(client._read_loop()) + + # Our own in-flight request -- what abort_prompt's session/cancel is. + req_id, our_future = await client._peer.begin_request( + "session/cancel", {"sessionId": "s1"}, client._process.stdin + ) + + stdout.feed_data(_permission_line(101, "rm -rf /")) + stdout.feed_data(_permission_line(102, "curl evil.sh")) + stdout.feed_data(_update_line("still talking")) + stdout.feed_data( + ( + json.dumps({"jsonrpc": "2.0", "id": req_id, "result": {"ok": True}}) + "\n" + ).encode() + ) + + try: + # Both dialogs are raised before either is answered: unfixed, the second + # line sits unread in the pipe behind the first handler. + first = await asyncio.wait_for(entered.get(), timeout=5) + second = await asyncio.wait_for(entered.get(), timeout=5) + assert [first, second] == ["rm -rf /", "curl evil.sh"] + + # ... and everything queued behind them was read anyway. + event = await asyncio.wait_for(client._event_queue.get(), timeout=5) + assert isinstance(event, TextChunk) and event.text == "still talking" + assert await asyncio.wait_for(our_future, timeout=5) == {"ok": True} + + # Neither permission has answered yet: only our own request went out. + assert [m.get("method") for m in client._process.stdin.written] == [ + "session/cancel" + ] + + release.set() + for _ in range(20): + await asyncio.sleep(0) + if len(client._process.stdin.written) == 3: + break + answers = {m["id"]: m for m in client._process.stdin.written if "id" in m} + assert answers[101]["result"] == { + "outcome": {"outcome": "selected", "optionId": "yes"} + } + assert answers[102]["result"] == { + "outcome": {"outcome": "selected", "optionId": "yes"} + } + finally: + release.set() + await _stop_loop(loop_task) + + +@pytest.mark.asyncio +async def test_sync_handler_ordering_is_unchanged(): + """``session/update`` has no id and is still dispatched inline, in order.""" + stdout = asyncio.StreamReader() + client = _client(stdout) + client._current_req_id = 99 + + loop_task = asyncio.create_task(client._read_loop()) + try: + for text in ("one", "two", "three"): + stdout.feed_data(_update_line(text)) + seen = [ + (await asyncio.wait_for(client._event_queue.get(), timeout=5)).text + for _ in range(3) + ] + assert seen == ["one", "two", "three"] + finally: + await _stop_loop(loop_task) + + +@pytest.mark.asyncio +async def test_a_raising_async_handler_fails_its_request_not_the_loop(): + stdout = asyncio.StreamReader() + client = _client(stdout) + + async def boom(**kw): + raise RuntimeError("handler exploded") + + client._peer.register_handler("session/request_permission", boom) + loop_task = asyncio.create_task(client._read_loop()) + try: + stdout.feed_data(_permission_line(101, "rm -rf /")) + for _ in range(50): + await asyncio.sleep(0) + if client._process.stdin.written: + break + (resp,) = client._process.stdin.written + assert resp["id"] == 101 + assert "handler exploded" in resp["error"]["message"] + + # The connection is untouched: the next line still dispatches. + client._current_req_id = 99 + stdout.feed_data(_update_line("alive")) + event = await asyncio.wait_for(client._event_queue.get(), timeout=5) + assert isinstance(event, TextChunk) and event.text == "alive" + assert client.alive + finally: + await _stop_loop(loop_task) + + +@pytest.mark.asyncio +async def test_cancel_all_cancels_an_in_flight_handler(): + """Teardown must not leave a parked confirmation task behind.""" + peer = JSONRPCPeer() + writer = _FakeStdin() + started = asyncio.Event() + + async def never_answers(**kw): + started.set() + await asyncio.Event().wait() + + peer.register_handler("session/request_permission", never_answers) + await peer.handle_line(_permission_line(101, "rm -rf /").decode(), writer) + await asyncio.wait_for(started.wait(), timeout=5) + + (task,) = list(peer._handler_tasks) + peer.cancel_all() + with pytest.raises(asyncio.CancelledError): + await task + assert task.cancelled() + assert writer.written == [] # a cancelled handler answers nothing From 3cf5edeca0ee4a6416bc0401a28c57b25e23388a Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 15:59:35 +0300 Subject: [PATCH 081/154] Cache each bot's controller configs across bots-page refreshes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bots-page enrichment fanned out one get_bot_controller_configs per bot on every refresh — once per 30s per server, for data that changes only when someone edits a config. Each bot's configs now come from a (server, bot_name)-keyed TTL cache with in-flight coalescing, so a fleet of N bots costs N upstream requests per minute instead of per poll, and two refreshes racing on a cold cache share one call per bot. Keyed per bot, never per batch: a newly deployed bot fetches only itself. A call that raises is never cached, and entries for bots that have left the fleet are pruned on each fan-out. The two config-editing bots routes drop the affected entries so a UI edit shows up immediately; the TTL stays the real freshness bound, since the same configs are also edited from Telegram and MCP. --- condor/fetchers/__init__.py | 11 +- condor/fetchers/bots.py | 114 +++++++++++-- condor/web/routes/bots.py | 15 +- tests/test_bots_ctrl_configs_cache.py | 220 ++++++++++++++++++++++++++ 4 files changed, 348 insertions(+), 12 deletions(-) create mode 100644 tests/test_bots_ctrl_configs_cache.py diff --git a/condor/fetchers/__init__.py b/condor/fetchers/__init__.py index 1554e7075..7b413409a 100644 --- a/condor/fetchers/__init__.py +++ b/condor/fetchers/__init__.py @@ -19,7 +19,16 @@ depend on who asks) or its subject is immutable. Every such cache must be keyed so one server's — or one Gateway network's — answer can never be served for another, carry a comment saying why the caller cannot hold it - instead, and be listed here. There are seven today: + instead, and be listed here. There are eight today: + + * ``bots._ctrl_configs_cache`` — one bot's controller configs as fetched, + keyed ``(server, bot_name)`` so a new bot invalidates nothing else, + ``_CTRL_CONFIGS_TTL`` 60s, in-flight coalesced, never caching a + failure. Pruned of departed bots on every fan-out; + ``clear_ctrl_configs_cache()`` / ``invalidate_ctrl_configs(client, + bot_name=None)``, which the two config-editing bots routes call so a + UI edit does not wait out the TTL. The TTL, not that hook, is the + freshness bound: Telegram and MCP edit the same configs. * ``bot_performance._raw_snapshot_cache`` — the whole-server controller-performance rows as fetched (``fetch_latest_snapshots``), diff --git a/condor/fetchers/bots.py b/condor/fetchers/bots.py index 4d1393c8a..e6ee9d436 100644 --- a/condor/fetchers/bots.py +++ b/condor/fetchers/bots.py @@ -2,9 +2,15 @@ import asyncio import logging +import time from typing import Any, NamedTuple, Optional -from condor.fetchers.bot_performance import fetch_latest_snapshots +from condor.asyncutil import SingleFlight + +# ``_server_key`` is the package's one rule for "which server does this client +# talk to" (base_url, never id(client), whose reuse after GC would hand one +# server's answer to another); shared rather than re-derived here. +from condor.fetchers.bot_performance import _server_key, fetch_latest_snapshots logger = logging.getLogger(__name__) @@ -291,6 +297,95 @@ async def _with_enrichment_timeout(coro, label: str, default: Any) -> Any: return default +# ── Controller-config cache (PERF-578) ── +# +# One entry per (server, bot) — never per batch: the fan-out below is re-run +# whenever the fleet changes, and a batch key would make a single newly deployed +# bot invalidate every other bot's configs (PERF-600). +# +# A bot's controller configs are near-static — they change only when someone +# edits a config — while this fan-out runs on every BOTS_ENRICHMENT refresh (SDS +# polls it every 30s per server), costing one upstream request per bot each +# time. The TTL is the real freshness bound rather than the invalidation below, +# because the same configs are also mutated from Telegram (handlers/bots/menu.py) +# and MCP (mcp_servers/hummingbot_api/tools/bot_management.py), which cannot call +# in here; 60s matches the ttl ServerDataService already applies to the whole +# enrichment, so the read path is no staler than it already was, and it halves +# the request volume of a 30s poll. Stamped at insert, after the await, so a +# round-trip slower than the TTL cannot write an already-stale entry (CORR-584). +# +# The fan-out itself is deliberately unbounded, as it was before: it is one +# gather of small GETs already capped by ENRICHMENT_TIMEOUT, and bounding it +# would serialise the *cold* path — the one case where the page has nothing to +# render — to save requests the cache now removes anyway. +_CTRL_CONFIGS_TTL = 60.0 +_ctrl_configs_cache: dict[tuple[str, str], tuple[float, list[dict]]] = {} +_ctrl_configs_inflight = SingleFlight() + + +def clear_ctrl_configs_cache() -> None: + """Drop every cached bot controller-config list (tests, reconfiguration).""" + _ctrl_configs_cache.clear() + _ctrl_configs_inflight.clear() + + +def invalidate_ctrl_configs(client, bot_name: Optional[str] = None) -> None: + """Forget cached controller configs for one bot, or for the whole server. + + Called by the routes that edit a controller config so an edit made in the + UI shows up on the next bots page instead of after the TTL. ``bot_name`` is + optional because a *saved* config is edited by id, with no bot attached: the + server's entries are dropped wholesale in that case. + """ + server = _server_key(client) + if not server: + return + for key in [ + k + for k in _ctrl_configs_cache + if k[0] == server and (bot_name is None or k[1] == bot_name) + ]: + _ctrl_configs_cache.pop(key, None) + + +async def _fetch_one_bot_configs(client, bot_name: str) -> list[dict]: + """The raw upstream call for one bot's controller configs.""" + configs = await client.controllers.get_bot_controller_configs(bot_name) + if not isinstance(configs, list): + return [] + return [cfg for cfg in configs if isinstance(cfg, dict)] + + +async def _get_bot_configs(client, bot_name: str) -> list[dict]: + """One bot's controller configs, TTL-cached per server and coalesced. + + A call that raises propagates and is never cached: the next caller retries. + A client with no ``base_url`` (test doubles) cannot be told apart from any + other server's, so it is not cached at all. The returned list is shared + between callers and must be treated as read-only. + """ + server = _server_key(client) + if not server: + return await _fetch_one_bot_configs(client, bot_name) + + key = (server, bot_name) + entry = _ctrl_configs_cache.get(key) + if entry is not None and time.monotonic() - entry[0] <= _CTRL_CONFIGS_TTL: + return entry[1] + + configs = await _ctrl_configs_inflight.run( + key, lambda: _fetch_one_bot_configs(client, bot_name) + ) + _ctrl_configs_cache[key] = (time.monotonic(), configs) + return configs + + +def _prune_ctrl_configs(server: str, live: set[str]) -> None: + """Forget cached configs for bots that have left the fleet.""" + for key in [k for k in _ctrl_configs_cache if k[0] == server and k[1] not in live]: + _ctrl_configs_cache.pop(key, None) + + async def _fetch_ctrl_configs(client, bot_names: list[str]) -> dict[str, dict]: """Controller configs for the given bots, keyed by config id and by name.""" configs_map: dict[str, dict] = {} @@ -299,19 +394,18 @@ async def _fetch_ctrl_configs(client, bot_names: list[str]) -> dict[str, dict]: async def _get_one(bn: str): try: - configs = await client.controllers.get_bot_controller_configs(bn) - if isinstance(configs, list): - for cfg in configs: - cid = cfg.get("id") or cfg.get("controller_id", "") - if cid: - configs_map[cid] = cfg - cname = cfg.get("controller_name", "") - if cname and cname != cid: - configs_map[cname] = cfg + for cfg in await _get_bot_configs(client, bn): + cid = cfg.get("id") or cfg.get("controller_id", "") + if cid: + configs_map[cid] = cfg + cname = cfg.get("controller_name", "") + if cname and cname != cid: + configs_map[cname] = cfg except Exception: pass await asyncio.gather(*[_get_one(bn) for bn in bot_names]) + _prune_ctrl_configs(_server_key(client), set(bot_names)) return configs_map diff --git a/condor/web/routes/bots.py b/condor/web/routes/bots.py index 3f84e5a67..0773a8ae1 100644 --- a/condor/web/routes/bots.py +++ b/condor/web/routes/bots.py @@ -9,7 +9,12 @@ from fastapi import APIRouter, Depends, HTTPException from condor.controller_configs import clean_config_for_save -from condor.fetchers.bots import BotsEnrichment, build_bots_page, extract_bots_list +from condor.fetchers.bots import ( + BotsEnrichment, + build_bots_page, + extract_bots_list, + invalidate_ctrl_configs, +) from condor.server_data_service import ServerDataType, get_server_data_service from condor.web.auth import require_server_access from condor.web.models import ( @@ -439,6 +444,10 @@ async def update_controller_config( ) raise upstream_error("Failed to save controller config", e) + # A saved config is edited by id, with no bot attached, so drop this + # server's whole controller-config cache rather than guess the holder. + invalidate_ctrl_configs(client) + record_ui_deed( user, verb="manage_controllers:upsert", @@ -867,6 +876,10 @@ async def update_bot_controller_config_endpoint( ) raise upstream_error("Failed to save controller config", e) + # The edit lands in this bot's live configs: re-fetch them on the next + # bots page instead of serving the pre-edit copy until the TTL expires. + invalidate_ctrl_configs(client, bot_name) + record_ui_deed( user, verb="manage_bots:update_config", diff --git a/tests/test_bots_ctrl_configs_cache.py b/tests/test_bots_ctrl_configs_cache.py new file mode 100644 index 000000000..93caaf9a2 --- /dev/null +++ b/tests/test_bots_ctrl_configs_cache.py @@ -0,0 +1,220 @@ +"""PERF-578: the bots-page enrichment caches each bot's controller configs. + +``_fetch_ctrl_configs`` used to issue one ``get_bot_controller_configs`` per bot +on *every* enrichment refresh (SDS polls it every 30s per server) for data that +changes only when someone edits a config. It now reads through a per-``(server, +bot)`` TTL cache with in-flight coalescing. + +Every fake upstream call awaits a real suspension point, so two "concurrent" +callers genuinely interleave: without it they would run to completion one after +the other and the coalescing test would pass against the unfixed code too. +""" + +import asyncio + +import pytest + +import condor.fetchers.bots as bots_mod +from condor.fetchers.bots import ( + clear_ctrl_configs_cache, + fetch_bots_enrichment, + invalidate_ctrl_configs, +) + +SERVER_URL = "http://hb.test:8000" + + +def _bots(*names: str) -> list[dict]: + return [{"bot_name": n} for n in names] + + +class _Client: + """Upstream stub counting controller-config calls, one per bot.""" + + def __init__(self, base_url: str = SERVER_URL, *, fail_next: bool = False): + self.base_url = base_url + self.config_calls: list[str] = [] + self.fail_next = fail_next + self.connector = "binance" + self.gate: asyncio.Event | None = None + + @property + def controllers(self): + return self + + @property + def bot_orchestration(self): + return self + + async def get_bot_controller_configs(self, bot_name): + self.config_calls.append(bot_name) + if self.gate is not None: + await self.gate.wait() + else: + # A real client always suspends; without this two concurrent + # callers would run sequentially and never exercise coalescing. + await asyncio.sleep(0) + if self.fail_next: + self.fail_next = False + raise RuntimeError("upstream down") + return [ + { + "id": f"ctrl_{bot_name}", + "controller_name": f"name_{bot_name}", + "connector_name": self.connector, + "trading_pair": "BTC-USDT", + } + ] + + async def get_bot_runs(self, **_kw): + return {"data": {}} + + async def get_latest_controller_performance(self): + return {"data": []} + + +@pytest.fixture(autouse=True) +def _clean_cache(): + clear_ctrl_configs_cache() + yield + clear_ctrl_configs_cache() + + +@pytest.mark.asyncio +async def test_second_enrichment_within_ttl_reuses_cached_configs(): + """Two refreshes inside the TTL cost one upstream call per bot, total.""" + client = _Client() + first = await fetch_bots_enrichment(client, _bots("alpha", "beta")) + second = await fetch_bots_enrichment(client, _bots("alpha", "beta")) + + assert sorted(client.config_calls) == ["alpha", "beta"] + assert first.ctrl_configs == second.ctrl_configs + assert second.ctrl_configs["ctrl_alpha"]["trading_pair"] == "BTC-USDT" + + +@pytest.mark.asyncio +async def test_concurrent_cold_enrichments_coalesce_into_one_call_per_bot(): + """Two refreshes racing on a cold cache share one call per bot, not two.""" + client = _Client() + client.gate = asyncio.Event() + + first = asyncio.ensure_future(fetch_bots_enrichment(client, _bots("alpha", "beta"))) + second = asyncio.ensure_future( + fetch_bots_enrichment(client, _bots("alpha", "beta")) + ) + # Let both callers reach the upstream call before any of them completes. + for _ in range(10): + await asyncio.sleep(0) + assert client.gate is not None + client.gate.set() + a, b = await asyncio.gather(first, second) + + assert sorted(client.config_calls) == ["alpha", "beta"] + assert a.ctrl_configs == b.ctrl_configs + assert "ctrl_alpha" in a.ctrl_configs and "ctrl_beta" in a.ctrl_configs + + +@pytest.mark.asyncio +async def test_ttl_expiry_refetches(monkeypatch): + """Past the TTL the configs are fetched again.""" + client = _Client() + now = [1000.0] + + class _Clock: + @staticmethod + def monotonic(): + return now[0] + + monkeypatch.setattr(bots_mod, "time", _Clock) + + await fetch_bots_enrichment(client, _bots("alpha")) + await fetch_bots_enrichment(client, _bots("alpha")) + now[0] += bots_mod._CTRL_CONFIGS_TTL + 1 + await fetch_bots_enrichment(client, _bots("alpha")) + + assert client.config_calls.count("alpha") == 2 + + +@pytest.mark.asyncio +async def test_a_new_bot_only_fetches_itself(): + """Per-bot keying: adding a bot does not invalidate the others.""" + client = _Client() + await fetch_bots_enrichment(client, _bots("alpha")) + await fetch_bots_enrichment(client, _bots("alpha", "beta")) + + assert client.config_calls == ["alpha", "beta"] + + +@pytest.mark.asyncio +async def test_failed_fetch_is_not_cached(): + """A call that raises is retried by the next refresh, not remembered.""" + client = _Client(fail_next=True) + first = await fetch_bots_enrichment(client, _bots("alpha")) + assert first.ctrl_configs == {} + + second = await fetch_bots_enrichment(client, _bots("alpha")) + assert client.config_calls == ["alpha", "alpha"] + assert second.ctrl_configs["ctrl_alpha"]["connector_name"] == "binance" + + +@pytest.mark.asyncio +async def test_invalidation_shows_the_edited_config(): + """After a config edit invalidates the bot, the next page shows the edit.""" + client = _Client() + await fetch_bots_enrichment(client, _bots("alpha", "beta")) + + client.connector = "kucoin" + invalidate_ctrl_configs(client, "alpha") + page = await fetch_bots_enrichment(client, _bots("alpha", "beta")) + + assert client.config_calls == ["alpha", "beta", "alpha"] + assert page.ctrl_configs["ctrl_alpha"]["connector_name"] == "kucoin" + assert page.ctrl_configs["ctrl_beta"]["connector_name"] == "binance" + + +@pytest.mark.asyncio +async def test_server_wide_invalidation_drops_every_bot(): + """Editing a saved config by id drops the whole server's entries.""" + client = _Client() + await fetch_bots_enrichment(client, _bots("alpha", "beta")) + + invalidate_ctrl_configs(client) + await fetch_bots_enrichment(client, _bots("alpha", "beta")) + + assert sorted(client.config_calls) == ["alpha", "alpha", "beta", "beta"] + + +@pytest.mark.asyncio +async def test_two_servers_never_share_an_answer(): + """The cache is keyed by server: one server's configs never serve another.""" + a = _Client("http://a.test:8000") + b = _Client("http://b.test:8000") + b.connector = "kucoin" + + await fetch_bots_enrichment(a, _bots("alpha")) + page = await fetch_bots_enrichment(b, _bots("alpha")) + + assert b.config_calls == ["alpha"] + assert page.ctrl_configs["ctrl_alpha"]["connector_name"] == "kucoin" + + +@pytest.mark.asyncio +async def test_departed_bots_are_pruned(): + """A bot that leaves the fleet does not linger in the cache.""" + client = _Client() + await fetch_bots_enrichment(client, _bots("alpha", "beta")) + await fetch_bots_enrichment(client, _bots("alpha")) + + assert (SERVER_URL, "beta") not in bots_mod._ctrl_configs_cache + assert (SERVER_URL, "alpha") in bots_mod._ctrl_configs_cache + + +@pytest.mark.asyncio +async def test_client_without_base_url_is_not_cached(): + """An unidentifiable client shares no key with anyone: it always fetches.""" + client = _Client(base_url="") + await fetch_bots_enrichment(client, _bots("alpha")) + await fetch_bots_enrichment(client, _bots("alpha")) + + assert client.config_calls == ["alpha", "alpha"] + assert not bots_mod._ctrl_configs_cache From d68c9eacc01b22d77743e78c6e62a0b3cbf4e54f Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 16:06:49 +0300 Subject: [PATCH 082/154] (perf) page agent performance executors 500 at a time, like the layer that owns the endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetch_agent_performance_batch declared its own PAGE_SIZE = 50 against the same client.executors.search_executors that condor/fetchers/executors.py walks at EXECUTORS_PAGE_SIZE = 500. Nothing justified the smaller page — the comment above it justifies the per-agent fan-out, not the size — so an agent with 1,000 executors cost 21 sequential requests instead of 3, on every agent tick and every 30s web rollup, fanned out over every active agent at once. Take the constant from the layer that owns the endpoint instead of re-declaring one, and restate the safety cap in the unit it is actually measured in: max_items was MAX_PAGES * PAGE_SIZE, which would have silently become 100,000 rows when the page size moved. It is now an explicit MAX_ROWS = 10_000, exactly where it was. No reported figure moves: the same rows arrive in the same order, only in fewer requests, and the cap is unchanged. --- condor/agents/performance.py | 11 ++- tests/test_agent_performance_paging.py | 95 ++++++++++++++++++++++++ tests/test_pagination_cursor_progress.py | 5 +- 3 files changed, 107 insertions(+), 4 deletions(-) create mode 100644 tests/test_agent_performance_paging.py diff --git a/condor/agents/performance.py b/condor/agents/performance.py index 06f8b6454..23519c3d4 100644 --- a/condor/agents/performance.py +++ b/condor/agents/performance.py @@ -15,6 +15,7 @@ from typing import Any from condor.fetchers._pagination import walk_pages +from condor.fetchers.executors import EXECUTORS_PAGE_SIZE log = logging.getLogger(__name__) @@ -376,10 +377,14 @@ async def fetch_agent_performance_batch( # the backend sometimes returned partial data for some controller_ids, # causing sessions with many executors to appear as zero in the rollup # while the per-session endpoint showed the correct numbers. - PAGE_SIZE = 50 + # The page size belongs to the layer that owns this endpoint, not to this + # call site: asking for 50 where ``fetchers.executors`` asks 500 of the same + # ``search_executors`` cost 10x the sequential round trips for the same rows. + PAGE_SIZE = EXECUTORS_PAGE_SIZE # Safety cap, expressed in rows: the walker counts what it accumulated, not # how many times it looped, and its own terminal guards end a stalled walk. - MAX_PAGES = 200 # → 10,000 executors per agent + # Stated as rows so it stays put when the page size moves. + MAX_ROWS = 10_000 # executors per agent async def _fetch_rows(aid: str) -> list[dict]: rows: list[dict] = [] @@ -388,7 +393,7 @@ async def _fetch_rows(aid: str) -> list[dict]: partial(client.executors.search_executors, controller_ids=[aid]), _extract_executors_list, page_size=PAGE_SIZE, - max_items=MAX_PAGES * PAGE_SIZE, + max_items=MAX_ROWS, ): for ex in page: if isinstance(ex, dict): diff --git a/tests/test_agent_performance_paging.py b/tests/test_agent_performance_paging.py new file mode 100644 index 000000000..fc8cb0f36 --- /dev/null +++ b/tests/test_agent_performance_paging.py @@ -0,0 +1,95 @@ +"""PERF-599: the agent performance walk pages at the fetchers layer's page size. + +``fetch_agent_performance_batch`` used to declare its own ``PAGE_SIZE = 50`` +against the same ``client.executors.search_executors`` that +``condor.fetchers.executors`` walks 500 rows at a time — ten times the +sequential round trips for identical rows, on every agent tick and every 30s +web rollup, fanned out over every active agent at once. + +These tests pin the two halves of that change: the page size now comes from the +fetchers' own constant, and the safety cap stays where it was (10,000 rows per +agent) rather than moving with the page size. +""" + +import asyncio + +from condor.agents.performance import fetch_agent_performance_batch +from condor.fetchers.executors import EXECUTORS_PAGE_SIZE + +MAX_ROWS = 10_000 # the per-agent row cap, pinned here so a drift fails loudly + + +class PagingClient: + """Serves a fixed history in cursor-advancing pages of exactly ``limit``. + + The cursor always advances and is always present, so the walk ends only on + a short/empty page — which is what makes the recorded call count a faithful + measure of the page size actually asked for. + """ + + def __init__(self, total_rows: int): + self._rows = [ + {"id": f"e{i}", "status": "TERMINATED"} for i in range(total_rows) + ] + self.calls: list[dict] = [] + self.executors = self._Executors(self) + + class _Executors: + def __init__(self, outer): + self._outer = outer + + async def search_executors(self, **kwargs): + outer = self._outer + outer.calls.append(kwargs) + offset = int(kwargs.get("cursor") or 0) + limit = kwargs["limit"] + page = outer._rows[offset : offset + limit] + return {"executors": page, "next_cursor": str(offset + len(page))} + + @property + def limits(self) -> list[int]: + return [call["limit"] for call in self.calls] + + +def test_a_thousand_row_history_costs_three_requests_not_twenty_one(): + """500 + 500 + a short final page, instead of 20 pages of 50 plus one.""" + client = PagingClient(1_000) + + out = asyncio.run(fetch_agent_performance_batch(client, ["agent-1"])) + + assert client.limits == [EXECUTORS_PAGE_SIZE] * 3 + assert len(client.calls) == 3, f"walked in {len(client.calls)} requests" + assert out["agent-1"].trade_count == 1_000 + + +def test_every_row_of_the_history_still_lands_exactly_once(): + """A bigger page must not drop, duplicate or reorder the rows it carries.""" + client = PagingClient(1_234) + + perf = asyncio.run(fetch_agent_performance_batch(client, ["agent-1"]))["agent-1"] + + ids = [row["id"] for row in perf.executors] + assert ids == [f"e{i}" for i in range(1_234)] + + +def test_the_row_cap_stays_at_ten_thousand(): + """The cap is expressed in rows, so it does not scale with the page size.""" + client = PagingClient(MAX_ROWS + 2_500) + + perf = asyncio.run(fetch_agent_performance_batch(client, ["agent-1"]))["agent-1"] + + assert perf.trade_count == MAX_ROWS + assert len(perf.executors) == MAX_ROWS + # The walker clamps each page to what is left of the cap, so no request + # fetches rows that would be discarded, and none overshoots it. + assert sum(client.limits) == MAX_ROWS + assert client.limits == [EXECUTORS_PAGE_SIZE] * (MAX_ROWS // EXECUTORS_PAGE_SIZE) + + +def test_the_walk_asks_for_no_hardcoded_page_size(): + """The limit on the wire is the fetchers' constant, not a local literal.""" + client = PagingClient(10) + + asyncio.run(fetch_agent_performance_batch(client, ["agent-1"])) + + assert client.limits == [EXECUTORS_PAGE_SIZE] diff --git a/tests/test_pagination_cursor_progress.py b/tests/test_pagination_cursor_progress.py index 62f033869..5e8198fb8 100644 --- a/tests/test_pagination_cursor_progress.py +++ b/tests/test_pagination_cursor_progress.py @@ -13,6 +13,7 @@ import asyncio from condor.agents.performance import fetch_agent_performance_batch +from condor.fetchers.executors import EXECUTORS_PAGE_SIZE from condor.web.ws_manager import WebSocketManager # The page sizes each loop asks for, mirrored here so the fake can answer with a @@ -20,7 +21,9 @@ # cursor guard). WS_FIRST_PAGE = 50 WS_NEXT_PAGE = 500 -PERF_PAGE_SIZE = 50 +# Tracks the walk's real page size (PERF-599 moved it to the fetchers' +# constant): a page shorter than the limit ends the walk before the guard. +PERF_PAGE_SIZE = EXECUTORS_PAGE_SIZE def _page(start, count): From b2332471723efb51f790eb9cef81d721674477ba Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 16:13:31 +0300 Subject: [PATCH 083/154] Stop re-walking the fleet's whole history because a PnL cell ticked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fleet chart's snapshot filter was memoized on the `controllers` array, and that array is rebuilt from scratch — fresh objects and all — on every ~5s `bots` frame. Live PnL moves on every one of them, so structural sharing could never hold it still and the memo was invalidated roughly six times more often than the thing it actually reads: `perfHistory` only changes on the 30s `controller_perf` frame. Each of those runs walked the fleet's entire performance history, a `controllerKey` and a `Date.parse` per row, and `.filter()` handed back a new array every time — so every downstream fold keyed on `fleet.snapshots` re-aggregated too, to redraw exactly the same chart. Membership and deploy time are everything the predicate reads. Splitting that identity out as a roster signature and keying the deploy map on it leaves a PnL-only frame referentially identical all the way down, and the walk happens only when the roster or a deploy time genuinely changes. The rows that come out are unchanged, CORR-241's bot+controller keying included; only the number of times they are computed is. Lifted into its own hook so that promise can be tested: the test asserts the returned array is the same array across a PnL-only frame and that `Date.parse` is not called at all, both of which fail on the old shape. --- .../src/hooks/useActiveSnapshots.test.tsx | 193 ++++++++++++++++++ frontend/src/hooks/useActiveSnapshots.ts | 102 +++++++++ frontend/src/hooks/useFleetData.ts | 32 +-- 3 files changed, 303 insertions(+), 24 deletions(-) create mode 100644 frontend/src/hooks/useActiveSnapshots.test.tsx create mode 100644 frontend/src/hooks/useActiveSnapshots.ts diff --git a/frontend/src/hooks/useActiveSnapshots.test.tsx b/frontend/src/hooks/useActiveSnapshots.test.tsx new file mode 100644 index 000000000..5e48b50b7 --- /dev/null +++ b/frontend/src/hooks/useActiveSnapshots.test.tsx @@ -0,0 +1,193 @@ +/** + * The fleet chart's snapshot filter, and the frames it must ignore. + * + * The promise under test is not "the same rows come out" — that was always + * true. It is that the *work* does not happen: a `bots` frame that moved only a + * PnL cell must not walk the fleet's whole performance history again, and must + * not hand the downstream folds a new array to re-aggregate (PERF-334). Both + * are asserted the only way they can be asserted from outside — the returned + * array is referentially identical, and `Date.parse`, which the filter calls + * once per history row, is not called at all. + * + * Against the pre-fix memo — keyed on the `controllers` array, which the socket + * rebuilds from scratch every ~5s — both of those fail: a fresh array is a new + * dependency, so the walk re-runs and `.filter()` returns a new array. + * + * @vitest-environment jsdom + */ + +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { ControllerInfo, ControllerPerformanceSnapshot } from "@/lib/api"; +import { useActiveSnapshots } from "./useActiveSnapshots"; + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +/** A live controller off the `bots` payload. */ +const ctrl = ( + bot: string, + id: string, + deployedAt: string | null, + pnl = 0, +): ControllerInfo => ({ + controller_name: id, + controller_type: "generic", + controller_id: id, + bot_name: bot, + status: "running", + connector: "binance", + trading_pair: "SOL-USDC", + realized_pnl_quote: 0, + unrealized_pnl_quote: 0, + global_pnl_quote: pnl, + global_pnl_pct: 0, + volume_traded: 0, + close_type_counts: {}, + positions_summary: [], + deployed_at: deployedAt, + config: {}, +}); + +/** One stored row of controller-performance history. */ +const snap = (bot: string, id: string, timestamp: string): ControllerPerformanceSnapshot => ({ + timestamp, + bot_name: bot, + controller_id: id, + controller_name: id, + connector: "binance", + trading_pair: "SOL-USDC", + realized_pnl_quote: 0, + unrealized_pnl_quote: 0, + global_pnl_quote: 0, + global_pnl_pct: 0, + volume_traded: 0, + positions_summary: [], +}); + +let container: HTMLDivElement; +let root: Root; +/** Every value the hook has returned, newest last. */ +let seen: ControllerPerformanceSnapshot[][]; + +function Probe({ + snapshots, + controllers, +}: { + snapshots: ControllerPerformanceSnapshot[] | undefined; + controllers: ControllerInfo[]; +}) { + seen.push(useActiveSnapshots(snapshots, controllers)); + return null; +} + +function draw(snapshots: ControllerPerformanceSnapshot[] | undefined, controllers: ControllerInfo[]) { + act(() => { + root.render(); + }); + return seen[seen.length - 1]; +} + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + seen = []; +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + vi.restoreAllMocks(); +}); + +describe("useActiveSnapshots", () => { + const history = [ + snap("bot-a", "c1", "2026-09-01T00:00:00Z"), + snap("bot-a", "c1", "2026-09-02T00:00:00Z"), + snap("bot-b", "c2", "2026-09-02T00:00:00Z"), + ]; + + it("does no work at all on a bots frame that moved only PnL", () => { + const roster = [ctrl("bot-a", "c1", "2026-08-31T00:00:00Z"), ctrl("bot-b", "c2", null)]; + const first = draw(history, roster); + expect(first).toHaveLength(3); + + // The next `bots` frame: a wholly fresh array of wholly fresh objects, as + // `shared-socket.ts` builds one every ~5s, differing only in live PnL. + const parse = vi.spyOn(Date, "parse"); + const nextFrame = [ + ctrl("bot-a", "c1", "2026-08-31T00:00:00Z", 12.34), + ctrl("bot-b", "c2", null, -5.5), + ]; + expect(nextFrame).not.toBe(roster); + const second = draw(history, nextFrame); + + // Same array, not merely equal: every downstream fold keyed on + // `fleet.snapshots` therefore does not re-run either. + expect(second).toBe(first); + // And the history was not re-walked: the filter parses a timestamp per row. + expect(parse).not.toHaveBeenCalled(); + }); + + it("re-runs when a controller joins the roster", () => { + const first = draw(history, [ctrl("bot-a", "c1", null)]); + expect(first).toHaveLength(2); + + const second = draw(history, [ctrl("bot-a", "c1", null), ctrl("bot-b", "c2", null)]); + expect(second).not.toBe(first); + expect(second).toHaveLength(3); + }); + + it("re-runs when a deploy time changes", () => { + const first = draw(history, [ctrl("bot-a", "c1", "2026-08-31T00:00:00Z")]); + expect(first).toHaveLength(2); + + const second = draw(history, [ctrl("bot-a", "c1", "2026-09-01T12:00:00Z")]); + expect(second).not.toBe(first); + expect(second.map((s) => s.timestamp)).toEqual(["2026-09-02T00:00:00Z"]); + }); + + it("cuts each bot at its own deploy time when two share a controller id (CORR-241)", () => { + const shared = [ + snap("old-bot", "c1", "2026-09-01T00:00:00Z"), + snap("old-bot", "c1", "2026-09-05T00:00:00Z"), + snap("new-bot", "c1", "2026-09-01T00:00:00Z"), + snap("new-bot", "c1", "2026-09-05T00:00:00Z"), + ]; + const out = draw(shared, [ + ctrl("old-bot", "c1", "2026-08-30T00:00:00Z"), + ctrl("new-bot", "c1", "2026-09-04T00:00:00Z"), + ]); + + expect(out.map((s) => `${s.bot_name}@${s.timestamp}`)).toEqual([ + "old-bot@2026-09-01T00:00:00Z", + "old-bot@2026-09-05T00:00:00Z", + "new-bot@2026-09-05T00:00:00Z", + ]); + }); + + it("drops snapshots whose controller is gone, and keeps a row with no deploy time", () => { + const out = draw(history, [ctrl("bot-b", "c2", null)]); + expect(out.map((s) => s.bot_name)).toEqual(["bot-b"]); + }); + + it("returns one stable empty array before any controller has arrived", () => { + const first = draw(history, []); + const second = draw(history, []); + expect(first).toEqual([]); + expect(second).toBe(first); + }); + + it("returns one stable empty array while the history is still loading", () => { + const roster = [ctrl("bot-a", "c1", null)]; + const first = draw(undefined, roster); + const second = draw(undefined, [ctrl("bot-a", "c1", null, 9)]); + expect(first).toEqual([]); + expect(second).toBe(first); + }); +}); diff --git a/frontend/src/hooks/useActiveSnapshots.ts b/frontend/src/hooks/useActiveSnapshots.ts new file mode 100644 index 000000000..5d1928645 --- /dev/null +++ b/frontend/src/hooks/useActiveSnapshots.ts @@ -0,0 +1,102 @@ +/** + * The fleet chart's snapshot filter, held still between `controller_perf` frames. + * + * The set of performance snapshots the fleet browser draws is + * `perfHistory.snapshots` narrowed to the controllers that are currently + * deployed, each one cut at its own deploy time. Both halves of that verdict + * come from the live `bots` payload — and that payload is re-broadcast whole + * every ~5s (`hummingbot_ws.py` subscribes with `update_interval=5.0`, and + * `shared-socket.ts` writes a fresh `controllers` array of fresh objects into + * the cache for each frame). Live PnL cells move on every one of those frames, + * so structural sharing can never hold the array still. + * + * Memoizing the filter on the `controllers` array therefore re-ran it roughly + * every 5 seconds, walking the fleet's *entire* history — thousands to tens of + * thousands of rows at the sampling intervals PERF-238 picks — with a + * `controllerKey` and a `Date.parse` per row, and handing every downstream fold + * a brand-new array to re-aggregate. All of that to answer the same question + * with the same answer: `perfHistory` itself only changes on the 30s + * `controller_perf` frame. + * + * So the identity the filter actually depends on is split out of the payload + * (PERF-334). Membership and deploy time are everything the predicate reads; + * nothing else about a controller can change its verdict. Keying the deploy map + * on a roster signature — the controller keys joined to their deploy times — + * means a frame that moved only PnL leaves the map, and therefore the filtered + * array, referentially identical, and the walk happens only when the roster or + * a deploy time genuinely changes. + */ +import { useMemo } from "react"; + +import type { ControllerInfo, ControllerPerformanceSnapshot } from "@/lib/api"; +import { controllerKey } from "@/lib/controller-identity"; + +/** Held still, so "no controllers yet" is not a new array every render. */ +const EMPTY_SNAPSHOTS: ControllerPerformanceSnapshot[] = []; + +/** + * Everything about the roster that can change the filter's verdict, as a string. + * + * Which controllers exist and when each was deployed — nothing else. Cheap to + * recompute every render (one pass over a fleet-sized list of controllers, not + * over the history), and equal across frames that moved only PnL. + */ +export function deployRosterSignature(controllers: ControllerInfo[]): string { + return controllers.map((c) => `${controllerKey(c)}:${c.deployed_at ?? ""}`).join("|"); +} + +/** + * Active controller key → its deploy time in ms (0 when unknown). + * + * Keyed by bot + controller, because a bare controller id is a config id two + * bots can share: one map entry per id meant last-write-wins on the deploy + * time, so an hour-old bot truncated its five-day sibling's history to an hour + * of points (CORR-241). + */ +export function buildDeployByKey(controllers: ControllerInfo[]): Map { + const deployByKey = new Map(); + for (const ctrl of controllers) { + const deployMs = ctrl.deployed_at ? Date.parse(ctrl.deployed_at) : 0; + deployByKey.set(controllerKey(ctrl), deployMs); + } + return deployByKey; +} + +/** The snapshots belonging to an active controller, from its deploy time on. */ +export function filterActiveSnapshots( + snapshots: ControllerPerformanceSnapshot[], + deployByKey: Map, +): ControllerPerformanceSnapshot[] { + return snapshots.filter((snap) => { + const key = controllerKey(snap); + if (!key || !deployByKey.has(key)) return false; + const deployMs = deployByKey.get(key)!; + if (!deployMs) return true; // no deploy time known, keep it + const snapMs = Date.parse(snap.timestamp) || 0; + return snapMs >= deployMs; + }); +} + +/** + * The filtered snapshot set, recomputed only when it can actually differ. + * + * Stable across every `bots` frame that left the roster and its deploy times + * alone, which is nearly all of them. + */ +export function useActiveSnapshots( + snapshots: ControllerPerformanceSnapshot[] | undefined, + controllers: ControllerInfo[], +): ControllerPerformanceSnapshot[] { + const signature = deployRosterSignature(controllers); + + // Keyed on the signature, not the array: the array is fresh every frame and + // the signature is not. `controllers` is read here only to rebuild the map + // the signature already decided has changed. + // eslint-disable-next-line react-hooks/exhaustive-deps + const deployByKey = useMemo(() => buildDeployByKey(controllers), [signature]); + + return useMemo(() => { + if (!snapshots || deployByKey.size === 0) return EMPTY_SNAPSHOTS; + return filterActiveSnapshots(snapshots, deployByKey); + }, [snapshots, deployByKey]); +} diff --git a/frontend/src/hooks/useFleetData.ts b/frontend/src/hooks/useFleetData.ts index bb76d8173..06da3965f 100644 --- a/frontend/src/hooks/useFleetData.ts +++ b/frontend/src/hooks/useFleetData.ts @@ -16,6 +16,7 @@ import type { DeedIndex, FleetOwner } from "@/lib/agent-attribution"; import type { ExecutorPaging } from "@/components/perf/PerfBrowser"; import type { Population } from "@/lib/perf-tree"; import { controllerKey } from "@/lib/controller-identity"; +import { useActiveSnapshots } from "@/hooks/useActiveSnapshots"; import { historyRowBudget } from "@/lib/history-pagination"; import { HISTORY_REFETCH_MS, @@ -254,30 +255,13 @@ export function useFleetData( [controllers], ); - // Filter performance snapshots to only active controllers and current run - const activeSnapshots = useMemo(() => { - if (!perfHistory?.snapshots || controllers.length === 0) return []; - - // Build set of active controller keys and their deploy times. Keyed by - // bot + controller, because a bare controller id is a config id two bots - // can share: one map entry per id meant last-write-wins on the deploy - // time, so an hour-old bot truncated its five-day sibling's history to an - // hour of points (CORR-241). - const activeControllers = new Map(); // key -> deployedAt ms - for (const ctrl of controllers) { - const deployMs = ctrl.deployed_at ? Date.parse(ctrl.deployed_at) : 0; - activeControllers.set(controllerKey(ctrl), deployMs); - } - - return perfHistory.snapshots.filter((snap) => { - const key = controllerKey(snap); - if (!key || !activeControllers.has(key)) return false; - const deployMs = activeControllers.get(key)!; - if (!deployMs) return true; // no deploy time known, keep it - const snapMs = Date.parse(snap.timestamp) || 0; - return snapMs >= deployMs; - }); - }, [perfHistory, controllers]); + // Filter performance snapshots to only active controllers and current run. + // + // Keyed on the roster's *identity* rather than the `controllers` array, which + // is rebuilt from scratch on every ~5s `bots` frame: see `useActiveSnapshots` + // for why walking the whole history six times more often than it changes was + // the only thing that bought (PERF-334). + const activeSnapshots = useActiveSnapshots(perfHistory?.snapshots, controllers); // ── The executors the browser hangs under those controllers ── From d3207a41278544568f8cdb6b54761f3ca2c8eca2 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 16:19:59 +0300 Subject: [PATCH 084/154] Stop prefetching two caches no component can read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit usePrefetchData warmed a 5,000-row candles entry and a ["connectors", server] entry on every app load and every server switch, and nothing could ever read either one. The candles key came from candlesQuery with only a start bound, so its endTime slot was null. The one other caller, ExecutorChart, always passes both bounds, so its key never has a null end; the /trade chart does not use react-query at all, fetching straight into candleStore with an unbucketed start, so not even the URL matched. The entry was fetched and then garbage-collected, and it is the heaviest request the dashboard makes. ["connectors", server] was likewise declared by no component, and that prefetch was the only caller of api.getConnectors, which goes with it. Neither removal costs a warm cache, because neither cache was ever read: /trade still issues its own candles request exactly as before, and ExecutorChart's key is untouched. App load now issues eight requests per server instead of ten. The remaining prefetches — bots, executors, connected-exchanges and its trading-rules fan-out, and the settings-* set — all have real observers and stay. --- frontend/src/hooks/usePrefetchData.test.tsx | 130 ++++++++++++++++++++ frontend/src/hooks/usePrefetchData.ts | 66 ++-------- frontend/src/lib/api.ts | 5 - 3 files changed, 140 insertions(+), 61 deletions(-) create mode 100644 frontend/src/hooks/usePrefetchData.test.tsx diff --git a/frontend/src/hooks/usePrefetchData.test.tsx b/frontend/src/hooks/usePrefetchData.test.tsx new file mode 100644 index 000000000..2fab24426 --- /dev/null +++ b/frontend/src/hooks/usePrefetchData.test.tsx @@ -0,0 +1,130 @@ +/** + * That app-load prefetching only warms keys a component can actually read + * (PERF-335). + * + * `usePrefetchData` fires on every app load and again on every server switch, + * so each request in it is paid for repeatedly. Two used to buy nothing: a + * 5,000-row candles fetch — the heaviest request the dashboard makes — stored + * under a `candlesQuery` key whose `endTime` slot is null, which no reader ever + * asks for (ExecutorChart always passes both bounds; TradeChart doesn't use + * react-query at all), and `["connectors", server]`, a key no component + * declares. Both entries were only ever garbage-collected. + * + * The assertion is therefore a request count, not a key-shape check: with a + * server selected, the hook must issue the prefetches that have observers and + * *no* candles or connectors call. Counting `api` calls rather than the query + * cache is deliberate — a dead cache entry is invisible, the round trip is not. + * + * Needs a DOM, so this file overrides vitest's default `node` environment. + * + * @vitest-environment jsdom + */ + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ServerContext } from "@/hooks/useServer"; + +/** Every `api` member the prefetch hook may reach for, each counted. */ +const calls = { + getExecutors: vi.fn(async () => []), + getBots: vi.fn(async () => []), + getConnectedExchanges: vi.fn(async () => ["binance"]), + getTradingRules: vi.fn(async () => ({})), + getSettingsServers: vi.fn(async () => []), + getCredentials: vi.fn(async () => []), + getAvailableConnectors: vi.fn(async () => []), + // Kept in the stub on purpose: if the hook ever calls these again, the test + // must fail on the count, not blow up on an undefined member. + getCandles: vi.fn(async () => []), + getConnectors: vi.fn(async () => []), +}; + +vi.mock("@/lib/api", () => ({ api: calls })); + +const { usePrefetchData } = await import("./usePrefetchData"); + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +function Probe() { + usePrefetchData(); + return null; +} + +let container: HTMLDivElement; +let root: Root; + +/** Mounts the hook with `server` selected and lets its prefetches settle. */ +async function mount(server: string | null) { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false, gcTime: 0 } }, + }); + await act(async () => { + root.render( + + {} }}> + + + , + ); + }); + // The trading-rules fan-out hangs off a resolved promise, so give the + // microtask queue a turn before counting. + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); +} + +describe("usePrefetchData", () => { + beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + for (const fn of Object.values(calls)) fn.mockClear(); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + }); + + afterEach(() => { + act(() => root.unmount()); + container.remove(); + }); + + it("spends no request on the candles and connectors keys nothing observes", async () => { + await mount("alpha"); + + expect(calls.getCandles).not.toHaveBeenCalled(); + expect(calls.getConnectors).not.toHaveBeenCalled(); + }); + + it("still warms every prefetch that has a real observer", async () => { + await mount("alpha"); + + expect(calls.getExecutors).toHaveBeenCalledTimes(1); + expect(calls.getBots).toHaveBeenCalledTimes(1); + expect(calls.getConnectedExchanges).toHaveBeenCalledTimes(1); + expect(calls.getTradingRules).toHaveBeenCalledWith("alpha", "binance"); + expect(calls.getSettingsServers).toHaveBeenCalledTimes(1); + expect(calls.getCredentials).toHaveBeenCalledTimes(1); + // spot + perpetual + expect(calls.getAvailableConnectors).toHaveBeenCalledTimes(2); + }); + + it("issues eight requests per server, not the ten it used to", async () => { + await mount("alpha"); + + const total = Object.values(calls).reduce((n, fn) => n + fn.mock.calls.length, 0); + expect(total).toBe(8); + }); + + it("issues nothing at all until a server is selected", async () => { + await mount(null); + + const total = Object.values(calls).reduce((n, fn) => n + fn.mock.calls.length, 0); + expect(total).toBe(0); + }); +}); diff --git a/frontend/src/hooks/usePrefetchData.ts b/frontend/src/hooks/usePrefetchData.ts index 2478b081e..8d1a8f0df 100644 --- a/frontend/src/hooks/usePrefetchData.ts +++ b/frontend/src/hooks/usePrefetchData.ts @@ -3,36 +3,21 @@ import { useEffect } from "react"; import { useServer } from "@/hooks/useServer"; import { api } from "@/lib/api"; -import { candlesQuery, executorsQuery } from "@/lib/queryClient"; -import { GRID_STORAGE_KEY } from "@/lib/sessionState"; - -const DEFAULT_CONNECTOR = "binance_perpetual"; -const DEFAULT_PAIR = "BTC-USDT"; -const DEFAULT_INTERVAL = "5m"; -const DEFAULT_LOOKBACK = 3 * 86400; // 3 days - -function getTradeDefaults() { - try { - const raw = localStorage.getItem(GRID_STORAGE_KEY); - if (!raw) return { connector: DEFAULT_CONNECTOR, pair: DEFAULT_PAIR, interval: DEFAULT_INTERVAL, lookback: DEFAULT_LOOKBACK }; - const saved = JSON.parse(raw); - return { - connector: saved.connector || DEFAULT_CONNECTOR, - pair: saved.pair || DEFAULT_PAIR, - interval: saved.interval || DEFAULT_INTERVAL, - lookback: saved.lookbackSeconds || DEFAULT_LOOKBACK, - }; - } catch { - return { connector: DEFAULT_CONNECTOR, pair: DEFAULT_PAIR, interval: DEFAULT_INTERVAL, lookback: DEFAULT_LOOKBACK }; - } -} +import { executorsQuery } from "@/lib/queryClient"; /** * Prefetches core data when the app loads so pages render instantly * instead of showing a loading state on first visit. * - * Executors, bots, connectors, trading rules, and default candles - * are all fetched eagerly as soon as a server is selected. + * Every prefetch here must warm a key some component actually observes, + * otherwise it is a request whose only destination is the garbage collector. + * Two used not to (PERF-335): a 5,000-row candles fetch keyed with a null + * `endTime`, which no `candlesQuery` caller ever asks for (ExecutorChart always + * passes both bounds, and TradeChart bypasses react-query entirely, fetching + * straight into `candleStore`), and `["connectors", server]`, which no + * component declares. Warming the /trade chart, if ever wanted again, has to go + * through `candleStore.mergeCandles` under `candleChannelKey(...)` — the path + * that chart actually reads. */ export function usePrefetchData() { const { server } = useServer(); @@ -41,8 +26,6 @@ export function usePrefetchData() { useEffect(() => { if (!server) return; - const defaults = getTradeDefaults(); - // Core data queryClient.prefetchQuery({ queryKey: executorsQuery(server).queryKey, @@ -54,13 +37,6 @@ export function usePrefetchData() { queryFn: () => api.getBots(server), }); - // Prefetch candle connectors list (for market data dropdowns) - queryClient.prefetchQuery({ - queryKey: ["connectors", server], - queryFn: () => api.getConnectors(server), - staleTime: 5 * 60 * 1000, - }); - // Prefetch trading rules only for connected exchanges (with credentials), // not all candle connectors — avoids 404s for unconfigured connectors queryClient @@ -81,28 +57,6 @@ export function usePrefetchData() { }) .catch(() => {}); - // Prefetch candles for the default trade pair. The key carries this - // window, so it only ever serves a chart asking for the same range. - const candles = candlesQuery( - server, - defaults.connector, - defaults.pair, - defaults.interval, - Math.floor(Date.now() / 1000) - defaults.lookback, - ); - queryClient.prefetchQuery({ - queryKey: candles.queryKey, - queryFn: () => - api.getCandles( - server, - defaults.connector, - defaults.pair, - defaults.interval, - 5000, - candles.startTime, - ), - }); - // Prefetch settings data so Settings page loads instantly queryClient.prefetchQuery({ queryKey: ["settings-servers"], diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index ec308ea24..84df369ba 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -2630,11 +2630,6 @@ export const api = { `/api/v1/servers/${encodeURIComponent(server)}/positions`, ), - getConnectors: (server: string) => - apiFetch( - `/api/v1/servers/${encodeURIComponent(server)}/market/connectors`, - ), - getConnectedExchanges: (server: string) => apiFetch( `/api/v1/servers/${encodeURIComponent(server)}/market/connected-exchanges`, From a80a875d89bd408cb288b8e551b0e0085489c53a Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 16:26:55 +0300 Subject: [PATCH 085/154] (perf) let the socket, not a 10s poll, keep the agent's executors current MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit useAgentExecutors subscribes `executors:` and then reads the very key the shared socket writes every frame into — so its `refetchInterval: 10000, // Fallback polling` was not a fallback at all: an open agent session or StrategyWorkbench re-downloaded the whole newest-500 executors list six times a minute purely to overwrite data that had arrived two seconds earlier. And because react-query drives a shared key at the shortest interval any of its observers asks for, that 10s also pulled Portfolio's deliberate 60s down with it whenever the two were co-mounted. Both observers now name one constant, EXECUTORS_REFETCH_MS, so the cadence of this key cannot silently disagree with itself again. Nothing else changes: the subscription already keeps the rows live within ~2s, which the new test pins alongside the request count. --- frontend/src/hooks/useAgentExecutors.test.tsx | 153 ++++++++++++++++++ frontend/src/hooks/useAgentExecutors.ts | 11 +- frontend/src/lib/queryClient.ts | 13 ++ frontend/src/pages/Portfolio.tsx | 4 +- 4 files changed, 176 insertions(+), 5 deletions(-) create mode 100644 frontend/src/hooks/useAgentExecutors.test.tsx diff --git a/frontend/src/hooks/useAgentExecutors.test.tsx b/frontend/src/hooks/useAgentExecutors.test.tsx new file mode 100644 index 000000000..24285dbcf --- /dev/null +++ b/frontend/src/hooks/useAgentExecutors.test.tsx @@ -0,0 +1,153 @@ +/** + * That the agent's executor view rides the socket and only nets under it with + * REST once a minute (PERF-336). + * + * The hook subscribes `executors:` and then reads the *unfiltered* + * executors key — the very key `shared-socket.ts` writes every frame into, at + * the backend's 2s cadence. It nevertheless declared a 10s `refetchInterval` on + * it, so an open agent session re-downloaded the whole newest-500 list six + * times a minute to overwrite data two seconds old; and because react-query + * drives a shared key at the shortest interval any observer asks for, it also + * dragged Portfolio's deliberate 60s down to 10s. + * + * So what is pinned here is the request actually issued: how many times + * `api.getExecutors` is called over three minutes of wall clock, and that a + * socket write still reaches the caller with no request at all. + * + * Needs a DOM, so this file overrides vitest's default `node` environment. + * + * @vitest-environment jsdom + */ + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { ExecutorInfo } from "@/lib/api"; + +const getExecutors = vi.fn(); + +vi.mock("@/lib/api", () => ({ + api: { getExecutors: (...args: unknown[]) => getExecutors(...args) }, +})); + +// The subscription itself is not under test — only the REST cadence beside it. +vi.mock("@/hooks/useWebSocket", () => ({ useCondorWebSocket: () => undefined })); + +const { executorsQuery } = await import("@/lib/queryClient"); +const { useAgentExecutors } = await import("./useAgentExecutors"); + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +const SERVER = "srv"; + +/** An executor belonging to `controllerId`. */ +function executor(id: string, controllerId: string): ExecutorInfo { + return { + id, + type: "position_executor", + connector: "binance", + trading_pair: "SOL-USDC", + side: "BUY", + status: "active", + close_type: "", + pnl: 0, + volume: 0, + timestamp: 0, + controller_id: controllerId, + cum_fees_quote: 0, + net_pnl_pct: 0, + entry_price: 0, + current_price: 0, + close_timestamp: 0, + custom_info: {}, + config: {}, + } as ExecutorInfo; +} + +/** Renders what the hook hands back, so the DOM is the record of it. */ +function Harness() { + const { executors } = useAgentExecutors(SERVER, ["ctrl-1"]); + return <>{executors.map((e) => e.id).join(",")}; +} + +let container: HTMLDivElement; +let root: Root; +let client: QueryClient; + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + vi.useFakeTimers(); + getExecutors.mockReset().mockResolvedValue([executor("e1", "ctrl-1")]); + client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + client.clear(); + vi.useRealTimers(); +}); + +async function mount() { + await act(async () => { + root.render( + + + , + ); + }); + await act(async () => { + await vi.advanceTimersByTimeAsync(0); + }); +} + +/** Runs `ms` of wall clock, letting every poll it schedules resolve. */ +async function elapse(ms: number) { + await act(async () => { + await vi.advanceTimersByTimeAsync(ms); + }); +} + +describe("useAgentExecutors", () => { + it("issues one unfiltered executors request per minute, not six", async () => { + await mount(); + expect(getExecutors).toHaveBeenCalledTimes(1); // the initial read + + // Well past the old 10s cadence — which would have polled three times here. + await elapse(59_000); + expect(getExecutors).toHaveBeenCalledTimes(1); + + await elapse(2_000); + expect(getExecutors).toHaveBeenCalledTimes(2); + + // Three minutes of an open agent session: three requests, not eighteen. + await elapse(120_000); + expect(getExecutors).toHaveBeenCalledTimes(4); + }); + + it("still shows a socket frame's executors without any request", async () => { + await mount(); + expect(container.textContent).toBe("e1"); + const before = getExecutors.mock.calls.length; + + // What `shared-socket.ts` does with an `executors:` frame. + await act(async () => { + client.setQueryData(executorsQuery(SERVER).queryKey, [ + executor("e1", "ctrl-1"), + executor("e2", "ctrl-1"), + executor("e3", "other-ctrl"), + ]); + await vi.advanceTimersByTimeAsync(0); + }); + + expect(container.textContent).toBe("e1,e2"); + expect(getExecutors).toHaveBeenCalledTimes(before); + }); +}); diff --git a/frontend/src/hooks/useAgentExecutors.ts b/frontend/src/hooks/useAgentExecutors.ts index 47facbd25..791bc64d5 100644 --- a/frontend/src/hooks/useAgentExecutors.ts +++ b/frontend/src/hooks/useAgentExecutors.ts @@ -3,13 +3,15 @@ import { useQuery } from "@tanstack/react-query"; import { useCondorWebSocket } from "@/hooks/useWebSocket"; import { type ExecutorInfo, api } from "@/lib/api"; -import { executorsQuery } from "@/lib/queryClient"; +import { EXECUTORS_REFETCH_MS, executorsQuery } from "@/lib/queryClient"; /** * Hook to get real-time executor data for an agent by subscribing to the * existing executors:{server} WS channel and filtering by controller IDs. * - * Falls back to REST polling if WS is not connected. + * REST is the net under the socket, not the update path: it re-reads the list + * once a minute in case the socket is down, at the one cadence every observer + * of this shared key agrees on (`EXECUTORS_REFETCH_MS`). */ export function useAgentExecutors( serverName: string | null | undefined, @@ -26,7 +28,10 @@ export function useAgentExecutors( queryKey: executorsQuery(serverName).queryKey, queryFn: () => api.getExecutors(serverName!), enabled: !!serverName, - refetchInterval: 10000, // Fallback polling + // The socket is the update path, not a fallback: every `executors:` + // frame lands on this key. REST is only the net under it, at the cadence + // every other observer of this key uses. + refetchInterval: EXECUTORS_REFETCH_MS, }); // Filter executors to those matching the agent's controller IDs diff --git a/frontend/src/lib/queryClient.ts b/frontend/src/lib/queryClient.ts index 21724ad69..7d195541d 100644 --- a/frontend/src/lib/queryClient.ts +++ b/frontend/src/lib/queryClient.ts @@ -130,6 +130,19 @@ export type ExecutorsQueryKey = [ pair: string, ]; +/** + * Poll cadence for an executors entry that the socket already feeds. + * + * The `executors:` frame is the update path — `shared-socket.ts` writes + * every one of them (~2s) straight into the unfiltered key — so REST is only + * the net underneath it. It has to be one constant because react-query drives a + * shared key at the *shortest* interval any of its observers asks for: while a + * hook polling that key at 10s was mounted, it pulled Portfolio's deliberate + * 60s down with it and the whole newest-500 list was re-downloaded six times a + * minute to overwrite data two seconds old (PERF-336). + */ +export const EXECUTORS_REFETCH_MS = 60_000; + export function executorsQuery( server: string | null | undefined, opts: { controllerId?: string; pair?: string } = {}, diff --git a/frontend/src/pages/Portfolio.tsx b/frontend/src/pages/Portfolio.tsx index a086e48a6..447335b5d 100644 --- a/frontend/src/pages/Portfolio.tsx +++ b/frontend/src/pages/Portfolio.tsx @@ -30,7 +30,7 @@ import { type PortfolioHistoryResponse, } from "@/lib/api"; import { formatCurrency, formatCurrencyPnl, formatCurrencyVolume, isExecutorActive } from "@/lib/formatters"; -import { executorsQuery } from "@/lib/queryClient"; +import { EXECUTORS_REFETCH_MS, executorsQuery } from "@/lib/queryClient"; import { getThemeColors } from "@/lib/theme-colors"; // ── Formatters ── @@ -809,7 +809,7 @@ export function Portfolio() { queryKey: executorsQuery(server).queryKey, queryFn: () => api.getExecutors(server!), enabled: !!server, - refetchInterval: 60000, + refetchInterval: EXECUTORS_REFETCH_MS, placeholderData: keepPreviousData, }); From 2fc0c3ffe82275ad6e3a5180f3bfb1bd8abd00d0 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 16:34:26 +0300 Subject: [PATCH 086/154] Fold the performance history once at a scope that splits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PerfBrowser built two chart candidates side by side. `ownerChart` runs `aggregatePnlSeries` once per owner and once more over the union of their spine keys; `controllerPoints` then ran it over that same union again, because `buildTree`'s invariant is that the children's spines partition the scope's own. Only one of the two is ever drawn — the JSX tests `ownerChart` first — so at the default fleet view an entire extra O(instants x controllers) fold, with a sort per controller, was computed and discarded on every WS frame that mints a new snapshots array. Both candidates are now gated on the branch that actually draws them. `controllerPoints` returns [] when `ownerChart` is present, and the `series` memo skips `resolvePerfSeries` entirely rather than letting it fall through to an `executorSeries` walk of every closed outcome in the terminated population. `chartData` and `notice` are read only inside the `!ownerChart` branch, so nothing rendered moves. No figure changes. The surviving folds are called with the same arguments and their output is untouched, so the drawn numbers are bit-identical; the discarded work is simply not done. --- .../perf/PerfBrowser.foldOnce.test.tsx | 280 ++++++++++++++++++ frontend/src/components/perf/PerfBrowser.tsx | 60 +++- 2 files changed, 325 insertions(+), 15 deletions(-) create mode 100644 frontend/src/components/perf/PerfBrowser.foldOnce.test.tsx diff --git a/frontend/src/components/perf/PerfBrowser.foldOnce.test.tsx b/frontend/src/components/perf/PerfBrowser.foldOnce.test.tsx new file mode 100644 index 000000000..22310489b --- /dev/null +++ b/frontend/src/components/perf/PerfBrowser.foldOnce.test.tsx @@ -0,0 +1,280 @@ +/** + * One fold per line, and not one more (PERF-338). + * + * `PerfBrowser` builds two chart candidates side by side: `ownerChart`, which + * is `aggregatePnlSeries` once per owner plus once over the union of their + * spine keys, and `controllerPoints`, which is `aggregatePnlSeries` over that + * same union again. The render only ever draws one of them — the JSX tests + * `ownerChart` first — so at every scope that splits, the second fold of the + * whole performance history was computed and thrown away, on every WS frame + * that mints a new `snapshots` array. + * + * `PerfBrowser.owners.test.tsx` already pins *which* chart is drawn. What this + * file pins is the work behind it: the count of folds per recompute, and that + * `resolvePerfSeries` — whose last resort walks every closed outcome — is not + * consulted at all for a chart that is not on screen. + * + * Nothing here is about the numbers: `aggregatePnlSeries` is called with the + * same arguments as before and its output is untouched, so the drawn figures + * are the ones `owners.test.tsx` already asserts. + * + * Needs a DOM, so this file overrides vitest's default `node` environment. + * + * @vitest-environment jsdom + */ + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { MemoryRouter } from "react-router-dom"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { + ControllerInfo, + ControllerPerformanceSnapshot, +} from "@/lib/api"; +import type { FleetOwner } from "@/lib/agent-attribution"; + +/** + * The counters, hoisted so the `vi.mock` factories below — which run before + * any module-level `const` in this file — can reach them. + */ +const spy = vi.hoisted(() => ({ + /** One entry per `aggregatePnlSeries` call: its key set, joined. */ + folds: [] as string[], + /** How many times the chart's source was resolved. */ + resolves: 0, +})); + +// Counting wrappers, not stubs: the real fold still runs, so the chart the +// assertions below read is the real one. +vi.mock("@/lib/pnl-chart", async () => { + const actual = await vi.importActual("@/lib/pnl-chart"); + return { + ...actual, + aggregatePnlSeries: ((...args) => { + spy.folds.push([...args[1]].sort().join("+")); + return actual.aggregatePnlSeries(...args); + }) as typeof actual.aggregatePnlSeries, + }; +}); + +vi.mock("@/lib/perf-history", async () => { + const actual = + await vi.importActual("@/lib/perf-history"); + return { + ...actual, + resolvePerfSeries: ((...args) => { + spy.resolves += 1; + return actual.resolvePerfSeries(...args); + }) as typeof actual.resolvePerfSeries, + }; +}); + +// Everything the browser asks the API for is beside the point here. +vi.mock("@/lib/api", () => ({ + api: new Proxy({}, { get: () => () => Promise.resolve({}) }), +})); + +vi.mock("@/components/bots/PnlEvolutionChart", () => ({ + PnlEvolutionChart: () =>
, +})); +vi.mock("@/components/bots/ControllerPnlChart", () => ({ ControllerPnlChart: () => null })); +vi.mock("@/components/charts/ExecutorChart", () => ({ ExecutorChart: () => null })); +vi.mock("@/components/editor/EditorModal", () => ({ EditorModal: () => null })); +vi.mock("@/components/bots/LogsSection", () => ({ LogsSection: () => null })); +vi.mock("@/components/bots/DeployBotDialog", () => ({ DeployBotDialog: () => null })); +vi.mock("@/components/bots/ArchivedBotDetail", () => ({ ArchivedBotDetail: () => null })); +vi.mock("@/components/perf/YamlConfigEditor", () => ({ YamlConfigEditor: () => null })); +vi.mock("@/hooks/useWebSocket", () => ({ useCondorWebSocket: () => {} })); + +const { PerfBrowser } = await import("@/components/perf/PerfBrowser"); + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +const DEPLOYED = new Date(Date.now() - 3 * 3_600_000).toISOString(); + +function controller(over: Partial): ControllerInfo { + return { + controller_name: "pmm_simple", + controller_type: "", + controller_id: "c1", + bot_name: "alpha-mm-1", + status: "running", + connector: "binance", + trading_pair: "SOL-USDC", + realized_pnl_quote: 0, + unrealized_pnl_quote: 0, + global_pnl_quote: 0, + global_pnl_pct: 0, + volume_traded: 0, + close_type_counts: {}, + positions_summary: [], + deployed_at: DEPLOYED, + config: {}, + ...over, + } as ControllerInfo; +} + +function snap( + bot: string, + id: string, + minutesAgo: number, + pnl: number, +): ControllerPerformanceSnapshot { + return { + timestamp: new Date(Date.now() - minutesAgo * 60_000).toISOString(), + bot_name: bot, + controller_id: id, + controller_name: "pmm_simple", + connector: "binance", + trading_pair: "SOL-USDC", + realized_pnl_quote: pnl, + unrealized_pnl_quote: 0, + global_pnl_quote: pnl, + global_pnl_pct: 0, + volume_traded: 0, + positions_summary: [], + } as unknown as ControllerPerformanceSnapshot; +} + +function owner(slug: string, name: string, bot: string): FleetOwner { + return { + runKey: `${slug}.mm`, + agentSlug: slug, + agentName: name, + strategySlug: "mm", + strategyName: "MM", + namespace: `${slug}-mm`, + declaredBots: [bot], + agentIds: [], + live: null, + } as unknown as FleetOwner; +} + +const CONTROLLERS = [ + controller({ controller_id: "c1", bot_name: "alpha-mm-1", global_pnl_quote: 30 }), + controller({ controller_id: "c3", bot_name: "alpha-mm-2", global_pnl_quote: 12 }), + controller({ + controller_id: "c2", + bot_name: "beta-mm-1", + trading_pair: "BTC-USDT", + connector: "kucoin", + global_pnl_quote: -8, + }), +]; + +const SNAPSHOTS = [ + snap("alpha-mm-1", "c1", 120, 10), + snap("alpha-mm-1", "c1", 60, 30), + snap("alpha-mm-2", "c3", 120, 4), + snap("alpha-mm-2", "c3", 60, 12), + snap("beta-mm-1", "c2", 120, -2), + snap("beta-mm-1", "c2", 60, -8), +]; + +const OWNERS = [owner("alpha", "Alpha", "alpha-mm-1"), owner("beta", "Beta", "beta-mm-1")]; + +let container: HTMLDivElement; +let root: Root; + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + Element.prototype.scrollIntoView = () => {}; + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + spy.folds = []; + spy.resolves = 0; +}); + +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); +}); + +const qc = () => new QueryClient({ defaultOptions: { queries: { retry: false } } }); + +async function draw(url: string, snapshots: ControllerPerformanceSnapshot[]) { + await act(async () => { + root.render( + + + ({ value, converted: true })} + currencySymbol="$" + snapshots={snapshots} + owners={OWNERS} + deeds={{ bots: {}, since: 1 }} + /> + + , + ); + }); + for (let i = 0; i < 3; i++) { + await act(async () => { + await new Promise((r) => setTimeout(r, 0)); + }); + } +} + +/** + * A WS frame: the same history in a new array, which is what + * `useFleetData`'s `activeSnapshots` mints whenever the bots socket speaks. + * Everything the memos below depend on is re-read; nothing the reader sees + * changes. + */ +async function newFrame(url: string) { + await draw(url, SNAPSHOTS); + spy.folds = []; + spy.resolves = 0; + await draw(url, [...SNAPSHOTS]); +} + +const pick = (selector: string) => container.querySelector(selector); + +describe("a scope that splits", () => { + it("folds the history once per line and once for the Total, and not a third time over the union", async () => { + await newFrame("/bots"); + + // The chart on screen is still the split one. + expect(pick("[data-owner-chart]")).toBeTruthy(); + expect(pick("[data-aggregate-chart]")).toBeNull(); + + // One fold per owner, plus the Total. Before PERF-338 there were four: the + // union was folded twice, once for the Total and once for a candidate the + // render never reaches. + expect(spy.folds.length).toBe(OWNERS.length + 1); + + // And the union — the widest and most expensive of them — exactly once. + const union = spy.folds[0]; + expect(union.split("+").length).toBe(CONTROLLERS.length); + expect(spy.folds.filter((keys) => keys === union)).toEqual([union]); + }); + + it("does not resolve a chart source for a chart it does not draw", async () => { + await newFrame("/bots"); + + // `resolvePerfSeries`' last resort walks every closed outcome in scope. + // Nothing downstream of it is rendered here, so it is not asked. + expect(spy.resolves).toBe(0); + }); +}); + +describe("a scope that does not split", () => { + it("still folds once and still resolves its own aggregate series", async () => { + // One agent, one bot: there is nothing below it for a second line to be, + // so the single aggregate curve is the chart — exactly as before. + await newFrame("/bots?scope=agent:beta.mm"); + + expect(pick("[data-owner-chart]")).toBeNull(); + expect(pick("[data-aggregate-chart]")).toBeTruthy(); + expect(spy.folds.length).toBe(1); + expect(spy.resolves).toBeGreaterThan(0); + }); +}); diff --git a/frontend/src/components/perf/PerfBrowser.tsx b/frontend/src/components/perf/PerfBrowser.tsx index fd6ffd11c..95e6a1409 100644 --- a/frontend/src/components/perf/PerfBrowser.tsx +++ b/frontend/src/components/perf/PerfBrowser.tsx @@ -105,7 +105,11 @@ import { type GroupAxis, } from "@/lib/perf-grouping"; import { GroupByPicker } from "@/components/perf/GroupByPicker"; -import { resolvePerfSeries, scopeInterval } from "@/lib/perf-history"; +import { + resolvePerfSeries, + scopeInterval, + type PerfSeriesResult, +} from "@/lib/perf-history"; import { chartNotice } from "@/lib/perf-notices"; import { buildPositionRows, parseSide, type PositionRow } from "@/lib/perf-positions"; import { groupSpine } from "@/components/agent/floor/floor"; @@ -195,6 +199,15 @@ const FLEET_SCOPE = "all"; /** Held still, so an absent fleet map is not a new array on every render. */ const EMPTY_OWNERS: FleetOwner[] = []; +/** + * The series a splitting scope does not have. + * + * `unsupported` is `false` because nothing reads it: the flag exists so the + * chart's notice can say *why* a fallback was taken, and at a splitting scope + * neither the notice nor the chart it belongs to is rendered (PERF-338). + */ +const EMPTY_SERIES: PerfSeriesResult = { points: [], source: "none", unsupported: false }; + /** * How many finished runs are warmed when the reader switches to Terminated, * and how many walks run at once. @@ -1869,6 +1882,14 @@ export function PerfBrowser({ /** The controller-history candidate: what this scope drew before FEAT-087. */ const controllerPoints = useMemo(() => { + // This scope splits, so the chart on screen is `OwnerPnlChart` (the JSX + // below tests `ownerChart` first) and this candidate is never read. It is + // also the *same fold*: `ownerSeries` already ran `aggregatePnlSeries` over + // the union of the children's spine keys, which `buildTree` guarantees + // partitions this scope's own — so computing it here re-folded the entire + // performance history on every WS frame and threw the answer away + // (PERF-338). + if (ownerChart) return []; // A *live* controller draws its own finer series (see `ControllerPnlChart`). // A finished one does not: its curve is already in the run's cached history, // over the run's real window rather than deploy-to-now, and folding it here @@ -1907,7 +1928,7 @@ export function PerfBrowser({ } return aggregatePnlSeries(snapshots, scopedKeys, scopedControllers, convert); }, [ - activeCtrl, population, snapshots, scopedKeys, scopedControllers, + ownerChart, activeCtrl, population, snapshots, scopedKeys, scopedControllers, convert, runHistory, scopeRun, archiveOnlyController, ]); @@ -1921,19 +1942,28 @@ export function PerfBrowser({ */ const series = useMemo( () => - resolvePerfSeries({ - snapshots: execHistory?.supported === false ? undefined : execHistory?.snapshots, - controllerPoints, - // Only the terminated population has closes to fold. A running scope - // whose executors have all closed is a contradiction the tree does not - // produce, and offering the fold there would draw a "closed outcomes" - // curve under a live controller. - outcomes: population === "terminated" ? scope.leaves : undefined, - supported: perfCapability?.supported, - convert, - cv, - }), - [execHistory, controllerPoints, population, scope, perfCapability, convert, cv], + // Same gate as `controllerPoints` above, and for the same reason: when + // this scope splits, neither `chartData` nor `notice` reaches the render. + // Stopping at the candidates would still leave `resolvePerfSeries` to + // fold `scope.leaves` through `executorSeries` in the terminated + // population — a second discarded walk. Nothing is resolved instead + // (PERF-338). + ownerChart + ? EMPTY_SERIES + : resolvePerfSeries({ + snapshots: + execHistory?.supported === false ? undefined : execHistory?.snapshots, + controllerPoints, + // Only the terminated population has closes to fold. A running + // scope whose executors have all closed is a contradiction the tree + // does not produce, and offering the fold there would draw a + // "closed outcomes" curve under a live controller. + outcomes: population === "terminated" ? scope.leaves : undefined, + supported: perfCapability?.supported, + convert, + cv, + }), + [ownerChart, execHistory, controllerPoints, population, scope, perfCapability, convert, cv], ); const chartData = series.points; From 80f20c44ca5b518e0fb375aa30154e5c462688b2 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 16:40:37 +0300 Subject: [PATCH 087/154] A shut Breakdown band folds nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The band cuts the scope's spine two ways, by instrument and by venue, and each cut is a full extra pass over that spine: groupSpine buckets every leaf and then reads each bucket back. Their only consumer renders strictly under band === "breakdown", and the band is shut on arrival every time and deliberately not remembered — so for the whole of a typical session both cuts were folded and thrown away, once per WS frame and once per minute of the clock. Both memos now yield a held-still empty result until the band is open. groupSpine itself is untouched and, when the band does open, is called with exactly the arguments it was called with before, so not one figure in the two tables moves. PERF-340 --- .../perf/PerfBrowser.breakdown.test.tsx | 242 ++++++++++++++++++ frontend/src/components/perf/PerfBrowser.tsx | 34 ++- 2 files changed, 271 insertions(+), 5 deletions(-) create mode 100644 frontend/src/components/perf/PerfBrowser.breakdown.test.tsx diff --git a/frontend/src/components/perf/PerfBrowser.breakdown.test.tsx b/frontend/src/components/perf/PerfBrowser.breakdown.test.tsx new file mode 100644 index 000000000..29157cd94 --- /dev/null +++ b/frontend/src/components/perf/PerfBrowser.breakdown.test.tsx @@ -0,0 +1,242 @@ +/** + * A shut band folds nothing (PERF-340). + * + * The Breakdown band cuts the scope's spine two ways — by instrument and by + * venue — and each cut is a full extra pass over that spine: `groupSpine` + * buckets every leaf and then runs `readSpine` per bucket. Their only consumer + * is ``, which renders strictly under `band === "breakdown"`, + * and the band is shut on arrival every time and deliberately not remembered. + * So for the whole of a typical session both cuts were folded and thrown away, + * once per WS-driven `scope.leaves` change and once per 60s clock tick. + * + * What this file pins is the work, not the numbers: `groupSpine` is wrapped in + * a counter, not replaced, so the tables the second half asserts are the real + * ones, folded by the real fold with the real converter. `Σ buckets == the + * scope's own fold` is `floor.test.ts`' business and is untouched here. + * + * Needs a DOM, so this file overrides vitest's default `node` environment. + * + * @vitest-environment jsdom + */ + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { MemoryRouter } from "react-router-dom"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { ControllerInfo } from "@/lib/api"; + +/** Hoisted so the `vi.mock` factory below — which runs first — can reach it. */ +const spy = vi.hoisted(() => ({ /** One entry per `groupSpine` call. */ cuts: 0 })); + +// A counting wrapper, not a stub: the real bucketing still runs, so the tables +// read at the foot of this file are the ones the reader would see. +vi.mock("@/components/agent/floor/floor", async () => { + const actual = + await vi.importActual( + "@/components/agent/floor/floor", + ); + return { + ...actual, + groupSpine: ((...args) => { + spy.cuts += 1; + return actual.groupSpine(...args); + }) as typeof actual.groupSpine, + }; +}); + +// Everything the browser asks the API for is beside the point here. +vi.mock("@/lib/api", () => ({ + api: new Proxy({}, { get: () => () => Promise.resolve({}) }), +})); + +vi.mock("@/components/bots/PnlEvolutionChart", () => ({ PnlEvolutionChart: () => null })); +vi.mock("@/components/bots/ControllerPnlChart", () => ({ ControllerPnlChart: () => null })); +vi.mock("@/components/charts/ExecutorChart", () => ({ ExecutorChart: () => null })); +vi.mock("@/components/editor/EditorModal", () => ({ EditorModal: () => null })); +vi.mock("@/components/bots/LogsSection", () => ({ LogsSection: () => null })); +vi.mock("@/components/bots/DeployBotDialog", () => ({ DeployBotDialog: () => null })); +vi.mock("@/components/bots/ArchivedBotDetail", () => ({ ArchivedBotDetail: () => null })); +vi.mock("@/components/perf/YamlConfigEditor", () => ({ YamlConfigEditor: () => null })); +vi.mock("@/hooks/useWebSocket", () => ({ useCondorWebSocket: () => {} })); + +const { PerfBrowser } = await import("@/components/perf/PerfBrowser"); + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +const DEPLOYED = new Date(Date.now() - 3 * 3_600_000).toISOString(); + +function position(amount: number, price: number, side: string) { + return { amount, breakeven_price: price, side }; +} + +function controller(over: Partial): ControllerInfo { + return { + controller_name: "pmm_simple", + controller_type: "", + controller_id: "c1", + bot_name: "alpha-mm-1", + status: "running", + connector: "binance", + trading_pair: "SOL-USDC", + realized_pnl_quote: 0, + unrealized_pnl_quote: 0, + global_pnl_quote: 0, + global_pnl_pct: 0, + volume_traded: 0, + close_type_counts: {}, + positions_summary: [], + deployed_at: DEPLOYED, + config: {}, + ...over, + } as ControllerInfo; +} + +// Two instruments over two venues, with exposures far enough apart that the +// ranking below is a statement about the fold and not about a tie-break: +// SOL-USDC/binance is +2200 net long, BTC-USDT/kucoin is -5000 net short. +const CONTROLLERS = [ + controller({ + controller_id: "c1", + global_pnl_quote: 30, + positions_summary: [position(10, 200, "LONG")], + }), + controller({ + controller_id: "c3", + bot_name: "alpha-mm-2", + global_pnl_quote: 12, + positions_summary: [position(1, 200, "LONG")], + }), + controller({ + controller_id: "c2", + bot_name: "beta-mm-1", + trading_pair: "BTC-USDT", + connector: "kucoin", + global_pnl_quote: -8, + positions_summary: [position(0.1, 50_000, "SHORT")], + }), +]; + +let container: HTMLDivElement; +let root: Root; + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + Element.prototype.scrollIntoView = () => {}; + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + spy.cuts = 0; +}); + +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); +}); + +const qc = () => new QueryClient({ defaultOptions: { queries: { retry: false } } }); + +async function settle() { + for (let i = 0; i < 3; i++) { + await act(async () => { + await new Promise((r) => setTimeout(r, 0)); + }); + } +} + +async function draw(controllers: ControllerInfo[]) { + await act(async () => { + root.render( + + + ({ value, converted: true })} + currencySymbol="$" + snapshots={[]} + owners={[]} + deeds={{ bots: {}, since: 1 }} + /> + + , + ); + }); + await settle(); +} + +const pick = (selector: string) => container.querySelector(selector); + +const buckets = (which: string) => + [...container.querySelectorAll(`[data-breakdown="${which}"] [data-bucket]`)].map((li) => + li.getAttribute("data-bucket"), + ); + +async function click(selector: string) { + const el = pick(selector); + expect(el).toBeTruthy(); + await act(async () => { + el!.click(); + }); + await settle(); +} + +describe("the Breakdown band while it is shut", () => { + it("does not cut the spine at all", async () => { + await draw(CONTROLLERS); + + // The tab is offered — there is a spine to cut — but nothing has cut it. + expect(pick("[data-breakdown-toggle]")).toBeTruthy(); + expect(pick("[data-breakdown='pair']")).toBeNull(); + expect(spy.cuts).toBe(0); + }); + + it("does not cut it on a WS frame either", async () => { + await draw(CONTROLLERS); + spy.cuts = 0; + + // A new `controllers` array carrying the same fleet, which is what the + // bots socket mints on every frame: every dep of the two memos is re-read. + await draw([...CONTROLLERS]); + + expect(spy.cuts).toBe(0); + }); +}); + +describe("the Breakdown band once it is opened", () => { + it("cuts the spine twice and ranks both tables by absolute exposure", async () => { + await draw(CONTROLLERS); + await click("[data-breakdown-toggle]"); + + // One cut per table, and not one more. + expect(spy.cuts).toBe(2); + + // The short book is the bigger position, so it leads both tables. + expect(buckets("pair")).toEqual(["BTC-USDT", "SOL-USDC"]); + expect(buckets("venue")).toEqual(["kucoin", "binance"]); + expect(pick("[data-not-measured]")).toBeTruthy(); + }); + + it("stops cutting when the band is shut again, leaving the other tabs whole", async () => { + await draw(CONTROLLERS); + await click("[data-breakdown-toggle]"); + spy.cuts = 0; + + // `toggleBand` on the open band closes it. + await click("[data-breakdown-toggle]"); + expect(pick("[data-breakdown='pair']")).toBeNull(); + + // And a frame arriving over the shut band folds nothing. + await draw([...CONTROLLERS]); + expect(spy.cuts).toBe(0); + + // The band's other occupants are still offered and still open. + await click("[data-positions-toggle]"); + expect(pick("[data-breakdown='pair']")).toBeNull(); + expect(spy.cuts).toBe(0); + }); +}); diff --git a/frontend/src/components/perf/PerfBrowser.tsx b/frontend/src/components/perf/PerfBrowser.tsx index 95e6a1409..8251665b7 100644 --- a/frontend/src/components/perf/PerfBrowser.tsx +++ b/frontend/src/components/perf/PerfBrowser.tsx @@ -112,7 +112,7 @@ import { } from "@/lib/perf-history"; import { chartNotice } from "@/lib/perf-notices"; import { buildPositionRows, parseSide, type PositionRow } from "@/lib/perf-positions"; -import { groupSpine } from "@/components/agent/floor/floor"; +import { groupSpine, type FloorBucket } from "@/components/agent/floor/floor"; import { mergeOwnerRows, ownerSeries } from "@/lib/owner-series"; import { aggregatePnlSeries, snapshotsFromRunHistory } from "@/lib/pnl-chart"; import { buildAttributor, runWindows } from "@/lib/run-attribution"; @@ -208,6 +208,15 @@ const EMPTY_OWNERS: FleetOwner[] = []; */ const EMPTY_SERIES: PerfSeriesResult = { points: [], source: "none", unsupported: false }; +/** + * The cuts a shut Breakdown band does not have. + * + * Held still for the same reason `EMPTY_OWNERS` is: the gate below returns it + * on every render where the band is closed, and a fresh `[]` each time would + * make the memo a no-op (PERF-340). + */ +const EMPTY_BUCKETS: FloorBucket[] = []; + /** * How many finished runs are warmed when the reader switches to Terminated, * and how many walks run at once. @@ -1848,13 +1857,28 @@ export function PerfBrowser({ // cut it: `leaf.connector` is on the leaf but is deliberately not a // `GroupAxis` (see `groupSpine`), and an instrument breakdown at a scope // already grouped by pair is still the honest answer for that scope. + // + // Both are gated on the band being open (PERF-340). Each `groupSpine` is a + // second and third full pass over the scope's spine — bucket, then + // `readSpine` per bucket — and the band is shut on arrival every time and + // deliberately not remembered, so for the whole of a typical session the two + // cuts were folded and thrown away on every WS frame and every clock tick. + // Nothing but `` reads them, and it only renders under the + // same guard, so the gate is invisible to the numbers. + const showBreakdown = band === "breakdown"; const byPair = useMemo( - () => groupSpine(scopedLeaves, (leaf) => leaf.pair || UNKNOWN_LABEL, cv, now), - [scopedLeaves, cv, now], + () => + showBreakdown + ? groupSpine(scopedLeaves, (leaf) => leaf.pair || UNKNOWN_LABEL, cv, now) + : EMPTY_BUCKETS, + [showBreakdown, scopedLeaves, cv, now], ); const byVenue = useMemo( - () => groupSpine(scopedLeaves, (leaf) => leaf.connector || UNKNOWN_LABEL, cv, now), - [scopedLeaves, cv, now], + () => + showBreakdown + ? groupSpine(scopedLeaves, (leaf) => leaf.connector || UNKNOWN_LABEL, cv, now) + : EMPTY_BUCKETS, + [showBreakdown, scopedLeaves, cv, now], ); /** From 511caf08e0cae2706cf1b49761e8d899a0c0cbce Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 16:49:27 +0300 Subject: [PATCH 088/154] Stop reading a finished run's agent detail every five seconds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `GET /agents/{slug}` builds a summary per strategy and prices each one's sessions through the Hummingbot API, so PERF-305 gated its poll on "some strategy is running" — but only where the strategies grid and the knowledge banner read the key. The workspace declared it twice more with a flat 5s interval, and react-query takes the shortest interval among a key's observers, so the gate never held on the one surface a reader leaves open longest: reading a finished run cost the whole fan-out every five seconds, forever. Both workspace declarations now carry the same gate. Nothing live on the screen came off that key anyway — the countdown and the cadence are the separately polled strategy record, and the run rail is its own disk-only poll. Gating alone would have been a trap, though: the lifecycle controls invalidated only the strategy record, so an idle agent's gate would have stayed shut over a loop the reader had just started, with nothing left to reopen it and the header's "Live" badge wrong until a reload. Starting, stopping, pausing and resuming now invalidate the agent detail too. The two declarations also take a cadence of `staleTime`, because the page resolves the key before it renders the screen and the screen's observer would otherwise mount onto data react-query calls stale — one open, two requests. Counted rather than argued: a minute of reading an idle agent is 15 requests before and 1 after. --- .../agent/AgentControls.gate.test.tsx | 153 +++++++++++++++ .../src/components/agent/AgentControls.tsx | 34 +++- .../agent/workspace/AgentRunScreen.tsx | 17 +- .../src/pages/AgentWorkspace.poll.test.tsx | 184 ++++++++++++++++++ frontend/src/pages/AgentWorkspace.tsx | 15 +- 5 files changed, 395 insertions(+), 8 deletions(-) create mode 100644 frontend/src/components/agent/AgentControls.gate.test.tsx create mode 100644 frontend/src/pages/AgentWorkspace.poll.test.tsx diff --git a/frontend/src/components/agent/AgentControls.gate.test.tsx b/frontend/src/components/agent/AgentControls.gate.test.tsx new file mode 100644 index 000000000..eebb515d7 --- /dev/null +++ b/frontend/src/components/agent/AgentControls.gate.test.tsx @@ -0,0 +1,153 @@ +/** + * What a lifecycle control makes stale (PERF-343). + * + * Gating `["agent", slug]` on "some strategy is running" only works if starting + * one re-opens the gate. The controls used to invalidate `["strategy", …]` + * alone, which was enough while the agent key polled unconditionally and is not + * enough now: an idle agent's gate is closed, so nothing would ever ask again to + * learn that the loop the reader just started is running — the "Live" badge and + * the delete guard would stay wrong until a reload. + * + * So this pins the pair. A gated observer of the key sits beside the real + * controls; resuming a paused loop must both re-read the key immediately and + * leave the 5s cadence running afterwards. + * + * Needs a DOM, so this file overrides vitest's default `node` environment. + * + * @vitest-environment jsdom + */ + +import { QueryClient, QueryClientProvider, useQuery } from "@tanstack/react-query"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { AgentDetail } from "@/lib/api"; + +const getAgent = vi.fn(async () => DETAIL); +const resumeStrategy = vi.fn(async () => ({})); + +vi.mock("@/lib/api", () => ({ + api: { + getAgent: () => getAgent(), + getServers: () => Promise.resolve([]), + resumeStrategy: (...a: unknown[]) => resumeStrategy(...(a as [])), + pauseStrategy: () => Promise.resolve({}), + stopStrategy: () => Promise.resolve({}), + startStrategy: () => Promise.resolve({}), + }, +})); + +const { AgentControls } = await import("./AgentControls"); + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +let DETAIL: AgentDetail; + +function detail(status: string): AgentDetail { + return { + slug: "brigado", + name: "Brigado", + description: "", + agent_md: "", + agent_key: "claude-code", + tools: [], + when_to_consult: "", + server_required: false, + server_name: "", + strategies: [{ slug: "brl_mm", name: "BRL MM", status, instances: [] }], + } as unknown as AgentDetail; +} + +/** The gate every observer of this key now declares (PERF-305/PERF-343). */ +function GatedReader() { + useQuery({ + queryKey: ["agent", "brigado"], + queryFn: () => getAgent(), + refetchInterval: (q) => + (q.state.data as AgentDetail | undefined)?.strategies.some( + (s) => s.status === "running", + ) + ? 5000 + : false, + }); + return null; +} + +let container: HTMLDivElement; +let root: Root; + +function mount() { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false, gcTime: 0 } }, + }); + act(() => { + root.render( + + + + , + ); + }); +} + +async function elapse(ms: number) { + await act(async () => { + await vi.advanceTimersByTimeAsync(ms); + }); +} + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + vi.useFakeTimers(); + getAgent.mockClear(); + resumeStrategy.mockClear(); + DETAIL = detail("paused"); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + vi.useRealTimers(); +}); + +describe("resuming a loop", () => { + it("re-reads the agent detail and re-arms its gated poll", async () => { + mount(); + await elapse(0); + expect(getAgent).toHaveBeenCalledTimes(1); + + // Idle: the gate is shut, so nothing but the control can reopen it. + await elapse(30_000); + expect(getAgent).toHaveBeenCalledTimes(1); + + DETAIL = detail("running"); + const resume = [...container.querySelectorAll("button")].find((b) => + b.textContent?.includes("Resume"), + ); + expect(resume).toBeTruthy(); + await act(async () => { + resume!.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + await elapse(0); + + expect(resumeStrategy).toHaveBeenCalledTimes(1); + // The invalidation, not a poll: no cadence has elapsed. + expect(getAgent).toHaveBeenCalledTimes(2); + + // And the gate is open again on what it read back. + await elapse(5_000); + expect(getAgent).toHaveBeenCalledTimes(3); + }); +}); diff --git a/frontend/src/components/agent/AgentControls.tsx b/frontend/src/components/agent/AgentControls.tsx index 9e45719ef..089bbf5f9 100644 --- a/frontend/src/components/agent/AgentControls.tsx +++ b/frontend/src/components/agent/AgentControls.tsx @@ -1,4 +1,9 @@ -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { + type QueryClient, + useMutation, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; import { Clock, MessageSquareText, @@ -15,6 +20,25 @@ import { useState } from "react"; import { useEscapeKey } from "@/hooks/useEscapeKey"; import { api } from "@/lib/api"; +/** + * What a start/stop/pause/resume just made stale. + * + * Two keys, not one. The strategy's own record is the obvious half; the agent + * detail is the half that used to be missed, and it is load-bearing because + * `strategies[].status` is what the workspace reads for the "Live" badge and + * the delete guard — and what re-arms the gated `["agent", slug]` poll + * (PERF-343). Invalidating only the strategy left an idle agent's gate closed + * over a loop that had just started, with nothing left to reopen it. + */ +function invalidateLifecycle( + queryClient: QueryClient, + slug: string, + sslug: string, +) { + queryClient.invalidateQueries({ queryKey: ["strategy", slug, sslug] }); + queryClient.invalidateQueries({ queryKey: ["agent", slug] }); +} + // ── Start Session Dialog ── export function StartSessionDialog({ @@ -80,7 +104,7 @@ export function StartSessionDialog({ return api.startStrategy(slug, sslug, config, context); }, onSuccess: () => { - queryClient.invalidateQueries({ queryKey: ["strategy", slug, sslug] }); + invalidateLifecycle(queryClient, slug, sslug); onClose(); }, }); @@ -328,17 +352,17 @@ export function AgentControls({ slug, sslug, status, defaultContext, agentConfig const stopMut = useMutation({ mutationFn: () => api.stopStrategy(slug, sslug), onSuccess: () => { - queryClient.invalidateQueries({ queryKey: ["strategy", slug, sslug] }); + invalidateLifecycle(queryClient, slug, sslug); setConfirmStop(false); }, }); const pauseMut = useMutation({ mutationFn: () => api.pauseStrategy(slug, sslug), - onSuccess: () => queryClient.invalidateQueries({ queryKey: ["strategy", slug, sslug] }), + onSuccess: () => invalidateLifecycle(queryClient, slug, sslug), }); const resumeMut = useMutation({ mutationFn: () => api.resumeStrategy(slug, sslug), - onSuccess: () => queryClient.invalidateQueries({ queryKey: ["strategy", slug, sslug] }), + onSuccess: () => invalidateLifecycle(queryClient, slug, sslug), }); const loading = stopMut.isPending || pauseMut.isPending || resumeMut.isPending; diff --git a/frontend/src/components/agent/workspace/AgentRunScreen.tsx b/frontend/src/components/agent/workspace/AgentRunScreen.tsx index 6c9c1604d..eb95ea124 100644 --- a/frontend/src/components/agent/workspace/AgentRunScreen.tsx +++ b/frontend/src/components/agent/workspace/AgentRunScreen.tsx @@ -147,11 +147,26 @@ export function AgentRunScreen({ // the keys, which is the only reason three regions polling at 5s is one poll // — and the reason the page can hold its own `["agent", slug]` for the header // without buying a second one. + // + // Gated the same way `AgentStrategies` and `AgentKnowledge` gate this key + // (PERF-305/PERF-343): `GET /agents/{slug}` prices every strategy's sessions + // through the Hummingbot API, and react-query takes the *shortest* interval + // among a key's observers — so a flat 5s here silently overrode their gate on + // the screen a reader leaves open longest. Nothing live on this screen comes + // off this key: the countdown and the cadence are the separately polled + // `["strategy", slug, sslug]`, and the rail is `["agent-runs", ...]`. const { data: agent, isLoading } = useQuery({ queryKey: ["agent", slug], queryFn: () => api.getAgent(slug), enabled: !!slug, - refetchInterval: 5000, + // The page resolves this key before it renders the screen, so the + // screen's observer mounts a tick later onto data react-query would + // otherwise call stale and re-fetch — one open, two requests. A cadence's + // worth of freshness makes the second mount reuse the first read; the + // interval below refetches on its own timer regardless. + staleTime: 5000, + refetchInterval: (q) => + q.state.data?.strategies.some((s) => s.status === "running") ? 5000 : false, }); // The rail's window, not a filter (FEAT-111). An install that has been diff --git a/frontend/src/pages/AgentWorkspace.poll.test.tsx b/frontend/src/pages/AgentWorkspace.poll.test.tsx new file mode 100644 index 000000000..7975ad486 --- /dev/null +++ b/frontend/src/pages/AgentWorkspace.poll.test.tsx @@ -0,0 +1,184 @@ +/** + * What `/agents/:slug` costs while you read it (PERF-343). + * + * `GET /agents/{slug}` is the expensive read on this route: it builds a summary + * per strategy and prices each one's sessions through the Hummingbot API. + * PERF-305 established that the key must only poll while something is looping, + * and gated it in `AgentStrategies` and `AgentKnowledge` — but react-query takes + * the *shortest* interval declared among a key's observers, so the workspace's + * two ungated `refetchInterval: 5000` declarations (the page's, for the header's + * "Live" badge and the delete guard, and the screen's, for the strategy picker's + * labels) overrode that gate on the surface a reader leaves open longest. + * + * So the assertion is a request count, taken through the real pair: this file + * mounts the page, which mounts the screen, so both declarations are live on the + * key at once and a gate on only one of them still fails. An idle agent is read + * once and then left alone; a running one still refreshes every 5s, which is the + * half that stops a component that never polls at all from passing. + * + * Needs a DOM, so this file overrides vitest's default `node` environment. + * + * @vitest-environment jsdom + */ + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { MemoryRouter, Route, Routes } from "react-router-dom"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { AgentDetail } from "@/lib/api"; + +const getAgent = vi.fn(async () => DETAIL); + +vi.mock("@/lib/api", () => ({ + api: { + getAgent: () => getAgent(), + getAgentRuns: () => Promise.resolve([]), + getStrategy: () => Promise.resolve(null), + getConversationDeployments: () => Promise.resolve([]), + getSessionJournal: () => Promise.resolve({ content: "" }), + getSessionActions: () => Promise.resolve({ actions: [] }), + getSessionReport: () => Promise.resolve({ report: null }), + getStrategySessionExecutors: () => Promise.resolve({ executors: [] }), + }, + CHAT_SLUG: "condor", +})); + +// The disclosure bodies each fetch their own world and none of them is what is +// being counted here; the same stubs `AgentRunScreen.test.tsx` uses. +const stub = () => () => null; +vi.mock("@/components/agent/workspace/NowView", () => ({ NowView: stub() })); +vi.mock("@/components/agent/workspace/MoneyView", () => ({ MoneyView: stub() })); +vi.mock("@/components/agent/workspace/AgentFleet", () => ({ + AgentFleet: stub(), +})); +vi.mock("@/components/agent/workspace/PlaybookView", () => ({ + PlaybookView: stub(), +})); +vi.mock("@/components/agent/lab/RunRail", () => ({ RunRail: stub() })); +vi.mock("@/components/agent/lab/RunOverview", () => ({ + RunOverview: stub(), + ExperimentDetail: stub(), +})); +vi.mock("@/components/agent/AgentSessionContent", () => ({ + SnapshotDetail: stub(), +})); +vi.mock("@/components/agent/DelegationSheet", () => ({ + DelegationSheet: stub(), +})); +// The header's own pickers fetch models and servers; the badge it draws from +// this key is covered where the key is invalidated, not here. +vi.mock("@/components/agent/workspace/WorkspaceHeader", () => ({ + WorkspaceHeader: () => null, +})); + +const { AgentWorkspace } = await import("./AgentWorkspace"); + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +let DETAIL: AgentDetail; + +function detail(...statuses: string[]): AgentDetail { + return { + slug: "brigado", + name: "Brigado", + description: "", + agent_md: "", + agent_key: "claude-code", + tools: [], + when_to_consult: "", + server_required: false, + server_name: "brigado_2", + strategies: statuses.map((status, i) => ({ + slug: `s${i}`, + name: `Loop ${i}`, + status, + instances: [], + })), + } as unknown as AgentDetail; +} + +let container: HTMLDivElement; +let root: Root; + +function mount() { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false, gcTime: 0 } }, + }); + act(() => { + root.render( + + + + } /> + + + , + ); + }); +} + +/** Let `ms` of polling elapse, flushing the fetches it schedules. */ +async function elapse(ms: number) { + await act(async () => { + await vi.advanceTimersByTimeAsync(ms); + }); +} + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + vi.useFakeTimers(); + getAgent.mockClear(); + DETAIL = detail("stopped"); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + vi.useRealTimers(); +}); + +describe("the workspace's agent-detail poll", () => { + it("reads an idle agent once and then stops asking", async () => { + DETAIL = detail("stopped", "idle"); + mount(); + await elapse(0); + expect(getAgent).toHaveBeenCalledTimes(1); + + // A minute of an open tab — twelve cadences — is still the one read. + await elapse(60_000); + expect(getAgent).toHaveBeenCalledTimes(1); + }); + + it("keeps the 5s cadence while a strategy is running", async () => { + DETAIL = detail("stopped", "running"); + mount(); + await elapse(0); + expect(getAgent).toHaveBeenCalledTimes(1); + + await elapse(5_000); + expect(getAgent).toHaveBeenCalledTimes(2); + + await elapse(5_000); + expect(getAgent).toHaveBeenCalledTimes(3); + }); + + it("goes quiet once the last loop stops", async () => { + DETAIL = detail("running"); + mount(); + await elapse(0); + + DETAIL = detail("stopped"); + await elapse(5_000); + const afterStop = getAgent.mock.calls.length; + + await elapse(60_000); + expect(getAgent).toHaveBeenCalledTimes(afterStop); + }); +}); diff --git a/frontend/src/pages/AgentWorkspace.tsx b/frontend/src/pages/AgentWorkspace.tsx index c9af0b3e8..d93c73b03 100644 --- a/frontend/src/pages/AgentWorkspace.tsx +++ b/frontend/src/pages/AgentWorkspace.tsx @@ -60,7 +60,11 @@ export function AgentWorkspace() { const adapter = useWorkspaceUrl(searchParams, setSearchParams); // The header's, and the two guards below. The same `["agent", slug]` the body - // reads, so react-query serves both from one poll rather than two. + // reads, so react-query serves both from one poll rather than two — and on + // the same gate, because sharing a key means sharing the shortest interval + // declared on it (PERF-343). Only a live loop can move the "Live" badge or + // the delete guard; the strategy controls invalidate this key when they + // start one, so the gate re-arms without a reload. const { data: agent, isLoading, @@ -69,7 +73,14 @@ export function AgentWorkspace() { queryKey: ["agent", slug], queryFn: () => api.getAgent(slug), enabled: !!slug, - refetchInterval: 5000, + // The page resolves this key before it renders the screen, so the + // screen's observer mounts a tick later onto data react-query would + // otherwise call stale and re-fetch — one open, two requests. A cadence's + // worth of freshness makes the second mount reuse the first read; the + // interval below refetches on its own timer regardless. + staleTime: 5000, + refetchInterval: (q) => + q.state.data?.strategies.some((s) => s.status === "running") ? 5000 : false, }); const deleteAgentMutation = useMutation({ From 9d4875fae55873cdda502af04df2a02b0933a6f3 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 16:56:12 +0300 Subject: [PATCH 089/154] Stop rebuilding the executor hover card on every crosshair move MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both chart crosshair handlers ran renderOverlayTooltipHtml — a ~100-line string build that reads the theme, JSON.parses a triple-barrier config and escapes a dozen rows — then assigned it to innerHTML and read offsetHeight back, on every pointer move while the cursor sat anywhere inside an executor box. That is a full teardown, reparse and forced layout of the card roughly sixty times a second to print exactly the same thing. The card is a pure function of the overlay and the two display-currency formatters, so createOverlayTooltipView now keys the drawn card on those three and rewrites the DOM only when one of them changes identity, reusing the height it measured then. A refetch that moves the PnL gives the overlay a new identity through react-query's structural sharing, and a currency switch mints new formatters, so both still repaint on the first move after they land; hiding the card forgets it, so re-entering the same overlay draws a fresh one. The two formatter closures are memoized rather than minted per render for the same reason the position lines' already were: as a cache key, a closure rebuilt every render is no key at all. --- .../src/components/charts/ExecutorChart.tsx | 32 ++- frontend/src/components/trade/TradeChart.tsx | 27 ++- .../executor-overlays.tooltip-view.test.ts | 188 ++++++++++++++++++ frontend/src/lib/executor-overlays.ts | 76 +++++++ 4 files changed, 306 insertions(+), 17 deletions(-) create mode 100644 frontend/src/lib/executor-overlays.tooltip-view.test.ts diff --git a/frontend/src/components/charts/ExecutorChart.tsx b/frontend/src/components/charts/ExecutorChart.tsx index 2c32a581b..89f8d1729 100644 --- a/frontend/src/components/charts/ExecutorChart.tsx +++ b/frontend/src/components/charts/ExecutorChart.tsx @@ -7,10 +7,10 @@ import { useRates } from "@/hooks/useRates"; import { api, type ExecutorInfo } from "@/lib/api"; import { computeMultiOverlays, + createOverlayTooltipView, getExecutorColor, getOverlayTimeRange, getPoolAddress, - renderOverlayTooltipHtml, type ExecutorOverlay, } from "@/lib/executor-overlays"; import { tsToSeconds } from "@/lib/formatters"; @@ -123,10 +123,21 @@ export function ExecutorChart({ const quoteCurrency = tradingPair.split("-")[1] || "USDT"; const quoteCurrencies = useMemo(() => [quoteCurrency], [quoteCurrency]); const { formatPnlValue, formatValue } = useRates(quoteCurrencies); + // Both are memoized, not minted per render: the cached tooltip card keys on + // the formatter identities, so a fresh closure each render would rebuild the + // card each render without changing a character of it. + const convertValue = useMemo( + () => (val: number) => formatValue(val, quoteCurrency), + [formatValue, quoteCurrency], + ); + const convertPnl = useMemo( + () => (val: number) => formatPnlValue(val, quoteCurrency), + [formatPnlValue, quoteCurrency], + ); const convertValueRef = useRef<(val: number) => string>(() => ""); const convertPnlRef = useRef<(val: number) => string>(() => ""); - convertValueRef.current = (val: number) => formatValue(val, quoteCurrency); - convertPnlRef.current = (val: number) => formatPnlValue(val, quoteCurrency); + convertValueRef.current = convertValue; + convertPnlRef.current = convertPnl; // Compute overlays const overlays = useMemo(() => computeMultiOverlays(executors), [executors]); @@ -217,19 +228,22 @@ export function ExecutorChart({ }); seriesRef.current = series; + // One cached tooltip card per chart instance, torn down with the chart. + const tooltipView = createOverlayTooltipView(); + // Crosshair tooltip handler chart.subscribeCrosshairMove((param) => { const tooltip = tooltipRef.current; if (!tooltip || !containerRef.current) return; if (!param.time || !param.point || param.point.x < 0 || param.point.y < 0) { - tooltip.style.display = "none"; + tooltipView.hide(tooltip); return; } const crosshairTime = typeof param.time === "number" ? param.time : 0; if (!crosshairTime) { - tooltip.style.display = "none"; + tooltipView.hide(tooltip); return; } @@ -278,20 +292,20 @@ export function ExecutorChart({ } if (!bestOverlay) { - tooltip.style.display = "none"; + tooltipView.hide(tooltip); return; } - tooltip.innerHTML = renderOverlayTooltipHtml(bestOverlay, { + // Rebuilds the card only when the hovered overlay or a formatter + // changed; otherwise this is the cached height and two style writes. + const tooltipH = tooltipView.show(tooltip, bestOverlay, { formatValue: convertValueRef.current, formatPnl: convertPnlRef.current, }); - tooltip.style.display = "block"; // Position tooltip on opposite side of cursor (fixed/viewport coords) const containerRect = containerRef.current.getBoundingClientRect(); const tooltipW = 280; - const tooltipH = tooltip.offsetHeight || 200; const cursorInRightHalf = param.point.x > containerRect.width / 2; let left = cursorInRightHalf ? containerRect.left + param.point.x - tooltipW - 16 diff --git a/frontend/src/components/trade/TradeChart.tsx b/frontend/src/components/trade/TradeChart.tsx index 774147461..51bce0a25 100644 --- a/frontend/src/components/trade/TradeChart.tsx +++ b/frontend/src/components/trade/TradeChart.tsx @@ -6,7 +6,7 @@ import { useRates } from "@/hooks/useRates"; import { api, type ConsolidatedPosition } from "@/lib/api"; import { candleChannelKey, candleStore } from "@/lib/candle-store"; import type { ChartLineSlot, ExtraLine, PickSlot } from "@/components/executor/types"; -import { getExecutorColor, renderOverlayTooltipHtml, type ExecutorOverlay } from "@/lib/executor-overlays"; +import { createOverlayTooltipView, getExecutorColor, type ExecutorOverlay } from "@/lib/executor-overlays"; import { getThemeColors, pnlHexColor } from "@/lib/theme-colors"; import { roundToPricePrecision } from "@/lib/formatters"; import { createDragHitPrimitive, type DragTarget } from "./priceLineDrag"; @@ -158,9 +158,17 @@ export function TradeChart({ () => (val: number) => formatPnlValue(val, quoteCurrency), [formatPnlValue, quoteCurrency], ); + // `convertValue` is memoized for the same reason `convertPnl` is, plus one: + // the tooltip view below keys its cached card on the formatter identities, so + // a closure minted fresh on every render would rebuild the card on every + // render for no change in what it says. + const convertValue = useMemo( + () => (val: number) => formatValue(val, quoteCurrency), + [formatValue, quoteCurrency], + ); const convertValueRef = useRef<(val: number) => string>(() => ""); const convertPnlRef = useRef<(val: number) => string>(() => ""); - convertValueRef.current = (val: number) => formatValue(val, quoteCurrency); + convertValueRef.current = convertValue; convertPnlRef.current = convertPnl; const [chartReady, setChartReady] = useState(false); @@ -286,11 +294,14 @@ export function TradeChart({ // Track the pointer's price/time for click-to-set, the measure tool and // the executor tooltip + // One cached tooltip card per chart instance, torn down with the chart. + const tooltipView = createOverlayTooltipView(); + chart.subscribeCrosshairMove((param) => { if (!param.point || !param.seriesData) { cursorPriceRef.current = null; crosshairTimeRef.current = null; - if (tooltipRef.current) tooltipRef.current.style.display = "none"; + if (tooltipRef.current) tooltipView.hide(tooltipRef.current); // Leave the measure box/badge frozen at their last position — a // measurement persists until cleared (click / Esc), so moving off // the pane edge doesn't make it vanish. @@ -369,7 +380,7 @@ export function TradeChart({ const crosshairTime = typeof param.time === "number" ? param.time : 0; if (!crosshairTime || !param.point || param.point.x < 0 || param.point.y < 0) { - tooltip.style.display = "none"; + tooltipView.hide(tooltip); return; } @@ -413,20 +424,20 @@ export function TradeChart({ } if (!bestOverlay) { - tooltip.style.display = "none"; + tooltipView.hide(tooltip); return; } - tooltip.innerHTML = renderOverlayTooltipHtml(bestOverlay, { + // Rebuilds the card only when the hovered overlay or a formatter + // changed; otherwise this is the cached height and two style writes. + const tooltipH = tooltipView.show(tooltip, bestOverlay, { formatValue: convertValueRef.current, formatPnl: convertPnlRef.current, }); - tooltip.style.display = "block"; // Position tooltip using viewport-fixed coords (rendered via portal) const containerRect = containerRef.current.getBoundingClientRect(); const tooltipW = 280; - const tooltipH = tooltip.offsetHeight || 200; const cursorInRightHalf = param.point.x > containerRect.width / 2; let left = cursorInRightHalf ? containerRect.left + param.point.x - tooltipW - 16 diff --git a/frontend/src/lib/executor-overlays.tooltip-view.test.ts b/frontend/src/lib/executor-overlays.tooltip-view.test.ts new file mode 100644 index 000000000..6fdf490a1 --- /dev/null +++ b/frontend/src/lib/executor-overlays.tooltip-view.test.ts @@ -0,0 +1,188 @@ +/** + * @vitest-environment jsdom + * + * The cached tooltip card (PERF-348). + * + * Both crosshair handlers used to rebuild the hover card — a ~100-line string + * build, an `innerHTML` reparse of ~20 nodes and the forced layout of the + * `offsetHeight` read that follows it — on every pointer move while the cursor + * sat inside an executor box, ~60 times a second to say exactly the same thing. + * `createOverlayTooltipView` rewrites the DOM only when one of the three + * arguments that produced the current card changed identity. + * + * These tests count the writes and the layout reads on the element itself, so + * they measure the real `renderOverlayTooltipHtml` going into the real DOM + * rather than a stand-in. The staleness cases matter at least as much as the + * count: a card that stops following the executor's PnL would be a worse bug + * than the rebuild it saves. + */ + +import { beforeEach, describe, expect, it } from "vitest"; + +import { + createOverlayTooltipView, + type ExecutorOverlay, + type OverlayTooltipFormatters, +} from "./executor-overlays"; + +function overlay(patch: Partial = {}): ExecutorOverlay { + return { + executorId: "abc123def456", + type: "position", + side: "buy", + status: "running", + closeType: "", + pnl: 12.5, + pnlPct: 0.0125, + volume: 1000, + fees: 1.25, + timeRange: { start: 1700000000, end: 1700003600 }, + config: {}, + ...patch, + } as ExecutorOverlay; +} + +const usd: OverlayTooltipFormatters = { + formatValue: (v: number) => `$${v.toFixed(2)}`, + formatPnl: (v: number) => `${v >= 0 ? "+" : ""}$${v.toFixed(2)}`, +}; + +/** A tooltip div that counts the two expensive things the handler does to it. */ +function makeTooltip(height = 148) { + const el = document.createElement("div"); + const counts = { writes: 0, layouts: 0 }; + let html = ""; + Object.defineProperty(el, "innerHTML", { + get: () => html, + set: (v: string) => { + html = v; + counts.writes += 1; + }, + configurable: true, + }); + Object.defineProperty(el, "offsetHeight", { + get: () => { + counts.layouts += 1; + // A hidden card measures 0 in the browser too — that is what the view's + // 200 fallback exists for. + return el.style.display === "none" ? 0 : height; + }, + configurable: true, + }); + return { el, counts, html: () => html }; +} + +describe("createOverlayTooltipView", () => { + let view: ReturnType; + + beforeEach(() => { + view = createOverlayTooltipView(); + }); + + it("builds the card once for a hover that never leaves the same overlay", () => { + const { el, counts } = makeTooltip(); + const o = overlay(); + + const heights = Array.from({ length: 50 }, () => view.show(el, o, usd)); + + // 50 crosshair moves, one card build and one forced layout. + expect(counts.writes).toBe(1); + expect(counts.layouts).toBe(1); + expect(new Set(heights)).toEqual(new Set([148])); + expect(el.style.display).toBe("block"); + }); + + it("swaps the card on the first move that changes the winning overlay", () => { + const { el, counts, html } = makeTooltip(); + const a = overlay({ executorId: "aaaaaaaaaa11" }); + const b = overlay({ executorId: "bbbbbbbbbb22" }); + + view.show(el, a, usd); + view.show(el, a, usd); + expect(html()).toContain("aaaaaaaaaa"); + + view.show(el, b, usd); + expect(counts.writes).toBe(2); + expect(html()).toContain("bbbbbbbbbb"); + expect(html()).not.toContain("aaaaaaaaaa"); + + // ...and back again, on the first move that swings the hit test back. + view.show(el, a, usd); + expect(counts.writes).toBe(3); + expect(html()).toContain("aaaaaaaaaa"); + }); + + it("repaints when a refetch lands a new PnL on the same executor", () => { + const { el, counts, html } = makeTooltip(); + + view.show(el, overlay({ pnl: 12.5 }), usd); + expect(html()).toContain("+$12.50"); + + // react-query's structural sharing hands the overlay a new identity exactly + // when one of its values actually changed, so this is what a refetch does. + for (let i = 0; i < 10; i++) view.show(el, overlay({ pnl: 41.25 }), usd); + + expect(html()).toContain("+$41.25"); + expect(html()).not.toContain("+$12.50"); + // One rebuild for the first card, one per new overlay object after it. + expect(counts.writes).toBe(11); + }); + + it("reprints in the new currency when the display formatters change", () => { + const { el, counts, html } = makeTooltip(); + const o = overlay(); + const eur: OverlayTooltipFormatters = { + formatValue: (v: number) => `€${v.toFixed(2)}`, + formatPnl: (v: number) => `${v >= 0 ? "+" : ""}€${v.toFixed(2)}`, + }; + + view.show(el, o, usd); + expect(html()).toContain("+$12.50"); + + view.show(el, o, eur); + view.show(el, o, eur); + expect(counts.writes).toBe(2); + expect(html()).toContain("+€12.50"); + }); + + it("re-renders after the pointer leaves the pane and comes back to the same overlay", () => { + const { el, counts, html } = makeTooltip(); + const o = overlay(); + + view.show(el, o, usd); + view.hide(el); + expect(el.style.display).toBe("none"); + + view.show(el, o, usd); + expect(counts.writes).toBe(2); + expect(el.style.display).toBe("block"); + expect(html()).toContain("abc123def4"); + }); + + it("keeps the flip-and-clamp maths honest: the reused height is the measured one", () => { + const { el } = makeTooltip(312); + const o = overlay(); + + // First move measures; every later move must position against that same + // number, not against the 200 fallback or a zero. + expect(view.show(el, o, usd)).toBe(312); + expect(view.show(el, o, usd)).toBe(312); + expect(view.show(el, o, usd)).toBe(312); + }); + + it("falls back to 200 when the card cannot be measured", () => { + const { el } = makeTooltip(0); + expect(view.show(el, overlay(), usd)).toBe(200); + }); + + it("measures per element, so a second tooltip node gets its own card", () => { + const first = makeTooltip(100); + const second = makeTooltip(260); + const o = overlay(); + + expect(view.show(first.el, o, usd)).toBe(100); + expect(view.show(second.el, o, usd)).toBe(260); + expect(second.counts.writes).toBe(1); + expect(second.html()).toContain("abc123def4"); + }); +}); diff --git a/frontend/src/lib/executor-overlays.ts b/frontend/src/lib/executor-overlays.ts index dbad663ba..680b59f1c 100644 --- a/frontend/src/lib/executor-overlays.ts +++ b/frontend/src/lib/executor-overlays.ts @@ -787,3 +787,79 @@ export function renderOverlayTooltipHtml( ${detailRows ? `
${detailRows}
` : ""} `; } + +/** + * The tooltip element as the crosshair handlers drive it: show it over an + * overlay, or hide it (PERF-348). + * + * `renderOverlayTooltipHtml` above is a ~100-line string build that reads the + * theme, `JSON.parse`s a config and escapes a dozen rows, and both charts used + * to run it — plus the `innerHTML` reparse of ~20 nodes and the forced layout + * of the `offsetHeight` read that follows it — on *every* crosshair move, i.e. + * ~60/s for as long as the pointer sits anywhere inside an executor box. The + * card is a pure function of its arguments, so all but the first of those are + * identical work. + * + * `show()` therefore rewrites the DOM only when one of the three arguments + * that produced the current card changed identity, and otherwise reuses the + * height it measured then, leaving only the `left`/`top` writes per move. + * + * Staleness is bounded by that key being the render's *whole* input: + * + * - the overlay object — rebuilt by `computeMultiOverlays` on each executors + * refetch, and react-query's structural sharing gives it a new identity + * exactly when one of its values (PnL included) actually changed, so a new + * PnL repaints the card on the first move after it lands; + * - both formatters — a display-currency switch mints new ones, so the card + * reprints in the new currency. + * + * `hide()` forgets the card as well, so re-entering the same overlay draws a + * fresh one rather than trusting markup left over from before. + */ +export interface OverlayTooltipView { + /** Draw `o` in `el` (rebuilding only if needed) and return the card's height. */ + show(el: HTMLElement, o: ExecutorOverlay, formatters: OverlayTooltipFormatters): number; + /** Hide `el` and forget what was drawn in it. */ + hide(el: HTMLElement): void; +} + +export function createOverlayTooltipView(): OverlayTooltipView { + let el: HTMLElement | null = null; + let overlay: ExecutorOverlay | null = null; + let formatValue: OverlayTooltipFormatters["formatValue"] | null = null; + let formatPnl: OverlayTooltipFormatters["formatPnl"] | null = null; + let height = 0; + + return { + show(nextEl, nextOverlay, formatters) { + if ( + nextEl !== el || + nextOverlay !== overlay || + formatters.formatValue !== formatValue || + formatters.formatPnl !== formatPnl + ) { + nextEl.innerHTML = renderOverlayTooltipHtml(nextOverlay, formatters); + el = nextEl; + overlay = nextOverlay; + formatValue = formatters.formatValue; + formatPnl = formatters.formatPnl; + height = 0; + } + nextEl.style.display = "block"; + // Measured only after the card is displayed — `offsetHeight` is 0 while + // it is `display:none`, which is what the 200 fallback is for. + if (!height) height = nextEl.offsetHeight || 200; + return height; + }, + hide(nextEl) { + nextEl.style.display = "none"; + if (nextEl === el) { + el = null; + overlay = null; + formatValue = null; + formatPnl = null; + height = 0; + } + }, + }; +} From 9fce125e453631af282a3f5613281914eaba66cb Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 17:14:13 +0300 Subject: [PATCH 090/154] Warm one connector's config map on hover, not all of them on load The Keys tab's exchange grid prefetched a config map for every connector the moment the list resolved -- 30-40 requests for "spot", of which the user reads exactly one. /settings/connectors/{name}/config-map is the one settings route with no cache behind it, so each was a fresh Condor to Hummingbot round trip, and the browser's 6-per-origin limit queued the one config map that mattered behind dozens nobody would ever open. Move the identical prefetch onto the connector button's mouseenter/focus, under the same query key and staleTime, so a click after a hover still hits a warm cache and a click without one falls back to the query that was always there, spinner and all. Nothing else reads that cache, so the requests that stop being issued were unread by construction. --- .../settings/ApiKeysSettings.test.tsx | 189 ++++++++++++++++++ .../components/settings/ApiKeysSettings.tsx | 26 ++- 2 files changed, 205 insertions(+), 10 deletions(-) create mode 100644 frontend/src/components/settings/ApiKeysSettings.test.tsx diff --git a/frontend/src/components/settings/ApiKeysSettings.test.tsx b/frontend/src/components/settings/ApiKeysSettings.test.tsx new file mode 100644 index 000000000..8c0775200 --- /dev/null +++ b/frontend/src/components/settings/ApiKeysSettings.test.tsx @@ -0,0 +1,189 @@ +/** + * How many config-map requests the Keys tab actually puts on the wire (PERF-349). + * + * `/settings/connectors/{name}/config-map` is the one settings route with no + * cache behind it (condor/web/routes/settings.py:659) — every call is a fresh + * Condor→Hummingbot round trip. The exchange grid used to prefetch one for + * *every* connector the moment the list resolved, so picking "spot" fired + * 30-40 uncached requests, all but one of which nobody would ever read, and + * the browser's 6-per-origin limit queued the one that mattered behind them. + * + * These tests count requests rather than inspecting hooks, because the whole + * defect lived in the traffic: a prefetch that never resolves and a prefetch + * that is never issued look identical from inside the component. `fetch` is + * therefore the only stub — the real `api` layer builds the URLs, so the + * assertions are about literal paths. + * + * Needs a DOM, so this file overrides vitest's default `node` environment. + * + * @vitest-environment jsdom + */ + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ServerContext } from "@/hooks/useServer"; +import { ApiKeysSettings } from "./ApiKeysSettings"; + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +let container: HTMLDivElement; +let root: Root; +let requested: string[] = []; + +/** Enough connectors that a per-connector prefetch is unmistakable in the count. */ +const SPOT = ["binance", "kucoin", "okx", "gate_io", "kraken", "mexc"]; + +const CONFIG_MAP = { + config_map: { + api_key: { type: "str", required: true, prompt: "Enter your API key" }, + api_secret: { type: "str", required: true, prompt: "Enter your API secret" }, + }, +}; + +function respond(path: string): unknown { + if (path.endsWith("/api/v1/servers")) { + return [{ name: "prod", host: "h", port: 8000, online: true, permission: "owner" }]; + } + if (path.includes("/settings/credentials")) return { credentials: [] }; + if (path.includes("/config-map")) return CONFIG_MAP; + if (path.includes("/settings/connectors")) { + return { connectors: SPOT.map((name) => ({ name, type: "spot" })) }; + } + if (path.includes("/gateway/wallets")) return { wallets: [] }; + if (path.includes("/gateway/status")) return { running: false }; + return {}; +} + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + requested = []; + vi.stubGlobal( + "fetch", + vi.fn(async (path: string) => { + requested.push(path); + return new Response(JSON.stringify(respond(path)), { + headers: { "Content-Type": "application/json" }, + }); + }), + ); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); +}); + +/** Every config-map request issued so far, as the connector names they name. */ +function configMapCalls(): string[] { + return requested + .filter((u) => u.includes("/config-map")) + .map((u) => u.match(/connectors\/([^/]+)\/config-map/)![1]); +} + +async function flush() { + for (let i = 0; i < 25; i++) { + await act(async () => { + await new Promise((r) => setTimeout(r, 0)); + }); + } +} + +/** The type cards carry a description under their label, so match on the prefix. */ +function button(label: string): HTMLButtonElement { + const all = [...container.querySelectorAll("button")]; + const found = + all.find((b) => b.textContent?.trim() === label) ?? + all.find((b) => b.textContent?.trim().startsWith(label)); + if (!found) { + const have = all.map((b) => b.textContent?.trim()).join(" | "); + throw new Error(`no button "${label}" — have: ${have}`); + } + return found as HTMLButtonElement; +} + +/** Mount the tab and walk it to the spot exchange grid. */ +async function openSpotGrid() { + const qc = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }); + await act(async () => { + root.render( + + {} }}> + + + , + ); + }); + await flush(); + await act(async () => button("Add API Key").click()); + await act(async () => button("spot").click()); + await flush(); +} + +describe("ApiKeysSettings config-map traffic", () => { + it("loads the spot grid without fetching a single config map", async () => { + await openSpotGrid(); + + // The list itself is fetched exactly once... + expect(requested.filter((u) => u.includes("&type=spot"))).toHaveLength(1); + // ...and nothing is speculatively pulled for the six connectors in it. + expect(configMapCalls()).toEqual([]); + }); + + it("warms exactly the hovered connector, and only once", async () => { + await openSpotGrid(); + + await act(async () => { + button("kucoin").dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + }); + await flush(); + + expect(configMapCalls()).toEqual(["kucoin"]); + + // A second hover inside the 30-minute staleTime is served from cache. + await act(async () => { + button("kucoin").dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + }); + await flush(); + + expect(configMapCalls()).toEqual(["kucoin"]); + }); + + it("renders the hovered connector's fields with no spinner on click", async () => { + await openSpotGrid(); + + await act(async () => { + button("okx").dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + }); + await flush(); + await act(async () => button("okx").click()); + + // No flush between the click and the assertion: the fields are there on the + // very first render because the hover already filled the cache. + expect(container.textContent).toContain("api_key"); + expect(container.textContent).not.toContain("Loading fields..."); + expect(configMapCalls()).toEqual(["okx"]); + }); + + it("still loads fields for a connector clicked without ever being hovered", async () => { + await openSpotGrid(); + + await act(async () => button("kraken").click()); + expect(container.textContent).toContain("Loading fields..."); + + await flush(); + + expect(container.textContent).toContain("api_secret"); + expect(configMapCalls()).toEqual(["kraken"]); + }); +}); diff --git a/frontend/src/components/settings/ApiKeysSettings.tsx b/frontend/src/components/settings/ApiKeysSettings.tsx index 42931873a..9521ed161 100644 --- a/frontend/src/components/settings/ApiKeysSettings.tsx +++ b/frontend/src/components/settings/ApiKeysSettings.tsx @@ -9,7 +9,7 @@ import { Star, Wallet, } from "lucide-react"; -import { useEffect, useMemo, useState } from "react"; +import { useCallback, useMemo, useState } from "react"; import { InlineConfirm } from "@/components/ui/InlineConfirm"; import { useServer } from "@/hooks/useServer"; @@ -107,18 +107,22 @@ export function ApiKeysSettings() { staleTime: 30 * 60 * 1000, }); - // Prefetch config-maps for all connectors when the exchange list loads - useEffect(() => { - const connectors: ConnectorInfo[] = connectorsData?.connectors ?? []; - if (!server || connectors.length === 0) return; - for (const c of connectors) { + // Warm the config-map cache for the ONE connector the pointer/focus is on, rather + // than every connector in the list (PERF-349): /settings/connectors/{name}/config-map + // is uncached on the backend, so a whole-list prefetch was 30-40 fresh round trips + // for a config map the user was never going to open. Same key and staleTime as the + // real query below, so a click after a hover still hits a warm cache. + const prefetchConfigMap = useCallback( + (name: string) => { + if (!server) return; qc.prefetchQuery({ - queryKey: ["settings-config-map", server, c.name], - queryFn: () => api.getConnectorConfigMap(server, c.name), + queryKey: ["settings-config-map", server, name], + queryFn: () => api.getConnectorConfigMap(server, name), staleTime: 30 * 60 * 1000, }); - } - }, [connectorsData, server, qc]); + }, + [qc, server], + ); const invalidate = () => qc.invalidateQueries({ queryKey: ["settings-credentials", server] }); @@ -369,6 +373,8 @@ export function ApiKeysSettings() {
diff --git a/frontend/src/components/chat/DockSection.tsx b/frontend/src/components/chat/DockSection.tsx index 07ad497ae..8d08c3209 100644 --- a/frontend/src/components/chat/DockSection.tsx +++ b/frontend/src/components/chat/DockSection.tsx @@ -4,11 +4,14 @@ import { ChevronDown, ChevronRight } from "lucide-react"; * One pane of a dock. * * Open, it takes a fixed share of the column — `flex-1 basis-0`, so two open - * panes are half and half no matter what is in them. Sizing from content - * instead (`flex-auto`) looks tidier on a quiet conversation and is unusable on - * a busy one: every task that starts or routine that finishes moves the divider, - * so the row you were reading slides out from under the cursor. A boundary that - * never moves is worth more than one that is always optimally placed. + * panes are half and half no matter what is in them, and `share` is the + * reader's own answer to the same question when the dock gives them a seam to + * drag (see `DockSplit`). Sizing from content instead (`flex-auto`) looks + * tidier on a quiet conversation and is unusable on a busy one: every task that + * starts or routine that finishes moves the divider, so the row you were + * reading slides out from under the cursor. A boundary that never moves — or + * that moves only when it is dragged — is worth more than one that is always + * optimally placed. * * The body owns the scrollbar, so the header never leaves the viewport whatever * the list does. @@ -27,6 +30,7 @@ export function DockSection({ hint, count, open, + share, onToggle, children, }: { @@ -36,11 +40,18 @@ export function DockSection({ hint: string; count?: number; open: boolean; + /** + * How much of the column this pane gets, against its sibling's share — the + * dragged split, when the dock has one. Omitted, open panes share evenly. + */ + share?: number; onToggle: () => void; children: React.ReactNode; }) { return (
+
+ +
+
+ ); +} + +const pane = (id: string) => + container.querySelector(`[data-testid="${id}"]`)!; +const seam = () => container.querySelector('[role="separator"]')!; +/** What the pane actually grows by — the number the drag is for. */ +const grow = (id: string) => Number(pane(id).style.flexGrow); + +async function render() { + await act(async () => { + root.render(); + }); + for (const [id, top] of [ + ["top", 100], + ["bottom", 300], + ] as const) { + Object.defineProperty(pane(id), "offsetHeight", { + value: 200, + configurable: true, + }); + pane(id).getBoundingClientRect = () => + ({ top, bottom: top + 200, height: 200 }) as DOMRect; + } +} + +/** Grab the seam and let go at `clientY`. */ +async function drag(clientY: number) { + await act(async () => { + seam().dispatchEvent( + new MouseEvent("mousedown", { bubbles: true, clientY: 300 }), + ); + }); + await act(async () => { + document.dispatchEvent(new MouseEvent("mousemove", { clientY })); + }); + await act(async () => { + document.dispatchEvent(new MouseEvent("mouseup")); + }); +} + +async function press(key: string) { + await act(async () => { + seam().dispatchEvent(new KeyboardEvent("keydown", { bubbles: true, key })); + }); +} + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + localStorage.clear(); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); +}); + +describe("the seam between two dock sections", () => { + it("opens even, and moves where the pointer let go", async () => { + await render(); + expect(grow("top")).toBe(0.5); + + // 400px of pane starting at y=100; letting go at 400 leaves 300 above. + await drag(400); + expect(grow("top")).toBeCloseTo(0.75); + expect(grow("bottom")).toBeCloseTo(0.25); + }); + + it("keeps a header and a row in the section being squeezed", async () => { + await render(); + + // Dragged clean off the top of the panel: the pane above still keeps its + // 96px floor, because a section collapsed to nothing is the toggle's job + // and not something a drag should be able to do by accident. + await drag(-500); + expect(grow("top")).toBeCloseTo(96 / 400); + + await drag(5000); + expect(grow("top")).toBeCloseTo((400 - 96) / 400); + }); + + it("remembers the split, and gives it back on the next mount", async () => { + await render(); + await drag(400); + expect(localStorage.getItem(KEY)).toBe("0.75"); + + await act(async () => root.unmount()); + root = createRoot(container); + await render(); + expect(grow("top")).toBeCloseTo(0.75); + }); + + it("steps with the arrow keys and resets on a double-click", async () => { + await render(); + + // Down grows the section above — the seam moves the way the arrow points. + await press("ArrowDown"); + expect(grow("top")).toBeCloseTo(0.52); + await press("ArrowUp"); + await press("ArrowUp"); + expect(grow("top")).toBeCloseTo(0.48); + + await act(async () => { + seam().dispatchEvent(new MouseEvent("dblclick", { bubbles: true })); + }); + expect(grow("top")).toBe(0.5); + }); + + it("reads a stored split back inside the envelope it allows", async () => { + // A hand-edited or stale value cannot hand one section the whole panel. + localStorage.setItem(KEY, "0.99"); + await render(); + expect(grow("top")).toBeCloseTo(0.85); + + await act(async () => root.unmount()); + root = createRoot(container); + localStorage.setItem(KEY, "not a number"); + await render(); + expect(grow("top")).toBe(0.5); + }); +}); diff --git a/frontend/src/components/chat/DockSplitHandle.tsx b/frontend/src/components/chat/DockSplitHandle.tsx new file mode 100644 index 000000000..9ada1064b --- /dev/null +++ b/frontend/src/components/chat/DockSplitHandle.tsx @@ -0,0 +1,105 @@ +import { useRef } from "react"; + +import { MAX_SPLIT_FRAC, MIN_SPLIT_FRAC } from "@/hooks/useDockSplit"; +import { useResizeDrag } from "@/hooks/useResizeDrag"; + +/** + * The seam *between* two open dock sections, and the way to move it. + * + * `DockResizeHandle` one module over moves the edge between a dock and the + * conversation — how wide the column is. This one moves the boundary inside it: + * how much of that column each section gets. + * + * The sections opened at a fixed even split, which is the right default and was + * the only thing on offer: a reader who wanted the execution table tall had to + * collapse the portfolio away entirely, and "put that away" is a different + * request from "give me more room for this". So the boundary is dragged, and + * remembered per browser under one key (see `DESK_SPLIT_KEY`). + * + * Where the seam *is* is `useDockSplit`'s, a hook away, because the shares + * belong to whoever draws the sections — this reports where the pointer went + * and nothing else. + */ + +/** Header plus a couple of rows — under this a section says nothing at all. */ +const MIN_SECTION_PX = 96; + +/** + * Drag it, or step it with the arrow keys; double-click puts it back. + * + * Rendered as the sibling between the two sections, so the frame it measures is + * the two elements on either side of it — taken once, at `mousedown`, because a + * table that grows a row mid-drag must not move the frame the pointer is being + * read against. The clamp is in pixels for the same reason the pane's is: a + * floor in percent means something different in a short panel than a tall one, + * and what matters is that neither section falls below a header and a row. + */ +export function DockSplitHandle({ + frac, + setFrac, + defaultFrac, + label, +}: { + frac: number; + setFrac: (f: number) => void; + /** Where a double-click puts it back to. */ + defaultFrac: number; + /** What this seam separates, for the reader who cannot see it. */ + label: string; +}) { + const geom = useRef({ top: 0, avail: 1 }); + + const { onMouseDown: startDrag, isDragging } = useResizeDrag({ + axis: "y", + value: 0, // `compute` is absolute; the drag has no starting size to grow. + onChange: (px) => setFrac(px / geom.current.avail), + min: MIN_SECTION_PX, + max: () => geom.current.avail - MIN_SECTION_PX, + compute: (coord) => coord - geom.current.top, + cursor: "row-resize", + lockUserSelect: true, + }); + + const onMouseDown = (e: React.MouseEvent) => { + const handle = e.currentTarget; + const above = handle.previousElementSibling as HTMLElement | null; + const below = handle.nextElementSibling as HTMLElement | null; + if (above && below) { + geom.current = { + top: above.getBoundingClientRect().top, + avail: above.offsetHeight + below.offsetHeight, + }; + } + startDrag(e); + }; + + const onKeyDown = (e: React.KeyboardEvent) => { + // Down grows the section above, up gives the room back — the seam moves the + // way the arrow points. + if (e.key === "ArrowDown") setFrac(frac + 0.02); + else if (e.key === "ArrowUp") setFrac(frac - 0.02); + else return; + e.preventDefault(); + }; + + return ( +
setFrac(defaultFrac)} + title="Drag to resize — double-click to reset" + // Pulled up over the section border above it, so the grabbable strip is + // the seam the reader can see rather than a gap below it. + className={`-mt-1 h-1.5 shrink-0 cursor-row-resize transition-colors hover:bg-[var(--color-primary)]/30 focus:outline-none focus-visible:bg-[var(--color-primary)]/30 ${ + isDragging ? "bg-[var(--color-primary)]/30" : "" + }`} + /> + ); +} diff --git a/frontend/src/hooks/useDockSplit.ts b/frontend/src/hooks/useDockSplit.ts new file mode 100644 index 000000000..ab60a2ffc --- /dev/null +++ b/frontend/src/hooks/useDockSplit.ts @@ -0,0 +1,50 @@ +import { useCallback, useEffect, useState } from "react"; + +/** + * How a dock's two open sections divide the column, remembered per browser. + * + * Kept as a *fraction* of the panel rather than a height, for the reason the + * workspace pane's split is: the panel's own height changes with the window, + * and a stored measurement would leave the seam where a taller window put it. + * + * The state lives with whoever *draws* the sections, because the shares are + * theirs to apply — `DockSplitHandle` only reports where the pointer went. + */ + +/** The envelope the stored fraction may take, whatever the panel's height. */ +export const MIN_SPLIT_FRAC = 0.15; +export const MAX_SPLIT_FRAC = 0.85; + +function clampFrac(f: number, fallback: number): number { + if (!Number.isFinite(f)) return fallback; + return Math.max(MIN_SPLIT_FRAC, Math.min(MAX_SPLIT_FRAC, f)); +} + +export function useDockSplit(key: string, fallback = 0.5) { + const [frac, setFracState] = useState(() => { + try { + const stored = localStorage.getItem(key); + return stored === null + ? fallback + : clampFrac(parseFloat(stored), fallback); + } catch { + // Unreadable storage is a browser that has never dragged the seam. + return fallback; + } + }); + + const setFrac = useCallback( + (f: number) => setFracState(clampFrac(f, fallback)), + [fallback], + ); + + useEffect(() => { + try { + localStorage.setItem(key, String(frac)); + } catch { + /* private mode; the split just lasts the session */ + } + }, [key, frac]); + + return { frac, setFrac, defaultFrac: fallback }; +} diff --git a/frontend/src/lib/sessionState.test.ts b/frontend/src/lib/sessionState.test.ts index 73278fa7c..4fdfaec8e 100644 --- a/frontend/src/lib/sessionState.test.ts +++ b/frontend/src/lib/sessionState.test.ts @@ -20,6 +20,7 @@ import { clearSessionState, DEX_DEPTH_COLLAPSED_KEY, DEX_NETWORK_KEY, + DESK_SPLIT_KEY, DISPLAY_CURRENCY_KEY, DOCK_PANES_KEY, DOCK_WIDTH_KEY, @@ -63,6 +64,7 @@ const DEVICE = [ BROWSE_HINT_KEY, PNL_HIDDEN_SERIES_KEY, DOCK_WIDTH_KEY, + DESK_SPLIT_KEY, PANE_FRAC_KEY, PANE_FRAC_TUNE_KEY, CHAT_RAIL_OPEN_KEY, diff --git a/frontend/src/lib/sessionState.ts b/frontend/src/lib/sessionState.ts index 946e6577c..75ad4a883 100644 --- a/frontend/src/lib/sessionState.ts +++ b/frontend/src/lib/sessionState.ts @@ -150,6 +150,21 @@ export const DOCK_WIDTH_KEY = "condor.dock.width"; */ export const ACCOUNT_DOCK_KEY = "condor.dock.account"; +/** + * Where the reader put the seam between the desk's two sections, as the + * fraction of the panel the top one (Portfolio) keeps. + * + * The sections opened at an even split and could only ever be even: a reader + * who wanted the execution table tall had to collapse the portfolio away + * entirely, which is a different question than "give me more of it". So the + * boundary is theirs to drag, and remembered per browser like every other + * geometry here. + * + * KEPT, for the reason `DOCK_WIDTH_KEY` and `PANE_FRAC_KEY` are: it says how + * this screen is divided and nothing about whose balances were behind it. + */ +export const DESK_SPLIT_KEY = "condor.dock.account.split"; + /** * Which of an agent run screen's disclosures are open, as a JSON array of * section ids — Runs, Detail, Money, Fleet and Playbook (FEAT-119). From 46db4df2b677ccb179e7c543e4998faec3ecc121 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 22:27:42 +0300 Subject: [PATCH 095/154] Name a controller row instead of titling it, so its tooltip lands on the row A focusable was the one box a scrolled table gets Chrome's tooltip placement wrong for: clicking a row's title put the wrapped bubble at the table's top corner, hundreds of pixels from what it described. The row now carries the description as its accessible name (aria-label); the tooltip moves to the cell that truncates, which is the cell the browser can anchor it beside. --- .../src/components/chat/DockExecution.test.tsx | 15 +++++++++++++-- frontend/src/components/chat/DockExecution.tsx | 14 ++++++++++++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/chat/DockExecution.test.tsx b/frontend/src/components/chat/DockExecution.test.tsx index 292a6b8dc..fe6415ad8 100644 --- a/frontend/src/components/chat/DockExecution.test.tsx +++ b/frontend/src/components/chat/DockExecution.test.tsx @@ -153,8 +153,19 @@ const toggleOn = (label: string) => * single one: a fleet running one bot must not spend a chevron saying so. */ const botHeaders = () => [...container.querySelectorAll("[data-bot-group]")]; -const rowFor = (title: string) => - container.querySelector(`[title="${title}"]`)!; +/** + * The row — or the group header — that says `label`. + * + * A controller row carries it as its accessible name (the tooltip itself is on + * the cell that truncates, which is where the browser can place it beside what + * it describes); a bot group header is a button and says it as a `title`. + */ +const rowFor = (label: string) => { + const el = container.querySelector( + `[aria-label="${label}"], [title="${label}"]`, + )!; + return el.closest("[data-controller-row]") ?? el; +}; const counts = () => container.querySelector('[data-testid="execution-counts"]')!.textContent ?? ""; const text = () => container.textContent ?? ""; diff --git a/frontend/src/components/chat/DockExecution.tsx b/frontend/src/components/chat/DockExecution.tsx index 83b02ef07..808f4813b 100644 --- a/frontend/src/components/chat/DockExecution.tsx +++ b/frontend/src/components/chat/DockExecution.tsx @@ -696,13 +696,22 @@ function ControllerRow({ const scope = controllerNodeId(leaf) ?? row.id; const stopped = leaf.status === "stopped"; + /** The row said in full — its cell's tooltip, and its accessible name. */ + const described = `${leaf.label} on ${leaf.bot}${stopped ? " — paused" : ""}`; + return ( ` inside a scrolled table + // is the one box Chrome gets wrong: clicking a row put a wrapped bubble + // up at the table's top corner, hundreds of pixels from what it + // described. The name is what a `role="button"` actually needs; the + // tooltip belongs to the cell that truncates, one line down. + aria-label={described} onClick={() => onOpen(scope)} onKeyDown={(e) => { if (e.key !== "Enter" && e.key !== " ") return; @@ -718,10 +727,11 @@ function ControllerRow({ > {/* The column that absorbs the slack, and the only one allowed to truncate: two rows under the same bot are told apart by nothing else, - so it gets every pixel the numbers do not need — and the row's `title` + so it gets every pixel the numbers do not need — and its own `title` says it in full when even that is not enough. It is indented by its depth, which is what makes the nesting readable without a rule. */} From b791d24661883aada048c6ba8aa8ccd1e9e63096 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 22:28:04 +0300 Subject: [PATCH 096/154] Give the Total its own zero-aligned axis, and point at a line to see its share A Total of twelve controllers is twelve controllers tall, so one shared axis drew the lines the reader came for as a flat braid across the middle of the pane. The Total now scales on its own right-hand axis -- the gutter the pane already reserved for the activity pane's position axis, so both panes' plot areas stay identical -- and alignedZeroDomains picks the two domains so a single dashed line at zero is true of both. The activity pane below stays fleet-level (twelve bar series is not a picture anyone reads), but mergeOwnerRows now folds each owner's volume and book alongside its PnL, so pointing at a line -- or its legend chip -- fills in that line's exact share inside the fleet's own bar and draws its book over the fleet's, in the line's colour. Nothing is drawn per-line until a line is picked. FEAT-119 --- .../src/components/perf/OwnerPnlChart.tsx | 251 +++++++++++++++++- frontend/src/lib/owner-series.test.ts | 113 ++++++++ frontend/src/lib/owner-series.ts | 156 ++++++++++- frontend/src/lib/pnl-chart-pane.tsx | 54 +++- 4 files changed, 556 insertions(+), 18 deletions(-) diff --git a/frontend/src/components/perf/OwnerPnlChart.tsx b/frontend/src/components/perf/OwnerPnlChart.tsx index 7a075fc9e..e9b6ec83f 100644 --- a/frontend/src/components/perf/OwnerPnlChart.tsx +++ b/frontend/src/components/perf/OwnerPnlChart.tsx @@ -21,18 +21,24 @@ import { formatAxisCurrency, formatAxisTime, formatCurrencyPnl, + formatCurrencyVolume, formatDateTime, pnlTextClass, } from "@/lib/formatters"; import { + alignedZeroDomains, nearestSeries, + niceTicks, ownerDataKey, + ownerPositionKey, + ownerVolumeKey, parseBaseline, parseBasis, rebaseRows, seriesColor, shortenLabels, type FloorChartRow, + type SeriesScale, } from "@/lib/owner-series"; import { AXIS_WIDTH, @@ -67,7 +73,23 @@ import { getThemeColors } from "@/lib/theme-colors"; * The one contract that must not be broken: **both panes reserve `AXIS_WIDTH` * on the left and on the right**, or their plot areas differ and the synced * cursor in one points at a different instant than the other. Nothing throws - * when that happens. + * when that happens. The right-hand gutter is where the Total's own axis went + * (FEAT-119) — it was reserved and empty before, so the second axis costs no + * width and breaks no alignment. + * + * **Two axes, one zero.** A Total of twelve controllers is twelve controllers + * tall, so one shared axis draws the lines the reader came for as a flat braid + * across the middle of the pane. The Total therefore has its own scale, and + * `alignedZeroDomains` chooses the two so that a single dashed line at zero is + * true of both — a chart about whether a number is above or below zero cannot + * afford two different zeros. + * + * **The pane below stays fleet-level.** Volume and the book are drawn once, + * for the whole scope, because twelve bar series and twelve areas is not a + * picture anyone reads. The per-line answer is attached to the *question* + * instead: point at a line (or its legend chip) and that line's share of each + * bar is filled in inside it, its own book is drawn over the fleet's, and the + * tooltip prints both as numbers. * * The Total line is not asserted to equal the sum of the owner lines — it is * folded by the same function over the union of their keys, and @@ -209,6 +231,28 @@ export function OwnerPnlChart({ visible.length > 1 ? visible[visible.length - 1].time - visible[0].time : 0; const fmtTimeAxis = useCallback((v: number) => formatAxisTime(v, spanMs), [spanMs]); + /** + * The picked line's share of each volume bar, in the line's own colour + * (FEAT-119). + * + * The pane below draws the *fleet's* flow and the fleet's book, because + * twelve bar series and twelve areas would be a pane nobody can read. What a + * reader actually asks is narrower than that and only while they are asking + * it: *how much of this trading is the bot I am pointing at?* So the answer + * is drawn where the question is asked — inside the bar already on screen, + * for the one series the cursor (or a legend chip) has picked, and nowhere at + * all the rest of the time. + */ + const highlight = useCallback( + (row: PnlChartPoint) => { + if (!focus || focus === "total") return null; + const share = (row as FloorChartRow)[ownerVolumeKey(focus)]; + if (typeof share !== "number") return null; + return { value: share, color: seriesColor(keys.indexOf(focus)) }; + }, + [focus, keys], + ); + // ── The activity pane (step 6) ── // // Its geometry is the same geometry PnlEvolutionChart's activity pane draws @@ -220,7 +264,7 @@ export function OwnerPnlChart({ volumeBar, positionDomain, positionZeroOffset, - } = useActivityPane(visible as PnlChartPoint[], spanMs); + } = useActivityPane(visible as PnlChartPoint[], spanMs, highlight); const hasPosition = rows.some((row) => row.position !== 0); // ── The stated gap between the chart and the strip ── @@ -259,6 +303,40 @@ export function OwnerPnlChart({ const drawn = keys.filter((key) => !hidden.has(key) && !muted.has(key)); const totalDrawn = !hidden.has("total") && !muted.has("total"); + /** + * The two y domains, sharing a zero (FEAT-119). + * + * The Total of twelve controllers is twelve controllers tall, so one axis + * scaled to it draws the lines the reader came for as a flat braid — the + * whole pane spent on a series whose shape is already legible at a twelfth of + * the height. It moves to its own axis on the right, in the gutter the pane + * was already reserving for the activity pane's position axis, so the two + * panes' plot areas stay identical and their synced cursors keep pointing at + * the same instant. + * + * `null` — nothing finite to measure, or the Total hidden — leaves both + * domains to recharts, which is the single-axis picture this chart had. + */ + const scales = useMemo(() => { + if (!totalDrawn) return null; + const lines: number[] = []; + const totals: number[] = []; + for (const row of drawnRows) { + for (const key of drawn) { + const value = row[ownerDataKey(key)]; + if (typeof value === "number") lines.push(value); + } + if (typeof row.total === "number") totals.push(row.total); + } + const domains = alignedZeroDomains(lines, totals); + if (!domains) return null; + return { + ...domains, + lineTicks: niceTicks(domains.owner), + totalTicks: niceTicks(domains.total), + }; + }, [drawnRows, drawn, totalDrawn]); + // The same 65/35 split `PnlEvolutionChart` takes, off the same measured box, // so switching between the aggregate chart and this one does not resize the // report column under the reader. @@ -329,6 +407,7 @@ export function OwnerPnlChart({ } drawn={totalDrawn} strong + note="drawn on the right-hand axis, which shares this pane's zero" onToggle={() => toggle("total")} onFocus={() => setFocus("total")} onBlur={() => setFocus(null)} @@ -425,6 +504,8 @@ export function OwnerPnlChart({ height={1} /> - {/* The mirror of the activity pane's position axis. Both panes - must reserve the same gutters or they desync. */} + {/* The Total's own axis, in the gutter the pane reserves either + way — the mirror of the activity pane's position axis, which + both panes must reserve or their plot areas differ and the + synced cursors point at different instants. Its ticks are + the Total's colour: an axis nobody can attribute to a series + is worse than no second axis at all. */} + {/* One line for both axes: `alignedZeroDomains` is what makes + that true, and without it this line would mean one thing for + the lines and another for the Total. */} )} + {/* The picked line's own book over the fleet's, on the fleet's + axis so the two are read against each other — the position + half of what the bar highlight says about volume, and drawn + for exactly as long as the same question is being asked. */} + {hasPosition && focus !== null && focus !== "total" && ( + + )}
@@ -609,6 +724,11 @@ export function OwnerPnlChart({ )}

+ {/* One line, and it has to stay one line: the card is `h-full + overflow-hidden` over a height that accounts for the two panes + and not for this, so a second line is a clipped line. What the + second line would have said — point at a line for its share of + the bars — the tooltip says instead, where it is being asked. */} {bucketLabel ? `Bars are volume traded per ${bucketLabel} bucket. ` : ""} The lines are folded from controller performance history, one call per line — the same fold this page draws when you walk into one of them @@ -636,6 +756,7 @@ function LegendChip({ drawn, strong = false, onToggle, + note, onFocus, onBlur, focused, @@ -649,6 +770,8 @@ function LegendChip({ drawn: boolean; strong?: boolean; onToggle: () => void; + /** Anything else true of this series — which axis it is on, in the Total's case. */ + note?: string; /** Hovering a chip picks the same series pointing at its line does. */ onFocus?: () => void; onBlur?: () => void; @@ -663,7 +786,7 @@ function LegendChip({ onClick={onToggle} onMouseEnter={onFocus} onMouseLeave={onBlur} - title={drawn ? `Hide ${full ?? label}` : `Show ${full ?? label}`} + title={`${drawn ? "Hide" : "Show"} ${full ?? label}${note ? ` — ${note}` : ""}`} className={`flex items-center gap-1.5 rounded px-1 py-0.5 text-[11px] tabular-nums transition-opacity hover:bg-[var(--color-surface-hover)] ${ drawn ? "" : "opacity-40" } ${focused ? "bg-[var(--color-surface-hover)]" : ""}`} @@ -715,6 +838,8 @@ function Toggle({ interface TooltipPayload { dataKey?: string | number; value?: number; + /** The whole row the entry was read from — every field, drawn or not. */ + payload?: FloorChartRow; } /** @@ -755,6 +880,9 @@ function OwnerTooltip({ labels, keys, format, + symbol, + bucketLabel, + hasPosition, showTotal, visible, onFocus, @@ -767,13 +895,25 @@ function OwnerTooltip({ labels: Map; keys: readonly string[]; format: (value: number) => string; + /** For the two figures that are never a percentage: volume and the book. */ + symbol: string; + /** How long one bar is, so "traded" is a quantity and not a mystery. */ + bucketLabel?: string; + hasPosition: boolean; showTotal: boolean; visible: boolean; onFocus: (key: string | null) => void; }) { - // The chart's own y scale, which is the only thing that can say where a - // value is *drawn*. A hook, so it runs before every early return below. + // Where a value is *drawn* — the only thing that can answer which line the + // cursor is on. Two scales because the Total has an axis of its own, and a + // hook each, so both run before every early return below. const scale = useYAxisScale(); + const totalScale = useYAxisScale("total"); + const scales = useMemo(() => { + const by = new Map([["total", totalScale]]); + for (const key of keys) by.set(key, scale); + return by; + }, [keys, scale, totalScale]); const by = useMemo(() => { const out = new Map(); @@ -799,7 +939,7 @@ function OwnerTooltip({ }, [by, keys, showTotal]); const live = active === true && visible; - const focused = live ? nearestSeries(pickable, coordinate?.y, scale) : null; + const focused = live ? nearestSeries(pickable, coordinate?.y, scales) : null; useEffect(() => { onFocus(focused); }, [focused, onFocus]); @@ -807,6 +947,10 @@ function OwnerTooltip({ if (!live || pickable.size === 0) return null; const total = by.get("total"); + // Every entry carries the whole row, which is where the fields nothing in + // this pane draws — the bucket's volume, the book, and each owner's share of + // them — are read from. + const row = payload?.find((entry) => entry.payload)?.payload; const when = (

{typeof label === "number" ? formatDateTime(label) : ""} @@ -826,6 +970,19 @@ function OwnerTooltip({ {name(focused)}{" "} {format(value)}

+ {/* What the pane below is showing about this one line, said in + numbers: its share of the bar the cursor is over, and its own + book inside the fleet's (FEAT-119). Only for a real owner — the + Total's share of the fleet is the fleet. */} + {focused !== "total" && ( + + )} {showTotal && focused !== "total" && typeof total === "number" && (

Total {format(total)} @@ -862,13 +1019,85 @@ function OwnerTooltip({ ))} {rest > 0 && (

- +{rest} more — point at a line to read it + +{rest} more — point at a line to read it and its share of the bars

)}
); } +/** + * One line's share of the pane below it, at the hovered instant (FEAT-119). + * + * The activity pane is fleet-level and stays fleet-level: twelve bar series and + * twelve areas is not a picture. What the reader wants instead is an + * attribution of the *one* bar and the *one* book they can already see, to the + * *one* line they are pointing at — so it is printed here, beside that line's + * PnL, and the bar highlights its share at the same moment. + * + * A share is only stated where it means something. Volume is a non-negative + * flow, so a percentage of the bucket is a fact; the book is signed, and "15% + * of a fleet that is net flat" is a number with no meaning, so the position row + * prints the two quantities and lets them be compared. + * + * Nothing is printed for a bucket in which this owner did not trade — a row of + * zeroes reads as a measurement, and the interesting silence is the same + * silence the bar shows by not being highlighted. + */ +function Attribution({ + row, + owner, + symbol, + bucketLabel, + hasPosition, +}: { + row: FloorChartRow | undefined; + owner: string; + symbol: string; + bucketLabel?: string; + hasPosition: boolean; +}) { + if (!row) return null; + const traded = row[ownerVolumeKey(owner)]; + const bucket = row.volumeDelta; + const book = row[ownerPositionKey(owner)]; + + const share = + typeof traded === "number" && typeof bucket === "number" && bucket > 0 + ? Math.round((traded / bucket) * 100) + : null; + + return ( + <> + {typeof traded === "number" && traded > 0 && ( +

+ Traded{bucketLabel ? ` ${bucketLabel}` : ""}{" "} + + {formatCurrencyVolume(traded, symbol)} + + {share !== null && ( + <> + {" · "} + {share}% of {formatCurrencyVolume(bucket, symbol)} + + )} +

+ )} + {hasPosition && typeof book === "number" && book !== 0 && ( +

+ Book{" "} + + {formatCurrencyVolume(book, symbol)} + + {typeof row.position === "number" && row.position !== 0 && ( + <> of {formatCurrencyVolume(row.position, symbol)} + )} +

+ )} + + ); +} + /** The windows the chips offer, and what each one means as a `TimeRange`. */ const WINDOWS = [ { value: "1d", label: "1D", ms: 24 * 3_600_000 }, diff --git a/frontend/src/lib/owner-series.test.ts b/frontend/src/lib/owner-series.test.ts index fdcfd4338..802ef2793 100644 --- a/frontend/src/lib/owner-series.test.ts +++ b/frontend/src/lib/owner-series.test.ts @@ -18,10 +18,14 @@ import { describe, expect, it } from "vitest"; import type { ControllerInfo, ControllerPerformanceSnapshot } from "@/lib/api"; import { FOCUS_RADIUS_PX, + alignedZeroDomains, mergeOwnerRows, nearestSeries, + niceTicks, ownerDataKey, + ownerPositionKey, ownerSeries, + ownerVolumeKey, parseBaseline, parseBasis, rebaseRows, @@ -153,6 +157,94 @@ describe("the flow and the stock", () => { expect(totalDelta).toBeLessThan(lifetime); expect(lifetime).toBeGreaterThan(0); }); + + // What the hover claims about a bar: "this line traded N of the M in this + // bucket". It is only a fact if the owners' shares are the bar — the same + // fold, the same bucket, the same clamp — rather than an apportionment. + it("splits each bar between the owners it belongs to, exactly", () => { + const series = ownerSeries(SNAPSHOTS, OWNERS, []); + const { rows, keys } = mergeOwnerRows([series.total], series.owners); + + for (const row of rows) { + const shares = keys.reduce((sum, key) => { + const value = row[ownerVolumeKey(key)]; + return sum + (typeof value === "number" ? value : 0); + }, 0); + expect(shares).toBeCloseTo(row.volumeDelta, 9); + } + // And it is a real split rather than a column of zeroes: alpha's second + // reading is the only trading in its bucket. + const traded = rows.find((row) => row.volumeDelta > 0)!; + expect(traded[ownerVolumeKey("alpha")]).toBeCloseTo(traded.volumeDelta, 9); + }); + + it("gives the share and the book the same gap rule as the line", () => { + const series = ownerSeries(SNAPSHOTS, OWNERS, []); + const { rows } = mergeOwnerRows([series.total], series.owners); + const first = rows[0]; + + expect(typeof first[ownerVolumeKey("alpha")]).toBe("number"); + expect(typeof first[ownerPositionKey("alpha")]).toBe("number"); + // Beta has not started: no line, and nothing to attribute to it either. + expect(first[ownerVolumeKey("beta")]).toBeUndefined(); + expect(first[ownerPositionKey("beta")]).toBeUndefined(); + }); +}); + +// ── The two axes (FEAT-119) ── +// +// The Total is twelve controllers tall, so it gets an axis of its own — and +// the moment it has one, the dashed line at zero means two different heights +// unless the two domains are chosen together. These are the arithmetic behind +// "one zero line, true of both". + +describe("alignedZeroDomains", () => { + /** Where zero falls in a domain, as a fraction of its height. */ + const zeroAt = ([min, max]: [number, number]) => -min / (max - min); + + it("puts zero at the same height on both axes", () => { + const domains = alignedZeroDomains([-25, 9, -3], [-210, 105])!; + expect(zeroAt(domains.owner)).toBeCloseTo(zeroAt(domains.total), 9); + }); + + it("leaves room for every value it was given", () => { + const domains = alignedZeroDomains([-25, 9], [-210, 105])!; + expect(domains.owner[0]).toBeLessThanOrEqual(-25); + expect(domains.owner[1]).toBeGreaterThanOrEqual(9); + expect(domains.total[0]).toBeLessThanOrEqual(-210); + expect(domains.total[1]).toBeGreaterThanOrEqual(105); + }); + + it("spends no pane on a side neither series is on", () => { + // A fleet that has only ever been down: zero is the top of both axes, not + // the middle of them. + const domains = alignedZeroDomains([-25, -3], [-210, -40])!; + expect(domains.owner[1]).toBe(0); + expect(domains.total[1]).toBe(0); + }); + + it("has no answer where there is nothing to scale", () => { + expect(alignedZeroDomains([], [1, 2])).toBeNull(); + expect(alignedZeroDomains([1, 2], [Number.NaN])).toBeNull(); + // Flat at zero: no height to divide, and recharts' own domain is as good. + expect(alignedZeroDomains([0, 0], [0, 0])).toBeNull(); + }); +}); + +describe("niceTicks", () => { + it("always includes zero, so the dashed line is labelled", () => { + expect(niceTicks([-26, 9.4])).toContain(0); + expect(niceTicks([-218.4, 109.2])).toContain(0); + expect(niceTicks([-0.026, 0.0094])).toContain(0); + }); + + it("steps in round numbers rather than in fifths of an odd domain", () => { + expect(niceTicks([-26, 9.4])).toEqual([-25, -20, -15, -10, -5, 0, 5]); + }); + + it("has no ticks for a domain with no height", () => { + expect(niceTicks([0, 0])).toEqual([]); + }); }); describe("the four toggle states", () => { @@ -295,6 +387,27 @@ describe("nearestSeries", () => { expect(nearestSeries(gappy, 440, scale)).toBeNull(); }); + it("compares in pixels across two axes, not in values", () => { + // The Total on its own axis: a tenth of the pixels per dollar. A single + // scale would place its -200 off the pane; per series, it is drawn at 400 + // and the cursor there is on it and not on alpha. + const totalScale = (value: number) => 500 - value / 10; + const scales = new Map([ + ["total", totalScale], + ["alpha", scale], + ["beta", scale], + ]); + const twoAxes = new Map([ + ["total", 1000], + ["alpha", 60], + ["beta", 20], + ]); + expect(nearestSeries(twoAxes, 400, scales)).toBe("total"); + expect(nearestSeries(twoAxes, 440, scales)).toBe("alpha"); + // A series the map has no scale for is not on screen to be pointed at. + expect(nearestSeries(twoAxes, 400, new Map([["alpha", scale]]))).toBeNull(); + }); + it("breaks a tie toward the first entry, which is the legend's order", () => { const tied = new Map([["alpha", 60], ["beta", 60]]); expect(nearestSeries(tied, 440, scale)).toBe("alpha"); diff --git a/frontend/src/lib/owner-series.ts b/frontend/src/lib/owner-series.ts index d7c46dd5e..0cdbe4be7 100644 --- a/frontend/src/lib/owner-series.ts +++ b/frontend/src/lib/owner-series.ts @@ -104,6 +104,23 @@ export function ownerDataKey(key: string): string { return `owner:${key}`; } +/** + * The `dataKey` an owner's **share of the bucket's trading** is carried under. + * + * Not a line: nothing draws twelve volume series at once, and that is the point + * — the activity pane draws the fleet's bar and this is what the *one* series + * the cursor is on contributed to it (FEAT-119). Same fold, same bucket, so the + * share is exact rather than apportioned. + */ +export function ownerVolumeKey(key: string): string { + return `vol:${key}`; +} + +/** The `dataKey` an owner's own book is carried under — the same idea as {@link ownerVolumeKey}. */ +export function ownerPositionKey(key: string): string { + return `pos:${key}`; +} + /** One merged row: every fleet field, plus one value per owner. */ export interface FloorChartRow extends PnlChartPoint { [key: string]: number; @@ -122,6 +139,13 @@ export interface FloorChartRow extends PnlChartPoint { * it would charge the same trading to every later bucket, so it is summed at * the instants that actually carry it and is zero everywhere else. * + * Each owner contributes three fields rather than one: its PnL under + * {@link ownerDataKey}, its book under {@link ownerPositionKey} and its share + * of the bucket's trading under {@link ownerVolumeKey}, each folded by the rule + * its fleet-level twin above is folded by. The last two are never drawn as + * twelve more series — they are what the hover reads when the cursor picks one + * line, which is the only moment they are wanted (FEAT-119). + * * `total` takes an array of series rather than one because a caller may hold * several folds that belong on one timeline — one per server, each folded with * its own currency converter, which is the shape the retired floor page had. @@ -186,17 +210,29 @@ export function mergeOwnerRows( } for (const key of keys) { let value = 0; + let book = 0; + let flow = 0; let seen = false; for (const series of byKey.get(key)!) { const point = at(series, t); if (!point) continue; seen = true; value += point.total; + book += point.position; + // The flow, under the fleet's own rule one loop up: charged to the + // bucket that recorded it and to no later one. An owner with no reading + // at `t` traded nothing in that bucket — it is a zero, not a gap, and + // that is exactly what makes the shares of one bar add up to it. + if (point.time === t) flow += point.volumeDelta; } // An owner that has not started yet contributes no point at all rather // than a zero: recharts draws a gap, which is the truth, where a zero // would draw a flat line along the axis for trading that had not begun. - if (seen) row[ownerDataKey(key)] = value; + if (seen) { + row[ownerDataKey(key)] = value; + row[ownerPositionKey(key)] = book; + row[ownerVolumeKey(key)] = flow; + } } rows.push(row); } @@ -302,6 +338,110 @@ export function rebaseRows( }; } +// ── Two axes, one zero (FEAT-119) ── + +/** A recharts y domain, as an explicit pair. */ +export type Domain = [number, number]; + +/** + * A domain for the owner lines and a domain for the Total that put **zero on + * the same pixel**. + * + * The Total of twelve controllers is about twelve times any one of them, so a + * single axis is scaled to the Total and the lines the reader came for are a + * flat braid across the middle of it (the picture FEAT-119 was opened over). + * The fix is the ordinary one — the Total on its own axis — and the ordinary + * cost of it is that the two axes then disagree about where zero is, on a chart + * whose whole subject is whether a number is above or below zero. A line + * crossing the dashed zero would mean one thing and the Total crossing it + * another, with nothing on screen saying so. + * + * So the two domains are not chosen independently. Both are split at the same + * fraction — the more demanding of the two sides' own split — and each is then + * scaled to whichever half needs the room. One dashed line at zero is therefore + * true of both axes, and the two curves' *shapes* stay comparable: only the + * unit differs, which is what the second axis' ticks say. + * + * `null` when either side has nothing finite to measure or is flat at zero: + * that is not an error, it is a pane with no scale to derive, and the caller + * leaves both domains to recharts. + */ +export function alignedZeroDomains( + owner: readonly number[], + total: readonly number[], + pad = 0.04, +): { owner: Domain; total: Domain } | null { + const o = zeroExtent(owner); + const t = zeroExtent(total); + if (!o || !t) return null; + + // How much of the pane sits above zero. The larger of the two, so neither + // side is cut off; the other simply gets more headroom than it needs. + const up = Math.max(upShare(o), upShare(t)); + const down = 1 - up; + const height = (e: Domain) => + Math.max(up > 0 ? e[1] / up : 0, down > 0 ? -e[0] / down : 0) * (1 + pad); + + const ho = height(o); + const ht = height(t); + if (!(ho > 0) || !(ht > 0)) return null; + return { owner: [-down * ho, up * ho], total: [-down * ht, up * ht] }; +} + +/** The values' extent, always straddling zero; `null` when none of them is a number. */ +function zeroExtent(values: readonly number[]): Domain | null { + let min = 0; + let max = 0; + let any = false; + for (const value of values) { + if (!Number.isFinite(value)) continue; + any = true; + if (value < min) min = value; + if (value > max) max = value; + } + return any ? [min, max] : null; +} + +/** The fraction of an extent that lies above zero. */ +function upShare([min, max]: Domain): number { + const span = max - min; + return span > 0 ? max / span : 0; +} + +/** + * Round tick values inside a domain, always including zero. + * + * An explicit domain costs the ticks recharts would have chosen: it divides + * whatever it is given into equal steps, so an aligned pair of domains — which + * are aligned precisely because they are *not* round — would print five + * arbitrary numbers per axis and no `$0` on either, on the two axes a dashed + * zero line runs between. Ticks of our own put the round numbers back and pin + * that line to a label on both sides of the pane. + */ +export function niceTicks([min, max]: Domain, count = 4): number[] { + const span = max - min; + if (!(span > 0) || !Number.isFinite(span)) return []; + const rough = span / Math.max(1, count); + const magnitude = 10 ** Math.floor(Math.log10(rough)); + // The largest round step that still fits, rather than the smallest that + // covers: an aligned domain is by nature an odd number, and rounding its + // step *up* leaves a 65%-tall pane with three labels on it. Then back off + // while that is too many, which bounds the count from the other side. + const ladder = [1, 2, 2.5, 5, 10].map((m) => m * magnitude); + let step = ladder.filter((s) => s <= rough).pop() ?? magnitude; + while (span / step > 2 * count) step = ladder.find((s) => s > step) ?? step * 2; + + // Counted in whole steps rather than accumulated, so the zero step is exactly + // 0 and not the 1e-11 that repeated addition of a step like 2.5e-3 lands on — + // which is a tick the axis would print, and a zero line it would miss. + const ticks: number[] = []; + const last = Math.floor(max / step + 1e-9); + for (let i = Math.ceil(min / step - 1e-9); i <= last; i++) { + ticks.push(Number((i * step).toPrecision(12))); + } + return ticks; +} + /** The eight categorical tokens, cycled — see the note beside them in index.css. */ export function seriesColor(index: number): string { return `var(--chart-series-${(Math.max(0, index) % 8) + 1})`; @@ -321,6 +461,9 @@ export function seriesColor(index: number): string { */ export const FOCUS_RADIUS_PX = 12; +/** What a y axis answers with: where a value is drawn, in pixels. */ +export type SeriesScale = ((value: number) => number | undefined) | null | undefined; + /** * The series the cursor is on, or `null` for a cursor on none. * @@ -340,15 +483,22 @@ export const FOCUS_RADIUS_PX = 12; export function nearestSeries( values: ReadonlyMap, cursorY: number | null | undefined, - scale: ((value: number) => number | undefined) | null | undefined, + scale: SeriesScale | ReadonlyMap, radius: number = FOCUS_RADIUS_PX, ): string | null { if (typeof cursorY !== "number" || !Number.isFinite(cursorY) || !scale) return null; + // One scale for a pane whose series share an axis; a scale *per series* for + // one whose Total is on its own (FEAT-119) — the same pixel comparison either + // way, which is the only way it can stay right across two axes. + const scaleOf = (key: string): SeriesScale => + scale instanceof Map ? scale.get(key) : (scale as SeriesScale); let best: string | null = null; let bestDistance = radius; for (const [key, value] of values) { if (!Number.isFinite(value)) continue; - const y = scale(value); + const on = scaleOf(key); + if (!on) continue; + const y = on(value); if (typeof y !== "number" || !Number.isFinite(y)) continue; const distance = Math.abs(y - cursorY); if (distance < bestDistance) { diff --git a/frontend/src/lib/pnl-chart-pane.tsx b/frontend/src/lib/pnl-chart-pane.tsx index 4fec7d36a..d4377a01f 100644 --- a/frontend/src/lib/pnl-chart-pane.tsx +++ b/frontend/src/lib/pnl-chart-pane.tsx @@ -27,6 +27,18 @@ import { type SamplingInterval, } from "@/lib/pnl-chart"; +/** + * The part of one bar that belongs to the series the reader is pointing at. + * + * `value` is in the bar's own units — a slice of the same bucket, not a + * fraction of it — because that is what the caller holds and what keeps the + * arithmetic here to one division against the bar recharts already sized. + */ +export interface BarHighlight { + value: number; + color: string; +} + export interface ActivityPane { /** Hand to the pane's ``; sizes the bars. */ onActivityResize: (width: number) => void; @@ -48,8 +60,13 @@ export interface ActivityPane { * * @param visible the points currently on screen * @param spanMs the window's time span, `last.time - first.time` + * @param highlight what part of each bar belongs to the picked series, if any */ -export function useActivityPane(visible: PnlChartPoint[], spanMs: number): ActivityPane { +export function useActivityPane( + visible: PnlChartPoint[], + spanMs: number, + highlight?: (row: PnlChartPoint) => BarHighlight | null, +): ActivityPane { // The measurement comes from the pane's own ResponsiveContainer, which is // already observing its size, rather than from a second observer of ours. It // is 0 until the first callback — and stays 0 where there is no layout at all @@ -87,7 +104,8 @@ export function useActivityPane(visible: PnlChartPoint[], spanMs: number): Activ (props: BarShapeProps) => { const width = barWidth ?? props.width; const x = props.x + props.width / 2 - width / 2; - return ( + const share = highlight?.(props.payload as PnlChartPoint) ?? null; + const bar = ( ); + // The picked series' share, drawn *inside* the bar from the baseline up + // rather than beside it or stacked on it: this is a part of the bucket + // already drawn, so the bar must keep its height and only fill in. + // + // In pixels by proportion, which is exact because this axis starts at + // zero — recharts has already mapped the whole bucket to `height`, and no + // second scale lookup can disagree with it. + const whole = Array.isArray(props.value) ? props.value[1] - props.value[0] : props.value; + if (!share || !(whole > 0) || !(share.value > 0)) return bar; + const height = props.height * Math.min(1, share.value / whole); + return ( + + {bar} + + + ); }, - [barWidth], + [barWidth, highlight], ); // The position axis is pinned across zero rather than left to recharts, so From 5f38516958ed2dcd0ae27cb7cccda3b9ebe423d9 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 23:34:39 +0300 Subject: [PATCH 097/154] Read venues through one shared hook instead of two hand-copied queries CreateExecutor and DexPool each declared their own ["venues", server] query with the same key, fetcher and 5-minute staleTime repeated literally. Extract useVenues() alongside the other market hooks so the key is single-sourced and a third page can't copy the literal with a drifted key or staleTime. Pure refactor: CreateExecutor's pending-window logic (credentialedConnectors, listsReady) is unchanged, and a new test asserts the two callers share one cached request for the same server. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LYuUt3zxZCa7on47NbDxRs --- .../src/components/market/useVenues.test.tsx | 102 ++++++++++++++++++ frontend/src/components/market/useVenues.ts | 29 +++++ frontend/src/pages/CreateExecutor.tsx | 12 +-- frontend/src/pages/DexPool.tsx | 10 +- 4 files changed, 139 insertions(+), 14 deletions(-) create mode 100644 frontend/src/components/market/useVenues.test.tsx create mode 100644 frontend/src/components/market/useVenues.ts diff --git a/frontend/src/components/market/useVenues.test.tsx b/frontend/src/components/market/useVenues.test.tsx new file mode 100644 index 000000000..70722d109 --- /dev/null +++ b/frontend/src/components/market/useVenues.test.tsx @@ -0,0 +1,102 @@ +/** + * `useVenues` is the one declaration of the `["venues", server]` query + * (ARCH-355) — CreateExecutor (Trade) and DexPool both call it instead of each + * hand-copying `queryKey`/`queryFn`/`staleTime`. The point of extracting it is + * that two callers on the same server share one cached request rather than + * two independent ones that could drift apart; this proves that cache sharing + * directly, standing in for the two pages with two hook consumers under one + * `QueryClient`. + * + * Needs a DOM, so this file overrides vitest's default `node` environment. + * + * @vitest-environment jsdom + */ + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { VenueTraits } from "@/lib/connector-capabilities"; +import { useVenues } from "./useVenues"; + +const VENUES: VenueTraits[] = [ + { name: "binance", hummingbotMarketData: true, clmmLp: false, credentialed: true }, +]; + +const getVenues = vi.fn(async (server: string) => { + void server; + return VENUES; +}); + +vi.mock("@/lib/api", () => ({ + api: { + getVenues: (server: string) => getVenues(server), + }, +})); + +declare global { + // eslint-disable-next-line no-var + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +let container: HTMLDivElement; +let root: Root; + +/** Stands in for a page that reads venues, e.g. CreateExecutor or DexPool. */ +function Consumer({ label }: { label: string }) { + const { venues, isPending } = useVenues("srv-1"); + return ( +
+ {isPending ? "pending" : venues.map((v) => v.name).join(",")} +
+ ); +} + +async function renderBothPages() { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false, gcTime: 0 } }, + }); + await act(async () => { + root.render( + + + + , + ); + }); + // react-query resolves on a later macrotask than the render that asked. + for (let i = 0; i < 5; i++) { + await act(async () => { + await new Promise((r) => setTimeout(r, 0)); + }); + } +} + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + vi.clearAllMocks(); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); +}); + +describe("useVenues shares one cached request across callers", () => { + it("fetches venues once for two consumers on the same server", async () => { + await renderBothPages(); + + const trade = container.querySelector('[data-consumer="trade"]'); + const dexPool = container.querySelector('[data-consumer="dex-pool"]'); + expect(trade?.textContent).toBe("binance"); + expect(dexPool?.textContent).toBe("binance"); + + // The cache-hit assertion: two mounted consumers, one network request. + expect(getVenues).toHaveBeenCalledTimes(1); + expect(getVenues).toHaveBeenCalledWith("srv-1"); + }); +}); diff --git a/frontend/src/components/market/useVenues.ts b/frontend/src/components/market/useVenues.ts new file mode 100644 index 000000000..1336ac2be --- /dev/null +++ b/frontend/src/components/market/useVenues.ts @@ -0,0 +1,29 @@ +import { useQuery } from "@tanstack/react-query"; + +import { api } from "@/lib/api"; +import type { VenueTraits } from "@/lib/connector-capabilities"; + +/** + * Every venue the trade panel can offer, each with the traits the UI + * decisions rest on (see `VenueTraits`). The server dedups (a venue in both + * of its input lists is a Hummingbot connector), so there is no merge to get + * wrong here. + * + * CreateExecutor and DexPool both ask through this hook so they share one + * cached request per server, and so the `["venues", server]` key and its + * 5-minute `staleTime` are declared in exactly one place — a third caller + * cannot copy the literal with a drifted key or `staleTime` (ARCH-355). + * + * `isPending` stays true (and `venues` empty) until the first answer lands, + * so callers can tell "no venues yet" apart from "no venues at all" instead + * of flashing every venue as view-only while the query is in flight. + */ +export function useVenues(server: string | null | undefined) { + const { data: venues = [], isPending } = useQuery({ + queryKey: ["venues", server], + queryFn: () => api.getVenues(server!), + enabled: !!server, + staleTime: 5 * 60 * 1000, + }); + return { venues, isPending }; +} diff --git a/frontend/src/pages/CreateExecutor.tsx b/frontend/src/pages/CreateExecutor.tsx index 9a9f1219b..176676b25 100644 --- a/frontend/src/pages/CreateExecutor.tsx +++ b/frontend/src/pages/CreateExecutor.tsx @@ -17,6 +17,7 @@ import { import { NoServerCard } from "@/components/NoServerCard"; import { useTradingRules } from "@/components/market/useTradingRules"; +import { useVenues } from "@/components/market/useVenues"; import { PriceTicker } from "@/components/market/PriceTicker"; import { MarketDepthPanel } from "@/components/market/MarketDepthPanel"; import { MarketBrowser, type MarketPick } from "@/components/market/MarketBrowser"; @@ -196,14 +197,9 @@ export function CreateExecutor() { }); // One query, one answer: every venue the panel can offer, each with the traits - // the UI decisions below rest on. The server dedups (a venue in both of its input - // lists is a Hummingbot connector), so there is no merge to get wrong here. - const { data: venues = [], isPending: venuesPending } = useQuery({ - queryKey: ["venues", server], - queryFn: () => api.getVenues(server!), - enabled: !!server, - staleTime: 5 * 60 * 1000, - }); + // the UI decisions below rest on. Shared with DexPool through `useVenues`, so + // both pages read the same cached request instead of hand-copying the query. + const { venues, isPending: venuesPending } = useVenues(server); // The list has to be in before the panel may *correct* a selection: judging a // persisted venue against an empty list would bounce it on every reload and diff --git a/frontend/src/pages/DexPool.tsx b/frontend/src/pages/DexPool.tsx index f06f4e9f0..2d9f057fe 100644 --- a/frontend/src/pages/DexPool.tsx +++ b/frontend/src/pages/DexPool.tsx @@ -22,6 +22,7 @@ import { LPConfigPanel } from "@/components/executor/LPConfigPanel"; import { useLpConfig } from "@/components/executor/lp-config"; import { OrderConfigPanel } from "@/components/executor/OrderConfigPanel"; import { useOrderConfig } from "@/components/executor/order-config"; +import { useVenues } from "@/components/market/useVenues"; import { TradeBottomPane } from "@/components/trade/TradeBottomPane"; import { TradeChart, type ChartPriceAxis } from "@/components/trade/TradeChart"; import { useDexUpstream } from "@/hooks/useDexUpstream"; @@ -208,12 +209,9 @@ export function DexPool() { }, }); - const { data: venues = [] } = useQuery({ - queryKey: ["venues", server], - queryFn: () => api.getVenues(server!), - enabled: !!server, - staleTime: 5 * 60 * 1000, - }); + // Shared with CreateExecutor through `useVenues`, so both pages read the same + // cached request instead of hand-copying the query. + const { venues } = useVenues(server); // Bins move with every swap through the active bin, so the server caches them // for a minute and this polls at the same cadence — every viewer of the pool From 2d46126324210b6476d5b8154b561a71cecda327 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 23:43:15 +0300 Subject: [PATCH 098/154] Invalidate venues and connected-exchanges alongside settings-credentials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Credential mutations (add, delete, and the Hyperliquid connect flow) only invalidated ["settings-credentials", server]. TanStack matches invalidation by prefix, so that provably could not reach ["venues", server] or ["connected-exchanges", server] — both held at a 5-minute staleTime — and the Trade page kept rendering its view-only overlay for a venue whose keys were just saved, for up to 5 minutes (CORR-353, issue #238). Add invalidateCredentialQueries(client, server) in lib/queryClient.ts as the one shared helper for the full set, and point ApiKeysSettings at it instead of its local single-key invalidate. Export venuesQueryKey from useVenues.ts (ARCH-355) so the helper doesn't hand-copy the literal. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LYuUt3zxZCa7on47NbDxRs --- frontend/src/components/market/useVenues.ts | 11 +- .../ApiKeysSettings.invalidate.test.tsx | 188 ++++++++++++++++++ .../components/settings/ApiKeysSettings.tsx | 8 +- frontend/src/lib/queryClient.test.ts | 48 +++++ frontend/src/lib/queryClient.ts | 26 +++ 5 files changed, 277 insertions(+), 4 deletions(-) create mode 100644 frontend/src/components/settings/ApiKeysSettings.invalidate.test.tsx diff --git a/frontend/src/components/market/useVenues.ts b/frontend/src/components/market/useVenues.ts index 1336ac2be..26678f860 100644 --- a/frontend/src/components/market/useVenues.ts +++ b/frontend/src/components/market/useVenues.ts @@ -18,9 +18,18 @@ import type { VenueTraits } from "@/lib/connector-capabilities"; * so callers can tell "no venues yet" apart from "no venues at all" instead * of flashing every venue as view-only while the query is in flight. */ +/** + * The `["venues", server]` key, single-sourced (ARCH-355) so a caller that + * only needs to invalidate the entry — a credential mutation, not a render — + * does not have to copy the literal (CORR-353). + */ +export function venuesQueryKey(server: string | null | undefined) { + return ["venues", server] as const; +} + export function useVenues(server: string | null | undefined) { const { data: venues = [], isPending } = useQuery({ - queryKey: ["venues", server], + queryKey: venuesQueryKey(server), queryFn: () => api.getVenues(server!), enabled: !!server, staleTime: 5 * 60 * 1000, diff --git a/frontend/src/components/settings/ApiKeysSettings.invalidate.test.tsx b/frontend/src/components/settings/ApiKeysSettings.invalidate.test.tsx new file mode 100644 index 000000000..7cf5445b0 --- /dev/null +++ b/frontend/src/components/settings/ApiKeysSettings.invalidate.test.tsx @@ -0,0 +1,188 @@ +/** + * Guards the credential-mutation invalidation set through the real component + * (CORR-353), not just the shared helper in isolation. + * + * `addMut` and `deleteMut`'s `onSuccess` used to invalidate only + * `["settings-credentials", server]`. This seeds `["venues", server]` and + * `["connected-exchanges", server]` the way `usePrefetchData` warms them on + * load, then asserts a successful `addCredential` / `deleteCredential` + * reaches all three: + * + * - `settings-credentials` has an active observer in this harness (the tab's + * own `useQuery`), so `invalidateQueries` refetches it immediately — the + * assertion there is an extra GET, the same shape as the existing refetch + * tests below. + * - `venues` and `connected-exchanges` have no mounted observer here (nothing + * on the Settings page reads them), so invalidation only marks them stale + * for a later fetch — the assertion there is `isInvalidated`, exactly as + * the item's Notes describe. + * + * Needs a DOM, so this file overrides vitest's default `node` environment. + * + * @vitest-environment jsdom + */ + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { venuesQueryKey } from "@/components/market/useVenues"; +import { ServerContext } from "@/hooks/useServer"; +import { ApiKeysSettings } from "./ApiKeysSettings"; + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +const SERVER = "prod"; +let container: HTMLDivElement; +let root: Root; +let requested: string[] = []; +/** Toggled per test: whether GET credentials reports binance already added. */ +let hasBinanceCredential = false; + +const CONNECTORS = { connectors: [{ name: "binance", type: "spot" }] }; +const CONFIG_MAP = { config_map: { api_key: { type: "str", required: true } } }; + +function respond(path: string, init?: RequestInit): unknown { + if (path.endsWith("/api/v1/servers")) { + return [{ name: SERVER, host: "h", port: 8000, online: true, permission: "owner" }]; + } + if (path.includes("/settings/credentials") && init?.method === "POST") return { added: true }; + if (path.includes("/settings/credentials") && init?.method === "DELETE") return { deleted: true }; + if (path.includes("/settings/credentials")) { + return { + credentials: hasBinanceCredential + ? [{ connector_name: "binance", connector_type: "spot" }] + : [], + }; + } + if (path.includes("/config-map")) return CONFIG_MAP; + if (path.includes("/settings/connectors")) return CONNECTORS; + if (path.includes("/gateway/wallets")) return { wallets: [] }; + if (path.includes("/gateway/status")) return { running: false }; + return {}; +} + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + requested = []; + hasBinanceCredential = false; + vi.stubGlobal( + "fetch", + vi.fn(async (path: string, init?: RequestInit) => { + requested.push(`${init?.method ?? "GET"} ${path}`); + return new Response(JSON.stringify(respond(path, init)), { + headers: { "Content-Type": "application/json" }, + }); + }), + ); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); +}); + +async function flush() { + for (let i = 0; i < 25; i++) { + await act(async () => { + await new Promise((r) => setTimeout(r, 0)); + }); + } +} + +const credentialGetCalls = () => + requested.filter((u) => u.startsWith("GET") && u.includes("/settings/credentials")).length; + +const isInvalidated = (qc: QueryClient, key: readonly unknown[]) => + qc.getQueryState(key as unknown[])?.isInvalidated === true; + +function clickContaining(text: string): HTMLButtonElement { + const found = [...container.querySelectorAll("button")].find((b) => + b.textContent?.includes(text), + ); + if (!found) throw new Error(`no button containing "${text}"`); + return found as HTMLButtonElement; +} + +function button(label: string): HTMLButtonElement { + const all = [...container.querySelectorAll("button")]; + const found = all.find( + (b) => b.getAttribute("aria-label") === label || b.textContent?.trim() === label, + ); + if (!found) { + const have = all.map((b) => b.getAttribute("aria-label") ?? b.textContent?.trim()).join(" | "); + throw new Error(`no button "${label}" — have: ${have}`); + } + return found as HTMLButtonElement; +} + +/** Mounts the tab and warms `venues`/`connected-exchanges` the way usePrefetchData does. */ +async function mountKeysTab() { + const qc = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }); + for (const key of [venuesQueryKey(SERVER), ["connected-exchanges", SERVER]]) { + qc.setQueryData(key, []); + } + await act(async () => { + root.render( + + {} }}> + + + , + ); + }); + await flush(); + return qc; +} + +describe("ApiKeysSettings credential mutations invalidate the full set", () => { + it("reaches settings-credentials, venues and connected-exchanges after addCredential", async () => { + const qc = await mountKeysTab(); + const callsBeforeAdd = credentialGetCalls(); + + await act(async () => button("Add API Key").click()); + await act(async () => clickContaining("spot").click()); + await flush(); + await act(async () => clickContaining("binance").click()); + await flush(); + await act(async () => button("Add Credential").click()); + await flush(); + + expect(requested.some((u) => u.startsWith("POST") && u.includes("/settings/credentials"))).toBe( + true, + ); + // settings-credentials has an active observer here, so invalidation shows + // up as a refetch rather than a lingering `isInvalidated` flag. + expect(credentialGetCalls()).toBeGreaterThan(callsBeforeAdd); + // venues and connected-exchanges have no observer mounted on this page — + // invalidation only marks them stale for later. + expect(isInvalidated(qc, venuesQueryKey(SERVER))).toBe(true); + expect(isInvalidated(qc, ["connected-exchanges", SERVER])).toBe(true); + }); + + it("reaches settings-credentials, venues and connected-exchanges after deleteCredential", async () => { + hasBinanceCredential = true; + const qc = await mountKeysTab(); + const callsBeforeDelete = credentialGetCalls(); + + await act(async () => button("Delete credential").click()); + await act(async () => button("Confirm delete").click()); + await flush(); + + expect( + requested.some((u) => u.startsWith("DELETE") && u.includes("/settings/credentials/binance")), + ).toBe(true); + expect(credentialGetCalls()).toBeGreaterThan(callsBeforeDelete); + expect(isInvalidated(qc, venuesQueryKey(SERVER))).toBe(true); + expect(isInvalidated(qc, ["connected-exchanges", SERVER])).toBe(true); + }); +}); diff --git a/frontend/src/components/settings/ApiKeysSettings.tsx b/frontend/src/components/settings/ApiKeysSettings.tsx index e692b4601..52f0ca048 100644 --- a/frontend/src/components/settings/ApiKeysSettings.tsx +++ b/frontend/src/components/settings/ApiKeysSettings.tsx @@ -16,7 +16,7 @@ import { useServer } from "@/hooks/useServer"; import { OWNER_ONLY_HINT, useServerPermission } from "@/hooks/useServerPermission"; import { type ConnectorInfo, type CredentialInfo, type GatewayWalletGroup, api } from "@/lib/api"; import { CREDENTIAL_FIELD_PATTERNS } from "@/lib/credential-fields"; -import { credentialsQuery, gatewayWalletsQuery } from "@/lib/queryClient"; +import { credentialsQuery, gatewayWalletsQuery, invalidateCredentialQueries } from "@/lib/queryClient"; import { ConnectHyperliquid } from "./ConnectHyperliquid"; import { ImportGatewayWallet, type WalletChain } from "./ImportGatewayWallet"; @@ -125,8 +125,10 @@ export function ApiKeysSettings() { [qc, server], ); - const invalidate = () => - qc.invalidateQueries({ queryKey: credentialsQuery(server).queryKey }); + // Adding, deleting or (via ConnectHyperliquid, below) connecting a credential + // has to invalidate more than the credential list itself — see + // invalidateCredentialQueries (CORR-353). + const invalidate = () => invalidateCredentialQueries(qc, server); const addMut = useMutation({ mutationFn: () => diff --git a/frontend/src/lib/queryClient.test.ts b/frontend/src/lib/queryClient.test.ts index 6d6a76b90..81c12595b 100644 --- a/frontend/src/lib/queryClient.test.ts +++ b/frontend/src/lib/queryClient.test.ts @@ -16,12 +16,14 @@ import { QueryClient, QueryObserver } from "@tanstack/react-query"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { venuesQueryKey } from "@/components/market/useVenues"; import { CONTROLLER_PERF_ROOTS, controllerPerfHistoryAllQuery, controllerPerfHistoryQuery, credentialsQuery, executorsQuery, + invalidateCredentialQueries, invalidateServerScopedQueries, parseControllerPerfHistoryKey, parseExecutorsKey, @@ -164,6 +166,52 @@ describe("invalidateServerScopedQueries", () => { }); }); +/** + * Guards the credential-mutation invalidation set (CORR-353). + * + * `["settings-credentials", server]` used to be the only key a credential + * mutation invalidated. TanStack matches invalidation keys by prefix, so that + * provably could not reach `["venues", server]` or + * `["connected-exchanges", server]` — both held at a multi-minute staleTime — + * and the Trade page kept its pre-credential view-only verdict for up to 5 + * minutes after a save. This pins all three keys to one shared call so a + * caller cannot invalidate only part of the set again. + */ +describe("invalidateCredentialQueries", () => { + const SERVER = "prod"; + const KEYS = () => [ + credentialsQuery(SERVER).queryKey, + venuesQueryKey(SERVER), + ["connected-exchanges", SERVER], + ]; + + it("invalidates the credential list, venues and connected-exchanges together", () => { + for (const key of KEYS()) client.setQueryData(key, "cached"); + + invalidateCredentialQueries(client, SERVER); + + for (const key of KEYS()) { + expect(isInvalidated(key as unknown[]), JSON.stringify(key)).toBe(true); + } + }); + + it("does not touch another server's entries", () => { + for (const key of KEYS()) client.setQueryData(key, "cached"); + const otherKeys = [ + credentialsQuery(NEXT).queryKey, + venuesQueryKey(NEXT), + ["connected-exchanges", NEXT], + ]; + for (const key of otherKeys) client.setQueryData(key, "cached"); + + invalidateCredentialQueries(client, SERVER); + + for (const key of otherKeys) { + expect(isInvalidated(key as unknown[]), JSON.stringify(key)).toBe(false); + } + }); +}); + /** * Pins the executors key contract (ARCH-227). * diff --git a/frontend/src/lib/queryClient.ts b/frontend/src/lib/queryClient.ts index 64a4ddb5b..a49c117d1 100644 --- a/frontend/src/lib/queryClient.ts +++ b/frontend/src/lib/queryClient.ts @@ -1,5 +1,7 @@ import { QueryClient } from "@tanstack/react-query"; +import { venuesQueryKey } from "@/components/market/useVenues"; + /** * App-wide TanStack Query cache. * @@ -363,3 +365,27 @@ export function gatewayWalletsQuery(server: string | null | undefined) { staleTime: CREDENTIALS_STALE_MS, }; } + +/** + * Invalidates every cached answer a credential mutation (add/delete a CEX + * key, or the Hyperliquid connect flow) can change: the credential list + * itself, the venue traits the Trade page's `credentialed` gate reads + * (`useVenues` -> `caps.canTrade` in lib/connector-capabilities.ts), and the + * connected-exchange list `usePrefetchData` warms. All three are held at a + * multi-minute `staleTime`, so without this the Trade page kept rendering the + * view-only overlay for a venue whose keys were just saved — up to 5 minutes + * after the save, because only `["settings-credentials", server]` was + * invalidated and TanStack matches by prefix, so it provably cannot touch the + * other two (CORR-353, issue #238). + * + * One shared call rather than each caller hand-listing the keys: a second + * hand-written list is exactly how this drifted in the first place. + */ +export function invalidateCredentialQueries( + client: QueryClient, + server: string | null | undefined, +) { + client.invalidateQueries({ queryKey: credentialsQuery(server).queryKey }); + client.invalidateQueries({ queryKey: venuesQueryKey(server) }); + client.invalidateQueries({ queryKey: ["connected-exchanges", server] }); +} From f9539f21f9765d134c1a24f1b1c2410074bef4a0 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 23:48:34 +0300 Subject: [PATCH 099/154] Invalidate Hyperliquid's own caches at the save, not the referral dismissal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ConnectHyperliquid wrote credentials directly and relied on the parent's onDone to invalidate — but onDone only fires from the referral prompt's "Link code" / "Skip" buttons, so a user who saves and navigates away left the Keys list, venues and connected-exchanges stale. Call the shared invalidateCredentialQueries helper right after the saves settle instead. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LYuUt3zxZCa7on47NbDxRs --- .../settings/ConnectHyperliquid.test.tsx | 170 ++++++++++++++++++ .../settings/ConnectHyperliquid.tsx | 9 + 2 files changed, 179 insertions(+) create mode 100644 frontend/src/components/settings/ConnectHyperliquid.test.tsx diff --git a/frontend/src/components/settings/ConnectHyperliquid.test.tsx b/frontend/src/components/settings/ConnectHyperliquid.test.tsx new file mode 100644 index 000000000..285f9a0aa --- /dev/null +++ b/frontend/src/components/settings/ConnectHyperliquid.test.tsx @@ -0,0 +1,170 @@ +/** + * CORR-354: ConnectHyperliquid used to invalidate the credential-derived caches + * only from the parent's `onDone`, which only fires when the user clicks + * "Link code" or "Skip" on the post-save referral prompt. A user who saves + * credentials and then navigates away (the common case — the account usually + * has no referrer yet, so the prompt shows) left the Keys list, venues and + * connected-exchanges stale. + * + * This exercises the connect flow directly (not through ApiKeysSettings) and + * asserts the shared `invalidateCredentialQueries` helper runs right after the + * saves settle — before the referral prompt is dismissed, and even when it + * never is. + * + * Needs a DOM, so this file overrides vitest's default `node` environment. + * + * @vitest-environment jsdom + */ + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { venuesQueryKey } from "@/components/market/useVenues"; + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +const SERVER = "prod"; + +const addCredential = vi.fn<(server: string, data: unknown) => Promise<{ added: boolean }>>(); +vi.mock("@/lib/api", () => ({ + api: { addCredential: (server: string, data: unknown) => addCredential(server, data) }, +})); + +const discoverWallets = vi.fn(); +const connectWallet = vi.fn(); +vi.mock("@/lib/wallet/evm", async () => { + const actual = await vi.importActual("@/lib/wallet/evm"); + return { + ...actual, + discoverWallets: () => discoverWallets(), + connectWallet: (...a: unknown[]) => connectWallet(...a), + }; +}); + +const connectHyperliquid = vi.fn(); +const hasHyperliquidReferrer = vi.fn(); +vi.mock("@/lib/wallet/hyperliquid", async () => { + const actual = + await vi.importActual("@/lib/wallet/hyperliquid"); + return { + ...actual, + connectHyperliquid: (...a: unknown[]) => connectHyperliquid(...a), + hasHyperliquidReferrer: (...a: unknown[]) => hasHyperliquidReferrer(...a), + }; +}); + +const { ConnectHyperliquid } = await import("./ConnectHyperliquid"); + +const CONNECTION = { + mainAddress: "0xmain", + agentAddress: "0xagent", + agentPrivateKey: "0xkey", + validUntil: Date.now() + 1000, +}; + +let container: HTMLDivElement; +let root: Root; + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + addCredential.mockReset(); + discoverWallets.mockReset().mockResolvedValue([ + { uuid: "w1", name: "Rabby", provider: {} }, + ]); + connectWallet.mockReset().mockResolvedValue("0xmain"); + connectHyperliquid.mockReset().mockResolvedValue(CONNECTION); + // No referrer yet — the "available" referral card renders and stays until + // the user clicks through it (or never does, which is exactly this bug). + hasHyperliquidReferrer.mockReset().mockResolvedValue(false); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); +}); + +async function flush() { + for (let i = 0; i < 25; i++) { + await act(async () => { + await new Promise((r) => setTimeout(r, 0)); + }); + } +} + +const isInvalidated = (qc: QueryClient, key: readonly unknown[]) => + qc.getQueryState(key as unknown[])?.isInvalidated === true; + +async function mountAndConnect(qc: QueryClient) { + await act(async () => { + root.render( + + {}} onDone={() => {}} /> + , + ); + }); + await flush(); + const button = [...container.querySelectorAll("button")].find((b) => + b.textContent?.includes("Rabby"), + ); + if (!button) throw new Error("wallet button not found"); + await act(async () => button.click()); + await flush(); +} + +function newClient() { + const qc = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }); + for (const key of [venuesQueryKey(SERVER), ["connected-exchanges", SERVER]]) { + qc.setQueryData(key, []); + } + return qc; +} + +describe("ConnectHyperliquid invalidates at the save, not at the referral dismissal", () => { + it("invalidates once both connectors save, before the referral prompt is touched", async () => { + addCredential.mockResolvedValue({ added: true }); + const qc = newClient(); + + await mountAndConnect(qc); + + expect(addCredential).toHaveBeenCalledTimes(2); + expect(isInvalidated(qc, venuesQueryKey(SERVER))).toBe(true); + expect(isInvalidated(qc, ["connected-exchanges", SERVER])).toBe(true); + // The referral card is up (no click on "Link code" / "Skip" happened) — + // invalidation ran regardless. + expect(container.textContent).toContain("Link code"); + }); + + it("still invalidates when only one of the two connectors saves", async () => { + addCredential.mockImplementation(async (_server, data: unknown) => { + const { connector_name } = data as { connector_name: string }; + if (connector_name === "hyperliquid_perpetual") throw new Error("boom"); + return { added: true }; + }); + const qc = newClient(); + + await mountAndConnect(qc); + + expect(isInvalidated(qc, venuesQueryKey(SERVER))).toBe(true); + expect(isInvalidated(qc, ["connected-exchanges", SERVER])).toBe(true); + }); + + it("does not invalidate when both connectors fail to save", async () => { + addCredential.mockRejectedValue(new Error("boom")); + const qc = newClient(); + + await mountAndConnect(qc); + + expect(isInvalidated(qc, venuesQueryKey(SERVER))).toBe(false); + expect(isInvalidated(qc, ["connected-exchanges", SERVER])).toBe(false); + }); +}); diff --git a/frontend/src/components/settings/ConnectHyperliquid.tsx b/frontend/src/components/settings/ConnectHyperliquid.tsx index e91c28a2f..d9cbf2ee8 100644 --- a/frontend/src/components/settings/ConnectHyperliquid.tsx +++ b/frontend/src/components/settings/ConnectHyperliquid.tsx @@ -1,7 +1,9 @@ +import { useQueryClient } from "@tanstack/react-query"; import { AlertCircle, ArrowLeft, Check, Loader2, Sparkles, Wallet } from "lucide-react"; import { useEffect, useState } from "react"; import { api } from "@/lib/api"; +import { invalidateCredentialQueries } from "@/lib/queryClient"; import { type DiscoveredWallet, connectWallet, discoverWallets } from "@/lib/wallet/evm"; import { AGENT_VALIDITY_DAYS, @@ -60,6 +62,7 @@ export function ConnectHyperliquid({ onBack: () => void; onDone: () => void; }) { + const qc = useQueryClient(); const [wallets, setWallets] = useState([]); const [scanning, setScanning] = useState(true); const [accountName, setAccountName] = useState(() => defaultAgentName()); @@ -143,6 +146,12 @@ export function ConnectHyperliquid({ const reason = (failed[0].result as PromiseRejectedResult).reason as { message?: string }; throw new Error(reason?.message || "Failed to save Hyperliquid credentials."); } + + // At least one connector saved — refresh the credential-derived caches now, + // at the write, rather than waiting on the "done" screen's referral prompt + // (which the user may dismiss by navigating away instead of clicking through). + invalidateCredentialQueries(qc, server); + if (failed.length > 0) { const reason = (failed[0].result as PromiseRejectedResult).reason as { message?: string }; setPartial( From b7353d60ce0a5f3c4c9b3ddacfe5175c60e08919 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 23:51:42 +0300 Subject: [PATCH 100/154] Point the /executors redirect at the running population, not terminated The terminated leaf walk skips every active executor by construction, so the old redirect showed the whole history minus the half a reader most likely wanted, with no hint the live ones were filtered rather than absent. Land on the default (live) population instead; the terminated one is one click away in the sidebar. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LYuUt3zxZCa7on47NbDxRs --- frontend/src/App.tsx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6ac80d5b0..2b1906a11 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -108,11 +108,15 @@ export default function App() { } /> {/* Executors are a scope of the browser now, not a page (FEAT-086). The listing this replaces was the whole - history, live and archived together, which is what the - Terminated population grouped by type is. */} + history, live and archived together — which is two + populations now, and a redirect can only choose one. + This one lands on the live fleet grouped by type + (omitting `?population` is the running default, per + `parsePopulation`); the terminated half is one click + away in the sidebar. */} } + element={} /> } /> } /> From a23e41b0968f2e337e6409b18322f06141714b40 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 9 Sep 2026 23:56:07 +0300 Subject: [PATCH 101/154] Give the two wrong-shape useRates mocks a real convert(), and lock it in with a money assertion ConvertFn returns { value, converted }, but these mocks returned a bare number, so quoteConverter's .value read undefined and every folded figure came out NaN. Both tests only passed because their assertions were on names, not numbers. Fix the mock shape in both files and add a Realized/ Volume money assertion to Bots.population.test.tsx's terminated case so the fix is actually exercised. --- .../src/components/agent/StrategyWorkbench.labels.test.tsx | 2 +- frontend/src/pages/Bots.population.test.tsx | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/agent/StrategyWorkbench.labels.test.tsx b/frontend/src/components/agent/StrategyWorkbench.labels.test.tsx index eb4c3b552..d6c0645d9 100644 --- a/frontend/src/components/agent/StrategyWorkbench.labels.test.tsx +++ b/frontend/src/components/agent/StrategyWorkbench.labels.test.tsx @@ -82,7 +82,7 @@ vi.mock("@/hooks/useFleetData", () => ({ runs: [], terminatedControllers: [], deeds: null, - convert: (v: number) => v, + convert: (v: number) => ({ value: v, converted: true }), currencySymbol: "$", rateFormatPnl: String, rateFormatValue: String, diff --git a/frontend/src/pages/Bots.population.test.tsx b/frontend/src/pages/Bots.population.test.tsx index 4f5b4f682..2465adf10 100644 --- a/frontend/src/pages/Bots.population.test.tsx +++ b/frontend/src/pages/Bots.population.test.tsx @@ -61,7 +61,7 @@ vi.mock("@/hooks/useWebSocket", () => ({ useCondorWebSocket: () => {} })); vi.mock("@/hooks/useRates", () => ({ useRates: () => ({ rates: {}, - convert: (v: number) => v, + convert: (v: number) => ({ value: v, converted: true }), formatValue: (v: number) => `$${v}`, formatPnlValue: (v: number) => `$${v}`, formatValueDetailed: (v: number) => `$${v}`, @@ -198,6 +198,11 @@ describe("/bots with no live fleet", () => { expect(text()).toContain("mm-sol-1"); expect(text()).toContain("grid-alpha"); expect(text()).not.toContain("No bots running"); + // Locks the fix in: with a correctly-shaped `convert` mock, the folded + // Realized ($12 → "+$12.00") and Volume ($5,000 → "$5.0K") tiles are real + // money, not `NaN`. + expect(text()).toContain("+$12.00"); + expect(text()).toContain("$5.0K"); }); it("keeps the population toggle reachable when the terminated set is empty too", async () => { From 9ea6ca2f51a93c437ca333a1a270ac6eceb6bf77 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 00:00:12 +0300 Subject: [PATCH 102/154] Hold runs/terminatedControllers at module-level empty arrays, not fresh ones useFleetData returned runs: runsData?.runs ?? [] and terminatedControllers: terminatedData?.controllers ?? []. Both queries are disabled on the default Running population, so every host render minted a new array identity there, defeating the leavesFor memo the same way EMPTY_OWNERS already guards against. PerfBrowser's own = [] parameter defaults for snapshots, executors, runs and terminatedControllers had the same latent shape, currently unreached since both mount sites pass the props. Reuse the held-constant pattern for all of them, plus bots for consistency. --- frontend/src/components/perf/PerfBrowser.tsx | 20 ++++++++++++++++---- frontend/src/hooks/useFleetData.ts | 15 ++++++++++++--- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/perf/PerfBrowser.tsx b/frontend/src/components/perf/PerfBrowser.tsx index 4ac122daf..18605227a 100644 --- a/frontend/src/components/perf/PerfBrowser.tsx +++ b/frontend/src/components/perf/PerfBrowser.tsx @@ -199,6 +199,18 @@ const FLEET_SCOPE = "all"; /** Held still, so an absent fleet map is not a new array on every render. */ const EMPTY_OWNERS: FleetOwner[] = []; +/** Held still, so an unpassed `snapshots` prop is not a new array on every render. */ +const EMPTY_SNAPSHOTS: ControllerPerformanceSnapshot[] = []; + +/** Held still, so an unpassed `executors` prop is not a new array on every render. */ +const EMPTY_EXECUTORS: ExecutorInfo[] = []; + +/** Held still, so an unpassed `runs` prop is not a new array on every render. */ +const EMPTY_RUNS: BotRunInfo[] = []; + +/** Held still, so an unpassed `terminatedControllers` prop is not a new array on every render. */ +const EMPTY_TERMINATED: ControllerInfo[] = []; + /** * The series a splitting scope does not have. * @@ -537,12 +549,12 @@ export function PerfBrowser({ server, convert, currencySymbol, - snapshots = [], + snapshots = EMPTY_SNAPSHOTS, truncated = false, - executors = [], + executors = EMPTY_EXECUTORS, paging, - runs = [], - terminatedControllers = [], + runs = EMPTY_RUNS, + terminatedControllers = EMPTY_TERMINATED, owners = EMPTY_OWNERS, deeds = null, rateFormatPnl, diff --git a/frontend/src/hooks/useFleetData.ts b/frontend/src/hooks/useFleetData.ts index 06da3965f..42557d815 100644 --- a/frontend/src/hooks/useFleetData.ts +++ b/frontend/src/hooks/useFleetData.ts @@ -43,6 +43,15 @@ const EXECUTOR_PAGES = 4; /** Held still, so a failed or pending fleet map is not a new prop every render. */ const EMPTY_OWNERS: FleetOwner[] = []; +/** Held still, so the disabled Running-population runs query is not a new array every render. */ +const EMPTY_RUNS: BotRunInfo[] = []; + +/** Held still, so the disabled Running-population terminated-controllers query is not a new array every render. */ +const EMPTY_TERMINATED: ControllerInfo[] = []; + +/** Held still, so a not-yet-loaded bot list is not a new array every render. */ +const EMPTY_BOTS: BotSummary[] = []; + /** Everything `PerfBrowser` reports on, and the state of fetching it. */ export interface FleetData { controllers: ControllerInfo[]; @@ -352,13 +361,13 @@ export function useFleetData( return { controllers: sortedControllers, - bots: data?.bots ?? [], + bots: data?.bots ?? EMPTY_BOTS, executors, paging, snapshots: activeSnapshots, truncated: perfHistory?.truncated ?? false, - runs: runsData?.runs ?? [], - terminatedControllers: terminatedData?.controllers ?? [], + runs: runsData?.runs ?? EMPTY_RUNS, + terminatedControllers: terminatedData?.controllers ?? EMPTY_TERMINATED, owners: fleet?.owners ?? EMPTY_OWNERS, deeds: fleet?.deeds ?? null, convert, From 191ad35e26185b43d5d5bb1fcbc3165bfe2863a6 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 00:06:59 +0300 Subject: [PATCH 103/154] Invalidate VENUES alongside CONNECTORS and PORTFOLIO on credential writes Adding or removing API keys never invalidated VENUES, so the trade panel kept showing "view only" for up to the 10-minute TTL after a key was added, or kept offering a venue whose key was just removed. Declare CREDENTIAL_DERIVED once in server_data_service.py and spread it at both settings.py writers instead of hand-listing the affected types per call site. Also move add_credential's invalidate outside the try/except so a failure there can no longer be swallowed and misreported as a failed credential add. --- condor/server_data_service.py | 15 +++++++++++++ condor/web/routes/settings.py | 20 ++++++++++------- tests/test_web_credentials_owner_only.py | 28 ++++++++++++++++++++++-- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/condor/server_data_service.py b/condor/server_data_service.py index 34d304f6e..cf30988f7 100644 --- a/condor/server_data_service.py +++ b/condor/server_data_service.py @@ -142,6 +142,21 @@ def interval_for(self, params: Dict[str, str]) -> float: } +#: Everything computed from an account's credential list. Adding or removing a +#: key changes all of it at once: CONNECTORS *is* the credentialed list, VENUES +#: derives its `credentialed` trait from that same list (fetch_venues calls +#: fetch_available_cex_connectors, condor/fetchers/connectors.py:173), and +#: PORTFOLIO is the balances of those accounts. VENUES is the one nothing +#: re-polls — it is not in auto_subscribe_servers' core_types — so forgetting it +#: here strands the trade panel behind a stale view-only overlay for a full +#: 600s TTL, in every browser (issue #238). +CREDENTIAL_DERIVED: Tuple[ServerDataType, ...] = ( + ServerDataType.CONNECTORS, + ServerDataType.VENUES, + ServerDataType.PORTFOLIO, +) + + # ============================================ # CACHE KEY & ENTRY # ============================================ diff --git a/condor/web/routes/settings.py b/condor/web/routes/settings.py index d61b48574..e10b2d60e 100644 --- a/condor/web/routes/settings.py +++ b/condor/web/routes/settings.py @@ -6,7 +6,11 @@ from fastapi import APIRouter, Depends, HTTPException, Query -from condor.server_data_service import ServerDataType, get_server_data_service +from condor.server_data_service import ( + CREDENTIAL_DERIVED, + ServerDataType, + get_server_data_service, +) from condor.web.auth import ( get_current_user, require_owner, @@ -690,14 +694,15 @@ async def add_credential( connector_name=req.connector_name, credentials=req.credentials, ) - # Invalidate configured connectors cache - get_server_data_service().invalidate(server, ServerDataType.CONNECTORS) - return {"added": True, "result": result} except Exception as e: logger.exception( "Failed to add credentials for '%s' on '%s'", req.connector_name, server ) raise upstream_error("Failed to add credentials", e) + # Outside the try: the credential write already succeeded, and an + # exception here must not be swallowed and re-reported as a failed add. + get_server_data_service().invalidate(server, *CREDENTIAL_DERIVED) + return {"added": True, "result": result} @router.delete("/credentials/{connector}") @@ -715,10 +720,9 @@ async def delete_credential( account_name="master_account", connector_name=connector, ) - # Invalidate configured connectors + portfolio caches so the removed key disappears immediately - sds = get_server_data_service() - sds.invalidate(server, ServerDataType.CONNECTORS) - sds.invalidate(server, ServerDataType.PORTFOLIO) + # Invalidate every credential-derived cache so the removed key + # disappears immediately (including VENUES' `credentialed` trait). + get_server_data_service().invalidate(server, *CREDENTIAL_DERIVED) return {"deleted": True, "result": result} except Exception as e: logger.exception( diff --git a/tests/test_web_credentials_owner_only.py b/tests/test_web_credentials_owner_only.py index ec4240483..5bbb71c46 100644 --- a/tests/test_web_credentials_owner_only.py +++ b/tests/test_web_credentials_owner_only.py @@ -77,8 +77,9 @@ class FakeSDS: def __init__(self): self.invalidated = [] - def invalidate(self, server, data_type): - self.invalidated.append((server, data_type)) + def invalidate(self, server, *data_types): + for data_type in data_types: + self.invalidated.append((server, data_type)) @pytest.fixture @@ -124,6 +125,29 @@ def test_owner_can_add_and_delete_credentials(env): assert (SERVER, ServerDataType.PORTFOLIO) in sds.invalidated +def test_add_and_delete_invalidate_the_same_credential_derived_set(env): + """CORR-613: VENUES' `credentialed` trait derives from the same credential + list as CONNECTORS, but nothing re-polls it — the writers must invalidate + it explicitly, on both add and delete, or the trade panel serves a stale + view-only overlay for up to 600s (issue #238).""" + app, client, sds = env + expected = { + ServerDataType.CONNECTORS, + ServerDataType.VENUES, + ServerDataType.PORTFOLIO, + } + + assert post_credential(as_user(app, OWNER)).status_code == 200 + added_types = {dt for (srv, dt) in sds.invalidated if srv == SERVER} + assert added_types == expected + + sds.invalidated.clear() + + assert delete_credential(as_user(app, OWNER)).status_code == 200 + deleted_types = {dt for (srv, dt) in sds.invalidated if srv == SERVER} + assert deleted_types == expected + + def test_shared_trader_cannot_add_or_delete_credentials(env): app, client, _ = env http = as_user(app, TRADER) From 92b81353d718aa2c07270cba34910265703cf4fd Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 00:17:16 +0300 Subject: [PATCH 104/154] Invalidate VENUES on the gateway stop and observed-running transitions, not the start/restart 200 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stopping Gateway kept offering DEX venues as tradable and starting it kept them missing from the trade and LP panels for up to the 10-minute VENUES TTL, since none of the three lifecycle routes touched the cache. gateway_stop now invalidates VENUES right after a successful stop (outside the try/except, so an invalidation error is never misreported as a failed stop) since container.stop() blocks until the new answer is already true. gateway_start/gateway_restart deliberately invalidate nothing on their 200 — they return right after a detached container run with no readiness wait, so invalidating there would refetch mid-boot and cache a gateway-less venue list for a fresh 600s. Instead gateway_status tracks the last observed running state per server and invalidates VENUES the moment it sees the false -> true transition. The frontend's own ["venues", server] query (staleTime 5m) is a separate cache the backend invalidation cannot reach, so GatewaySettings.tsx now invalidates it on the same transition its existing 10s status poll observes. --- condor/web/routes/settings.py | 25 +- .../components/settings/GatewaySettings.tsx | 17 +- tests/test_web_gateway_venues_invalidation.py | 216 ++++++++++++++++++ 3 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 tests/test_web_gateway_venues_invalidation.py diff --git a/condor/web/routes/settings.py b/condor/web/routes/settings.py index e10b2d60e..11a167716 100644 --- a/condor/web/routes/settings.py +++ b/condor/web/routes/settings.py @@ -144,6 +144,14 @@ async def set_default_server(name: str, user: WebUser = Depends(get_current_user # ── Gateway ── +#: Last `running` state observed per server, so `gateway_status` can detect the +#: false → true transition — the only moment Gateway can actually answer +#: `list_networks()`, which decides VENUES' `clmm_lp` trait and half of its +#: `credentialed` trait (condor/fetchers/connectors.py:47, CORR-614). Invalidating +#: on `gateway_start`/`gateway_restart`'s 200 instead would refetch mid-boot and +#: cache a gateway-less answer for a fresh 600s. +_gateway_was_running: dict[str, bool] = {} + @router.get("/gateway/status") async def gateway_status( @@ -162,8 +170,12 @@ async def gateway_status( result["image"] = info.get("image", None) result["created_at"] = info.get("created", info.get("created_at", None)) result["container_status"] = info.get("status", None) + if is_running and not _gateway_was_running.get(server, False): + get_server_data_service().invalidate(server, ServerDataType.VENUES) + _gateway_was_running[server] = is_running return result except Exception: + _gateway_was_running[server] = False return {"running": False, "info": None} @@ -216,6 +228,11 @@ async def gateway_start( # host. That is the owner's decision, not a shared trader's. _require_owner(cm, user.id, server) client = await _get_client(cm, server) + # Deliberately does not invalidate VENUES here: this returns right after a + # detached `containers.run` with no readiness wait, so a refetch triggered + # now lands mid-boot, `list_networks` 503s, and `fetch_venues` would cache + # a gateway-less list for a fresh 600s. `gateway_status` invalidates once + # Gateway is actually observed running (CORR-614). try: result = await client.gateway.start( { @@ -241,10 +258,14 @@ async def gateway_stop( client = await _get_client(cm, server) try: result = await client.gateway.stop() - return {"stopped": True, "result": result} except Exception as e: logger.exception("Failed to stop gateway on '%s'", server) raise upstream_error("Failed to stop gateway", e) + # Outside the try: the stop already succeeded — `container.stop()` blocks, + # so the new answer (no venues) is already true — and an invalidation + # error here must not be reported as a failed stop (CORR-614). + get_server_data_service().invalidate(server, ServerDataType.VENUES) + return {"stopped": True, "result": result} @router.post("/gateway/restart") @@ -256,6 +277,8 @@ async def gateway_restart( # A restart drops in-flight DEX orders for everyone on the server. _require_owner(cm, user.id, server) client = await _get_client(cm, server) + # Same reasoning as gateway_start: no invalidation here, `gateway_status` + # catches the real transition once the container is back up (CORR-614). try: result = await client.gateway.restart() return {"restarted": True, "result": result} diff --git a/frontend/src/components/settings/GatewaySettings.tsx b/frontend/src/components/settings/GatewaySettings.tsx index 7a42c1c70..7d6f2bdec 100644 --- a/frontend/src/components/settings/GatewaySettings.tsx +++ b/frontend/src/components/settings/GatewaySettings.tsx @@ -10,8 +10,9 @@ import { RefreshCw, Square, } from "lucide-react"; -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; +import { venuesQueryKey } from "@/components/market/useVenues"; import { useServer } from "@/hooks/useServer"; import { OWNER_ONLY_HINT, useServerPermission } from "@/hooks/useServerPermission"; import { api } from "@/lib/api"; @@ -338,6 +339,20 @@ export function GatewaySettings() { refetchInterval: 10000, }); + // The server invalidates its own VENUES cache the moment Gateway is first + // observed running again (CORR-614), but this browser's own `["venues", + // server]` query (staleTime 5m) is a separate cache that nothing else + // refetches — invalidate it on the same false → true transition this + // 10s poll already observes, or the Trade/LP panels keep the DEX venues + // missing for up to 5 more minutes after Gateway comes back up. + const wasRunningRef = useRef(undefined); + useEffect(() => { + if (status?.running && wasRunningRef.current === false) { + qc.invalidateQueries({ queryKey: venuesQueryKey(server) }); + } + wasRunningRef.current = status?.running; + }, [status?.running, server, qc]); + const { data: logsData, isFetching: fetchingLogs } = useQuery({ queryKey: ["gateway-logs", server], queryFn: () => api.getGatewayLogs(server!), diff --git a/tests/test_web_gateway_venues_invalidation.py b/tests/test_web_gateway_venues_invalidation.py new file mode 100644 index 000000000..dceac1b97 --- /dev/null +++ b/tests/test_web_gateway_venues_invalidation.py @@ -0,0 +1,216 @@ +"""Gateway lifecycle must invalidate VENUES only when the answer can actually +change (CORR-614). + +`_chartable_gateway_networks` (condor/fetchers/connectors.py:47) calls +`client.gateway.list_networks()`, and its result decides each venue's +`clmm_lp` trait and half of its `credentialed` trait. VENUES has no +subscriber (server_data_service.py) so its 600s TTL is the only thing that +ever clears it on its own — the lifecycle routes must invalidate it +explicitly, but only at the moment Gateway can truthfully answer: + +- `gateway_stop` blocks until the container is down, so the new (gateway-less) + answer is already true by the time the route returns — invalidate right away, + and outside the try/except so an invalidation bug is never misreported as a + failed stop. +- `gateway_start`/`gateway_restart` return right after a detached + `containers.run` with no readiness wait — invalidating on their 200 would + refetch mid-boot and cache a gateway-less list for a fresh 600s. They must + invalidate nothing; `gateway_status` catches the real transition instead. +""" + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +import condor.web.routes.settings as settings_routes +from condor.server_data_service import ServerDataType +from condor.web.auth import get_current_user +from condor.web.models import WebUser +from config_manager import ServerPermission + +SERVER = "alpha" +OWNER = WebUser(id=1, username="owner", first_name="O", role="user") +TRADER = WebUser(id=2, username="trader", first_name="T", role="user") + + +class FakeGateway: + def __init__(self): + self.calls = [] + self.status_sequence = [] + + async def start(self, cfg): + self.calls.append("start") + return {"ok": True} + + async def stop(self): + self.calls.append("stop") + return {"ok": True} + + async def restart(self): + self.calls.append("restart") + return {"ok": True} + + async def get_status(self): + self.calls.append("status") + return self.status_sequence.pop(0) + + +class FakeClient: + def __init__(self): + self.gateway = FakeGateway() + + +class FakeConfigManager: + def __init__(self, client): + self._client = client + + def get_server_permission(self, user_id, server_name): + if server_name != SERVER: + return None + if user_id == OWNER.id: + return ServerPermission.OWNER + if user_id == TRADER.id: + return ServerPermission.TRADER + return None + + def has_server_access(self, user_id, server_name, min_permission=None): + return self.get_server_permission(user_id, server_name) is not None + + def is_admin(self, user_id): + return False + + async def get_client(self, server_name): + return self._client + + +class FakeSDS: + def __init__(self): + self.invalidated = [] + + def invalidate(self, server, *data_types): + for data_type in data_types: + self.invalidated.append((server, data_type)) + + +class ExplodingSDS: + """Raises on invalidate — used to pin that the call sits outside the + try/except around the upstream call.""" + + def invalidate(self, server, *data_types): + raise RuntimeError("cache backend unavailable") + + +@pytest.fixture +def env(monkeypatch): + client = FakeClient() + cm = FakeConfigManager(client) + sds = FakeSDS() + monkeypatch.setattr(settings_routes, "get_config_manager", lambda: cm) + monkeypatch.setattr("condor.web.auth.get_config_manager", lambda: cm) + monkeypatch.setattr( + "condor.web.routes.settings.get_server_data_service", lambda: sds + ) + # Module-level transition tracker — reset so tests don't bleed into each other. + settings_routes._gateway_was_running.clear() + app = FastAPI() + app.include_router(settings_routes.router) + return app, client, sds + + +def as_user(app, user): + app.dependency_overrides[get_current_user] = lambda: user + return TestClient(app) + + +P = {"server": SERVER} + + +def test_stop_invalidates_venues(env): + app, client, sds = env + resp = as_user(app, OWNER).post("/settings/gateway/stop", params=P) + assert resp.status_code == 200 + assert client.gateway.calls == ["stop"] + assert sds.invalidated == [(SERVER, ServerDataType.VENUES)] + + +def test_stop_invalidation_sits_outside_the_try_except(env): + """An invalidation failure must not be reported as a failed gateway stop — + the stop itself already succeeded by the time invalidate() runs.""" + app, client, _ = env + app.dependency_overrides[get_current_user] = lambda: OWNER + http = TestClient(app, raise_server_exceptions=False) + + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "condor.web.routes.settings.get_server_data_service", + lambda: ExplodingSDS(), + ) + resp = http.post("/settings/gateway/stop", params=P) + + # The backend call happened and is not what failed; if invalidate() were + # inside the try/except this would come back as a 502 with detail + # "Failed to stop gateway: ...". + assert client.gateway.calls == ["stop"] + assert resp.status_code == 500 + assert "Failed to stop gateway" not in resp.text + + +def test_start_does_not_invalidate_venues_on_the_200(env): + """Starting returns before Gateway is actually up — invalidating here would + make fetch_venues cache a gateway-less answer for a fresh 600s.""" + app, client, sds = env + resp = as_user(app, OWNER).post( + "/settings/gateway/start", + params=P, + json={"image": "hummingbot/gateway:latest", "port": 15888}, + ) + assert resp.status_code == 200 + assert client.gateway.calls == ["start"] + assert sds.invalidated == [] + + +def test_restart_does_not_invalidate_venues_on_the_200(env): + app, client, sds = env + resp = as_user(app, OWNER).post("/settings/gateway/restart", params=P) + assert resp.status_code == 200 + assert client.gateway.calls == ["restart"] + assert sds.invalidated == [] + + +def test_status_invalidates_venues_only_on_the_false_to_true_transition(env): + app, client, sds = env + client.gateway.status_sequence = [ + {"running": False}, # down: no transition, no invalidation + {"running": False}, # still down + {"running": True}, # <-- the transition: invalidate + {"running": True}, # already known running: no repeat invalidation + ] + http = as_user(app, TRADER) # reads stay at TRADER + + assert http.get("/settings/gateway/status", params=P).json()["running"] is False + assert sds.invalidated == [] + + assert http.get("/settings/gateway/status", params=P).json()["running"] is False + assert sds.invalidated == [] + + assert http.get("/settings/gateway/status", params=P).json()["running"] is True + assert sds.invalidated == [(SERVER, ServerDataType.VENUES)] + + assert http.get("/settings/gateway/status", params=P).json()["running"] is True + assert sds.invalidated == [(SERVER, ServerDataType.VENUES)] # unchanged + + +def test_status_error_is_treated_as_not_running_for_the_transition(env): + """A transient status-fetch failure must not get skipped over as if the + gateway were still up, or the next real transition would be missed.""" + app, client, sds = env + settings_routes._gateway_was_running[SERVER] = True + http = as_user(app, TRADER) + + async def raising_get_status(): + raise RuntimeError("upstream unreachable") + + client.gateway.get_status = raising_get_status + resp = http.get("/settings/gateway/status", params=P) + assert resp.json() == {"running": False, "info": None} + assert settings_routes._gateway_was_running[SERVER] is False From c4722332afc791c0227514ce82bb52cd73426c15 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 00:25:42 +0300 Subject: [PATCH 105/154] Flush the SDS data cache on server edit/delete, not just the client and status memo _invalidate_server_caches was the shared chokepoint for modify_server and delete_server but only cleared the pooled client, status memo, and failure cooldown. The SDS cache is keyed by server name, not host, so repointing a server at a new host/port kept serving the old host's portfolio/venues/ connectors/bots data under the same name until each type's TTL expired on its own. Extend the chokepoint to also invalidate the SDS cache for that name, wrapped so a cache-layer failure can never fail a config write. Also drops the now-redundant SDS invalidation in the Telegram delete path, since ConfigManager.delete_server covers it directly. --- config_manager.py | 17 +++++++++++++- handlers/config/servers.py | 9 ++------ tests/test_server_status_probe.py | 38 +++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/config_manager.py b/config_manager.py index 2f7b4751b..a871591d0 100644 --- a/config_manager.py +++ b/config_manager.py @@ -485,15 +485,30 @@ def get_or_create_web_jwt_secret(self) -> str: return secret def _invalidate_server_caches(self, name: str): - """Drop the pooled client and memoized status for a server. + """Drop every cached artefact of a server: the pooled client, the memoized + status, and the whole SDS data cache for that name. Called whenever the server's credentials or existence change, so a cached client or status can never outlive the config it was built from. + + The data cache is keyed by server *name* (CacheKey, server_data_service.py), + never by host, so repointing a server at a different host/port would + otherwise keep serving the previous host's answers under the same name + until each type's TTL expires on its own. """ self._clients.pop(name, None) self._status_cache.pop(name, None) # New credentials deserve a real attempt, not the old failure's cooldown. self._client_failures.pop(name, None) + try: + # Lazy, mirroring the direction SDS already imports this module in + # (server_data_service.py) — no cycle, and a cache layer must never + # be able to fail a config write. + from condor.server_data_service import get_server_data_service + + get_server_data_service().invalidate_server(name) + except Exception: + logger.debug("SDS invalidation skipped for '%s'", name, exc_info=True) async def get_client(self, name: str = None): """Get or create API client for a server.""" diff --git a/handlers/config/servers.py b/handlers/config/servers.py index 9a5a92514..9b0b4168c 100644 --- a/handlers/config/servers.py +++ b/handlers/config/servers.py @@ -523,13 +523,8 @@ async def delete_server( # Invalidate cache if we deleted the server that was in use if was_current: invalidate_cache(context.user_data, "all") - # Also invalidate SDS (server-scoped) - try: - from condor.server_data_service import get_server_data_service - - get_server_data_service().invalidate_server(server_name) - except Exception: - pass + # SDS is invalidated by ConfigManager.delete_server itself now + # (config_manager.py _invalidate_server_caches). logger.info( f"Cache invalidated after deleting current server '{server_name}'" ) diff --git a/tests/test_server_status_probe.py b/tests/test_server_status_probe.py index 5ef18cc5b..305a06b75 100644 --- a/tests/test_server_status_probe.py +++ b/tests/test_server_status_probe.py @@ -14,7 +14,9 @@ import pytest +import condor.server_data_service as sds_module import config_manager as cm_module +from condor.server_data_service import CacheKey, ServerDataService, ServerDataType from config_manager import ConfigManager @@ -81,6 +83,21 @@ def factory(monkeypatch): return fake +@pytest.fixture +def sds(monkeypatch): + """An isolated ServerDataService, wired in place of the process singleton.""" + instance = ServerDataService() + monkeypatch.setattr(sds_module, "get_server_data_service", lambda: instance) + return instance + + +def _seed_sds(sds: ServerDataService, server: str): + key = CacheKey.make(server, ServerDataType.PORTFOLIO) + sds._cache[key] = sds_module.CacheEntry( + key=key, value={"USD": 1}, fetched_at=time.time() + ) + + def _pool(cm: ConfigManager, name: str, client: FakeClient): cm._clients[name] = (client, time.time()) @@ -254,6 +271,27 @@ async def test_deleting_a_server_clears_its_memo(cm, factory): assert "prod" not in cm._clients +def test_modifying_a_server_clears_its_sds_cache(cm, sds): + """Repointing a server at a new host must not keep serving the old one's + cached SDS data (portfolio, venues, connectors, ...) under the same name.""" + _seed_sds(sds, "prod") + assert any(k.server == "prod" for k in sds._cache) + + cm.modify_server("prod", host="new-host") + + assert not any(k.server == "prod" for k in sds._cache) + + +def test_deleting_a_server_clears_its_sds_cache(cm, sds): + """A deleted server leaves no SDS entries behind for a later namesake.""" + _seed_sds(sds, "prod") + assert any(k.server == "prod" for k in sds._cache) + + cm.delete_server("prod") + + assert not any(k.server == "prod" for k in sds._cache) + + @pytest.mark.asyncio async def test_expired_pooled_client_is_not_probed(cm, factory): """A pooled entry past _client_ttl is dead weight: probe a fresh client.""" From 7863b0033bf0b5da1bad77ba236d23d273cfcdd2 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 00:34:06 +0300 Subject: [PATCH 106/154] Refuse start_time/end_time on clmm_positions instead of silently dropping them gateway_clmm.search_positions has no time-window parameter at any layer, so search_history("clmm_positions", start_time=..., end_time=...) was returning the unfiltered newest-50 positions while looking like a time-windowed history. Extends the CORR-563/CORR-569 refusal pattern already used for perp_positions and orders' offset/cursor mismatches. --- mcp_servers/hummingbot_api/tools/history.py | 9 ++- tests/test_mcp_search_history_filters.py | 78 ++++++++++++++++++--- 2 files changed, 77 insertions(+), 10 deletions(-) diff --git a/mcp_servers/hummingbot_api/tools/history.py b/mcp_servers/hummingbot_api/tools/history.py index 9d4bf98a4..275423c30 100644 --- a/mcp_servers/hummingbot_api/tools/history.py +++ b/mcp_servers/hummingbot_api/tools/history.py @@ -41,7 +41,10 @@ "offset", "cursor", ), - "clmm_positions": ("cursor",), + # gateway_clmm.search_positions has no time-window parameter at any layer + # (CORR-618): forwarding start_time/end_time would silently return the + # unfiltered newest-50 positions while looking like a time-windowed history. + "clmm_positions": ("cursor", "start_time", "end_time"), } _FILTER_ALTERNATIVES: dict[str, str] = { @@ -56,7 +59,9 @@ ), "clmm_positions": ( "clmm_positions is offset-paginated: use offset= (the value printed at " - "the end of the previous page) rather than a cursor." + "the end of the previous page) rather than a cursor. The CLMM position " + "store has no time-window filter either: read created_at/closed_at on " + 'the returned rows, or use data_type="orders" for time-windowed history.' ), } diff --git a/tests/test_mcp_search_history_filters.py b/tests/test_mcp_search_history_filters.py index a27cab8e2..5cf1bdb12 100644 --- a/tests/test_mcp_search_history_filters.py +++ b/tests/test_mcp_search_history_filters.py @@ -51,9 +51,21 @@ async def search_orders(self, **kwargs): return {"data": [], "pagination": {"has_more": False}} +class RecordingGatewayClmm: + """Records every outgoing gateway_clmm.search_positions call.""" + + def __init__(self): + self.search_calls = [] + + async def search_positions(self, **kwargs): + self.search_calls.append(kwargs) + return {"data": []} + + class RecordingClient: def __init__(self): self.trading = RecordingTrading() + self.gateway_clmm = RecordingGatewayClmm() @pytest.fixture @@ -68,24 +80,45 @@ async def fake_get_client(): return client.trading +@pytest.fixture +def clmm_calls(monkeypatch): + """Drive the server-level tool against a recording client, gateway_clmm side.""" + client = RecordingClient() + + async def fake_get_client(): + return client + + monkeypatch.setattr(hb_server.hummingbot_client, "get_client", fake_get_client) + return client.gateway_clmm + + @pytest.mark.parametrize( - "filters, expected_names", + "data_type, filters, expected_names", [ - ({"status": "CLOSED"}, ["status"]), + ("perp_positions", {"status": "CLOSED"}, ["status"]), + ( + "perp_positions", + {"start_time": 1757000000, "end_time": 1757600000}, + ["start_time", "end_time"], + ), + ("perp_positions", {"trading_pairs": ["SOL-USDT"]}, ["trading_pairs"]), + ("perp_positions", {"offset": 50}, ["offset"]), + # CORR-618: gateway_clmm.search_positions has no time-window parameter at + # any layer, so clmm_positions must refuse start_time/end_time rather than + # silently returning the unfiltered newest-50 positions. ( + "clmm_positions", {"start_time": 1757000000, "end_time": 1757600000}, ["start_time", "end_time"], ), - ({"trading_pairs": ["SOL-USDT"]}, ["trading_pairs"]), - ({"offset": 50}, ["offset"]), ], ) -def test_perp_positions_refuses_filters_it_cannot_honour( - client_calls, filters, expected_names +def test_positions_branches_refuse_filters_they_cannot_honour( + client_calls, data_type, filters, expected_names ): """The tool raises naming the parameter, and no request is sent.""" with pytest.raises(ToolError) as excinfo: - asyncio.run(hb_server.search_history(data_type="perp_positions", **filters)) + asyncio.run(hb_server.search_history(data_type=data_type, **filters)) message = str(excinfo.value) for name in expected_names: @@ -95,7 +128,7 @@ def test_perp_positions_refuses_filters_it_cannot_honour( # into "Failed to search history: ...". assert "silently ignored" in message - # Acceptance criterion: the perp branch never reaches the positions endpoint. + # Acceptance criterion: neither branch reaches its backend endpoint. assert client_calls.position_calls == [] @@ -121,6 +154,35 @@ def test_perp_positions_still_works_with_supported_filters(client_calls): assert "current open book" in output +def test_clmm_positions_still_reaches_search_positions_with_supported_filters( + clmm_calls, +): + """CORR-618: refusing start_time/end_time must not touch the supported filters.""" + output = asyncio.run( + hb_server.search_history( + data_type="clmm_positions", + network="mainnet-beta", + connector_names=["raydium"], + trading_pairs=["SOL-USDC"], + status="OPEN", + offset=10, + ) + ) + + assert clmm_calls.search_calls == [ + { + "limit": 50, + "offset": 10, + "refresh": False, + "network": "mainnet-beta", + "connector": "raydium", + "trading_pair": "SOL-USDC", + "status": "OPEN", + } + ] + assert "No CLMM positions found" in output + + def test_orders_branch_still_forwards_every_filter(client_calls): """The guard is perp-only: orders genuinely sends its filters to the backend.""" asyncio.run( From 40b5e19f872c4fcea63a25d02c3d2c06c19a634d Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 00:41:36 +0300 Subject: [PATCH 107/154] Clear stopping marks for the failed ids on a partial stop_controllers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit manage_bot_execution/_set_kill_switches returns 200 with a non-empty "failed" map whenever at least one controller's kill-switch write succeeded — it only raises when nothing did. That success path never ran the except-branch cleanup, so a partly failed stop left the failed controller's mark in _stopping_controllers, and overlay_stopping_state only clears on manual_kill_switch=True, which a failed write never sets. The operator lost the retry button for the full 300s TTL. Clear the mark for every id in result["failed"] on the success path, leaving succeeded ids marked stopping as before. Also move cm.get_client inside the try in both stop_bot_endpoint and stop_controllers_endpoint so a client-resolution failure can't leak a mark either. --- condor/web/routes/bots.py | 14 +++-- ...t_stop_controllers_failure_clears_marks.py | 62 ++++++++++++++++++- 2 files changed, 69 insertions(+), 7 deletions(-) diff --git a/condor/web/routes/bots.py b/condor/web/routes/bots.py index 0773a8ae1..8b9ff5ca0 100644 --- a/condor/web/routes/bots.py +++ b/condor/web/routes/bots.py @@ -737,11 +737,10 @@ async def stop_bot_endpoint( # Mark as stopping immediately so UI reflects it mark_bot_stopping(name, bot_name) - client = await cm.get_client(name) - from mcp_servers.hummingbot_api.tools.bot_management import manage_bot_execution try: + client = await cm.get_client(name) result = await manage_bot_execution( client=client, bot_name=bot_name, @@ -770,11 +769,10 @@ async def stop_controllers_endpoint( # Mark controllers as stopping immediately mark_controllers_stopping(name, bot_name, body.controller_names) - client = await cm.get_client(name) - from mcp_servers.hummingbot_api.tools.bot_management import manage_bot_execution try: + client = await cm.get_client(name) result = await manage_bot_execution( client=client, bot_name=bot_name, @@ -792,6 +790,14 @@ async def stop_controllers_endpoint( ) raise upstream_error("Failed to stop controllers", e) + # manage_bot_execution can return 200 with a partial failure (some + # controllers' config writes rejected) — it only raises when *nothing* + # succeeded. Those failed ids never flip manual_kill_switch, so without + # this the overlay would keep painting them "stopping" for the full TTL, + # locking the retry control out from under the operator. + for controller_id in result.get("failed", {}): + clear_controller_stopping(name, bot_name, controller_id) + record_ui_deed( user, verb="manage_bots:stop_controllers", diff --git a/tests/test_stop_controllers_failure_clears_marks.py b/tests/test_stop_controllers_failure_clears_marks.py index 14366a9d9..e2de40286 100644 --- a/tests/test_stop_controllers_failure_clears_marks.py +++ b/tests/test_stop_controllers_failure_clears_marks.py @@ -38,7 +38,9 @@ class _Controllers: """The controllers sub-API ``_set_kill_switches`` actually drives.""" - def __init__(self, update_error: Exception | None): + def __init__(self, update_error: Exception | dict[str, Exception] | None): + # A plain Exception fails every controller (total failure); a dict + # fails only the controllers named as keys (partial failure). self._update_error = update_error self.updated: list[str] = [] @@ -49,8 +51,11 @@ async def get_bot_controller_configs(self, _bot_name): ] async def update_bot_controller_config(self, _bot_name, config_name, _update): - if self._update_error is not None: - raise self._update_error + error = self._update_error + if isinstance(error, dict): + error = error.get(config_name) + if error is not None: + raise error self.updated.append(config_name) return {"updated": True} @@ -187,3 +192,54 @@ def test_the_bot_and_controller_failure_paths_agree(bind_client): with pytest.raises(HTTPException): _stop_controllers() assert get_stopping_controllers(SERVER) == set() + + +def test_a_partial_failure_clears_only_the_failed_marks(bind_client): + """CORR-619: manage_bot_execution returns 200 with a non-empty ``failed`` + when at least one controller succeeded — the ``except`` branch never + runs, so the success-path cleanup must clear the failed ids itself while + leaving the succeeded ones marked stopping. + """ + failing, ok = CONTROLLERS[0], CONTROLLERS[1] + client = bind_client(_FakeClient({failing: RuntimeError("backend rejected")})) + + result = _stop_controllers() + + assert result["succeeded"] == [ok] + assert result["failed"] == {failing: "backend rejected"} + + stopping = get_stopping_controllers(SERVER) + assert stopping == {f"{BOT}:{ok}"}, ( + "the succeeded controller must stay marked stopping and the failed " + f"one must not; got {stopping}" + ) + assert client.controllers.updated == [ok] + + +def test_a_client_resolution_failure_leaves_no_marks(monkeypatch): + """cm.get_client failing (outside the upstream call) must not leak marks + for either endpoint — it now happens inside the ``try`` block. + """ + + class _FailingCM: + def has_server_access(self, *_args, **_kwargs): + return True + + async def get_client(self, _name): + raise RuntimeError("server unreachable") + + monkeypatch.setattr(bots_module, "get_config_manager", lambda: _FailingCM()) + + bots_module.clear_bot_stopping(SERVER, BOT) + for cid in CONTROLLERS: + bots_module.clear_controller_stopping(SERVER, BOT, cid) + + with pytest.raises(HTTPException): + asyncio.run( + bots_module.stop_bot_endpoint(name=SERVER, bot_name=BOT, user=_USER) + ) + assert get_stopping_bots(SERVER) == set() + + with pytest.raises(HTTPException): + _stop_controllers() + assert get_stopping_controllers(SERVER) == set() From 958318e64c8bd6e4a067a161586d261c65b2fef0 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 01:06:59 +0300 Subject: [PATCH 108/154] Stop racing the 0.5s stderr settle window in the ACP dead-child test _drain_stderr and the read loop's EOF detection are two tasks reading two independent pipes of the same dying child, created together and raced by start(); under heavy load the drain can lose, leaving the stderr marker (and, separately, the command name) out of the exception message even though nothing is broken. Split the stderr-content assertion into its own deterministic test that feeds the drain via a StreamReader first -- like the existing tail-bounding test already does -- so it exercises the same _read_loop/_stderr_detail message-building code with no timing dependency at all, and drop the command-name assertion from the real-subprocess test since production only guarantees it on one of two legitimate exception shapes for a child that dies this fast. --- tests/runtime/test_acp_stderr_in_errors.py | 46 ++++++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/tests/runtime/test_acp_stderr_in_errors.py b/tests/runtime/test_acp_stderr_in_errors.py index afb916360..cb94d1791 100644 --- a/tests/runtime/test_acp_stderr_in_errors.py +++ b/tests/runtime/test_acp_stderr_in_errors.py @@ -22,12 +22,50 @@ async def test_a_child_that_dies_talking_to_stderr_says_so_in_the_error(): client = ACPClient(command=f"echo '{_MARKER}' >&2; exit 127") - with pytest.raises(ConnectionError) as excinfo: + # The type stays the one CORR-329 established (ConnectionResetError from a + # stdin write racing the child's own exit still qualifies -- it subclasses + # ConnectionError). The exact *message* -- whether it carries the stderr + # tail, whether it carries the command -- depends on which of several + # concurrent tasks reading/writing this dying child's pipes asyncio + # happens to schedule first, so it is not asserted on here; the + # deterministic test below exercises that same message-building code + # without racing anything (CORR-621). + with pytest.raises(ConnectionError): await asyncio.wait_for(client.start(), timeout=30) - # The type stays the one CORR-329 established; only the message grew. - assert _MARKER in str(excinfo.value) - assert client.command in str(excinfo.value) + +@pytest.mark.asyncio +async def test_the_dead_childs_stderr_tail_reaches_the_disconnect_error(): + """Same message-building code as above, minus the settle-window race. + + ``_read_loop``'s EOF handling and ``_drain_stderr`` are created together in + :meth:`ACPClient.start` and race each other against two independent pipes + of the same dying child, so under heavy load the drain can lose and + ``test_a_child_that_dies_talking_to_stderr_says_so_in_the_error`` would red + a suite in which nothing is broken (CORR-621). Make the assertion + independent of that race instead of widening ``_STDERR_SETTLE_TIMEOUT``: + feed the drain deterministically, like + ``test_the_kept_tail_is_bounded_in_lines_and_in_width`` already does, then + run the exact same disconnect path ``start()`` runs on a dead child. + """ + client = ACPClient(command="true") + stdout = asyncio.StreamReader() + stdout.feed_eof() + stderr = asyncio.StreamReader() + stderr.feed_data(f"{_MARKER}\n".encode()) + stderr.feed_eof() + client._process = type("_P", (), {"stdout": stdout, "stderr": stderr})() # type: ignore[assignment] + + # Drained to completion -- and to a done task -- before the read loop ever + # looks at it, so _stderr_detail's settle wait is never even reached, let + # alone raced. + client._stderr_task = asyncio.create_task(client._drain_stderr()) + await client._stderr_task + await client._read_loop() + + assert client._peer._failure is not None + assert _MARKER in str(client._peer._failure) + assert client.command in str(client._peer._failure) @pytest.mark.asyncio From a6563b502934df32883cf40b7271925f832b7584 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 01:15:19 +0300 Subject: [PATCH 109/154] Let the routines menu reuse the mtime cache instead of re-importing every module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every /routines command and every "back to menu" callback went through _show_menu with force_reload=True, which blanked the discovery caches and re-executed all routine modules — ~6ms per render on a 14-routine install, and a latent hazard for any routine that ever keeps module-level state. Discovery is mtime-keyed, so a plain discover_routines() already picks up new, edited and deleted routine files on the next render. force_reload stays on the explicit Reload button, whose job is retrying a cached load *failure*: a routine whose mtime is unchanged but whose broken dependency was since fixed. --- handlers/routines/__init__.py | 5 +- tests/test_routine_discovery_cache.py | 71 +++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/handlers/routines/__init__.py b/handlers/routines/__init__.py index d2e34ffdc..3b85afbe4 100644 --- a/handlers/routines/__init__.py +++ b/handlers/routines/__init__.py @@ -902,7 +902,10 @@ async def _edit_or_send( async def _show_menu(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: """Show main routines menu.""" chat_id = update.effective_chat.id - routines = discover_routines(force_reload=True) + # Plain discovery: it is mtime-keyed, so new/edited/deleted routine files are + # still picked up on every render (PERF-620). ``force_reload=True`` is kept for + # the explicit Reload button, whose job is to retry cached load *failures*. + routines = discover_routines() all_instances = _get_instances(context) # Count running instances per routine diff --git a/tests/test_routine_discovery_cache.py b/tests/test_routine_discovery_cache.py index 1172cb87a..0582b8f67 100644 --- a/tests/test_routine_discovery_cache.py +++ b/tests/test_routine_discovery_cache.py @@ -7,8 +7,10 @@ and deletions on the next call — no restart needed. """ +import asyncio import os from pathlib import Path +from types import SimpleNamespace import routines.base as base from routines.base import discover_routines, discover_routines_from_path @@ -174,6 +176,75 @@ def test_force_reload_still_reimports_all(self, monkeypatch): assert len(calls) == len(own) +class TestRoutinesMenuRender: + """PERF-620: rendering the /routines menu no longer forces a full re-import. + + ``_show_menu`` used to pass ``force_reload=True``, so every ``/routines`` and + every "back to menu" callback re-executed every routine module. Discovery is + mtime-keyed, so a plain call already picks up new, edited and deleted files — + the forced reload only blanked the caches. It is still passed by the explicit + Reload button, which exists to retry a *cached load failure*. + """ + + @staticmethod + def _spy_discovery(monkeypatch, hr): + """Record the kwargs of each ``discover_routines`` call, still calling it.""" + seen: list[dict] = [] + real = hr.discover_routines + + def spy(*args, **kwargs): + seen.append(dict(kwargs)) + return real(*args, **kwargs) + + monkeypatch.setattr(hr, "discover_routines", spy) + return seen + + @staticmethod + def _callback_update(data: str): + async def noop(*args, **kwargs): + return None + + message = SimpleNamespace(edit_text=noop, reply_text=noop) + return SimpleNamespace( + effective_chat=SimpleNamespace(id=1), + message=None, + callback_query=SimpleNamespace(data=data, answer=noop, message=message), + ) + + def test_repeated_renders_do_not_reimport(self, monkeypatch): + import handlers.routines as hr + + discover_routines() # warm the mtime cache + + seen = self._spy_discovery(monkeypatch, hr) + reloaded: list[str] = [] + monkeypatch.setattr( + base.importlib, "reload", lambda m: reloaded.append(m.__name__) + ) + + update = self._callback_update("routines:menu") + context = SimpleNamespace(user_data={}) + asyncio.run(hr._show_menu(update, context)) + asyncio.run(hr._show_menu(update, context)) + + assert seen == [{}, {}] # no force_reload from the menu + assert reloaded == [] + + def test_reload_button_still_forces(self, monkeypatch): + import handlers.routines as hr + + discover_routines() # warm the mtime cache + + seen = self._spy_discovery(monkeypatch, hr) + + update = self._callback_update("routines:reload") + context = SimpleNamespace(user_data={}) + # __wrapped__ skips the @restricted auth check; the branch is what matters. + asyncio.run(hr.routines_callback_handler.__wrapped__(update, context)) + + assert seen[0] == {"force_reload": True} + + class TestRoutineStoreResolve: def test_resolve_agent_routine_does_not_reexec_siblings( self, tmp_path, monkeypatch From 7ac9389fd0a10e8880ead1060e4102e27dcfbbb0 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 01:23:57 +0300 Subject: [PATCH 110/154] The browser is the /bots page in both populations, empty or not An empty running fleet swapped PerfBrowser for a standalone deploy-only page, and the population toggle went with it. Nothing in the app links to ?population=terminated, so on a fleet whose bots had all stopped the run history, the closed executors and the archive drill-in were reachable only by hand-editing the URL -- the same stranding CORR-297 fixed on the terminated side. The gate also counted controllers alone, while the running population counts every active executor too: zero controllers plus one unattached executor holding open capital rendered as "No bots running", on the very page the execution dock links to in order to show that capital. The browser now always mounts. Deploy and Editor come from its own fleet header, so the page keeps only the broker diagnostic -- a bot the server can see reporting no controllers means the MQTT reports are not arriving -- and draws it as a banner above the browser instead of in place of it. --- frontend/src/pages/Bots.population.test.tsx | 127 ++++++++++++++++++- frontend/src/pages/Bots.tsx | 133 +++++++------------- 2 files changed, 168 insertions(+), 92 deletions(-) diff --git a/frontend/src/pages/Bots.population.test.tsx b/frontend/src/pages/Bots.population.test.tsx index 2465adf10..9b4c6085e 100644 --- a/frontend/src/pages/Bots.population.test.tsx +++ b/frontend/src/pages/Bots.population.test.tsx @@ -1,5 +1,5 @@ /** - * Which page `/bots` draws when the live fleet is empty (CORR-297). + * Which page `/bots` draws when the live fleet is empty (CORR-297, CORR-357). * * The empty state used to be gated on the *live* controller list alone, and * `PerfBrowser` returned `null` for the same reason — so a server whose bots @@ -9,6 +9,16 @@ * redirect into) were unreachable from the UI. An empty fleet is exactly when * the terminated population is worth reading. * + * CORR-297 fixed one direction only: the *running* side kept the standalone + * deploy-only page, which strands the reader just as badly. Nothing in the app + * links to `?population=terminated` (the `?tab=` and `/executors` routes are + * redirects), so on a fleet whose bots had all stopped the population toggle + * was gone with the browser and the history was reachable only by editing the + * URL — and because the running population counts active *executors* besides + * controllers, a standalone executor holding open capital was drawn as "No bots + * running". So the browser is now the page in both populations, and the only + * thing the page still says for itself is the broker diagnostic below. + * * Needs a DOM, so this file overrides vitest's default `node` environment. * * @vitest-environment jsdom @@ -17,11 +27,11 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { act } from "react"; import { createRoot, type Root } from "react-dom/client"; -import { MemoryRouter } from "react-router-dom"; +import { MemoryRouter, useLocation } from "react-router-dom"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ServerContext } from "@/hooks/useServer"; -import type { BotRunInfo, ControllerInfo } from "@/lib/api"; +import type { BotRunInfo, BotSummary, ControllerInfo, ExecutorInfo } from "@/lib/api"; const getBots = vi.fn(); const getBotRuns = vi.fn(); @@ -126,6 +136,45 @@ function runOf(over: Partial = {}): BotRunInfo { }; } +/** An active executor nobody claims: no `controller_id`, so it files under + * `(unattached)` — the running population counts it even with no controllers. */ +function activeExecutorOf(over: Partial = {}): ExecutorInfo { + return { + id: "ex-solo-1", + type: "position_executor", + connector: "binance", + trading_pair: "SOL-USDC", + side: "BUY", + status: "active", + close_type: "", + pnl: 12, + volume: 5000, + timestamp: Date.parse(HOUR_AGO) / 1000, + controller_id: "", + cum_fees_quote: 0, + net_pnl_pct: 0.004, + entry_price: 100, + current_price: 101, + close_timestamp: 0, + custom_info: {}, + config: {}, + ...over, + }; +} + +function botOf(over: Partial = {}): BotSummary { + return { + bot_name: "mm-sol-1", + status: "running", + num_controllers: 0, + error_count: 0, + deployed_at: HOUR_AGO, + error_logs: [], + general_logs: [], + ...over, + }; +} + let container: HTMLDivElement; let root: Root; @@ -153,6 +202,15 @@ afterEach(async () => { container.remove(); }); +/** Where the router is, so a click on a population segment can be checked. */ +function LocationProbe() { + const { search } = useLocation(); + return {search}; +} + +const locationSearch = () => + container.querySelector('[data-testid="location"]')?.textContent ?? ""; + async function render(search: string) { const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); await act(async () => { @@ -161,6 +219,7 @@ async function render(search: string) { {} }}> + , @@ -215,10 +274,66 @@ describe("/bots with no live fleet", () => { expect(text()).not.toContain("No bots running"); }); - it("still offers Deploy on the Running population", async () => { + it("draws the browser on an empty running fleet, not a deploy-only page", async () => { + await render(""); + + // Both sides of the population toggle are on screen, with Running on: the + // deploy-only page carried neither, so the terminated tree was reachable + // only by hand-editing the URL (CORR-357). + expect(button("Running")?.getAttribute("aria-pressed")).toBe("true"); + expect(button("Terminated")).toBeTruthy(); + // And the two fleet-wide actions the old page existed to carry: the + // browser's own header has them, lowercase. + expect(button("Deploy bot")).toBeTruthy(); + expect(button("Editor")).toBeTruthy(); + }); + + it("reaches the terminated tree from an empty running fleet in one click", async () => { + await render(""); + + await act(async () => { + button("Terminated")!.click(); + }); + + expect(locationSearch()).toContain("population=terminated"); + expect(button("Terminated")?.getAttribute("aria-pressed")).toBe("true"); + }); + + it("draws an active executor that no controller claims", async () => { + // Zero controllers, one executor holding open capital. The old gate counted + // controllers only, so this rendered as "No bots running" while the money + // was live — and the execution dock links here to show it. + getExecutorsPage.mockResolvedValue({ + executors: [activeExecutorOf()], + next_cursor: null, + }); + + await render(""); + + expect(text()).not.toContain("No bots running"); + // It has a row of its own in the scope tree, filed under the unattached bot, + // and the grain strip counts it: zero controllers, one executor. + expect(text()).toContain("unattached"); + expect(text()).toContain("0 controllers · 1 executor"); + // Its money folded into the fleet strip: +$12 net over $5,000 of volume. + expect(text()).toContain("+$12.00"); + expect(text()).toContain("$5.0K"); + }); + + it("keeps the broker diagnostic and the population toggle together", async () => { + // A bot the server can see, reporting no controller, means the MQTT reports + // are not arriving. That diagnostic is the one thing the page still says for + // itself — and it now says it *over* the browser rather than instead of it. + getBots.mockResolvedValue({ + bots: [botOf()], + controllers: [], + server_online: true, + }); + await render(""); - expect(text()).toContain("No bots running"); - expect(button("Deploy Bot")).toBeTruthy(); + expect(text()).toContain("mm-sol-1 is running but reporting no controllers"); + expect(text()).toContain("make doctor"); + expect(button("Terminated")).toBeTruthy(); }); }); diff --git a/frontend/src/pages/Bots.tsx b/frontend/src/pages/Bots.tsx index 66b8aa850..273050071 100644 --- a/frontend/src/pages/Bots.tsx +++ b/frontend/src/pages/Bots.tsx @@ -1,11 +1,7 @@ -import { Bot, Rocket, TerminalSquare } from "lucide-react"; -import { useState } from "react"; import { Navigate, useSearchParams } from "react-router-dom"; import { NoServerCard } from "@/components/NoServerCard"; import { PerfBrowser } from "@/components/perf/PerfBrowser"; -import { DeployBotDialog } from "@/components/bots/DeployBotDialog"; -import { EditorModal } from "@/components/editor/EditorModal"; import { FallbackSpinner } from "@/components/ui/FallbackSpinner"; import { useFleetData } from "@/hooks/useFleetData"; import { useServer } from "@/hooks/useServer"; @@ -19,7 +15,7 @@ import { parsePopulation } from "@/lib/perf-tree"; * sidebar (fleet → bot → controller) *is* the page, and every bot-level action * that lived in the accordion is reachable from the scope it belongs to. * - * What is left here is a *host*: the empty states, and the browser over + * What is left here is a *host*: the no-data guards, and the browser over * `useFleetData` (FEAT-108). The fleet query and the performance-history walk * that used to live in this file are in that hook now, unchanged and under the * same query keys — so the agent workspace can mount the same browser over the @@ -39,13 +35,6 @@ export function Bots() { // old links land on the scope that answers them (FEAT-086). const tab = searchParams.get("tab"); const legacyRunsTab = tab === "runs" || tab === "archived"; - // Deploy lives in the browser's fleet-scope header — except when there is no - // fleet to scope, which is exactly when it is needed most (see below). The - // Editor sits beside it there and is stranded the same way: writing the - // controller you are about to deploy is the *first* thing an empty fleet - // needs, not something reachable only once a bot is already running. - const [showDeploy, setShowDeploy] = useState(false); - const [showEditor, setShowEditor] = useState(false); const fleet = useFleetData(server, { population }); @@ -83,17 +72,6 @@ export function Bots() { ); } - // Nothing to scope: the browser draws nothing without controllers, and the - // fleet header that carries Deploy is part of the browser — so the empty - // state has to carry the one action that gets out of it. - // - // Only for the *live* fleet, though. An empty fleet is exactly the state in - // which the Terminated population is worth reading — the run history, the - // closed executors and the archive drill-in all live there, and their queries - // above have already fetched them — so answering "No bots running" for - // `?population=terminated` strands the reader on the one screen that still - // has something to say (CORR-297). - // // "No controllers" and "no bots" are not the same thing, and saying the first // as the second is how a broker outage reads as an empty fleet. A controller // is reported over the server's MQTT broker; the bot list is not (Docker @@ -101,70 +79,53 @@ export function Bots() { // means the reports are not arriving — and that is worth naming here, on the // screen where the bot is missing, rather than leaving it to be found in the // API's logs. - const silentBots = fleet.bots; - if (population === "running" && fleet.controllers.length === 0) { - return ( -
- - {silentBots.length === 0 ? ( -

No bots running

- ) : ( -
-

- {silentBots.length === 1 - ? `${silentBots[0].bot_name} is running but reporting no controllers` - : `${silentBots.length} bots are running but reporting no controllers`} -

-

- Controller reports reach the API over its MQTT broker. Check that the broker - is up and that the API is connected to it — on the server,{" "} - make doctor names it. -

-
- )} -
- - -
- setShowDeploy(false)} server={server} /> - {showEditor && setShowEditor(false)} />} -
- ); - } + // + // A banner over the browser, not a page instead of it (CORR-357). The browser + // *is* the page in both populations: it carries the population toggle, the + // group-by picker, the filter bubbles and the terminated drill-in, so swapping + // it out on an empty live fleet left the run history reachable only by hand- + // editing the URL — and hid the active executors that the running population + // counts besides controllers, i.e. open capital, behind "No bots running". + const silentBots = population === "running" && fleet.controllers.length === 0 ? fleet.bots : []; return ( - +
+ {silentBots.length > 0 && ( +
+

+ {silentBots.length === 1 + ? `${silentBots[0].bot_name} is running but reporting no controllers` + : `${silentBots.length} bots are running but reporting no controllers`} +

+

+ Controller reports reach the API over its MQTT broker. Check that the broker is + up and that the API is connected to it — on the server,{" "} + make doctor names it. +

+
+ )} +
+ +
+
); } From 5d6288f6f6cf23f6f87c390b12251c103e9b760e Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 01:58:47 +0300 Subject: [PATCH 111/154] Own the frame queue and the 2D context in the backfill chart test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit StrictMode's double mount fires the chart-init effect twice in one tick, so `import("lightweight-charts")` is requested twice concurrently — and Vitest's mocker answers the stub to the first request and raw-imports the real library for the second. Two plain mounts in one act() split the same way, and neither a static nor an awaited warm-up import changes it, so the file cannot win that race from the outside. The real ChartWidget then built against a jsdom with no 2D canvas and scheduled a draw frame that fired after teardown, outside every test: ~20% of full runs exited 1 with "2 unhandled errors" and 1993 tests green. So the file now owns both things the library needs and jsdom lacks — the frame queue, with teardown asserting no frame outlived the tree, and a 2D context — and asserts the stub answered on the single-mount path, where it can. 11 consecutive full runs: 1993 passed, 0 unhandled errors, 0 canvas warnings. --- .../trade/TradeChart.backfill.test.tsx | 85 +++++++++++++++++-- 1 file changed, 79 insertions(+), 6 deletions(-) diff --git a/frontend/src/components/trade/TradeChart.backfill.test.tsx b/frontend/src/components/trade/TradeChart.backfill.test.tsx index 9ce1ada15..e5549492d 100644 --- a/frontend/src/components/trade/TradeChart.backfill.test.tsx +++ b/frontend/src/components/trade/TradeChart.backfill.test.tsx @@ -10,12 +10,28 @@ * The production contract is the other half: one backfill per distinct set of * the effect's six dependencies, and no refetch when the props do not change. * + * The same double mount fires the chart-init effect twice in one tick, so + * `import("lightweight-charts")` is requested twice concurrently — and Vitest's + * mocker answers the stub to the first request and raw-imports the real library + * for the second. It is the concurrency, not StrictMode: two plain mounts in one + * `act()` split the same way, and neither a static nor an awaited warm-up import + * of the module changes it. The real `ChartWidget` then builds against a jsdom + * that has no 2D canvas and schedules a draw frame, which fired after teardown, + * outside every test, and exited the whole suite 1 with "2 unhandled errors" + * while every test reported green (CORR-360). + * + * So this file owns the two things the chart library needs and jsdom does not + * have: the frame queue — nothing runs unless we run it, and teardown proves no + * frame outlived the tree — and a 2D context. Whichever module answers the + * import, it can no longer leave anything behind. + * * @vitest-environment jsdom */ +import * as charts from "lightweight-charts"; import { act, StrictMode, type ComponentProps } from "react"; import { createRoot, type Root } from "react-dom/client"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from "vitest"; import { TradeChart } from "./TradeChart"; @@ -93,6 +109,27 @@ declare global { var IS_REACT_ACT_ENVIRONMENT: boolean; } +/** The stub's factory, read back to prove the component got the stub. */ +const createChartStub = charts.createChart as unknown as Mock; + +/** Frames the render scheduled, so teardown can prove none outlive the tree. */ +const pendingFrames = new Map(); + +/** + * jsdom ships no canvas backend, and asking it for one only prints + * "Not implemented: HTMLCanvasElement's getContext()" and answers null — which a + * real chart widget dereferences. The calls just have to answer; this window + * belongs to this file alone, so it is patched once and never restored. + */ +HTMLCanvasElement.prototype.getContext = (() => + new Proxy({} as Record, { + get(target, prop) { + if (typeof prop !== "string") return undefined; + if (!(prop in target)) target[prop] = vi.fn(() => ({ width: 0 })); + return target[prop]; + }, + })) as unknown as typeof HTMLCanvasElement.prototype.getContext; + let container: HTMLDivElement; let root: Root; @@ -122,15 +159,45 @@ async function render(strict: boolean, extra: Partial { + await new Promise((resolve) => setTimeout(resolve, 0)); + }); + // A single mount asks for the chart library once, so the stub is what must + // have answered — the module identity is a result here, not an assumption. + // The StrictMode path asks twice at once, which Vitest's mocker cannot serve + // (see the docblock); the doubles above are what make that harmless. + if (!strict) { + expect(createChartStub).toHaveBeenCalled(); + expect(container.querySelector("canvas")).toBeNull(); + } } beforeEach(() => { globalThis.IS_REACT_ACT_ENVIRONMENT = true; - globalThis.ResizeObserver = class { - observe() {} - unobserve() {} - disconnect() {} - } as unknown as typeof ResizeObserver; + vi.stubGlobal( + "ResizeObserver", + class { + observe() {} + unobserve() {} + disconnect() {} + }, + ); + // Own the frame queue instead of letting jsdom fire it off a timer: a frame + // that runs after its test is an exception belonging to no test at all. + let nextFrame = 1; + pendingFrames.clear(); + vi.stubGlobal("requestAnimationFrame", (cb: FrameRequestCallback) => { + const id = nextFrame++; + pendingFrames.set(id, cb); + return id; + }); + vi.stubGlobal("cancelAnimationFrame", (id: number) => { + pendingFrames.delete(id); + }); + createChartStub.mockClear(); store.mergeCandles.mockClear(); store.setDuration.mockClear(); apiState.getCandles.mockReset(); @@ -143,6 +210,12 @@ beforeEach(() => { afterEach(async () => { await act(async () => root.unmount()); container.remove(); + // Unmounting destroys the chart, which cancels its own frame, so the queue has + // to be empty. Drain it either way, then report what was still queued. + const queued = [...pendingFrames.keys()]; + pendingFrames.clear(); + expect(queued).toEqual([]); + vi.unstubAllGlobals(); vi.useRealTimers(); }); From 9f8f25de26b3d6173ae16ed3394f637fb07519be Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 02:13:55 +0300 Subject: [PATCH 112/154] Log every snippet a tick runs, and refuse one in dry-run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `run_code` stayed outside both predicates: the action log never kept a row for the one tool that can change anything, and the unattended gate keyed its dry-run block off `is_dangerous_tool_call`, so a snippet fell through to the auto-approve tail — a dry run could execute a real, irreversible mutation through `client.*` while promising that nothing mutates. It stays ungated by name on purpose: since ARCH-308 a snippet is how a tick reads a market, so a confirmation here would land on every candle read. Instead it is classified by action — `run` executes, `history`/`get` only read stored runs — which the log records and the dry-run branch refuses. --- condor/agents/risk.py | 19 ++++++++ condor/code_runner.py | 11 +++++ condor/runtime/danger.py | 65 ++++++++++++++++++++++++--- tests/test_agent_actions.py | 89 +++++++++++++++++++++++++++++++++++++ tests/test_risk_gate.py | 69 ++++++++++++++++++++++++++++ 5 files changed, 247 insertions(+), 6 deletions(-) diff --git a/condor/agents/risk.py b/condor/agents/risk.py index 2fe3fe4a5..11adabad3 100644 --- a/condor/agents/risk.py +++ b/condor/agents/risk.py @@ -20,6 +20,7 @@ DANGEROUS_CLMM_ACTIONS, LEVERAGE_TOOL, LEVERAGED_EXECUTOR_TOOLS, + is_code_execution_call, is_dangerous_tool_call, tool_call_input, tool_call_name, @@ -855,6 +856,24 @@ async def callback(tool_call: dict, options: list[dict]) -> dict: "create_*_executor tool instead", ) + # A snippet is not in `DANGEROUS_TOOLS` and must not be — it is how a + # tick reads a market (ARCH-308), and gating it by name would put a + # confirmation in front of every candle read. But it runs arbitrary + # Python holding the unrestricted API client, so in dry-run it is the one + # remaining way to mutate the world for real: every *named* write above + # is refused there, and `await client.gateway.start(...)` inside a + # snippet was not (SEC-616). Dry-run's promise is that nothing mutates, + # so the tool that can mutate anything does not auto-approve there. + # Reading past runs (`history`, `get`) changes nothing and stays free. + if execution_mode == "dry_run" and is_code_execution_call(tool_call): + return deny( + tool_call_name(tool_call), + "this session runs in dry-run mode, where nothing mutates, and a " + "snippet holds the unrestricted API client — read what you need " + "with the read-only tools instead", + level=logging.INFO, + ) + # Auto-approve everything else for opt in options: if opt.get("kind") in ("allow_once", "allow_always"): diff --git a/condor/code_runner.py b/condor/code_runner.py index 4593eee10..35a86ce05 100644 --- a/condor/code_runner.py +++ b/condor/code_runner.py @@ -18,6 +18,17 @@ the whole process, so every caller must gate it at least as tightly as ``condor/web/routes/code.py`` does (SEC-151); "the caller is authenticated" is not a gate. + +Which means the ``code_run`` preference that gate honors is **admin-equivalent +by construction**, not a narrower permission that happens to include code: a +snippet reaches the API client, the config, every ``condor.*`` module and the +process itself, so granting it grants everything a confirmation prompt stands in +front of elsewhere (SEC-616). Grant it as you would admin. A snippet is not +confirmed per call on purpose — gating it by name would put a prompt in front of +every agent tick's market read — so this grant, the per-run record in +``condor/code_runs.py`` and the action-log row +(:func:`condor.runtime.danger.is_code_execution_call`) are the whole containment; +unattended dry-run sessions refuse it outright in ``condor/agents/risk.py``. """ from __future__ import annotations diff --git a/condor/runtime/danger.py b/condor/runtime/danger.py index a83ed7714..ced5d6f7e 100644 --- a/condor/runtime/danger.py +++ b/condor/runtime/danger.py @@ -242,6 +242,20 @@ #: no trace at all. Recording them is the log's question, not the gate's. MUTATING_CONTROLLER_ACTIONS = {"upsert", "delete"} READ_ONLY_CONTROLLER_ACTIONS = {"list", "describe"} +#: The snippet runner. Deliberately *not* in ``DANGEROUS_TOOLS`` and not to be +#: added: since ARCH-308 a tick reads a market it can compute on only through +#: ``client.market_data.*`` inside a snippet, so a name gate here would put a +#: confirmation in front of every tick's candle read (SEC-616). What it does get +#: is a log row, and a refusal in the one mode that promises nothing mutates. +CODE_RUN_TOOL = "run_code" +#: ``run_code``'s three actions split by what they touch: ``run`` executes a +#: snippet holding the unrestricted API client, and the other two only read runs +#: already stored. +MUTATING_CODE_RUN_ACTIONS = {"run"} +READ_ONLY_CODE_RUN_ACTIONS = {"history", "get"} +#: How much of a snippet's first line the log row carries. A summary is one line +#: on a page, and the whole source is in the code-run store anyway. +MAX_SNIPPET_HEAD_CHARS = 80 #: `manage_gateway_config` is recorded on its *action*, not its resource type: #: what it edits is what the gate weighs, and whether it edited at all is what #: the log weighs. @@ -520,6 +534,25 @@ def is_mutating_tool_call(tool_call: dict[str, Any]) -> bool: } +def is_code_execution_call(tool_call: dict[str, Any]) -> bool: + """Does this call *execute* a snippet? (SEC-616) + + ``run_code(action="run")`` hands arbitrary Python the unrestricted API + client, so it can do anything any other tool can do and several things none + of them can. ``history`` and ``get`` only read runs already stored. + + True on an action this module cannot read, which is both the fail-open rule + its siblings follow *and* the tool's own default: ``action`` omitted means + ``run``. The log uses this as an extra row; the unattended gate uses it to + refuse in dry-run, where failing this way is failing closed. + """ + if tool_call_name(tool_call) != CODE_RUN_TOOL: + return False + return _is_mutating_action( + tool_call, MUTATING_CODE_RUN_ACTIONS, READ_ONLY_CODE_RUN_ACTIONS + ) + + def is_recordable_tool_call(tool_call: dict[str, Any]) -> bool: """Should the action log keep a row for this call? (FEAT-102) @@ -529,11 +562,14 @@ def is_recordable_tool_call(tool_call: dict[str, Any]) -> bool: functions answering nearly the same question will drift, and this shape makes the gate's set structurally a subset that cannot fall behind. - Its one extra today is ``manage_controllers``. The gate excludes that tool - entirely and should keep excluding it — widening the gate would put a new - confirmation prompt in front of a running fleet — but a bot's controllers - are *written* by exactly these calls, so a log that drops them cannot say - how a fleet was built or which of its config writes were rejected. + Its extras are ``manage_controllers`` and ``run_code``. The gate excludes + both tools entirely and should keep excluding them — widening the gate would + put a new confirmation prompt in front of a running fleet, and in front of + every tick's market read — but a bot's controllers are *written* by exactly + these calls, so a log that drops them cannot say how a fleet was built or + which of its config writes were rejected, and a snippet can change anything + at all, so a log that drops it is silent about the one tool that can + (SEC-616). Fails open the same way its siblings do: an action this module has not heard of is recorded rather than dropped. @@ -546,7 +582,7 @@ def is_recordable_tool_call(tool_call: dict[str, Any]) -> bool: tool_call, MUTATING_CONTROLLER_ACTIONS, READ_ONLY_CONTROLLER_ACTIONS ) - return False + return is_code_execution_call(tool_call) def format_tool_summary(tool_call: dict[str, Any]) -> str: @@ -729,5 +765,22 @@ def format_tool_summary(tool_call: dict[str, Any]) -> str: ) return f"Controller {target}: {action} '{name}'" + if tool_name == CODE_RUN_TOOL: + # Never gated either, so this line is written for the log (SEC-616). The + # label is what the caller said the snippet was for and the first line is + # what it actually starts doing — enough to tell a candle read from a + # `client.gateway.start(...)` without carrying a whole script into the + # log; the full source is in the code-run store (`condor/code_runs.py`). + action = input_data.get("action") or "run" + if action not in MUTATING_CODE_RUN_ACTIONS: + return f"Code run: {action}" + label = str(input_data.get("label") or "").strip() + code = str(input_data.get("code") or "") + head = next((line.strip() for line in code.splitlines() if line.strip()), "") + if len(head) > MAX_SNIPPET_HEAD_CHARS: + head = head[:MAX_SNIPPET_HEAD_CHARS] + "…" + what = f"Run snippet '{label}'" if label else "Run snippet" + return f"{what}: {head}" if head else f"{what} (no code)" + # Generic fallback return tool_name diff --git a/tests/test_agent_actions.py b/tests/test_agent_actions.py index 09b7167de..7e21c4724 100644 --- a/tests/test_agent_actions.py +++ b/tests/test_agent_actions.py @@ -7,6 +7,7 @@ """ import json +import typing import pytest @@ -555,6 +556,94 @@ def test_the_confirmation_gate_is_untouched_by_the_log_growing(): ) +# ── The snippet that can change anything (SEC-616) ── + + +def test_a_snippet_a_tick_ran_is_recorded_with_its_label(): + """``run_code`` is outside the gate on purpose and was outside the log too. + + The one tool that can do anything any other tool can — it holds the + unrestricted API client — left the log that answers "what changed the world" + completely silent. + """ + calls = [ + folded( + "mcp__condor__run_code", + label="open a CLMM on SOL-USDC", + code="pos = await client.gateway.clmm_open(...)\nprint(pos)", + ) + ] + + (action,) = actions_from_tool_calls(calls, tick=4, at=1.0) + + assert action.tool == "run_code" + assert action.verb == "run_code" + assert action.ok is True + assert action.summary == ( + "Run snippet 'open a CLMM on SOL-USDC': " + "pos = await client.gateway.clmm_open(...)" + ) + + +def test_a_failed_snippet_is_recorded_with_its_error(): + calls = [folded("run_code", status="failed", code="await client.boom()")] + calls[0]["output"] = "AttributeError: boom" + + (action,) = actions_from_tool_calls(calls, tick=4, at=1.0) + + assert action.ok is False + assert action.error == "AttributeError: boom" + assert action.summary == "Run snippet: await client.boom()" + + +def test_reading_past_runs_back_is_not_recorded(): + """``history`` and ``get`` read the store; they execute nothing.""" + calls = [ + folded("run_code", action="history", limit=20), + folded("run_code", action="get", run_id="cr_1"), + ] + assert actions_from_tool_calls(calls, tick=1, at=1.0) == [] + + +def test_the_snippet_predicate_fails_open_and_matches_the_tool_default(): + """An unreadable action is recorded — and ``action`` omitted *is* a run.""" + from condor.runtime.danger import is_recordable_tool_call + + assert is_recordable_tool_call({"tool": "run_code", "input": None}) + assert is_recordable_tool_call({"tool": "run_code", "input": {"code": "1"}}) + assert is_recordable_tool_call({"tool": "run_code", "input": {"action": "sudo"}}) + + +def test_the_snippet_runner_is_still_not_gated(): + """Criterion: no confirmation is introduced on the tick path. + + Since ARCH-308 a snippet is how a tick reads a market, so a name gate here + would prompt a human on every candle read. + """ + from condor.runtime.danger import DANGEROUS_TOOLS, is_dangerous_tool_call + + assert "run_code" not in DANGEROUS_TOOLS + for action in ("run", "history", "get", "something_new"): + call = {"tool": "run_code", "input": {"action": action, "code": "x"}} + assert is_dangerous_tool_call(call) is False, action + assert is_dangerous_tool_call({"tool": "run_code", "input": None}) is False + + +def test_the_snippet_action_sets_match_the_registered_tool(): + """Every action literal the tool accepts is classified, or the fail-open + rule would record a read of the run history as a change to the world.""" + from condor.runtime.danger import ( + MUTATING_CODE_RUN_ACTIONS, + READ_ONLY_CODE_RUN_ACTIONS, + ) + from mcp_servers.condor.server import run_code + + fn = getattr(run_code, "fn", run_code) + literals = set(typing.get_args(fn.__annotations__["action"])) + assert MUTATING_CODE_RUN_ACTIONS | READ_ONLY_CODE_RUN_ACTIONS == literals + assert not (MUTATING_CODE_RUN_ACTIONS & READ_ONLY_CODE_RUN_ACTIONS) + + def test_the_controller_action_sets_match_the_registered_tool(): """Every action literal the tool accepts is classified, or the fail-open rule would silently record a read.""" diff --git a/tests/test_risk_gate.py b/tests/test_risk_gate.py index 7be296314..64cb5ba09 100644 --- a/tests/test_risk_gate.py +++ b/tests/test_risk_gate.py @@ -11,6 +11,7 @@ import pytest from condor.agents.risk import ( + RefusalLog, RiskEngine, RiskLimits, RiskState, @@ -735,3 +736,71 @@ def test_amm_guide_load_is_not_risk_checked(): assert result["outcome"]["outcome"] == "selected" assert state.total_exposure == 0 + + +# --------------------------------------------------------------------------- +# run_code is ungated by name on purpose (it is how a tick reads a market since +# ARCH-308), which left one way to mutate the world from inside a dry run: a +# snippet holding the unrestricted API client (SEC-616). +# --------------------------------------------------------------------------- + + +def _code_call(**args) -> dict: + return {"tool": "mcp__condor__run_code", "input": args} + + +def test_a_dry_run_cannot_execute_a_snippet(): + engine = RiskEngine(RiskLimits()) + refusals = RefusalLog() + callback = auto_approve_with_risk_check( + engine, RiskState(), execution_mode="dry_run", refusals=refusals + ) + + result = asyncio.run( + callback( + _code_call(code="await client.gateway.start({'image': 'x'})"), _OPTIONS + ) + ) + + assert result["outcome"]["outcome"] == "cancelled" + (noted,) = refusals.drain() + assert noted["tool"] == "run_code" + assert "dry-run" in noted["reason"] + + +def test_a_dry_run_refuses_a_snippet_whose_action_cannot_be_read(): + """The tool's own default is ``run``, so an absent action is an execution.""" + callback = auto_approve_with_risk_check( + RiskEngine(RiskLimits()), RiskState(), execution_mode="dry_run" + ) + + for call in ( + _code_call(code="print(1)"), + {"tool": "run_code", "input": None}, + _code_call(action=None, code="print(1)"), + ): + result = asyncio.run(callback(call, _OPTIONS)) + assert result["outcome"]["outcome"] == "cancelled", call + + +def test_a_dry_run_still_reads_its_past_runs(): + callback = auto_approve_with_risk_check( + RiskEngine(RiskLimits()), RiskState(), execution_mode="dry_run" + ) + + for call in (_code_call(action="history"), _code_call(action="get", run_id="cr_1")): + result = asyncio.run(callback(call, _OPTIONS)) + assert result["outcome"]["outcome"] == "selected", call + + +def test_a_live_loop_still_runs_snippets_without_a_confirmation(): + """The refusal is dry-run's, not a new gate: loop mode is unchanged.""" + callback = auto_approve_with_risk_check( + RiskEngine(RiskLimits()), RiskState(), execution_mode="loop" + ) + + result = asyncio.run( + callback(_code_call(code="await client.market_data.candles(...)"), _OPTIONS) + ) + + assert result["outcome"]["outcome"] == "selected" From 0cbf309bc6da66db1ce69dffea5d21e0d1a04df8 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 02:23:32 +0300 Subject: [PATCH 113/154] An empty fleet is an absence the browser states, not a fold of zeros PerfBrowser had no sentence for an empty population. A server that has never run a bot got a strip of $0.00 tiles, "No performance history available" and a 10px muted "Nothing in scope." in the sidebar -- a line written for a different case (every filter ticked off, a window with nothing in it) and not drawn at all once the sidebar is collapsed. So a measurement of zero stood in for an absence, and the one diagnostic that tells a broker outage apart from an empty fleet lived outside components/perf entirely. Both answers are the browser's now, in the report pane every reader has: "No bots running" with a Deploy call to action, or -- winning where it applies, because which of the two it is is the question -- the silent-bots line naming the MQTT broker and make doctor. The condition reads rawLeaves, the population before the filters and the grain: a tree emptied by a pair filter is a narrowing the reader performed, and a fleet holding one unclaimed active executor is open capital rather than an empty server, however few controllers report it. The stopgap banner CORR-357 left in Bots.tsx is gone with it, or the page would say the same thing twice; /bots is a host again and says nothing for itself. --- frontend/src/components/perf/PerfBrowser.tsx | 79 ++++++++++++++++++++ frontend/src/pages/Bots.population.test.tsx | 75 ++++++++++++++++++- frontend/src/pages/Bots.tsx | 79 +++++++------------- 3 files changed, 175 insertions(+), 58 deletions(-) diff --git a/frontend/src/components/perf/PerfBrowser.tsx b/frontend/src/components/perf/PerfBrowser.tsx index 18605227a..e2050673f 100644 --- a/frontend/src/components/perf/PerfBrowser.tsx +++ b/frontend/src/components/perf/PerfBrowser.tsx @@ -211,6 +211,9 @@ const EMPTY_RUNS: BotRunInfo[] = []; /** Held still, so an unpassed `terminatedControllers` prop is not a new array on every render. */ const EMPTY_TERMINATED: ControllerInfo[] = []; +/** Held still, so "no bot is silent" is not a new array on every render. */ +const EMPTY_BOTS: BotSummary[] = []; + /** * The series a splitting scope does not have. * @@ -1112,6 +1115,36 @@ export function PerfBrowser({ filters.execTypes.length > 0 || filters.agents.length > 0; + /** + * The bots the server can see that are reporting no controller at all. + * + * "No controllers" and "no bots" are not the same thing, and saying the first + * as the second is how a broker outage reads as an empty fleet. A controller + * is reported over the server's MQTT broker; the bot list is not (Docker + * answers that one). So a bot that is up while no controller report arrives + * means the reports are not arriving — worth naming on the screen where the + * controller is missing, rather than leaving it to be found in the API's logs. + * + * This used to be the page's sentence, drawn beside `/bots` instead of in the + * browser (CORR-357); it belongs here, where the records it is about are. + */ + const silentBots = population === "running" && controllers.length === 0 ? bots : EMPTY_BOTS; + + /** + * Nothing in the live population at all — and therefore an absence to state, + * not a fold of zeros to draw (CORR-356). + * + * Read off `rawLeaves`, the population *before* the filters and the grain. A + * tree emptied by a pair filter or by `Controllers`-only granularity is a + * narrowing the reader performed and `ScopeTree` already has the sentence for + * it ("Nothing in scope."); only the raw population can say the server is not + * running anything. `rawLeaves` is also the right count rather than + * `controllers.length`: `runningLeaves` pushes a leaf per active executor too, + * and an unclaimed executor is open capital, so a fleet with one of those is + * not empty however few controllers report it (CORR-357). + */ + const emptyPopulation = population === "running" && rawLeaves.length === 0; + /** * The one bot every row on screen belongs to, when there is one. * @@ -2916,6 +2949,52 @@ export function PerfBrowser({ a short one the reader can reach the rows that no longer do rather than have them hang off the bottom of the screen. */}
+ {/* What an empty live population *means*, said before any number is + drawn (CORR-356). Without it a server that has never run a bot + reads as a measurement of zero: a strip of `$0.00` tiles and "No + performance history available", with a 10px muted "Nothing in + scope." in the sidebar as the only hint — and that line is + written for a different case (every filter ticked off, a window + with nothing in it) and is not drawn at all once the sidebar is + collapsed. So it goes in the report pane, which every reader has. + + The broker diagnostic wins where it applies, because telling an + outage apart from an empty fleet is the higher-value half, and it + is drawn *over* the numbers rather than instead of them: a silent + bot can sit beside a live unattached executor, and that executor's + money is real. */} + {silentBots.length > 0 ? ( +
+

+ {silentBots.length === 1 + ? `${silentBots[0].bot_name} is running but reporting no controllers` + : `${silentBots.length} bots are running but reporting no controllers`} +

+

+ Controller reports reach the API over its MQTT broker. Check that the broker is + up and that the API is connected to it — on the server,{" "} + make doctor names it. +

+
+ ) : emptyPopulation ? ( +
+

No bots running

+

+ Nothing is deployed on this server and no executor is open, so the figures below + are an absence rather than a result. What has already run is under Terminated. +

+ {scope.kind === "fleet" && ( + + )} +
+ ) : null} {/* Headline numbers first: the chart below is the shape of these. */}
{/* A fixed set of tiles, not a set that depends on what the scope diff --git a/frontend/src/pages/Bots.population.test.tsx b/frontend/src/pages/Bots.population.test.tsx index 9b4c6085e..a15cbf5c5 100644 --- a/frontend/src/pages/Bots.population.test.tsx +++ b/frontend/src/pages/Bots.population.test.tsx @@ -16,8 +16,11 @@ * was gone with the browser and the history was reachable only by editing the * URL — and because the running population counts active *executors* besides * controllers, a standalone executor holding open capital was drawn as "No bots - * running". So the browser is now the page in both populations, and the only - * thing the page still says for itself is the broker diagnostic below. + * running". So the browser is now the page in both populations, and the page + * itself says nothing: both answers an empty live fleet is owed — "No bots + * running" and the broker diagnostic — are the browser's own, drawn in its + * report pane (CORR-356), which is the one part of it a collapsed sidebar and a + * rooted host both still have. * * Needs a DOM, so this file overrides vitest's default `node` environment. * @@ -242,6 +245,30 @@ function button(label: string): HTMLButtonElement | undefined { ) as HTMLButtonElement | undefined; } +/** An icon-only button, which is named by its tooltip. */ +function titled(title: string): HTMLButtonElement { + const found = [...container.querySelectorAll("button")].find( + (b) => b.getAttribute("title") === title, + ); + if (!found) throw new Error(`no button titled ${title}`); + return found as HTMLButtonElement; +} + +/** The browser's pair filter, the cheapest way to empty a tree on purpose. */ +function pairFilter(): HTMLInputElement { + const found = container.querySelector('input[placeholder="Filter pair…"]'); + if (!found) throw new Error("no pair filter"); + return found as HTMLInputElement; +} + +/** React tracks its own value, so a typed character has to go in through the + * native setter before the input event it listens for. */ +function type(input: HTMLInputElement, value: string) { + const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")!.set!; + setter.call(input, value); + input.dispatchEvent(new Event("input", { bubbles: true })); +} + describe("/bots with no live fleet", () => { it("draws the terminated population instead of the empty state", async () => { getTerminatedControllers.mockResolvedValue({ @@ -286,6 +313,44 @@ describe("/bots with no live fleet", () => { // browser's own header has them, lowercase. expect(button("Deploy bot")).toBeTruthy(); expect(button("Editor")).toBeTruthy(); + // The answer the old page carried, now the browser's and in the report pane + // rather than only the sidebar's 10px "Nothing in scope." (CORR-356): an + // empty fleet is an absence, not the strip of $0.00 tiles beside it. + expect(text()).toContain("No bots running"); + }); + + it("states the empty answer with the sidebar collapsed", async () => { + await render(""); + + // `ScopeTree` draws nothing at all in compact mode, so a reader who has + // collapsed the sidebar is the case the sidebar's sentence cannot answer. + await act(async () => { + titled("Collapse sidebar").click(); + }); + + expect(text()).not.toContain("Nothing in scope."); + expect(text()).toContain("No bots running"); + }); + + it("keeps a filter-emptied tree a narrowing, not an empty fleet", async () => { + // A real live controller, hidden by a pair nobody trades. The tree is empty + // for a reason the reader chose, and the raw population is not — so the + // answer is the sidebar's, and "No bots running" would be a lie. + getBots.mockResolvedValue({ + bots: [botOf({ num_controllers: 1 })], + controllers: [terminatedControllerOf({ status: "running" })], + server_online: true, + }); + + await render(""); + expect(text()).not.toContain("No bots running"); + + await act(async () => { + type(pairFilter(), "zzz-nothing"); + }); + + expect(text()).toContain("Nothing in scope."); + expect(text()).not.toContain("No bots running"); }); it("reaches the terminated tree from an empty running fleet in one click", async () => { @@ -322,8 +387,9 @@ describe("/bots with no live fleet", () => { it("keeps the broker diagnostic and the population toggle together", async () => { // A bot the server can see, reporting no controller, means the MQTT reports - // are not arriving. That diagnostic is the one thing the page still says for - // itself — and it now says it *over* the browser rather than instead of it. + // are not arriving. The browser says that in its report pane now (CORR-356), + // beside the toggle rather than instead of it — and it says it there rather + // than "No bots running", because which of the two it is is the question. getBots.mockResolvedValue({ bots: [botOf()], controllers: [], @@ -334,6 +400,7 @@ describe("/bots with no live fleet", () => { expect(text()).toContain("mm-sol-1 is running but reporting no controllers"); expect(text()).toContain("make doctor"); + expect(text()).not.toContain("No bots running"); expect(button("Terminated")).toBeTruthy(); }); }); diff --git a/frontend/src/pages/Bots.tsx b/frontend/src/pages/Bots.tsx index 273050071..77d1f936c 100644 --- a/frontend/src/pages/Bots.tsx +++ b/frontend/src/pages/Bots.tsx @@ -72,60 +72,31 @@ export function Bots() { ); } - // "No controllers" and "no bots" are not the same thing, and saying the first - // as the second is how a broker outage reads as an empty fleet. A controller - // is reported over the server's MQTT broker; the bot list is not (Docker - // answers that one). So a bot the server can see, reporting no controller, - // means the reports are not arriving — and that is worth naming here, on the - // screen where the bot is missing, rather than leaving it to be found in the - // API's logs. - // - // A banner over the browser, not a page instead of it (CORR-357). The browser - // *is* the page in both populations: it carries the population toggle, the - // group-by picker, the filter bubbles and the terminated drill-in, so swapping - // it out on an empty live fleet left the run history reachable only by hand- - // editing the URL — and hid the active executors that the running population - // counts besides controllers, i.e. open capital, behind "No bots running". - const silentBots = population === "running" && fleet.controllers.length === 0 ? fleet.bots : []; - + // Nothing is said here about an empty fleet or a silent bot: both are the + // browser's sentences now, drawn in its report pane where the records they are + // about would be (CORR-356). The page owning them is what made an empty live + // fleet a page *instead of* the browser, which stranded the whole terminated + // drill-in behind a hand-edited URL (CORR-357). return ( -
- {silentBots.length > 0 && ( -
-

- {silentBots.length === 1 - ? `${silentBots[0].bot_name} is running but reporting no controllers` - : `${silentBots.length} bots are running but reporting no controllers`} -

-

- Controller reports reach the API over its MQTT broker. Check that the broker is - up and that the API is connected to it — on the server,{" "} - make doctor names it. -

-
- )} -
- -
-
+ ); } From c9a6d5328652d3c98f93fc9f2923777396bc8984 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 02:31:07 +0300 Subject: [PATCH 114/154] Record that routine definitions are install-wide, not per-user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SEC-617 asked whether one install's approved users share an agent workspace or are tenants. The code already answers it: GET /agents lists every agent unscoped, and _strategy_principal states that agents and their strategies are a single global store, "not partitioned by owner the way conversations and delegations are". So a routine's name, description, fields schema and source being readable by any approved user is the design, not a gap in it — and scoping the routine slice alone would invert the incoherence, leaving you unable to read the source of a routine belonging to an agent you can already list. Documented on both routes and in routine_store's module docstring, with the line to hold when editing them: a definition is public to the install, a run and its output belong to whoever made it. Also names list_routines' owner_id for what it is — a report-count scope, not a visibility filter — and notes that routine_source_roots() is path confinement rather than authorization. No behaviour change. --- condor/routine_store.py | 29 +++++++++++++++++++++++++++++ condor/web/routes/routines.py | 23 +++++++++++++++++++++-- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/condor/routine_store.py b/condor/routine_store.py index bcf3eaf63..f70e8ff86 100644 --- a/condor/routine_store.py +++ b/condor/routine_store.py @@ -2,6 +2,27 @@ Bridges Telegram handler and web API so both can see the same instances, schedule runs, and read results. + +**Routine definitions are install-wide, not per-user (SEC-617).** One Condor +install is a single shared agent workspace, not a set of tenants: every +approved user sees every agent (``GET /agents`` calls ``list_all()`` unscoped) +and ``_strategy_principal`` in ``web/routes/agents.py`` states it outright — +agents and their strategies "are a single global store, not partitioned by +owner the way conversations and delegations are". So a routine's *definition* — +its name, description, ``fields`` schema and source — is readable by any +approved user, including the agent-prefixed ones, and that is the design rather +than a gap in it. Scoping the routine slice alone would make the model +incoherent in the other direction: you could not read the source of a routine +belonging to an agent you can already list and inspect. + +What *is* per-user is the activity on top of those definitions, and that +partitioning has shipped: reports (SEC-196, SEC-593), conversations, sessions, +delegations, and the instances below, which ``routes/routines.py`` filters with +``_owns``. The line to hold when editing this module is therefore: a definition +is public to the install, a *run* and its output belong to whoever made it. +Promoting definitions to per-user is a tenancy decision for the whole agent +layer — ``GET /agents`` and the strategy routes first — not something to +retrofit here. """ from __future__ import annotations @@ -348,6 +369,14 @@ def _get_report_counts(self, owner_id: int | None = None) -> dict[str, int]: return {} def list_routines(self, owner_id: int | None = None) -> list[dict]: + """Every discovered routine, with the caller's report tally on each. + + ``owner_id`` scopes the ``report_count`` only — it is the report filter + of :meth:`_get_report_counts`, not a visibility filter on the rows. The + row set is deliberately the whole install's: see the module docstring + (SEC-617) for why definitions are install-wide while the runs counted + beside them are per-user. + """ all_routines = self._discover_all() report_counts = self._get_report_counts(owner_id) out = [] diff --git a/condor/web/routes/routines.py b/condor/web/routes/routines.py index 2def4f9aa..91fcffc30 100644 --- a/condor/web/routes/routines.py +++ b/condor/web/routes/routines.py @@ -133,7 +133,16 @@ def _authorized_instance(instance_id: str, user: WebUser) -> dict: @router.get("") async def list_routines(user: WebUser = Depends(get_current_user)): - """List all discovered routines with their fields.""" + """List all discovered routines with their fields. + + Deliberately every routine the install has, agent-prefixed ones included, + for any approved user (SEC-617). Routine *definitions* are install-wide + because the whole agent layer is: ``GET /agents`` lists them unscoped and + ``_strategy_principal`` in ``routes/agents.py`` records that agents and + strategies are one global store. The owner filter below reaches only the + ``report_count`` on each row — the per-user part of this response — exactly + as ``GET /reports`` scopes the same tally (SEC-593). + """ store = get_routine_store() return store.list_routines(owner_id=report_owner_filter(user)) @@ -350,7 +359,17 @@ async def get_routine_source( routine_name: str, user: WebUser = Depends(get_current_user), ): - """Return the source code of a routine.""" + """Return the source code of a routine. + + Readable by any approved user, for every agent's routines as well as the + general and ``_shared`` libraries (SEC-617). That follows from routine + definitions being install-wide — see ``list_routines`` above and the + ``routine_store`` module docstring — and not from the confinement below, + which answers a different question: ``routine_source_roots()`` is a *path* + allowlist stopping ``..`` and symlink escapes out of the dirs discovery + reads, never an authorization check. Agent homes stay out of those roots, + so a journal or memory store next door is not in scope either way. + """ store = get_routine_store() all_routines = store._discover_all() routine = all_routines.get(routine_name) From 0b2abd8a637904d3b201973266aed332a098e017 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 09:07:50 +0300 Subject: [PATCH 115/154] Refuse a routine write or run in dry-run, the same door one over from a snippet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SEC-616 refused `run_code` in a dry run on the principle that a tool holding the unrestricted API client must not auto-approve where nothing may mutate. It enforced that on exactly one door. `manage_routines` sits in COMMON_TOOLS, is mounted on the tick seat, and its `create_routine` writes a Python file that `run` then executes holding the same client — so a dry run could write `await client.gateway.start(...)` into a routine and run it for real. Rather than a second special case, the dry-run policy becomes one function. `dry_run_refusal` answers why a call must not auto-approve there, over the tools the gate deliberately does not name; the risk gate asks it once and a third such tool never has to reach that file. `manage_routines` gets the action split its siblings have, pinned against the registered Literal so a new action cannot default to allowed, and the writes it names are now recorded in the action log too: "wrote a routine and ran it" is exactly the sequence a reader needs. Reads stay free — list, describe, read_routine, get_instance, list_instances — so a rehearsal can still see what exists and what it does. `stop` does not: a dry run can reach neither `start` nor `run_async`, so the only instance it could stop belongs to a live seat, and killing that is a mutation, not a brake. Loop and attended modes are unchanged, and no confirmation is introduced. --- condor/agents/risk.py | 37 +++++----- condor/runtime/danger.py | 132 +++++++++++++++++++++++++++++++++--- tests/test_agent_actions.py | 86 +++++++++++++++++++++++ tests/test_risk_gate.py | 99 +++++++++++++++++++++++++++ 4 files changed, 327 insertions(+), 27 deletions(-) diff --git a/condor/agents/risk.py b/condor/agents/risk.py index 11adabad3..086802554 100644 --- a/condor/agents/risk.py +++ b/condor/agents/risk.py @@ -20,7 +20,7 @@ DANGEROUS_CLMM_ACTIONS, LEVERAGE_TOOL, LEVERAGED_EXECUTOR_TOOLS, - is_code_execution_call, + dry_run_refusal, is_dangerous_tool_call, tool_call_input, tool_call_name, @@ -856,23 +856,24 @@ async def callback(tool_call: dict, options: list[dict]) -> dict: "create_*_executor tool instead", ) - # A snippet is not in `DANGEROUS_TOOLS` and must not be — it is how a - # tick reads a market (ARCH-308), and gating it by name would put a - # confirmation in front of every candle read. But it runs arbitrary - # Python holding the unrestricted API client, so in dry-run it is the one - # remaining way to mutate the world for real: every *named* write above - # is refused there, and `await client.gateway.start(...)` inside a - # snippet was not (SEC-616). Dry-run's promise is that nothing mutates, - # so the tool that can mutate anything does not auto-approve there. - # Reading past runs (`history`, `get`) changes nothing and stays free. - if execution_mode == "dry_run" and is_code_execution_call(tool_call): - return deny( - tool_call_name(tool_call), - "this session runs in dry-run mode, where nothing mutates, and a " - "snippet holds the unrestricted API client — read what you need " - "with the read-only tools instead", - level=logging.INFO, - ) + # Neither `run_code` nor `manage_routines` is in `DANGEROUS_TOOLS`, and + # neither must be — a snippet is how a tick reads a market (ARCH-308) + # and a routine is ordinary tick work, so gating either by name would + # put a confirmation in front of every candle read. But both run + # arbitrary Python holding the unrestricted API client, which makes them + # the ways to mutate the world for real from inside a dry run: every + # *named* write above is refused there, `await client.gateway.start(...)` + # inside a snippet was not (SEC-616), and neither was writing that same + # line into a routine and running it (SEC-626). + # + # Which calls those are is `danger.py`'s policy and not this gate's, so + # a third such tool never has to reach this file. Reads on both tools — + # past runs, the routine list, a routine's source — change nothing and + # stay free. + if execution_mode == "dry_run": + refusal = dry_run_refusal(tool_call) + if refusal: + return deny(tool_call_name(tool_call), refusal, level=logging.INFO) # Auto-approve everything else for opt in options: diff --git a/condor/runtime/danger.py b/condor/runtime/danger.py index ced5d6f7e..410ab2e3e 100644 --- a/condor/runtime/danger.py +++ b/condor/runtime/danger.py @@ -253,6 +253,42 @@ #: already stored. MUTATING_CODE_RUN_ACTIONS = {"run"} READ_ONLY_CODE_RUN_ACTIONS = {"history", "get"} +#: The routine library. The *other* door onto arbitrary Python, and for the same +#: reason as ``run_code``: a routine is a Python file with an ``async run()``, +#: ``create_routine`` writes one and ``run`` executes it holding the same +#: unrestricted API client. Deliberately not in ``DANGEROUS_TOOLS`` either — +#: running a routine is ordinary tick work and a name gate would prompt a human +#: for every one of them (SEC-626). +ROUTINE_TOOL = "manage_routines" +#: ``manage_routines``' twelve actions split by what they touch. The writes are +#: the two halves of the same capability: ``create_routine`` / ``edit_routine`` +#: put Python on disk, ``run`` / ``run_async`` / ``start`` execute it, and +#: ``delete_routine`` takes it away again. +#: +#: ``stop`` is here rather than with the reads, which is the one place this set +#: parts company with the module's rule that a brake is never stood in front of. +#: A brake is a session stopping *its own* activity, and a dry run has none: it +#: cannot reach ``start`` or ``run_async``, so the only instance it could stop +#: belongs to a live seat, and killing that is a mutation of the shared runtime +#: rather than this session braking. +MUTATING_ROUTINE_ACTIONS = { + "run", + "run_async", + "start", + "stop", + "create_routine", + "edit_routine", + "delete_routine", +} +#: The routine library's reads: what exists, what it takes, what it says, and +#: what a run already produced. None of them execute anything. +READ_ONLY_ROUTINE_ACTIONS = { + "list", + "describe", + "read_routine", + "get_instance", + "list_instances", +} #: How much of a snippet's first line the log row carries. A summary is one line #: on a page, and the whole source is in the code-run store anyway. MAX_SNIPPET_HEAD_CHARS = 80 @@ -553,6 +589,63 @@ def is_code_execution_call(tool_call: dict[str, Any]) -> bool: ) +def is_mutating_routine_call(tool_call: dict[str, Any]) -> bool: + """Does this call *write or execute* a routine? (SEC-626) + + The sibling of :func:`is_code_execution_call` over the other door onto + arbitrary Python. A routine is a Python file, ``create_routine`` writes one + and ``run`` executes it holding the same unrestricted API client, so the two + tools have the same reach and get the same answer. + + Fails closed the same way, and for the same two reasons: an action this + module has not heard of is a new one, and a newly added write must not + default to allowed. + """ + if tool_call_name(tool_call) != ROUTINE_TOOL: + return False + return _is_mutating_action( + tool_call, MUTATING_ROUTINE_ACTIONS, READ_ONLY_ROUTINE_ACTIONS + ) + + +def dry_run_refusal(tool_call: dict[str, Any]) -> str | None: + """Why a dry run must not auto-approve this call, or ``None`` to let it by. + + Dry-run's promise is that nothing mutates, and the *gate* refuses every + named write there already. This is the rest of that promise: the tools the + gate deliberately does not name, because naming them would put a + confirmation in front of ordinary tick work, yet which can each mutate + anything at all once they run. + + Kept as one function returning one reason rather than a branch per tool in + the caller, because it is a policy and not a special case: SEC-616 refused + ``run_code`` here and SEC-626 found the identical hole one door over in + ``manage_routines``. A third such tool is a line in this function, and the + unattended gate does not have to learn about it. + + The refusal is per *action*, not per tool, so what a dry run needs in order + to rehearse at all — listing routines, reading their source and their config + schema, reading back a past run or a past snippet — stays free. Widening + that read-only half is how a dry run gets a new capability without getting + the ability to write. + """ + if is_code_execution_call(tool_call): + return ( + "this session runs in dry-run mode, where nothing mutates, and a " + "snippet holds the unrestricted API client — read what you need " + "with the read-only tools instead" + ) + + if is_mutating_routine_call(tool_call): + return ( + "this session runs in dry-run mode, where nothing mutates, and a " + "routine is Python holding the same unrestricted API client as a " + "snippet — 'list', 'describe' and 'read_routine' still work" + ) + + return None + + def is_recordable_tool_call(tool_call: dict[str, Any]) -> bool: """Should the action log keep a row for this call? (FEAT-102) @@ -562,14 +655,16 @@ def is_recordable_tool_call(tool_call: dict[str, Any]) -> bool: functions answering nearly the same question will drift, and this shape makes the gate's set structurally a subset that cannot fall behind. - Its extras are ``manage_controllers`` and ``run_code``. The gate excludes - both tools entirely and should keep excluding them — widening the gate would - put a new confirmation prompt in front of a running fleet, and in front of - every tick's market read — but a bot's controllers are *written* by exactly - these calls, so a log that drops them cannot say how a fleet was built or - which of its config writes were rejected, and a snippet can change anything - at all, so a log that drops it is silent about the one tool that can - (SEC-616). + Its extras are ``manage_controllers``, ``run_code`` and ``manage_routines``. + The gate excludes all three tools entirely and should keep excluding them — + widening the gate would put a new confirmation prompt in front of a running + fleet, and in front of every tick's market read — but a bot's controllers + are *written* by exactly these calls, so a log that drops them cannot say + how a fleet was built or which of its config writes were rejected, and a + snippet can change anything at all, so a log that drops it is silent about + the one tool that can (SEC-616). A routine is the same Python behind a + different door, so it is recorded on the same argument (SEC-626): "wrote a + routine and ran it" is precisely the sequence a reader needs to see. Fails open the same way its siblings do: an action this module has not heard of is recorded rather than dropped. @@ -582,7 +677,7 @@ def is_recordable_tool_call(tool_call: dict[str, Any]) -> bool: tool_call, MUTATING_CONTROLLER_ACTIONS, READ_ONLY_CONTROLLER_ACTIONS ) - return is_code_execution_call(tool_call) + return is_code_execution_call(tool_call) or is_mutating_routine_call(tool_call) def format_tool_summary(tool_call: dict[str, Any]) -> str: @@ -782,5 +877,24 @@ def format_tool_summary(tool_call: dict[str, Any]) -> str: what = f"Run snippet '{label}'" if label else "Run snippet" return f"{what}: {head}" if head else f"{what} (no code)" + if tool_name == ROUTINE_TOOL: + # Never gated either, so this line too is written for the log (SEC-626). + # It has to name the routine: a tick that writes one and runs it makes + # two calls that "manage_routines" twice over describes as nothing, and + # which library it landed in is half of what a routine *is* — an + # agent-local script and a shared one under the same name are different + # code. For `stop` and `get_instance` the name is an instance id, which + # is the right thing to print for those anyway. + action = input_data.get("action", "?") + name = str(input_data.get("name") or "").strip() or "?" + agent = input_data.get("agent") + if input_data.get("shared"): + where = " (shared)" + elif isinstance(agent, str) and agent: + where = f" ({agent})" + else: + where = "" + return f"Routine {action} '{name}'{where}" + # Generic fallback return tool_name diff --git a/tests/test_agent_actions.py b/tests/test_agent_actions.py index 7e21c4724..e620600f3 100644 --- a/tests/test_agent_actions.py +++ b/tests/test_agent_actions.py @@ -644,6 +644,92 @@ def test_the_snippet_action_sets_match_the_registered_tool(): assert not (MUTATING_CODE_RUN_ACTIONS & READ_ONLY_CODE_RUN_ACTIONS) +def folded_routine(**args): + """A folded ``manage_routines`` call. + + Its own ``name`` argument (the routine) collides with :func:`folded`'s first + positional (the tool), so the tool name is bound here instead. + """ + return { + "id": "tc_manage_routines", + "name": "manage_routines", + "status": "completed", + "kind": "mcp", + "input": args, + } + + +def test_a_tick_records_the_routines_it_writes_and_runs_and_not_the_ones_it_lists(): + """A routine is Python too, so "wrote one and ran it" has to leave a trace.""" + calls = [ + folded_routine(action="list"), + folded_routine(action="describe", name="scan"), + folded_routine(action="read_routine", name="scan"), + folded_routine(action="create_routine", name="pwn", code="x"), + folded_routine(action="run", name="pwn"), + ] + + actions = actions_from_tool_calls(calls, tick=3, at=1.0) + + assert [a.summary for a in actions] == [ + "Routine create_routine 'pwn'", + "Routine run 'pwn'", + ] + + +def test_a_routine_row_names_the_library_it_landed_in(): + """An agent-local routine and a shared one of the same name are different code.""" + calls = [ + folded_routine(action="edit_routine", name="scan", agent="brigado"), + folded_routine(action="create_routine", name="scan", shared=True), + ] + + assert [a.summary for a in actions_from_tool_calls(calls, tick=3, at=1.0)] == [ + "Routine edit_routine 'scan' (brigado)", + "Routine create_routine 'scan' (shared)", + ] + + +def test_the_routine_predicate_fails_open_on_an_action_it_cannot_read(): + from condor.runtime.danger import is_recordable_tool_call + + assert is_recordable_tool_call({"tool": "manage_routines", "input": None}) + assert is_recordable_tool_call({"tool": "manage_routines", "input": {"name": "x"}}) + assert is_recordable_tool_call( + {"tool": "manage_routines", "input": {"action": "publish_routine"}} + ) + + +def test_the_routine_tool_is_still_not_gated(): + """Criterion: no confirmation is introduced. Running a routine is tick work.""" + from condor.runtime.danger import DANGEROUS_TOOLS, is_dangerous_tool_call + + assert "manage_routines" not in DANGEROUS_TOOLS + for action in ("run", "start", "create_routine", "list", "something_new"): + call = {"tool": "manage_routines", "input": {"action": action, "name": "x"}} + assert is_dangerous_tool_call(call) is False, action + assert is_dangerous_tool_call({"tool": "manage_routines", "input": None}) is False + + +def test_the_routine_action_sets_match_the_registered_tool(): + """Every action literal the tool accepts is classified (SEC-626). + + Unclassified is *refused* in dry-run and recorded elsewhere, so the cost of + a drifting set is a rehearsal that stops working rather than a silent hole — + but a new action must be a deliberate choice either way. + """ + from condor.runtime.danger import ( + MUTATING_ROUTINE_ACTIONS, + READ_ONLY_ROUTINE_ACTIONS, + ) + from mcp_servers.condor.server import manage_routines + + fn = getattr(manage_routines, "fn", manage_routines) + literals = set(typing.get_args(fn.__annotations__["action"])) + assert MUTATING_ROUTINE_ACTIONS | READ_ONLY_ROUTINE_ACTIONS == literals + assert not (MUTATING_ROUTINE_ACTIONS & READ_ONLY_ROUTINE_ACTIONS) + + def test_the_controller_action_sets_match_the_registered_tool(): """Every action literal the tool accepts is classified, or the fail-open rule would silently record a read.""" diff --git a/tests/test_risk_gate.py b/tests/test_risk_gate.py index 64cb5ba09..8fb792105 100644 --- a/tests/test_risk_gate.py +++ b/tests/test_risk_gate.py @@ -804,3 +804,102 @@ def test_a_live_loop_still_runs_snippets_without_a_confirmation(): ) assert result["outcome"]["outcome"] == "selected" + + +# --------------------------------------------------------------------------- +# manage_routines is ungated by name for the same reason run_code is, and left +# the same hole one door over: a routine is Python, so a dry run could write one +# and execute it holding the unrestricted client (SEC-626). +# --------------------------------------------------------------------------- + + +def _routine_call(**args) -> dict: + return {"tool": "mcp__condor__manage_routines", "input": args} + + +@pytest.mark.parametrize( + "action", + ["run", "run_async", "start", "create_routine", "edit_routine", "delete_routine"], +) +def test_a_dry_run_cannot_write_or_execute_a_routine(action): + refusals = RefusalLog() + callback = auto_approve_with_risk_check( + RiskEngine(RiskLimits()), + RiskState(), + execution_mode="dry_run", + refusals=refusals, + ) + + result = asyncio.run( + callback( + _routine_call( + action=action, + name="pwn", + code="async def run(config, context):\n await client.gateway.start({})", + ), + _OPTIONS, + ) + ) + + assert result["outcome"]["outcome"] == "cancelled", action + (noted,) = refusals.drain() + assert noted["tool"] == "manage_routines" + assert "dry-run" in noted["reason"] + + +def test_a_dry_run_refuses_a_routine_stop_it_cannot_own(): + """It can reach neither `start` nor `run_async`, so the instance is a live seat's.""" + callback = auto_approve_with_risk_check( + RiskEngine(RiskLimits()), RiskState(), execution_mode="dry_run" + ) + + result = asyncio.run(callback(_routine_call(action="stop", name="ri_1"), _OPTIONS)) + + assert result["outcome"]["outcome"] == "cancelled" + + +def test_a_dry_run_refuses_a_routine_action_it_cannot_read(): + """Fails closed, so a newly added action cannot default to allowed.""" + callback = auto_approve_with_risk_check( + RiskEngine(RiskLimits()), RiskState(), execution_mode="dry_run" + ) + + for call in ( + {"tool": "manage_routines", "input": None}, + _routine_call(name="x"), + _routine_call(action=None, name="x"), + _routine_call(action="publish_routine", name="x"), + ): + result = asyncio.run(callback(call, _OPTIONS)) + assert result["outcome"]["outcome"] == "cancelled", call + + +@pytest.mark.parametrize( + "action", ["list", "describe", "read_routine", "get_instance", "list_instances"] +) +def test_a_dry_run_still_reads_the_routine_library(action): + """Rehearsal needs to see what exists and what it does; a read executes nothing.""" + callback = auto_approve_with_risk_check( + RiskEngine(RiskLimits()), RiskState(), execution_mode="dry_run" + ) + + result = asyncio.run(callback(_routine_call(action=action, name="scan"), _OPTIONS)) + + assert result["outcome"]["outcome"] == "selected", action + + +@pytest.mark.parametrize("mode", ["loop", "attended"]) +def test_other_modes_still_run_routines_without_a_confirmation(mode): + """The refusal is dry-run's, not a new gate: loop and attended are unchanged.""" + from condor.runtime.danger import DANGEROUS_TOOLS, is_dangerous_tool_call + + assert "manage_routines" not in DANGEROUS_TOOLS + callback = auto_approve_with_risk_check( + RiskEngine(RiskLimits()), RiskState(), execution_mode=mode + ) + + for action in ("run", "start", "create_routine", "list"): + call = _routine_call(action=action, name="scan") + assert is_dangerous_tool_call(call) is False, action + result = asyncio.run(callback(call, _OPTIONS)) + assert result["outcome"]["outcome"] == "selected", action From de03b0faff6102302d5eb87a818b7c63bb4dc551 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 09:07:53 +0300 Subject: [PATCH 116/154] Pin the clock tick test to a fixed hour, not the hour it happens to run in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `windowCutoff` is floored to the hour while the clock ticks every minute, so a bare tick moves the window on exactly one minute in sixty. The file installed its fake timers at the ambient wall clock, so once an hour it stood on that minute and asserted "a tick rebuilds nothing" against the one tick that legitimately rebuilds everything — red for ~61 s in every hour, and red for two or three consecutive suite runs when it hit, which is what made it read as a regression rather than as a clock. The timers now start at a fixed instant on minute 00, and the fixtures are measured from that instant rather than from `Date.now()` — they are module level, so left on the real clock they would have dated themselves before the pinned start. The crossing itself stops being untested: a fourth case sets the clock a minute short of the hour and asserts the tree *is* rebuilt, so the three "rebuilds nothing" cases cannot pass by the window having gone still. Verified by shifting the process clock and running the file at :00:00, :00:30, :30:00, :58:55, :59:05, :59:30 and :59:59 — green at all seven, where the old file was green at :30:00 and red at :59:30 under the same harness. --- .../perf/PerfBrowser.clockTick.test.tsx | 45 +++++++++++++++++-- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/perf/PerfBrowser.clockTick.test.tsx b/frontend/src/components/perf/PerfBrowser.clockTick.test.tsx index c50187e5f..78622ac60 100644 --- a/frontend/src/components/perf/PerfBrowser.clockTick.test.tsx +++ b/frontend/src/components/perf/PerfBrowser.clockTick.test.tsx @@ -94,6 +94,27 @@ declare global { /** The tick the clock is quantised to, and the window it used to drag with it. */ const CLOCK_TICK_MS = 60_000; const DAY_MS = 86_400_000; +const HOUR_MS = 3_600_000; + +/** + * The instant every clock in this file starts from, and why it is pinned. + * + * `windowCutoff` is floored to the *hour* while the clock ticks every *minute*, + * so a bare tick moves the cutoff — and with it `leavesFor`, `rawLeaves` and + * the tree — on exactly one minute in sixty: the one that crosses an hour + * boundary. Left on the ambient wall clock this file therefore asserted "a tick + * rebuilds nothing" while standing, once an hour, on the one tick that + * legitimately rebuilds everything, and failed for ~61 s in every hour. + * + * Landing on minute 00 leaves the whole hour between the start and the next + * boundary, so the `shouldAdvanceTime` drift below — real seconds, not minutes — + * cannot walk the test into that minute. The crossing itself is not left + * untested: `HOUR_EDGE` pins it deliberately. + */ +const NOW = Date.parse("2026-09-04T12:00:00Z"); + +/** A minute before the next hour: the one tick that *must* move the window. */ +const HOUR_EDGE = NOW + 59 * CLOCK_TICK_MS; function controller(over: Partial): ControllerInfo { return { @@ -111,7 +132,7 @@ function controller(over: Partial): ControllerInfo { volume_traded: 0, close_type_counts: {}, positions_summary: [], - deployed_at: new Date(Date.now() - 3 * 3_600_000).toISOString(), + deployed_at: new Date(NOW - 3 * HOUR_MS).toISOString(), config: {}, ...over, } as ControllerInfo; @@ -119,7 +140,7 @@ function controller(over: Partial): ControllerInfo { /** A closed executor that stopped `daysAgo` ago — the only thing the window cuts. */ function closed(id: string, daysAgo: number): ExecutorInfo { - const endedAt = Date.now() - daysAgo * DAY_MS; + const endedAt = NOW - daysAgo * DAY_MS; return { id, type: "position_executor", @@ -172,8 +193,9 @@ let root: Root; beforeEach(() => { // `shouldAdvanceTime` keeps the `setTimeout(0)` pumping below working while - // still letting the test drive the clock's own interval by hand. - vi.useFakeTimers({ shouldAdvanceTime: true }); + // still letting the test drive the clock's own interval by hand. `now` pins + // where in the hour that drift starts — see `NOW`. + vi.useFakeTimers({ shouldAdvanceTime: true, now: NOW }); globalThis.IS_REACT_ACT_ENVIRONMENT = true; Element.prototype.scrollIntoView = () => {}; container = document.createElement("div"); @@ -269,6 +291,21 @@ describe("a bare clock tick", () => { expect(spy.trees).toEqual([]); }); + + // The other three tests in this describe are only worth anything if a tick + // *can* rebuild the tree — otherwise they would still pass against a browser + // that had stopped tracking the window at all. This is the tick that must, + // and it is also the one this file used to land on by accident once an hour. + it("does rebuild when the tick crosses an hour, where the cutoff moves", async () => { + vi.setSystemTime(HOUR_EDGE); + await draw("/bots?population=terminated"); + spy.populations = 0; + spy.trees = []; + + await tick(); + + expect(spy.trees.length).toBeGreaterThan(0); + }); }); describe("the terminated window", () => { From dd9733e152bbbbc6340de536e2890a41a5d8d370 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 09:27:30 +0300 Subject: [PATCH 117/154] Hand every test the same lightweight-charts, at the resolver Every chart component resolves the library with a dynamic import(), so two mounts in one act() request it concurrently and Vitest's mocker answers a per-file vi.mock to one of them while raw-importing the real library for the other. A probe mounting two ExecutorCharts recorded 1 stub createChart call for 2 mounts and 7 real canvases: a real ChartWidget in a canvas-less jsdom, whose draw frame fires after teardown and exits the run 1 while every individual test still reports green. CORR-360 answered that in the one file it happened to bite. The other three chart tests held only by accident, each mounting once per act(); a second instance or a StrictMode wrapper re-broke them. So the substitution moves to test.alias, where it catches every resolution including the losing one. The four copy-pasted Proxy stubs collapse into one double that records what each chart was asked to draw, and a test declares the pixel-price scale its assertions read off. A test file can no longer reach the real library even by forgetting to mock it, and a new chart test inherits the double for free. lightweight-charts-double.test.tsx is the probe, inverted: delete the alias entry and it goes red in the open rather than intermittently, after teardown, somewhere else. --- .../charts/ExecutorChart.overlays.test.tsx | 68 ++---- .../trade/TradeChart.backfill.test.tsx | 78 ++---- .../components/trade/TradeChart.drag.test.tsx | 92 ++----- .../src/components/trade/TradeChart.test.tsx | 56 +---- .../test/lightweight-charts-double.test.tsx | 99 ++++++++ .../src/test/lightweight-charts-double.ts | 226 ++++++++++++++++++ frontend/vite.config.ts | 11 + 7 files changed, 402 insertions(+), 228 deletions(-) create mode 100644 frontend/src/test/lightweight-charts-double.test.tsx create mode 100644 frontend/src/test/lightweight-charts-double.ts diff --git a/frontend/src/components/charts/ExecutorChart.overlays.test.tsx b/frontend/src/components/charts/ExecutorChart.overlays.test.tsx index 960e6ddb6..cdef6854d 100644 --- a/frontend/src/components/charts/ExecutorChart.overlays.test.tsx +++ b/frontend/src/components/charts/ExecutorChart.overlays.test.tsx @@ -18,50 +18,13 @@ import { createRoot, type Root } from "react-dom/client"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { ExecutorInfo } from "@/lib/api"; +// The library itself is substituted at the resolver, for every test file at +// once (CORR-368) — `chartDouble` is only how this file reads back what the +// component drew. `lineSeriesAdded` counts the overlay churn under test: the +// candlestick series is created once at init and is not a line series. +import { chartDouble } from "@/test/lightweight-charts-double"; import { ExecutorChart } from "./ExecutorChart"; -const chartState = vi.hoisted(() => ({ - addSeries: 0, - removeSeries: 0, -})); - -vi.mock("lightweight-charts", () => { - /** Unknown members answer with a no-op spy; only the counters carry meaning. */ - const stub = (own: Record) => - new Proxy(own, { - get(target, prop) { - if (typeof prop !== "string" || prop === "then") return undefined; - if (!(prop in target)) target[prop] = vi.fn(); - return target[prop]; - }, - }); - - const LineSeries = { kind: "line" }; - const series = stub({ priceToCoordinate: vi.fn(() => 0), setData: vi.fn() }); - const timeScale = stub({ scrollPosition: vi.fn(() => 0) }); - const chart = stub({ - addSeries: vi.fn((kind: unknown) => { - // The candlestick series is created once at init; only the overlay line - // series are the churn under test. - if (kind === LineSeries) chartState.addSeries += 1; - return series; - }), - removeSeries: vi.fn(() => { - chartState.removeSeries += 1; - }), - timeScale: vi.fn(() => timeScale), - }); - - return { - createChart: vi.fn(() => chart), - CandlestickSeries: { kind: "candlestick" }, - LineSeries, - ColorType: { Solid: "solid" }, - CrosshairMode: { Normal: 0 }, - LineStyle: { Solid: 0, Dotted: 1, Dashed: 2 }, - }; -}); - vi.mock("@/hooks/useRates", () => ({ useRates: () => ({ formatPnlValue: (v: number) => String(v), @@ -133,8 +96,7 @@ describe("ExecutorChart overlay series", () => { beforeEach(() => { globalThis.IS_REACT_ACT_ENVIRONMENT = true; queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); - chartState.addSeries = 0; - chartState.removeSeries = 0; + chartDouble.reset(); container = document.createElement("div"); document.body.appendChild(container); root = createRoot(container); @@ -149,7 +111,7 @@ describe("ExecutorChart overlay series", () => { const executor = closedExecutor(); await render([executor]); - const drawn = chartState.addSeries; + const drawn = chartDouble.lineSeriesAdded; expect(drawn).toBeGreaterThan(0); // What the detail panel does on every mousemove of a divider drag, and what @@ -158,29 +120,29 @@ describe("ExecutorChart overlay series", () => { await render([{ ...executor }]); await render([{ ...executor }]); - expect(chartState.addSeries).toBe(drawn); - expect(chartState.removeSeries).toBe(0); + expect(chartDouble.lineSeriesAdded).toBe(drawn); + expect(chartDouble.seriesRemoved).toBe(0); }); it("redraws when the segment the executor draws actually moves", async () => { const executor = closedExecutor(); await render([executor]); - const drawn = chartState.addSeries; + const drawn = chartDouble.lineSeriesAdded; await render([{ ...executor, custom_info: { close_price: 191 } }]); - expect(chartState.addSeries).toBe(drawn * 2); - expect(chartState.removeSeries).toBe(drawn); + expect(chartDouble.lineSeriesAdded).toBe(drawn * 2); + expect(chartDouble.seriesRemoved).toBe(drawn); }); it("redraws when an executor joins the group", async () => { const executor = closedExecutor(); await render([executor]); - const drawn = chartState.addSeries; + const drawn = chartDouble.lineSeriesAdded; await render([executor, closedExecutor({ id: "exec-2", entry_price: 170 })]); - expect(chartState.addSeries).toBeGreaterThan(drawn); - expect(chartState.removeSeries).toBe(drawn); + expect(chartDouble.lineSeriesAdded).toBeGreaterThan(drawn); + expect(chartDouble.seriesRemoved).toBe(drawn); }); }); diff --git a/frontend/src/components/trade/TradeChart.backfill.test.tsx b/frontend/src/components/trade/TradeChart.backfill.test.tsx index e5549492d..6e3772f38 100644 --- a/frontend/src/components/trade/TradeChart.backfill.test.tsx +++ b/frontend/src/components/trade/TradeChart.backfill.test.tsx @@ -11,28 +11,28 @@ * the effect's six dependencies, and no refetch when the props do not change. * * The same double mount fires the chart-init effect twice in one tick, so - * `import("lightweight-charts")` is requested twice concurrently — and Vitest's - * mocker answers the stub to the first request and raw-imports the real library - * for the second. It is the concurrency, not StrictMode: two plain mounts in one - * `act()` split the same way, and neither a static nor an awaited warm-up import - * of the module changes it. The real `ChartWidget` then builds against a jsdom - * that has no 2D canvas and schedules a draw frame, which fired after teardown, - * outside every test, and exited the whole suite 1 with "2 unhandled errors" - * while every test reported green (CORR-360). + * `import("lightweight-charts")` is requested twice concurrently — and a + * per-file `vi.mock` answered the stub to the first request and raw-imported + * the real library for the second. It is the concurrency, not StrictMode: two + * plain mounts in one `act()` split the same way, and neither a static nor an + * awaited warm-up import of the module changes it. The real `ChartWidget` then + * builds against a jsdom that has no 2D canvas and schedules a draw frame, + * which fired after teardown, outside every test, and exited the whole suite 1 + * with "2 unhandled errors" while every test reported green (CORR-360). * - * So this file owns the two things the chart library needs and jsdom does not - * have: the frame queue — nothing runs unless we run it, and teardown proves no - * frame outlived the tree — and a 2D context. Whichever module answers the - * import, it can no longer leave anything behind. + * The library is now substituted at the resolver, so both requests get the same + * double and no real widget can be built at all (CORR-368). This file keeps its + * own frame queue and 2D context anyway: they are what proves the StrictMode + * mount leaves nothing behind, and they cost nothing. * * @vitest-environment jsdom */ -import * as charts from "lightweight-charts"; import { act, StrictMode, type ComponentProps } from "react"; import { createRoot, type Root } from "react-dom/client"; import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from "vitest"; +import { chartDouble, createChart } from "@/test/lightweight-charts-double"; import { TradeChart } from "./TradeChart"; const CANDLES = [ @@ -49,37 +49,6 @@ const apiState = vi.hoisted(() => ({ getCandles: vi.fn(), })); -vi.mock("lightweight-charts", () => { - /** Only the shape the component walks matters here; the rest answers no-ops. */ - const stub = (own: Record) => - new Proxy(own, { - get(target, prop) { - if (typeof prop !== "string" || prop === "then") return undefined; - if (!(prop in target)) target[prop] = vi.fn(); - return target[prop]; - }, - }); - - const series = stub({ - coordinateToPrice: vi.fn(() => 100), - priceToCoordinate: vi.fn(() => 0), - createPriceLine: vi.fn(() => ({})), - }); - const chart = stub({ - addSeries: vi.fn(() => series), - timeScale: vi.fn(() => stub({})), - }); - - return { - createChart: vi.fn(() => chart), - CandlestickSeries: {}, - LineSeries: {}, - ColorType: { Solid: "solid" }, - CrosshairMode: { Normal: 0 }, - LineStyle: { Solid: 0, Dotted: 1, Dashed: 2 }, - }; -}); - vi.mock("@/hooks/useCandleStore", () => ({ useCandleStore: () => ({ candles: [], @@ -109,8 +78,8 @@ declare global { var IS_REACT_ACT_ENVIRONMENT: boolean; } -/** The stub's factory, read back to prove the component got the stub. */ -const createChartStub = charts.createChart as unknown as Mock; +/** The double's factory, read back to prove the component reached the chart. */ +const createChartStub = createChart as unknown as Mock; /** Frames the render scheduled, so teardown can prove none outlive the tree. */ const pendingFrames = new Map(); @@ -165,14 +134,12 @@ async function render(strict: boolean, extra: Partial { await new Promise((resolve) => setTimeout(resolve, 0)); }); - // A single mount asks for the chart library once, so the stub is what must - // have answered — the module identity is a result here, not an assumption. - // The StrictMode path asks twice at once, which Vitest's mocker cannot serve - // (see the docblock); the doubles above are what make that harmless. - if (!strict) { - expect(createChartStub).toHaveBeenCalled(); - expect(container.querySelector("canvas")).toBeNull(); - } + // Both paths ask for the chart library — the StrictMode one twice at once — + // and the resolver hands the double to every request, so the component got a + // chart and jsdom got no canvas. The module identity is a result here, not an + // assumption, and it holds on the concurrent path too. + expect(createChartStub).toHaveBeenCalled(); + expect(container.querySelector("canvas")).toBeNull(); } beforeEach(() => { @@ -197,7 +164,8 @@ beforeEach(() => { vi.stubGlobal("cancelAnimationFrame", (id: number) => { pendingFrames.delete(id); }); - createChartStub.mockClear(); + // Discards the charts the last test built and reinstates the price scale. + chartDouble.reset({ toPrice: () => 100 }); store.mergeCandles.mockClear(); store.setDuration.mockClear(); apiState.getCandles.mockReset(); diff --git a/frontend/src/components/trade/TradeChart.drag.test.tsx b/frontend/src/components/trade/TradeChart.drag.test.tsx index 2ac17937e..61bc5d177 100644 --- a/frontend/src/components/trade/TradeChart.drag.test.tsx +++ b/frontend/src/components/trade/TradeChart.drag.test.tsx @@ -16,6 +16,11 @@ import { createRoot, type Root } from "react-dom/client"; import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from "vitest"; import type { PickSlot } from "@/components/executor/types"; +// The library itself is substituted at the resolver, for every test file at +// once (CORR-368); this file only declares the invertible price scale the drag +// is measured on and reads back the primitives, options and crosshair handler +// the component installed. +import { chartDouble } from "@/test/lightweight-charts-double"; import { TradeChart } from "./TradeChart"; /** A flat, invertible scale: price 700 sits at row 300, price 800 at row 200. */ @@ -30,60 +35,6 @@ const START_PRICE = 700; // row 300 const END_PRICE = 800; // row 200 const LIMIT_PRICE = 600; // row 400 -const chartState = vi.hoisted(() => ({ - series: null as unknown, - options: {} as Record, - primitives: [] as unknown[], - crosshairCb: null as ((param: unknown) => void) | null, -})); - -vi.mock("lightweight-charts", () => { - const stub = (own: Record) => - new Proxy(own, { - get(target, prop) { - if (typeof prop !== "string" || prop === "then") return undefined; - if (!(prop in target)) target[prop] = vi.fn(); - return target[prop]; - }, - }); - - const series = stub({ - coordinateToPrice: vi.fn((y: number) => 1000 - y), - priceToCoordinate: vi.fn((price: number) => 1000 - price), - createPriceLine: vi.fn(() => ({})), - attachPrimitive: vi.fn((p: unknown) => { - chartState.primitives.push(p); - (p as { attached?: (a: unknown) => void }).attached?.({ series }); - }), - detachPrimitive: vi.fn((p: unknown) => { - chartState.primitives = chartState.primitives.filter((x) => x !== p); - (p as { detached?: () => void }).detached?.(); - }), - }); - chartState.series = series; - - const timeScale = stub({}); - const chart = stub({ - addSeries: vi.fn(() => series), - timeScale: vi.fn(() => timeScale), - subscribeCrosshairMove: vi.fn((cb: (param: unknown) => void) => { - chartState.crosshairCb = cb; - }), - applyOptions: vi.fn((opts: Record) => { - Object.assign(chartState.options, opts); - }), - }); - - return { - createChart: vi.fn(() => chart), - CandlestickSeries: {}, - LineSeries: {}, - ColorType: { Solid: "solid" }, - CrosshairMode: { Normal: 0 }, - LineStyle: { Solid: 0, Dotted: 1, Dashed: 2 }, - }; -}); - vi.mock("@/hooks/useCandleStore", () => ({ useCandleStore: () => ({ candles: [], @@ -175,7 +126,7 @@ async function drag(fromY: number, ...throughY: number[]) { /** Park the crosshair on a pixel row, which is what the click branches read. */ async function hoverAt(y: number) { await act(async () => { - chartState.crosshairCb?.({ + chartDouble.crosshairCb?.({ point: { x: 300, y }, seriesData: new Map(), time: 1_700_000_000, @@ -197,9 +148,7 @@ beforeEach(() => { unobserve() {} disconnect() {} } as unknown as typeof ResizeObserver; - chartState.options = {}; - chartState.primitives = []; - chartState.crosshairCb = null; + chartDouble.reset({ toPrice: priceAtY, toCoordinate: yAtPrice }); onPriceSet = vi.fn<(field: PickSlot, price: number) => void>(); container = document.createElement("div"); document.body.appendChild(container); @@ -305,10 +254,10 @@ describe("TradeChart price-line drag", () => { fire("pointerdown", yAtPrice(START_PRICE)); fire("pointermove", 340); }); - expect(chartState.options).toMatchObject({ handleScroll: false, handleScale: false }); + expect(chartDouble.options).toMatchObject({ handleScroll: false, handleScale: false }); await act(async () => { fire("pointerup", 340); }); - expect(chartState.options).toMatchObject({ handleScroll: true, handleScale: true }); + expect(chartDouble.options).toMatchObject({ handleScroll: true, handleScale: true }); }); it("takes the crosshair down for the length of the drag", async () => { @@ -319,10 +268,10 @@ describe("TradeChart price-line drag", () => { await act(async () => { fire("pointerdown", yAtPrice(START_PRICE)); }); // A press that has not travelled is still a candidate click, and keeps it. - expect(chartState.options.crosshair).toBeUndefined(); + expect(chartDouble.options.crosshair).toBeUndefined(); await act(async () => { fire("pointermove", 340); }); - expect(chartState.options).toMatchObject({ + expect(chartDouble.options).toMatchObject({ crosshair: { horzLine: { visible: false, labelVisible: false }, vertLine: { visible: false, labelVisible: false }, @@ -330,7 +279,7 @@ describe("TradeChart price-line drag", () => { }); await act(async () => { fire("pointerup", 340); }); - expect(chartState.options).toMatchObject({ + expect(chartDouble.options).toMatchObject({ crosshair: { horzLine: { visible: true, labelVisible: true }, vertLine: { visible: true, labelVisible: true }, @@ -346,7 +295,7 @@ describe("TradeChart price-line drag", () => { fire("pointercancel", 340); }); - expect(chartState.options).toMatchObject({ + expect(chartDouble.options).toMatchObject({ crosshair: { horzLine: { visible: true }, vertLine: { visible: true } }, }); }); @@ -357,10 +306,10 @@ describe("TradeChart price-line drag", () => { fire("pointerdown", yAtPrice(START_PRICE)); fire("pointermove", 340); }); - expect(chartState.options).toMatchObject({ handleScroll: false }); + expect(chartDouble.options).toMatchObject({ handleScroll: false }); await act(async () => { root.unmount(); }); - expect(chartState.options).toMatchObject({ handleScroll: true, handleScale: true }); + expect(chartDouble.options).toMatchObject({ handleScroll: true, handleScale: true }); // The afterEach unmount must stay harmless. root = createRoot(container); @@ -389,7 +338,7 @@ describe("TradeChart price-line drag", () => { expect(onPriceSet).toHaveBeenCalledTimes(1); expect(onPriceSet).toHaveBeenCalledWith("end", START_PRICE); // And the press handed panning straight back. - expect(chartState.options).toMatchObject({ handleScroll: true, handleScale: true }); + expect(chartDouble.options).toMatchObject({ handleScroll: true, handleScale: true }); }); it("writes nothing for a wiggle that stays inside the click slop", async () => { @@ -437,7 +386,7 @@ describe("TradeChart price-line drag", () => { describe("TradeChart drag hover cursor", () => { it("asks for a resize cursor over a draggable line and nothing elsewhere", async () => { await render(); - const primitive = chartState.primitives[0] as { + const primitive = chartDouble.primitives[0] as { hitTest?: (x: number, y: number) => { cursorStyle?: string; externalId: string } | null; }; expect(primitive).toBeTruthy(); @@ -459,16 +408,17 @@ describe("TradeChart drag hover cursor", () => { */ describe("TradeChart price-line styling", () => { function priceLineOpts(): Record[] { - const series = chartState.series as { createPriceLine: Mock }; + const series = chartDouble.series as { createPriceLine: Mock }; return series.createPriceLine.mock.calls.map(([opts]) => opts as Record); } it("draws both range bounds solid and the limit dotted", async () => { - (chartState.series as { createPriceLine: Mock }).createPriceLine.mockClear(); + // No `mockClear()` first: `chartDouble.reset()` gave this test its own + // chart, so every recorded price line is one this render drew. await render({ lineLabels: { start: "Lower", end: "Upper", limit: "Lower limit" } }); const byTitle = new Map(priceLineOpts().map((opts) => [opts.title, opts])); - // The mocked module numbers the styles Solid: 0, Dotted: 1, Dashed: 2. + // The double numbers the styles Solid: 0, Dotted: 1, Dashed: 2. expect(byTitle.get("Lower")?.lineStyle).toBe(0); expect(byTitle.get("Upper")?.lineStyle).toBe(0); expect(byTitle.get("Lower limit")?.lineStyle).toBe(1); diff --git a/frontend/src/components/trade/TradeChart.test.tsx b/frontend/src/components/trade/TradeChart.test.tsx index c4265efea..cafb8a2cf 100644 --- a/frontend/src/components/trade/TradeChart.test.tsx +++ b/frontend/src/components/trade/TradeChart.test.tsx @@ -17,6 +17,10 @@ import { createRoot, type Root } from "react-dom/client"; import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from "vitest"; import type { PickSlot } from "@/components/executor/types"; +// The library itself is substituted at the resolver, for every test file at +// once (CORR-368); this file only declares the price scale its clicks are read +// off and reads back the crosshair handler the component subscribed. +import { chartDouble } from "@/test/lightweight-charts-double"; import { TradeChart } from "./TradeChart"; /** Pixel row → price, steep enough that two clicks 60px apart cannot collide. */ @@ -25,52 +29,6 @@ function priceAtY(y: number): number { return PRICE_AT_Y_200 - (y - 200) * 10; } -const chartState = vi.hoisted(() => ({ - series: null as unknown, - crosshairCb: null as ((param: unknown) => void) | null, -})); - -vi.mock("lightweight-charts", () => { - /** - * The chart pieces the component reaches for are many and mostly irrelevant - * here, so unknown members answer with a no-op spy; only the price mapping and - * the crosshair subscription carry real behaviour. - */ - const stub = (own: Record) => - new Proxy(own, { - get(target, prop) { - if (typeof prop !== "string" || prop === "then") return undefined; - if (!(prop in target)) target[prop] = vi.fn(); - return target[prop]; - }, - }); - - const series = stub({ - coordinateToPrice: vi.fn((y: number) => priceAtY(y)), - priceToCoordinate: vi.fn(() => 0), - createPriceLine: vi.fn(() => ({})), - }); - chartState.series = series; - - const timeScale = stub({}); - const chart = stub({ - addSeries: vi.fn(() => series), - timeScale: vi.fn(() => timeScale), - subscribeCrosshairMove: vi.fn((cb: (param: unknown) => void) => { - chartState.crosshairCb = cb; - }), - }); - - return { - createChart: vi.fn(() => chart), - CandlestickSeries: {}, - LineSeries: {}, - ColorType: { Solid: "solid" }, - CrosshairMode: { Normal: 0 }, - LineStyle: { Solid: 0, Dotted: 1, Dashed: 2 }, - }; -}); - vi.mock("@/hooks/useCandleStore", () => ({ useCandleStore: () => ({ candles: [], @@ -136,9 +94,9 @@ async function render( /** Move the crosshair to a pixel row, optionally over a candle. */ async function moveTo(y: number, overBar = true) { const seriesData = new Map(); - if (overBar) seriesData.set(chartState.series, HOVERED_BAR); + if (overBar) seriesData.set(chartDouble.series, HOVERED_BAR); await act(async () => { - chartState.crosshairCb?.({ + chartDouble.crosshairCb?.({ point: { x: 300, y }, seriesData, time: overBar ? 1_700_000_000 : undefined, @@ -194,7 +152,7 @@ beforeEach(() => { unobserve() {} disconnect() {} } as unknown as typeof ResizeObserver; - chartState.crosshairCb = null; + chartDouble.reset({ toPrice: priceAtY }); onPriceSet = vi.fn<(field: PickSlot, price: number) => void>(); container = document.createElement("div"); document.body.appendChild(container); diff --git a/frontend/src/test/lightweight-charts-double.test.tsx b/frontend/src/test/lightweight-charts-double.test.tsx new file mode 100644 index 000000000..671b0f3f7 --- /dev/null +++ b/frontend/src/test/lightweight-charts-double.test.tsx @@ -0,0 +1,99 @@ +/** + * No test can reach the real `lightweight-charts` (CORR-368). + * + * This file is the guard on the `test.alias` in `vite.config.ts`. Delete that + * entry and both tests below go red immediately, in the open — rather than the + * suite starting to fail intermittently, after teardown, in whichever chart + * test happens to mount twice in one tick. + * + * The second test is the exact probe that reproduced the bug: before the alias + * it recorded 1 stub `createChart` call for 2 mounts and 7 real `` + * elements, because the second, concurrent `import("lightweight-charts")` was + * answered by the real library. + * + * @vitest-environment jsdom + */ + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ExecutorChart } from "@/components/charts/ExecutorChart"; +import { chartDouble, createChart } from "@/test/lightweight-charts-double"; + +// Deliberately NOT mocked here: the point is that the resolver already did it. +vi.mock("@/hooks/useRates", () => ({ + useRates: () => ({ + formatPnlValue: (v: number) => String(v), + formatValue: (v: number) => String(v), + }), +})); + +vi.mock("@/lib/api", () => ({ + api: { getCandles: vi.fn(async () => []) }, +})); + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +let containers: HTMLDivElement[]; +let roots: Root[]; +let queryClient: QueryClient; + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + globalThis.ResizeObserver = class { + observe() {} + unobserve() {} + disconnect() {} + } as unknown as typeof ResizeObserver; + queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + chartDouble.reset(); + containers = [0, 1].map(() => { + const el = document.createElement("div"); + document.body.appendChild(el); + return el; + }); + roots = containers.map((el) => createRoot(el)); +}); + +afterEach(() => { + act(() => roots.forEach((root) => root.unmount())); + containers.forEach((el) => el.remove()); +}); + +describe("the lightweight-charts double", () => { + it("is what the specifier resolves to, with no vi.mock in this file", async () => { + const resolved = await import("lightweight-charts"); + + expect(resolved.createChart as unknown).toBe(createChart); + }); + + it("answers both of two chart mounts made in one act()", async () => { + await act(async () => { + for (const root of roots) { + root.render( + + + , + ); + } + }); + // The chart module is imported dynamically; let both promises land. + await act(async () => { + await Promise.resolve(); + }); + + expect(createChart).toHaveBeenCalledTimes(2); + expect(chartDouble.charts).toHaveLength(2); + // A real chart widget paints into canvases the double never creates. + expect(containers.reduce((n, el) => n + el.querySelectorAll("canvas").length, 0)).toBe(0); + }); +}); diff --git a/frontend/src/test/lightweight-charts-double.ts b/frontend/src/test/lightweight-charts-double.ts new file mode 100644 index 000000000..1ccbd8e0b --- /dev/null +++ b/frontend/src/test/lightweight-charts-double.ts @@ -0,0 +1,226 @@ +/** + * The only `lightweight-charts` any test ever gets (CORR-368). + * + * Every runtime resolution of the library in `src/` is a *dynamic* `import()` + * (`TradeChart`, `ExecutorChart`, `AgentPnlChart`). When two chart mounts land + * in one `act()` — a `StrictMode` double mount, a detail panel beside a fleet + * chart, two roots in one test — the module is requested twice concurrently, + * and Vitest's mocker answers a per-file `vi.mock` stub to the first request + * while raw-importing the real library for the second. The real `ChartWidget` + * then builds against a jsdom that has no 2D canvas and schedules a draw frame + * that fires after teardown, outside every test: the run exits 1 with unhandled + * errors while every individual test reports green (CORR-360 fixed one file + * defensively; this replaces that whole class of accident). + * + * So the substitution happens at the resolver instead: `vite.config.ts` maps + * `lightweight-charts` to this file for the whole test run, so *every* + * resolution — including the losing concurrent one — lands here. A test file + * cannot reach the real library even by forgetting to mock it, and a new chart + * test gets the double for free. `lightweight-charts-double.test.tsx` holds + * that wiring to its promise. + * + * Tests reach the recorded state through `chartDouble`, importing it by path + * (`@/test/lightweight-charts-double`) so TypeScript still checks the + * production code against the library's real types. + */ + +import { vi } from "vitest"; + +type Dict = Record; + +/** + * Unknown members answer with a no-op spy, so a chart can walk any shape the + * component reaches for; only the members below carry real behaviour. `then` + * is excluded because these objects travel through promise chains and a + * callable `then` would make one look thenable. + */ +function stub(own: Dict): Dict { + return new Proxy(own, { + get(target, prop) { + if (typeof prop !== "string" || prop === "then") return undefined; + if (!(prop in target)) target[prop] = vi.fn(); + return target[prop]; + }, + }); +} + +/** + * The pixel↔price mapping a test wants. Charts are canvas paint with no + * geometry in jsdom, so the double invents one and each test declares the scale + * its assertions need via `chartDouble.reset()`. + */ +export interface PriceScale { + /** Pixel row → price, what `series.coordinateToPrice` answers. */ + toPrice: (y: number) => number; + /** Price → pixel row, what `series.priceToCoordinate` answers. */ + toCoordinate: (price: number) => number; +} + +const FLAT_SCALE: PriceScale = { toPrice: () => 0, toCoordinate: () => 0 }; +let scale: PriceScale = { ...FLAT_SCALE }; + +/** One created chart and everything the component did to it. */ +interface ChartRecord { + /** The element `createChart` was handed. */ + container: unknown; + /** The options `createChart` was handed — *not* merged into `options`. */ + initialOptions: Dict; + chart: Dict; + /** One series object shared by every `addSeries` call on this chart. */ + series: Dict; + timeScale: Dict; + /** The series-type sentinel of each `addSeries` call, in order. */ + addedSeries: unknown[]; + removedSeries: unknown[]; + /** Everything `chart.applyOptions` has merged in, latest write winning. */ + options: Dict; + /** Primitives currently attached to the series. */ + primitives: unknown[]; + crosshairHandlers: ((param: unknown) => void)[]; +} + +const charts: ChartRecord[] = []; + +const last = (): ChartRecord | undefined => charts[charts.length - 1]; + +// ── The module surface the components import ──────────────────────────────── + +/** Series-type sentinels: identity is all the components and the double use. */ +export const AreaSeries = { seriesType: "Area" }; +export const BarSeries = { seriesType: "Bar" }; +export const BaselineSeries = { seriesType: "Baseline" }; +export const CandlestickSeries = { seriesType: "Candlestick" }; +export const HistogramSeries = { seriesType: "Histogram" }; +export const LineSeries = { seriesType: "Line" }; + +export const ColorType = { Solid: "solid", VerticalGradient: "gradient" }; +export const CrosshairMode = { Normal: 0, Magnet: 1, Hidden: 2 }; +export const LineStyle = { + Solid: 0, + Dotted: 1, + Dashed: 2, + LargeDashed: 3, + SparseDotted: 4, +}; +export const LineType = { Simple: 0, WithSteps: 1, Curved: 2 }; +export const PriceScaleMode = { Normal: 0, Logarithmic: 1 }; +export const TickMarkType = { + Year: 0, + Month: 1, + DayOfMonth: 2, + Time: 3, + TimeWithSeconds: 4, +}; + +export const createChart = vi.fn((container: unknown, initialOptions: Dict = {}) => { + const record: ChartRecord = { + container, + initialOptions, + chart: {}, + series: {}, + timeScale: {}, + addedSeries: [], + removedSeries: [], + options: {}, + primitives: [], + crosshairHandlers: [], + }; + + const series = stub({ + coordinateToPrice: vi.fn((y: number) => scale.toPrice(y)), + priceToCoordinate: vi.fn((price: number) => scale.toCoordinate(price)), + createPriceLine: vi.fn(() => ({})), + attachPrimitive: vi.fn((primitive: unknown) => { + record.primitives.push(primitive); + (primitive as { attached?: (param: unknown) => void }).attached?.({ series }); + }), + detachPrimitive: vi.fn((primitive: unknown) => { + record.primitives = record.primitives.filter((p) => p !== primitive); + (primitive as { detached?: () => void }).detached?.(); + }), + }); + + const timeScale = stub({ scrollPosition: vi.fn(() => 0) }); + + const chart = stub({ + // Real charts mint a series per call; one shared object per chart is enough + // here and lets a test read `chartDouble.series` without caring which of a + // component's several series it got. + addSeries: vi.fn((seriesType: unknown) => { + record.addedSeries.push(seriesType); + return series; + }), + removeSeries: vi.fn((removed: unknown) => { + record.removedSeries.push(removed); + }), + timeScale: vi.fn(() => timeScale), + applyOptions: vi.fn((options: Dict) => { + Object.assign(record.options, options); + }), + subscribeCrosshairMove: vi.fn((cb: (param: unknown) => void) => { + record.crosshairHandlers.push(cb); + }), + unsubscribeCrosshairMove: vi.fn((cb: (param: unknown) => void) => { + record.crosshairHandlers = record.crosshairHandlers.filter((h) => h !== cb); + }), + }); + + record.chart = chart; + record.series = series; + record.timeScale = timeScale; + charts.push(record); + return chart; +}); + +// ── The handle tests read ─────────────────────────────────────────────────── + +export const chartDouble = { + /** Every chart created since the last `reset()`, in creation order. */ + get charts(): readonly ChartRecord[] { + return charts; + }, + /** The most recently created chart, or `null` before anything mounted. */ + get chart(): Dict | null { + return last()?.chart ?? null; + }, + /** The most recent chart's series — the object its `addSeries` handed back. */ + get series(): Dict | null { + return last()?.series ?? null; + }, + get timeScale(): Dict | null { + return last()?.timeScale ?? null; + }, + /** What the most recent chart's `applyOptions` calls have merged in. */ + get options(): Dict { + return last()?.options ?? {}; + }, + /** Primitives currently attached to the most recent chart's series. */ + get primitives(): readonly unknown[] { + return last()?.primitives ?? []; + }, + /** The last crosshair handler the most recent chart subscribed. */ + get crosshairCb(): ((param: unknown) => void) | null { + return last()?.crosshairHandlers.at(-1) ?? null; + }, + /** Line series added across every chart — the overlay churn under test. */ + get lineSeriesAdded(): number { + return charts.reduce( + (n, c) => n + c.addedSeries.filter((type) => type === LineSeries).length, + 0, + ); + }, + /** Series removed across every chart. */ + get seriesRemoved(): number { + return charts.reduce((n, c) => n + c.removedSeries.length, 0); + }, + + /** + * Forget every chart and reinstate the price scale. Call it in `beforeEach`, + * passing the mapping this file's assertions read prices off. + */ + reset(priceScale: Partial = {}): void { + charts.length = 0; + createChart.mockClear(); + scale = { ...FLAT_SCALE, ...priceScale }; + }, +}; diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 395795308..2aa0c253a 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -21,6 +21,17 @@ export default defineConfig({ // components/ui/AnchoredMenu.test.tsx). environment: "node", include: ["src/**/*.test.{ts,tsx}"], + // `lightweight-charts` is resolved dynamically by every chart component, so + // two mounts in one `act()` request it concurrently and Vitest's mocker + // answers a per-file `vi.mock` to one of them and raw-imports the real + // library for the other — a real chart widget in a canvas-less jsdom, whose + // draw frame fires after teardown and exits the run 1 while every test + // reports green (CORR-360, CORR-368). Substituting at the resolver instead + // of per file catches every resolution, including the losing one, so no + // test can reach the real library even by forgetting to mock it. + alias: { + "lightweight-charts": path.resolve(__dirname, "./src/test/lightweight-charts-double.ts"), + }, }, server: { proxy: { From e49f174aaa97a36a24c9c0b0ce18c620fe65d6e2 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 09:29:24 +0300 Subject: [PATCH 118/154] Give a rehearsal a candle it can read without holding the client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ARCH-308 deleted the candle, order book and funding tools because each answered in a rendered table, so every number taken out of one had to be fetched again through run_code to be computed on. That left client.market_data.* inside a snippet as the only structured market read. SEC-616 then refused run_code in dry-run, because a tool holding the unrestricted API client must not auto-approve where nothing may mutate, and SEC-626 refused manage_routines one door over for the same reason. Correct each time, and between them they left a dry run with no structured candle read at all — unable to rehearse the very decision the five shared playbooks teach, which is the whole point of one. get_market_data is the read-only path back. It answers in rows — {timestamp, open, high, low, close, volume} floats, the same numbers a snippet gets — so it does not reopen ARCH-308's objection, which was to prose and not to the existence of a tool. And it takes parameters instead of code, which is what makes it safe where a snippet is not: there is nothing in it to write with, in any mode. It delegates to condor.fetchers.market_data, so the payload shapes and the fallback ladder are the ones every other candle consumer already uses. Keeping it read-only is danger.py's job, not the gate's. Its mutating action set is named and empty, its read-only set is pinned against the tool's registered Literal, and dry_run_refusal refuses an action classified in neither — so a future action that does more than read cannot become callable in a rehearsal by being added to the tool and forgotten. risk.py is untouched: the dry-run policy is still one function, asked once. The tick preloads it in every mode rather than only in dry-run, because it is also the cheaper read wherever the candles are the answer and a tick should not have to know which mode it is in to reach for it. The playbooks now say which tool to use when: the rows for a plain read, the snippet for anything with arithmetic in it. What was flatly wrong is corrected — the Condor agent's own notes claimed no candle tool existed. Closes CORR-625. --- .../_shared/skills/dca_into_position/SKILL.md | 4 + .../skills/directional_position/SKILL.md | 4 +- .../skills/market_data_with_code/SKILL.md | 34 ++- .../_shared/skills/open_lp_position/SKILL.md | 4 + agents/_shared/skills/run_a_grid/SKILL.md | 4 + agents/adaptive_grid_trader/AGENT.md | 1 + agents/condor/AGENT.md | 3 +- agents/market_making_expert/AGENT.md | 1 + .../strategies/hip_3_mm_operator/strategy.md | 4 +- agents/meteora_launch_lp/AGENT.md | 1 + agents/smart_money_flow/AGENT.md | 1 + agents/solana_dex_lp_expert/AGENT.md | 1 + agents/xrpl_market_maker/AGENT.md | 1 + condor/agents/prompts.py | 15 +- condor/runtime/danger.py | 46 +++- mcp_servers/hummingbot_api/profiles.py | 25 +- mcp_servers/hummingbot_api/server.py | 85 ++++++- .../hummingbot_api/tools/market_data.py | 116 ++++++++- tests/test_acp_permission_gate.py | 6 + tests/test_agent_actions.py | 39 +++ tests/test_mcp_market_data_tool.py | 229 ++++++++++++++++++ tests/test_mcp_tool_profiles.py | 46 +++- tests/test_risk_gate.py | 72 ++++++ 23 files changed, 700 insertions(+), 42 deletions(-) create mode 100644 tests/test_mcp_market_data_tool.py diff --git a/agents/_shared/skills/dca_into_position/SKILL.md b/agents/_shared/skills/dca_into_position/SKILL.md index 0507025d2..579b7eae6 100644 --- a/agents/_shared/skills/dca_into_position/SKILL.md +++ b/agents/_shared/skills/dca_into_position/SKILL.md @@ -31,6 +31,10 @@ get_prices(connector_name="binance_perpetual", trading_pairs=["BTC-USDT"]) df = await client.market_data.get_candles_last_days("binance_perpetual", "BTC-USDT", days=30, interval="4h") ``` +In a **dry run** `run_code` is refused. Read the same series with the tool instead — +`get_market_data(action="candles", connector_name="binance_perpetual", +trading_pair="BTC-USDT", interval="4h", max_records=180)`. + The candles matter more here than for most tools: they tell you how far this pair actually falls in a normal pullback, which is exactly the spacing question. diff --git a/agents/_shared/skills/directional_position/SKILL.md b/agents/_shared/skills/directional_position/SKILL.md index 9721f64f1..a93ad2c34 100644 --- a/agents/_shared/skills/directional_position/SKILL.md +++ b/agents/_shared/skills/directional_position/SKILL.md @@ -52,7 +52,9 @@ Work the other way round: the broken level, whatever the trade is actually predicting. Use candles if you need the pair's typical swing: `run_code`: `await client.market_data.get_candles_last_days(connector, pair, days=7, - interval="1h")`. + interval="1h")`. In a dry run `run_code` is refused — use + `get_market_data(action="candles", connector_name=connector, trading_pair=pair, + interval="1h", max_records=168)`, which returns the same rows. 2. **That distance is the stop**, expressed as a fraction of entry. 3. **Size so that distance costs an acceptable amount.** The stop sets the risk per unit; the amount sets how many units. Adjust the amount, never the stop. diff --git a/agents/_shared/skills/market_data_with_code/SKILL.md b/agents/_shared/skills/market_data_with_code/SKILL.md index 01e1212b0..f961fbcba 100644 --- a/agents/_shared/skills/market_data_with_code/SKILL.md +++ b/agents/_shared/skills/market_data_with_code/SKILL.md @@ -3,10 +3,12 @@ name: market_data_with_code description: Fetch and analyse market data — use run_code for anything beyond a single raw value; canned snippets for common queries. when_to_use: 'User asks for market data: price, candles, funding rate, order book, - RSI/EMA/VWAP, comparisons across assets or venues, any derived calculation. The - only raw market data tool left is get_prices, and it is only appropriate for a single, - direct lookup with no computation. For everything else — indicators, multi-asset, - multi-venue, aggregations — write a Python snippet and call run_code.' + RSI/EMA/VWAP, comparisons across assets or venues, any derived calculation. Two + raw market data tools are left — get_prices for a single direct quote, and + get_market_data for plain OHLCV candles — and both are only appropriate when the + raw value IS the answer. For everything else — indicators, multi-asset, multi-venue, + aggregations — write a Python snippet and call run_code. In a dry run get_market_data + is the only candle read there is: run_code does not execute there.' created: '2026-09-02T15:29:27Z' source: chat --- @@ -44,6 +46,7 @@ If you are reading this because a market data request just came in, do this chec |---|---|---| | Single price, one venue | `get_prices` MCP tool | 100 | | Single price inside run_code | `client.market_data.get_prices` | 150 | +| Plain candles, no math on them | `get_market_data` MCP tool | 200 | | Order book — one or many venues | `run_code` → `get_order_book` | 350–500 | | Funding rate — one or many venues | `run_code` → `get_funding_info` | 350–500 | | Indicators (RSI / EMA / ATR / VWAP) | `run_code` → `get_candles_last_days` + pandas_ta | varies | @@ -53,6 +56,29 @@ If you are reading this because a market data request just came in, do this chec --- +### In a dry run, `run_code` does not run + +A snippet and a routine both hold the unrestricted API client, so a dry-run +session auto-approves neither — the refusal says so and the tick goes on. That +leaves exactly three market reads in a rehearsal: `get_prices`, +`explore_geckoterminal`, and + +``` +get_market_data(action="candles", connector_name="binance_perpetual", + trading_pair="SOL-USDT", interval="1h", max_records=168) +``` + +which returns `candles` as rows of `{timestamp, open, high, low, close, volume}` +floats — the same numbers `client.market_data.get_candles` gives you, without the +snippet. `action="historical_candles"` takes a unix `start_time`/`end_time` range; +`action="connectors"` lists which venues serve OHLCV at all. + +Do the arithmetic in your head from the rows and say what you *would* have +computed. Do not try to route around the refusal by writing the snippet into a +routine — that door is closed too. + +--- + ### API reference **Order book** diff --git a/agents/_shared/skills/open_lp_position/SKILL.md b/agents/_shared/skills/open_lp_position/SKILL.md index 9d02e449d..dc2da5f8b 100644 --- a/agents/_shared/skills/open_lp_position/SKILL.md +++ b/agents/_shared/skills/open_lp_position/SKILL.md @@ -73,6 +73,10 @@ Anchor it on realised movement rather than a guess: df = await client.market_data.get_candles_last_days("binance", "SOL-USDT", days=7, interval="1h") ``` +In a **dry run** `run_code` is refused. Read the same series with the tool instead — +`get_market_data(action="candles", connector_name="binance", trading_pair="SOL-USDT", +interval="1h", max_records=168)`. + Take the recent high/low. Then: - **Actively managed (hours to a day):** roughly ±0.5–1× the pair's daily range around diff --git a/agents/_shared/skills/run_a_grid/SKILL.md b/agents/_shared/skills/run_a_grid/SKILL.md index f77557f2e..df7cc2a6e 100644 --- a/agents/_shared/skills/run_a_grid/SKILL.md +++ b/agents/_shared/skills/run_a_grid/SKILL.md @@ -29,6 +29,10 @@ Do this before anything else. A grid in a trend is a slow loss with extra steps. df = await client.market_data.get_candles_last_days("binance", "SOL-USDT", days=3, interval="1h") ``` +In a **dry run** `run_code` is refused. Read the same series with the tool instead — +`get_market_data(action="candles", connector_name="binance", trading_pair="SOL-USDT", +interval="1h", max_records=72)` — and judge the regime off the rows. + Read the series, don't just take the min and max: - **Ranging** — price crosses its own mid repeatedly, highs and lows cluster in a band, diff --git a/agents/adaptive_grid_trader/AGENT.md b/agents/adaptive_grid_trader/AGENT.md index 992debf76..718e21b07 100644 --- a/agents/adaptive_grid_trader/AGENT.md +++ b/agents/adaptive_grid_trader/AGENT.md @@ -5,6 +5,7 @@ description: Expert in multi-timeframe adaptive grid trading with safety-first o agent_key: claude-acp:opus tools: - get_prices +- get_market_data - get_portfolio_overview - create_grid_executor - list_executors diff --git a/agents/condor/AGENT.md b/agents/condor/AGENT.md index e42f1465d..0d0d0c552 100644 --- a/agents/condor/AGENT.md +++ b/agents/condor/AGENT.md @@ -11,7 +11,8 @@ You are Condor, a trading assistant. Do NOT explore the codebase — use MCP too ## MCP Tools **mcp-hummingbot** — Trading API (pre-configured, call directly): -- `get_prices` — latest quote for one or more pairs. Everything else about a market — candles, order book, funding rate — is read as structured data with `client.market_data.*` inside `run_code` (see the `market_data_with_code` skill); there is no raw candle, book or funding tool +- `get_prices` — latest quote for one or more pairs +- `get_market_data` — plain OHLCV candles as rows (`candles`, `historical_candles`, `connectors`). Reach for it when the candles ARE the answer; when you are going to compute on them — indicators, regimes, several venues compared — read them with `client.market_data.*` inside `run_code` instead (see the `market_data_with_code` skill) and do the arithmetic where the data is. There is still no raw order book or funding-rate tool, and in a dry run `run_code` does not execute, which makes `get_market_data` the only candle read there - `get_portfolio_overview` — balances, positions, orders - `create_position_executor` / `create_grid_executor` / `create_dca_executor` / `create_order_executor` / `create_lp_executor` — deploy trading executors. A single market/limit order is `create_order_executor` (`execution_strategy` MARKET / LIMIT / LIMIT_MAKER); there is no `place_order` tool - `list_executors` / `get_executor` / `stop_executor` — monitor and stop running executors diff --git a/agents/market_making_expert/AGENT.md b/agents/market_making_expert/AGENT.md index 60c4634c3..dd14e5966 100644 --- a/agents/market_making_expert/AGENT.md +++ b/agents/market_making_expert/AGENT.md @@ -5,6 +5,7 @@ description: Market making specialist — regime detection, spread calibration, agent_key: claude-acp:sonnet tools: - get_prices +- get_market_data - get_portfolio_overview - list_executors - get_executor diff --git a/agents/market_making_expert/strategies/hip_3_mm_operator/strategy.md b/agents/market_making_expert/strategies/hip_3_mm_operator/strategy.md index f72ad12e0..b722985ad 100644 --- a/agents/market_making_expert/strategies/hip_3_mm_operator/strategy.md +++ b/agents/market_making_expert/strategies/hip_3_mm_operator/strategy.md @@ -97,7 +97,9 @@ your wider levels. `clearinghouseState {"dex":"xyz"}` (shows only per-dex position margin, $0 when flat). - **Trading hours:** many xyz markets close off-hours (empty book). Scanner filters them; re-check the live book before deploying. -- **Data:** candles via `run_code` (`client.market_data.get_candles_last_days`); live book via Hyperliquid `l2Book` +- **Data:** candles via `run_code` (`client.market_data.get_candles_last_days`), or + `get_market_data(action="candles", ...)` when the rows are all you need — that tool is + the only candle read in a dry run, where `run_code` is refused; live book via Hyperliquid `l2Book` `{"type":"l2Book","coin":"xyz:DRAM"}` — **lowercase prefix + UPPERCASE token**, no `-USD` (both `XYZ:...` and `xyz:dram` return null). diff --git a/agents/meteora_launch_lp/AGENT.md b/agents/meteora_launch_lp/AGENT.md index 5db68f95f..cd767c0cc 100644 --- a/agents/meteora_launch_lp/AGENT.md +++ b/agents/meteora_launch_lp/AGENT.md @@ -10,6 +10,7 @@ tools: - explore_dex_pools - get_portfolio_overview - get_prices +- get_market_data - send_notification - manage_routines - manage_agents diff --git a/agents/smart_money_flow/AGENT.md b/agents/smart_money_flow/AGENT.md index 5ee0dcb89..c42de37c3 100644 --- a/agents/smart_money_flow/AGENT.md +++ b/agents/smart_money_flow/AGENT.md @@ -16,6 +16,7 @@ tools: - list_positions_held - get_portfolio_overview - get_prices +- get_market_data - search_history - manage_agents - manage_strategies diff --git a/agents/solana_dex_lp_expert/AGENT.md b/agents/solana_dex_lp_expert/AGENT.md index ff53660be..00a7e27d0 100644 --- a/agents/solana_dex_lp_expert/AGENT.md +++ b/agents/solana_dex_lp_expert/AGENT.md @@ -16,6 +16,7 @@ tools: - resolve_orphaned_position - get_portfolio_overview - get_prices +- get_market_data - search_history - manage_routines - manage_agents diff --git a/agents/xrpl_market_maker/AGENT.md b/agents/xrpl_market_maker/AGENT.md index ff9c05a7b..116ef2916 100644 --- a/agents/xrpl_market_maker/AGENT.md +++ b/agents/xrpl_market_maker/AGENT.md @@ -5,6 +5,7 @@ description: On-ledger market making specialist for the XRPL CLOB — reference agent_key: claude-acp:sonnet tools: - get_prices +- get_market_data - get_portfolio_overview - explore_geckoterminal - create_order_executor diff --git a/condor/agents/prompts.py b/condor/agents/prompts.py index 267fd0558..990706d23 100644 --- a/condor/agents/prompts.py +++ b/condor/agents/prompts.py @@ -223,11 +223,18 @@ def _build_tool_preload( """ tools = [ "mcp__mcp-hummingbot__get_prices", - # The candle, order book and funding readers are not mounted any more - # (ARCH-308): market data a tick computes on is read as structured rows - # with ``client.market_data.*`` inside run_code, so run_code is what a - # tick has to arrive holding. + # The table-rendering candle, order book and funding readers are not + # mounted any more (ARCH-308): market data a tick computes on is read as + # structured rows with ``client.market_data.*`` inside run_code, so + # run_code is what a tick has to arrive holding. "mcp__condor__run_code", + # …except in a dry run, where a snippet is refused for holding the + # unrestricted client (SEC-616) and so is a routine (SEC-626), which + # between them left a rehearsal with no candle read at all. Preloaded in + # every mode rather than only that one: it is the cheaper read wherever + # the candles are the answer, and a tick should not have to know which + # mode it is in to reach for it (CORR-625). + "mcp__mcp-hummingbot__get_market_data", ] if is_controller_mode: # Read-only bot/controller queries stay available in dry-run; the diff --git a/condor/runtime/danger.py b/condor/runtime/danger.py index 410ab2e3e..092f8cfc8 100644 --- a/condor/runtime/danger.py +++ b/condor/runtime/danger.py @@ -289,6 +289,26 @@ "get_instance", "list_instances", } +#: The candle reader, and the answer to what refusing the two tools above costs +#: a rehearsal (CORR-625). Both of those are refused in dry-run for holding the +#: unrestricted API client, and since ARCH-308 they were between them the only +#: structured market read a tick had — so a dry run could not rehearse a +#: candle-driven decision at all, which is the whole point of one. +#: +#: This tool is the read-only path back. It takes parameters rather than code, +#: so unlike a snippet there is nothing in it to write with, and it is listed +#: here rather than left unnamed for one reason: an *unclassified* action on it +#: is refused in dry-run. A future action that does more than read cannot become +#: callable in a rehearsal by being added to the tool and forgotten here. +MARKET_DATA_TOOL = "get_market_data" +#: Empty on purpose, and the assertion this whole entry exists to make: the tool +#: has no write half. It is kept as a name rather than inlined as ``set()`` so +#: that the day one is added, there is somewhere obvious to put it. +MUTATING_MARKET_DATA_ACTIONS: set[str] = set() +#: Reading candles, reading a candle range, and asking which connectors serve +#: OHLCV at all. Pinned against the tool's registered ``Literal`` by a test, the +#: same way the routine and snippet sets are. +READ_ONLY_MARKET_DATA_ACTIONS = {"candles", "historical_candles", "connectors"} #: How much of a snippet's first line the log row carries. A summary is one line #: on a page, and the whole source is in the code-run store anyway. MAX_SNIPPET_HEAD_CHARS = 80 @@ -625,15 +645,18 @@ def dry_run_refusal(tool_call: dict[str, Any]) -> str | None: The refusal is per *action*, not per tool, so what a dry run needs in order to rehearse at all — listing routines, reading their source and their config - schema, reading back a past run or a past snippet — stays free. Widening - that read-only half is how a dry run gets a new capability without getting - the ability to write. + schema, reading back a past run or a past snippet, reading candles — stays + free. Widening that read-only half is how a dry run gets a new capability + without getting the ability to write, and ``get_market_data`` below is that + widening rather than an exception to it (CORR-625): it appears here not to + be allowed — an unnamed tool is already allowed — but so that an action + added to it later has to be classified before a rehearsal can call it. """ if is_code_execution_call(tool_call): return ( "this session runs in dry-run mode, where nothing mutates, and a " - "snippet holds the unrestricted API client — read what you need " - "with the read-only tools instead" + "snippet holds the unrestricted API client — read the market with " + "get_market_data and the other read-only tools instead" ) if is_mutating_routine_call(tool_call): @@ -643,6 +666,19 @@ def dry_run_refusal(tool_call: dict[str, Any]) -> str | None: "snippet — 'list', 'describe' and 'read_routine' still work" ) + if tool_call_name(tool_call) == MARKET_DATA_TOOL and _is_mutating_action( + tool_call, MUTATING_MARKET_DATA_ACTIONS, READ_ONLY_MARKET_DATA_ACTIONS + ): + # Every action this module knows on this tool reads, so reaching here at + # all means an action it does *not* know — a new one, or an unreadable + # argument. Fails closed like its siblings: the cost is a rehearsal that + # says so, against a write that a rehearsal promised could not happen. + return ( + "this session runs in dry-run mode, and get_market_data was asked " + "for something this build does not know is a read — use " + "'candles', 'historical_candles' or 'connectors'" + ) + return None diff --git a/mcp_servers/hummingbot_api/profiles.py b/mcp_servers/hummingbot_api/profiles.py index 2978c56c7..2726b7dd5 100644 --- a/mcp_servers/hummingbot_api/profiles.py +++ b/mcp_servers/hummingbot_api/profiles.py @@ -24,6 +24,7 @@ "set_account_position_mode_and_leverage": "Set position mode and leverage", "search_history": "Search historical trades, orders and funding", "get_prices": "Latest price for one or more pairs", + "get_market_data": "OHLCV candles as rows — the read a dry run can make", "manage_controllers": "Controller templates and saved configs (design-time)", "manage_bots": "Deploy, monitor and control controller-based bots", "create_position_executor": "Open a directional position with SL/TP — spends funds", @@ -57,20 +58,26 @@ #: size a position, run it and report on it. This is the whole surface minus the #: two rings below. #: -#: No raw candle, order book or funding-rate reader is in it (ARCH-308). Those -#: three returned a rendered table — a string a model can read and cannot compute -#: on — so reaching for one bought a number that then had to be re-fetched through -#: ``run_code`` to be averaged, charted or compared across venues. The structured -#: equivalents are one ``run_code`` snippet away (``client.market_data.*``, which -#: returns dicts and takes an ``asyncio.gather`` across venues), and a tool absent -#: from the list is the only form of that advice a model cannot skip. ``get_prices`` -#: stays: a single quote, read once and not computed on, is the one case the text -#: answers completely. +#: No *table-rendering* market reader is in it (ARCH-308). ``get_candles``, +#: ``get_order_book`` and ``get_funding_rate`` returned a string a model can read +#: and cannot compute on, so reaching for one bought a number that then had to be +#: re-fetched through ``run_code`` to be averaged, charted or compared across +#: venues. That path is still the one for anything with arithmetic in it +#: (``client.market_data.*`` returns dicts and takes an ``asyncio.gather`` across +#: venues), and a table tool absent from the list is the only form of that advice +#: a model cannot skip. +#: +#: ``get_prices`` stays: a single quote, read once and not computed on, is the one +#: case the text answers completely. ``get_market_data`` joins it for the other +#: (CORR-625): it answers candles in rows rather than prose, so it buys no second +#: read, and it is the only candle path left in a dry run, where a snippet and a +#: routine are both refused for holding the unrestricted client. TRADING_TOOLS: tuple[str, ...] = ( "get_portfolio_overview", "set_account_position_mode_and_leverage", "search_history", "get_prices", + "get_market_data", "manage_controllers", "manage_bots", "create_position_executor", diff --git a/mcp_servers/hummingbot_api/server.py b/mcp_servers/hummingbot_api/server.py index 2d40f4d4f..5fadda01d 100644 --- a/mcp_servers/hummingbot_api/server.py +++ b/mcp_servers/hummingbot_api/server.py @@ -383,13 +383,22 @@ async def search_history( # Market Data Tools # -# One tool, on purpose (ARCH-308). The candle, order book and funding-rate -# readers that used to sit here returned a rendered table, so anything computed -# from one — a spread, an indicator, three venues compared — had to be fetched a -# second time through ``run_code`` to get numbers back. That is now the only -# path: ``client.market_data.*`` inside a snippet returns dicts and gathers -# across venues in one round trip. ``get_prices`` survives because a quote read -# once and not computed on is answered completely by its own text. +# Two tools, and neither renders a series (ARCH-308). The candle, order book and +# funding-rate readers that used to sit here returned a rendered *table*, so +# anything computed from one — a spread, an indicator, three venues compared — +# had to be fetched a second time through ``run_code`` to get numbers back. That +# objection was to prose, not to a tool: ``client.market_data.*`` inside a +# snippet stays the path for anything with arithmetic in it, and ``get_prices`` +# survives because a quote read once and not computed on is answered completely +# by its own text. +# +# ``get_market_data`` is the third case, and it exists because of dry-run +# (CORR-625). A snippet holds the unrestricted API client and so does a routine, +# so a rehearsal auto-approves neither (SEC-616, SEC-626) — which left a dry run +# with no structured candle read at all, unable to rehearse the very decision the +# shared playbooks teach. It answers in rows, so nothing read from it is read +# twice, and it takes parameters instead of code, so there is nothing in it to +# write with. @handle_errors("get prices") @@ -417,6 +426,68 @@ async def get_prices(connector_name: str, trading_pairs: list[str]) -> str: ) +@handle_errors("get market data") +async def get_market_data( + action: Literal["candles", "historical_candles", "connectors"], + connector_name: str = "", + trading_pair: str = "", + interval: str = "1m", + max_records: int = 200, + start_time: int | None = None, + end_time: int | None = None, +) -> dict[str, Any]: + """Read OHLCV candles back as rows, without writing any code. + + Returns numbers, not a table: `candles` is a list of + `{timestamp, open, high, low, close, volume}` floats, ready to be read off + or handed to a snippet. + + WHEN TO USE THIS INSTEAD OF `run_code`: when the candles ARE the answer — + the last N bars of one pair, the range they cover, whether a connector even + serves OHLCV. When you are going to *compute* on them — an indicator, a + regime, a spread across three venues, anything with pandas in it — write the + snippet instead: `client.market_data.*` inside `run_code` gathers venues in + one round trip and does the arithmetic where the data already is. + + IN A DRY RUN THIS IS THE ONLY CANDLE READ. A snippet and a routine both hold + the unrestricted API client, so neither runs in a rehearsal; this tool has no + write path at all and runs in every mode. + + Actions: + - "candles": the most recent `max_records` candles (needs connector_name, + trading_pair) + - "historical_candles": a unix time range (needs start_time; end_time + optional) + - "connectors": which connectors serve OHLCV at all — check before asking a + DEX connector for candles, because most do not serve them + + Args: + action: What to read. + connector_name: Exchange connector, e.g. 'binance_perpetual'. + trading_pair: Pair to read, e.g. 'SOL-USDT'. + interval: Candle interval — '1m', '5m', '1h', '4h', '1d'. + max_records: Rows to return (default 200, max 1000). + start_time: Range start, unix epoch seconds (historical_candles). + end_time: Range end, unix epoch seconds. Defaults to now. + + Example: + - get_market_data("candles", "binance_perpetual", "SOL-USDT", interval="1h", + max_records=168) + """ + client = await hummingbot_client.get_client() + + return await market_data_tools.get_market_data( + client=client, + action=action, + connector_name=connector_name, + trading_pair=trading_pair, + interval=interval, + max_records=max_records, + start_time=start_time, + end_time=end_time, + ) + + @handle_errors("manage controllers") async def manage_controllers( action: Literal["list", "describe", "upsert", "delete"], diff --git a/mcp_servers/hummingbot_api/tools/market_data.py b/mcp_servers/hummingbot_api/tools/market_data.py index 1120328dc..ffff15523 100644 --- a/mcp_servers/hummingbot_api/tools/market_data.py +++ b/mcp_servers/hummingbot_api/tools/market_data.py @@ -1,17 +1,39 @@ """ Market data operations business logic. -One function, for the one market data tool this server still exposes (ARCH-308). -Candles, funding rates and order books are read as structured data through -``client.market_data.*`` inside ``run_code``; nothing here renders them into a -table any more, because nothing asks for one. +Two functions, and neither renders a series into a table (ARCH-308). ``get_prices`` +answers a single quote in text because a quote read once and not computed on is +answered completely by its text. ``get_market_data`` answers candles in **rows** — +a list of ``{timestamp, open, high, low, close, volume}`` dicts, the same shape +``client.market_data.*`` returns inside ``run_code`` — so nothing read here has to +be fetched a second time to be averaged or compared. + +Its reason to exist is dry-run (CORR-625). A snippet and a routine both hold the +unrestricted API client, so a rehearsal auto-approves neither (SEC-616, SEC-626), +which left a dry run with no structured candle read at all. This tool takes +parameters, not code: there is nothing in it to write with, in any mode. """ from datetime import datetime from typing import Any +from condor.fetchers.market_data import ( + fetch_candle_connectors, + fetch_historical_candles, +) +from mcp_servers.hummingbot_api.exceptions import ToolError from mcp_servers.hummingbot_api.formatters import format_prices_as_table +#: The tool's actions. Every one of them reads; the set is deliberately closed, +#: and ``condor.runtime.danger`` pins it against this literal so an action added +#: here without a thought about what it touches cannot quietly become callable +#: in a dry run. +MARKET_DATA_ACTIONS = ("candles", "historical_candles", "connectors") + +#: Rows per candle request. The cap is context, not throughput: a tick that asks +#: for 5000 1m candles spends its whole window reading them back. +MAX_CANDLE_RECORDS = 1000 + async def get_prices( client: Any, connector_name: str, trading_pairs: list[str] @@ -47,3 +69,89 @@ async def get_prices( "connector_name": connector_name, "timestamp": time_str, } + + +async def get_market_data( + client: Any, + action: str, + connector_name: str = "", + trading_pair: str = "", + interval: str = "1m", + max_records: int = 200, + start_time: int | None = None, + end_time: int | None = None, +) -> dict[str, Any]: + """Read candles as rows, or list the connectors that serve them. + + Delegates to the fetchers every other candle consumer in Condor already uses + (``condor.fetchers.market_data``), so this tool inherits their payload-shape + handling and their fallback ladder rather than restating it: the two candle + actions are the same fetcher with and without a time range. + + Args: + client: Hummingbot API client + action: One of :data:`MARKET_DATA_ACTIONS` + connector_name: Exchange connector name (candle actions) + trading_pair: Pair to read (candle actions) + interval: Candle interval, e.g. "1m", "1h", "4h", "1d" + max_records: How many rows to return, capped at :data:`MAX_CANDLE_RECORDS` + start_time / end_time: Unix epoch seconds, for "historical_candles" + + Returns: + A dict whose candle rows are floats, not a rendered table. + """ + if action == "connectors": + return { + "action": action, + "connectors": await fetch_candle_connectors(client), + } + + if action not in ("candles", "historical_candles"): + raise ToolError( + f"unknown action {action!r} — expected one of " + f"{', '.join(MARKET_DATA_ACTIONS)}" + ) + + if not connector_name: + raise ToolError(f"connector_name is required for the {action} action") + if not trading_pair: + raise ToolError(f"trading_pair is required for the {action} action") + + limit = max(1, min(int(max_records), MAX_CANDLE_RECORDS)) + + if action == "historical_candles": + if start_time is None: + raise ToolError( + "start_time is required for the historical_candles action — use " + "action='candles' for the most recent window instead" + ) + rows = await fetch_historical_candles( + client, + connector_name, + trading_pair, + interval, + start_time=start_time, + end_time=end_time, + limit=limit, + fallback_on_error=True, + ) + else: + # No start_time skips the ranged call entirely and the `limit` fallback + # answers, which is exactly "the most recent `limit` candles". + rows = await fetch_historical_candles( + client, + connector_name, + trading_pair, + interval, + start_time=None, + limit=limit, + ) + + return { + "action": action, + "connector_name": connector_name, + "trading_pair": trading_pair, + "interval": interval, + "count": len(rows), + "candles": rows, + } diff --git a/tests/test_acp_permission_gate.py b/tests/test_acp_permission_gate.py index 2dd2e823c..2ef2451e9 100644 --- a/tests/test_acp_permission_gate.py +++ b/tests/test_acp_permission_gate.py @@ -407,6 +407,12 @@ def test_dry_run_cancels_a_swap_but_not_a_quote(): "executor_defaults", # edits a local preferences file; creates nothing "explore_dex_pools", "explore_geckoterminal", + # Reads candles and nothing else, which is the whole reason it exists: a dry + # run may call neither a snippet nor a routine, so this is the market read + # left to a rehearsal (CORR-625). Its mutating half in danger.py is empty, + # and an action added to it without landing in the read-only set there is + # refused in dry-run rather than waved through. + "get_market_data", } #: Verbs that mean "this call changes something out in the world". diff --git a/tests/test_agent_actions.py b/tests/test_agent_actions.py index e620600f3..9a08a2537 100644 --- a/tests/test_agent_actions.py +++ b/tests/test_agent_actions.py @@ -644,6 +644,45 @@ def test_the_snippet_action_sets_match_the_registered_tool(): assert not (MUTATING_CODE_RUN_ACTIONS & READ_ONLY_CODE_RUN_ACTIONS) +def test_the_market_data_action_sets_match_the_registered_tool(): + """The candle reader has no write half, and this is what keeps it that way. + + ``get_market_data`` exists so a dry run can read a market at all (CORR-625), + and it earns that by taking parameters rather than code. Its mutating set is + empty, so an action added to the tool and not classified here is refused in + dry-run rather than waved through — but only as long as the two sets still + partition what the tool actually accepts. + """ + from condor.runtime.danger import ( + MUTATING_MARKET_DATA_ACTIONS, + READ_ONLY_MARKET_DATA_ACTIONS, + ) + from mcp_servers.hummingbot_api.server import get_market_data + + fn = getattr(get_market_data, "fn", get_market_data) + literals = set(typing.get_args(fn.__annotations__["action"])) + assert MUTATING_MARKET_DATA_ACTIONS == set() + assert READ_ONLY_MARKET_DATA_ACTIONS == literals + + +def test_the_candle_reader_is_neither_gated_nor_logged_as_a_change(): + """It reads. A confirmation in front of it, or a row in the action log + claiming it changed something, would both be wrong.""" + from condor.runtime.danger import ( + DANGEROUS_TOOLS, + is_dangerous_tool_call, + is_mutating_tool_call, + is_recordable_tool_call, + ) + + assert "get_market_data" not in DANGEROUS_TOOLS + for action in ("candles", "historical_candles", "connectors"): + call = {"tool": "get_market_data", "input": {"action": action}} + assert is_dangerous_tool_call(call) is False, action + assert is_mutating_tool_call(call) is False, action + assert is_recordable_tool_call(call) is False, action + + def folded_routine(**args): """A folded ``manage_routines`` call. diff --git a/tests/test_mcp_market_data_tool.py b/tests/test_mcp_market_data_tool.py new file mode 100644 index 000000000..7ba3c06fe --- /dev/null +++ b/tests/test_mcp_market_data_tool.py @@ -0,0 +1,229 @@ +"""``get_market_data``: the candle read that survives a dry run (CORR-625). + +ARCH-308 deleted the candle, order book and funding tools because they answered +in a rendered table, leaving ``client.market_data.*`` inside ``run_code`` as the +only structured read. SEC-616 then refused ``run_code`` in dry-run for holding +the unrestricted API client, and SEC-626 refused ``manage_routines`` one door +over for the same reason — so a rehearsal could not read a candle at all. + +This tool is the way back, and the two properties it has to keep are pinned +here: it answers in **rows** (or ARCH-308's objection returns), and it reaches +for nothing but the client's candle readers (or dry-run's promise is worth +nothing). The gate side of that — which actions a dry run may call, and what +happens to one nobody classified — lives in test_risk_gate.py. + +The repo has no async test setup, so the coroutines are driven with +asyncio.run(). +""" + +import asyncio + +import pytest + +from mcp_servers.hummingbot_api.exceptions import ToolError +from mcp_servers.hummingbot_api.tools.market_data import ( + MARKET_DATA_ACTIONS, + MAX_CANDLE_RECORDS, + get_market_data, +) + + +class FakeMarketData: + """Only the three readers the tool is allowed to reach. + + Anything else — an order, a swap, a gateway call — is an ``AttributeError``, + which is the assertion this class exists to make. + """ + + def __init__(self, rows=None, connectors=None): + self._rows = rows if rows is not None else [] + self._connectors = connectors or [] + self.calls = [] + + async def get_candles(self, connector_name, trading_pair, interval, limit): + self.calls.append( + ("get_candles", connector_name, trading_pair, interval, limit) + ) + return self._rows + + async def get_historical_candles( + self, connector_name, trading_pair, interval, start_time=None, end_time=None + ): + self.calls.append( + ( + "get_historical_candles", + connector_name, + trading_pair, + interval, + start_time, + end_time, + ) + ) + return {"data": self._rows} + + async def get_available_candle_connectors(self): + self.calls.append(("get_available_candle_connectors",)) + return self._connectors + + +class FakeClient: + """A client with a market data reader and nothing else at all.""" + + def __init__(self, market_data): + self.market_data = market_data + + def __getattr__(self, name): + raise AssertionError( + f"get_market_data reached client.{name} — it may only read candles" + ) + + +ROWS = [ + { + "timestamp": 1_757_000_000, + "open": 1, + "high": 2, + "low": 0.5, + "close": 1.5, + "volume": 100, + }, + [1_757_003_600, 1.5, 2.5, 1.0, 2.0, 200], +] + + +def _run(**kwargs): + md = FakeMarketData(**kwargs.pop("fake", {})) + result = asyncio.run(get_market_data(client=FakeClient(md), **kwargs)) + return result, md + + +def test_candles_come_back_as_rows_of_floats_and_not_a_table(): + """ARCH-308's objection was prose. Nothing read here is read twice.""" + result, _ = _run( + action="candles", + connector_name="binance", + trading_pair="SOL-USDC", + fake={"rows": ROWS}, + ) + + assert result["count"] == 2 + assert result["candles"] == [ + { + "timestamp": 1_757_000_000.0, + "open": 1.0, + "high": 2.0, + "low": 0.5, + "close": 1.5, + "volume": 100.0, + }, + { + "timestamp": 1_757_003_600.0, + "open": 1.5, + "high": 2.5, + "low": 1.0, + "close": 2.0, + "volume": 200.0, + }, + ] + assert all(isinstance(v, float) for v in result["candles"][0].values()) + + +def test_a_plain_candle_read_asks_for_a_window_and_not_a_range(): + """No ``start_time`` means "the most recent N", which is the fetcher's + fallback rung, not a ranged query with an invented start.""" + _, md = _run( + action="candles", + connector_name="binance", + trading_pair="SOL-USDC", + interval="1h", + max_records=168, + fake={"rows": ROWS}, + ) + + assert md.calls == [("get_candles", "binance", "SOL-USDC", "1h", 168)] + + +def test_a_historical_read_passes_the_range_through(): + _, md = _run( + action="historical_candles", + connector_name="binance", + trading_pair="SOL-USDC", + interval="4h", + start_time=1_757_000_000, + end_time=1_757_600_000, + fake={"rows": ROWS}, + ) + + assert md.calls == [ + ( + "get_historical_candles", + "binance", + "SOL-USDC", + "4h", + 1_757_000_000, + 1_757_600_000, + ) + ] + + +def test_the_row_count_is_capped_so_one_read_cannot_eat_a_tick(): + _, md = _run( + action="candles", + connector_name="binance", + trading_pair="SOL-USDC", + max_records=50_000, + fake={"rows": ROWS}, + ) + + assert md.calls[0][-1] == MAX_CANDLE_RECORDS + + +def test_connectors_answers_which_venues_serve_ohlcv_at_all(): + """Most DEX connectors do not, and asking is cheaper than a failed read.""" + result, md = _run(action="connectors", fake={"connectors": ["binance", "kucoin"]}) + + assert result == {"action": "connectors", "connectors": ["binance", "kucoin"]} + assert md.calls == [("get_available_candle_connectors",)] + + +@pytest.mark.parametrize( + "kwargs,missing", + [ + ({"action": "candles", "trading_pair": "SOL-USDC"}, "connector_name"), + ({"action": "candles", "connector_name": "binance"}, "trading_pair"), + ( + { + "action": "historical_candles", + "connector_name": "binance", + "trading_pair": "SOL-USDC", + }, + "start_time", + ), + ], +) +def test_a_read_missing_what_it_needs_says_which_argument(kwargs, missing): + with pytest.raises(ToolError, match=missing): + _run(**kwargs) + + +def test_an_action_the_tool_does_not_have_is_an_error_not_an_empty_read(): + """The gate refuses an unclassified action in dry-run; in every other mode + the tool itself has to say so rather than answer with nothing.""" + with pytest.raises(ToolError, match="unknown action"): + _run(action="subscribe", connector_name="binance", trading_pair="SOL-USDC") + + +def test_every_action_the_tool_advertises_actually_works(): + """``MARKET_DATA_ACTIONS`` is what ``danger.py``'s read-only set is written + against, so an entry in it that no branch handles would classify a call as a + read that then fails.""" + assert set(MARKET_DATA_ACTIONS) == {"candles", "historical_candles", "connectors"} + for action in MARKET_DATA_ACTIONS: + result, _ = _run( + action=action, + connector_name="binance", + trading_pair="SOL-USDC", + start_time=1_757_000_000, + fake={"rows": ROWS, "connectors": ["binance"]}, + ) + assert result["action"] == action diff --git a/tests/test_mcp_tool_profiles.py b/tests/test_mcp_tool_profiles.py index 9d9ca6932..2b2d4be3c 100644 --- a/tests/test_mcp_tool_profiles.py +++ b/tests/test_mcp_tool_profiles.py @@ -29,6 +29,7 @@ "set_account_position_mode_and_leverage", "search_history", "get_prices", + "get_market_data", "manage_controllers", "manage_bots", "create_position_executor", @@ -214,12 +215,12 @@ def test_the_tick_preload_always_carries_a_way_to_read_a_market( ): """The other direction, which the test above cannot see (ARCH-308). - Since no ring mounts a candle, order book or funding reader any more, the - only way a tick reads a market it can compute on is ``run_code`` over - ``client.market_data.*``. Drop that one name from the preload and every - assertion above still passes while the tick goes blind past a single price - — so the name is pinned here rather than left to whoever edits the list - next. + Since no ring mounts a table-rendering candle, order book or funding reader + any more, the only way a tick reads a market it can *compute* on is + ``run_code`` over ``client.market_data.*``. Drop that one name from the + preload and every assertion above still passes while the tick goes blind + past a single price — so the name is pinned here rather than left to whoever + edits the list next. """ from condor.agents.prompts import _build_tool_preload @@ -231,6 +232,39 @@ def test_the_tick_preload_always_carries_a_way_to_read_a_market( assert "mcp__condor__run_code" in line +@pytest.mark.parametrize("is_experiment", [True, False]) +@pytest.mark.parametrize("is_controller_mode", [True, False]) +def test_the_tick_preload_carries_a_market_read_a_dry_run_can_actually_make( + is_experiment, is_controller_mode +): + """Preloaded is not the same as callable (CORR-625). + + The test above pins ``run_code``, and in a dry run ``run_code`` is refused + for holding the unrestricted API client — as is ``manage_routines``, the + other door onto the same Python. Both stay in the preload for their + read-only halves, so a preload that named only those would satisfy every + assertion here while a rehearsal could not read a single candle. + + So this asserts the stronger thing: the preload names a market read, and the + dry-run policy in ``danger.py`` lets that exact call through. + """ + from condor.agents.prompts import _build_tool_preload + from condor.runtime.danger import dry_run_refusal + + line = _build_tool_preload( + is_dry_run=True, + is_experiment=is_experiment, + is_controller_mode=is_controller_mode, + ) + name = "mcp__mcp-hummingbot__get_market_data" + + assert name in line + assert name.rsplit("__", 1)[-1] in _registered(hb_server, "tick") + for action in ("candles", "historical_candles", "connectors"): + call = {"tool": name, "input": {"action": action, "trading_pair": "SOL-USDC"}} + assert dry_run_refusal(call) is None, action + + def test_the_manage_trading_agent_funnel_is_in_no_profile(): """FEAT-068 split it; no ring may resurrect the name.""" for name in CONDOR_PROFILES: diff --git a/tests/test_risk_gate.py b/tests/test_risk_gate.py index 8fb792105..95eae565a 100644 --- a/tests/test_risk_gate.py +++ b/tests/test_risk_gate.py @@ -903,3 +903,75 @@ def test_other_modes_still_run_routines_without_a_confirmation(mode): assert is_dangerous_tool_call(call) is False, action result = asyncio.run(callback(call, _OPTIONS)) assert result["outcome"]["outcome"] == "selected", action + + +# --------------------------------------------------------------------------- +# Refusing both doors onto arbitrary Python left a rehearsal unable to read a +# candle. `get_market_data` is the way back: parameters, not code (CORR-625). +# --------------------------------------------------------------------------- + + +def _market_call(**args) -> dict: + return {"tool": "mcp__mcp-hummingbot__get_market_data", "input": args} + + +@pytest.mark.parametrize("action", ["candles", "historical_candles", "connectors"]) +def test_a_dry_run_reads_candles_without_a_refusal(action): + """The point of the item: a rehearsal reads a market with no `run_code`.""" + refusals = RefusalLog() + callback = auto_approve_with_risk_check( + RiskEngine(RiskLimits()), + RiskState(), + execution_mode="dry_run", + refusals=refusals, + ) + + result = asyncio.run( + callback( + _market_call( + action=action, + connector_name="binance_perpetual", + trading_pair="SOL-USDT", + interval="1h", + start_time=1_757_000_000, + ), + _OPTIONS, + ) + ) + + assert result["outcome"]["outcome"] == "selected", action + assert refusals.drain() == [] + + +def test_a_dry_run_refuses_a_market_data_action_it_cannot_read(): + """The read-only path stays read-only: an action nobody classified is not a + read this build can vouch for, so it fails closed like its siblings.""" + callback = auto_approve_with_risk_check( + RiskEngine(RiskLimits()), RiskState(), execution_mode="dry_run" + ) + + for call in ( + {"tool": "get_market_data", "input": None}, + _market_call(trading_pair="SOL-USDT"), + _market_call(action=None), + _market_call(action="subscribe"), + ): + result = asyncio.run(callback(call, _OPTIONS)) + assert result["outcome"]["outcome"] == "cancelled", call + + +@pytest.mark.parametrize("mode", ["loop", "attended", "run_once"]) +def test_the_candle_reader_needs_no_confirmation_in_any_mode(mode): + """It is a read, so it is never a prompt — dry-run included.""" + from condor.runtime.danger import DANGEROUS_TOOLS, is_dangerous_tool_call + + assert "get_market_data" not in DANGEROUS_TOOLS + callback = auto_approve_with_risk_check( + RiskEngine(RiskLimits()), RiskState(), execution_mode=mode + ) + + call = _market_call( + action="candles", connector_name="binance", trading_pair="SOL-USDC" + ) + assert is_dangerous_tool_call(call) is False + assert asyncio.run(callback(call, _OPTIONS))["outcome"]["outcome"] == "selected" From f52b8605faaa2666c19382de07c553e43dacdd78 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 09:44:11 +0300 Subject: [PATCH 119/154] Name a pseudo-run row with the map's word, not its slug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pseudo-owner is built with an empty namespace and the emptiness is load-bearing: `in_namespace` refuses one in both languages, so no `condor-ui-…` bot can ever exist. That removes the entire justification `runKeyLabel` gives for preferring slugs — there is nothing beneath the row for `ui` to be matched against — and the sidebar read `condor / ui` and `condor / chat` as two unrelated systems rather than two doors of the same one. The words were already on the wire the whole time, in every pseudo-owner's `strategy_name`, and only `ownerTitle` ever said them. So an owner row is now named in exactly one place, `ownerRowLabel`: slugs for a strategy, the map's own words for a chat, a delegation or the dashboard, and slugs again for an owner the map no longer holds so a stale deep link names something rather than nothing. `agentBucketLabel` is that plus the two labels for the rows that are not runs, and the sidebar row, its filter bubble, the chat's filter chip, the Dock's execution rows and the Money view's terms all come through it. That last one cost a vocabulary: the reconciliation had its own private map of pseudo-slugs to words. It is gone, and with it the second copy of names Python already owns. `PSEUDO_STRATEGIES` survives as what it always was underneath — a list of reserved *slugs* — and moves next to the run-key vocabulary it belongs to. Run keys are untouched. Three doors stay three keys, three rows and three `?scope=agent:` values; only the displayed text changes. --- .../src/components/agent/floor/floor.test.ts | 5 +- .../components/agent/workspace/AgentFleet.tsx | 6 +- .../agent/workspace/MoneyView.test.tsx | 16 +- .../components/agent/workspace/MoneyView.tsx | 3 +- .../agent/workspace/reconcile.test.ts | 73 ++++--- .../components/agent/workspace/reconcile.ts | 59 ++---- .../src/components/chat/DockExecution.tsx | 4 +- .../src/components/chat/executionTree.test.ts | 46 ++++- frontend/src/components/chat/executionTree.ts | 31 ++- frontend/src/components/perf/PerfBrowser.tsx | 18 +- .../perf/ScopeTree.agentRows.test.tsx | 179 ++++++++++++++++++ frontend/src/components/perf/ScopeTree.tsx | 32 +++- .../src/components/perf/agentFilter.test.ts | 65 ++++++- frontend/src/components/perf/agentFilter.ts | 36 +++- frontend/src/lib/agent-attribution.test.ts | 46 +++++ frontend/src/lib/agent-attribution.ts | 72 ++++++- frontend/src/lib/pageFacts.ts | 6 +- 17 files changed, 586 insertions(+), 111 deletions(-) create mode 100644 frontend/src/components/perf/ScopeTree.agentRows.test.tsx diff --git a/frontend/src/components/agent/floor/floor.test.ts b/frontend/src/components/agent/floor/floor.test.ts index 6baeabdd3..33bfb74a8 100644 --- a/frontend/src/components/agent/floor/floor.test.ts +++ b/frontend/src/components/agent/floor/floor.test.ts @@ -174,7 +174,7 @@ describe("an agent's fold is the agent entire", () => { it("prints the same number as the home's row when one strategy is in scope", () => { const leaves = [leaf({ agent: "alpha.mm", how: "namespace", net: 64.12, volume: 2_549 })]; - const input = { leaves, deeds: DEEDS, convert: cv, now: NOW, symbol: "$" }; + const input = { leaves, deeds: DEEDS, owners: [], convert: cv, now: NOW, symbol: "$" }; const home = foldRows(foldTargets([agent()], null)[0].targets, input); const floor = foldRows(floorTargets([agent()], null)[0].targets, input); @@ -188,7 +188,7 @@ describe("an agent's fold is the agent entire", () => { leaf({ agent: "alpha.mm", how: "namespace", net: 100, volume: 10 }), leaf({ agent: "alpha.grid", how: "namespace", net: 25, volume: 5 }), ]; - const input = { leaves, deeds: DEEDS, convert: cv, now: NOW, symbol: "$" }; + const input = { leaves, deeds: DEEDS, owners: [], convert: cv, now: NOW, symbol: "$" }; expect(foldRows(foldTargets([two], null)[0].targets, input).get("alpha")!.net).toBe(100); // The whole reason `floorTargets` exists: `alpha.grid` is attributed, so it @@ -225,6 +225,7 @@ describe("an agent's fold is the agent entire", () => { const fold = foldRows([{ slug: "alpha", strategy: null }], { leaves, deeds: DEEDS, + owners: [], convert: cv, now: NOW, symbol: "$", diff --git a/frontend/src/components/agent/workspace/AgentFleet.tsx b/frontend/src/components/agent/workspace/AgentFleet.tsx index d7a514daa..4430aaa6a 100644 --- a/frontend/src/components/agent/workspace/AgentFleet.tsx +++ b/frontend/src/components/agent/workspace/AgentFleet.tsx @@ -5,7 +5,7 @@ import { Link, useSearchParams } from "react-router-dom"; import { PerfBrowser } from "@/components/perf/PerfBrowser"; import { useFleetData } from "@/hooks/useFleetData"; import { useServer } from "@/hooks/useServer"; -import { attributionOf, runKeyLabel } from "@/lib/agent-attribution"; +import { attributionOf, ownerRowLabel } from "@/lib/agent-attribution"; import type { AgentRunRow } from "@/lib/api"; import { parsePopulation } from "@/lib/perf-tree"; @@ -132,7 +132,7 @@ export function AgentFleet({
Showing {counts.mine.toLocaleString()} of {counts.total.toLocaleString()}{" "} - {counts.total === 1 ? "controller" : "controllers"} — {runKeyLabel(runKey)} + {counts.total === 1 ? "controller" : "controllers"} — {ownerRowLabel(fleet.owners, runKey)} 's - {runKeyLabel(runKey)}'s bots run on{" "} + {ownerRowLabel(fleet.owners, runKey)}'s bots run on{" "} {serverName} + + )} + + {/* Mic / Stop button */} + {isRecording ? ( + ) : isTranscribing ? ( +
+ +
+ ) : ( + - - )} + )} - {/* Mic / Stop button */} - {isRecording ? ( - - ) : isTranscribing ? ( -
- -
- ) : ( - - )} + {/* Stop — only while an answer is in flight. It stays even though + the composer is now live, because stopping without redirecting is + still a thing users want (and Esc does the same). */} + {isStreaming && ( + + )} - {/* Stop — only while an answer is in flight. It stays even though the - composer is now live, because stopping without redirecting is still - a thing users want (and Esc does the same). */} - {isStreaming && ( + {/* Send — enabled mid-answer, because that is the whole feature. + The tooltip says what it will do before the user finds out: + sending discards the answer in flight and redirects the same + session. */} - )} - - {/* Send — enabled mid-answer, because that is the whole feature. The - tooltip says what it will do before the user finds out: sending - discards the answer in flight and redirects the same session. */} - +
From 916fb04c7114fc9162fbe2e0f9fe95e26f3acaaa Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 12:39:55 +0300 Subject: [PATCH 130/154] Make a strategy's session count open its newest session, as the dry-run count already did In both strategy views the dry-run count was a link to the newest dry run and the session count beside it was plain text, so the pair answered a click differently. The session count now opens the newest session (`run=s:N`); zero sessions stays a label, since there is nothing to open. The target is named explicitly: a bare Runs link falls back to the newest loop run, which can be a dry run. On the run screen's Playbook band both counts are now selections the screen makes (AgentRunScreen.showRun) rather than links. The old link wrote `?open=runs`, which replaced the open set and shut the Playbook under the cursor; the new move adds Runs to the set and scrolls to it, because Runs draws above the Playbook and would otherwise open out of sight. It scrolls immediately when Runs is already open, where an effect keyed on the open set would never fire. Run links are now written in the canonical `s:N` / `e:N` form via formatRunId; the legacy `e2` form still parses. --- .../agent/StrategyWorkbench.labels.test.tsx | 44 +++++++++++++++ .../components/agent/StrategyWorkbench.tsx | 37 ++++++++++-- .../agent/workspace/AgentRunScreen.test.tsx | 56 ++++++++++++++++++- .../agent/workspace/AgentRunScreen.tsx | 44 +++++++++++++-- .../agent/workspace/PlaybookView.test.tsx | 28 ++++++++++ .../agent/workspace/PlaybookView.tsx | 56 +++++++++++++++---- 6 files changed, 243 insertions(+), 22 deletions(-) diff --git a/frontend/src/components/agent/StrategyWorkbench.labels.test.tsx b/frontend/src/components/agent/StrategyWorkbench.labels.test.tsx index d6c0645d9..d3bdbacb2 100644 --- a/frontend/src/components/agent/StrategyWorkbench.labels.test.tsx +++ b/frontend/src/components/agent/StrategyWorkbench.labels.test.tsx @@ -69,6 +69,7 @@ vi.mock("@/hooks/useAgentExecutors", () => ({ })); const { StrategyWorkbench } = await import("./StrategyWorkbench"); +const { api } = await import("@/lib/api"); vi.mock("@/hooks/useFleetData", () => ({ useFleetData: () => ({ @@ -204,3 +205,46 @@ describe("in the chat's pane", () => { } }); }); + +/** + * The meta strip's two counts answer a click the same way. + * + * Only the dry-run count used to be a link. The session count beside it was a + * label in the same pill, so a reader who tried it first learned that neither + * one went anywhere. + */ +describe("the meta strip's counts", () => { + let base: Awaited>; + beforeEach(async () => { + base = await api.getStrategy("brigado", "fleet_op"); + }); + afterEach(() => { + vi.mocked(api.getStrategy).mockImplementation(async () => base); + }); + + const linkTo = (text: string) => + [...host.querySelectorAll("a")].find((a) => a.textContent?.includes(text)); + const params = (link: HTMLAnchorElement | undefined) => + new URL(link?.getAttribute("href") ?? "", "http://condor").searchParams; + + it("open the newest session and the newest dry run", async () => { + vi.mocked(api.getStrategy).mockResolvedValue({ + ...base, + sessions: [{ number: 1 }, { number: 4 }], + experiments: [{ number: 2 }], + } as typeof base); + await render(false); + + const sessions = params(linkTo("2 sessions")); + expect(sessions.get("run")).toBe("s:4"); + expect(sessions.get("strategy")).toBe("fleet_op"); + expect(sessions.get("open")).toBe("runs"); + expect(params(linkTo("1 dry run")).get("run")).toBe("e:2"); + }); + + it("leave zero sessions a label — there is no session to open", async () => { + await render(false); + expect(linkTo("session")).toBeUndefined(); + expect(host.textContent).toContain("0 sessions"); + }); +}); diff --git a/frontend/src/components/agent/StrategyWorkbench.tsx b/frontend/src/components/agent/StrategyWorkbench.tsx index 3e9b9699d..150db44c9 100644 --- a/frontend/src/components/agent/StrategyWorkbench.tsx +++ b/frontend/src/components/agent/StrategyWorkbench.tsx @@ -13,7 +13,7 @@ import { import { ConfirmDialog } from "@/components/agent/ConfirmDialog"; import { DeployedFleet } from "@/components/agent/DeployedFleet"; import { LoopPulse } from "@/components/agent/LoopPulse"; -import { isLiveRun, runFacts, runLabel } from "@/components/agent/lab/runs"; +import { formatRunId, isLiveRun, runFacts, runLabel } from "@/components/agent/lab/runs"; import { DiscardChangesDialog } from "@/components/editor/EditorDialogs"; import { ReportBrowser } from "@/components/routines/ReportBrowser"; import { ExecutorChart } from "@/components/charts/ExecutorChart"; @@ -239,6 +239,9 @@ export function StrategyWorkbench({ } const liveInstance = instances.find((i) => i.status === "running") ?? instances[0] ?? null; + const newestDryRun = strategy.experiments.length + ? Math.max(...strategy.experiments.map((e) => e.number)) + : 0; const actionClass = "flex items-center gap-1.5 rounded-lg border border-[var(--color-border)] bg-[var(--color-surface)] px-2.5 py-1.5 text-xs font-semibold text-[var(--color-text-muted)] transition-all hover:border-[var(--color-primary)]/50 hover:text-[var(--color-primary)]"; @@ -339,15 +342,39 @@ export function StrategyWorkbench({ {/* Meta strip */}
- - {strategy.sessions.length} session{strategy.sessions.length !== 1 ? "s" : ""} - + {/* A door to the newest session, as the dry-run count beside it is to + the newest dry run. Only that one used to be, so two counts in one + strip answered a click differently. */} + {strategy.sessions.length > 0 ? ( + + {strategy.sessions.length} session{strategy.sessions.length !== 1 ? "s" : ""} + + ) : ( + + 0 sessions + + )} {/* Said beside the sessions rather than only behind a button: a strategy whose whole history is one dry run used to read as one that had never run. */} {strategy.experiments.length > 0 && ( e.number))}` })} + to={labUrl({ + run: formatRunId({ + kind: "experiment", + number: newestDryRun, + id: String(newestDryRun), + }), + })} className="flex items-center gap-1 rounded border border-amber-500/30 bg-amber-500/10 px-2.5 py-1 text-amber-400 transition-colors hover:bg-amber-500/20" > diff --git a/frontend/src/components/agent/workspace/AgentRunScreen.test.tsx b/frontend/src/components/agent/workspace/AgentRunScreen.test.tsx index b4069532a..165ac4ef2 100644 --- a/frontend/src/components/agent/workspace/AgentRunScreen.test.tsx +++ b/frontend/src/components/agent/workspace/AgentRunScreen.test.tsx @@ -71,8 +71,17 @@ vi.mock("@/components/agent/workspace/MoneyView", () => ({ vi.mock("@/components/agent/workspace/AgentFleet", () => ({ AgentFleet: stub("fleet"), })); +// The Playbook's stub keeps one control: a count that names a run, which is the +// one move the band asks of the screen around it. vi.mock("@/components/agent/workspace/PlaybookView", () => ({ - PlaybookView: stub("playbook"), + PlaybookView: ({ onOpenRun }: { onOpenRun: (run: string) => void }) => { + mounted.push("playbook"); + return ( +
+
+ ); + }, })); vi.mock("@/components/agent/lab/RunRail", () => ({ RunRail: stub("rail") })); vi.mock("@/components/agent/lab/RunOverview", () => ({ @@ -447,3 +456,48 @@ describe("the index down the side (FEAT-120)", () => { expect(container.textContent).toContain("no strategies yet"); }); }); + +describe("a count in the Playbook that names a run", () => { + // jsdom has no layout and no `scrollIntoView`, so one is lent for these tests + // alone: the scroll is the half of this move a reader would miss without it. + let scrolled: Element[]; + beforeEach(() => { + scrolled = []; + Element.prototype.scrollIntoView = function (this: Element) { + scrolled.push(this); + }; + }); + afterEach(() => { + Reflect.deleteProperty(Element.prototype, "scrollIntoView"); + }); + + const count = () => + container.querySelector("[data-open-run]")!; + const scrolledTo = () => + scrolled.map((el) => el.getAttribute("data-section-body")); + + it("opens Runs beside the Playbook, on that run of this strategy", async () => { + await render("/?open=playbook"); + await click(count()); + + const params = new URLSearchParams(search()); + // Beside, not instead: the link this replaced wrote `?open=runs` and shut + // the Playbook under the reader's cursor. + expect(params.get("open")).toBe("runs.playbook"); + expect(params.get("run")).toBe("s:3"); + expect(params.get("strategy")).toBe("brl_mm"); + expect(bodies()).toEqual(["answers", "rail", "playbook"]); + }); + + it("brings Runs on screen, whether or not it was already open", async () => { + await render("/?open=playbook"); + await click(count()); + expect(scrolledTo()).toEqual(["runs"]); + + // Runs is open now, so `open` does not move on the second click — the case + // a scroll keyed on `open` alone would silently skip. + scrolled = []; + await click(count()); + expect(scrolledTo()).toEqual(["runs"]); + }); +}); diff --git a/frontend/src/components/agent/workspace/AgentRunScreen.tsx b/frontend/src/components/agent/workspace/AgentRunScreen.tsx index eb95ea124..5f43575f7 100644 --- a/frontend/src/components/agent/workspace/AgentRunScreen.tsx +++ b/frontend/src/components/agent/workspace/AgentRunScreen.tsx @@ -17,6 +17,7 @@ import { PlaybookView } from "@/components/agent/workspace/PlaybookView"; import { SectionRail } from "@/components/agent/workspace/SectionRail"; import { SECTION_META } from "@/components/agent/workspace/sectionMeta"; import { + serializeSections, useSections, type SectionId, } from "@/components/agent/workspace/sections"; @@ -276,11 +277,7 @@ export function AgentRunScreen({ if (!id) return; pendingScroll.current = null; if (!open.includes(id)) return; - bodyRef.current - ?.querySelector(`[data-section-body="${id}"]`) - // Guarded: jsdom has no layout, so it implements no `scrollIntoView`, - // and a rail click in a test must not throw for want of a viewport. - ?.scrollIntoView?.({ block: "start", behavior: "smooth" }); + scrollToSection(bodyRef.current, id); }, [open]); // The same action as the band's own header, reached from the index: one rule @@ -294,6 +291,33 @@ export function AgentRunScreen({ [toggle], ); + /** + * A run named from inside a band — the Playbook's session and dry-run counts. + * + * The selection a rail row makes, plus two things a door from further down + * the page owes the reader. It opens Runs *beside* the band it was clicked + * from, because `?open=` is a set and the old link replaced it, closing the + * Playbook under the reader's cursor. And it brings Runs on screen, because + * the band draws above the Playbook: opened from there, it grows the page + * out of sight and the click looks like it did nothing. + * + * Scrolls now when Runs is already open — `open` will not change, so the + * effect above would never spend the request. + */ + const showRun = useCallback( + (run: string) => { + const runsOpen = open.includes("runs"); + setParams({ + strategy: sslug, + run, + open: serializeSections([...open, "runs"]), + }); + if (runsOpen) scrollToSection(bodyRef.current, "runs"); + else pendingScroll.current = "runs"; + }, + [open, setParams, sslug], + ); + // The page has already guarded this by the time it mounts the screen — the // query is shared and warm — so this only shows on a hard reload racing it. if (isLoading || !agent) { @@ -498,6 +522,7 @@ export function AgentRunScreen({ sslug={sslug} strategy={strategy} onDeleted={() => setParams({ strategy: null })} + onOpenRun={showRun} /> ) : (

@@ -545,6 +570,15 @@ export function AgentRunScreen({ ); } +/** Bring one band's top to the top of the screen's own scroller. */ +function scrollToSection(body: HTMLElement | null, id: SectionId): void { + body + ?.querySelector(`[data-section-body="${id}"]`) + // Guarded: jsdom has no layout, so it implements no `scrollIntoView`, + // and a rail click in a test must not throw for want of a viewport. + ?.scrollIntoView?.({ block: "start", behavior: "smooth" }); +} + /** * One band of evidence, which renders nothing at all until it is opened. * diff --git a/frontend/src/components/agent/workspace/PlaybookView.test.tsx b/frontend/src/components/agent/workspace/PlaybookView.test.tsx index 032ee38e6..c101da85e 100644 --- a/frontend/src/components/agent/workspace/PlaybookView.test.tsx +++ b/frontend/src/components/agent/workspace/PlaybookView.test.tsx @@ -63,6 +63,8 @@ declare global { let container: HTMLDivElement; let root: Root; +/** What a count in the band asks the screen to open. */ +const onOpenRun = vi.fn(); const STRATEGY = { slug: "pmm_king", @@ -110,6 +112,7 @@ function render(strategy: StrategyDetail = STRATEGY) { sslug="pmm_king" strategy={strategy} onDeleted={() => {}} + onOpenRun={onOpenRun} /> , @@ -184,3 +187,28 @@ describe("PlaybookView", () => { expect(sw?.textContent).toContain("resumes on restart"); }); }); + +describe("the run counts", () => { + beforeEach(() => onOpenRun.mockClear()); + + const count = (pattern: RegExp) => + [...container.querySelectorAll("button")].find((b) => + pattern.test(b.textContent?.trim() ?? ""), + ); + + it("open the newest session and the newest dry run", () => { + // Only the dry-run count used to go anywhere; the session count beside it + // was a label in the same pill. + render(); + act(() => count(/^2 sessions$/)!.click()); + expect(onOpenRun).toHaveBeenLastCalledWith("s:2"); + act(() => count(/^1 dry run$/)!.click()); + expect(onOpenRun).toHaveBeenLastCalledWith("e:1"); + }); + + it("leave zero sessions a label — there is no session to open", () => { + render({ ...STRATEGY, sessions: [] }); + expect(count(/^\d+ sessions?$/)).toBeUndefined(); + expect(container.textContent).toContain("0 sessions"); + }); +}); diff --git a/frontend/src/components/agent/workspace/PlaybookView.tsx b/frontend/src/components/agent/workspace/PlaybookView.tsx index 438d5c9e4..7e9bc80cf 100644 --- a/frontend/src/components/agent/workspace/PlaybookView.tsx +++ b/frontend/src/components/agent/workspace/PlaybookView.tsx @@ -12,12 +12,12 @@ import { X, } from "lucide-react"; import { useMemo, useState } from "react"; -import { Link } from "react-router-dom"; import ReactMarkdown from "react-markdown"; import remarkGfm from "remark-gfm"; import { MarkdownEditor } from "@/components/agent/AgentOverviewTab"; import { ConfirmDialog } from "@/components/agent/ConfirmDialog"; +import { formatRunId } from "@/components/agent/lab/runs"; import { DiscardChangesDialog } from "@/components/editor/EditorDialogs"; import { ReportBrowser } from "@/components/routines/ReportBrowser"; import { countdown } from "@/lib/agent-attribution"; @@ -55,6 +55,7 @@ export function PlaybookView({ sslug, strategy, onDeleted, + onOpenRun, }: { slug: string; sslug: string; @@ -62,6 +63,8 @@ export function PlaybookView({ strategy: StrategyDetail; /** The host's move after a delete: the run screen drops `?strategy=`. */ onDeleted: () => void; + /** Select a run of this strategy (`s:3`, `e:1`) and bring Runs on screen. */ + onOpenRun: (run: string) => void; }) { const queryClient = useQueryClient(); const [showRoutines, setShowRoutines] = useState(false); @@ -83,6 +86,10 @@ export function PlaybookView({ refetchInterval: 5000, }); + const sessions = strategy.sessions.length; + const newestSession = sessions + ? Math.max(...strategy.sessions.map((s) => s.number)) + : 0; const dryRuns = strategy.experiments.length; const newestDryRun = dryRuns ? Math.max(...strategy.experiments.map((e) => e.number)) @@ -106,22 +113,49 @@ export function PlaybookView({

)}
- - {strategy.sessions.length} session - {strategy.sessions.length === 1 ? "" : "s"} - + {/* Both counts are doors to the newest run of their kind. Only the + dry-run one used to be, so a pair of counts read as one link and + one label. They are buttons and not links because the move is a + selection on this screen: it opens the Runs band *beside* this + one and scrolls to it, which only the screen knows how to do. */} + {sessions > 0 ? ( + + ) : ( + 0 sessions + )} {/* A strategy whose whole history is one dry run used to read as - one that had never run, so the count is a link and not a note. */} + one that had never run, so the count is a door and not a note. */} {dryRuns > 0 && ( - + onOpenRun( + formatRunId({ + kind: "experiment", + number: newestDryRun, + id: String(newestDryRun), + }), + ) + } className="flex items-center gap-1 rounded border border-amber-500/30 bg-amber-500/10 px-2 py-0.5 text-amber-500 transition-colors hover:bg-amber-500/20" > {dryRuns} dry run{dryRuns === 1 ? "" : "s"} - + )} {strategy.slug} {strategy.agent_id && {strategy.agent_id}} From 247cdafcfbd9e71e8f3267b03f91019103ee8122 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 13:17:31 +0300 Subject: [PATCH 131/154] Give PTB's startup calls 20s instead of 5s, so a slow link no longer kills the process at getMe --- main.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/main.py b/main.py index 4c95d70b7..18cd21f63 100644 --- a/main.py +++ b/main.py @@ -1020,9 +1020,15 @@ def main() -> None: # In local mode there is no token and nothing polls; the placeholder exists # only so the Application (and with it job_queue, CallbackContext and the # handler registry) can be built at all. Nothing ever calls Telegram with it. + # PTB's 5s defaults kill startup on a slow link: initialize() calls getMe, + # and one connect that takes >5s raises TimedOut and exits the process. application = ( Application.builder() .token(TELEGRAM_TOKEN or "0:local") + .connect_timeout(20) + .read_timeout(20) + .get_updates_connect_timeout(20) + .get_updates_read_timeout(30) .persistence(persistence) .concurrent_updates(True) .build() From 0d21fe30be2ce2b92a67e07aaf53d75a2000b82a Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 13:17:31 +0300 Subject: [PATCH 132/154] An approval reads as a paused command on the composer: it previews the call, counts down to the auto-deny, and says a chat reply cancels it --- condor/runtime/confirmations.py | 10 ++ condor/web/routes/chat_ws.py | 7 + .../components/chat/ApprovalPrompt.test.tsx | 121 +++++++++++++++ .../src/components/chat/ApprovalPrompt.tsx | 144 ++++++++++++++++++ frontend/src/components/chat/ChatThread.tsx | 59 +++---- .../useChatSocket.confirmations.test.tsx | 15 ++ frontend/src/hooks/useChatSocket.ts | 30 ++++ frontend/src/lib/api.ts | 6 + tests/test_chat_ws_permissions.py | 29 ++++ 9 files changed, 382 insertions(+), 39 deletions(-) create mode 100644 frontend/src/components/chat/ApprovalPrompt.test.tsx create mode 100644 frontend/src/components/chat/ApprovalPrompt.tsx diff --git a/condor/runtime/confirmations.py b/condor/runtime/confirmations.py index 00797aa4f..5154e5fe2 100644 --- a/condor/runtime/confirmations.py +++ b/condor/runtime/confirmations.py @@ -89,12 +89,22 @@ def to_wire(self) -> dict: "user_id": self.user_id, "summary": self.summary, "origin": self.origin, + # The call itself, normalized once here so every surface previews + # the same thing the gate judged: the bare tool name and its + # arguments, or ``None`` when they could not be read. A summary line + # alone is what made the prompt read like a notice rather than a + # command waiting to run. + "tool": danger.tool_call_name(self.tool_call), + "input": danger.tool_call_input(self.tool_call), "tool_call": self.tool_call, "options": self.options, "status": self.status.value, "selected_option_id": self.selected_option_id, "created_at": self.created_at, "expires_at": self.expires_at, + # Relative, so a surface can count down without trusting its own + # clock to agree with this one about what ``expires_at`` means. + "expires_in": max(0.0, self.expires_at - time.time()), } diff --git a/condor/web/routes/chat_ws.py b/condor/web/routes/chat_ws.py index 1356f01fe..a7ff13058 100644 --- a/condor/web/routes/chat_ws.py +++ b/condor/web/routes/chat_ws.py @@ -289,6 +289,7 @@ def __init__(self, ws: WebSocket): self._ws = ws async def deliver(self, pending: PendingConfirmation) -> None: + wire = pending.to_wire() await _send_turn( self._ws, pending.user_id, @@ -305,6 +306,12 @@ async def deliver(self, pending: PendingConfirmation) -> None: # Which agent, on which server, is asking. The slot addresses # the request; this says out loud what the user is authorizing. "origin": pending.origin, + # The call itself and the time left to answer it, so the prompt + # previews a command that is paused rather than reading like a + # notice the user can leave for later. + "tool": wire["tool"], + "input": wire["input"], + "expires_in": wire["expires_in"], }, ) diff --git a/frontend/src/components/chat/ApprovalPrompt.test.tsx b/frontend/src/components/chat/ApprovalPrompt.test.tsx new file mode 100644 index 000000000..3e9de60f3 --- /dev/null +++ b/frontend/src/components/chat/ApprovalPrompt.test.tsx @@ -0,0 +1,121 @@ +/** + * The approval prompt must read as a paused command, not a warning. + * + * Users left agents "running" while they sat blocked on an approval that + * looked like a notice — often right after typing "confirm" in chat. These pin + * the parts that make it unmistakable: it previews the call, counts down to the + * automatic deny, says a chat reply is not an answer, and answers by id. + * + * @vitest-environment jsdom + */ + +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { PermissionRequest } from "@/hooks/useChatSocket"; +import { ApprovalPrompt } from "./ApprovalPrompt"; + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +let container: HTMLDivElement; +let root: Root; + +const NOW = 1_700_000_000_000; + +function request(partial: Partial = {}): PermissionRequest { + return { + request_id: "abc123", + summary: "BUY 1 SOL-USDC (MARKET) on binance", + origin: "brigado on moneymaker", + tool: "place_order", + input: { trading_pair: "SOL-USDC", amount: 1 }, + deadline: NOW / 1000 + 90, + ...partial, + }; +} + +function render(req: PermissionRequest, onResolve = vi.fn()) { + act(() => root.render()); + return onResolve; +} + +function button(name: string): HTMLButtonElement { + const match = [...container.querySelectorAll("button")].find( + (b) => b.textContent === name, + ); + if (!match) throw new Error(`no "${name}" button`); + return match; +} + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + vi.useFakeTimers(); + vi.setSystemTime(NOW); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + vi.useRealTimers(); +}); + +describe("ApprovalPrompt", () => { + it("says the agent is paused and previews the call it would run", () => { + render(request()); + + const text = container.textContent ?? ""; + expect(text).toContain("Paused — waiting for your approval"); + expect(text).toContain("brigado on moneymaker wants to run:"); + expect(text).toContain("BUY 1 SOL-USDC (MARKET) on binance"); + expect(container.querySelector("pre")?.textContent).toContain('"amount": 1'); + }); + + it("says a chat reply is not the answer", () => { + render(request()); + + expect(container.textContent).toContain("sending one cancels this call"); + }); + + it("answers by request id", () => { + const onResolve = render(request()); + + act(() => button("Allow").click()); + expect(onResolve).toHaveBeenLastCalledWith("abc123", true); + + act(() => button("Deny").click()); + expect(onResolve).toHaveBeenLastCalledWith("abc123", false); + }); + + it("counts down to the automatic deny", () => { + render(request()); + expect(container.textContent).toContain("1:30"); + + act(() => vi.advanceTimersByTime(31_000)); + expect(container.textContent).toContain("0:59"); + }); + + it("stops offering Allow once the deadline has passed", () => { + const onResolve = render(request({ deadline: NOW / 1000 + 2 })); + + act(() => vi.advanceTimersByTime(3_000)); + + expect(container.textContent).toContain("Approval timed out"); + expect(() => button("Allow")).toThrow(); + act(() => button("Dismiss").click()); + expect(onResolve).toHaveBeenLastCalledWith("abc123", false); + }); + + it("renders an older backend's summary-only request without inventing a preview", () => { + render(request({ tool: undefined, input: undefined, deadline: undefined, origin: "" })); + + expect(container.textContent).toContain("The agent wants to run:"); + expect(container.querySelector("pre")).toBeNull(); + expect(button("Allow")).toBeTruthy(); + }); +}); diff --git a/frontend/src/components/chat/ApprovalPrompt.tsx b/frontend/src/components/chat/ApprovalPrompt.tsx new file mode 100644 index 000000000..5155a59f5 --- /dev/null +++ b/frontend/src/components/chat/ApprovalPrompt.tsx @@ -0,0 +1,144 @@ +import { useEffect, useState } from "react"; +import { Hand } from "lucide-react"; + +import type { PermissionRequest } from "@/hooks/useChatSocket"; +import { formatToolName } from "@/lib/formatters"; + +/** + * Whole seconds until `deadline`, ticking once a second; null when the + * backend did not say. Stops ticking at zero — nothing changes after that. + */ +function useSecondsLeft(deadline?: number): number | null { + const [now, setNow] = useState(() => Date.now() / 1000); + const running = deadline !== undefined && deadline > now; + useEffect(() => { + if (!running) return; + const timer = setInterval(() => setNow(Date.now() / 1000), 1000); + return () => clearInterval(timer); + }, [running]); + if (deadline === undefined) return null; + return Math.max(0, Math.ceil(deadline - now)); +} + +function mmss(seconds: number): string { + const m = Math.floor(seconds / 60); + const s = seconds % 60; + return `${m}:${String(s).padStart(2, "0")}`; +} + +/** + * A tool call that is paused until the user answers it. + * + * This used to be an amber strip above the transcript with a triangle in it — + * the same furniture as a warning — and it was read as one: users left the + * agent "running" while it sat blocked on a click, often right after typing + * "confirm" in chat and reasonably believing that was the approval. So it is + * built as the opposite of a notice, closer to a terminal's permission prompt: + * + * - it sits on the composer, where the user's attention already is, and + * cannot scroll out of view; + * - its header says the agent is paused and on whom; + * - it previews the call itself — tool and arguments — not just a summary line; + * - it counts down to the automatic deny, which a notice never does; + * - it says out loud that a chat reply is not an answer. On the dashboard a + * message sent mid-turn steers the agent, and steering *denies* the pending + * call (`condor.runtime.client.prompt`), so "confirm" typed here cancels it. + */ +export function ApprovalPrompt({ + request, + onResolve, +}: { + request: PermissionRequest; + onResolve: (requestId: string, approved: boolean) => void; +}) { + const secondsLeft = useSecondsLeft(request.deadline); + const expired = secondsLeft === 0; + const args = + request.input && Object.keys(request.input).length > 0 + ? JSON.stringify(request.input, null, 2) + : null; + + return ( +
+
+ {!expired && ( +
+ +
+

+ {request.origin ? `${request.origin} wants to run:` : "The agent wants to run:"} +

+

{request.summary}

+ {request.tool && ( +
+
+ {formatToolName(request.tool)} +
+ {args && ( + 
+                {args}
+
+ )} +
+ )} +
+ +
+ {expired ? ( + <> +

+ Nobody answered in time, so it was not run. +

+ + + ) : ( + <> + + +

+ Nothing runs until you choose. A chat reply is not an answer — + sending one cancels this call. +

+ + )} +
+
+ ); +} diff --git a/frontend/src/components/chat/ChatThread.tsx b/frontend/src/components/chat/ChatThread.tsx index 0272c32b6..dfbce9a82 100644 --- a/frontend/src/components/chat/ChatThread.tsx +++ b/frontend/src/components/chat/ChatThread.tsx @@ -1,9 +1,10 @@ import { useCallback, useEffect, useMemo, useRef } from "react"; import { AlertTriangle, Bot, Brain, Loader2, MessageSquare, X } from "lucide-react"; -import type { ChatSlot } from "@/hooks/useChatSocket"; +import type { ChatSlot, PermissionRequest } from "@/hooks/useChatSocket"; import { CHAT_SLUG, type AgentSummary, type ChatAgentOption } from "@/lib/api"; import { speakerNames } from "@/lib/agentColor"; +import { ApprovalPrompt } from "./ApprovalPrompt"; import { ChatInput } from "./ChatInput"; import { ChatMessageView } from "./ChatMessage"; import { Starters, type Starter } from "./Starters"; @@ -61,11 +62,7 @@ export function ChatThread({ * identical to a dropped one is the bug this replaces. */ isQueued?: boolean; - permissionRequest: { - request_id: string; - summary: string; - origin?: string; - } | null; + permissionRequest: PermissionRequest | null; onResolvePermission: (requestId: string, approved: boolean) => void; switchError?: string | null; onDismissSwitchError?: () => void; @@ -211,38 +208,6 @@ export function ChatThread({ return ( <> - {/* Permission request banner */} - {permissionRequest && ( -
-
- -
-

- Confirm action - {permissionRequest.origin ? ` — ${permissionRequest.origin}` : ""} -

-

- {permissionRequest.summary} -

-
- - -
-
-
-
- )} - {/* Switch failure — the header still shows whoever is actually answering */} {switchError && (
@@ -340,6 +305,16 @@ export function ChatThread({ {/* Input */} {slot && (
+ {/* On the composer, not above the transcript: the paused call sits + where the user is already looking and cannot scroll away. */} + {permissionRequest && ( + + )} { }); }); + it("carries the call and a local deadline, so the prompt can preview and count down", async () => { + getPendingConfirmations.mockResolvedValue([ + { ...stranded, tool: "place_order", input: { amount: 1 }, expires_in: 90 }, + ]); + const before = Date.now() / 1000; + + await arrive(); + + const req = chat().permissionRequests.s1; + expect(req.tool).toBe("place_order"); + expect(req.input).toEqual({ amount: 1 }); + expect(req.deadline).toBeGreaterThanOrEqual(before + 90); + expect(req.deadline).toBeLessThanOrEqual(Date.now() / 1000 + 90); + }); + it("files an approval with no slot where an unaddressed one goes", async () => { getPendingConfirmations.mockResolvedValue([{ ...stranded, slot_id: "" }]); diff --git a/frontend/src/hooks/useChatSocket.ts b/frontend/src/hooks/useChatSocket.ts index de66155dd..c737afd03 100644 --- a/frontend/src/hooks/useChatSocket.ts +++ b/frontend/src/hooks/useChatSocket.ts @@ -216,6 +216,21 @@ export interface PermissionRequest { summary: string; /** Which agent, on which server, raised it. Empty when unattributable. */ origin?: string; + /** The bare tool name, previewed like the command it is. */ + tool?: string; + /** Its arguments; null when the backend could not read them. */ + input?: Record | null; + /** + * When the runtime denies it unanswered, in *this* browser's epoch seconds — + * derived from the server's relative `expires_in` on arrival, so a skewed + * local clock cannot make the countdown lie. Absent from an older backend. + */ + deadline?: number; +} + +/** A relative `expires_in` from the wire, as a local deadline. */ +export function deadlineFrom(expiresIn: unknown): number | undefined { + return typeof expiresIn === "number" ? Date.now() / 1000 + expiresIn : undefined; } /** @@ -1057,6 +1072,9 @@ export function useChatSocket() { request_id: p.id, summary: p.summary, origin: p.origin || "", + tool: p.tool, + input: p.input, + deadline: deadlineFrom(p.expires_in), }; added = true; } @@ -1666,6 +1684,9 @@ export function useChatSocket() { request_id: data.request_id as string, summary: data.summary as string, origin: (data.origin as string) || "", + tool: typeof data.tool === "string" ? data.tool : undefined, + input: (data.input as Record | null | undefined) ?? null, + deadline: deadlineFrom(data.expires_in), }, })); break; @@ -1677,6 +1698,15 @@ export function useChatSocket() { // answer that trailed off — the alternative the old dead composer // avoided by never letting this happen at all. if (!slotId) break; + // Steering denies whatever the turn was waiting on + // (`condor.runtime.client.prompt`), so an approval still on screen + // would offer an Allow that can no longer do anything. + setPermissionRequests((prev) => { + if (!(slotId in prev)) return prev; + const next = { ...prev }; + delete next[slotId]; + return next; + }); flushChunks(slotId); updateSlotMessages(slotId, (prev) => { const msgs = settleToolCalls(prev); diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 0a2d27458..cdda52279 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -2280,6 +2280,12 @@ export interface PendingConfirmation { summary: string; origin: string; expires_at: number; + /** Seconds until the runtime denies it, measured on the server's clock. */ + expires_in?: number; + /** The bare tool name the gate judged. */ + tool?: string; + /** Its arguments, or null when they could not be read. */ + input?: Record | null; } export const api = { diff --git a/tests/test_chat_ws_permissions.py b/tests/test_chat_ws_permissions.py index 80a61a865..73c78f689 100644 --- a/tests/test_chat_ws_permissions.py +++ b/tests/test_chat_ws_permissions.py @@ -102,6 +102,35 @@ def test_delivered_request_names_the_slot_that_raised_it(registry): # The rest of the shape is a live contract with the shipped dashboard. assert frame["request_id"] == pending.id assert frame["summary"] == "buy" + # What the prompt previews: the call the gate judged, and the time left. + assert frame["tool"] == "place_order" + assert frame["input"] == {} + assert 0 < frame["expires_in"] <= 30 + + +def test_delivered_request_previews_the_normalized_call(registry): + """The dashboard shows the bare tool name and the parsed arguments.""" + call = {"tool": "mcp__mcp-hummingbot__place_order", "input": '{"amount": 1}'} + pending = registry.register("web:1:conv-abc", USER_A, "buy", call, OPTIONS, 30) + ws = _FakeWS() + + asyncio.run(WebSocketChannel(ws).deliver(pending)) + + (frame,) = ws.sent + assert frame["tool"] == "place_order" + assert frame["input"] == {"amount": 1} + + +def test_unreadable_arguments_are_sent_as_none(registry): + """No invented preview: arguments the gate could not read stay unread.""" + call = {"tool": "place_order", "input": "not json"} + pending = registry.register("web:1:conv-abc", USER_A, "buy", call, OPTIONS, 30) + ws = _FakeWS() + + asyncio.run(WebSocketChannel(ws).deliver(pending)) + + (frame,) = ws.sent + assert frame["input"] is None def test_unparseable_session_key_still_delivers(registry): From 86036c97c1a0d1a370ef541b3eaaa492f759f107 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 13:40:21 +0300 Subject: [PATCH 133/154] Move CI onto the Node 24 releases of its actions, and build the frontend on Node 22 now that 20 is EOL --- .github/workflows/ci.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7678cf7c..09239aa4c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,13 +11,13 @@ jobs: name: Backend Lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v4 + - uses: astral-sh/setup-uv@v10.0.1 with: version: "latest" - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.12" @@ -34,13 +34,13 @@ jobs: name: Backend Import Check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v4 + - uses: astral-sh/setup-uv@v10.0.1 with: version: "latest" - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.12" @@ -60,13 +60,13 @@ jobs: name: Backend Tests runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v4 + - uses: astral-sh/setup-uv@v10.0.1 with: version: "latest" - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.12" @@ -83,11 +83,11 @@ jobs: run: working-directory: frontend steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: - node-version: "20" + node-version: "22" cache: "npm" cache-dependency-path: frontend/package-lock.json From bd13057b407cabae5ad6d8d59ff128a012ceb624 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 18:44:26 +0300 Subject: [PATCH 134/154] Take the Gateway container off the MCP surface: the dashboard already starts, stops, restarts and tails it behind the owner check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit manage_gateway_container is gone, with its schema, formatter, profile entry and its SEC-565 confirmation gate. A failed Gateway call now points the user at Settings → Gateway → Logs instead of at the tool. --- condor/runtime/danger.py | 53 --------- condor/runtime/toolsets.py | 4 +- handlers/agents/_shared.py | 1 - .../hummingbot_api/formatters/__init__.py | 2 - .../hummingbot_api/formatters/gateway.py | 33 ------ mcp_servers/hummingbot_api/middleware.py | 5 +- mcp_servers/hummingbot_api/profiles.py | 14 ++- mcp_servers/hummingbot_api/schemas.py | 39 ------- mcp_servers/hummingbot_api/server.py | 28 ----- mcp_servers/hummingbot_api/tools/gateway.py | 60 +--------- tests/test_acp_permission_gate.py | 72 +----------- tests/test_dangerous_gate_names_resolve.py | 15 --- tests/test_hummingbot_mcp_tools.py | 104 +++--------------- tests/test_mcp_tool_profiles.py | 3 +- 14 files changed, 35 insertions(+), 398 deletions(-) diff --git a/condor/runtime/danger.py b/condor/runtime/danger.py index 092f8cfc8..211d6d2fc 100644 --- a/condor/runtime/danger.py +++ b/condor/runtime/danger.py @@ -49,7 +49,6 @@ "manage_clmm", # every action that moves liquidity "manage_amm", # every action that moves liquidity "manage_gateway_config", # only writes to networks/connectors; see below - "manage_gateway_container", # only the lifecycle actions; see below "control_agent", # only `start`, which launches an unattended trading loop # The executor family is gated by NAME (FEAT-062), the same way the swap family # is: a create and a stop each have their own tool, so there is no `action` to @@ -126,29 +125,6 @@ # so the gate has to know both or `start_agent` walks straight past it. DANGEROUS_CONTROL_ACTIONS = {"start", "start_agent"} -# Actions within manage_gateway_container that require confirmation (SEC-565). -# This is the only registered tool that controls a container lifecycle, and the -# gate weighs what a call does rather than whether it signs: `manage_gateway_config` -# is gated while signing nothing, and toolsets.py already calls container control an -# owner-only operator action. -# -# `stop` and `restart` take Gateway down underneath any live CLMM/LP executor that -# needs it to manage or close a position. `start` is the wider one: the impl passes -# the caller's `config` dict straight to the API host (tools/gateway.py), and that -# dict names the Docker `image` — so an ungated `start` lets a model whose input -# includes untrusted tool output (pool names, bot logs, GeckoTerminal rows) choose -# what runs on the host holding the exchange API keys and the Gateway wallet keys. -# -# `get_status` and `get_logs` stay on the fast path. `get_logs` in particular is the -# escape hatch every opaque Gateway failure points at (middleware.GATEWAY_LOG_HINT), -# and putting a prompt in front of reading a log would put one in front of diagnosis. -# -# Unlike control_agent there is no second spelling to cover: the action is a plain -# five-member Literal validated by pydantic (schemas.GatewayContainerRequest) and the -# impl dispatches on it directly, with no alias resolver. An unreadable or missing -# action still fails closed. -DANGEROUS_CONTAINER_ACTIONS = {"start", "stop", "restart"} - # Resource types within manage_gateway_config whose *writes* require confirmation # (SEC-566). This set used to be empty, justified by "everything it touches is # Gateway's own symbol/address mapping". That is true of two of the four resources @@ -214,10 +190,6 @@ "set_state", } -#: `manage_gateway_container`'s writes. Identical to its confirmation set: the -#: three lifecycle actions are everything it can change, and the other two read. -MUTATING_CONTAINER_ACTIONS = DANGEROUS_CONTAINER_ACTIONS - # The reads of each dispatch tool, named explicitly. They are what keeps the # fail-open rule below from recording a `manage_bots(action="status")`: an # action in neither set is one this module has not heard of, and *that* is what @@ -232,9 +204,6 @@ "quote_liquidity", } READ_ONLY_CONTROL_ACTIONS = {"list", "list_agents", "get_state"} -#: `manage_gateway_container`'s reads, named so the fail-open rule below does not -#: record a status poll or a log tail as a change to the world. -READ_ONLY_CONTAINER_ACTIONS = {"get_status", "get_logs"} #: `manage_controllers`' writes and reads. The tool is outside the *gate* #: entirely and stays there (it writes controller templates and saved configs, #: never a running bot), but a fleet is *built* out of these calls: the twelve @@ -491,9 +460,6 @@ def is_dangerous_tool_call(tool_call: dict[str, Any]) -> bool: if tool_name == "manage_gateway_config": return _is_dangerous_config_call(tool_call) - if tool_name == "manage_gateway_container": - return _has_dangerous_action(tool_call, DANGEROUS_CONTAINER_ACTIONS) - if tool_name == "control_agent": return _has_dangerous_action(tool_call, DANGEROUS_CONTROL_ACTIONS) @@ -563,11 +529,6 @@ def is_mutating_tool_call(tool_call: dict[str, Any]) -> bool: tool_call, MUTATING_CONTROL_ACTIONS, READ_ONLY_CONTROL_ACTIONS ) - if tool_name == "manage_gateway_container": - return _is_mutating_action( - tool_call, MUTATING_CONTAINER_ACTIONS, READ_ONLY_CONTAINER_ACTIONS - ) - if tool_name == "manage_gateway_config": # Recorded unless it is one of the two reads. The resource type is read # only to stay a superset of the gate, which fails closed on a missing @@ -837,20 +798,6 @@ def format_tool_summary(tool_call: dict[str, Any]) -> str: return f"Gateway config: {action} {resource} '{target}', setting {keys}" return f"Gateway config: {action} {resource}" - if tool_name == "manage_gateway_container": - # `start` and `restart` hand a caller-chosen Docker image to the API host, - # so the line names the image: the human is approving what will run on the - # box that holds the exchange and wallet keys, not the word "start". - action = input_data.get("action", "?") - config = input_data.get("config") - image = config.get("image") if isinstance(config, dict) else None - if action in ("start", "restart"): - what = "Start" if action == "start" else "Restart" - return f"{what} the Gateway container with image {image or 'default'}" - if action == "stop": - return "Stop the Gateway container (live LP positions lose their manager)" - return f"Gateway container: {action}" - if tool_name in ("manage_clmm", "manage_amm"): action = input_data.get("action", "?") kind = "CLMM" if tool_name == "manage_clmm" else "AMM" diff --git a/condor/runtime/toolsets.py b/condor/runtime/toolsets.py index 41c8f7277..a8588523d 100644 --- a/condor/runtime/toolsets.py +++ b/condor/runtime/toolsets.py @@ -344,8 +344,8 @@ def candidates(): server_name = next((name for name in candidates() if usable(name)), None) # The admin ring belongs to the server's owner (SEC-252). ``full`` is the - # only profile that mounts ADMIN_TOOLS on mcp-hummingbot — configure_server, - # manage_gateway_config, manage_gateway_container — and those act on the + # only profile that mounts ADMIN_TOOLS on mcp-hummingbot — configure_server + # and manage_gateway_config — and those act on the # resolved server with the OWNER's credentials, injected into the env below, # while enforcing no permission of their own: the subprocess has no notion of # the calling user, so nothing downstream can. Every other surface already diff --git a/handlers/agents/_shared.py b/handlers/agents/_shared.py index ebb2fe9bc..bdafdc3a3 100644 --- a/handlers/agents/_shared.py +++ b/handlers/agents/_shared.py @@ -37,7 +37,6 @@ DANGEROUS_BOT_ACTIONS, DANGEROUS_CLMM_ACTIONS, DANGEROUS_CONFIG_RESOURCES, - DANGEROUS_CONTAINER_ACTIONS, DANGEROUS_CONTROL_ACTIONS, DANGEROUS_TOOLS, is_dangerous_tool_call, diff --git a/mcp_servers/hummingbot_api/formatters/__init__.py b/mcp_servers/hummingbot_api/formatters/__init__.py index f906209c6..d185e5003 100644 --- a/mcp_servers/hummingbot_api/formatters/__init__.py +++ b/mcp_servers/hummingbot_api/formatters/__init__.py @@ -47,7 +47,6 @@ format_clmm_result, format_gateway_clmm_pool_result, format_gateway_config_result, - format_gateway_container_result, format_gateway_swap_result, ) @@ -67,7 +66,6 @@ __all__ = [ # Gateway formatters - "format_gateway_container_result", "format_gateway_config_result", "format_gateway_swap_result", "format_gateway_clmm_pool_result", diff --git a/mcp_servers/hummingbot_api/formatters/gateway.py b/mcp_servers/hummingbot_api/formatters/gateway.py index 6f54c53fa..5643a68f9 100644 --- a/mcp_servers/hummingbot_api/formatters/gateway.py +++ b/mcp_servers/hummingbot_api/formatters/gateway.py @@ -8,39 +8,6 @@ from .table_builder import ColumnDef, TableBuilder -def format_gateway_container_result(result: dict[str, Any]) -> str: - """Format gateway container action results into a human-readable string.""" - result_action = result.get("action", "") - - if result_action == "get_status": - status = result.get("status", {}) - running = status.get("running", False) - container_id = status.get("container_id") - created_at = status.get("created_at") - - container_id_display = f"{container_id[:12]}..." if container_id else "None" - created_at_display = created_at[:19] if created_at else "None" - - return ( - f"Gateway Container Status:\n\n" - f"Status: {'Running ✓' if running else 'Stopped ✗'}\n" - f"Container ID: {container_id_display}\n" - f"Image: {status.get('image') or 'None'}\n" - f"Port: {status.get('port') or 'None'}\n" - f"Created: {created_at_display}" - ) - - elif result_action == "get_logs": - logs = result.get("logs", "No logs available") - return f"Gateway Container Logs:\n\n{logs}" - - elif result_action in ["start", "stop", "restart"]: - message = result.get("message", "") - return f"Gateway Container: {message}" - - return f"Gateway Container Result: {result}" - - def format_gateway_config_result(result: dict[str, Any]) -> str: """Format gateway config action results into a human-readable string.""" result_resource_type = result.get("resource_type", "") diff --git a/mcp_servers/hummingbot_api/middleware.py b/mcp_servers/hummingbot_api/middleware.py index 84875261a..7169a695a 100644 --- a/mcp_servers/hummingbot_api/middleware.py +++ b/mcp_servers/hummingbot_api/middleware.py @@ -17,7 +17,10 @@ T = TypeVar("T") -GATEWAY_LOG_HINT = "\n\n💡 Check gateway logs for more details: manage_gateway_container(action='get_logs')" +GATEWAY_LOG_HINT = ( + "\n\n💡 Gateway's logs usually say why. They are in the Condor dashboard " + "(Settings → Gateway → Logs); point the user there." +) def handle_errors( diff --git a/mcp_servers/hummingbot_api/profiles.py b/mcp_servers/hummingbot_api/profiles.py index 2726b7dd5..879fb81bb 100644 --- a/mcp_servers/hummingbot_api/profiles.py +++ b/mcp_servers/hummingbot_api/profiles.py @@ -51,7 +51,6 @@ "manage_clmm": "Direct CLMM position operations", "configure_server": "Repoint this seat at another Hummingbot API server", "manage_gateway_config": "Read and edit Gateway's chains, tokens and wallets", - "manage_gateway_container": "Gateway container status, start, stop and logs", } #: The trading surface: everything an autonomous tick needs to read a market, @@ -113,14 +112,17 @@ "manage_clmm", ) -#: Infrastructure. Repointing the API server, rewriting Gateway's config and -#: restarting its container are operator actions with a human in front of them: -#: the chat, or a standalone host. No agent's tool list names one, and the chat's -#: own context prompt already says not to call ``configure_server``. +#: Infrastructure. Repointing the API server and rewriting Gateway's config are +#: operator actions with a human in front of them: the chat, or a standalone +#: host. No agent's tool list names one, and the chat's own context prompt +#: already says not to call ``configure_server``. +#: +#: The Gateway container has no tool at all. Starting, stopping, restarting it +#: and reading its logs happen in the dashboard (Settings → Gateway), behind the +#: server-owner check, and nowhere a model can reach. ADMIN_TOOLS: tuple[str, ...] = ( "configure_server", "manage_gateway_config", - "manage_gateway_container", ) #: profile name → the tools it registers. ``full`` is the default because this diff --git a/mcp_servers/hummingbot_api/schemas.py b/mcp_servers/hummingbot_api/schemas.py index 425761a27..c9073f241 100644 --- a/mcp_servers/hummingbot_api/schemas.py +++ b/mcp_servers/hummingbot_api/schemas.py @@ -15,45 +15,6 @@ # ============================================================================== -class GatewayContainerRequest(BaseModel): - """Request model for Gateway container management with progressive disclosure. - - This model supports container lifecycle management: - - get_status: Check if Gateway is running and get container details - - start: Start Gateway container with configuration - - stop: Stop Gateway container - - restart: Restart Gateway (optionally with new configuration) - - get_logs: Retrieve Gateway container logs - """ - - action: Literal["get_status", "start", "stop", "restart", "get_logs"] = Field( - description="Action to perform on Gateway container" - ) - - config: dict[str, Any] | None = Field( - default=None, - description="Gateway configuration (used for 'start', optional for 'restart'). " - "The Hummingbot API runs Gateway secured (TLS + mTLS) and manages the " - "certificates/passphrase itself using its own CONFIG_PASSWORD (hummingbot-api " - "SEC-048), so no passphrase is needed here. " - "Fields: image (Docker image, default: hummingbot/gateway:development), " - "port (exposed port, default: 15888).", - examples=[ - { - "image": "hummingbot/gateway:development", - "port": 15888, - } - ], - ) - - tail: int | None = Field( - default=100, - ge=1, - le=200, - description="Number of log lines to retrieve (only for 'get_logs' action, default: 100, max: 200)", - ) - - class GatewayConfigRequest(BaseModel): """Request model for Gateway configuration management. diff --git a/mcp_servers/hummingbot_api/server.py b/mcp_servers/hummingbot_api/server.py index 5fadda01d..ba924c3c5 100644 --- a/mcp_servers/hummingbot_api/server.py +++ b/mcp_servers/hummingbot_api/server.py @@ -19,7 +19,6 @@ format_clmm_result, format_gateway_clmm_pool_result, format_gateway_config_result, - format_gateway_container_result, format_gateway_swap_result, ) from mcp_servers.hummingbot_api.hummingbot_client import hummingbot_client @@ -30,7 +29,6 @@ CLMMRequest, GatewayCLMMRequest, GatewayConfigRequest, - GatewayContainerRequest, GatewaySwapRequest, ) from mcp_servers.hummingbot_api.settings import DEFAULT_TOOL_PROFILE, settings @@ -45,9 +43,6 @@ from mcp_servers.hummingbot_api.tools.gateway import ( manage_gateway_config as manage_gateway_config_impl, ) -from mcp_servers.hummingbot_api.tools.gateway import ( - manage_gateway_container as manage_gateway_container_impl, -) from mcp_servers.hummingbot_api.tools.gateway_amm import manage_amm_impl from mcp_servers.hummingbot_api.tools.gateway_clmm import ( explore_gateway_clmm_pools as explore_gateway_clmm_pools_impl, @@ -1668,29 +1663,6 @@ async def manage_gateway_config( return format_gateway_config_result(result) -@handle_errors("manage Gateway container") -async def manage_gateway_container( - action: Literal["get_status", "start", "stop", "restart", "get_logs"], - config: dict[str, Any] | None = None, - tail: int = 100, -) -> str: - """Gateway container lifecycle — status, start, stop, restart, and logs. - - `get_logs` is what the hint on a failed Gateway call points at: when a swap or an LP - action fails with an error that does not say why, the container log usually does. - - Args: - action: get_status | start | stop | restart | get_logs. - config: Gateway configuration. Used by 'start', optional for 'restart'. - tail: Log lines to retrieve for 'get_logs' (1-200, default 100). - """ - request = GatewayContainerRequest(action=action, config=config, tail=tail) - - client = await hummingbot_client.get_client() - result = await manage_gateway_container_impl(client, request) - return format_gateway_container_result(result) - - @handle_errors("manage AMM", GATEWAY_LOG_HINT) async def manage_amm( action: ( diff --git a/mcp_servers/hummingbot_api/tools/gateway.py b/mcp_servers/hummingbot_api/tools/gateway.py index 6c954a237..b74e299a9 100644 --- a/mcp_servers/hummingbot_api/tools/gateway.py +++ b/mcp_servers/hummingbot_api/tools/gateway.py @@ -6,69 +6,11 @@ from typing import Any from mcp_servers.hummingbot_api.exceptions import ToolError -from mcp_servers.hummingbot_api.schemas import ( - GatewayConfigRequest, - GatewayContainerRequest, -) +from mcp_servers.hummingbot_api.schemas import GatewayConfigRequest logger = logging.getLogger("hummingbot-mcp") -async def manage_gateway_container( - client: Any, request: GatewayContainerRequest -) -> dict[str, Any]: - """Manage Gateway container lifecycle operations. - - Supports: - - get_status: Check Gateway container status - - start: Start Gateway with configuration - - stop: Stop Gateway container - - restart: Restart Gateway (optionally with new config) - - get_logs: Get container logs - """ - if request.action == "get_status": - result = await client.gateway.get_status() - return {"action": "get_status", "status": result} - - elif request.action == "start": - if not request.config: - raise ToolError( - "Configuration is required to start Gateway. " - "Provide 'config' with at least 'image' and optionally 'port' and 'environment'." - ) - - result = await client.gateway.start(request.config) - return { - "action": "start", - "message": "Gateway started successfully", - "result": result, - } - - elif request.action == "stop": - result = await client.gateway.stop() - return { - "action": "stop", - "message": "Gateway stopped successfully", - "result": result, - } - - elif request.action == "restart": - result = await client.gateway.restart(request.config) - return { - "action": "restart", - "message": "Gateway restarted successfully", - "result": result, - "config_updated": request.config is not None, - } - - elif request.action == "get_logs": - result = await client.gateway.get_logs(tail=request.tail or 100) - return {"action": "get_logs", "tail": request.tail or 100, "logs": result} - - else: - raise ToolError(f"Unknown action: {request.action}") - - async def manage_gateway_config( client: Any, request: GatewayConfigRequest ) -> dict[str, Any]: diff --git a/tests/test_acp_permission_gate.py b/tests/test_acp_permission_gate.py index 2ef2451e9..914121afe 100644 --- a/tests/test_acp_permission_gate.py +++ b/tests/test_acp_permission_gate.py @@ -392,11 +392,6 @@ def test_dry_run_cancels_a_swap_but_not_a_quote(): # broadcast through, and one to `connectors` sets the slippage every later # swap inherits (SEC-566) "manage_gateway_config", - # start/stop/restart of the Gateway container (SEC-565). It signs nothing, - # which is not the question: stopping it strands live CLMM/LP executors, and - # starting it hands a caller-chosen Docker image to the host that holds the - # exchange API keys and the Gateway wallet keys. - "manage_gateway_container", } #: Tools that read, or that only write config the trading loop must be told to @@ -485,15 +480,12 @@ def test_every_mutating_action_of_a_fund_moving_tool_is_dangerous(): f"{tool_name}({action}) mutates but is auto-approved; " "add it to the matching DANGEROUS_* set in condor/runtime/danger.py" ) - # 3 AMM + 3 CLMM + 5 bot + container `stop` today: a floor, so a signature - # refactor that silently stops yielding actions fails instead of passing - # vacuously. Only `stop` of the container tool is counted here — `start` and - # `restart` match no MUTATING_PREFIXES entry ("start_" has the underscore), which - # is why test_stopping_or_restarting_gateway_asks_a_human below names all three. + # 3 AMM + 3 CLMM + 5 bot today: a floor, so a signature refactor that silently + # stops yielding actions fails instead of passing vacuously. # Neither the swap nor the executor family is counted: they have no `action` since # FEAT-064 and FEAT-062 and are gated by name instead (see # test_swap_signing_action_is_dangerous and tests/test_dangerous_gate_names_resolve.py). - assert checked >= 12, f"only {checked} mutating actions found — enumeration broke" + assert checked >= 11, f"only {checked} mutating actions found — enumeration broke" # --------------------------------------------------------------------------- @@ -543,64 +535,6 @@ def test_control_agent_with_unreadable_arguments_fails_closed(): assert is_dangerous_tool_call(call), f"{raw!r} slipped past the gate" -# --------------------------------------------------------------------------- -# manage_gateway_container's lifecycle actions must ask first (SEC-565) -# --------------------------------------------------------------------------- - -CONTAINER = "mcp__mcp-hummingbot__manage_gateway_container" - - -def test_stopping_or_restarting_gateway_asks_a_human(): - """A seat that is not authorized by a human cannot take Gateway down. - - Before SEC-565 this tool reached no gate at all: ``is_dangerous_tool_call`` - fell through to ``return False`` and the ACP callback auto-approved, so a - stop under a live CLMM position was neither confirmed nor logged. - """ - for action in ("stop", "restart", "start"): - channel = _CapturingChannel(answer=False) - result = _drive_acp(_acp_request(CONTAINER, {"action": action}), channel) - assert ( - len(channel.delivered) == 1 - ), f"manage_gateway_container({action}) ran with no confirmation" - assert ( - result["outcome"]["outcome"] == "cancelled" - ), f"manage_gateway_container({action}) proceeded after a refusal" - - -def test_the_gateway_start_prompt_names_the_image(): - """`start` hands a caller-chosen image to the host holding the keys.""" - channel = _CapturingChannel(answer=False) - _drive_acp( - _acp_request( - CONTAINER, - {"action": "start", "config": {"image": "evil/gateway:latest"}}, - ), - channel, - ) - - assert channel.delivered[0].summary == ( - "Start the Gateway container with image evil/gateway:latest" - ) - - -def test_reading_gateway_status_or_logs_never_asks(): - """`get_logs` is the escape hatch every opaque Gateway failure points at.""" - for action in ("get_status", "get_logs"): - channel = _CapturingChannel(answer=True) - result = _drive_acp(_acp_request(CONTAINER, {"action": action}), channel) - assert ( - not channel.delivered - ), f"manage_gateway_container({action}) raised a confirmation" - assert result["outcome"]["outcome"] == "selected" - - -def test_gateway_container_with_unreadable_arguments_fails_closed(): - for raw in (None, "not json", ["stop"], {}, {"action": 7}): - call = normalize_tool_call(_acp_request(CONTAINER, raw)) - assert is_dangerous_tool_call(call), f"{raw!r} slipped past the gate" - - # --------------------------------------------------------------------------- # manage_gateway_config's network/connector writes must ask first (SEC-566) # --------------------------------------------------------------------------- diff --git a/tests/test_dangerous_gate_names_resolve.py b/tests/test_dangerous_gate_names_resolve.py index 16b7b40ca..c2ae7dc68 100644 --- a/tests/test_dangerous_gate_names_resolve.py +++ b/tests/test_dangerous_gate_names_resolve.py @@ -22,7 +22,6 @@ DANGEROUS_BOT_ACTIONS, DANGEROUS_CLMM_ACTIONS, DANGEROUS_CONFIG_RESOURCES, - DANGEROUS_CONTAINER_ACTIONS, DANGEROUS_CONTROL_ACTIONS, DANGEROUS_TOOLS, is_dangerous_tool_call, @@ -73,7 +72,6 @@ def test_gated_actions_exist_on_their_tools(): ("manage_clmm", DANGEROUS_CLMM_ACTIONS), ("manage_amm", DANGEROUS_AMM_ACTIONS), ("manage_bots", DANGEROUS_BOT_ACTIONS), - ("manage_gateway_container", DANGEROUS_CONTAINER_ACTIONS), # SEC-566 gates manage_gateway_config on a resource *and* an action: the # exemption is spelled out as the read-only actions, so a rename of `get` # would silently start prompting on every read rather than silently stop @@ -667,18 +665,6 @@ def test_an_ungated_config_edit_is_recorded(): ) -def test_a_gateway_container_lifecycle_call_is_gated_and_recorded(): - """SEC-565: the three lifecycle actions; the two reads stay off both lists.""" - for action in ("start", "stop", "restart"): - call = _call("manage_gateway_container", action=action) - assert is_dangerous_tool_call(call), f"{action} is auto-approved" - assert is_mutating_tool_call(call), f"{action} leaves no row" - for action in ("get_status", "get_logs"): - call = _call("manage_gateway_container", action=action) - assert not is_dangerous_tool_call(call), f"{action} raised a confirmation" - assert not is_mutating_tool_call(call), f"{action} was recorded as a write" - - def test_the_log_fails_open_where_the_gate_fails_closed(): """An action nobody has heard of is recorded, not dropped.""" assert is_mutating_tool_call({"tool": "manage_bots", "input": None}) @@ -708,7 +694,6 @@ def _every_plausible_call() -> list[dict]: "manage_clmm", "manage_amm", "manage_gateway_config", - "manage_gateway_container", ): for action in _action_literals(tool): calls.append(_call(tool, action=action, resource_type="tokens")) diff --git a/tests/test_hummingbot_mcp_tools.py b/tests/test_hummingbot_mcp_tools.py index 4b83117f2..6ca52b990 100644 --- a/tests/test_hummingbot_mcp_tools.py +++ b/tests/test_hummingbot_mcp_tools.py @@ -124,6 +124,22 @@ def test_the_backtesting_tools_are_gone(): import mcp_servers.hummingbot_api.tools.backtesting # noqa: F401 +def test_the_gateway_container_tool_is_gone(): + """The dashboard (Settings → Gateway) is the one place to run the container. + + It already did every action the tool had, logs included, behind the + server-owner check. So the hint on a failed Gateway call points the user + there instead of at a tool. + """ + import mcp_servers.hummingbot_api.server as server + from mcp_servers.hummingbot_api.middleware import GATEWAY_LOG_HINT + from mcp_servers.hummingbot_api.profiles import TOOL_DESCRIPTIONS + + assert not hasattr(server, "manage_gateway_container") + assert "manage_gateway_container" not in TOOL_DESCRIPTIONS + assert "manage_gateway_container" not in GATEWAY_LOG_HINT + + POOL_LATENCY = 0.05 @@ -289,93 +305,5 @@ def test_lp_branch_is_skipped_when_not_requested(): assert not any(s["title"] == "LP Positions (CLMM)" for s in result["sections"]) -class FakeGatewayContainerApi: - """The three ``client.gateway`` calls the container tool's branches make.""" - - def __init__(self): - self.calls = [] - - async def get_status(self): - self.calls.append("get_status") - return { - "running": True, - "container_id": "abc123def4567890", - "image": "hummingbot/gateway:latest", - "port": 15888, - "created_at": "2026-09-08T12:00:00.000000Z", - } - - async def get_logs(self, tail): - self.calls.append(("get_logs", tail)) - return "gateway | ERROR the swap reverted" - - async def restart(self, config): - self.calls.append(("restart", config)) - return {"ok": True} - - -class FakeGatewayContainerClient: - def __init__(self, gateway): - self.gateway = gateway - - -class FakeClientHolder: - """Stands in for the ``hummingbot_client`` singleton server.py awaits.""" - - def __init__(self, client): - self._client = client - - async def get_client(self): - return self._client - - -def _stub_gateway_container(monkeypatch): - from mcp_servers.hummingbot_api import server as hb_server - - gateway = FakeGatewayContainerApi() - monkeypatch.setattr( - hb_server, - "hummingbot_client", - FakeClientHolder(FakeGatewayContainerClient(gateway)), - ) - return hb_server, gateway - - -def test_manage_gateway_container_get_status_reaches_the_impl_and_formatter(): - """CORR-561: the wrapper's body called two names server.py never imported. - - Registration succeeded (the tool is in ADMIN_TOOLS and the resolver only - needs the wrapper), so nothing caught it until a call ran the body and - @handle_errors reformatted the NameError into "Failed to manage Gateway - container: name 'manage_gateway_container_impl' is not defined". - """ - monkeypatch = pytest.MonkeyPatch() - try: - hb_server, gateway = _stub_gateway_container(monkeypatch) - output = asyncio.run(hb_server.manage_gateway_container(action="get_status")) - finally: - monkeypatch.undo() - - assert gateway.calls == ["get_status"], "the impl branch never ran" - assert "Gateway Container Status" in output - assert "Running" in output - assert "abc123def456" in output - - -def test_manage_gateway_container_get_logs_is_a_working_escape_hatch(): - """GATEWAY_LOG_HINT points every opaque swap/LP failure at this action.""" - monkeypatch = pytest.MonkeyPatch() - try: - hb_server, gateway = _stub_gateway_container(monkeypatch) - output = asyncio.run( - hb_server.manage_gateway_container(action="get_logs", tail=25) - ) - finally: - monkeypatch.undo() - - assert gateway.calls == [("get_logs", 25)] - assert "the swap reverted" in output - - if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/test_mcp_tool_profiles.py b/tests/test_mcp_tool_profiles.py index 2b2d4be3c..2d1290e04 100644 --- a/tests/test_mcp_tool_profiles.py +++ b/tests/test_mcp_tool_profiles.py @@ -54,7 +54,7 @@ "explore_geckoterminal", } HB_LIQUIDITY = {"manage_amm", "manage_clmm"} -HB_ADMIN = {"configure_server", "manage_gateway_config", "manage_gateway_container"} +HB_ADMIN = {"configure_server", "manage_gateway_config"} HB_PROFILES = { "tick": HB_TRADING, @@ -133,7 +133,6 @@ def test_every_tool_the_module_defines_lands_in_some_profile(module): [ "configure_server", "manage_gateway_config", - "manage_gateway_container", "manage_amm", "manage_clmm", ], From 1e5f535749ff16696466bb78fbf9108c1303ef30 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 18:56:12 +0300 Subject: [PATCH 135/154] Make Gateway's RPC and connector settings dashboard-only: manage_gateway_config reads networks and connectors and can no longer write them The tool loses its 'update' action and the config_updates payload, so an agent cannot name the write at all rather than being asked to confirm it. With the funds-path writes gone there is nothing left for the SEC-566 gate to stop, so it goes too. Tokens and pools stay editable, and are still recorded in the action log. --- condor/agents/actions.py | 4 +- condor/runtime/danger.py | 107 +++--------------- handlers/agents/_shared.py | 1 - mcp_servers/TOOL_STYLE.md | 5 +- .../hummingbot_api/formatters/gateway.py | 2 +- mcp_servers/hummingbot_api/profiles.py | 2 +- mcp_servers/hummingbot_api/schemas.py | 23 ++-- mcp_servers/hummingbot_api/server.py | 20 ++-- mcp_servers/hummingbot_api/tools/gateway.py | 48 +------- tests/test_acp_permission_gate.py | 85 ++------------ tests/test_dangerous_gate_names_resolve.py | 89 +++------------ tests/test_hummingbot_mcp_tools.py | 24 ++++ 12 files changed, 92 insertions(+), 318 deletions(-) diff --git a/condor/agents/actions.py b/condor/agents/actions.py index 8b83277f8..479231287 100644 --- a/condor/agents/actions.py +++ b/condor/agents/actions.py @@ -66,8 +66,8 @@ # The tools that carry their verb in an ``action`` argument. For these the # queryable key is ``tool:action`` — ``manage_bots`` alone would collapse a # deploy and a stop into one verb. ``manage_gateway_config`` is here for the -# same reason even though the *gate* reads its resource type instead: the log's -# question is whether it edited, and the action is what says so. +# same reason even though it is never gated: the log's question is whether it +# edited a token or a pool, and the action is what says so. _DISPATCH_TOOLS = frozenset( { "manage_bots", diff --git a/condor/runtime/danger.py b/condor/runtime/danger.py index 211d6d2fc..f2cff0dc6 100644 --- a/condor/runtime/danger.py +++ b/condor/runtime/danger.py @@ -48,7 +48,6 @@ "execute_swap", # every call signs; quote/status/search are separate tools "manage_clmm", # every action that moves liquidity "manage_amm", # every action that moves liquidity - "manage_gateway_config", # only writes to networks/connectors; see below "control_agent", # only `start`, which launches an unattended trading loop # The executor family is gated by NAME (FEAT-062), the same way the swap family # is: a create and a stop each have their own tool, so there is no `action` to @@ -125,40 +124,20 @@ # so the gate has to know both or `start_agent` walks straight past it. DANGEROUS_CONTROL_ACTIONS = {"start", "start_agent"} -# Resource types within manage_gateway_config whose *writes* require confirmation -# (SEC-566). This set used to be empty, justified by "everything it touches is -# Gateway's own symbol/address mapping". That is true of two of the four resources -# and false of the other two, which is the correction: -# -# - `tokens` and `pools` stay ungated. Adding or deleting one edits a symbol → -# address mapping. It moves no funds and changes nothing on-chain, so gating it -# would put a human in front of a config edit while the trades that edit enables -# stay where they are. `chains` and `wallets` stay ungated too — both are -# read-only over MCP since FEAT-065 (a wallet is imported in the dashboard, and -# `add` no longer takes a private key anywhere the model can reach). -# - `networks` and `connectors` are gated, because an `update` there is not a -# mapping: a network config carries `nodeURL`, the RPC endpoint every transaction -# from this server is signed against and broadcast through, and a connector config -# carries settings such as allowed slippage that every later swap inherits. The -# dashboard already treats exactly this write as privileged — the web route calls -# it "a server-wide change" and demands OWNER (condor/web/routes/settings.py) — -# while the MCP path let a model repoint it with no human in the loop. Tool output -# is untrusted input, so "the prompt says don't call this" is not a control -# (SEC-253). -# -# The gate is resource *and* action: `list`/`get` on a gated resource stay on the -# fast path, because reading which RPC a chain is on is how a model diagnoses a -# failed swap, and a prompt in front of a read buys nothing. An unreadable -# `resource_type` — and, on a gated resource, an unreadable `action` — still fails -# closed. See :func:`_is_dangerous_config_call`. -DANGEROUS_CONFIG_RESOURCES: set[str] = {"networks", "connectors"} +# There is no gate on `manage_gateway_config`. Its `networks` and `connectors` +# are read-only over MCP: a network config carries `nodeURL`, the RPC every +# transaction is broadcast through, and a connector config the slippage every +# swap inherits, so those writes belong to the server owner in Condor and the +# tool takes no `update` action or payload at all. What is left to write is the +# `tokens` and `pools` symbol → address mapping, which moves no funds and needs +# no human. The same goes for `chains` and `wallets`, both read-only since FEAT-065. # ── What changed the world (FEAT-097) ── # # The sets above answer "should a human approve this". The log asks a different # question — "did this change anything" — and the two deliberately differ: -# `manage_gateway_config` gates only its two funds-path resources (SEC-566), and +# `manage_gateway_config`'s token and pool edits are never gated, and # the brakes (`stop`, `pause`, `resume`, `shutdown`) are ungated on purpose. A # log built on the confirmation predicate would therefore be silent about every # config edit and every brake, which is the exact silence the log exists to end. @@ -281,9 +260,8 @@ #: How much of a snippet's first line the log row carries. A summary is one line #: on a page, and the whole source is in the code-run store anyway. MAX_SNIPPET_HEAD_CHARS = 80 -#: `manage_gateway_config` is recorded on its *action*, not its resource type: -#: what it edits is what the gate weighs, and whether it edited at all is what -#: the log weighs. +#: `manage_gateway_config` is recorded on its *action*: whether it edited a token +#: or a pool at all is what the log weighs. READ_ONLY_CONFIG_ACTIONS = {"list", "get"} @@ -413,37 +391,6 @@ def _executor_amount(tool_name: str, input_data: dict[str, Any]) -> str: return f" of {amount}" if amount is not None else "" -def _is_dangerous_config_call(tool_call: dict[str, Any]) -> bool: - """Whether a ``manage_gateway_config`` call writes a funds-path resource (SEC-566). - - Resource *and* action, because neither half alone is the right gate. Resource - alone would prompt on `get networks`, the read a model does to diagnose a - failed swap; action alone would prompt on `add tokens`, a symbol → address - mapping that moves nothing. What needs a human is a *write* to `networks` or - `connectors`: the RPC every transaction is broadcast through, and the slippage - every later swap inherits. - - Fails closed twice over (SEC-093). Unreadable arguments, or a missing or - non-string ``resource_type``, are dangerous whatever the action claims to be — - a call we cannot classify is never let through on the strength of the half of - it we can read. On a resource we *can* read and that is gated, a missing or - non-string ``action`` is dangerous too, so a write cannot hide behind an - unparseable action string. - """ - input_data = tool_call_input(tool_call) - if input_data is None: - return True - resource = input_data.get("resource_type") - if not isinstance(resource, str) or not resource: - return True - if resource not in DANGEROUS_CONFIG_RESOURCES: - return False - action = input_data.get("action") - if not isinstance(action, str) or not action: - return True - return action not in READ_ONLY_CONFIG_ACTIONS - - def is_dangerous_tool_call(tool_call: dict[str, Any]) -> bool: """Check if a tool call requires user confirmation.""" tool_name = tool_call_name(tool_call) @@ -457,9 +404,6 @@ def is_dangerous_tool_call(tool_call: dict[str, Any]) -> bool: if tool_name == "manage_amm": return _has_dangerous_action(tool_call, DANGEROUS_AMM_ACTIONS) - if tool_name == "manage_gateway_config": - return _is_dangerous_config_call(tool_call) - if tool_name == "control_agent": return _has_dangerous_action(tool_call, DANGEROUS_CONTROL_ACTIONS) @@ -530,15 +474,8 @@ def is_mutating_tool_call(tool_call: dict[str, Any]) -> bool: ) if tool_name == "manage_gateway_config": - # Recorded unless it is one of the two reads. The resource type is read - # only to stay a superset of the gate, which fails closed on a missing - # one: a call neither of us can parse is recorded rather than dropped. - input_data = tool_call_input(tool_call) - if input_data is None: - return True - resource = input_data.get("resource_type") - if not isinstance(resource, str) or not resource: - return True + # Recorded unless it is one of the two reads: a token or pool edit is a + # write to Gateway's config even though nothing gates it. return _is_mutating_action(tool_call, set(), READ_ONLY_CONFIG_ACTIONS) # Gated by name, and every one of them writes: an order, a signature, an @@ -776,26 +713,10 @@ def format_tool_summary(tool_call: dict[str, Any]) -> str: return f"Swap {side} {amount} {pair}" if tool_name == "manage_gateway_config": - # The wallet import/remove summaries lived here until the tool stopped - # accepting a private key at all (FEAT-065); wallets are read-only now. + # Never gated, so this line is written for the log. Wallets went read-only + # in FEAT-065, and networks and connectors went the same way after SEC-566. resource = input_data.get("resource_type", "?") action = input_data.get("action", "?") - if resource in DANGEROUS_CONFIG_RESOURCES and action not in ( - READ_ONLY_CONFIG_ACTIONS - ): - # The gated half (SEC-566). "update networks" is not what the human is - # approving — the target and the keys are, because one of those keys is - # `nodeURL`, the RPC every later transaction is broadcast through. - target = ( - input_data.get("network_id") or input_data.get("connector_name") or "?" - ) - updates = input_data.get("config_updates") - keys = ( - ", ".join(str(key) for key in updates) - if isinstance(updates, dict) and updates - else "?" - ) - return f"Gateway config: {action} {resource} '{target}', setting {keys}" return f"Gateway config: {action} {resource}" if tool_name in ("manage_clmm", "manage_amm"): diff --git a/handlers/agents/_shared.py b/handlers/agents/_shared.py index bdafdc3a3..619b3c981 100644 --- a/handlers/agents/_shared.py +++ b/handlers/agents/_shared.py @@ -36,7 +36,6 @@ DANGEROUS_AMM_ACTIONS, DANGEROUS_BOT_ACTIONS, DANGEROUS_CLMM_ACTIONS, - DANGEROUS_CONFIG_RESOURCES, DANGEROUS_CONTROL_ACTIONS, DANGEROUS_TOOLS, is_dangerous_tool_call, diff --git a/mcp_servers/TOOL_STYLE.md b/mcp_servers/TOOL_STYLE.md index 69fce40c8..a352e7f26 100644 --- a/mcp_servers/TOOL_STYLE.md +++ b/mcp_servers/TOOL_STYLE.md @@ -46,7 +46,10 @@ the gate can classify on its own: unreadable-argument ambiguity to fail closed on. - A tool that only sometimes moves funds is gated on one literal field, and the gate fails closed when that field cannot be read — `manage_clmm` and `manage_amm` on - `action`, `manage_gateway_config` on `resource_type`. + `action`. +- A write no agent should make, even behind a confirmation, is not a parameter at all: + `manage_gateway_config` has no `update` action, so the RPC, connector settings and + wallets are changed by the server owner in Condor. - **A read tool must be safe by name.** If a name can be both, it is two tools. - Every gated name must resolve to a really-registered tool, and every gated literal to a real member of that tool's `Literal` — `tests/test_dangerous_gate_names_resolve.py` diff --git a/mcp_servers/hummingbot_api/formatters/gateway.py b/mcp_servers/hummingbot_api/formatters/gateway.py index 5643a68f9..cbeac6b2b 100644 --- a/mcp_servers/hummingbot_api/formatters/gateway.py +++ b/mcp_servers/hummingbot_api/formatters/gateway.py @@ -71,7 +71,7 @@ def format_gateway_config_result(result: dict[str, Any]) -> str: output += f"- {chain_name}: {address}\n" return output - elif result_action in ["add", "delete", "update"]: + elif result_action in ["add", "delete"]: message = result.get("result", {}).get("message", "") return f"Gateway Config {result_action.title()}: {message}" diff --git a/mcp_servers/hummingbot_api/profiles.py b/mcp_servers/hummingbot_api/profiles.py index 879fb81bb..d0add0fda 100644 --- a/mcp_servers/hummingbot_api/profiles.py +++ b/mcp_servers/hummingbot_api/profiles.py @@ -50,7 +50,7 @@ "manage_amm": "Direct AMM pool operations and pool creation", "manage_clmm": "Direct CLMM position operations", "configure_server": "Repoint this seat at another Hummingbot API server", - "manage_gateway_config": "Read and edit Gateway's chains, tokens and wallets", + "manage_gateway_config": "Read Gateway's config; edit its tokens and pools", } #: The trading surface: everything an autonomous tick needs to read a market, diff --git a/mcp_servers/hummingbot_api/schemas.py b/mcp_servers/hummingbot_api/schemas.py index c9073f241..3d4adf947 100644 --- a/mcp_servers/hummingbot_api/schemas.py +++ b/mcp_servers/hummingbot_api/schemas.py @@ -22,17 +22,20 @@ class GatewayConfigRequest(BaseModel): Resource Types: - chains: Blockchain chains (get all chains) - - networks: Network configurations (list, get, update) - format: 'chain-network' + - networks: Network configurations (list, get) - format: 'chain-network' - tokens: Token configurations (list, add, delete, save) per network - - connectors: DEX connector configurations (list, get, update) + - connectors: DEX connector configurations (list, get) - pools: Liquidity pools (list, add, delete, save) per connector/network - wallets: Configured wallets per chain (list only — a wallet is added or removed in the Condor dashboard, never through an agent) + Networks and connectors are read-only here. Their config holds the RPC every + transaction is broadcast through and the slippage every swap inherits, so it + is changed by the server owner in Condor, never through an agent. + Actions: - list: List available resources - get: Get specific resource configuration - - update: Update resource configuration - add: Add new resource (tokens, pools) - requires full details - delete: Delete resource (tokens, pools) - save: Save resource by address only (tokens, pools) - auto-fetches details @@ -42,7 +45,7 @@ class GatewayConfigRequest(BaseModel): "chains", "networks", "tokens", "connectors", "pools", "wallets" ] = Field(description="Type of resource to manage") - action: Literal["list", "get", "update", "add", "delete", "save"] = Field( + action: Literal["list", "get", "add", "delete", "save"] = Field( description="Action to perform on the resource" ) @@ -61,18 +64,6 @@ class GatewayConfigRequest(BaseModel): examples=["meteora", "raydium", "orca", "uniswap", "pancakeswap"], ) - # Configuration data - config_updates: dict[str, Any] | None = Field( - default=None, - description="Configuration updates as key-value pairs. " - "Keys can be in snake_case or camelCase. " - "Required for 'update' action", - examples=[ - {"slippage_pct": 0.5, "timeout": 30000}, - {"node_url": "https://api.mainnet-beta.solana.com"}, - ], - ) - # Token-specific fields token_address: str | None = Field( default=None, diff --git a/mcp_servers/hummingbot_api/server.py b/mcp_servers/hummingbot_api/server.py index ba924c3c5..ab99ab346 100644 --- a/mcp_servers/hummingbot_api/server.py +++ b/mcp_servers/hummingbot_api/server.py @@ -1586,10 +1586,9 @@ async def manage_gateway_config( resource_type: Literal[ "chains", "networks", "tokens", "connectors", "pools", "wallets" ], - action: Literal["list", "get", "update", "add", "delete", "save"], + action: Literal["list", "get", "add", "delete", "save"], network_id: str | None = None, connector_name: str | None = None, - config_updates: dict[str, Any] | None = None, token_address: str | None = None, token_symbol: str | None = None, token_decimals: int | None = None, @@ -1602,7 +1601,7 @@ async def manage_gateway_config( network: str | None = None, chain: str | None = None, ) -> str: - """Read and edit Gateway's own configuration — chains, networks, tokens, connectors, pools, wallets. + """Read Gateway's own configuration, and edit the tokens and pools it knows. This is Gateway's config, not the chain. Adding or deleting a token here changes the symbol -> address mapping Gateway resolves against; it moves no funds and touches @@ -1614,19 +1613,21 @@ async def manage_gateway_config( Resource types: - chains: every blockchain Gateway knows - - networks: network config, ids in 'chain-network' form ('solana-mainnet-beta') - - tokens: the per-network symbol/address/decimals mapping (list, add, delete) - - connectors: DEX connector config - - pools: the named pool registry (list, add) + - networks: list/get only, ids in 'chain-network' form ('solana-mainnet-beta'). + The RPC endpoint and the rest of a network's config are changed by the user + in the Condor dashboard (Settings → Gateway), never by an agent. + - tokens: the per-network symbol/address/decimals mapping (list, add, delete, save) + - connectors: list/get only. A connector's settings (slippage and the like) are + changed by the server owner in Condor, never by an agent. + - pools: the named pool registry (list, add, delete, save) - wallets: list only. Add or remove wallets in the Condor dashboard (Settings → Gateway) — a private key must never be sent through chat. Args: resource_type: Which part of Gateway's config to act on. - action: list | get | update | add | delete | save. + action: list | get | add | delete | save. network_id: Network id in 'chain-network' form. Required for token and pool actions. connector_name: DEX connector name ('meteora', 'raydium', 'uniswap'). - config_updates: Key-value updates for 'update'/'save'. token_address: Token contract address. Required to add or delete a token. token_symbol: Token symbol. Required to add a token. token_decimals: Token decimals (6 for USDC, 18 for WETH). Required to add a token. @@ -1644,7 +1645,6 @@ async def manage_gateway_config( action=action, network_id=network_id, connector_name=connector_name, - config_updates=config_updates, token_address=token_address, token_symbol=token_symbol, token_decimals=token_decimals, diff --git a/mcp_servers/hummingbot_api/tools/gateway.py b/mcp_servers/hummingbot_api/tools/gateway.py index b74e299a9..afe1be725 100644 --- a/mcp_servers/hummingbot_api/tools/gateway.py +++ b/mcp_servers/hummingbot_api/tools/gateway.py @@ -18,9 +18,9 @@ async def manage_gateway_config( Resource Types: - chains: Get all blockchain chains - - networks: List/get/update network configurations (format: 'chain-network') + - networks: List/get network configurations (format: 'chain-network'), read-only - tokens: List/add/delete tokens per network - - connectors: List/get/update DEX connector configurations + - connectors: List/get DEX connector configurations, read-only - pools: List/add liquidity pools per connector/network - wallets: List the configured wallets per chain (read-only) """ @@ -56,28 +56,11 @@ async def manage_gateway_config( "result": result, } - elif request.action == "update": - if not request.network_id: - raise ToolError("network_id is required for 'update' network action") - if not request.config_updates: - raise ToolError( - "config_updates is required for 'update' network action" - ) - - result = await client.gateway.update_network_config( - request.network_id, request.config_updates - ) - return { - "resource_type": "networks", - "action": "update", - "network_id": request.network_id, - "result": result, - } - else: raise ToolError( f"Action '{request.action}' not supported for networks. " - f"Supported: list, get, update" + "Supported: list, get. A network's config, its RPC endpoint " + "included, is changed in the Condor dashboard (Settings → Gateway)." ) # ============================================ @@ -193,30 +176,11 @@ async def manage_gateway_config( "result": result, } - elif request.action == "update": - if not request.connector_name: - raise ToolError( - "connector_name is required for 'update' connector action" - ) - if not request.config_updates: - raise ToolError( - "config_updates is required for 'update' connector action" - ) - - result = await client.gateway.update_connector_config( - request.connector_name, request.config_updates - ) - return { - "resource_type": "connectors", - "action": "update", - "connector_name": request.connector_name, - "result": result, - } - else: raise ToolError( f"Action '{request.action}' not supported for connectors. " - f"Supported: list, get, update" + "Supported: list, get. A connector's settings are changed by the " + "server owner in Condor, not over MCP." ) # ============================================ diff --git a/tests/test_acp_permission_gate.py b/tests/test_acp_permission_gate.py index 914121afe..555731d3b 100644 --- a/tests/test_acp_permission_gate.py +++ b/tests/test_acp_permission_gate.py @@ -388,10 +388,6 @@ def test_dry_run_cancels_a_swap_but_not_a_quote(): "manage_amm", "manage_bots", "manage_clmm", - # a write to `networks` repoints `nodeURL`, the RPC every transaction is - # broadcast through, and one to `connectors` sets the slippage every later - # swap inherits (SEC-566) - "manage_gateway_config", } #: Tools that read, or that only write config the trading loop must be told to @@ -399,6 +395,9 @@ def test_dry_run_cancels_a_swap_but_not_a_quote(): #: ``test_every_action_gated_tool_is_classified`` below. NON_FUND_MOVING_TOOLS = { "manage_controllers", # writes controller templates, never a running bot + # Writes only the token/pool symbol → address mapping. Networks and connectors + # (the RPC and the slippage) are read-only over MCP and set in Condor. + "manage_gateway_config", "executor_defaults", # edits a local preferences file; creates nothing "explore_dex_pools", "explore_geckoterminal", @@ -536,68 +535,18 @@ def test_control_agent_with_unreadable_arguments_fails_closed(): # --------------------------------------------------------------------------- -# manage_gateway_config's network/connector writes must ask first (SEC-566) +# manage_gateway_config never asks: the RPC and slippage are set in Condor # --------------------------------------------------------------------------- CONFIG = "mcp__mcp-hummingbot__manage_gateway_config" -def test_repointing_the_rpc_endpoint_asks_a_human_and_is_refused(): - """A seat that is not authorized by a human cannot move the funds path. +def test_reading_gateway_config_or_editing_a_token_never_asks(): + """What the tool can still do is read, or edit a symbol → address mapping. - Before SEC-566 ``DANGEROUS_CONFIG_RESOURCES`` was empty, so this call was - auto-approved: a model could repoint `nodeURL` — the RPC every transaction - from this server is signed against and broadcast through — with no prompt, - while the dashboard demanded OWNER for the identical write. + Repointing the RPC or a connector's slippage used to reach this gate + (SEC-566). That write is no longer an action of the tool at all. """ - for resource, target in ( - ("networks", {"network_id": "solana-mainnet-beta"}), - ("connectors", {"connector_name": "jupiter"}), - ): - channel = _CapturingChannel(answer=False) - result = _drive_acp( - _acp_request( - CONFIG, - { - "resource_type": resource, - "action": "update", - **target, - "config_updates": {"nodeURL": "https://evil.example/rpc"}, - }, - ), - channel, - ) - assert ( - len(channel.delivered) == 1 - ), f"manage_gateway_config(update {resource}) ran with no confirmation" - assert ( - result["outcome"]["outcome"] == "cancelled" - ), f"manage_gateway_config(update {resource}) proceeded after a refusal" - - -def test_the_network_update_prompt_names_the_network_and_the_keys(): - """The human approves a `nodeURL` change, not the words "update networks".""" - channel = _CapturingChannel(answer=False) - _drive_acp( - _acp_request( - CONFIG, - { - "resource_type": "networks", - "action": "update", - "network_id": "solana-mainnet-beta", - "config_updates": {"nodeURL": "https://evil.example/rpc"}, - }, - ), - channel, - ) - - assert channel.delivered[0].summary == ( - "Gateway config: update networks 'solana-mainnet-beta', setting nodeURL" - ) - - -def test_reading_a_network_config_or_editing_a_token_never_asks(): - """Reads stay silent, and a token edit is a symbol → address mapping.""" for args in ( {"resource_type": "networks", "action": "get", "network_id": "solana-mainnet"}, {"resource_type": "connectors", "action": "list"}, @@ -610,24 +559,6 @@ def test_reading_a_network_config_or_editing_a_token_never_asks(): assert result["outcome"]["outcome"] == "selected" -def test_gateway_config_with_an_unreadable_resource_or_action_fails_closed(): - """SEC-093/SEC-566: neither half of the gate can be defeated by junk.""" - for raw in ( - None, - "not json", - ["networks"], - {}, - {"resource_type": 7}, - {"action": "update"}, - # A gated resource whose action cannot be read is a write. - {"resource_type": "networks"}, - {"resource_type": "networks", "action": 7}, - {"resource_type": "connectors", "action": ""}, - ): - call = normalize_tool_call(_acp_request(CONFIG, raw)) - assert is_dangerous_tool_call(call), f"{raw!r} slipped past the gate" - - # --------------------------------------------------------------------------- # A summary that raises must not become a silent "no" (CORR-294) # --------------------------------------------------------------------------- diff --git a/tests/test_dangerous_gate_names_resolve.py b/tests/test_dangerous_gate_names_resolve.py index c2ae7dc68..b48c79d26 100644 --- a/tests/test_dangerous_gate_names_resolve.py +++ b/tests/test_dangerous_gate_names_resolve.py @@ -21,7 +21,6 @@ DANGEROUS_AMM_ACTIONS, DANGEROUS_BOT_ACTIONS, DANGEROUS_CLMM_ACTIONS, - DANGEROUS_CONFIG_RESOURCES, DANGEROUS_CONTROL_ACTIONS, DANGEROUS_TOOLS, is_dangerous_tool_call, @@ -72,10 +71,8 @@ def test_gated_actions_exist_on_their_tools(): ("manage_clmm", DANGEROUS_CLMM_ACTIONS), ("manage_amm", DANGEROUS_AMM_ACTIONS), ("manage_bots", DANGEROUS_BOT_ACTIONS), - # SEC-566 gates manage_gateway_config on a resource *and* an action: the - # exemption is spelled out as the read-only actions, so a rename of `get` - # would silently start prompting on every read rather than silently stop - # gating — but both halves have to keep resolving, so both are pinned. + # Not a gate but the log's read set: a rename of `get` would record every + # config read as a write, so the reads have to keep resolving too. ("manage_gateway_config", READ_ONLY_CONFIG_ACTIONS), ): available = _action_literals(tool_name) @@ -277,36 +274,19 @@ def _config_resource_literals() -> set[str]: } -def test_gateway_config_gates_the_funds_path_resources_and_nothing_else(): - """SEC-566: a write to `networks`/`connectors` asks; a token or pool edit does not. +def test_gateway_config_has_no_write_left_that_needs_a_human(): + """The funds-path writes are gone from the tool rather than gated on it. - The gate used to be an empty resource set, justified by "everything this tool - touches is Gateway's own symbol/address mapping". Two of the four resources are - not: a network config carries `nodeURL`, the RPC every transaction is broadcast - through, and a connector config carries the slippage every later swap inherits. - The dashboard already demands OWNER for exactly that write; the MCP path asked - nobody. `tokens` and `pools` really are a mapping and stay ungated, so a human is - not put in front of a config edit while the trades it enables run unattended. + `networks` and `connectors` used to be gated on `update` (SEC-566): a network + config carries `nodeURL`, the RPC every transaction is broadcast through, and + a connector config the slippage every swap inherits. Those writes now belong + to the server owner in Condor and the tool cannot name them, so every call it + accepts is a read or a token/pool mapping edit, and none of them asks. """ - resources = _config_resource_literals() - assert DANGEROUS_CONFIG_RESOURCES <= resources, ( - f"gated resource(s) the tool has no such value for: " - f"{sorted(DANGEROUS_CONFIG_RESOURCES - resources)}" - ) - assert DANGEROUS_CONFIG_RESOURCES == {"networks", "connectors"} - - for resource in DANGEROUS_CONFIG_RESOURCES: - for action in _action_literals("manage_gateway_config") - ( - READ_ONLY_CONFIG_ACTIONS - ): - assert is_dangerous_tool_call( - { - "tool": "manage_gateway_config", - "input": {"resource_type": resource, "action": action}, - } - ), f"{resource}/{action} repoints the funds path with no confirmation" + assert "update" not in _action_literals("manage_gateway_config") + assert "manage_gateway_config" not in DANGEROUS_TOOLS - for resource in resources - DANGEROUS_CONFIG_RESOURCES: + for resource in _config_resource_literals(): for action in _action_literals("manage_gateway_config"): assert not is_dangerous_tool_call( { @@ -316,44 +296,6 @@ def test_gateway_config_gates_the_funds_path_resources_and_nothing_else(): ), f"{resource}/{action} should not need confirmation" -def test_gateway_config_reads_stay_on_the_fast_path(): - """Reading a gated resource is how a failed swap gets diagnosed (SEC-566). - - The gate is resource *and* action for this reason alone: a prompt in front of - `get networks` would put one in front of finding out which RPC a chain is on. - """ - for resource in DANGEROUS_CONFIG_RESOURCES: - for action in READ_ONLY_CONFIG_ACTIONS: - assert not is_dangerous_tool_call( - { - "tool": "manage_gateway_config", - "input": {"resource_type": resource, "action": action}, - } - ), f"{resource}/{action} raised a confirmation for a read" - - -def test_gateway_config_fails_closed_on_an_unreadable_action(): - """SEC-566: on a gated resource, an action we cannot read is a write. - - The resource half is not enough on its own — once the gate started reading an - action, an unparseable one would otherwise fall through the read-only test and - be waved past. - """ - for bad in ({}, {"action": None}, {"action": 7}, {"action": ""}): - for resource in DANGEROUS_CONFIG_RESOURCES: - call = { - "tool": "manage_gateway_config", - "input": {"resource_type": resource, **bad}, - } - assert is_dangerous_tool_call(call), f"{resource}/{bad} slipped past" - - -def test_gateway_config_fails_closed_on_an_unreadable_resource(): - """SEC-093: a call whose resource_type cannot be read is treated as dangerous.""" - for bad in ({}, {"resource_type": None}, {"resource_type": 7}, {"action": "add"}): - assert is_dangerous_tool_call({"tool": "manage_gateway_config", "input": bad}) - - # --------------------------------------------------------------------------- # control_agent: starting a loop is the third capital path (SEC-275) # --------------------------------------------------------------------------- @@ -654,8 +596,8 @@ def test_the_ungated_brakes_are_recorded(): def test_an_ungated_config_edit_is_recorded(): - """A token edit is ungated on purpose (SEC-566); the log keeps it anyway.""" - for action in ("add", "delete", "update", "save"): + """A token edit is ungated on purpose; the log keeps it anyway.""" + for action in ("add", "delete", "save"): call = _call("manage_gateway_config", action=action, resource_type="tokens") assert not is_dangerous_tool_call(call) assert is_mutating_tool_call(call) @@ -700,8 +642,7 @@ def _every_plausible_call() -> list[dict]: calls.append(_call(tool, action=action)) for resource in _config_resource_literals(): calls.append(_call("manage_gateway_config", resource_type=resource)) - # Every real (resource, action) pair, so the SEC-566 gate's own combinations - # — not just the fail-closed ones — are held to `dangerous ⊆ mutating`. + # Every real (resource, action) pair, held to `dangerous ⊆ mutating`. for action in _action_literals("manage_gateway_config"): calls.append( _call("manage_gateway_config", resource_type=resource, action=action) diff --git a/tests/test_hummingbot_mcp_tools.py b/tests/test_hummingbot_mcp_tools.py index 6ca52b990..9e8d1cf08 100644 --- a/tests/test_hummingbot_mcp_tools.py +++ b/tests/test_hummingbot_mcp_tools.py @@ -140,6 +140,30 @@ def test_the_gateway_container_tool_is_gone(): assert "manage_gateway_container" not in GATEWAY_LOG_HINT +def test_no_agent_can_write_a_network_or_connector_config(): + """The RPC and a connector's slippage are the server owner's, set in Condor. + + A write there repoints every later transaction, so it is not something a + model may do even behind a confirmation. The tool cannot name it at all. + The action and the payload that carried it are both gone. + """ + import inspect + + import pydantic + + import mcp_servers.hummingbot_api.server as server + from mcp_servers.hummingbot_api.schemas import GatewayConfigRequest + + params = inspect.signature(server.manage_gateway_config).parameters + assert "config_updates" not in params + assert "update" not in str(params["action"].annotation) + assert "config_updates" not in GatewayConfigRequest.model_fields + + for resource in ("networks", "connectors"): + with pytest.raises(pydantic.ValidationError): + GatewayConfigRequest(resource_type=resource, action="update") + + POOL_LATENCY = 0.05 From de39391c1ef5eda245bfb98513099c20a9cec254 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 19:35:29 +0300 Subject: [PATCH 136/154] Count the tokens every chat turn spends and keep the running total with the conversation: both clients fold their backend's exact usage into a lifetime counter, the funnel charges each turn the delta, and meta.json holds the sum (FEAT-120) Cancelled, stale, abandoned and errored turns all count; tokens spent outside a turn (/compact, the eager context) land on the next one. input_tokens is inclusive of cache on both backends. prompt_done and DONE only gain a key. --- condor/acp/client.py | 57 ++- condor/acp/pydantic_ai_client.py | 95 ++++- condor/acp/usage.py | 151 ++++++++ condor/llm/openrouter_models.py | 14 + condor/runtime/client.py | 22 +- condor/runtime/conversations.py | 53 ++- condor/runtime/sessions.py | 26 ++ condor/web/routes/chat_ws.py | 4 + tests/runtime/test_session_api.py | 9 +- tests/test_conversations_api.py | 26 ++ tests/test_token_usage.py | 578 ++++++++++++++++++++++++++++++ 11 files changed, 1018 insertions(+), 17 deletions(-) create mode 100644 condor/acp/usage.py create mode 100644 tests/test_token_usage.py diff --git a/condor/acp/client.py b/condor/acp/client.py index 9a1d3c893..6265e7394 100644 --- a/condor/acp/client.py +++ b/condor/acp/client.py @@ -19,6 +19,7 @@ from typing import Any, AsyncIterator, Awaitable, Callable from .jsonrpc import JSONRPCPeer +from .usage import TokenUsage log = logging.getLogger(__name__) @@ -532,6 +533,11 @@ def __init__( self._read_loop_ended = False self._event_queue: asyncio.Queue[ACPEvent] = asyncio.Queue() self._current_req_id: int | None = None # tracks in-flight prompt request + # Everything the agent has reported spending, for this client's whole + # life (FEAT-120): cancelled and stale turns included, since tokens + # burned for an answer nobody read are still burned. The session takes + # per-turn deltas of it; nothing here knows what a turn is. + self.usage = TokenUsage() # A turn the agent has not settled and that nobody is streaming any # more: one that ignored ``session/cancel``, or one whose consumer # walked away (a WS drop, a page reload, a cancelled prompt task). @@ -1091,6 +1097,11 @@ async def prompt_stream( self._current_req_id = req_id def _on_response(fut: asyncio.Future) -> None: + # Counted before the stale check: a turn we cancelled locally is + # still settled by the adapter with its usage, and those tokens + # were spent whether or not anyone is left to read the answer. + if not fut.cancelled() and fut.exception() is None: + self._fold_prompt_usage(fut.result()) # Only enqueue PromptDone if this is still the current prompt if self._current_req_id != req_id: return # stale response from an aborted prompt — ignore @@ -1202,10 +1213,18 @@ def _on_session_update( # (PERF-332). Terminal events never come through here, so a parked # consumer is still unblocked: the read loop, ``_cancel_locally`` and # ``_on_response`` put their ``PromptDone`` on the queue directly. + # + # ``usage_update`` is the exception, and so it is read first: it + # reaches no consumer, it only moves the counter, and the adapter + # sends one for a background task's result after the turn has settled + # — exactly when there is no ``_current_req_id`` (FEAT-120). + kind = update.get("sessionUpdate") + if kind == "usage_update": + self._note_usage_update(update) + return if self._current_req_id is None: return - kind = update.get("sessionUpdate") if kind == "agent_message_chunk": content = update.get("content", {}) text = content.get("text", "") @@ -1251,6 +1270,42 @@ def _on_session_update( ) ) + def _fold_prompt_usage(self, result: Any) -> None: + """Add a ``session/prompt`` response's per-turn ``usage`` to the counter. + + Runs inside a future's done callback, where an exception would be + logged by asyncio and the ``PromptDone`` after it never enqueued — + leaving the consumer parked until its timeout. So it cannot raise. + """ + try: + if isinstance(result, dict): + self.usage = self.usage + TokenUsage.from_acp(result.get("usage")) + except Exception: # noqa: BLE001 - see docstring + log.warning("Could not count prompt usage", exc_info=True) + + def _note_usage_update(self, update: dict[str, Any]) -> None: + """Take the context reading and the cost from a ``usage_update``. + + ``used`` is the context occupancy after the last assistant message and + ``size`` the window; both are latest values. ``cost.amount`` is the + SDK's ``total_cost_usd``, which is cumulative for the Claude process — + one process per client — so it is *assigned*, through ``max`` to keep + it monotonic, never added. Its tokens are not here: they arrive on the + prompt response (and a background task's never do; see + :class:`TokenUsage`). + """ + used = update.get("used") + size = update.get("size") + if isinstance(used, int) and not isinstance(used, bool): + self.usage.context_used = used + if isinstance(size, int) and not isinstance(size, bool) and size > 0: + self.usage.context_size = size + cost = update.get("cost") + if isinstance(cost, dict) and cost.get("currency", "USD") == "USD": + amount = cost.get("amount") + if isinstance(amount, (int, float)) and not isinstance(amount, bool): + self.usage.cost_usd = max(self.usage.cost_usd, float(amount)) + async def _on_request_permission( self, sessionId: str = "", diff --git a/condor/acp/pydantic_ai_client.py b/condor/acp/pydantic_ai_client.py index 34d489bdd..f2265fca9 100644 --- a/condor/acp/pydantic_ai_client.py +++ b/condor/acp/pydantic_ai_client.py @@ -29,6 +29,7 @@ ToolCallEvent, ToolCallUpdate, ) +from .usage import TokenUsage log = logging.getLogger(__name__) @@ -437,6 +438,10 @@ def __init__( # There is no protocol to notify here (the "agent" is a library call), # so cancelling the run *is* the cancel. self._abort_requested = False + # Everything this client's runs have read and written, for its whole + # life (FEAT-120). The session takes per-turn deltas of it; nothing + # here knows what a turn is. + self.usage = TokenUsage() async def _build_model(self) -> Any: """Build the pydantic-ai model object with sensible defaults. @@ -988,10 +993,13 @@ async def prompt_stream( # event the dashboard already received. blocked_ids: set[str] = set() - async with self._agent.iter( - self._build_user_prompt(text, images), - message_history=self._message_history, - ) as run: + async with ( + self._agent.iter( + self._build_user_prompt(text, images), + message_history=self._message_history, + ) as run, + self._usage_counted(run), + ): async for node in run: if self._abort_requested: aborted = True @@ -1034,6 +1042,85 @@ async def prompt_stream( yield TextChunk(text=self._format_error(e)) yield PromptDone(stop_reason="error") + @contextlib.asynccontextmanager + async def _usage_counted(self, run: Any) -> AsyncIterator[None]: + """Count the run's usage however its block is left. + + On the way out of the run, not beside the history accumulation: a + finished run, a stopped one and one that raised all spent their + requests, and so does one whose consumer walked away mid-answer (a WS + drop, a page reload), which closes this generator at a ``yield`` and + never reaches the lines after the node loop. The run is still open + here, so ``PromptDone`` — yielded after this exits — already sees the + turn's tokens. + """ + try: + yield + finally: + self._fold_usage(run) + + def _fold_usage(self, run: Any) -> None: + """Add one run's tokens, price and context reading to :attr:`usage`. + + pydantic-ai's ``input_tokens`` is already inclusive of cache, which is + the shape :class:`TokenUsage` stores, so the counters go in unconverted. + + The price is all or nothing per run: one response the bundled + ``genai_prices`` table cannot price (every local model, a brand-new + id) makes the run's cost unknown rather than understated, and it is + counted in ``unpriced_turns`` instead. + + Never raises: accounting must not cost the user their answer. + """ + try: + from pydantic_ai.messages import ModelResponse + + ru = run.usage() + new_messages = ( + run.result.new_messages() + if run.result is not None + else run.new_messages() + ) + responses = [m for m in new_messages if isinstance(m, ModelResponse)] + cost = 0.0 + priced = True + for response in responses: + try: + cost += float(response.cost().total_price) + except Exception: # noqa: BLE001 - LookupError, the model_name assert + priced = False + break + context_used = None + if responses: + last = responses[-1].usage + context_used = (last.input_tokens + last.output_tokens) or None + self.usage = self.usage + TokenUsage( + input_tokens=ru.input_tokens, + output_tokens=ru.output_tokens, + cache_read_tokens=ru.cache_read_tokens, + cache_write_tokens=ru.cache_write_tokens, + cost_usd=cost if priced else 0.0, + unpriced_turns=0 if priced else 1, + context_used=context_used, + context_size=self._context_size(), + ) + except Exception: # noqa: BLE001 - see docstring + log.warning("Could not count usage for %s", self.model_name, exc_info=True) + + def _context_size(self) -> int | None: + """The model's context window, when it is known without a request. + + Only OpenRouter publishes one in a catalog Condor already fetches; a + local server or a natively resolved provider reports none, and the + readout then shows occupancy without a denominator. + """ + prefix, _, model_id = self.model_name.partition(":") + if prefix != "openrouter" or not model_id: + return None + from condor.llm.openrouter_models import cached_context_length + + return cached_context_length(model_id) + def _build_user_prompt(self, text: str, images: list | None) -> Any: """Assemble the user turn: images first, then the text. diff --git a/condor/acp/usage.py b/condor/acp/usage.py new file mode 100644 index 000000000..ae1ca6364 --- /dev/null +++ b/condor/acp/usage.py @@ -0,0 +1,151 @@ +"""What a model has read and written, in one shape for every backend (FEAT-120). + +Both clients already receive exact usage from their backend — the ACP adapter on +every ``session/prompt`` response and ``usage_update`` notification, pydantic-ai +on every run — and this is the one type they fold it into. It lives in +``condor.acp`` because both clients import it and ``condor.runtime`` already +depends on this package, never the reverse. +""" + +from __future__ import annotations + +from dataclasses import dataclass, fields +from typing import Any + +# Summed by ``+`` and subtracted by ``-``. The context readings are not: they +# are the latest value, and a sum of two occupancies means nothing. +_COUNTERS = ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", + "cost_usd", + "unpriced_turns", +) + + +@dataclass +class TokenUsage: + """A running tally of tokens and their price at API rates. + + **``input_tokens`` is inclusive**: every prompt token the model read, cache + hits and cache writes included, so ``total_tokens`` means "everything the + model read and wrote" on both backends. pydantic-ai (like OpenAI's + ``prompt_tokens``) already counts that way. Anthropic does not — its + ``input_tokens`` excludes both cache counters — so the ACP fold adds them + back in (:meth:`from_acp`). ``cache_*`` are therefore subsets of + ``input_tokens``, never additions to it. + + ``cost_usd`` is what the tokens would cost at API prices, summed over + whatever could be priced. A Claude subscription is not billed per token, so + this is an estimate and never a spend. ``unpriced_turns`` counts the runs + whose price could not be known (a local model, an id the bundled price + table does not have): a chat with both is a lower bound. + + The ACP adapter keeps a background task's result out of the turn's token + counts and reports only its cost and context. Those tokens are missing here + while their cost is not; reconstructing them is not worth it. + """ + + input_tokens: int = 0 + output_tokens: int = 0 + cache_read_tokens: int = 0 + cache_write_tokens: int = 0 + cost_usd: float = 0.0 + unpriced_turns: int = 0 + # Latest-value fields: overwritten, never summed. + context_used: int | None = None + context_size: int | None = None + + @property + def total_tokens(self) -> int: + return self.input_tokens + self.output_tokens + + def is_zero(self) -> bool: + """Nothing was counted. The context readings do not count as usage.""" + return not any(getattr(self, name) for name in _COUNTERS) + + def __add__(self, other: TokenUsage) -> TokenUsage: + summed = { + name: getattr(self, name) + getattr(other, name) for name in _COUNTERS + } + return TokenUsage( + **summed, + context_used=( + other.context_used + if other.context_used is not None + else self.context_used + ), + context_size=( + other.context_size + if other.context_size is not None + else self.context_size + ), + ) + + def __sub__(self, other: TokenUsage) -> TokenUsage: + """What ``self`` has that ``other`` had not. Floored at zero. + + The context readings are ``self``'s: a delta taken now carries the + latest occupancy, not a difference of two. + """ + return TokenUsage( + **{ + name: max(getattr(self, name) - getattr(other, name), 0) + for name in _COUNTERS + }, + context_used=self.context_used, + context_size=self.context_size, + ) + + def to_dict(self) -> dict[str, Any]: + out: dict[str, Any] = {f.name: getattr(self, f.name) for f in fields(self)} + # Float subtraction leaves dust (0.41 - 0.4 = 0.00999…); a micro-dollar + # is below anything the readout shows. + out["cost_usd"] = round(self.cost_usd, 6) + out["total_tokens"] = self.total_tokens + return out + + @classmethod + def from_dict(cls, data: dict | None) -> TokenUsage: + """Tolerant both ways: unknown keys are ignored, missing ones read as 0. + + A meta written before FEAT-120 has no ``usage`` at all, which reads as + zero — "unknown before this existed", not a real reading. + """ + if not isinstance(data, dict): + return cls() + usage = cls() + for name in _COUNTERS: + value = data.get(name) + if isinstance(value, (int, float)) and not isinstance(value, bool): + setattr(usage, name, float(value) if name == "cost_usd" else int(value)) + for name in ("context_used", "context_size"): + value = data.get(name) + if isinstance(value, int) and not isinstance(value, bool): + setattr(usage, name, value) + return usage + + @classmethod + def from_acp(cls, usage: Any) -> TokenUsage: + """A ``session/prompt`` response's ``usage``, with input made inclusive. + + The adapter reports Anthropic's split — ``inputTokens`` excludes + ``cachedReadTokens`` and ``cachedWriteTokens`` (its own ``totalTokens`` + is the sum of all four) — so they are added back into input here. + """ + if not isinstance(usage, dict): + return cls() + + def count(key: str) -> int: + value = usage.get(key) + return int(value) if isinstance(value, (int, float)) else 0 + + cache_read = count("cachedReadTokens") + cache_write = count("cachedWriteTokens") + return cls( + input_tokens=count("inputTokens") + cache_read + cache_write, + output_tokens=count("outputTokens"), + cache_read_tokens=cache_read, + cache_write_tokens=cache_write, + ) diff --git a/condor/llm/openrouter_models.py b/condor/llm/openrouter_models.py index 8a7b17b58..a0fb0af7a 100644 --- a/condor/llm/openrouter_models.py +++ b/condor/llm/openrouter_models.py @@ -98,6 +98,20 @@ async def fetch_models(force_refresh: bool = False) -> list[OpenRouterModel]: return models +def cached_context_length(model_id: str) -> int | None: + """The context window of ``model_id`` if the catalog is already cached. + + Never fetches: it runs at the end of every chat turn (FEAT-120), which must + not grow a network call. A stale cache still answers, since a model's + context window does not move by the hour. ``None`` when the catalog was + never fetched in this process or does not list the model. + """ + if _cache is None: + return None + model = find_model_by_slug(_cache[1], model_id) + return model.context_length if model and model.context_length > 0 else None + + def format_button_label(model: OpenRouterModel) -> str: """Short label for inline keyboard buttons. Telegram max ~30 chars looks good.""" label = model.name or model.slug diff --git a/condor/runtime/client.py b/condor/runtime/client.py index 07badeb74..53f78fef9 100644 --- a/condor/runtime/client.py +++ b/condor/runtime/client.py @@ -171,6 +171,20 @@ def _model_of(agent_key: str) -> str: return agent_key.split(":", 1)[1] if ":" in agent_key else agent_key +def _attach_usage(done: RuntimeEvent, session, recorder) -> None: + """Charge this turn what the client spent since the last one (FEAT-120). + + Taken here, the fifth concern this funnel owns on the same argument as the + other four: every chat turn on every surface ends here, so the clients only + count and never have to know what a turn is — and none of the half-dozen + places that synthesize a ``PromptDone`` has to learn about usage. Riding the + DONE event rather than a new one keeps the wire shape: consumers gain a key. + """ + turn_usage = session.take_usage() + done.data["usage"] = turn_usage.to_dict() + recorder.note_usage(turn_usage) + + async def prompt( key: SessionKey, req: PromptRequest, @@ -341,6 +355,7 @@ async def prompt( tool_kinds[kind] = tool_kinds.get(kind, 0) + 1 elif runtime_event.type is EventType.DONE: outcome = "done" + _attach_usage(runtime_event, session, recorder) recorder.observe(runtime_event) yield runtime_event except Exception as exc: # noqa: BLE001 - surfaced to the caller as an event @@ -349,8 +364,13 @@ async def prompt( failure = RuntimeEvent.error(str(exc), session_key=raw_key) recorder.observe(failure) yield failure - yield RuntimeEvent.done("error", session_key=raw_key) + done = RuntimeEvent.done("error", session_key=raw_key) + _attach_usage(done, session, recorder) + yield done finally: + # A turn that never reached DONE — the abandoned generator below — is + # counted here or not at all. After a DONE this takes nothing. + recorder.note_usage(session.take_usage()) # Not on DONE: the dashboard abandons this generator constantly (page # reload, abort_prompt cancelling the task, WS disconnect), and an # abandoned async generator only ever gets GeneratorExit. Losing the diff --git a/condor/runtime/conversations.py b/condor/runtime/conversations.py index 9bcea882c..e553b883a 100644 --- a/condor/runtime/conversations.py +++ b/condor/runtime/conversations.py @@ -47,6 +47,7 @@ fold_tool_call_event, normalize_tool_title, ) +from condor.acp.usage import TokenUsage from condor.fsutil import atomic_write_bytes from condor.runtime.events import EventType from condor.runtime.registry_file import read_status, write_status @@ -336,6 +337,15 @@ class ConversationMeta(BaseModel): default=False, description="Did the pass actually learn something?" ) + # ── Token usage (FEAT-120) ── + # The running total, in ``TokenUsage.to_dict()``'s shape. A plain dict and + # not a nested model because ``write_status`` merges top-level keys. Empty + # on every conversation older than this, which reads as "not measured". + usage: dict = Field( + default_factory=dict, + description="Running TokenUsage total; {} = unknown (before FEAT-120).", + ) + class TurnEntry(BaseModel): """One line of the transcript. @@ -418,6 +428,14 @@ class TurnEntry(BaseModel): "repeated here. Empty = order not recorded (pre-ARCH-330 turns)." ), ) + usage: dict = Field( + default_factory=dict, + description=( + "What this turn cost, in TokenUsage.to_dict()'s shape, on the last " + "entry the turn wrote. Includes anything spent since the previous " + "turn outside one (/compact). Empty = not measured (pre-FEAT-120)." + ), + ) # ── Paths ── @@ -702,6 +720,12 @@ def append_turn(user_id: int, conv_id: str, entry: TurnEntry) -> None: fields["title"] = _truncate(entry.text, TITLE_MAX_CHARS) if entry.role == "assistant" and entry.text: fields["last_snippet"] = _truncate(entry.text, SNIPPET_MAX_CHARS) + if entry.usage: + # Beside ``turn_count`` and on exactly its terms: one read-modify-write + # per turn, merged under ``write_status``'s lock. + fields["usage"] = ( + TokenUsage.from_dict(meta.usage) + TokenUsage.from_dict(entry.usage) + ).to_dict() write_status(conv_dir, META_FILENAME, **fields) @@ -994,6 +1018,9 @@ def __init__( # what was in them. self._events: list[dict] = [] self._error = "" + # What this turn cost, handed in by the funnel (FEAT-120) — once at + # DONE and once more in its ``finally``, which after a DONE adds zero. + self._usage = TokenUsage() # Stays empty unless a DONE arrives: an abandoned generator never # reports an ending, and "unknown" is the honest record of that. self._stop = "" @@ -1081,6 +1108,11 @@ def observe(self, event) -> None: elif event.type == EventType.DONE: self._stop = event.stop_reason + def note_usage(self, usage: TokenUsage) -> None: + """Add to what this turn is charged. Never writes.""" + if self.enabled: + self._usage = self._usage + usage + def _note_thought(self, text: str) -> None: """Extend the run's trailing reasoning step, or open a new one. @@ -1170,13 +1202,11 @@ def flush(self) -> None: attachments=self._attachments, ) ) - append_turn(self.user_id, self.conv_id, opening) + entries = [opening] text = "".join(self._text) tools = self._recorded_calls() if text or tools or self._thought: - append_turn( - self.user_id, - self.conv_id, + entries.append( TurnEntry( role="assistant", text=text, @@ -1185,14 +1215,17 @@ def flush(self) -> None: events=self._recorded_events(tools), stop_reason=self._stop, **self._attribution(), - ), + ) ) elif self._error: - append_turn( - self.user_id, - self.conv_id, - TurnEntry(role="system", text=self._error, kind="error"), - ) + entries.append(TurnEntry(role="system", text=self._error, kind="error")) + # On the *last* entry written — the answer, else the error, else the + # opening line — so a turn that failed or never answered still adds + # what it spent to the conversation's total. + if not self._usage.is_zero(): + entries[-1].usage = self._usage.to_dict() + for entry in entries: + append_turn(self.user_id, self.conv_id, entry) except Exception: # noqa: BLE001 - recording must not break a prompt log.warning("Could not record turn for %s", self.conv_id, exc_info=True) diff --git a/condor/runtime/sessions.py b/condor/runtime/sessions.py index 3b06e928d..f55c894ea 100644 --- a/condor/runtime/sessions.py +++ b/condor/runtime/sessions.py @@ -12,6 +12,7 @@ import asyncio import logging from contextlib import asynccontextmanager +from copy import copy from dataclasses import dataclass, field from datetime import datetime, timezone @@ -20,6 +21,7 @@ from condor.acp import ACPClient, PermissionCallback, PromptDone from condor.acp.client import ToolCallEvent, ToolCallUpdate, fold_tool_call_event from condor.acp.pydantic_ai_client import PydanticAIClient +from condor.acp.usage import TokenUsage from condor.agents import deeds from condor.agents.agent import identity_header as agent_identity_header from condor.runtime import binding, conversations @@ -161,6 +163,10 @@ class AgentSession: user_data: dict | None = None _lock: asyncio.Lock = field(default_factory=asyncio.Lock) _abort_event: asyncio.Event = field(default_factory=asyncio.Event) + # How much of ``client.usage`` has already been handed to a turn + # (FEAT-120). Per session, and a session owns exactly one client for its + # life, so this baseline never straddles two counters. + _usage_recorded: TokenUsage = field(default_factory=TokenUsage) def info(self) -> SessionInfo: """Serializable view of this session.""" @@ -181,6 +187,26 @@ def info(self) -> SessionInfo: conversation_id=self.conversation_id, ) + def take_usage(self) -> TokenUsage: + """What the client has spent since the last take. Advances the baseline. + + Idempotent per turn: a second call with nothing new in between returns + zero counters (``context_*`` still carry the latest reading). Anything + the client spent outside a recorded turn — ``/compact``, the eager + context prompt, a cancelled turn's late response — is in the next take, + so it is charged one turn late but never lost. + + A client that never learned to count (a test double, a future backend) + records nothing rather than failing the turn. + """ + current = getattr(self.client, "usage", None) + if not isinstance(current, TokenUsage): + return TokenUsage() + now = copy(current) + delta = now - self._usage_recorded + self._usage_recorded = now + return delta + async def prompt_stream( self, text: str, diff --git a/condor/web/routes/chat_ws.py b/condor/web/routes/chat_ws.py index a7ff13058..63e091dcb 100644 --- a/condor/web/routes/chat_ws.py +++ b/condor/web/routes/chat_ws.py @@ -231,6 +231,10 @@ def _to_ws_message(event: RuntimeEvent, slot_id: str) -> dict | None: "event": "prompt_done", "slot_id": slot_id, "stop_reason": event.stop_reason, + # What this turn cost (FEAT-120), added to the total the dashboard + # seeded from the conversation's meta. An added key: absent (null) + # on a DONE the funnel did not charge, which the client skips. + "usage": event.field("usage"), } if event.type == EventType.ERROR: return { diff --git a/tests/runtime/test_session_api.py b/tests/runtime/test_session_api.py index 082484a5c..bfa0e50d6 100644 --- a/tests/runtime/test_session_api.py +++ b/tests/runtime/test_session_api.py @@ -213,7 +213,14 @@ def test_ws_contract_unchanged(): done = _to_ws_message( RuntimeEvent.from_acp(PromptDone(stop_reason="end_turn")), "s" ) - assert done == {"event": "prompt_done", "slot_id": "s", "stop_reason": "end_turn"} + # ``usage`` is an added key (FEAT-120): what the funnel charged the turn, + # null on a DONE it did not charge — this one never went through it. + assert done == { + "event": "prompt_done", + "slot_id": "s", + "stop_reason": "end_turn", + "usage": None, + } err = _to_ws_message(RuntimeEvent.error("nope"), "slot1") assert err == {"event": "error", "slot_id": "slot1", "message": "nope"} diff --git a/tests/test_conversations_api.py b/tests/test_conversations_api.py index 222c7afc2..e1e10a5f4 100644 --- a/tests/test_conversations_api.py +++ b/tests/test_conversations_api.py @@ -73,6 +73,32 @@ def test_get_returns_meta_and_transcript(store): assert body["turns"][0]["text"] == "what is my pnl?" +def test_the_running_token_total_rides_the_meta(store): + """FEAT-120: no route change — the total is on the model both routes dump.""" + from condor.acp.usage import TokenUsage + + meta = new_conversation(USER.id, "web", agent_key="claude-code") + append_turn(USER.id, meta.id, TurnEntry(role="user", text="hello")) + append_turn( + USER.id, + meta.id, + TurnEntry( + role="assistant", + text="hi back", + usage=TokenUsage( + input_tokens=1200, output_tokens=30, cost_usd=0.02 + ).to_dict(), + ), + ) + + body = _client(USER).get(f"/conversations/{meta.id}").json() + assert body["meta"]["usage"]["total_tokens"] == 1230 + assert body["meta"]["usage"]["cost_usd"] == 0.02 + assert body["turns"][-1]["usage"]["input_tokens"] == 1200 + listed = _client(USER).get("/conversations").json() + assert listed[0]["usage"]["input_tokens"] == 1200 + + def test_unknown_conversation_is_404(store): assert _client(USER).get("/conversations/nope").status_code == 404 diff --git a/tests/test_token_usage.py b/tests/test_token_usage.py new file mode 100644 index 000000000..1cc9fae62 --- /dev/null +++ b/tests/test_token_usage.py @@ -0,0 +1,578 @@ +"""A conversation knows how many tokens it has spent (FEAT-120). + +Both agent backends already receive exact usage — the ACP adapter on every +``session/prompt`` response and ``usage_update``, pydantic-ai on every run — and +used to throw it away. The design splits the work three ways and these tests +pin each part where it lives: + +- **counting** in the client, a lifetime counter that sees every token, + cancelled and abandoned turns included; +- **attribution** in the funnel, which charges each turn the counter's delta + at DONE and again in its ``finally``; +- **persistence** in the Recorder and ``append_turn``, beside ``turn_count``. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +from decimal import Decimal +from types import SimpleNamespace + +import pytest +from pydantic_ai import Agent +from pydantic_ai.messages import ModelResponse, TextPart +from pydantic_ai.models.function import AgentInfo, FunctionModel +from pydantic_ai.usage import RequestUsage + +from condor.acp.client import ACPClient, PromptDone, TextChunk +from condor.acp.pydantic_ai_client import PydanticAIClient +from condor.acp.usage import TokenUsage +from condor.llm import openrouter_models +from condor.runtime import PromptRequest, SessionKey +from condor.runtime import client as runtime +from condor.runtime import conversations +from condor.runtime import sessions as sessions_module +from condor.runtime.conversations import ( + TurnEntry, + append_turn, + get_conversation, + new_conversation, + read_transcript, +) +from condor.runtime.events import EventType, RuntimeEvent + +USER = 4120 + + +# ── The type ── + + +def test_adding_sums_counters_and_keeps_the_latest_context_reading(): + a = TokenUsage( + input_tokens=100, + output_tokens=10, + cost_usd=0.5, + context_used=100, + context_size=200_000, + ) + b = TokenUsage(input_tokens=50, output_tokens=5, unpriced_turns=1, context_used=150) + + total = a + b + + assert (total.input_tokens, total.output_tokens) == (150, 15) + assert total.cost_usd == 0.5 + assert total.unpriced_turns == 1 + assert total.context_used == 150, "latest reading, never a sum" + assert total.context_size == 200_000, "an absent reading keeps the last one" + assert total.total_tokens == 165 + + +def test_subtracting_floors_at_zero_and_carries_the_newer_context(): + now = TokenUsage(input_tokens=100, cost_usd=0.41, context_used=900) + before = TokenUsage(input_tokens=120, cost_usd=0.40, context_used=100) + + delta = now - before + + assert delta.input_tokens == 0 + assert delta.to_dict()["cost_usd"] == pytest.approx(0.01) + assert delta.context_used == 900 + + +def test_the_dict_round_trip_is_tolerant_both_ways(): + usage = TokenUsage( + input_tokens=3, + output_tokens=4, + cache_read_tokens=1, + cost_usd=0.25, + context_size=8, + ) + assert TokenUsage.from_dict(usage.to_dict()) == usage + assert TokenUsage.from_dict(None) == TokenUsage() + assert TokenUsage.from_dict({}) == TokenUsage() + # A newer build's key and a garbled value are ignored, not fatal. + assert TokenUsage.from_dict( + {"input_tokens": 5, "mystery": 1, "context_used": "lots"} + ) == TokenUsage(input_tokens=5) + + +def test_acp_input_is_made_inclusive_of_cache(): + """Anthropic's ``inputTokens`` excludes both cache counters; Condor's includes them.""" + wire = { + "inputTokens": 10, + "outputTokens": 5, + "cachedReadTokens": 1000, + "cachedWriteTokens": 200, + "totalTokens": 1215, + } + + usage = TokenUsage.from_acp(wire) + + assert usage.input_tokens == 1210 + assert usage.cache_read_tokens == 1000 + assert usage.cache_write_tokens == 200 + assert usage.total_tokens == wire["totalTokens"] + + +# ── pydantic-ai ── + + +def _answer(messages: list, info: AgentInfo) -> ModelResponse: + return ModelResponse( + parts=[TextPart("SOL-USDC")], + usage=RequestUsage(input_tokens=1000, output_tokens=50, cache_read_tokens=400), + ) + + +def _pai(model: str = "openai:gpt-4o") -> PydanticAIClient: + client = PydanticAIClient(model) + client._agent = Agent(FunctionModel(_answer)) + return client + + +def _drive(client: PydanticAIClient) -> list: + async def run() -> list: + return [event async for event in client.prompt_stream("go")] + + return asyncio.run(run()) + + +def test_pydantic_ai_tokens_accumulate_across_runs(): + client = _pai() + + _drive(client) + _drive(client) + + # Already inclusive on this backend: cache is not added a second time. + assert client.usage.input_tokens == 2000 + assert client.usage.cache_read_tokens == 800 + assert client.usage.output_tokens == 100 + assert client.usage.context_used == 1050 + + +def test_an_unpriceable_model_counts_tokens_and_no_cost(): + """``FunctionModel`` stamps an id no price table knows, like any local model.""" + client = _pai() + + _drive(client) + + assert client.usage.total_tokens == 1050 + assert client.usage.cost_usd == 0 + assert client.usage.unpriced_turns == 1 + + +def test_a_priced_model_sums_its_cost(monkeypatch): + monkeypatch.setattr( + ModelResponse, + "cost", + lambda self: SimpleNamespace(total_price=Decimal("0.0125")), + ) + client = _pai() + + _drive(client) + _drive(client) + + assert client.usage.cost_usd == pytest.approx(0.025) + assert client.usage.unpriced_turns == 0 + + +def test_a_stopped_run_still_counts(): + client = _pai() + + async def run() -> list: + events = [] + async for event in client.prompt_stream("go"): + events.append(event) + if isinstance(event, TextChunk): + await client.abort_prompt() + return events + + events = asyncio.run(run()) + + assert events[-1] == PromptDone(stop_reason="cancelled") + assert client.usage.input_tokens == 1000 + + +def test_a_run_whose_consumer_walked_away_still_counts(): + """A WS drop closes the generator at a ``yield``, before the node loop ends.""" + client = _pai() + + async def run() -> None: + stream = client.prompt_stream("go") + async for event in stream: + if isinstance(event, TextChunk): + break + # Unwinding pydantic-ai's run from a closed generator raises on its own + # (its anyio cancel scope; it did before FEAT-120 too, and in + # production the finalizer only logs it). What this pins is that the + # usage was folded first: the fold is the innermost exit. + with contextlib.suppress(RuntimeError, GeneratorExit): + await stream.aclose() + + asyncio.run(run()) + + assert client.usage.input_tokens == 1000 + + +def test_the_openrouter_window_comes_from_the_cached_catalog_only(monkeypatch): + monkeypatch.setattr(openrouter_models, "_cache", None) + assert openrouter_models.cached_context_length("x/y") is None, "never fetches" + + catalog = [openrouter_models.OpenRouterModel("x/y", "Y", 128_000, 0.0, 0.0)] + monkeypatch.setattr(openrouter_models, "_cache", (0.0, catalog)) + client = _pai("openrouter:x/y") + + _drive(client) + + assert client.usage.context_size == 128_000 + assert _pai("ollama:qwen3")._context_size() is None + + +# ── ACP ── + + +class _FakeStdin: + def __init__(self) -> None: + self.written: list[dict] = [] + + def write(self, data: bytes) -> None: + self.written.append(json.loads(data.decode())) + + async def drain(self) -> None: + pass + + +class _FakeProcess: + def __init__(self, stdout: asyncio.StreamReader) -> None: + self.stdout = stdout + self.stdin = _FakeStdin() + self.returncode = None + + +def _acp() -> tuple[ACPClient, asyncio.StreamReader]: + """A real client over a fake pipe: the peer, the read loop and the done callback are real.""" + stdout = asyncio.StreamReader() + client = ACPClient(command="true") + client._process = _FakeProcess(stdout) # type: ignore[assignment] + client._session_id = "s1" + return client, stdout + + +async def _prompt_id(client: ACPClient) -> int: + for _ in range(500): + for message in client._process.stdin.written: + if message.get("method") == "session/prompt": + return message["id"] + await asyncio.sleep(0) + raise AssertionError("session/prompt was never sent") + + +def _response(req_id: int, result: dict) -> bytes: + return ( + json.dumps({"jsonrpc": "2.0", "id": req_id, "result": result}) + "\n" + ).encode() + + +_WIRE_USAGE = { + "inputTokens": 3, + "outputTokens": 40, + "cachedReadTokens": 30_000, + "cachedWriteTokens": 2_000, + "totalTokens": 32_043, +} + + +def test_a_prompt_responses_usage_is_counted_inclusively(): + async def scenario() -> tuple[ACPClient, list]: + client, stdout = _acp() + reader = asyncio.create_task(client._read_loop()) + stream_events: list = [] + + async def consume() -> None: + async for event in client.prompt_stream("hi"): + stream_events.append(event) + + task = asyncio.create_task(consume()) + req_id = await _prompt_id(client) + stdout.feed_data( + _response(req_id, {"stopReason": "end_turn", "usage": _WIRE_USAGE}) + ) + await asyncio.wait_for(task, timeout=5) + reader.cancel() + return client, stream_events + + client, events = asyncio.run(scenario()) + + assert events[-1] == PromptDone(stop_reason="end_turn") + assert client.usage.input_tokens == 32_003 + assert client.usage.total_tokens == _WIRE_USAGE["totalTokens"] + + +def test_a_stale_response_from_a_cancelled_turn_still_counts(): + """The screen ended at the local cancel; the tokens arrive afterwards.""" + + async def scenario() -> tuple[TokenUsage, TokenUsage, list]: + client, stdout = _acp() + reader = asyncio.create_task(client._read_loop()) + stream_events: list = [] + + async def consume() -> None: + async for event in client.prompt_stream("hi"): + stream_events.append(event) + + task = asyncio.create_task(consume()) + req_id = await _prompt_id(client) + # The agent ignored session/cancel: the fallback ends the turn here. + client._cancel_locally(req_id) + await asyncio.wait_for(task, timeout=5) + before = client.usage + + stdout.feed_data( + _response(req_id, {"stopReason": "cancelled", "usage": _WIRE_USAGE}) + ) + for _ in range(500): + if not client.usage.is_zero(): + break + await asyncio.sleep(0) + reader.cancel() + return before, client.usage, stream_events + + before, after, events = asyncio.run(scenario()) + + assert events[-1] == PromptDone(stop_reason="cancelled") + assert before.is_zero() + assert after.output_tokens == 40 + assert after.input_tokens == 32_003 + + +def _usage_update(amount: float, used: int = 45_000, size: int = 200_000) -> dict: + return { + "sessionUpdate": "usage_update", + "used": used, + "size": size, + "cost": {"amount": amount, "currency": "USD"}, + } + + +def test_a_usage_update_with_no_turn_in_flight_still_moves_the_counter(): + """A background task's result lands after the turn settled — its cost is only here.""" + client = ACPClient(command="true") + assert client._current_req_id is None + + client._on_session_update("s1", _usage_update(0.41)) + + assert client.usage.cost_usd == 0.41 + assert client.usage.context_used == 45_000 + assert client.usage.context_size == 200_000 + assert client._event_queue.empty(), "the counter moves; nothing is relayed" + + +def test_acp_cost_is_cumulative_and_never_goes_down(): + client = ACPClient(command="true") + client._current_req_id = 7 + + client._on_session_update("s1", _usage_update(0.41)) + client._on_session_update("s1", _usage_update(0.30)) + assert client.usage.cost_usd == 0.41 + + client._on_session_update("s1", _usage_update(0.55, used=60_000)) + assert client.usage.cost_usd == 0.55 + assert client.usage.context_used == 60_000 + assert client._event_queue.empty() + + +# ── Attribution and persistence ── + + +class _MeteredClient: + """A chat client whose counter moves while it answers, like a real one's. + + A script step is either an event to yield or a ``TokenUsage`` to spend. + """ + + def __init__(self) -> None: + self.alive = True + self.usage = TokenUsage() + self.script: list = [] + + async def prompt_stream(self, text, images=None): + for step in self.script: + await asyncio.sleep(0) + if isinstance(step, TokenUsage): + self.usage = self.usage + step + elif isinstance(step, Exception): + raise step + else: + yield step + + async def abort_prompt(self) -> None: + pass + + +def _spend(i: int, o: int) -> TokenUsage: + return TokenUsage(input_tokens=i, output_tokens=o) + + +@pytest.fixture +def chat(monkeypatch): + """One live web session over a metered client, answering into a real conversation.""" + monkeypatch.setattr(conversations, "_live_recorders", set()) + meta = new_conversation(USER, "web", agent_key="claude-code") + key = SessionKey.web(USER, meta.id) + client = _MeteredClient() + session = sessions_module.AgentSession( + key=key, + agent_key="claude-code", + client=client, + user_id=USER, + conversation_id=meta.id, + ) + # No spec, so the funnel's staleness check has nothing to rebuild from. + monkeypatch.setattr(sessions_module, "_sessions", {str(key): session}) + return SimpleNamespace(key=key, client=client, conv_id=meta.id) + + +def _turn(chat, text: str = "go") -> list[RuntimeEvent]: + async def run() -> list[RuntimeEvent]: + return [e async for e in runtime.prompt(chat.key, PromptRequest(text=text))] + + return asyncio.run(run()) + + +def _done(events: list[RuntimeEvent]) -> RuntimeEvent: + return [e for e in events if e.type is EventType.DONE][-1] + + +def test_the_done_event_carries_what_the_turn_spent(chat): + chat.client.script = [ + TextChunk(text="hi"), + _spend(1000, 50), + PromptDone(stop_reason="end_turn"), + ] + + events = _turn(chat) + + assert _done(events).field("usage")["total_tokens"] == 1050 + + +def test_the_prompt_done_frame_carries_the_turns_usage_as_an_added_key(): + from condor.web.routes.chat_ws import _to_ws_message + + done = RuntimeEvent.done("end_turn", session_key="k") + done.data["usage"] = _spend(10, 2).to_dict() + + frame = _to_ws_message(done, "s1") + + assert frame["event"] == "prompt_done" + assert frame["stop_reason"] == "end_turn" + assert frame["usage"]["total_tokens"] == 12 + # A DONE the funnel never charged still makes a frame, with nothing to add. + assert _to_ws_message(RuntimeEvent.done("cancelled"), "s1")["usage"] is None + + +def test_two_turns_add_up_on_the_conversation(chat): + chat.client.script = [_spend(1000, 50), TextChunk(text="a"), PromptDone("end_turn")] + _turn(chat) + chat.client.script = [ + _spend(2000, 100), + TextChunk(text="b"), + PromptDone("end_turn"), + ] + _turn(chat) + + meta = get_conversation(USER, chat.conv_id) + assert meta.usage["input_tokens"] == 3000 + assert meta.usage["output_tokens"] == 150 + answers = [t for t in read_transcript(USER, chat.conv_id) if t.role == "assistant"] + assert [t.usage["total_tokens"] for t in answers] == [1050, 2100] + user_turns = [t for t in read_transcript(USER, chat.conv_id) if t.role == "user"] + assert all(t.usage == {} for t in user_turns), "stamped once, on the last entry" + + +def test_an_abandoned_turn_is_still_charged(chat): + """A page reload mid-answer: the funnel's generator only ever sees GeneratorExit.""" + chat.client.script = [ + TextChunk(text="half an ans"), + _spend(700, 7), + TextChunk(text="wer"), + TextChunk(text=" never finished"), + ] + + async def walk_away() -> None: + stream = runtime.prompt(chat.key, PromptRequest(text="go")) + seen = 0 + async for event in stream: + if event.type is EventType.TEXT: + seen += 1 + if seen == 2: + break + await stream.aclose() + + asyncio.run(walk_away()) + + (answer,) = [ + t for t in read_transcript(USER, chat.conv_id) if t.role == "assistant" + ] + assert answer.usage["total_tokens"] == 707 + assert get_conversation(USER, chat.conv_id).usage["total_tokens"] == 707 + + +def test_a_turn_that_errored_is_charged_on_its_error_entry(chat): + chat.client.script = [_spend(300, 0), RuntimeError("upstream 500")] + + events = _turn(chat) + + assert _done(events).stop_reason == "error" + assert _done(events).field("usage")["input_tokens"] == 300 + last = read_transcript(USER, chat.conv_id)[-1] + assert (last.role, last.kind) == ("system", "error") + assert last.usage["input_tokens"] == 300 + assert get_conversation(USER, chat.conv_id).usage["input_tokens"] == 300 + + +def test_tokens_spent_between_turns_land_on_the_next_one(chat): + """``/compact`` prompts the client outside the funnel; its tokens are not lost.""" + chat.client.script = [_spend(100, 0), TextChunk(text="a"), PromptDone("end_turn")] + _turn(chat) + chat.client.usage = chat.client.usage + _spend(500, 20) # the /compact summary + chat.client.script = [_spend(10, 1), TextChunk(text="b"), PromptDone("end_turn")] + _turn(chat) + + answers = [t for t in read_transcript(USER, chat.conv_id) if t.role == "assistant"] + assert [t.usage["input_tokens"] for t in answers] == [100, 510] + assert get_conversation(USER, chat.conv_id).usage["total_tokens"] == 631 + + +def test_a_client_that_does_not_count_records_nothing(chat): + """A test double or a future backend: no ``usage`` attribute, no stamp, no crash.""" + del chat.client.usage + chat.client.script = [TextChunk(text="a"), PromptDone("end_turn")] + + events = _turn(chat) + + assert _done(events).field("usage")["total_tokens"] == 0 + assert all(t.usage == {} for t in read_transcript(USER, chat.conv_id)) + assert get_conversation(USER, chat.conv_id).usage == {} + + +def test_a_meta_written_before_usage_existed_loads_and_merges_from_zero(): + meta = new_conversation(USER, "web", agent_key="claude-code") + meta_path = conversations._conv_dir(USER, meta.id) / conversations.META_FILENAME + on_disk = json.loads(meta_path.read_text()) + on_disk.pop("usage", None) + meta_path.write_text(json.dumps(on_disk)) + + assert get_conversation(USER, meta.id).usage == {} + + append_turn( + USER, + meta.id, + TurnEntry(role="assistant", text="x", usage=_spend(7, 3).to_dict()), + ) + + assert get_conversation(USER, meta.id).usage["total_tokens"] == 10 + + +def test_a_transcript_line_older_than_usage_still_parses(): + assert TurnEntry.model_validate({"role": "user", "text": "hi"}).usage == {} From 5921d004a101cb5f46bfff39bd956a07d604068f Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 10 Sep 2026 19:35:29 +0300 Subject: [PATCH 137/154] =?UTF-8?q?Show=20a=20conversation's=20tokens,=20c?= =?UTF-8?q?ontext=20and=20=E2=89=88=20API=20cost=20under=20the=20chat:=20s?= =?UTF-8?q?eeded=20from=20the=20stored=20total=20on=20hydrate,=20advanced?= =?UTF-8?q?=20by=20each=20prompt=5Fdone=20(FEAT-120)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cost hides for a chat nothing in which could be priced (a local model), reads ≥ once only part of it could, and the context turns amber past 80%. --- frontend/src/components/chat/ChatThread.tsx | 3 + .../src/components/chat/ConversationUsage.tsx | 65 ++++++ frontend/src/hooks/useChatSocket.ts | 31 +++ .../src/hooks/useChatSocket.usage.test.tsx | 221 ++++++++++++++++++ frontend/src/lib/api.ts | 25 ++ frontend/src/lib/usage.test.ts | 69 ++++++ frontend/src/lib/usage.ts | 71 ++++++ 7 files changed, 485 insertions(+) create mode 100644 frontend/src/components/chat/ConversationUsage.tsx create mode 100644 frontend/src/hooks/useChatSocket.usage.test.tsx create mode 100644 frontend/src/lib/usage.test.ts create mode 100644 frontend/src/lib/usage.ts diff --git a/frontend/src/components/chat/ChatThread.tsx b/frontend/src/components/chat/ChatThread.tsx index dfbce9a82..a806c2c1f 100644 --- a/frontend/src/components/chat/ChatThread.tsx +++ b/frontend/src/components/chat/ChatThread.tsx @@ -7,6 +7,7 @@ import { speakerNames } from "@/lib/agentColor"; import { ApprovalPrompt } from "./ApprovalPrompt"; import { ChatInput } from "./ChatInput"; import { ChatMessageView } from "./ChatMessage"; +import { ConversationUsage } from "./ConversationUsage"; import { Starters, type Starter } from "./Starters"; /** How close to the end still counts as "following the answer", in pixels. */ @@ -219,6 +220,8 @@ export function ChatThread({
)} + + {/* Messages area */}
{/* `min-h-full` so an empty state can centre itself in the viewport diff --git a/frontend/src/components/chat/ConversationUsage.tsx b/frontend/src/components/chat/ConversationUsage.tsx new file mode 100644 index 000000000..8f4c9ac84 --- /dev/null +++ b/frontend/src/components/chat/ConversationUsage.tsx @@ -0,0 +1,65 @@ +import type { TokenUsage } from "@/lib/api"; +import { formatCurrency } from "@/lib/formatters"; +import { CONTEXT_WARN_RATIO, costDisplay, formatTokens } from "@/lib/usage"; + +const COST_TITLE = "≈ API cost; a Claude subscription is not billed per token"; + +/** + * What this conversation has cost so far, on one muted line (FEAT-120). + * + * `38.2k tokens · context 45k / 200k · ≈ $0.41`. Every part is only as + * specific as the backend could be: the context reading appears once one was + * reported and gains its window only when the window is known, and the cost is + * hidden where nothing could be priced (a local model) and marked `≥` where + * only some of it could. Hidden entirely until a turn has been measured, so a + * conversation older than the measurement shows nothing rather than a zero. + */ +export function ConversationUsage({ usage }: { usage?: TokenUsage }) { + if (!usage || usage.total_tokens <= 0) return null; + + const cost = costDisplay(usage); + const used = usage.context_used; + const size = usage.context_size; + // The moment `/compact` becomes worth knowing about. + const nearlyFull = used != null && size != null && size > 0 && used / size > CONTEXT_WARN_RATIO; + const breakdown = [ + `in ${usage.input_tokens.toLocaleString()}`, + `out ${usage.output_tokens.toLocaleString()}`, + `cache read ${usage.cache_read_tokens.toLocaleString()}`, + `cache write ${usage.cache_write_tokens.toLocaleString()}`, + ].join(" · "); + + return ( +
+ {formatTokens(usage.total_tokens)} tokens + {used != null && ( + <> + · + + context {formatTokens(used)} + {size != null && ` / ${formatTokens(size)}`} + + + )} + {cost && ( + <> + · + + {cost.lowerBound ? "≥ " : "≈ "} + {formatCurrency(usage.cost_usd)} + + + )} +
+ ); +} diff --git a/frontend/src/hooks/useChatSocket.ts b/frontend/src/hooks/useChatSocket.ts index c737afd03..aed6c9008 100644 --- a/frontend/src/hooks/useChatSocket.ts +++ b/frontend/src/hooks/useChatSocket.ts @@ -7,8 +7,10 @@ import { type AppNotification, type ConversationTurn, type NotificationsResponse, + type TokenUsage, } from "@/lib/api"; import { useAuth } from "@/lib/auth"; +import { addUsage } from "@/lib/usage"; import { namesATool, toolCallState } from "@/lib/formatters"; import { collectViewFacts, renderViewBlock } from "@/lib/viewFacts"; import { WS_AUTH_SUBPROTOCOL } from "@/lib/websocket"; @@ -252,6 +254,12 @@ export interface ChatSlot { * lands, which is what makes a new chat feel warm instead of loading. */ pending?: boolean; + /** + * What this conversation has cost so far (FEAT-120): seeded from the stored + * total on hydrate and advanced by each `prompt_done`. Absent until the + * backend has measured something. + */ + usage?: TokenUsage; } let msgIdCounter = 0; @@ -1207,6 +1215,16 @@ export function useChatSocket() { : null; try { const detail = await api.getConversation(conversationId); + // The stored total, not a sum of what this tab happened to see: a + // resync re-seeds, so a turn answered from Telegram or another tab + // converges here. Before the empty-transcript return below, which is + // about messages only. + const usage = addUsage(undefined, detail.meta?.usage); + if (usage && usage.total_tokens > 0) { + setSlots((prev) => + prev.map((s) => (s.info.slot_id === slotId ? { ...s, usage } : s)), + ); + } const restored = turnsToMessages(detail.turns, conversationId); // An empty transcript never wipes the screen: on a resync that would // trade a missed note for a lost conversation. @@ -1796,6 +1814,19 @@ export function useChatSocket() { // mid-answer, and its composer stays locked until its own turn is // done. stopStreaming(slotId); + // What the turn cost, onto the total (FEAT-120). A frame without + // it — an older backend, a DONE the funnel did not charge — leaves + // the total where it was. + const turnUsage = data.usage as Partial | null | undefined; + if (turnUsage) { + setSlots((prev) => + prev.map((s) => + s.info.slot_id === slotId + ? { ...s, usage: addUsage(s.usage, turnUsage) } + : s, + ), + ); + } } break; diff --git a/frontend/src/hooks/useChatSocket.usage.test.tsx b/frontend/src/hooks/useChatSocket.usage.test.tsx new file mode 100644 index 000000000..ecba22de1 --- /dev/null +++ b/frontend/src/hooks/useChatSocket.usage.test.tsx @@ -0,0 +1,221 @@ +/** + * A slot knows what its conversation has cost (FEAT-120). + * + * The stored total is the truth — it survives a reload and it includes turns + * answered from Telegram or another tab — so a slot is seeded from it whenever + * the transcript is read, and each `prompt_done` adds the turn it closes. These + * pin both halves, and that a reconnect's re-read re-seeds rather than keeping + * a figure only this tab believed. + * + * Needs a DOM, so this file overrides vitest's default `node` environment. + * + * @vitest-environment jsdom + */ + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, useEffect } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ServerContext } from "@/hooks/useServer"; + +const getConversation = vi.fn(); + +vi.mock("@/lib/api", () => ({ + api: { + listConversations: () => Promise.resolve([]), + getSessionOptions: () => Promise.resolve({ default_agent: "claude-code" }), + getConversation: (...args: unknown[]) => getConversation(...args), + }, +})); + +vi.mock("@/lib/auth", () => ({ + useAuth: () => ({ token: "jwt", user: { id: 7 } }), +})); + +const { useChatSocket } = await import("./useChatSocket"); + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +class FakeSocket { + static last: FakeSocket | null = null; + + static readonly CONNECTING = 0; + static readonly OPEN = 1; + static readonly CLOSING = 2; + static readonly CLOSED = 3; + readyState = FakeSocket.OPEN; + onopen: (() => void) | null = null; + onclose: (() => void) | null = null; + onmessage: ((ev: { data: string }) => void) | null = null; + + constructor() { + FakeSocket.last = this; + } + send() {} + close() { + this.readyState = FakeSocket.CLOSED; + } + deliver(frame: Record) { + this.onmessage?.({ data: JSON.stringify(frame) }); + } +} + +const sock = () => FakeSocket.last!; + +const holder: { current: ReturnType | null } = { + current: null, +}; +const chat = () => holder.current!; + +function Harness() { + const state = useChatSocket(); + useEffect(() => { + holder.current = state; + }); + return null; +} + +let container: HTMLDivElement; +let root: Root; + +async function settle() { + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + }); +} + +/** The one session the server knows about in these tests. */ +const ROSTER = [{ slot_id: "s1", conversation_id: "c1", agent_key: "k" }]; + +const roster = () => ({ event: "sessions_list", sessions: ROSTER }); + +/** The token total of the slot under test. */ +const usageOf = () => chat().slots.find((s) => s.info.slot_id === "s1")?.usage; + +const TURNS = [{ role: "user", text: "run the audit", ts: "1", tool_calls: [] }]; + +/** Open the page and let the first roster hydrate the one live session. */ +async function arrive() { + act(() => { + root.render( + + {} }}> + + + , + ); + }); + act(() => { + chat().connect(); + sock().onopen?.(); + }); + act(() => { + sock().deliver(roster()); + }); + await settle(); +} + +/** Drop the socket and bring it back, exactly as the browser would. */ +function reconnect() { + act(() => { + sock().readyState = FakeSocket.CLOSED; + sock().onclose?.(); + }); + act(() => { + chat().connect(); + sock().onopen?.(); + }); +} + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + vi.stubGlobal("WebSocket", FakeSocket); + FakeSocket.last = null; + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + getConversation.mockResolvedValue({ + meta: { usage: { input_tokens: 1000, output_tokens: 50, cost_usd: 0.1 } }, + turns: TURNS, + }); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); + vi.clearAllMocks(); +}); + +describe("a conversation's token total", () => { + it("is seeded from the stored total when the transcript hydrates", async () => { + await arrive(); + + expect(usageOf()?.total_tokens).toBe(1050); + expect(usageOf()?.cost_usd).toBeCloseTo(0.1); + }); + + it("advances by what each turn cost", async () => { + await arrive(); + + act(() => { + sock().deliver({ + event: "prompt_done", + slot_id: "s1", + stop_reason: "end_turn", + usage: { + input_tokens: 200, + output_tokens: 10, + cost_usd: 0.05, + context_used: 5000, + context_size: 200000, + }, + }); + }); + + expect(usageOf()?.total_tokens).toBe(1260); + expect(usageOf()?.cost_usd).toBeCloseTo(0.15); + expect(usageOf()?.context_used).toBe(5000); + expect(usageOf()?.context_size).toBe(200000); + }); + + it("stays put on a prompt_done that carries no usage", async () => { + await arrive(); + + act(() => { + sock().deliver({ event: "prompt_done", slot_id: "s1", stop_reason: "cancelled" }); + }); + + expect(usageOf()?.total_tokens).toBe(1050); + }); + + it("re-seeds on a reconnect, so a turn answered elsewhere converges", async () => { + await arrive(); + // Telegram answered a turn while this tab's socket was down. + getConversation.mockResolvedValue({ + meta: { usage: { input_tokens: 5000, output_tokens: 100 } }, + turns: TURNS, + }); + + reconnect(); + act(() => { + sock().deliver(roster()); + }); + await settle(); + + expect(usageOf()?.total_tokens).toBe(5100); + }); + + it("shows nothing for a conversation older than the measurement", async () => { + getConversation.mockResolvedValue({ meta: {}, turns: TURNS }); + + await arrive(); + + expect(usageOf()).toBeUndefined(); + }); +}); diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index cdda52279..d09ae0ae5 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1746,6 +1746,31 @@ export interface ConversationMeta { share_id?: string; share_revision?: number; shared_at?: string | null; + /** Running token total (FEAT-120). `{}` or absent on a conversation older + * than the measurement, which reads as "not measured", not as zero. */ + usage?: Partial; +} + +/** + * What a model read and wrote — the backend's `TokenUsage.to_dict()`. + * + * `input_tokens` is inclusive of cache on every backend, so `total_tokens` is + * everything read and written; the two `cache_*` counters are subsets of it. + * `cost_usd` is an estimate at API prices, never a spend — a Claude + * subscription is not billed per token — and `unpriced_turns` counts the runs + * no price was known for (a local model), which makes the cost a lower bound. + * The `context_*` pair is the latest reading, not a sum. + */ +export interface TokenUsage { + input_tokens: number; + output_tokens: number; + cache_read_tokens: number; + cache_write_tokens: number; + cost_usd: number; + unpriced_turns: number; + context_used: number | null; + context_size: number | null; + total_tokens: number; } export interface ConversationTurn { diff --git a/frontend/src/lib/usage.test.ts b/frontend/src/lib/usage.test.ts new file mode 100644 index 000000000..fcdc03c0e --- /dev/null +++ b/frontend/src/lib/usage.test.ts @@ -0,0 +1,69 @@ +/** + * The dashboard's token fold must be the backend's (FEAT-120), or a live tab + * drifts from a reloaded one; and the cost figure must never claim more than + * is known. + */ + +import { describe, expect, it } from "vitest"; + +import type { TokenUsage } from "@/lib/api"; +import { addUsage, costDisplay, formatTokens } from "./usage"; + +const usage = (over: Partial = {}): TokenUsage => ({ + input_tokens: 0, + output_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + cost_usd: 0, + unpriced_turns: 0, + context_used: null, + context_size: null, + total_tokens: 0, + ...over, +}); + +describe("addUsage", () => { + it("sums the counters and keeps the latest context reading", () => { + const total = addUsage( + usage({ input_tokens: 1000, output_tokens: 50, cost_usd: 0.1, context_used: 1000, context_size: 200000 }), + { input_tokens: 200, output_tokens: 10, cost_usd: 0.05, context_used: 5000 }, + ); + expect(total?.input_tokens).toBe(1200); + expect(total?.output_tokens).toBe(60); + expect(total?.total_tokens).toBe(1260); + expect(total?.cost_usd).toBeCloseTo(0.15); + expect(total?.context_used).toBe(5000); + // A turn that did not report a window keeps the one already known. + expect(total?.context_size).toBe(200000); + }); + + it("reads a pre-FEAT-120 meta's `{}` as zero and nothing as nothing", () => { + expect(addUsage(undefined, {})?.total_tokens).toBe(0); + expect(addUsage(undefined, undefined)).toBeUndefined(); + expect(addUsage(usage({ input_tokens: 5 }), null)?.input_tokens).toBe(5); + }); +}); + +describe("costDisplay", () => { + it("hides the cost of a chat nothing in which could be priced", () => { + expect(costDisplay(usage({ unpriced_turns: 3 }))).toBeNull(); + }); + + it("marks a chat that moved from a priced model to an unpriced one as a lower bound", () => { + expect(costDisplay(usage({ cost_usd: 0.41, unpriced_turns: 1 }))).toEqual({ lowerBound: true }); + }); + + it("shows a fully priced chat plainly", () => { + expect(costDisplay(usage({ cost_usd: 0.41 }))).toEqual({ lowerBound: false }); + }); +}); + +describe("formatTokens", () => { + it("reads at a glance", () => { + expect(formatTokens(950)).toBe("950"); + expect(formatTokens(38200)).toBe("38.2k"); + expect(formatTokens(45000)).toBe("45k"); + expect(formatTokens(200000)).toBe("200k"); + expect(formatTokens(1_200_000)).toBe("1.2M"); + }); +}); diff --git a/frontend/src/lib/usage.ts b/frontend/src/lib/usage.ts new file mode 100644 index 000000000..0dd231c18 --- /dev/null +++ b/frontend/src/lib/usage.ts @@ -0,0 +1,71 @@ +/** + * A conversation's token total, folded the way the backend folds it (FEAT-120). + * + * The dashboard seeds a slot from the conversation's stored total and then adds + * each turn's `prompt_done.usage` to it, so these rules have to be the Python + * `TokenUsage.__add__` rules exactly or a live tab would drift from a reloaded + * one: counters sum, the context reading is the latest one that was reported. + */ + +import type { TokenUsage } from "@/lib/api"; + +const COUNTERS = [ + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", + "cost_usd", + "unpriced_turns", +] as const; + +/** Past this share of the window the context reading turns amber: the moment + * `/compact` becomes worth knowing about. */ +export const CONTEXT_WARN_RATIO = 0.8; + +const num = (v: unknown): number => (typeof v === "number" && Number.isFinite(v) ? v : 0); +const reading = (v: unknown): number | null => + typeof v === "number" && Number.isFinite(v) ? v : null; + +/** + * `total` plus one turn. Either side may be partial wire data — a meta written + * before FEAT-120 carries `{}` — and a missing side changes nothing. + */ +export function addUsage( + total: Partial | null | undefined, + turn: Partial | null | undefined, +): TokenUsage | undefined { + if (!total && !turn) return undefined; + const a = total ?? {}; + const b = turn ?? {}; + const next = {} as TokenUsage; + for (const key of COUNTERS) next[key] = num(a[key]) + num(b[key]); + next.context_used = reading(b.context_used) ?? reading(a.context_used); + next.context_size = reading(b.context_size) ?? reading(a.context_size); + next.total_tokens = next.input_tokens + next.output_tokens; + return next; +} + +/** + * How the cost figure shows, if at all. + * + * - `null` — a chat whose every priced figure is zero *because* nothing could be + * priced (a local model). "$0.00" there would be a claim, not a reading. + * - `lowerBound` — some turns were priced and some were not (a chat that moved + * from Claude to a local model), so the figure is at least this much. + */ +export function costDisplay(u: TokenUsage): { lowerBound: boolean } | null { + if (u.cost_usd <= 0 && u.unpriced_turns > 0) return null; + return { lowerBound: u.cost_usd > 0 && u.unpriced_turns > 0 }; +} + +/** `38.2k`, `1.2M` — a token count at a glance. */ +export function formatTokens(n: number): string { + if (n >= 1_000_000) return `${trim(n / 1_000_000)}M`; + if (n >= 1_000) return `${trim(n / 1_000)}k`; + return String(Math.round(n)); +} + +function trim(v: number): string { + // One decimal below 100, none above: `38.2k`, `200k`, never `200.0k`. + return v >= 100 ? String(Math.round(v)) : v.toFixed(1).replace(/\.0$/, ""); +} From 5045ab06a5359a3589a45190c19ca16b4ad021fe Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 11 Sep 2026 14:57:25 +0300 Subject: [PATCH 138/154] Stand the chat composer on the transcript's own ground: drop the hairline and the column-wide surface strip it sat on, raise the box to a surface card, and pad it like the scroll area so it lines up with the turns above --- .../src/components/chat/ApprovalPrompt.tsx | 2 +- frontend/src/components/chat/ChatBubble.tsx | 2 +- frontend/src/components/chat/ChatInput.tsx | 31 +++----- frontend/src/components/chat/ChatThread.tsx | 78 ++++++++++--------- frontend/src/index.css | 4 +- 5 files changed, 56 insertions(+), 61 deletions(-) diff --git a/frontend/src/components/chat/ApprovalPrompt.tsx b/frontend/src/components/chat/ApprovalPrompt.tsx index 5155a59f5..cc33c2273 100644 --- a/frontend/src/components/chat/ApprovalPrompt.tsx +++ b/frontend/src/components/chat/ApprovalPrompt.tsx @@ -63,7 +63,7 @@ export function ApprovalPrompt({ role="region" aria-label="Approval needed" aria-live="assertive" - className="mx-3 mb-2 overflow-hidden rounded-lg border-2 border-[var(--color-primary)] bg-[var(--color-surface)] shadow-lg" + className="mb-2 overflow-hidden rounded-lg border-2 border-[var(--color-primary)] bg-[var(--color-surface)] shadow-lg" >
{!expired && ( diff --git a/frontend/src/components/chat/ChatBubble.tsx b/frontend/src/components/chat/ChatBubble.tsx index b2479bffe..745493b7d 100644 --- a/frontend/src/components/chat/ChatBubble.tsx +++ b/frontend/src/components/chat/ChatBubble.tsx @@ -391,7 +391,7 @@ function BubbleHero({ {/* Its own draft namespace, not the workspace hero's: the bubble asks about the page you are standing on, and a question typed here is not the one waiting in the full chat. */} - +
diff --git a/frontend/src/components/chat/ChatInput.tsx b/frontend/src/components/chat/ChatInput.tsx index 71ddb9577..2adc2305d 100644 --- a/frontend/src/components/chat/ChatInput.tsx +++ b/frontend/src/components/chat/ChatInput.tsx @@ -62,13 +62,6 @@ interface ChatInputProps { * call sites that have no conversation to hang a draft on want. */ draftKey?: string; - /** - * Drop the deck — the hairline, the surface and its padding — and render only - * the box. For a composer that sits in an empty state rather than under a - * transcript: there the deck is not the floor of a column but a slab of a - * different colour laid across the middle of the hero. - */ - bare?: boolean; } type RecordingState = "idle" | "recording" | "transcribing"; @@ -82,7 +75,6 @@ export function ChatInput({ placeholder = "Ask Condor...", leading, draftKey, - bare, }: ChatInputProps) { const [value, setValue] = useState(() => readDraft(draftKey)); const textareaRef = useRef(null); @@ -377,17 +369,13 @@ export function ChatInput({ const isTranscribing = recordingState === "transcribing"; return ( - // The composer is a deck the transcript sits on, not a card floating over - // it: a hairline the width of the column, its own surface below it, and the - // field recessed into that surface. Only the ring is gold, and only on - // focus. A `bare` composer has no transcript to sit on, so no deck. -
+ // No deck of its own: the box stands on the same ground as the transcript + // above it, so the chat reads as one surface with a field at its foot. The + // deck it used to sit on was a hairline and a strip of `--color-surface` + // only as wide as the text column — a slab laid across the bottom of the + // pane rather than the floor of it. Only the ring is gold, and only on + // focus. +
{voiceError &&

{voiceError}

} {fileError &&

{fileError}

} {/* One composer chrome, owned here — the hero and the thread both get this @@ -415,7 +403,10 @@ export function ChatInput({ acceptFiles(e.dataTransfer?.files ?? null); }} data-testid="composer-box" - className={`@container flex flex-col gap-1.5 rounded-xl border bg-[var(--chat-inset)] px-2 py-1.5 transition-colors ${ + // Raised one step off the ground, not recessed into it: with no deck + // around it the box is the only chrome here, and a surface card is + // what says "type here" on the page ground in both themes. + className={`@container flex flex-col gap-1.5 rounded-xl border bg-[var(--color-surface)] px-2 py-1.5 shadow-sm transition-colors ${ focused || dragging ? "border-[var(--color-primary)]/40 ring-1 ring-[var(--color-primary)]/20" : "border-[var(--color-border)]" diff --git a/frontend/src/components/chat/ChatThread.tsx b/frontend/src/components/chat/ChatThread.tsx index a806c2c1f..055fe4a34 100644 --- a/frontend/src/components/chat/ChatThread.tsx +++ b/frontend/src/components/chat/ChatThread.tsx @@ -305,45 +305,49 @@ export function ChatThread({
- {/* Input */} + {/* Input — the same `px-4` as the transcript's scroll area, so the box + lines up with the turns above it at every width instead of running + to the pane's edges on a narrow one. */} {slot && ( -
- {/* On the composer, not above the transcript: the paused call sits - where the user is already looking and cannot scroll away. */} - {permissionRequest && ( - +
+ {/* On the composer, not above the transcript: the paused call sits + where the user is already looking and cannot scroll away. */} + {permissionRequest && ( + + )} + - )} - +
)} diff --git a/frontend/src/index.css b/frontend/src/index.css index 460fb97dc..8312922c2 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -47,7 +47,7 @@ /* Chat surfaces. `--chat-inset` is the recessed ground *inside* an answer: code fences, - table headers, the composer's field. It is defined per theme rather than + table headers. It is defined per theme rather than reused from `--color-bg` because the answer itself now sits on the page ground, and "one step off the page" points in opposite directions by theme — down in light, up in dark, where `--color-bg` is already all but @@ -58,7 +58,7 @@ the cards that also render `.chat-markdown`. `--chat-rule` is the hairline: the h2 underline, the fence border, the - composer's top edge, the gutter of a turn nobody is speaking in. + gutter of a turn nobody is speaking in. `--on-primary` is what may be written on `--color-primary`. White on the gold is 2.22:1 here and 3.26:1 in light — both under the 4.5:1 floor, From 5be862db6557aee41b53941088d911eb221f2681 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 11 Sep 2026 16:12:59 +0300 Subject: [PATCH 139/154] Stop the DEX pool pager from paging stale rows: keep the previous page up only while the same listing loads its next page, so a ticker in Search or a tab switch no longer leaves another listing's rows under a live Next (#241) --- frontend/src/pages/Dex.paging.test.tsx | 209 +++++++++++++++++++++++++ frontend/src/pages/Dex.tsx | 40 +++-- 2 files changed, 234 insertions(+), 15 deletions(-) create mode 100644 frontend/src/pages/Dex.paging.test.tsx diff --git a/frontend/src/pages/Dex.paging.test.tsx b/frontend/src/pages/Dex.paging.test.tsx new file mode 100644 index 000000000..c50a0aec6 --- /dev/null +++ b/frontend/src/pages/Dex.paging.test.tsx @@ -0,0 +1,209 @@ +/** + * That the pool browser's pager only ever pages the listing on screen (#241). + * + * The table keeps the previous page up while the next one loads, and that + * placeholder used to be carried anywhere — into another tab, and into a query + * that never runs at all: a ticker typed into Search is not an address, so the + * search is disabled, and Trending's rows (with Trending's `has_more`) stayed + * under a pager whose Next moved the counter while the rows never changed. + * + * Only the data edges are stubbed — `@/lib/api`, the upstream-budget hook and + * the LP positions strip above the table; the page's own keying is under test. + * + * @vitest-environment jsdom + */ + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { MemoryRouter } from "react-router-dom"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ServerContext } from "@/hooks/useServer"; +import type { PoolSummary } from "@/lib/api"; +import { Dex } from "./Dex"; + +const getDexPools = vi.fn(); + +vi.mock("@/lib/api", () => ({ + api: { + getDexChains: async () => [], + getDexVenues: async () => [], + getDexPools: (...args: unknown[]) => getDexPools(...args), + getDexPoolByAddress: async () => null, + getDexPoolsByAddress: async () => ({ pools: [] }), + }, +})); + +const upstream = { + limited: false, + retryIn: 0, + requestsLastMinute: 0, + budget: 30, + report: () => {}, +}; +vi.mock("@/hooks/useDexUpstream", () => ({ useDexUpstream: () => upstream })); +vi.mock("@/components/dex/LpPositions", () => ({ LpPositions: () => null })); +vi.mock("@/components/dex/UpstreamNotice", () => ({ UpstreamNotice: () => null })); + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +function pool(base: string, quote: string, dex: string): PoolSummary { + return { + address: `${base}${quote}${dex}`, + name: `${base} / ${quote}`, + source: "gecko", + dex_id: dex, + network: "solana-mainnet-beta", + gecko_network: "solana", + gateway_network: "solana-mainnet-beta", + base_symbol: base, + quote_symbol: quote, + base_token_symbol: base, + quote_token_symbol: quote, + base_token_address: `${base}mint`, + quote_token_address: `${quote}mint`, + trading_pair: `${base}mint-${quote}`, + lp_provider: null, + lp_supported: false, + tradable: true, + has_bins: false, + reserve_usd: 1_000_000, + volume_24h: 500_000, + price_change_24h: 1, + }; +} + +const TRENDING = { + pools: [pool("SOL", "USDC", "orca"), pool("BONK", "SOL", "raydium")], + has_more: true, +}; + +/** A reply the test answers when it chooses to, to observe the page in flight. */ +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((r) => (resolve = r)); + return { promise, resolve }; +} + +let container: HTMLDivElement; +let root: Root; + +async function settle(ms = 0) { + await act(() => new Promise((r) => setTimeout(r, ms))); + await act(() => new Promise((r) => setTimeout(r, 0))); +} + +async function render() { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + await act(async () => { + root.render( + + {} }}> + + + + + , + ); + }); + await settle(); +} + +function button(label: string): HTMLButtonElement | undefined { + return [...container.querySelectorAll("button")].find( + (b) => b.textContent?.trim() === label, + ); +} + +async function click(label: string) { + const target = button(label); + expect(target, `no "${label}" button`).toBeDefined(); + await act(async () => target!.click()); + await settle(); +} + +async function type(value: string) { + const input = container.querySelector("input[placeholder^='Paste']") as HTMLInputElement; + const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")!.set!; + await act(async () => { + setter.call(input, value); + input.dispatchEvent(new Event("input", { bubbles: true })); + }); +} + +const text = () => container.textContent ?? ""; + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + localStorage.clear(); + getDexPools.mockReset(); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(async () => { + await act(() => root.unmount()); + container.remove(); +}); + +describe("Dex pool pager", () => { + it("does not leave Trending's rows under Search when the query is a ticker", async () => { + getDexPools.mockResolvedValue(TRENDING); + await render(); + expect(text()).toContain("BONK-SOL"); + expect(button("Next")?.disabled).toBe(false); + + await click("Search"); + expect(text()).not.toContain("BONK-SOL"); + expect(text()).toContain("Paste a pool or token address to find it."); + + await type("SOL-USDC"); + await settle(450); // past the search debounce + expect(text()).not.toContain("BONK-SOL"); + expect(text()).toContain("That is not a pool or token address."); + expect(button("Next")).toBeUndefined(); + // A ticker never reaches the backend: the only request was Trending's. + expect(getDexPools).toHaveBeenCalledTimes(1); + }); + + it("does not carry one tab's rows into another while it loads", async () => { + const top = deferred(); + getDexPools.mockImplementation((_server: string, params: { view: string }) => + params.view === "top" ? top.promise : Promise.resolve(TRENDING), + ); + await render(); + + await click("Top"); + expect(text()).not.toContain("BONK-SOL"); + expect(text()).toContain("Loading pools…"); + + top.resolve({ pools: [pool("JUP", "USDC", "meteora")], has_more: false }); + await settle(); + expect(text()).toContain("JUP-USDC"); + }); + + it("keeps the current page up while Next loads the one after it", async () => { + const second = deferred(); + getDexPools.mockImplementation((_server: string, params: { page: number }) => + params.page === 2 ? second.promise : Promise.resolve(TRENDING), + ); + await render(); + + await click("Next"); + expect(getDexPools).toHaveBeenLastCalledWith( + "srv", + expect.objectContaining({ view: "trending", page: 2 }), + ); + expect(text()).toContain("Page 2"); + expect(text()).toContain("BONK-SOL"); + + second.resolve({ pools: [pool("JUP", "USDC", "meteora")], has_more: false }); + await settle(); + expect(text()).toContain("JUP-USDC"); + expect(text()).not.toContain("BONK-SOL"); + }); +}); diff --git a/frontend/src/pages/Dex.tsx b/frontend/src/pages/Dex.tsx index 179c62275..809e62b27 100644 --- a/frontend/src/pages/Dex.tsx +++ b/frontend/src/pages/Dex.tsx @@ -140,25 +140,28 @@ export function Dex() { source.kind !== "favorites" && (source.kind === "gateway" || !isSearch || isAddress); + // Everything that picks *which* listing is showing; the page within it is + // appended below. + const listingKey = [ + "dex-pools", + server, + source.kind, + source.kind === "gecko" + ? source.view + : source.kind === "gateway" + ? source.connector + : "favorites", + isGateway ? "" : network, + effectiveQuery, + isGateway ? "" : dexes.join(","), + ]; + const { data: pagedPools, isFetching, dataUpdatedAt: poolsUpdatedAt, } = useQuery({ - queryKey: [ - "dex-pools", - server, - source.kind, - source.kind === "gecko" - ? source.view - : source.kind === "gateway" - ? source.connector - : "favorites", - isGateway ? "" : network, - effectiveQuery, - isGateway ? "" : dexes.join(","), - page, - ], + queryKey: [...listingKey, page], queryFn: () => api.getDexPools( server!, @@ -185,7 +188,14 @@ export function Dex() { ), enabled, staleTime: POOL_STALE_MS, - placeholderData: (prev) => prev, + // The previous page stays up while the next one loads — but only a page of + // this same listing. Carried across tabs, or into a query that will never + // run (a ticker typed into Search), it sat under a pager whose Next still + // worked while the rows never changed. + placeholderData: (prev, prevQuery) => + enabled && listingKey.every((part, i) => prevQuery?.queryKey[i] === part) + ? prev + : undefined, }); // The pasted address may be a *pool*, not a token. Both are 44 base58 From 040bb738225678a71acaaac562240b37be6cebd7 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 11 Sep 2026 16:23:49 +0300 Subject: [PATCH 140/154] Strip whitespace from Gateway network config values before saving them from the dashboard: a nodeURL saved as " https://..." passed Gateway's schema and then broke every Solana call, surfacing as /wallet/add failing with "Endpoint URL must start with http: or https:" (hummingbot/hummingbot-api#233, reported by @rapcmia) --- condor/web/models.py | 9 +- tests/test_gateway_network_config_strip.py | 100 +++++++++++++++++++++ 2 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 tests/test_gateway_network_config_strip.py diff --git a/condor/web/models.py b/condor/web/models.py index 8c96986a4..0bad86786 100644 --- a/condor/web/models.py +++ b/condor/web/models.py @@ -2,7 +2,7 @@ from typing import Any, Optional -from pydantic import BaseModel +from pydantic import BaseModel, field_validator from condor.fetchers.executors import build_executor_row, get_executor_type from condor.fetchers.models import ( @@ -742,6 +742,13 @@ class GatewayNetworkUpdateRequest(BaseModel): # The Gateway validates values against its own JSON schema. config: dict[str, Any] + @field_validator("config") + @classmethod + def _strip_string_values(cls, v: dict[str, Any]) -> dict[str, Any]: + # Gateway stores strings verbatim, so a pasted " https://..." nodeURL + # saves fine and then breaks every Solana call (hummingbot-api#233). + return {k: val.strip() if isinstance(val, str) else val for k, val in v.items()} + class GatewayWalletAddRequest(BaseModel): chain: str diff --git a/tests/test_gateway_network_config_strip.py b/tests/test_gateway_network_config_strip.py new file mode 100644 index 000000000..4f06eaee9 --- /dev/null +++ b/tests/test_gateway_network_config_strip.py @@ -0,0 +1,100 @@ +"""Network config saves strip surrounding whitespace (hummingbot-api#233). + +Gateway stores a network config's string values verbatim. A nodeURL saved as +" https://..." passes Gateway's schema, then the Solana client rejects it on +the next connection — surfacing as "Endpoint URL must start with http: or +https:" from /wallet/add, far from the save that caused it. The Telegram +flows already strip their input; the dashboard route must too. +""" + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +import condor.web.routes.settings as settings_routes +from condor.web.auth import get_current_user +from condor.web.models import WebUser +from config_manager import ServerPermission + +SERVER = "alpha" +OWNER = WebUser(id=1, username="owner", first_name="O", role="user") +NETWORK_ID = "solana-mainnet-beta" + + +class FakeGateway: + def __init__(self): + self.calls = [] + + async def update_network_config(self, network_id, config): + self.calls.append((network_id, config)) + return {"ok": True} + + +class FakeClient: + def __init__(self): + self.gateway = FakeGateway() + + +class FakeConfigManager: + def __init__(self, client): + self._client = client + + def get_server_permission(self, user_id, server_name): + return ServerPermission.OWNER if server_name == SERVER else None + + def has_server_access(self, user_id, server_name, min_permission=None): + return server_name == SERVER + + def is_admin(self, user_id): + return False + + async def get_client(self, server_name): + return self._client + + +@pytest.fixture +def env(monkeypatch): + client = FakeClient() + cm = FakeConfigManager(client) + monkeypatch.setattr(settings_routes, "get_config_manager", lambda: cm) + monkeypatch.setattr("condor.web.auth.get_config_manager", lambda: cm) + app = FastAPI() + app.include_router(settings_routes.router) + app.dependency_overrides[get_current_user] = lambda: OWNER + return TestClient(app), client + + +def test_string_values_reach_gateway_stripped(env): + http, client = env + resp = http.post( + f"/settings/gateway/networks/{NETWORK_ID}", + params={"server": SERVER}, + json={ + "config": { + "nodeURL": " https://api.mainnet-beta.solana.com\n", + "nativeCurrencySymbol": "SOL ", + } + }, + ) + assert resp.status_code == 200 + assert client.gateway.calls == [ + ( + NETWORK_ID, + { + "nodeURL": "https://api.mainnet-beta.solana.com", + "nativeCurrencySymbol": "SOL", + }, + ) + ] + + +def test_non_string_values_pass_through_untouched(env): + http, client = env + config = {"chainID": 101, "useHeliusRestRPC": False, "defaultNetworks": ["a"]} + resp = http.post( + f"/settings/gateway/networks/{NETWORK_ID}", + params={"server": SERVER}, + json={"config": config}, + ) + assert resp.status_code == 200 + assert client.gateway.calls == [(NETWORK_ID, config)] From ede31c2bf8115422bdb9954a95a887fa61732bca Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 11 Sep 2026 18:28:44 +0300 Subject: [PATCH 141/154] =?UTF-8?q?Tell=20agents=20which=20DEXs=20go=20thr?= =?UTF-8?q?ough=20Gateway:=20only=20AMM/CLMM/DLMM=20pools=20and=20swap=20r?= =?UTF-8?q?outers=20(meteora,=20raydium,=20orca,=20jupiter,=20uniswap,=20p?= =?UTF-8?q?ancakeswap)=20do;=20CLOB=20DEXs=20(hyperliquid,=20xrpl,=20dydx,?= =?UTF-8?q?=20injective,=20derive,=20dexalot)=20are=20native=20Hummingbot?= =?UTF-8?q?=20connectors=20used=20like=20a=20CEX,=20and=20some=20serve=20c?= =?UTF-8?q?andles=20=E2=80=94=20the=20skills,=20AGENT.md=20files=20and=20g?= =?UTF-8?q?et=5Fmarket=5Fdata=20docstring=20had=20lumped=20every=20DEX=20t?= =?UTF-8?q?ogether?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agents/_defaults/shutdown.md | 4 +- .../skills/market_data_with_code/SKILL.md | 6 ++- .../skills/verify_connector_support/SKILL.md | 43 +++++++++++++------ agents/condor/AGENT.md | 4 +- agents/condor/skills/log_analyzer/SKILL.md | 2 +- agents/xrpl_market_maker/AGENT.md | 4 ++ .../skills/xrpl_mm_deploy/SKILL.md | 6 ++- mcp_servers/hummingbot_api/server.py | 3 +- 8 files changed, 51 insertions(+), 21 deletions(-) diff --git a/agents/_defaults/shutdown.md b/agents/_defaults/shutdown.md index 273028899..9a51e1fb6 100644 --- a/agents/_defaults/shutdown.md +++ b/agents/_defaults/shutdown.md @@ -10,7 +10,9 @@ any orphan positions the policy says to close. You are the best-effort cleanup p running on top of that guaranteed floor. Now: - If any position is spot dust worth less than ~$5, leave it — not worth the fees. -- Cancel any stray Gateway / LP / resting orders you can find for this session. +- Cancel any stray resting orders (CEX or CLOB DEX — `hyperliquid`, `xrpl` and the like + are plain Hummingbot connectors, not Gateway) and close any leftover Gateway LP + position (AMM/CLMM pools) you can find for this session. - If any position that should be closed is still open, close it (stop its executor with keep_position off, or place a reduce-only order). - Notify the owner via `send_notification` with the final realized PnL and a short diff --git a/agents/_shared/skills/market_data_with_code/SKILL.md b/agents/_shared/skills/market_data_with_code/SKILL.md index f961fbcba..b08458fbb 100644 --- a/agents/_shared/skills/market_data_with_code/SKILL.md +++ b/agents/_shared/skills/market_data_with_code/SKILL.md @@ -52,7 +52,8 @@ If you are reading this because a market data request just came in, do this chec | Indicators (RSI / EMA / ATR / VWAP) | `run_code` → `get_candles_last_days` + pandas_ta | varies | | All tickers for a connector | `run_code` → `get_tickers` | 300 | | Multi-venue anything with math | `run_code` + `asyncio.gather` | ~500 | -| DEX candles | GeckoTerminal — DEX connectors don't serve OHLCV | varies | +| AMM/CLMM DEX candles (Gateway: meteora, raydium, orca, uniswap…) | GeckoTerminal — Gateway connectors don't serve OHLCV | varies | +| CLOB DEX candles (`hyperliquid_perpetual`, …) | `get_candles*` like any CEX — CLOB DEXs are Hummingbot connectors, not Gateway; check the candle list (`xrpl` has no feed) | varies | --- @@ -164,7 +165,8 @@ vol = await client.market_data.get_quote_volume_for_price("binance_perpetual", **Utilities** ```python -# Check which connectors serve OHLCV before using candles on a DEX connector +# Check which connectors serve OHLCV before relying on one — Gateway AMM/CLMM +# connectors never do; CLOB DEXs vary (hyperliquid_perpetual yes, xrpl no) candle_connectors = await client.market_data.get_available_candle_connectors() # → ["binance_perpetual", "binance", "okx_perpetual", ...] ``` diff --git a/agents/_shared/skills/verify_connector_support/SKILL.md b/agents/_shared/skills/verify_connector_support/SKILL.md index 3310ab0e3..3cad8ee29 100644 --- a/agents/_shared/skills/verify_connector_support/SKILL.md +++ b/agents/_shared/skills/verify_connector_support/SKILL.md @@ -3,12 +3,13 @@ name: verify_connector_support description: Check what a connector actually supports before using it; when candles are unavailable, source history from a proxy or GeckoTerminal instead when_to_use: 'User asks "can I use connector X?" or "does Y support Z?" — any capability - question about a connector or DEX. Also: any time OHLCV / candles / price history - is needed and the connector is not on the candle list — typically xrpl, meteora, - raydium, orca, uniswap, jupiter and other DEX connectors. Triggers — "Connector - ''X'' does not support candle data", setting up or backtesting an agent on a DEX - venue, "get me candles for on ", computing EMA/RSI/ATR on a non-candle - venue; ES — "no hay velas para ", "sin datos históricos en ".' + question about a connector or DEX, including whether a DEX goes through Gateway or + through Hummingbot directly. Also: any time OHLCV / candles / price history is needed + and the connector is not on the candle list — typically xrpl and the Gateway AMM/CLMM + connectors (meteora, raydium, orca, uniswap, jupiter…). Triggers — "Connector ''X'' + does not support candle data", setting up or backtesting an agent on a DEX venue, + "get me candles for on ", computing EMA/RSI/ATR on a non-candle venue; + ES — "no hay velas para ", "sin datos históricos en ".' created: '2026-08-12T11:51:59Z' source: chat --- @@ -17,12 +18,28 @@ source: chat Never guess connector capabilities from memory. Always pull the authoritative source first. +### First: which kind of DEX is it? + +"DEX" covers two different stacks, and only one of them goes through Gateway: + +| Kind | Examples | Runs through | Tools | +|---|---|---|---| +| **AMM / CLMM / DLMM pools, swap routers** | `meteora`, `raydium`, `orca`, `jupiter`, `uniswap`, `pancakeswap` | **Gateway** — `connector_name` is the network (`solana-mainnet-beta`); the DEX rides in `lp_provider` / `swap_provider` | `explore_dex_pools`, `manage_amm`, `manage_clmm`, `quote_swap` / `execute_swap`, `create_lp_executor`, `manage_gateway_config` | +| **CLOB (order-book) DEXs** | `hyperliquid`, `hyperliquid_perpetual`, `xrpl`, `dydx_v4_perpetual`, `injective_v2`, `injective_v2_perpetual`, `derive`, `derive_perpetual`, `dexalot` | **Hummingbot directly**, through the Hummingbot API — exactly like a CEX | `get_prices`, `get_portfolio_overview`, order book / trading rules via `run_code`, every executor, controllers and bots | + +A CLOB DEX is a plain Hummingbot connector: its credentials go in Settings → Keys (not a +Gateway wallet), it has no pools, and Gateway knows nothing about it. Never reach for +`manage_gateway_config`, `explore_dex_pools`, `manage_amm` / `manage_clmm` or +`quote_swap` / `execute_swap` for one. Rule of thumb: if you place limit orders on an +order book, it is a Hummingbot connector. + ### Capability lookup steps -1. **LP / CLMM questions** → read the `create_lp_executor` tool description — its `lp_provider` parameter lists the supported DEXs -2. **AMM swap / pool-creation questions** → `manage_amm()` (no action) — read the connector list -3. **Pool discovery questions** → `explore_dex_pools` tool description lists supported connectors -4. **Market data / candle questions** → see the **Candles** section below +1. **CLOB DEX questions** (`hyperliquid`, `xrpl`, …) → treat it as a CEX: balances via `get_portfolio_overview`, book and trading rules via `client.market_data.*` / `client.connectors.*` in `run_code`, deploy with executors or controllers +2. **LP / CLMM questions** (Gateway) → read the `create_lp_executor` tool description — its `lp_provider` parameter lists the supported DEXs +3. **AMM swap / pool-creation questions** (Gateway) → `manage_amm()` (no action) — read the connector list +4. **Pool discovery questions** (Gateway) → `explore_dex_pools` tool description lists supported connectors +5. **Market data / candle questions** → see the **Candles** section below Answer from what the guide actually says — not from what you remember. @@ -39,7 +56,7 @@ Available connectors: ['binance', 'binance_perpetual', 'kucoin', 'kraken', ...] This is a **hard capability gap, not a transient error**. Retrying, changing the interval, changing `days`, or reformatting the pair will never make it succeed. The failing loop this skill exists to stop: agent setup asks for candles on a DEX → error → retries → error → user has to interrupt by hand. -**Who has no candle feed:** `xrpl` and every AMM/CLMM DEX connector (`meteora`, `raydium`, `orca`, `uniswap`, `pancakeswap`, `jupiter`, …), plus any CEX not in the list the error prints. The list is the authority — never assume from the name. +**Who has no candle feed:** every Gateway connector (AMM/CLMM/DLMM: `meteora`, `raydium`, `orca`, `uniswap`, `pancakeswap`, `jupiter`, …), plus any CEX or CLOB DEX not in the list the error prints — `xrpl` is one. Being a DEX is not the test: CLOB DEXs such as `hyperliquid` / `hyperliquid_perpetual` are Hummingbot connectors and do serve candles. The list is the authority — never assume from the name. ### What still works on that connector @@ -47,8 +64,8 @@ Losing candles does **not** mean losing the venue. These remain live and correct - `get_prices(trading_pairs=[...])` — current price - `client.market_data.get_order_book(...)` — depth, and the `price_for_volume` / - `volume_for_price` slippage queries beside it -- `explore_dex_pools` — pool discovery, TVL, fees, APR (CLMM connectors) + `volume_for_price` slippage queries beside it (CLOB venues, `xrpl` included) +- `explore_dex_pools` — pool discovery, TVL, fees, APR (Gateway CLMM connectors only) - Trading itself: quoting, swaps, LP and executor deployment **Execute on the venue the user asked for, source the *history* elsewhere.** diff --git a/agents/condor/AGENT.md b/agents/condor/AGENT.md index 0d0d0c552..d4dd6b5fc 100644 --- a/agents/condor/AGENT.md +++ b/agents/condor/AGENT.md @@ -18,13 +18,15 @@ You are Condor, a trading assistant. Do NOT explore the codebase — use MCP too - `list_executors` / `get_executor` / `stop_executor` — monitor and stop running executors - `manage_bots` — start/stop/monitor bots - `manage_controllers` — controller configs -- `explore_dex_pools` / `explore_geckoterminal` — DEX discovery +- `explore_dex_pools` / `explore_geckoterminal` — pool discovery (Gateway CLMM pools / GeckoTerminal) - `manage_amm` — direct AMM liquidity & pool creation (Meteora DAMM v2 / Raydium CPMM / Uniswap V2) - `search_history` — historical trades and executor data - `set_account_position_mode_and_leverage` — futures config _Connecting/removing exchange API keys is not available to the assistant — keys are managed by the user in the Condor web dashboard (Settings → Keys)._ +_Two kinds of DEX, two stacks._ AMM/CLMM/DLMM pools and swap routers (`meteora`, `raydium`, `orca`, `jupiter`, `uniswap`, `pancakeswap`) run through **Gateway** — `explore_dex_pools`, `manage_amm`, `manage_clmm`, `create_lp_executor`, swaps on a network connector like `solana-mainnet-beta`. CLOB DEXs (`hyperliquid`, `hyperliquid_perpetual`, `xrpl`, `dydx_v4_perpetual`, `injective_v2`, `derive_perpetual`, `dexalot`, …) are **native Hummingbot connectors**, used exactly like a CEX — keys in Settings → Keys, trade with executors/controllers — and never touch Gateway. + **condor** — UI & utilities: - `send_notification` — send Telegram messages to the user - `manage_routines` — run/list analysis scripts diff --git a/agents/condor/skills/log_analyzer/SKILL.md b/agents/condor/skills/log_analyzer/SKILL.md index d0f0852b3..bc1a426e4 100644 --- a/agents/condor/skills/log_analyzer/SKILL.md +++ b/agents/condor/skills/log_analyzer/SKILL.md @@ -27,7 +27,7 @@ All via `client = await get_client(context._chat_id, context=context)`: | All active bots | `client.bot_orchestration.get_active_bots_status()` | `data{bot: {error_logs, general_logs, status, performance, ...}}` | | One bot | `client.bot_orchestration.get_bot_status(bot_name)` | same shape for a single bot (perf + logs + activity) | | One executor | `client.executors.get_executor_logs(executor_id, limit=100, level="ERROR")` | per-executor log entries | -| Gateway (DEX) | `client.gateway.get_logs(tail=100)` | gateway process logs | +| Gateway (AMM/CLMM DEXs: meteora, raydium, orca, jupiter, uniswap…) | `client.gateway.get_logs(tail=100)` | gateway process logs — never the place for CLOB DEXs (`hyperliquid`, `xrpl`, …): those are Hummingbot connectors, so their errors land in the bot / executor logs above | **Log entry shape** (each item in `error_logs` / `general_logs`): ```python diff --git a/agents/xrpl_market_maker/AGENT.md b/agents/xrpl_market_maker/AGENT.md index 116ef2916..ff0c1cefe 100644 --- a/agents/xrpl_market_maker/AGENT.md +++ b/agents/xrpl_market_maker/AGENT.md @@ -35,6 +35,10 @@ created_at: '2026-07-28T00:00:00Z' You make markets on the **XRPL on-ledger CLOB**. Undercut the AMM pool fee to win pathfinding flow; price off a CEX reference, never the ledger mid alone. +`xrpl` is a **native Hummingbot connector** — it trades through the Hummingbot API like a +CEX, **not through Gateway**. No `manage_gateway_config`, `explore_dex_pools` or +`quote_swap` / `execute_swap`; its credentials live in Settings → Keys. + ## Hard rules 1. **Fair value = CEX reference** (Bitget XRP-USDT for RLUSD/XRP). Never derive it from diff --git a/agents/xrpl_market_maker/skills/xrpl_mm_deploy/SKILL.md b/agents/xrpl_market_maker/skills/xrpl_mm_deploy/SKILL.md index acd169820..036a09c51 100644 --- a/agents/xrpl_market_maker/skills/xrpl_mm_deploy/SKILL.md +++ b/agents/xrpl_market_maker/skills/xrpl_mm_deploy/SKILL.md @@ -18,8 +18,10 @@ real controller failure (schema reject, deploy/status error, or no on-ledger ord 1. **Pair quality** — `explore_geckoterminal(action="top_pools", network="xrpl")`. Need real depth *and* turnover. RLUSD/XRP is currently the only pair with both. 2. **Issuer transfer fee = 0%** — non-zero fees can erase the whole spread. -3. **XRPL credentials configured** — without them the keyless connector has empty trading - rules and nothing can size. Stop and tell the user; executor mode will not rescue this. +3. **XRPL credentials configured** — as a Hummingbot connector key (Settings → Keys), not a + Gateway wallet: `xrpl` never goes through Gateway. Without them the keyless connector has + empty trading rules and nothing can size. Stop and tell the user; executor mode will not + rescue this. 4. **Balances** — `get_portfolio_overview()`. Need free XRP for reserves (1 + 0.2×offers), a trustline for the issued asset, and inventory on both sides. diff --git a/mcp_servers/hummingbot_api/server.py b/mcp_servers/hummingbot_api/server.py index ab99ab346..9823613d6 100644 --- a/mcp_servers/hummingbot_api/server.py +++ b/mcp_servers/hummingbot_api/server.py @@ -454,7 +454,8 @@ async def get_market_data( - "historical_candles": a unix time range (needs start_time; end_time optional) - "connectors": which connectors serve OHLCV at all — check before asking a - DEX connector for candles, because most do not serve them + DEX for candles: Gateway AMM/CLMM connectors never serve them, and CLOB + DEXs vary (`hyperliquid_perpetual` does, `xrpl` does not) Args: action: What to read. From 9b2b296cca61691b26a3736939e5b826e9776230 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 11 Sep 2026 18:31:11 +0300 Subject: [PATCH 142/154] Refuse a loop start whose model cannot run, and stop a failing tick from failing silently: the start route checks the resolved agent key up front (unknown provider, missing model id, unsaved custom endpoint, missing API key) and answers 422 instead of "started"; a dry run or run-once whose tick raises is recorded as a failed run in the Runs rail and self-stops as ERROR instead of completed; a loop's repeated tick error or risk block notifies the owner once per distinct reason instead of every tick; and _notify reaches dashboard launches through the owner's private chat instead of only a live bot neither caller passed --- condor/agents/engine.py | 114 ++++++++-- condor/runtime/llm_client.py | 56 +++++ condor/web/routes/agents.py | 17 ++ tests/test_agent_start_model_check.py | 299 ++++++++++++++++++++++++++ 4 files changed, 468 insertions(+), 18 deletions(-) create mode 100644 tests/test_agent_start_model_check.py diff --git a/condor/agents/engine.py b/condor/agents/engine.py index fa0173195..2b0c81677 100644 --- a/condor/agents/engine.py +++ b/condor/agents/engine.py @@ -75,6 +75,15 @@ def _supervisor(): return get_supervisor() +def resolve_agent_key(config: dict[str, Any], strategy, agent) -> str: + """The model a run uses: run config override > strategy override > Agent. + + Module-level so the start route can check the very key the engine will run + before building one — building an engine allocates a session on disk. + """ + return config.get("agent_key") or strategy.agent_key or agent.agent_key + + def get_engine(agent_id: str) -> TickEngine | None: return _supervisor().get(agent_id) @@ -114,6 +123,9 @@ class TickEngine: _shutting_down: bool = field(default=False, init=False) _last_tick_at: float = field(default=0.0, init=False) _last_error: str = field(default="", init=False) + # The block the owner was last told about, so a block lasting many ticks is + # announced once instead of on every one of them. + _last_block_reason: str = field(default="", init=False, repr=False) _last_skill_data: dict[str, Any] = field(default_factory=dict, init=False) _adoption_done: bool = field(default=False, init=False, repr=False) _mode_mismatch_noted: bool = field(default=False, init=False, repr=False) @@ -386,17 +398,28 @@ async def _loop(self) -> None: mode = self.config.get("execution_mode", "loop") while self._running: if not self._paused: + tick_error = "" try: await self._tick() self._last_error = "" except asyncio.CancelledError: raise except Exception as e: - self._last_error = str(e) + tick_error = str(e) or type(e).__name__ + # A loop with a broken model or server fails every tick the + # same way: tell the owner when the error starts or changes, + # not once a tick. A single-tick run reports below instead. + repeated = tick_error == self._last_error + self._last_error = tick_error log.exception("TickEngine %s tick error", self.agent_id) if self.journal: - self.journal.append_error(str(e)) - await self._notify(f"Agent {self.agent_id} tick error: {e}") + self.journal.append_error(tick_error) + if mode in ("dry_run", "run_once"): + self._record_failed_experiment(tick_error) + elif not repeated: + await self._notify( + f"Agent {self.agent_id} tick error: {tick_error}" + ) # A shutdown that started *inside* the tick (the hard risk # kill-switch) already ran its winddown, wrote the terminal @@ -416,6 +439,20 @@ async def _loop(self) -> None: # Single-tick modes: stop after first tick if mode in ("dry_run", "run_once"): label = "Dry run" if mode == "dry_run" else "Run-once" + if tick_error: + # A tick that raised is a failed run, not a completed one. + log.info( + "TickEngine %s: %s failed, self-stopping", + self.agent_id, + label, + ) + await self._notify( + f"Agent {self.agent_id}: {label} failed: {tick_error}" + ) + self._last_stop_reason = "error" + self._running = False + _supervisor().unregister(self.agent_id, LoopState.ERROR) + return log.info( "TickEngine %s: %s complete, self-stopping", self.agent_id, @@ -531,10 +568,13 @@ async def _tick(self) -> None: risk_state.block_reason, ) self.journal.record_tick("blocked: " + risk_state.block_reason) - await self._notify( - f"Agent {self.agent_id} blocked: {risk_state.block_reason}" - ) + if risk_state.block_reason != self._last_block_reason: + await self._notify( + f"Agent {self.agent_id} blocked: {risk_state.block_reason}" + ) + self._last_block_reason = risk_state.block_reason return + self._last_block_reason = "" # 5. Build prompt (server credentials are injected via env into MCP process) # Routine discovery is read fresh each tick, like the skills index right @@ -1084,12 +1124,8 @@ def _executor_owners(self) -> dict[str, str]: return owners def _agent_key(self) -> str: - """Resolve the model for this run: config override > strategy override > Agent.""" - return ( - self.config.get("agent_key") - or self.strategy.agent_key - or self.agent.agent_key - ) + """Resolve the model for this run (see :func:`resolve_agent_key`).""" + return resolve_agent_key(self.config, self.strategy, self.agent) def _resolve_server(self) -> tuple[str | None, dict | None]: """Resolve the server for this run, keyed on ``user_id`` (SEC-164). @@ -1165,12 +1201,54 @@ async def _get_client(self): return None async def _notify(self, message: str) -> None: - """Send a notification to the user via Telegram.""" - if hasattr(self, "_bot") and self._bot: - try: - await self._bot.send_message(chat_id=self.chat_id, text=message) - except Exception: - log.exception("Failed to send notification to chat %s", self.chat_id) + """Tell the run's owner, down the same ladder a delegation's notice takes. + + Only a live bot handed to ``start()`` used to count, and neither caller + (the start route, the boot restart) has one to hand, so every notice + here went nowhere. A dashboard launch carries no chat (``chat_id`` 0); + its owner's private chat is the one their user id names. + """ + chat_id = self.chat_id or self.user_id + if not chat_id: + return + from .delegate import resolve_bot + + try: + bot = resolve_bot(getattr(self, "_bot", None)) + await bot.send_message(chat_id=chat_id, text=message) + except Exception: + log.exception("Failed to send notification to chat %s", chat_id) + + def _record_failed_experiment(self, error: str) -> None: + """Write the dry run's file for a tick that raised before writing it. + + An experiment keeps no journal and its engine is dropped the moment it + stops, so a failed dry run used to leave nothing behind at all. The + error goes where a failed model call already puts it — the Agent + Response — which is what marks the run as failed in the Runs rail. + """ + from datetime import datetime, timezone + + from .journal import save_experiment_snapshot + + try: + save_experiment_snapshot( + agent_dir=self.strategy.home, + experiment_num=self.session_num, + execution_mode=self.config.get("execution_mode", "loop"), + timestamp=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC"), + system_prompt="", + response_text=f"(error: {error})", + tool_calls=[], + executors_data="The tick failed before collecting any.", + risk_state={}, + duration=time.time() - self._last_tick_at, + agent_key=self._agent_key(), + ) + except Exception: + log.exception( + "TickEngine %s: could not record the failed run", self.agent_id + ) def get_info(self) -> dict[str, Any]: """Return a summary dict for display.""" diff --git a/condor/runtime/llm_client.py b/condor/runtime/llm_client.py index af6ff594f..b8a661afb 100644 --- a/condor/runtime/llm_client.py +++ b/condor/runtime/llm_client.py @@ -96,3 +96,59 @@ def build_llm_client( model=model_pref or None, system_prompt=system_prompt, ) + + +async def agent_key_error( + agent_key: str, + *, + user_id: int | None = None, + base_url_override: str | None = None, +) -> str | None: + """Why ``agent_key`` cannot run here, or ``None`` when nothing says so yet. + + For a caller about to hand the key to something unattended — the loop + engine above all — that would rather refuse up front than start a run which + fails where nobody is watching. Only what is certain before a request goes + out counts: a provider no client knows, a key with no model id, a custom key + naming an endpoint this user never saved, a provider with no API key or base + URL. Those are exactly what a tick raises from its first ``start()``, and + they are raised here by the same code, so the two cannot disagree. A local + server that is down, or a model the provider does not serve, still fails at + run time: telling those apart takes a network call, and a server that is off + now may well be on by the first tick. + """ + key = (agent_key or "").strip() + if not key: + return None # no key = the ACP default, always addressable + if not pydantic_ai.is_pydantic_ai_model(key): + base = key.split(":", 1)[0] + if base.split("@", 1)[0] in pydantic_ai.PYDANTIC_AI_PREFIXES: + return f"no model id — use '{base}:'" + if base not in acp_client.ACP_COMMANDS: + # resolve_acp would quietly run Claude Code in its place. + known = sorted(acp_client.ACP_COMMANDS) + sorted( + pydantic_ai.PYDANTIC_AI_PREFIXES + ) + return f"unknown model provider '{base}' (known: {', '.join(known)})" + return None + + from condor.llm.readiness import LOCAL_PREFIXES + + try: + client = build_llm_client( + key, + user_id=user_id, + base_url_override=base_url_override, + # An explicit base URL is the endpoint, so the saved name is moot. + strict_custom_endpoint=not base_url_override, + ) + # A bare local key ("ollama:") asks the server which model to use. + if ( + pydantic_ai.model_prefix(key) in LOCAL_PREFIXES + and not key.partition(":")[2] + ): + return None + await client._build_model() + except Exception as e: + return str(e) or type(e).__name__ + return None diff --git a/condor/web/routes/agents.py b/condor/web/routes/agents.py index 06f5cb925..4543111b1 100644 --- a/condor/web/routes/agents.py +++ b/condor/web/routes/agents.py @@ -2846,6 +2846,23 @@ async def _start(agent, strategy, req: StartStrategyRequest, user_id: int) -> di if not cm.get_server(server_name): raise HTTPException(status_code=404, detail="Server not found") + # The model is held to the same up-front check as the server. The loop runs + # in the background, so a key that could not run used to answer "started" + # and then fail its first tick where nobody saw it. + from condor.agents.engine import resolve_agent_key + from condor.runtime.llm_client import agent_key_error + + agent_key = resolve_agent_key(config_dict, strategy, agent) + problem = await agent_key_error( + agent_key, + user_id=user_id, + base_url_override=config_dict.get("model_base_url") or None, + ) + if problem: + raise HTTPException( + status_code=422, detail=f"Model '{agent_key}' cannot run: {problem}" + ) + if req.trading_context: config_dict["trading_context"] = req.trading_context elif not config_dict.get("trading_context") and strategy.default_trading_context: diff --git a/tests/test_agent_start_model_check.py b/tests/test_agent_start_model_check.py new file mode 100644 index 000000000..290b6c072 --- /dev/null +++ b/tests/test_agent_start_model_check.py @@ -0,0 +1,299 @@ +"""A model that cannot run is refused at start, and a single tick that fails says so. + +The start route answered ``started: true`` before the engine had touched the +model, so ``control_agent(start)`` with a key naming an unsaved custom endpoint +reported success. The dry run then failed its only tick where nobody could see +it: no dry-run file (experiments keep no journal), no notice (``_notify`` only +ever used a live bot, which neither caller passes) and a final state of +*completed*. +""" + +import asyncio +import time + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +from condor.agents import engine as engine_module +from condor.agents.agent import Agent, AgentStore +from condor.agents.sessions_index import list_experiments +from condor.agents.strategy import Strategy, StrategyStore +from condor.runtime.loops import LoopSupervisor +from condor.runtime.registry_file import LoopState +from condor.web.auth import get_current_user +from condor.web.models import WebUser +from condor.web.routes import agents as routes + +USER = WebUser(id=555, username="u", first_name="U", role="user") +BROKEN = "custom@__broken_test_endpoint__:__broken_test_model__" + + +# ── The start route ── + + +class FakeConfigManager: + def is_admin(self, user_id): + return False + + def get_server(self, server_name): + return None + + def has_server_access(self, *a, **k): + return False + + +class FakeEngine: + spawned: list[dict] = [] + + def __init__(self, agent, strategy, config, chat_id, user_id): + self.agent_id = "brigado.scalp_e1" + self.session_num = 1 + FakeEngine.spawned.append(config) + + async def start(self): + return None + + +@pytest.fixture +def client(tmp_path, monkeypatch): + monkeypatch.setenv("CONDOR_AGENTS_ROOT", str(tmp_path)) + monkeypatch.setattr( + "config_manager.get_config_manager", lambda: FakeConfigManager() + ) + monkeypatch.setattr( + "condor.web.auth.get_config_manager", lambda: FakeConfigManager() + ) + # No saved custom endpoints for anyone, and no env fallback to hide that. + monkeypatch.setattr("condor.preferences.load_user_data_for", lambda uid: {}) + monkeypatch.delenv("CUSTOM_LLM_BASE_URL", raising=False) + monkeypatch.setattr(engine_module, "TickEngine", FakeEngine) + FakeEngine.spawned = [] + AgentStore().create(name="Brigado", description="BRL market making") + StrategyStore().create(agent_slug="brigado", name="Scalp") + + app = FastAPI() + app.include_router(routes.router) + app.dependency_overrides[get_current_user] = lambda: USER + return TestClient(app) + + +def _start(client, config): + return client.post( + "/agents/brigado/strategies/scalp/start", + json={"config": {"execution_mode": "dry_run", **config}}, + ) + + +def test_an_unsaved_custom_endpoint_is_refused_before_any_engine_exists(client): + resp = _start(client, {"agent_key": BROKEN}) + + assert resp.status_code == 422 + assert "No saved endpoint named '__broken_test_endpoint__'" in resp.json()["detail"] + assert FakeEngine.spawned == [] + + +@pytest.mark.parametrize( + "key, reason", + [ + ("nonsense:model", "unknown model provider 'nonsense'"), + ("openrouter", "no model id"), + ("custom@venice", "no model id"), + ], +) +def test_a_key_no_client_can_run_is_refused(client, key, reason): + """``resolve_acp`` would have run Claude Code in place of these.""" + resp = _start(client, {"agent_key": key}) + + assert resp.status_code == 422 + assert reason in resp.json()["detail"] + assert FakeEngine.spawned == [] + + +@pytest.mark.parametrize( + "key", ["", "claude-code", "claude-acp:opus", "codex", "ollama:llama3.1"] +) +def test_runnable_keys_still_start(client, key): + resp = _start(client, {"agent_key": key} if key else {}) + + assert resp.status_code == 200, resp.text + assert resp.json()["started"] is True + + +def test_an_explicit_base_url_stands_in_for_the_saved_endpoint(client): + """The engine lets ``model_base_url`` win over the named endpoint; so must the check.""" + resp = _start( + client, {"agent_key": BROKEN, "model_base_url": "http://127.0.0.1:9/v1"} + ) + + assert resp.status_code == 200, resp.text + + +# ── The engine ── + + +def _async(result): + async def go(*args, **kwargs): + return result + + return go + + +async def _empty_stream(): + return + yield # pragma: no cover -- makes this an async generator + + +class _FakeClient: + def __init__(self, fail_with: Exception | None = None): + self.fail_with = fail_with + + async def start(self): + if self.fail_with: + raise self.fail_with + + async def stop(self): + return None + + +@pytest.fixture +def supervisor(monkeypatch): + sup = LoopSupervisor() + finals: list[str] = [] + real_unregister = sup.unregister + + def spy(agent_id, final_state=LoopState.STOPPED): + finals.append(final_state) + real_unregister(agent_id, final_state) + + sup.unregister = spy + sup.finals = finals + monkeypatch.setattr(engine_module, "_supervisor", lambda: sup) + return sup + + +def _engine(tmp_path, monkeypatch, *, mode, chat_id=1, client=None): + monkeypatch.setenv("CONDOR_AGENTS_ROOT", str(tmp_path / "agents")) + monkeypatch.setenv("CONDOR_REPORTS_DIR", str(tmp_path / "reports")) + strategy = Strategy(agent_slug="brigado", name="Scalp") + strategy.home.mkdir(parents=True, exist_ok=True) + engine = engine_module.TickEngine( + agent=Agent(slug="brigado", name="Brigado", agent_key=BROKEN), + strategy=strategy, + config={"execution_mode": mode, "frequency_sec": 0}, + chat_id=chat_id, + user_id=42, + ) + notices: list[str] = [] + + async def notify(message): + notices.append(message) + + monkeypatch.setattr(engine, "_notify", notify) + monkeypatch.setattr(engine, "_get_client", _async(object())) + monkeypatch.setattr(engine, "_adopt_running_bots", _async(None)) + monkeypatch.setattr(engine, "_create_client", _async(client or _FakeClient())) + monkeypatch.setattr(engine, "_collect_stream", lambda *a, **k: _empty_stream()) + monkeypatch.setattr(engine.provider_registry, "run_core_providers", _async({})) + return engine, notices + + +def _run(engine): + async def go(): + await engine.start() + await engine._task + + asyncio.run(go()) + + +def test_a_dry_run_whose_model_fails_is_a_failed_run(tmp_path, monkeypatch, supervisor): + failing = _FakeClient( + RuntimeError("No base URL configured for the custom provider.") + ) + engine, notices = _engine(tmp_path, monkeypatch, mode="dry_run", client=failing) + + _run(engine) + + # A dry-run file exists and the Runs rail reads it as failed. + [run] = list_experiments(engine.strategy.home) + assert run["error"] is True + assert run["agent_key"] == BROKEN + # The owner hears it failed — once, and not that it completed. + assert notices == [ + f"Agent {engine.agent_id}: Dry run failed: " + "No base URL configured for the custom provider." + ] + assert supervisor.finals == [LoopState.ERROR] + assert engine._last_stop_reason == "error" + assert supervisor.all() == {} + + +def test_a_healthy_dry_run_still_completes(tmp_path, monkeypatch, supervisor): + engine, notices = _engine(tmp_path, monkeypatch, mode="dry_run") + + _run(engine) + + [run] = list_experiments(engine.strategy.home) + assert run["error"] is False + assert notices == [f"Agent {engine.agent_id}: Dry run complete."] + assert supervisor.finals == [LoopState.COMPLETED] + + +def test_a_loop_tells_its_owner_once_per_distinct_error( + tmp_path, monkeypatch, supervisor +): + engine, notices = _engine(tmp_path, monkeypatch, mode="loop") + errors = iter(["model down", "model down", "model down", "server gone"]) + + async def failing_tick(): + engine._last_tick_at = time.time() + err = next(errors, None) + if err == "server gone": + engine._running = False # last tick: let the loop return + raise RuntimeError(err) + + monkeypatch.setattr(engine, "_tick", failing_tick) + + _run(engine) + + assert notices == [ + f"Agent {engine.agent_id} tick error: model down", + f"Agent {engine.agent_id} tick error: server gone", + ] + # Every failure still reaches the journal, repeated or not. + assert engine.journal._path.read_text().count("model down") == 3 + + +def test_a_block_is_announced_once_not_every_tick(tmp_path, monkeypatch, supervisor): + from condor.agents.risk import RiskState + + engine, notices = _engine(tmp_path, monkeypatch, mode="loop") + monkeypatch.setattr( + engine.risk, + "get_state", + lambda tracker: RiskState(is_blocked=True, block_reason="max drawdown"), + ) + + for _ in range(3): + asyncio.run(engine._tick()) + + assert notices == [f"Agent {engine.agent_id} blocked: max drawdown"] + + +def test_notify_reaches_the_owner_of_a_dashboard_launch(tmp_path, monkeypatch): + """chat_id 0 is a web launch; the notice goes to the owner's own chat.""" + engine = engine_module.TickEngine.__new__(engine_module.TickEngine) + engine.chat_id, engine.user_id, engine.agent_id = 0, 42, "brigado.scalp_1" + sent: list[dict] = [] + + class Recorder: + async def send_message(self, **kw): + sent.append(kw) + + monkeypatch.setattr( + "condor.agents.delegate.resolve_bot", lambda bot=None: Recorder() + ) + + asyncio.run(engine_module.TickEngine._notify(engine, "hello")) + + assert sent == [{"chat_id": 42, "text": "hello"}] From e934287dfaa566dd2729af163ab8080996550adb Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 11 Sep 2026 18:31:12 +0300 Subject: [PATCH 143/154] Draw only the range the TradeChart picker names: the shared candle store keeps every candle any chart on the channel has loaded and never trims to one chart's range, so picking a shorter range after a longer one left the longer history on screen; the chart now draws the tail measured back from the newest candle, fully redraws and refits on a range change, and still leaves live ticks to the incremental update --- .../trade/TradeChart.range.test.tsx | 176 ++++++++++++++++++ frontend/src/components/trade/TradeChart.tsx | 47 +++-- 2 files changed, 204 insertions(+), 19 deletions(-) create mode 100644 frontend/src/components/trade/TradeChart.range.test.tsx diff --git a/frontend/src/components/trade/TradeChart.range.test.tsx b/frontend/src/components/trade/TradeChart.range.test.tsx new file mode 100644 index 000000000..ce2d6cc91 --- /dev/null +++ b/frontend/src/components/trade/TradeChart.range.test.tsx @@ -0,0 +1,176 @@ +/** + * The chart draws the range its picker names, not every candle the store holds. + * + * The candle store is shared by every chart on a channel and never trims to one + * chart's range: once a 1-day chart has loaded, the channel holds a day of + * candles for a 1-hour chart too. The chart drew all of them, and only ever + * redrew when older history arrived — so picking a shorter range after a longer + * one left the longer history on screen. + * + * @vitest-environment jsdom + */ + +import { act, type ComponentProps } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from "vitest"; + +import type { CandleData } from "@/lib/api"; +import { chartDouble } from "@/test/lightweight-charts-double"; +import { TradeChart } from "./TradeChart"; + +const HOUR = 3600; +const DAY = 24 * HOUR; +const NEWEST = 1_700_086_400; + +/** `count` 1m candles ending at `newest`, oldest first — the store's order. */ +function minutes(count: number, newest = NEWEST): CandleData[] { + return Array.from({ length: count }, (_, i) => ({ + timestamp: newest - (count - 1 - i) * 60, + open: 1, + high: 2, + low: 0.5, + close: 1.5, + volume: 1, + })); +} + +/** What a 1-day chart leaves in the shared store. */ +const A_DAY = minutes(1440); + +const store = vi.hoisted(() => ({ candles: [] as CandleData[] })); + +vi.mock("@/hooks/useCandleStore", () => ({ + useCandleStore: () => ({ + candles: store.candles, + isStale: false, + mergeCandles: vi.fn(), + setDuration: vi.fn(), + }), +})); + +vi.mock("@/hooks/useRates", () => ({ + useRates: () => ({ + formatPnlValue: (v: number) => String(v), + formatValue: (v: number) => String(v), + }), +})); + +vi.mock("@/lib/api", () => ({ + api: { getCandles: vi.fn(async () => []) }, +})); + +vi.mock("@/lib/candle-store", () => ({ + candleChannelKey: (...parts: unknown[]) => parts.join(":"), + candleStore: { onUpdate: vi.fn(() => () => {}) }, +})); + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +let container: HTMLDivElement; +let root: Root; + +const PROPS: ComponentProps = { + server: "local", + connector: "binance_perpetual", + pair: "BTC-USDT", + interval: "1m", + lookbackSeconds: HOUR, + startPrice: 100, + endPrice: 200, + limitPrice: 150, + side: 1, + minSpread: 0.001, + activePickField: null, + onPriceSet: vi.fn(), +}; + +async function render(lookbackSeconds: number) { + await act(async () => { + root.render(); + }); + // The chart library arrives through a dynamic import a few hops out; give it + // a macrotask so the series exists and the data effect has run. + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 0)); + }); +} + +/** The bar times of every full `setData()` so far, oldest render first. */ +function draws(): number[][] { + const setData = chartDouble.series!.setData as Mock; + return setData.mock.calls.map(([bars]) => (bars as { time: number }[]).map((b) => b.time)); +} + +function fits(): number { + return (chartDouble.timeScale!.fitContent as Mock).mock.calls.length; +} + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + vi.stubGlobal( + "ResizeObserver", + class { + observe() {} + unobserve() {} + disconnect() {} + }, + ); + vi.stubGlobal("requestAnimationFrame", () => 0); + vi.stubGlobal("cancelAnimationFrame", () => {}); + chartDouble.reset(); + store.candles = A_DAY; + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); +}); + +describe("TradeChart range", () => { + it("draws only the picked range when the store holds more", async () => { + await render(HOUR); + + const [drawn] = draws().slice(-1); + expect(drawn).toHaveLength(61); + expect(drawn[0]).toBe(NEWEST - HOUR); + expect(drawn[drawn.length - 1]).toBe(NEWEST); + }); + + it("redraws to a shorter range picked after a longer one", async () => { + await render(DAY); + expect(draws().slice(-1)[0]).toHaveLength(1440); + + await render(HOUR); + + expect(draws()).toHaveLength(2); + expect(draws()[1]).toHaveLength(61); + expect(draws()[1][0]).toBe(NEWEST - HOUR); + expect(fits()).toBe(2); + }); + + it("redraws back out to a longer range the store already holds", async () => { + await render(HOUR); + await render(DAY); + + expect(draws()).toHaveLength(2); + expect(draws()[1]).toHaveLength(1440); + }); + + it("leaves a live tick to the incremental update", async () => { + await render(HOUR); + expect(draws()).toHaveLength(1); + + // A newer bar appends and the window slides one bar forward with it. + store.candles = minutes(1441, NEWEST + 60); + await render(HOUR); + + expect(draws()).toHaveLength(1); + expect(fits()).toBe(1); + }); +}); diff --git a/frontend/src/components/trade/TradeChart.tsx b/frontend/src/components/trade/TradeChart.tsx index 51bce0a25..aa0f0a0a6 100644 --- a/frontend/src/components/trade/TradeChart.tsx +++ b/frontend/src/components/trade/TradeChart.tsx @@ -124,7 +124,6 @@ export function TradeChart({ const chartModuleRef = useRef(null); const chartRef = useRef(null); const seriesRef = useRef | null>(null); - const initializedRef = useRef(false); // Exact price under the pointer, never snapped to the hovered candle's close: // the crosshair is free vertically, so this is the price the axis label shows // and the only one a click or a measurement may report. @@ -201,7 +200,7 @@ export function TradeChart({ }); // ── Candle data from the singleton store (WS live + cached) ── - const { candles, mergeCandles, setDuration } = useCandleStore( + const { candles: storedCandles, mergeCandles, setDuration } = useCandleStore( server, connector, pair, @@ -209,6 +208,19 @@ export function TradeChart({ poolAddress, ); + // ── Only the picked range is drawn ── + // The store keeps every candle any chart on this channel has loaded — a + // longer range picked earlier, or another chart's — and never trims to one + // chart's range. So the chart draws the tail its own picker names, measured + // back from the newest candle rather than the clock, which would make render + // impure. + const candles = useMemo(() => { + if (!storedCandles.length) return storedCandles; + const cutoff = storedCandles[storedCandles.length - 1].timestamp - lookbackSeconds; + const start = storedCandles.findIndex((c) => c.timestamp >= cutoff); + return start <= 0 ? storedCandles : storedCandles.slice(start); + }, [storedCandles, lookbackSeconds]); + // ── Filter executor overlays to those within candle time range ── // Depend on the earliest candle timestamp (not the candles array, whose reference // changes on every WS tick) so filteredOverlays keeps a stable identity across @@ -541,10 +553,10 @@ export function TradeChart({ return () => observer.disconnect(); }, [chartReady]); - // Signature of the last full setData() render: channel key + earliest - // timestamp + count. Lets us tell a wholesale change (first load, pair/ - // interval switch, history backfill/prepend) apart from a live tick, where - // the listener below already applied a cheap series.update(). + // Signature of the last full setData() render: channel key + range + + // earliest timestamp + count. Lets us tell a wholesale change (first load, + // pair/interval/range switch, history backfill/prepend) apart from a live + // tick, where the listener below already applied a cheap series.update(). const lastSetDataSigRef = useRef(""); // ── Push candle data to chart (full setData only on structural changes) ── @@ -554,7 +566,7 @@ export function TradeChart({ const key = candleChannelKey(server, connector, pair, interval, poolAddress); const first = candles[0].timestamp; const prevSig = lastSetDataSigRef.current; - const [prevKey, prevFirstStr, prevLenStr] = prevSig.split("|"); + const [prevKey, prevRangeStr, prevFirstStr, prevLenStr] = prevSig.split("|"); const prevFirst = Number(prevFirstStr); const prevLen = Number(prevLenStr); @@ -562,6 +574,8 @@ export function TradeChart({ // newer appended bar. So a full setData() is only required when: // • first load for this chart instance (no prior signature), or // • the channel key changed (pair/interval/connector/server switch), or + // • the range changed — a shorter one drops bars from the front, which + // update() can't do any more than it can insert them, or // • the earliest candle moved back in time, i.e. older history was // prepended (REST backfill) — update() can't insert before the data. // A plain live tick keeps the same key and earliest timestamp (last-bar @@ -570,10 +584,11 @@ export function TradeChart({ // expensive map + setData over the whole array on every tick. const isFirstLoad = prevSig === ""; const keyChanged = prevKey !== key; + const rangeChanged = Number(prevRangeStr) !== lookbackSeconds; const historyPrepended = candles.length > prevLen && first < prevFirst; - const needsFullReset = isFirstLoad || keyChanged || historyPrepended; + const needsFullReset = isFirstLoad || keyChanged || rangeChanged || historyPrepended; - lastSetDataSigRef.current = `${key}|${first}|${candles.length}`; + lastSetDataSigRef.current = `${key}|${lookbackSeconds}|${first}|${candles.length}`; if (!needsFullReset) return; @@ -586,11 +601,10 @@ export function TradeChart({ })); seriesRef.current.setData(mapped); - if (!initializedRef.current) { - chartRef.current?.timeScale().fitContent(); - initializedRef.current = true; - } - }, [candles, chartReady, server, connector, pair, interval, poolAddress]); + // Every full render is a new picture — a market, a range, or the history a + // backfill just brought in — so the view fits it. Live ticks never get here. + chartRef.current?.timeScale().fitContent(); + }, [candles, chartReady, server, connector, pair, interval, poolAddress, lookbackSeconds]); // ── Real-time last candle update via candle store listener ── useEffect(() => { @@ -612,11 +626,6 @@ export function TradeChart({ return removeListener; }, [chartReady, server, connector, pair, interval, poolAddress]); - // ── Reset auto-fit on pair/interval/range change ── - useEffect(() => { - initializedRef.current = false; - }, [pair, interval, lookbackSeconds]); - // ── Update price precision ── useEffect(() => { if (!seriesRef.current || pricePrecision == null) return; From 09a4b9c5356cd5967701eb9b7ce6aae9019b6454 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 11 Sep 2026 18:31:12 +0300 Subject: [PATCH 144/154] Stop search_history offering an order cursor the backend handed straight back: hummingbot-api built every order's cursor from fields its rows do not carry, so each page returned "0:" and an agent following it looped over overlapping pages; a non-advancing cursor now ends the walk with a note that the rows may repeat, until the backend fix (hummingbot/hummingbot-api#234) is deployed --- mcp_servers/hummingbot_api/tools/history.py | 13 ++++++- tests/test_mcp_search_history_pagination.py | 42 +++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/mcp_servers/hummingbot_api/tools/history.py b/mcp_servers/hummingbot_api/tools/history.py index 275423c30..d7474b746 100644 --- a/mcp_servers/hummingbot_api/tools/history.py +++ b/mcp_servers/hummingbot_api/tools/history.py @@ -179,7 +179,18 @@ async def search_history( # model that followed it re-fetched page one forever and read the # identical rows as fresh history (CORR-569). following = next_cursor(result) - if following: + if following and following == cursor: + # hummingbot-api before its keyset-cursor fix built every order's + # cursor from fields its rows do not carry (timestamp/client_order_id + # instead of created_at/order_id), so every page handed back "0:" and + # the page after it overlapped the one before. Surfacing it again would + # send the model round the same page forever. + formatted_output += ( + "\n\n... the backend handed back the same cursor it was given, so it " + "cannot page any further and the rows above may repeat the previous " + "page. Narrow the search with start_time/end_time to reach older orders." + ) + elif following: formatted_output += ( f'\n\n... and more (use cursor="{following}" to see the next page)' ) diff --git a/tests/test_mcp_search_history_pagination.py b/tests/test_mcp_search_history_pagination.py index f02608e18..412de1986 100644 --- a/tests/test_mcp_search_history_pagination.py +++ b/tests/test_mcp_search_history_pagination.py @@ -126,6 +126,48 @@ def test_passing_the_cursor_back_returns_a_different_page(client_calls): assert "cursor=" not in second +def test_a_cursor_the_backend_hands_straight_back_is_not_offered_again(monkeypatch): + """hummingbot-api once answered every order page with the cursor ``0:``. + + Its cursor was built from ``timestamp``/``client_order_id``, which order rows do + not carry, so the page after ``0:`` was always the list minus its first row: it + repeated most of the previous page and handed ``0:`` back again. A tool that kept + printing ``use cursor="0:"`` sent the model round that page forever. + """ + + class StuckTrading: + def __init__(self): + self.order_calls = [] + + async def search_orders(self, **kwargs): + self.order_calls.append(kwargs) + return { + "data": PAGE_ONE, + "pagination": {"has_more": True, "next_cursor": "0:"}, + } + + client = PaginatingClient() + client.trading = StuckTrading() + + async def fake_get_client(): + return client + + monkeypatch.setattr(hb_server.hummingbot_client, "get_client", fake_get_client) + + first = asyncio.run(hb_server.search_history(data_type="orders", limit=3)) + # Page one cannot tell a stuck cursor from a real one yet: it has sent none. + assert 'cursor="0:"' in first + + second = asyncio.run( + hb_server.search_history(data_type="orders", limit=3, cursor="0:") + ) + + assert client.trading.order_calls[1]["cursor"] == "0:" + assert "cursor=" not in second, f"the stuck cursor was offered again: {second}" + assert "same cursor" in second + assert "may repeat the previous page" in second + + def test_orders_refuses_an_offset_it_would_silently_drop(client_calls): """search_orders has no offset parameter; accepting one is the CORR-563 bug.""" with pytest.raises(ToolError) as excinfo: From 4a7e316fee2fdf6a48a8d8f0058c6a6cf436c6ec Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 11 Sep 2026 20:09:49 +0300 Subject: [PATCH 145/154] =?UTF-8?q?Enforce=20an=20Agent's=20tools=20allowl?= =?UTF-8?q?ist=20on=20Claude/ACP=20seats=20too:=20only=20pydantic-ai=20eve?= =?UTF-8?q?r=20filtered=20by=20it,=20so=20every=20claude-acp=20agent=20mou?= =?UTF-8?q?nted=20the=20whole=20agent=20ring=20and=20its=20preload=20told?= =?UTF-8?q?=20it=20to=20ToolSearch=20all=2042=20tools=20(~39k=20tokens=20o?= =?UTF-8?q?n=20a=20Solana=20LP=20chat's=20first=20turn)=20while=20its=20AG?= =?UTF-8?q?ENT.md=20named=2021;=20the=20list's=20complement=20now=20joins?= =?UTF-8?q?=20the=20operator=20mutes=20as=20--mute-tools=20(toolsets.seat?= =?UTF-8?q?=5Fmutes),=20so=20neither=20MCP=20subprocess=20registers=20what?= =?UTF-8?q?=20the=20list=20leaves=20out=20on=20any=20backend,=20and=20both?= =?UTF-8?q?=20the=20chat=20and=20tick=20preloads=20name=20only=20what=20th?= =?UTF-8?q?e=20seat=20mounts=20(solana=5Fdex=5Flp=5Fexpert:=2042=20?= =?UTF-8?q?=E2=86=92=2028)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four stale stock lists are brought in line with what the agents are told to call — the framework family their inherited playbooks use (delegate, send_notification, trading_agent_journal_write, get_available_models, manage_memory) plus solana's own manage_clmm/manage_bots/list_positions_held — and tests/test_agent_tool_allowlists.py pins every stock list against its own files and that family so it cannot drift silently again. The "ACP runs unrestricted" statements are corrected everywhere, including the model-facing agent_builder and strategy_builder skills and the manage_agents / get_available_models docstrings; the Tools tab dims a tool the allowlist leaves out, flags it "not in allowlist" and drops its switch. --- .../_shared/skills/strategy_builder/SKILL.md | 7 +- agents/condor/skills/agent_builder/SKILL.md | 12 +- agents/market_making_expert/AGENT.md | 4 + agents/meteora_launch_lp/AGENT.md | 3 + agents/solana_dex_lp_expert/AGENT.md | 7 + agents/xrpl_market_maker/AGENT.md | 4 + condor/agents/agent_run.py | 8 +- condor/agents/prompts.py | 15 +- condor/runtime/context.py | 14 +- condor/runtime/llm_client.py | 5 +- condor/runtime/sessions.py | 9 +- condor/runtime/toolsets.py | 79 +++++++++-- condor/web/routes/agents.py | 20 ++- .../agent/AgentKnowledge.tools.test.tsx | 25 +++- .../src/components/agent/AgentKnowledge.tsx | 43 +++--- frontend/src/lib/api.ts | 11 +- mcp_servers/condor/server.py | 27 ++-- mcp_servers/condor/settings.py | 4 +- mcp_servers/hummingbot_api/server.py | 11 +- tests/test_agent_tool_allowlists.py | 131 ++++++++++++++++++ tests/test_mcp_tool_profiles.py | 27 +++- tests/test_specialist_tool_preload.py | 43 ++++++ tests/test_tool_mutes.py | 78 ++++++++++- 23 files changed, 493 insertions(+), 94 deletions(-) create mode 100644 tests/test_agent_tool_allowlists.py diff --git a/agents/_shared/skills/strategy_builder/SKILL.md b/agents/_shared/skills/strategy_builder/SKILL.md index c710a54a1..7145434b9 100644 --- a/agents/_shared/skills/strategy_builder/SKILL.md +++ b/agents/_shared/skills/strategy_builder/SKILL.md @@ -121,10 +121,9 @@ the wrong one strands live capital. This playbook ends at launch. **Model:** the strategy's `agent_key` defaults to yours; override per launch with `config={"agent_key": "…"}`. Never invent one — call `get_available_models` and pick from -what the operator actually has, or leave it inherited. A pydantic-ai key -(`ollama:`/`openai:`/`groq:`/`lmstudio:`/`openrouter:`/`custom@…`) enforces the `tools` -allowlist; an ACP key (`claude-code`/`claude-acp`/`gemini`/`copilot`) runs unrestricted, -with mutations still confirmation-gated. +what the operator actually has, or leave it inherited. Every key is held to your `tools` +allowlist — a tool it leaves out is never mounted — and an ACP key +(`claude-code`/`claude-acp`/`gemini`/`copilot`) still confirmation-gates mutations. **Server:** leave `server_name` empty unless the user pins it. A strategy you own runs on whichever server your agent resolves. diff --git a/agents/condor/skills/agent_builder/SKILL.md b/agents/condor/skills/agent_builder/SKILL.md index b109701b6..9f52b3ec6 100644 --- a/agents/condor/skills/agent_builder/SKILL.md +++ b/agents/condor/skills/agent_builder/SKILL.md @@ -228,9 +228,13 @@ reads the same shared playbook and knows its own domain better than you do. **Capability rule:** there isn't one. Every agent is delegable and loopable on any model; `when_to_consult` and owning a strategy are quality, not permission. The model -only changes *how* a run executes: a pydantic-ai key (`ollama:…`/`openai:…`/`groq:…`/ -`lmstudio:…`) enforces the `tools` allowlist; an ACP key (`claude-code`/`gemini`/ -`copilot`) runs unrestricted. Every run reached through `delegate` — `start` or `ask` — +only changes *how* a run executes, never what it may reach: the `tools` allowlist +binds on every key, ACP bridges included — a tool it leaves out is never mounted. So +an allowlist must name every tool the agent's own playbooks call *plus* the family the +inherited framework skills call (`delegate`, `send_notification`, `run_code`, +`manage_memory`, `manage_skill`, `manage_routines`, `trading_agent_journal_read`, +`trading_agent_journal_write`, `manage_agents`, `manage_strategies`, `control_agent`, +`get_available_models`); leave it empty for unrestricted. Every run reached through `delegate` — `start` or `ask` — is unattended: nobody is asked to approve its tool calls, so only hand work to agents and tasks you trust. @@ -250,7 +254,7 @@ operator actually has — call `get_available_models` and pick for the agent's j default to a hardcoded model.** The tool reports: - `acp_clis` — subscription/CLI bridges (`claude-code`, `gemini`, `copilot`, `codex`) and whether each CLI is installed. No API key or per-token cost (rides the operator's - Claude/ChatGPT subscription); runs unrestricted (does NOT enforce the `tools` allowlist). + Claude/ChatGPT subscription); still held to the agent's `tools` allowlist. **`available` means installed, not signed in** — each bridge needs its own interactive login that Condor cannot probe. Never recommend one as if it were ready; name it as an option and ask the user to confirm they use it. diff --git a/agents/market_making_expert/AGENT.md b/agents/market_making_expert/AGENT.md index dd14e5966..7d45479f3 100644 --- a/agents/market_making_expert/AGENT.md +++ b/agents/market_making_expert/AGENT.md @@ -17,7 +17,11 @@ tools: - manage_agents - manage_strategies - control_agent +- get_available_models +- delegate +- send_notification - trading_agent_journal_read +- trading_agent_journal_write - manage_memory - manage_skill - run_code diff --git a/agents/meteora_launch_lp/AGENT.md b/agents/meteora_launch_lp/AGENT.md index cd767c0cc..fd562342d 100644 --- a/agents/meteora_launch_lp/AGENT.md +++ b/agents/meteora_launch_lp/AGENT.md @@ -16,7 +16,10 @@ tools: - manage_agents - manage_strategies - control_agent +- get_available_models +- delegate - trading_agent_journal_read +- trading_agent_journal_write - manage_memory - manage_skill - run_code diff --git a/agents/solana_dex_lp_expert/AGENT.md b/agents/solana_dex_lp_expert/AGENT.md index 00a7e27d0..85c74a1bf 100644 --- a/agents/solana_dex_lp_expert/AGENT.md +++ b/agents/solana_dex_lp_expert/AGENT.md @@ -12,8 +12,11 @@ tools: - list_executors - get_executor - stop_executor +- list_positions_held - list_orphaned_positions - resolve_orphaned_position +- manage_clmm +- manage_bots - get_portfolio_overview - get_prices - get_market_data @@ -22,7 +25,11 @@ tools: - manage_agents - manage_strategies - control_agent +- get_available_models +- delegate +- send_notification - trading_agent_journal_read +- trading_agent_journal_write - manage_memory - manage_skill - run_code diff --git a/agents/xrpl_market_maker/AGENT.md b/agents/xrpl_market_maker/AGENT.md index ff0c1cefe..9d204bd33 100644 --- a/agents/xrpl_market_maker/AGENT.md +++ b/agents/xrpl_market_maker/AGENT.md @@ -18,7 +18,11 @@ tools: - manage_agents - manage_strategies - control_agent +- get_available_models +- delegate - trading_agent_journal_read +- trading_agent_journal_write +- manage_memory - manage_skill - send_notification - run_code diff --git a/condor/agents/agent_run.py b/condor/agents/agent_run.py index 5e7e030b5..4c753c1ac 100644 --- a/condor/agents/agent_run.py +++ b/condor/agents/agent_run.py @@ -166,10 +166,10 @@ async def run_agent_to_completion( ) # Build the client for the (possibly fallback) model through the shared - # factory (ARCH-192). A pydantic-ai model gets the agent's tool allowlist - # enforced; an ACP model (claude-code) cannot enforce an allowlist, so it - # runs unrestricted — acceptable for a delegation, which is unattended by - # design and only started for a trusted agent. The factory re-resolves the + # factory (ARCH-192). A pydantic-ai model also filters by the agent's tool + # allowlist client-side; an ACP model (claude-code) cannot, and need not: + # the MCP servers built above never mount what the allowlist leaves out + # (toolsets.seat_mutes), whatever model runs. The factory re-resolves the # custom endpoint (same lenient inputs as the healthcheck above), so a # fallback model never inherits the original's credentials. from condor.runtime.llm_client import build_llm_client diff --git a/condor/agents/prompts.py b/condor/agents/prompts.py index ad1aa84d2..d0d7ac3b3 100644 --- a/condor/agents/prompts.py +++ b/condor/agents/prompts.py @@ -9,6 +9,7 @@ import json import logging +from collections.abc import Collection from pathlib import Path from typing import Any @@ -211,7 +212,11 @@ def _clip(text: str, limit: int) -> str: def _build_tool_preload( - *, is_dry_run: bool, is_experiment: bool, is_controller_mode: bool = False + *, + is_dry_run: bool, + is_experiment: bool, + is_controller_mode: bool = False, + muted: Collection[str] = (), ) -> str: """ToolSearch preload line for ACP sessions. @@ -221,6 +226,10 @@ def _build_tool_preload( modes (dry_run / run_once) omit trading_agent_journal_write since they have no journal. Controller mode preloads the bot/controller tools it actually trades with — otherwise the agent burns a tick discovering them. + + ``muted`` is what the seat never mounts (``toolsets.seat_mutes``: operator + mutes plus whatever the Agent's allowlist leaves out), dropped from the line so + it names only tools the tick can actually call. """ tools = [ "mcp__mcp-hummingbot__get_prices", @@ -270,6 +279,7 @@ def _build_tool_preload( "mcp__condor__manage_skill", "mcp__condor__manage_routines", ] + tools = [t for t in tools if t.rsplit("__", 1)[-1] not in muted] return ( "IMPORTANT: At the very start, load ALL MCP tools in a single ToolSearch call:\n" f'ToolSearch(query="select:{",".join(tools)}")\n' @@ -525,11 +535,14 @@ def build_tick_prompt( # Tool preload is ACP-specific (ToolSearch); pydantic-ai auto-discovers MCP tools if not use_pydantic_ai: + from condor.runtime.toolsets import seat_mutes + sections.append( _build_tool_preload( is_dry_run=is_dry_run, is_experiment=is_experiment, is_controller_mode=is_controller_mode, + muted=seat_mutes(getattr(agent, "slug", "") or None), ) ) else: diff --git a/condor/runtime/context.py b/condor/runtime/context.py index 7f30604fb..7176f2201 100644 --- a/condor/runtime/context.py +++ b/condor/runtime/context.py @@ -151,7 +151,7 @@ def _chat_mcp_tools() -> tuple[str, ...]: ) -def chat_tool_preload(agent_key: str | None) -> str: +def chat_tool_preload(agent_key: str | None, agent_slug: str | None = None) -> str: """The ToolSearch preload line for a chat seat, or ``""`` when it needs none. ACP seats (Claude Code and friends) get MCP tools deferred: they must @@ -164,15 +164,25 @@ def chat_tool_preload(agent_key: str | None) -> str: Public because both chat branches need it: the coordinator's :func:`build_initial_context` and the specialist's ``bound_agent_context``, which skips that builder entirely (CORR-272). + + It names exactly what the seat mounts: the ring, minus what + :func:`~condor.runtime.toolsets.seat_mutes` subtracts for ``agent_slug`` — + the operator's mutes and whatever the Agent's allowlist leaves out. Naming + the whole ring cost a Claude specialist ~39k tokens of schemas on its first + turn, most of them for tools its allowlist never meant it to have. ``None`` + is the coordinator, whose mutes live under the chat's slug. """ from condor.acp.pydantic_ai_client import is_pydantic_ai_model + from condor.runtime.toolsets import seat_mutes if not agent_key or is_pydantic_ai_model(agent_key): return "" + muted = set(seat_mutes(agent_slug)) + tools = [t for t in _chat_mcp_tools() if t.rsplit("__", 1)[-1] not in muted] return ( "IMPORTANT: At the very start of the session (before your first response), " "load ALL MCP tools in a single ToolSearch call:\n" - f'ToolSearch(query="select:{",".join(_chat_mcp_tools())}")\n' + f'ToolSearch(query="select:{",".join(tools)}")\n' "This avoids repeated ToolSearch calls that waste context tokens. " "Do this silently without telling the user." ) diff --git a/condor/runtime/llm_client.py b/condor/runtime/llm_client.py index b8a661afb..51c1fc866 100644 --- a/condor/runtime/llm_client.py +++ b/condor/runtime/llm_client.py @@ -61,7 +61,10 @@ def build_llm_client( whichever client understands them. Both clients take the env and the system prompt — each over its own system-level channel (``_meta.systemPrompt`` for ACP, ``instructions`` for pydantic-ai), so a bound Agent keeps its identity - on either backend (ARCH-331). Only PydanticAI enforces the tool allowlist. + on either backend (ARCH-331). Only PydanticAI takes ``allowed_tools`` as a + client filter; ACP filters nothing, so for both the allowlist is enforced + where the MCP servers are built — :func:`condor.runtime.toolsets.seat_mutes` + keeps what it leaves out from ever being mounted. """ if pydantic_ai.is_pydantic_ai_model(agent_key): custom_url, api_key = resolve_custom_endpoint( diff --git a/condor/runtime/sessions.py b/condor/runtime/sessions.py index f55c894ea..258b71c1e 100644 --- a/condor/runtime/sessions.py +++ b/condor/runtime/sessions.py @@ -484,6 +484,7 @@ def bound_agent_context( ``control_agent`` was authorized to stop the whole time. ``agent_key`` falls back to the binding's own; the caller passes the resolved key, since a model picked in the UI overrides what the Agent front matter configured. + The slug narrows the line to what this Agent's seat actually mounts. """ sections = [ binding.agent_identity_context( @@ -491,7 +492,7 @@ def bound_agent_context( ), platform_formatting(platform), ] - preload = chat_tool_preload(agent_key or bound.agent_key) + preload = chat_tool_preload(agent_key or bound.agent_key, bound.agent_slug) if preload: sections.append(preload) return "\n\n".join(sections) @@ -756,8 +757,10 @@ async def _spawn_session( agent_key, mcp_servers=mcp_servers, permission_callback=permission_callback, - # A bound Agent's allowlist is enforced here exactly as it is on - # delegate and loop, so an Agent has the same reach in every mode. + # The client-side filter, which only pydantic-ai applies. On every + # backend the allowlist is also enforced one level down: the MCP + # subprocesses never mount what it leaves out (toolsets.seat_mutes), so + # an Agent has the same reach in every mode and on every model. allowed_tools=bound.tools or None, extra_env=extra_env, system_prompt=( diff --git a/condor/runtime/toolsets.py b/condor/runtime/toolsets.py index a8588523d..3db660195 100644 --- a/condor/runtime/toolsets.py +++ b/condor/runtime/toolsets.py @@ -77,10 +77,11 @@ def _env_entries(**values: Any) -> list[dict[str, str]]: def seat_profile(agent_slug: str | None, tick: bool) -> str: """Which tool profile a seat mounts (FEAT-066). - Tool allowlists are only enforced for pydantic-ai model keys; an ACP bridge - runs unrestricted, so for those seats the surface a session MOUNTS *is* the - permission model. One rule, in one place, for both subprocesses — they share - a profile vocabulary precisely so a seat is described here and nowhere else. + An ACP bridge filters no tool itself, so the surface a session MOUNTS *is* + the permission model: this ring, minus what :func:`seat_mutes` subtracts (the + operator's mutes and whatever the Agent's ``tools:`` allowlist leaves out). + One rule, in one place, for both subprocesses — they share a profile + vocabulary precisely so a seat is described here and nowhere else. - ``tick`` — the unattended loop. No Gateway config or container control, no repointing the API server, no direct liquidity moves outside an executor, @@ -153,12 +154,65 @@ def seat_tools(agent_slug: str | None, tick: bool = False) -> list[dict[str, Any ] +def _every_tool_name() -> set[str]: + """Every tool name any ring of either server can mount.""" + from mcp_servers.condor import profiles as condor_profiles + from mcp_servers.hummingbot_api import profiles as hummingbot_profiles + + return { + name + for module in (condor_profiles, hummingbot_profiles) + for ring in module.PROFILE_TOOLS.values() + for name in ring + } + + +def allowlist_mutes(agent_slug: str | None) -> set[str]: + """What an Agent's ``tools:`` allowlist leaves out, as mute names. + + Only pydantic-ai's ``_prepare_tools`` reads ``allowed_tools``; an ACP bridge + filters nothing, so on every Claude seat the list used to be decoration — the + seat mounted the whole ring and its preload named all of it, ~39k tokens of + schemas for tools the Agent was never meant to reach. Turning the complement + into mutes enforces the list at the one layer every backend shares: the + subprocess never registers what the list omits, so the model is never told + the tool exists. + + An empty list means unrestricted, as it always has; so does an unknown slug. + Names may be namespaced (``mcp__condor__delegate``), the form pydantic-ai + also accepts, and match on their last segment. + """ + if not agent_slug: + return set() + from condor.agents.agent import AgentStore + + agent = AgentStore().get(agent_slug) + if agent is None or not agent.tools: + return set() + allowed = {str(name).rsplit("__", 1)[-1] for name in agent.tools} + return _every_tool_name() - allowed + + +def seat_mutes(agent_slug: str | None) -> list[str]: + """Every tool this agent's seats must not mount, sorted. + + The operator's mutes (FEAT-091) plus what its allowlist leaves out. The one + answer to "what is subtracted", read by the spawner that builds argv and by + both preloads that name tools to the model: a preload naming a tool the + subprocess never registered spends a ToolSearch on nothing, and a spawner + and a preload that disagree are how the allowlist stopped meaning anything. + """ + from condor.memory.mutes import load_mutes + + return sorted(load_mutes(agent_slug)["tools"] | allowlist_mutes(agent_slug)) + + def _muted_tool_args(muted_tools: Sequence[str]) -> list[str]: """``--mute-tools a,b,c`` — or nothing at all when nothing is muted. - Nothing on the line is the point: an install where no operator has switched - a tool off spawns byte-identical argv to before FEAT-091 existed, so the flag - can never be blamed for a session that behaves differently. + Nothing on the line is the point: an agent nobody has curated and that names + no allowlist spawns byte-identical argv to before FEAT-091 existed, so the + flag can never be blamed for a session that behaves differently. Both servers are handed the *same* list, and each ignores the names it does not mount. A mute is one fact about one agent; splitting it per server would @@ -306,7 +360,6 @@ def build_mcp_servers_for_session( turn for ``on_complete="resume"`` to wake. See :func:`_condor_mcp_args` for why it travels on argv. """ - from condor.memory.mutes import load_mutes from config_manager import ( ServerPermission, get_config_manager, @@ -316,11 +369,13 @@ def build_mcp_servers_for_session( cm = get_config_manager() profile = seat_profile(agent_slug, tick) - # Read once, for both subprocesses: a mute is one fact about one agent, and - # reading the file twice is two answers to the same question. Sorted so the + # Read once, for both subprocesses: what an agent must not mount is one fact + # about one agent — its operator mutes plus whatever its allowlist leaves out + # — and reading it twice is two answers to the same question. Sorted so the # spawn line is stable between restarts, and empty for every agent nobody has - # curated — in which case neither builder puts a flag on the line at all. - muted_tools = sorted(load_mutes(agent_slug)["tools"]) + # curated that names no allowlist — in which case neither builder puts a flag + # on the line at all. + muted_tools = seat_mutes(agent_slug) # Resolve which hummingbot server to use (explicit override > user # preferences). Every candidate is held to existence *and* reach, because diff --git a/condor/web/routes/agents.py b/condor/web/routes/agents.py index 4543111b1..2cad62cb0 100644 --- a/condor/web/routes/agents.py +++ b/condor/web/routes/agents.py @@ -351,17 +351,15 @@ class StrategyCard(BaseModel): class ToolCard(BaseModel): - """One tool this Agent's seat actually mounts (FEAT-091). - - Not the AGENT.md allowlist: that list is only enforced for pydantic-ai model - keys, and an ACP bridge (claude-code, gemini, copilot) runs unrestricted, so - for most seats here the list is decoration. What the model is really handed - is what the two MCP subprocesses register — which is what this row is, and - what its switch turns off. - - ``allowlisted`` keeps the other statement visible instead of conflating the - two: it says the AGENT.md list names this tool, which is a pydantic-ai fact - about *filtering*, while ``muted`` is an operator fact about *mounting*. + """One tool of this Agent's seat's ring (FEAT-091), and what takes it away. + + Every row of the ring is listed, so a tool switched off can be switched back + on. Two things keep a row from being mounted, on every backend alike, since + the spawner turns both into ``--mute-tools`` (``toolsets.seat_mutes``): + ``muted`` is the operator's switch, and — when the Agent names an allowlist + (``AgentBrain.tools_unrestricted`` false) — ``allowlisted`` false means the + AGENT.md list leaves it out. What the model is handed is the rows that + neither removes. """ name: str diff --git a/frontend/src/components/agent/AgentKnowledge.tools.test.tsx b/frontend/src/components/agent/AgentKnowledge.tools.test.tsx index e3f6189a9..1726c284d 100644 --- a/frontend/src/components/agent/AgentKnowledge.tools.test.tsx +++ b/frontend/src/components/agent/AgentKnowledge.tools.test.tsx @@ -1,11 +1,9 @@ /** * The Tools tab is the seat's real mounted surface, with a switch (FEAT-091). * - * It used to echo the AGENT.md allowlist, which only binds pydantic-ai model - * keys — an ACP bridge runs unrestricted — so for most agents here the tab was - * telling the reader something untrue about what the model can reach. What is - * pinned here is the replacement: rows grouped by MCP server, the allowlist - * shown as one flag among the rows rather than as the whole list, the switch + * What is pinned here: rows grouped by MCP server, the allowlist shown as one + * flag among the rows rather than as the whole list, a row the allowlist leaves + * out dimmed with no switch (it is never mounted, on any backend), the switch * reaching the endpoint with `kind: "tool"`, and the muted row still visible so * the curation can be undone. * @@ -173,6 +171,23 @@ describe("the tools tab", () => { expect(container.textContent).toContain("Edit it in the Brain tab"); }); + it("dims what an allowlist leaves out, flagged and with no switch", async () => { + getAgentBrain.mockResolvedValue(brain({ allowlist: ["get_candles"] })); + await openTools(); + + // Left out of the list means never mounted: nothing to switch. + expect(rowFor("delegate").textContent).toContain("not in allowlist"); + expect(rowFor("delegate").querySelector('button[role="switch"]')).toBeNull(); + expect(rowFor("get_candles").textContent).not.toContain("not in allowlist"); + expect(switchFor("get_candles")).toBeTruthy(); + }); + + it("flags nothing as left out when no allowlist is written", async () => { + await openTools(); + expect(container.textContent).not.toContain("not in allowlist"); + expect(switchFor("delegate")).toBeTruthy(); + }); + it("points at the Brain tab when no allowlist is written", async () => { await openTools(); expect(container.textContent).toContain("AGENT.md names no allowlist"); diff --git a/frontend/src/components/agent/AgentKnowledge.tsx b/frontend/src/components/agent/AgentKnowledge.tsx index 86bcd8c00..b90045fab 100644 --- a/frontend/src/components/agent/AgentKnowledge.tsx +++ b/frontend/src/components/agent/AgentKnowledge.tsx @@ -1049,13 +1049,13 @@ const TOOL_SERVERS: { id: string; label: string }[] = [ ]; /** - * Every tool this agent's seat actually mounts, each with a switch (FEAT-091). + * Every tool this agent's seat could mount, each with a switch (FEAT-091). * - * Not the AGENT.md allowlist. That list only binds pydantic-ai model keys — an - * ACP bridge runs unrestricted — so a tab that only echoed it was telling most - * agents something untrue about what they can reach. The switch is the honest - * control: a muted tool is never registered on the subprocess, so the model is - * never told it exists, on every backend alike. + * The rows are the whole ring, so a switched-off tool can be switched back on. + * Two things keep a row from being mounted, on every backend alike: the + * operator's switch, and — when AGENT.md names an allowlist — being left out of + * it. Either way the tool is never registered on the subprocess, so the model + * is never told it exists. */ function ToolsTab({ brain, @@ -1072,6 +1072,10 @@ function ToolsTab({ ...server, tools: brain.tools.filter((t) => t.server === server.id), })).filter((group) => group.tools.length > 0); + // Under an allowlist, a tool it leaves out is never mounted (the spawner turns + // the omission into a mute), so its row has no switch worth throwing. + const offList = (t: AgentBrain["tools"][number]) => + !brain.tools_unrestricted && !t.allowlisted; return (
@@ -1082,8 +1086,8 @@ function ToolsTab({

{brain.tools_unrestricted - ? "AGENT.md names no allowlist, so nothing narrows this further. Naming tools there also narrows what the agent may call — edit it in the Brain tab, where it is written." - : "AGENT.md also names an allowlist, marked below. Edit it in the Brain tab, where it is written."} + ? "AGENT.md names no allowlist, so nothing narrows this further. Naming tools there limits the agent to exactly those — edit it in the Brain tab, where it is written." + : "AGENT.md names an allowlist: only the tools marked allowlisted are mounted, and the rest are never offered to the agent. Edit it in the Brain tab, where it is written."}

{brain.tools.length === 0 ? ( No tools mounted on this agent's seat. @@ -1109,20 +1113,27 @@ function ToolsTab({ allowlisted )} + {offList(t) && ( + + not in allowlist + + )} } subtitle={t.description} - dimmed={t.muted} + dimmed={t.muted || offList(t)} onAsk={onAskAgent && (() => onAskAgent(OPENER.tool(t.name)))} askTitle="Ask the agent how it uses this tool" toggle={ - onMute && { - on: !t.muted, - onChange: () => onMute(t.name, !t.muted), - title: t.muted - ? "Off for this agent — switch it back on" - : "On — switch it off for this agent", - } + onMute && !offList(t) + ? { + on: !t.muted, + onChange: () => onMute(t.name, !t.muted), + title: t.muted + ? "Off for this agent — switch it back on" + : "On — switch it off for this agent", + } + : undefined } /> ))} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index d09ae0ae5..e12d2fbc7 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1069,12 +1069,11 @@ export interface AgentDetail { * `getAgentMemory`), so opening the panel never pulls the whole library. */ /** - * One tool this agent's seat actually mounts (FEAT-091). + * One tool of this agent's seat's ring (FEAT-091). * - * Not the AGENT.md allowlist: that is only enforced for pydantic-ai model keys, - * and an ACP bridge (claude-code, gemini, copilot) runs unrestricted — so for - * most seats the list is decoration and this row is the real surface. Switching - * one off means the next session does not register it at all. + * Two things keep it from being mounted, on every backend alike: `muted`, and — + * when the agent names an allowlist (`tools_unrestricted` false) — `allowlisted` + * being false. Either way the next session does not register it at all. */ export interface ToolCard { name: string; @@ -1083,7 +1082,7 @@ export interface ToolCard { description: string; /** Switched off by the operator — the next session never mounts it. */ muted: boolean; - /** The AGENT.md allowlist names it (the pydantic-ai filter, not the mount). */ + /** The AGENT.md allowlist names it. Under an allowlist, a row it omits is never mounted. */ allowlisted: boolean; } diff --git a/mcp_servers/condor/server.py b/mcp_servers/condor/server.py index 407b62af2..f3e16f1c2 100644 --- a/mcp_servers/condor/server.py +++ b/mcp_servers/condor/server.py @@ -695,8 +695,8 @@ async def get_available_models( model (a capable OpenRouter model when its key is set, or a subscription ACP bridge the user confirms is signed in); simple report/watch loops or privacy/offline needs → a loaded local model or a cheap - OpenRouter one. Only pydantic-ai keys (openrouter:/ollama:/lmstudio:/openai:/ - groq:) enforce an agent's ``tools`` allowlist; ACP bridges run unrestricted. + OpenRouter one. An agent's ``tools`` allowlist binds on every model, ACP + bridges included: a tool it leaves out is never mounted. """ return await available_models_tool.get_available_models( openrouter_query, openrouter_limit @@ -767,9 +767,15 @@ async def manage_agents( instructions: The AGENT.md body — identity + domain knowledge (create/update). agent_key: Default LLM. Examples: "claude-code", "gemini", "copilot", "ollama:llama3.1", "ollama:qwen3:32b", "groq:llama-3.3-70b-versatile". - Any model can run an agent; a pydantic-ai key (e.g. "ollama:...") - additionally enforces the tools allowlist. Default "claude-code". - tools: Tool-name allowlist for the agent. Empty/None = unrestricted. + Any model can run an agent, and every one is held to the tools + allowlist. Default "claude-code". + tools: Tool-name allowlist for the agent. Empty/None = unrestricted. A + tool it omits is never mounted on any model, so keep the family the + inherited framework playbooks call — delegate, send_notification, + run_code, manage_memory, manage_skill, manage_routines, + trading_agent_journal_read, trading_agent_journal_write, + manage_agents, manage_strategies, control_agent, + get_available_models — plus every tool the agent's own playbooks name. when_to_consult: One-line hint describing when to route work to this agent. Purely for routing — every agent is delegable with or without it; it falls back to the description. @@ -1220,11 +1226,12 @@ async def trading_agent_journal_write( # ── Tool profiles (FEAT-066) ───────────────────────────────────────────────── # -# Tool allowlists are only enforced for pydantic-ai model keys; an ACP bridge -# (claude-code, gemini, copilot) runs unrestricted. For those seats the surface a -# session MOUNTS is the whole permission model, so which tools this process -# registers is a security boundary — hence explicit registration below instead of -# an ``@mcp.tool()`` decorator that fires for every seat at import. +# An ACP bridge (claude-code, gemini, copilot) filters no tool itself, so the +# surface a session MOUNTS is the whole permission model — the profile, minus the +# operator's mutes and whatever the Agent's allowlist leaves out, both arriving +# as ``--mute-tools``. Which tools this process registers is a security boundary +# — hence explicit registration below instead of an ``@mcp.tool()`` decorator +# that fires for every seat at import. # # The rings themselves — which tool sits in which one, and why — moved to # ``profiles.py`` as plain name strings (FEAT-091), because the web process has diff --git a/mcp_servers/condor/settings.py b/mcp_servers/condor/settings.py index ae178a8dd..d8b443a0f 100644 --- a/mcp_servers/condor/settings.py +++ b/mcp_servers/condor/settings.py @@ -44,8 +44,8 @@ class Settings: # with nobody watching any of them. Enforced in ``tools/delegate.py``. ask_target: bool = False # Which slice of the tool surface this process registers (FEAT-066). An ACP - # bridge runs unrestricted, so for those seats the mounted surface IS the - # permission model: see ``server.TOOL_PROFILES``. It is a separate flag from + # bridge filters no tool itself, so the mounted surface IS the permission + # model: see ``server.TOOL_PROFILES``. It is a separate flag from # the two above rather than derived from them, because the seat it narrows is # the *tick*, and neither ``agent_slug`` nor ``delegate_worker`` tells an # unattended loop apart from an attended chat with the same specialist. diff --git a/mcp_servers/hummingbot_api/server.py b/mcp_servers/hummingbot_api/server.py index 9823613d6..76fd68ed7 100644 --- a/mcp_servers/hummingbot_api/server.py +++ b/mcp_servers/hummingbot_api/server.py @@ -2143,11 +2143,12 @@ async def explore_geckoterminal( # ── Tool profiles (FEAT-066) ───────────────────────────────────────────────── # -# Tool allowlists are only enforced for pydantic-ai model keys; an ACP bridge -# (claude-code, gemini, copilot) runs unrestricted. For those seats the surface a -# session MOUNTS is the whole permission model, so which tools this process -# registers is a security boundary — hence explicit registration below instead of -# an ``@mcp.tool()`` decorator that fires for everyone at import. +# An ACP bridge (claude-code, gemini, copilot) filters no tool itself, so the +# surface a session MOUNTS is the whole permission model — the profile, minus the +# operator's mutes and whatever the Agent's allowlist leaves out, both arriving +# as ``--mute-tools``. Which tools this process registers is a security boundary +# — hence explicit registration below instead of an ``@mcp.tool()`` decorator +# that fires for everyone at import. # # The rings themselves — which tool sits in which one, and why — moved to # ``profiles.py`` as plain name strings (FEAT-091), because the web process has diff --git a/tests/test_agent_tool_allowlists.py b/tests/test_agent_tool_allowlists.py new file mode 100644 index 000000000..c6a6ca418 --- /dev/null +++ b/tests/test_agent_tool_allowlists.py @@ -0,0 +1,131 @@ +"""A stock Agent's ``tools:`` allowlist binds on every backend, so it must name +what the agent is actually told to call. + +It used to bind only pydantic-ai seats. On a Claude seat it was decoration, and it +drifted unnoticed: four of the six stock lists omitted tools their own playbooks +call — ``meteora_launch_lp``'s loop strategy journals with a tool its list never +named, ``solana_dex_lp_expert``'s shutdown notifies the owner with another. Now +the spawner never mounts what a list leaves out (``toolsets.seat_mutes``), so a +missing name is a step the agent cannot take. These pin every stock list against +the two sources that say what an agent needs. +""" + +import re +from pathlib import Path + +import pytest +import yaml + +from mcp_servers.condor import profiles as condor_profiles +from mcp_servers.hummingbot_api import profiles as hummingbot_profiles + +AGENTS = Path(__file__).resolve().parent.parent / "agents" + +#: What an attended specialist can mount at all. +AGENT_RING = set(condor_profiles.PROFILE_TOOLS["agent"]) | set( + hummingbot_profiles.PROFILE_TOOLS["agent"] +) + +#: What the framework skills every agent inherits (``agent_framework``, +#: ``strategy_builder``, ``operate_your_loop``, ``skill_authoring``, +#: ``self_improve``) and the tick's house prompt tell *any* agent to call. +FRAMEWORK = { + "delegate", + "send_notification", + "run_code", + "manage_memory", + "manage_skill", + "manage_routines", + "trading_agent_journal_read", + "trading_agent_journal_write", + "manage_agents", + "manage_strategies", + "control_agent", + "get_available_models", +} + +#: Tool names an agent's own files mention without calling them. +NOT_CALLS = { + "meteora_launch_lp": { + # "CLMM/DLMM LP → `create_lp_executor` / the Solana DEX LP agent. Never + # reach for those here." — and router swaps go to create_order_executor. + "create_lp_executor", + "create_order_executor", + # `manage_amm(action="quote_swap" | "execute_swap")`: actions, not tools. + "quote_swap", + "execute_swap", + }, + "xrpl_market_maker": { + # "No `manage_gateway_config`, `explore_dex_pools` or `quote_swap` / + # `execute_swap`" — XRPL is a native CLOB, not Gateway. + "explore_dex_pools", + "quote_swap", + "execute_swap", + }, +} + + +def _frontmatter(path: Path) -> dict: + return yaml.safe_load(path.read_text().split("---")[1]) or {} + + +def _allowlisted_agents() -> list[str]: + return sorted( + path.parent.name + for path in AGENTS.glob("*/AGENT.md") + if _frontmatter(path).get("tools") + ) + + +def _allowlist(slug: str) -> set[str]: + return set(_frontmatter(AGENTS / slug / "AGENT.md")["tools"]) + + +def _named_in_own_files(slug: str) -> dict[str, list[str]]: + """``{tool: [file, ...]}`` for every ring tool the agent's own markdown names.""" + home = AGENTS / slug + found: dict[str, list[str]] = {} + for path in sorted(home.rglob("*.md")): + text = path.read_text() + for tool in AGENT_RING: + if re.search(rf"\b{tool}\b", text): + found.setdefault(tool, []).append(str(path.relative_to(home))) + return found + + +ALLOWLISTED = _allowlisted_agents() + + +def test_there_are_lists_to_check(): + assert ALLOWLISTED, "no stock agent names an allowlist any more" + + +@pytest.mark.parametrize("slug", ALLOWLISTED) +def test_the_list_names_only_tools_the_seat_can_mount(slug): + """A typo, or a tool no ring mounts any more, is a name that grants nothing.""" + assert _allowlist(slug) <= AGENT_RING, sorted(_allowlist(slug) - AGENT_RING) + + +@pytest.mark.parametrize("slug", ALLOWLISTED) +def test_the_list_carries_the_framework_family(slug): + missing = FRAMEWORK - _allowlist(slug) + assert not missing, f"{slug} inherits playbooks that call {sorted(missing)}" + + +@pytest.mark.parametrize("slug", ALLOWLISTED) +def test_the_list_covers_what_the_agents_own_files_call(slug): + exempt = NOT_CALLS.get(slug, set()) + missing = { + tool: files + for tool, files in _named_in_own_files(slug).items() + if tool not in _allowlist(slug) and tool not in exempt + } + assert not missing, f"{slug}'s own files call tools its list leaves out: {missing}" + + +@pytest.mark.parametrize("slug", sorted(NOT_CALLS)) +def test_every_exemption_is_still_needed(slug): + """An exemption whose mention is gone would hide the next real one.""" + named = _named_in_own_files(slug) + stale = {t for t in NOT_CALLS[slug] if t not in named or t in _allowlist(slug)} + assert not stale, f"drop {sorted(stale)} from NOT_CALLS[{slug!r}]" diff --git a/tests/test_mcp_tool_profiles.py b/tests/test_mcp_tool_profiles.py index 2d1290e04..1f5c810e6 100644 --- a/tests/test_mcp_tool_profiles.py +++ b/tests/test_mcp_tool_profiles.py @@ -1,10 +1,10 @@ """What each seat mounts (FEAT-066). -Tool allowlists are only enforced for pydantic-ai model keys; an ACP bridge -(claude-code, gemini, copilot) runs unrestricted. For those seats the surface a -session MOUNTS is the whole permission model, so every profile's tool set is -pinned here as a golden list: a tool added to the wrong ring fails a test rather -than quietly widening the seat that trades with real capital. +An ACP bridge (claude-code, gemini, copilot) filters no tool itself, so the +surface a session MOUNTS is the whole permission model — the profile, minus the +operator's mutes and the Agent's allowlist complement — and every profile's tool +set is pinned here as a golden list: a tool added to the wrong ring fails a test +rather than quietly widening the seat that trades with real capital. """ import asyncio @@ -264,6 +264,23 @@ def test_the_tick_preload_carries_a_market_read_a_dry_run_can_actually_make( assert dry_run_refusal(call) is None, action +def test_the_tick_preload_drops_what_the_seat_never_mounts(): + """An allowlisted agent's tick must not be told of tools its seat never + registers — naming one spends the tick discovering it cannot call it.""" + from condor.agents.prompts import _build_tool_preload + + line = _build_tool_preload( + is_dry_run=False, + is_experiment=False, + muted={"create_grid_executor", "send_notification"}, + ) + names = set(re.search(r'select:([^"]+)"', line).group(1).split(",")) + + assert "mcp__mcp-hummingbot__create_grid_executor" not in names + assert "mcp__condor__send_notification" not in names + assert "mcp__mcp-hummingbot__create_lp_executor" in names + + def test_the_manage_trading_agent_funnel_is_in_no_profile(): """FEAT-068 split it; no ring may resurrect the name.""" for name in CONDOR_PROFILES: diff --git a/tests/test_specialist_tool_preload.py b/tests/test_specialist_tool_preload.py index 21b9e53ed..c05ee56f4 100644 --- a/tests/test_specialist_tool_preload.py +++ b/tests/test_specialist_tool_preload.py @@ -151,6 +151,49 @@ def test_the_preload_names_the_tools_the_shared_playbooks_call(tool): assert tool in chat_tool_preload(ACP_KEY) +def _preloaded(line: str) -> set[str]: + import re + + return set(re.search(r'select:([^"]+)"', line).group(1).split(",")) + + +def test_an_allowlisted_specialist_is_preloaded_only_its_list(): + """The ~39k-token first turn: a Claude LP specialist was preloaded all 42 + tools of the ring while its AGENT.md named 21 of them.""" + from condor.agents.agent import AgentStore + + slug = ( + AgentStore().create(name="Lister", tools=["get_prices", "control_agent"]).slug + ) + + assert _preloaded(chat_tool_preload(ACP_KEY, slug)) == { + "mcp__mcp-hummingbot__get_prices", + "mcp__condor__control_agent", + } + + +def test_the_bound_branch_preloads_its_own_seat(): + from condor.agents.agent import AgentStore + + bound = _specialist() + bound.agent_slug = AgentStore().create(name="Lister", tools=["get_prices"]).slug + + context = bound_agent_context(bound, 1, "web") + assert "mcp__mcp-hummingbot__get_prices" in context + assert "mcp__mcp-hummingbot__manage_clmm" not in context + + +def test_an_operator_mute_is_not_preloaded(): + """The coordinator's mutes live under the chat slug, which ``None`` names.""" + from condor.memory.mutes import set_muted + + set_muted(None, "tool", "manage_clmm", True) + + line = chat_tool_preload(ACP_KEY) + assert "mcp__mcp-hummingbot__manage_clmm" not in line + assert "mcp__mcp-hummingbot__manage_amm" in line + + def test_no_literal_tool_list_survives_in_context(): """Derivation is the fix; a literal creeping back is the regression. diff --git a/tests/test_tool_mutes.py b/tests/test_tool_mutes.py index ec69241d3..06dcf538e 100644 --- a/tests/test_tool_mutes.py +++ b/tests/test_tool_mutes.py @@ -1,8 +1,9 @@ """FEAT-091: a muted tool is never mounted. -The Tools tab stops being a read-only echo of the AGENT.md allowlist — which -only binds pydantic-ai seats, and is decoration on an ACP bridge — and becomes -the real mounted surface with a switch per row. Switching one off means the next +The Tools tab stops being a read-only echo of the AGENT.md allowlist — which then +bound only pydantic-ai seats, and was decoration on an ACP bridge until it was +folded into this same subtraction (3b) — and becomes the real mounted surface +with a switch per row. Switching one off means the next session's MCP subprocess never registers it, so the model is never told it exists, on every backend alike. @@ -294,6 +295,77 @@ def test_a_mute_belongs_to_one_agent(tmp_path): assert {r["name"] for r in seat_tools("spot") if r["muted"]} == set() +# ── 3b. the allowlist, enforced through the same subtraction ── + + +def _agent_allowing(*tools: str) -> str: + """An Agent on disk whose AGENT.md names ``tools`` as its allowlist.""" + from condor.agents.agent import AgentStore + + return AgentStore().create(name="Lister", tools=list(tools)).slug + + +def _muted_on(argv: list[str]) -> set[str]: + return set(argv[argv.index("--mute-tools") + 1].split(",")) + + +def test_an_allowlist_mutes_everything_it_leaves_out(monkeypatch): + """Only pydantic-ai ever filtered by the list; now it reaches argv, which + every backend obeys because an unregistered tool is one nobody is told of.""" + slug = _agent_allowing("get_prices", "delegate") + args = _session_args(monkeypatch, agent_slug=slug) + + for argv in args.values(): + muted = _muted_on(argv) + assert "get_prices" not in muted and "delegate" not in muted + assert {"manage_clmm", "execute_swap", "manage_agents"} <= muted + + +def test_end_to_end_an_allowlisted_seat_mounts_only_its_list(monkeypatch): + slug = _agent_allowing("get_prices", "delegate") + args = _session_args(monkeypatch, agent_slug=slug) + + mounted = {} + for name, module in ( + ("condor", condor_server), + ("mcp-hummingbot", hummingbot_server), + ): + argv = args[name] + profile = argv[argv.index("--profile") + 1] + mounted[name] = _registered(module, profile, _muted_on(argv)) + + assert mounted == {"condor": {"delegate"}, "mcp-hummingbot": {"get_prices"}} + + +def test_an_allowlist_and_an_operator_mute_add_up(monkeypatch): + slug = _agent_allowing("get_prices", "delegate") + set_muted(slug, "tool", "get_prices", True) + + muted = _muted_on(_session_args(monkeypatch, agent_slug=slug)["mcp-hummingbot"]) + assert "get_prices" in muted + assert "delegate" not in muted + + +def test_a_namespaced_allowlist_name_still_matches(monkeypatch): + """pydantic-ai accepts ``mcp__server__tool``; the mount has to agree.""" + slug = _agent_allowing("mcp__mcp-hummingbot__get_prices") + argv = _session_args(monkeypatch, agent_slug=slug)["mcp-hummingbot"] + assert "get_prices" not in _muted_on(argv) + + +def test_an_empty_allowlist_stays_unrestricted(monkeypatch): + slug = _agent_allowing() + for argv in _session_args(monkeypatch, agent_slug=slug).values(): + assert "--mute-tools" not in argv + + +def test_the_panel_switch_stays_the_operators_alone(): + """``muted`` is the operator's switch; the allowlist is its own flag, so a + tool the list leaves out is not reported as switched off.""" + slug = _agent_allowing("get_prices") + assert all(row["muted"] is False for row in seat_tools(slug)) + + # ── 4. through the route ── From dc0f78dbab9abaf036e1ca74ea358047b5844541 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Mon, 14 Sep 2026 15:06:35 +0300 Subject: [PATCH 146/154] =?UTF-8?q?Make=20usage=20telemetry=20opt-out=20af?= =?UTF-8?q?ter=20a=20notice=20instead=20of=20an=20opt-in=20question:=20an?= =?UTF-8?q?=20unanswered=20install=20stays=20at=20the=20ping=20floor=20unt?= =?UTF-8?q?il=20the=20admin=20has=20actually=20been=20told=20(the=20Telegr?= =?UTF-8?q?am=20boot=20message=20was=20delivered,=20or=20the=20dashboard?= =?UTF-8?q?=20strip=20rendered=20for=20the=20admin=20and=20posted=20POST?= =?UTF-8?q?=20/settings/telemetry/notice),=20then=20resolves=20to=20usage;?= =?UTF-8?q?=20the=20notice=20offers=20Got=20it=20and=20Turn=20off=20(a=20r?= =?UTF-8?q?ecorded=20refusal)=20and=20points=20to=20Settings=20=E2=86=92?= =?UTF-8?q?=20Privacy,=20a=20failed=20send=20turns=20nothing=20on,=20and?= =?UTF-8?q?=20an=20install=20that=20chose=20ping,=20refused,=20or=20is=20p?= =?UTF-8?q?inned=20by=20CONDOR=5FTELEMETRY=20keeps=20its=20answer;=20PRIVA?= =?UTF-8?q?CY.md=20and=20the=20README=20now=20describe=20the=20default=20a?= =?UTF-8?q?s=20usage-after-notice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PRIVACY.md | 84 ++++++----- README.md | 7 +- condor/telemetry/__init__.py | 7 +- condor/telemetry/consent.py | 82 ++++++++--- condor/telemetry/prompt.py | 105 +++++++------ condor/web/routes/settings.py | 31 +++- .../src/components/TelemetryConsentBanner.tsx | 78 +++++----- .../components/settings/TelemetrySettings.tsx | 18 +-- frontend/src/hooks/useTelemetry.ts | 25 +++- frontend/src/lib/api.ts | 33 +++-- main.py | 16 +- tests/test_telemetry.py | 138 ++++++++++++++++-- tests/test_telemetry_consent_web.py | 33 ++++- 13 files changed, 464 insertions(+), 193 deletions(-) diff --git a/PRIVACY.md b/PRIVACY.md index 2abfa864c..670aca5af 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -1,17 +1,21 @@ # Privacy Condor is self-hosted. It runs on your machine, holds your exchange API keys, -and places your orders. So what leaves that machine by default is the absolute -minimum — an anonymous "this install exists" — and everything beyond that is -opt-in. This document is the complete statement of both. - -**Short version:** a fresh install counts itself and nothing more. With no -consent recorded it runs at level `ping`: a random install id, the version, and -a periodic heartbeat — nothing about you, your users, or your trading. None of -the usage events are sent until an admin taps "yes" on a prompt. Batches go to -the project's collector at `https://telemetry.hummingbot.org/v1/events`, which -is fixed in the source and cannot be pointed elsewhere. An admin can turn it -off entirely — in Settings → Privacy, or with `CONDOR_TELEMETRY=off` in the +and places your orders. So what leaves that machine is anonymous, allowlisted, +and announced before it starts — and anything that is content is opt-in. This +document is the complete statement of both. + +**Short version:** a fresh install counts itself: a random install id, the +version, and a periodic heartbeat — nothing about you, your users, or your +trading. Anonymous **usage summaries** (which features are used, what breaks, +which models agents run) are **on by default, but only after the admin has been +told**: Condor shows one notice — a Telegram message on boot, or a strip on the +dashboard — saying so, with a "Got it" button and a way to turn it off. Until +that notice has been delivered, nothing beyond the install count is recorded. +Batches go to the project's collector at +`https://telemetry.hummingbot.org/v1/events`, which is fixed in the source and +cannot be pointed elsewhere. An admin can turn it off entirely — from the +notice, in Settings → Privacy, or with `CONDOR_TELEMETRY=off` in the environment — and a refusal, once recorded, is honoured across upgrades. There is one other way anything can leave, and it is completely separate: you @@ -32,17 +36,18 @@ meant to be read, not trusted. ## What is collected -There are three levels. The consent prompt chooses between `ping` and `usage` -— it has no "off" button, so ignoring it is never read as a refusal. `off` is a -deliberate act: the admin turns reporting off in Settings → Privacy, or the -operator sets `CONDOR_TELEMETRY=off`. You can change the answer later, in -either direction. +There are three levels. An install that has not answered is at `ping` until the +admin has been shown the notice, and at `usage` from then on. `off` is a +deliberate act: the admin presses "Turn off" on the notice or picks it in +Settings → Privacy, or the operator sets `CONDOR_TELEMETRY=off`. Ignoring the +notice is not a refusal — but it has been read, so it is not a secret either. +You can change the answer later, in either direction. | Level | What it sends | |---|---| -| `ping` | Only that this install exists: `install`, `heartbeat`, `version_change`, `shutdown`. **This is the default, and the floor** — the prompt has no "off" option. | -| `usage` | The above plus the feature, reliability and agent events below. Opt-in only. | -| `off` | Nothing, ever. The emitter is a no-op — no install id is created, nothing is buffered, nothing is written, nothing is sent. Reached by an admin turning reporting off in Settings → Privacy, or by `CONDOR_TELEMETRY=off` in the environment. | +| `ping` | Only that this install exists: `install`, `heartbeat`, `version_change`, `shutdown`. What an install sends **before the notice has reached the admin**, and what "Only count my install" in Settings keeps it at. | +| `usage` | The above plus the feature, reliability and agent events below. **The default once the notice has been shown**; opt out at any time. | +| `off` | Nothing, ever. The emitter is a no-op — no install id is created, nothing is buffered, nothing is written, nothing is sent. Reached by "Turn off" on the notice, by an admin turning reporting off in Settings → Privacy, or by `CONDOR_TELEMETRY=off` in the environment. | Every batch carries one context block describing the *deployment*, not you: @@ -120,8 +125,9 @@ per-install, the same person on two installs produces two unrelated hashes. ## Where it goes -At the default `ping` level, only the four adoption events and the envelope -above. Everything else needs an explicit opt-in. +Before the notice has been shown, and at `ping`, only the four adoption events +and the envelope above. The usage events follow once the admin has been told, +unless they turned it off. Batches are POSTed to `https://telemetry.hummingbot.org/v1/events`. That address is compiled into @@ -144,37 +150,40 @@ cat .condor/telemetry/outbox.jsonl | jq . ## How to change it — or turn it off entirely -Install counting (`ping`) is the floor for an install that has *not* answered: -it is not an option on the consent prompt, because an ignored prompt must not be -read as a refusal and the project needs an honest count of installs to know what -to support. A refusal, though, is a different thing from silence — an admin who -says no is obeyed, and that answer is written to `config.yml`, so it keeps -holding after an upgrade. To check what your install is doing: +The notice is the moment usage summaries begin for an install that has not +answered, and it is recorded as `noticed_at` in `config.yml` only once it has +actually been delivered — a Telegram message that failed to send, or a dashboard +nobody opened, turns nothing on. A refusal is a different thing from silence — +an admin who says no is obeyed, and that answer is written to `config.yml`, so +it keeps holding after an upgrade; the notice never overrides it, nor an install +that chose "Only count my install". To check what your install is doing: ```bash -# The authoritative answer. Prints "ping" on a default install, and -# "off" when CONDOR_TELEMETRY=off is set or the admin turned reporting off. +# The authoritative answer. Prints "ping" before the notice has been shown, +# "usage" after it, and "off" when CONDOR_TELEMETRY=off is set or the admin +# turned reporting off. uv run python -c "from condor.telemetry import consent; print(consent.level())" ``` Three ways to control it, in order of precedence: 1. **Environment** — `CONDOR_TELEMETRY=off` in your `.env`. The one full kill - switch: it overrides everything, suppresses the prompt, and is the right + switch: it overrides everything, suppresses the notice, and is the right answer for an install that must send nothing at all. 2. **The dashboard** — Settings → Privacy, which every seat can read and only the admin can change - (`GET`/`PUT /api/v1/settings/telemetry?level=ping|usage|off`). This is also - where an install that runs without Telegram is asked in the first place, - since it has no bot to be asked through. Choosing `off` here records a - refusal, which is the durable form of the kill switch: it is stored rather - than read from the environment of one process. + (`GET`/`PUT /api/v1/settings/telemetry?level=ping|usage|off`). An install + that runs without Telegram is shown the notice as a strip across the top of + the dashboard instead, since it has no bot to be told through. Choosing + `off` here — or "Turn off" on the notice — records a refusal, which is the + durable form of the kill switch: it is stored rather than read from the + environment of one process. 3. **`config.yml`** — edit the `telemetry` section directly: ```yaml telemetry: consent: granted # or `denied`, which forces level `off` - level: ping + level: ping # or `usage` ``` Downgrading from `usage` to `ping`, and turning reporting off, are both a @@ -345,7 +354,8 @@ Always, which nobody but that user can do for them. ## Changes to this document Adding anything to the collected list requires a change to `schema.py`, a change -to this file, and re-asking for consent. In particular, adding trading pairs +to this file, and a new notice to every install that has not turned reporting +off. In particular, adding trading pairs would make positions inferable from timing and must not be done quietly. The same applies to sharing, in its own terms. Sending anything a user has not diff --git a/README.md b/README.md index f69974050..9b6ebf0c3 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,10 @@ A Telegram bot for monitoring and trading with Hummingbot via the **Hummingbot A > > Full walkthrough: [Securing Condor and Hummingbot API with Tailscale](https://hummingbot.org/blog/posts/securing-condor-and-hummingbot-api-with-tailscale/) · [Hummingbot API Tailscale guide](https://hummingbot.org/hummingbot-api/tailscale/) -> **Privacy:** Condor counts installs anonymously — a random id, the version, -> and a heartbeat; nothing about you or your trading. Everything beyond that is -> **opt-in** from the one prompt it sends on first boot, and +> **Privacy:** Condor sends anonymous usage stats — a random id, the version, +> which features get used and what breaks; nothing about you or your trading. +> Usage summaries start only after the admin has been shown a notice saying so, +> they can be turned off from that notice or Settings → Privacy, and > `CONDOR_TELEMETRY=off` silences telemetry entirely. Your conversations are > never part of that: the only way one leaves is if you press Share on it and > confirm the redacted transcript, which `CONDOR_SHARING=off` disables outright. diff --git a/condor/telemetry/__init__.py b/condor/telemetry/__init__.py index 8fd202992..5e5114c04 100644 --- a/condor/telemetry/__init__.py +++ b/condor/telemetry/__init__.py @@ -11,9 +11,10 @@ ``ping``: the four adoption events (``install``, ``heartbeat``, ``version_change``, ``shutdown``) and the anonymous envelope, from the first boot, with no answer required. -- **Usage is opt-in, once, by the admin.** One inline-keyboard prompt on boot - offers full usage or install-count-only. The answer is durable and - reversible. +- **Usage is opt-out, after a notice.** The admin is told once — a Telegram + message on boot, or a strip on the dashboard — that usage summaries are on + and where to turn them off. Only a delivered notice turns them on; the answer + ("Got it", "Turn off", or a level in Settings) is durable and reversible. - **Allowlisted.** :mod:`condor.telemetry.schema` declares every event and every property. Anything undeclared is dropped by construction, which is what makes the "never collected" list in ``PRIVACY.md`` a property of the code. diff --git a/condor/telemetry/consent.py b/condor/telemetry/consent.py index e7902d119..307613ae2 100644 --- a/condor/telemetry/consent.py +++ b/condor/telemetry/consent.py @@ -5,19 +5,27 @@ clone), ``granted`` and ``denied``, stored in ``config.yml`` under ``telemetry`` alongside the install's identity. -Three rules matter more than the rest: +Four rules matter more than the rest: -**The floor is ``ping`` — for an install that has not answered.** Every install -that has said nothing is counted: it emits the four adoption events -(``install``, ``heartbeat``, ``version_change``, ``shutdown``) and nothing else. -The prompt decides one thing only — whether the ``usage`` events are added on -top. There is no "off" answer on the form, so silence is never read as refusal. +**The floor is ``ping`` — until the admin has been told.** An install that has +said nothing emits the four adoption events (``install``, ``heartbeat``, +``version_change``, ``shutdown``) and nothing else. That is all an install +sends before anyone could have read what it sends. + +**Usage is the default once the notice has been shown.** The admin is shown one +notice — the Telegram message next to the boot notification, or the dashboard +strip — saying usage summaries are on and where to turn them off. Its delivery +is recorded as ``noticed_at``, and from that moment an unanswered install +resolves to ``usage``. It is an opt-out, and the notice is what makes it one: no +usage event is ever recorded for an install that was never told. "Got it" just +records ``granted`` at ``usage`` so the notice stops showing. **A refusal is honoured, and survives an upgrade.** ``denied`` is not silence: -it is a recorded "no", written by :func:`deny` (the dashboard's off switch, and -older builds' "No thanks" button). It resolves to level ``off``, so an install -that refused under any build stays silent across upgrades and is never re-asked. -Re-enabling is an explicit act — :func:`set_level` with ``ping`` or ``usage``. +it is a recorded "no", written by :func:`deny` (the notice's "Turn off", the +dashboard's off switch, older builds' "No thanks"). It resolves to level +``off``, so an install that refused under any build stays silent across upgrades +and is never re-asked — and the notice cannot flip it, nor an install that chose +``ping``. Re-enabling is an explicit act — :func:`set_level`. **The environment wins.** ``CONDOR_TELEMETRY`` in ``utils/config.py`` overrides the stored answer in both directions: it can silence an install that granted @@ -45,10 +53,9 @@ GRANTED = "granted" DENIED = "denied" -# Answer -> level, the two buttons of the admin prompt. "off" is deliberately -# not one of them: install counting is the floor for an install that has *not* -# answered, so an ignored prompt must not be readable as a refusal. Refusing is -# a separate, explicit act — `deny()`. +# Answer -> level for the grantable answers: the notice's "Got it" and the two +# levels in Settings. "off" is deliberately not one of them — refusing is a +# separate, recorded act, `deny()`, so a stray answer can never be read as one. ANSWER_LEVELS = {"usage": USAGE, "ping": PING} _cached_level: str | None = None @@ -125,12 +132,13 @@ def _compute_level() -> str: stored = _section().get("level") return stored if stored in (PING, USAGE) else USAGE if stored_state == DENIED: - # A recorded "no" outranks the floor. Installs that refused under an - # older build carry this state forward, and an upgrade must not read - # their refusal as an unanswered prompt. + # A recorded "no" outranks everything stored. Installs that refused + # under an older build carry this state forward, and an upgrade must not + # read their refusal as an unanswered notice. return OFF - # Unanswered installs are still counted: ping is the floor. - return PING + # Unanswered: usage once the admin has been told it is on, and until then + # only the ping floor — nothing beyond a count before anyone could know. + return USAGE if notice_shown() else PING def refresh() -> str: @@ -286,7 +294,7 @@ def mark_feature_seen(feature: str) -> bool: """True the first time this install ever uses ``feature``, False after. Backs the ``feature_first_use`` activation funnel. Only called at level - ``usage``, so an install that has not opted in never accumulates the list. + ``usage``, so an install below it never accumulates the list. """ section = _section() seen = list(section.get("features_seen") or []) @@ -297,17 +305,43 @@ def mark_feature_seen(feature: str) -> bool: return True -# ── Prompt bookkeeping ─────────────────────────────────────────────────── +# ── Notice bookkeeping ─────────────────────────────────────────────────── + + +def notice_shown() -> bool: + """Has the admin been shown the notice that usage summaries are on?""" + return bool(_section().get("noticed_at")) + + +def mark_notice_shown() -> bool: + """Record that the notice reached the admin. True if this turned usage on. + + Called only once the notice has actually been delivered — after the + Telegram message was sent, or when the dashboard strip rendered for the + admin — never before, because this is the moment an unanswered install + starts sending usage events. It changes nothing for an install that has + already answered (a ``ping`` or a refusal stays exactly that) or whose level + the environment pins. + """ + if env_overridden() or state() != UNKNOWN or notice_shown(): + return False + _update(noticed_at=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())) + ensure_identity() + return True def should_prompt(version: str = "") -> bool: - """Has this install never been asked (or not since this version)?""" - if env_overridden() or state() != UNKNOWN: + """Is there a notice left to send (and not already tried on this version)?""" + if env_overridden() or state() != UNKNOWN or notice_shown(): return False asked = _section().get("prompted_version") return asked != (version or "unknown") def mark_prompted(version: str = "") -> None: - """Written *before* the prompt is sent, so a crash loop cannot re-ask forever.""" + """Written *before* the notice is sent, so a crash loop cannot re-send forever. + + Deliberately not :func:`mark_notice_shown`: an attempt is not a delivery, + and a send that fails must leave the install at the ping floor. + """ _update(prompted_version=version or "unknown") diff --git a/condor/telemetry/prompt.py b/condor/telemetry/prompt.py index 3a39d2df1..fc1aa13b2 100644 --- a/condor/telemetry/prompt.py +++ b/condor/telemetry/prompt.py @@ -1,16 +1,18 @@ -"""The one-tap consent prompt, and the callback that answers it. - -Opt-in only works if asking is cheap, so this is a single message with two -buttons next to the "Condor is online" notification the admin already gets. It -is sent at most once per version, the intent is written to disk *before* the -message goes out (a crash loop must not re-ask forever), and until it is -answered the install stays at the ``ping`` floor — counted, nothing more. - -Telegram is not the only surface that asks. A local-mode install has no bot to -message, so the dashboard asks instead — ``GET /api/v1/settings/telemetry`` -serves :data:`DISCLOSURE` to the consent card. Both surfaces render the same -copy from the same constant: a privacy claim written down twice is a privacy -claim that will eventually disagree with itself. +"""The telemetry notice, and the callback that answers it. + +A notice, not a question: one message next to the "Condor is online" +notification the admin already gets, saying that anonymous usage summaries are +on, what they contain, and how to turn them off. It has a "Got it" button and a +"Turn off" button. It is sent at most once per version until it is delivered, +the attempt is written to disk *before* the message goes out (a crash loop must +not re-send forever), and only a *delivered* notice moves an unanswered install +from the ``ping`` floor to ``usage`` — see :mod:`condor.telemetry.consent`. + +Telegram is not the only surface that tells. A local-mode install has no bot to +message, so the dashboard shows the same notice — ``GET +/api/v1/settings/telemetry`` serves :data:`DISCLOSURE` to the notice strip. Both +surfaces render the same copy from the same constant: a privacy claim written +down twice is a privacy claim that will eventually disagree with itself. """ from __future__ import annotations @@ -21,25 +23,25 @@ CALLBACK_PREFIX = "telemetry" -# The two answers, in the order both surfaces offer them. `off` is deliberately -# absent — install counting is the floor (see ``consent.ANSWER_LEVELS``). +# The two levels an admin can choose in Settings → Privacy, in the order it +# offers them. The notice itself offers "Got it" (usage) and "Turn off" (a +# recorded refusal, `consent.deny`); `off` is not a level here for that reason. OPTIONS = ( - {"level": "usage", "label": "Yes, share usage summaries"}, + {"level": "usage", "label": "Usage summaries and install count (default)"}, {"level": "ping", "label": "Only count my install"}, ) -# Everything an install is told before it answers. +# Everything an install is told. DISCLOSURE = { - "headline": "Help improve Condor?", - "always_on": ( - "Condor counts installs so the project knows it is used: a random id, " - "the version, and an uptime ping — nothing about you or your trading. " - "That is always on." + "headline": "Condor shares anonymous usage stats", + "summary": ( + "To learn what gets used and what breaks, Condor sends a random install " + "id, its version, which commands and screens are used, errors, and " + "which models agents run. Nothing about you or your trading." ), - "optional": ( - "It can also send an anonymous, allowlisted usage summary: which " - "commands and screens get used, what breaks, and which models agents " - "run. That part is up to you." + "opt_out": ( + "You can turn this off at any time in Settings \u2192 Privacy, or with " + "CONDOR_TELEMETRY=off." ), # A list in the browser, one sentence in Telegram. The last entry is phrased # to close the sentence :func:`_never_line` builds. @@ -56,8 +58,10 @@ ], "doc": ( "Full details in PRIVACY.md at the root of the repo, which also says " - "how to change this answer at any time." + "how to change this at any time." ), + "acknowledge": "Got it", + "turn_off": "Turn off", "options": [dict(option) for option in OPTIONS], } @@ -71,9 +75,9 @@ def _never_line() -> str: _TEXT = "\n\n".join( ( DISCLOSURE["headline"], - DISCLOSURE["always_on"], - DISCLOSURE["optional"], + DISCLOSURE["summary"], _never_line(), + DISCLOSURE["opt_out"], DISCLOSURE["doc"], ) ) @@ -82,21 +86,26 @@ def _never_line() -> str: def keyboard(): from telegram import InlineKeyboardButton, InlineKeyboardMarkup + from condor.telemetry import consent + return InlineKeyboardMarkup( [ [ InlineKeyboardButton( - option["label"], - callback_data=f"{CALLBACK_PREFIX}:{option['level']}", - ) + DISCLOSURE["acknowledge"], + callback_data=f"{CALLBACK_PREFIX}:{consent.USAGE}", + ), + InlineKeyboardButton( + DISCLOSURE["turn_off"], + callback_data=f"{CALLBACK_PREFIX}:{consent.OFF}", + ), ] - for option in OPTIONS ] ) async def maybe_prompt_admin(bot) -> bool: - """Ask the admin once, if there is anything to ask. Never raises.""" + """Tell the admin once, if there is anything to tell. Never raises.""" try: from utils.config import ADMIN_USER_ID @@ -110,20 +119,24 @@ async def maybe_prompt_admin(bot) -> bool: return False # Written first: if sending or the process dies right after, the admin - # gets asked again on the next version, not on the next boot loop. + # is told again on the next version, not on the next boot loop. consent.mark_prompted(version) await bot.send_message( chat_id=int(ADMIN_USER_ID), text=_TEXT, reply_markup=keyboard() ) + # Only a delivered notice turns usage on. A send that raised above + # leaves the install at the ping floor. + consent.mark_notice_shown() return True except Exception: # noqa: BLE001 - log.debug("Could not send the telemetry consent prompt", exc_info=True) + log.debug("Could not send the telemetry notice", exc_info=True) return False async def callback_handler(update, context) -> None: - """Handle ``telemetry:usage|ping|off``. Admin only — it is an install-wide - setting, and the admin owns the install.""" + """Handle ``telemetry:usage|off`` from the notice, and ``ping`` from a + prompt an older build sent. Admin only — it is an install-wide setting, and + the admin owns the install.""" query = update.callback_query try: await query.answer() @@ -145,11 +158,10 @@ async def callback_handler(update, context) -> None: answer = ( (query.data or "").split(":", 1)[1] if ":" in (query.data or "") else "" ) - # An "off" tap can only come from a prompt sent by an older version, - # where the button read "No thanks". That is a refusal, so it is - # recorded as one — the same answer the dashboard's off switch gives — - # rather than being rounded up to the floor. Anything else - # unrecognized still lands on ping via grant(). + # "Turn off" on the notice — or "No thanks" on a prompt an older build + # sent. Either is a refusal, so it is recorded as one — the same answer + # the dashboard's off switch gives — rather than being rounded up to the + # floor. Anything else unrecognized still lands on ping via grant(). if answer == consent.OFF: consent.deny() await query.edit_message_text( @@ -173,10 +185,9 @@ async def callback_handler(update, context) -> None: ) else: await query.edit_message_text( - "Thanks. Condor will send anonymous usage and reliability " - "events. No keys, addresses, pairs, amounts or prompts ever " - "leave this machine. PRIVACY.md says how to change or withdraw " - "this." + "Thanks. Condor sends anonymous usage and reliability events. " + "No keys, addresses, pairs, amounts or prompts ever leave this " + "machine. Settings \u2192 Privacy turns it off at any time." ) except Exception: # noqa: BLE001 log.exception("Telemetry consent callback failed") diff --git a/condor/web/routes/settings.py b/condor/web/routes/settings.py index 11a167716..80bab9c3c 100644 --- a/condor/web/routes/settings.py +++ b/condor/web/routes/settings.py @@ -896,10 +896,37 @@ async def get_telemetry_settings(user: WebUser = Depends(get_current_user)): "pending_events": emitter.buffered(), "privacy_doc": "PRIVACY.md", "can_change": cm.is_admin(user.id), + "notice_shown": consent.notice_shown(), "disclosure": DISCLOSURE, } +@router.post("/telemetry/notice") +async def mark_telemetry_notice_shown(user: WebUser = Depends(get_current_user)): + """Record that the dashboard showed the admin the telemetry notice. + + Posted by the notice strip when it renders. It is the dashboard's delivery + receipt — the equivalent of the Telegram message having been sent — and it + is what moves an unanswered install from the ping floor to ``usage``, so it + counts only for the admin, who is the one being told. It changes nothing + for an install that has already answered or whose level the environment + pins; ``consent.mark_notice_shown`` enforces both. + """ + from condor.telemetry import consent + + if not get_config_manager().is_admin(user.id): + raise HTTPException( + status_code=403, + detail="Telemetry is an install-wide setting; only the admin is notified", + ) + consent.mark_notice_shown() + return { + "level": consent.level(), + "consent": consent.state(), + "notice_shown": consent.notice_shown(), + } + + @router.put("/telemetry") async def set_telemetry_settings( level: str = Query(..., description="ping | usage | off"), @@ -907,8 +934,8 @@ async def set_telemetry_settings( ): """Change the install's telemetry level. Admin only, and reversible. - ``ping`` is the floor for an install that has never answered — silence is - not refusal — but ``off`` is a real answer here, because an admin who wants + An unanswered install is at ``ping`` until it has been shown the notice and + at ``usage`` after — silence is not refusal — but ``off`` is a real answer here, because an admin who wants this install to report nothing must have a way to say so in the product rather than only by editing ``.env``. It is recorded as a refusal in ``config.yml``, so it survives upgrades; ``ping``/``usage`` re-enable. diff --git a/frontend/src/components/TelemetryConsentBanner.tsx b/frontend/src/components/TelemetryConsentBanner.tsx index acc45132c..5f2d0a643 100644 --- a/frontend/src/components/TelemetryConsentBanner.tsx +++ b/frontend/src/components/TelemetryConsentBanner.tsx @@ -1,27 +1,46 @@ import { Loader2, ShieldCheck } from "lucide-react"; +import { useEffect, useRef } from "react"; import { Link } from "react-router-dom"; -import { shouldAskConsent, useSetTelemetryLevel, useTelemetry } from "@/hooks/useTelemetry"; -import type { TelemetryLevel } from "@/lib/api"; +import { + shouldAskConsent, + useMarkTelemetryNoticeShown, + useSetTelemetryLevel, + useTelemetry, +} from "@/hooks/useTelemetry"; /** - * The dashboard's half of the consent prompt. + * The dashboard's half of the telemetry notice. * - * A Telegram install gets asked once, next to the boot notification, because - * that is the one moment the admin is already looking. A local-mode install has - * no bot to be asked through, so the equivalent moment is the first dashboard - * it opens — which is this strip. + * A Telegram install is told once, next to the boot notification, because that + * is the one moment the admin is already looking. A local-mode install has no + * bot to be told through, so the equivalent moment is the first dashboard it + * opens — which is this strip. * - * There is no dismiss, because both buttons are answers and one click retires - * it forever; a "later" would only add a way to be asked again. It renders for - * nobody else: not for a non-admin seat, not once answered on either surface, - * and not when `CONDOR_TELEMETRY` has already decided. + * It is a notice, not a question: usage summaries are on, here is what they + * are, here is where to turn them off. Rendering it is the delivery, so the + * strip posts its own receipt on mount — that, not the click, is what moves an + * unanswered install off the ping floor. "Got it" only records the answer so + * the strip retires. It renders for nobody else: not for a non-admin seat, not + * once answered on either surface, and not when `CONDOR_TELEMETRY` has already + * decided. */ export function TelemetryConsentBanner() { const { data } = useTelemetry(); - const mutation = useSetTelemetryLevel(); + const setLevel = useSetTelemetryLevel(); + const markShown = useMarkTelemetryNoticeShown(); + const receiptSent = useRef(false); - if (!shouldAskConsent(data) || !data) return null; + const visible = shouldAskConsent(data); + const needsReceipt = visible && !data?.notice_shown; + + useEffect(() => { + if (!needsReceipt || receiptSent.current) return; + receiptSent.current = true; + markShown.mutate(); + }, [needsReceipt, markShown]); + + if (!visible || !data) return null; const { disclosure } = data; @@ -32,39 +51,32 @@ export function TelemetryConsentBanner() { {/* `min-w-0` so the copy wraps inside its own column instead of pushing the buttons onto a row of their own. */}

- {disclosure.headline}{" "} - {disclosure.optional}{" "} + {disclosure.headline}.{" "} + {disclosure.summary}{" "} - What is never sent + Turn off or see details

- {mutation.isPending && ( + {setLevel.isPending && ( )} - {disclosure.options.map((option, index) => ( - - ))} +
- {mutation.isError && ( + {setLevel.isError && (

- Could not save that. {(mutation.error as Error).message} + Could not save that. {(setLevel.error as Error).message}

)}
diff --git a/frontend/src/components/settings/TelemetrySettings.tsx b/frontend/src/components/settings/TelemetrySettings.tsx index 26ef2ac56..cc69e2ea0 100644 --- a/frontend/src/components/settings/TelemetrySettings.tsx +++ b/frontend/src/components/settings/TelemetrySettings.tsx @@ -24,10 +24,10 @@ export function TelemetrySettings() { const { disclosure, level, consent, env_overridden, can_change } = data; const locked = env_overridden || !can_change; - // At `unknown` no option is the answer yet, so none is shown as chosen — - // the install is at the ping floor, but nobody has said so. `denied` is an - // answer, and the one it selects is the off row below. - const chosen = consent === "granted" ? level : consent === "denied" ? "off" : null; + // Unanswered installs still have a real level — `ping` until the admin has + // been shown the notice, `usage` after — so the row that is in effect is the + // one shown as chosen. `denied` selects the off row below. + const chosen = consent === "denied" ? "off" : level; const offChosen = chosen === "off"; return ( @@ -38,8 +38,7 @@ export function TelemetrySettings() { {disclosure.headline}
-

{disclosure.always_on}

-

{disclosure.optional}

+

{disclosure.summary}

@@ -94,10 +93,9 @@ export function TelemetrySettings() { ); })} - {/* Not one of `disclosure.options`: the consent form has no "off" - button, because ignoring a prompt must not read as a refusal. - Saying no out loud is a different act, and it needs somewhere to - be said other than the operator's `.env`. */} + {/* Not one of `disclosure.options`: "off" is not a level but a + recorded refusal (`consent.deny`), the same answer the notice's + "Turn off" gives, and it survives upgrades. */}
- {/* Tab bar */} -
- {tabs.map((t) => ( - + {/* Narrow screens: one select, grouped the same way as the sidebar. */} + - {/* Tab content */} - {tab === "servers" && } - {tab === "gateway" && } - {tab === "keys" && } - {tab === "llm" && } - {tab === "voice" && } - {tab === "notifications" && } - {/* Two cards, not one switch. Telemetry is anonymous counts the admin - consents to install-wide; sharing is content only its author can hand - over. They are different promises, and merging the controls would - misrepresent one of them — the divider is where the copy says so. */} - {tab === "privacy" && ( -
- -
- -
+
+ + +
+ {/* Tab content */} + {tab === "servers" && } + {tab === "gateway" && } + {tab === "keys" && } + {tab === "llm" && } + {tab === "voice" && } + {tab === "notifications" && } + {/* Two cards, not one switch. Telemetry is anonymous counts the admin + consents to install-wide; sharing is content only its author can hand + over. They are different promises, and merging the controls would + misrepresent one of them — the divider is where the copy says so. */} + {tab === "privacy" && ( +
+ +
+ +
+
+ )} + {tab === "admin" && } + {tab === "updates" && }
- )} - {tab === "admin" && } - {tab === "updates" && } +
); } From fef62f7d660e40f87322a74b1c95a018e644a063 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 16 Sep 2026 15:46:40 +0300 Subject: [PATCH 151/154] Give the pydantic-ai image-order test a real client and a complete run stand-in: it built the client with __new__ and a _Run without result/usage()/new_messages(), so every turn in it raised into prompt_stream's error branch and passed only because it checks what agent.iter received; once 39cb7d50 made that branch read _mcp_servers, the missing attribute failed CI on PR #240. The client now goes through __init__, the stand-in run finishes cleanly, and the test requires both turns to end with end_turn --- tests/runtime/test_images_funnel.py | 34 +++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/tests/runtime/test_images_funnel.py b/tests/runtime/test_images_funnel.py index 3ba1e8233..c3f7aba1c 100644 --- a/tests/runtime/test_images_funnel.py +++ b/tests/runtime/test_images_funnel.py @@ -203,12 +203,18 @@ def test_the_pydantic_ai_prompt_is_a_list_with_the_image_first(): import contextlib from pydantic_ai.messages import BinaryContent + from pydantic_ai.usage import RunUsage from condor.acp import pydantic_ai_client as pac seen: list = [] class _Run: + # A finished run with nothing to say: enough of AgentRun for the turn to + # end cleanly, so the test can require end_turn rather than pass through + # the error branch. + result = None + def __aiter__(self): async def _empty(): return @@ -216,29 +222,39 @@ async def _empty(): return _empty() + def usage(self): + return RunUsage() + + def new_messages(self): + return [] + class _Agent: @contextlib.asynccontextmanager async def iter(self, prompt, **kwargs): seen.append(prompt) yield _Run() - client = pac.PydanticAIClient.__new__(pac.PydanticAIClient) + from condor.acp.client import PromptDone + + client = pac.PydanticAIClient(model="openai:gpt-4o") client._agent = _Agent() - client._request_semaphore = None - client._abort_requested = False - client._message_history = [] - client._permission_gate = type("G", (), {"reset": lambda self: None})() + + stop_reasons: list[str] = [] async def scenario(): - async for _ in client.prompt_stream( + async for event in client.prompt_stream( "read this", images=[PromptImage(data=PNG, mime="image/png")] ): - pass - async for _ in client.prompt_stream("and this?"): - pass + if isinstance(event, PromptDone): + stop_reasons.append(event.stop_reason) + async for event in client.prompt_stream("and this?"): + if isinstance(event, PromptDone): + stop_reasons.append(event.stop_reason) asyncio.run(scenario()) + assert stop_reasons == ["end_turn", "end_turn"] + assert seen[0] == [BinaryContent(data=PNG, media_type="image/png"), "read this"] assert seen[1] == "and this?", "a text-only turn stays a bare string" From 376214faeb5fef17bdcccfda8ae8b0035cb3f66e Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 16 Sep 2026 16:20:25 +0300 Subject: [PATCH 152/154] Merge an excluded agent into its local home instead of leaving it in the shipped library: the v2 split moves an agent's store into .condor/agents// before it moves untracked agents whole, so that move found its destination already there and skipped the agent, whose definition stayed under agents/ where the layering reads it as stock; on the maintainer's install brigado's strategies were refused as "ships with Condor" and could not be deleted. The whole-agent step now merges into an existing local home (a file only the leftover has moves, an identical one is dropped, a differing one keeps the local copy the product was already reading and sets the leftover aside under .condor/migration-backups/), and a v4 marker reruns that step once on installs whose v2 already skipped --- condor/migrations.py | 88 +++++++++++++++++++++++++++- tests/test_agents_split_migration.py | 66 +++++++++++++++++++++ 2 files changed, 151 insertions(+), 3 deletions(-) diff --git a/condor/migrations.py b/condor/migrations.py index aeb2cd0a5..c0d51a015 100644 --- a/condor/migrations.py +++ b/condor/migrations.py @@ -67,6 +67,20 @@ ``.condor/agents//delegations//``, in the per-task shape the readers already use. Ownerless is not worthless: these are full transcripts, tens of kilobytes of real output apiece. + +**v4: the untracked agents v2 skipped.** v2's step 1 runs first and moves an +excluded agent's ``store/`` and ``proposals/`` into ``.condor/agents//``, +so by step 2 that destination exists and the whole-directory move declined to +go over it. The agent's definition stayed in ``agents/``, where the layering +reads it as stock: its strategies could not be deleted ("ships with Condor") +and every file the product wrote for it forked down beside a stale copy. + +v4 runs step 2 again, and step 2 now merges into a local home that already +exists instead of skipping it. A file only the leftover has moves; one both +sides hold identically is dropped; one they hold differently keeps the local +copy — the one the product has been reading all along, since local shadows +stock — and sets the leftover aside under +``.condor/migration-backups/agents//`` rather than deleting it. """ from __future__ import annotations @@ -94,6 +108,8 @@ MARKER_FILENAME = ".migrated-v1" MARKER_V2_FILENAME = ".migrated-v2" MARKER_V3_FILENAME = ".migrated-v3" +MARKER_V4_FILENAME = ".migrated-v4" +BACKUPS_DIRNAME = "migration-backups" # Runtime output that lived under a *tracked* agent directory. Everything here # was already gitignored, so step 1 is a move with no git in it. @@ -131,6 +147,8 @@ class MigrationReport: agent_forks: int = 0 # v3 stranded_delegations: int = 0 + # v4: leftover copies that lost to a differing local file, set aside + agent_backups: int = 0 @property def total(self) -> int: @@ -143,6 +161,7 @@ def total(self) -> int: + self.agent_dirs + self.agent_forks + self.stranded_delegations + + self.agent_backups ) @@ -188,10 +207,18 @@ def ensure_migrated(agents_root: Path | None = None) -> MigrationReport: return report _write_marker(root, MARKER_V3_FILENAME, "FEAT-115") + if not (root / MARKER_V4_FILENAME).is_file(): + try: + _merge_leftover_agents(report, source) + except Exception: # noqa: BLE001 - same rule: never block a boot + log.exception("Leftover agent merge failed; leaving the tree in place") + return report + _write_marker(root, MARKER_V4_FILENAME, "FEAT-115") + if report.total or report.dropped_stubs: log.warning( "Runtime migrated to %s: %d conversations, %d delegations, " - "%d stranded delegations, " + "%d stranded delegations, %d set-aside agent files, " "%d state namespaces, %d telemetry files, %d agent artefacts, " "%d agent directories, %d hoisted forks " "(%d empty conversation stubs dropped, %d already present)", @@ -199,6 +226,7 @@ def ensure_migrated(agents_root: Path | None = None) -> MigrationReport: report.conversations, report.delegations, report.stranded_delegations, + report.agent_backups, report.state, report.telemetry, report.agent_artefacts, @@ -519,17 +547,71 @@ def _move_untracked_agents( go stale rather than wrong once the directory is gone; removing them is a manual tidy-up, not something a boot migration should do to someone's ``.git/``. + + Step 1 has usually created ``/`` by now (the agent's store), so + a local home that already exists is merged into, never a reason to skip. """ for agent_dir in _agent_dirs(stock_root): slug = agent_dir.name tracked = _git(repo_dir, "ls-files", "--", f"{prefix}/{slug}") if tracked is None or tracked.strip(): continue # git knows it: stock, or unreadable — either way, leave it - if _move(agent_dir, local_root / slug): + destination = local_root / slug + if _move(agent_dir, destination): report.agent_dirs += 1 log.info("Agent split: %s was untracked and is now this install's", slug) - else: + continue + backups = paths.runtime_root() / BACKUPS_DIRNAME / prefix / slug + _merge_tree(report, agent_dir, destination, backups) + if agent_dir.exists(): report.skipped += 1 + log.warning("Agent split: %s could not be emptied; see above", agent_dir) + else: + report.agent_dirs += 1 + log.info( + "Agent split: %s was untracked and is merged into %s", slug, destination + ) + + +def _merge_leftover_agents(report: MigrationReport, stock_root: Path) -> None: + """v4: step 2 again, for the installs whose v2 skipped an existing home.""" + repo_dir = stock_root.parent + if not stock_root.is_dir() or not (repo_dir / ".git").exists(): + return + _move_untracked_agents( + report, stock_root, paths.local_agents_root(), repo_dir, stock_root.name + ) + + +def _merge_tree(report: MigrationReport, src: Path, dst: Path, backups: Path) -> None: + """Empty ``src`` into ``dst``; the local side wins every disagreement. + + ``dst`` is what the product has been reading (local shadows stock), so a + differing leftover is set aside under ``backups`` rather than overwriting + it — and rather than being deleted, since nobody has looked at it. + ``__pycache__`` is regenerated output and is simply dropped. + """ + for child in sorted(src.iterdir()): + target = dst / child.name + if child.name == "__pycache__": + shutil.rmtree(child, ignore_errors=True) + elif _move(child, target): + pass + elif child.is_dir() and target.is_dir(): + _merge_tree(report, child, target, backups / child.name) + elif child.is_file() and target.is_file() and _same_bytes(child, target): + child.unlink() + elif _move(child, backups / child.name): + report.agent_backups += 1 + log.info("Agent split: kept %s, set %s aside", target, child) + _prune_if_empty(src) + + +def _same_bytes(a: Path, b: Path) -> bool: + try: + return a.read_bytes() == b.read_bytes() + except OSError: + return False def _hoist_modified_files( diff --git a/tests/test_agents_split_migration.py b/tests/test_agents_split_migration.py index 426ec3f4c..c2fd2a0c6 100644 --- a/tests/test_agents_split_migration.py +++ b/tests/test_agents_split_migration.py @@ -24,6 +24,7 @@ MARKER_FILENAME, MARKER_V2_FILENAME, MARKER_V3_FILENAME, + MARKER_V4_FILENAME, ensure_migrated, ) @@ -125,6 +126,71 @@ def test_an_excluded_agent_moves_whole_and_is_still_listed(repo): assert _dirty(repo) == "" +def test_an_excluded_agent_with_a_store_still_leaves_the_library(repo): + """Step 1 moves the store first, so step 2 finds a local home already there. + + It used to take that as "never overwrite" and skip the agent, leaving its + definition in ``agents/`` where the layering calls it shipped — and every + strategy under it undeletable. + """ + from condor.layering import resolves_to_stock + + agents = repo / "agents" + _write(agents / "brigado" / "AGENT.md", "---\nname: Brigado\n---\n\nBRL.\n") + _write(agents / "brigado" / "store" / "user_7" / "audit.log", "ran") + _write(agents / "brigado" / "strategies" / "fleet" / "strategy.md", "---\n---\n") + _write(repo / ".git" / "info" / "exclude", "agents/brigado/\n") + + report = ensure_migrated() + + local = paths.local_agents_root() / "brigado" + assert report.agent_dirs == 1 + assert not (agents / "brigado").exists() + assert (local / "AGENT.md").exists() + assert (local / "store" / "user_7" / "audit.log").read_text() == "ran" + assert not resolves_to_stock("brigado", "strategies", "fleet", "strategy.md") + + +def test_v4_merges_a_leftover_the_local_copy_winning(repo): + """A box whose v2 already skipped the agent: v4 finishes the job. + + Where both sides hold a file and disagree, the local one is what the product + has been reading, so it stays; the leftover is set aside, not deleted. + """ + agents = repo / "agents" + local = paths.local_agents_root() / "brigado" + runtime = paths.runtime_root() + for marker in (MARKER_FILENAME, MARKER_V2_FILENAME, MARKER_V3_FILENAME): + _write(runtime / marker, "done\n") + _write(repo / ".git" / "info" / "exclude", "agents/brigado/\n") + _write(agents / "brigado" / "AGENT.md", "old") + _write(agents / "brigado" / "skills" / "lp" / "SKILL.md", "same") + _write(agents / "brigado" / "routines" / "r.py", "only in the leftover") + _write(agents / "brigado" / "routines" / "__pycache__" / "r.pyc", "bytecode") + _write(agents / "brigado" / "strategies" / "fleet" / "strategy.md", "fleet") + _write(local / "AGENT.md", "edited") + _write(local / "skills" / "lp" / "SKILL.md", "same") + _write(local / "store" / "user_7" / "audit.log", "ran") + _write(local / "routines" / "__pycache__" / "r.pyc", "newer bytecode") + + report = ensure_migrated() + + assert not (agents / "brigado").exists() + assert (local / "AGENT.md").read_text() == "edited" + assert (local / "skills" / "lp" / "SKILL.md").read_text() == "same" + assert (local / "routines" / "r.py").read_text() == "only in the leftover" + assert (local / "strategies" / "fleet" / "strategy.md").exists() + # Regenerated output is dropped, never set aside as a "disagreement". + assert ( + local / "routines" / "__pycache__" / "r.pyc" + ).read_text() == "newer bytecode" + backup = runtime / "migration-backups" / "agents" / "brigado" / "AGENT.md" + assert backup.read_text() == "old" + assert report.agent_backups == 1 and report.agent_dirs == 1 + assert (runtime / MARKER_V4_FILENAME).is_file() + assert ensure_migrated().total == 0 + + def test_a_tracked_agent_is_left_where_it_is(repo): ensure_migrated() From 51f13d66a616f146a975642f4f4e25e6c06f3379 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 16 Sep 2026 16:32:49 +0300 Subject: [PATCH 153/154] Warn at startup when an inherited environment variable overrides .env: load_dotenv never overrides a set variable, so a tmux server started by another project handed its TELEGRAM_TOKEN to the Condor pane and Condor silently polled as that project's bot; main() now logs each .env key whose inherited value differs, naming both bot ids for TELEGRAM_TOKEN and never a secret --- main.py | 2 + tests/test_env_shadowing_warning.py | 53 ++++++++++++++++++++++++++ utils/config.py | 58 +++++++++++++++++++++++++++-- 3 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 tests/test_env_shadowing_warning.py diff --git a/main.py b/main.py index c5f73b37c..0b2d3b1f8 100644 --- a/main.py +++ b/main.py @@ -34,6 +34,7 @@ ConfigError, check_local_user, check_startup_config, + warn_shadowed_env, ) # Enable logging @@ -998,6 +999,7 @@ def main() -> None: except ConfigError as exc: logger.error("%s", exc) raise SystemExit(1) + warn_shadowed_env() # Reap any ACP/MCP subprocess trees orphaned by a prior hard kill (kill -9, # OOM, power loss) before we spawn our own — those bypass teardown(). diff --git a/tests/test_env_shadowing_warning.py b/tests/test_env_shadowing_warning.py new file mode 100644 index 000000000..519f7d0ae --- /dev/null +++ b/tests/test_env_shadowing_warning.py @@ -0,0 +1,53 @@ +"""An inherited variable overriding `.env` has to be audible. + +`load_dotenv` never overrides a set variable — the right precedence for a deploy +that injects its own environment. It went wrong on a dev machine: a tmux server +started by another project handed its `TELEGRAM_TOKEN` to the Condor pane, and +Condor polled as that project's bot. `/web` got no answer, the log had no error, +and Telegram held the messages for a bot nothing was polling. +""" + +import logging + +from utils.config import shadowed_env_keys, warn_shadowed_env + + +def test_a_differing_inherited_value_is_shadowing(): + assert shadowed_env_keys({"TELEGRAM_TOKEN": "1:a"}, {"TELEGRAM_TOKEN": "2:b"}) == [ + "TELEGRAM_TOKEN" + ] + + +def test_an_equal_value_is_not_shadowing(): + # A deploy that loads the same .env into the environment is not a conflict. + assert shadowed_env_keys({"TELEGRAM_TOKEN": "1:a"}, {"TELEGRAM_TOKEN": "1:a"}) == [] + + +def test_unset_and_valueless_keys_are_not_shadowing(): + assert shadowed_env_keys({"WEB_URL": "http://x"}, {}) == [] + assert shadowed_env_keys({"FOO": None}, {"FOO": "bar"}) == [] + assert shadowed_env_keys({}, {"PATH": "/usr/bin"}) == [] + + +def test_the_warning_names_both_bots_and_no_secret(caplog): + env = {"TELEGRAM_TOKEN": "6451353778:inherited-secret", "OPENAI_API_KEY": "sk-a"} + file_values = {"TELEGRAM_TOKEN": "8548509697:file-secret", "OPENAI_API_KEY": "sk-b"} + + with caplog.at_level(logging.WARNING, logger="utils.config"): + shadowed = warn_shadowed_env(env, file_values) + + assert shadowed == ["OPENAI_API_KEY", "TELEGRAM_TOKEN"] + text = caplog.text + assert "running as bot 6451353778, .env names bot 8548509697" in text + assert "OPENAI_API_KEY" in text + for secret in ("inherited-secret", "file-secret", "sk-a", "sk-b"): + assert secret not in text + + +def test_nothing_is_logged_when_env_agrees_with_the_file(caplog): + with caplog.at_level(logging.WARNING, logger="utils.config"): + assert ( + warn_shadowed_env({"TELEGRAM_TOKEN": "1:a"}, {"TELEGRAM_TOKEN": "1:a"}) + == [] + ) + assert caplog.text == "" diff --git a/utils/config.py b/utils/config.py index bd9b5986e..c8263babb 100644 --- a/utils/config.py +++ b/utils/config.py @@ -1,10 +1,14 @@ +import logging import os -from typing import Optional +from typing import Mapping, Optional from urllib.parse import urlparse -from dotenv import load_dotenv +from dotenv import dotenv_values, find_dotenv, load_dotenv -load_dotenv() +logger = logging.getLogger(__name__) + +_DOTENV_PATH = find_dotenv() +load_dotenv(_DOTENV_PATH) class ConfigError(RuntimeError): @@ -201,6 +205,54 @@ def check_startup_config(env=None) -> None: ) +def shadowed_env_keys( + file_values: Mapping[str, Optional[str]], env: Mapping[str, str] +) -> list[str]: + """The keys ``.env`` sets whose value in *env* is a different one. + + ``load_dotenv`` never overrides a variable that is already set, which is the + right precedence for a deploy that injects its own environment. The trouble + is a variable nobody meant to set: a tmux server started by another project + hands every pane that project's ``TELEGRAM_TOKEN``, and Condor then polls as + that project's bot, sees no message sent to its own, and logs nothing. + """ + return sorted( + key + for key, value in file_values.items() + if value is not None and key in env and env[key] != value + ) + + +def warn_shadowed_env(env=None, file_values=None) -> list[str]: + """Log each ``.env`` key the inherited environment overrides. Never a value. + + ``TELEGRAM_TOKEN`` also names the bot on both sides: the id before the + ``:`` is public (``getMe`` returns it), and it is the difference between + "some variable differs" and "you are running as another bot". + """ + env = os.environ if env is None else env + if file_values is None: + file_values = dotenv_values(_DOTENV_PATH) if _DOTENV_PATH else {} + shadowed = shadowed_env_keys(file_values, env) + for key in shadowed: + detail = "" + if key == "TELEGRAM_TOKEN": + detail = ( + f" — running as bot {env[key].split(':', 1)[0]}, " + f".env names bot {file_values[key].split(':', 1)[0]}" + ) + logger.warning( + "%s is set in the environment Condor was started from and overrides " + ".env%s. If .env is meant to win, start Condor where it is unset " + "(`unset %s`, or `tmux set-environment -gu %s` for new tmux windows).", + key, + detail, + key, + key, + ) + return shadowed + + def check_local_user(env=None, get_role=None) -> None: """In local mode, refuse to start unless the user it logs in as exists. From 1193c2847547c991cbf65a83bac7bb944852dfb4 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 16 Sep 2026 16:32:49 +0300 Subject: [PATCH 154/154] Give an agent row in the execution dock its totals under the Vol, Real., Unreal. and Net columns instead of spanning the whole table, hide the executor count when a fleet runs none, and label icon-only column headings with aria-label so copying the table no longer pastes their hidden hint text --- .../src/components/chat/DockExecution.tsx | 217 ++++++++++-------- 1 file changed, 122 insertions(+), 95 deletions(-) diff --git a/frontend/src/components/chat/DockExecution.tsx b/frontend/src/components/chat/DockExecution.tsx index 0424822eb..5add71005 100644 --- a/frontend/src/components/chat/DockExecution.tsx +++ b/frontend/src/components/chat/DockExecution.tsx @@ -371,17 +371,22 @@ export function DockExecution({ · {paused} paused )} - - {liveExecutors} executor{liveExecutors === 1 ? "" : "s"} - + {/* Only when there are some: a PMM fleet runs no executors at all, and + a standing "0 executors" beside the controller count reads as a + fault rather than as the shape of the strategy. */} + {liveExecutors > 0 && ( + + {liveExecutors} executor{liveExecutors === 1 ? "" : "s"} + + )}
{/* A table, not a list of cards. Every controller answers the same eight questions, and eight answers per row only stay comparable when they - are in eight columns. The agent and bot rows above them span the whole - table instead: they are headings with a fold attached, not eight more - numbers, and giving them the controller's columns would have put an - agent's total under a heading that says "Controller". It fits whatever + are in eight columns. A bot row spans the whole table: it is a heading + with a fold attached. An agent row puts its name across the first four + columns and its totals under Vol, Real., Unreal. and Net, so every sum + is labelled by the heading above it. The table fits whatever width the panel was dragged to and only scrolls sideways below `TABLE_MIN_PX`, where the controller name would otherwise be squeezed out of legibility. */} @@ -407,13 +412,17 @@ export function DockExecution({ key={col.key} scope="col" title={col.hint} + // An `aria-label` rather than an `sr-only` child: visually + // hidden text is still selected text, so copying the table + // pasted "Pause or start the controller" as a heading. + aria-label={col.label ? undefined : col.hint} className={`px-1.5 pb-1 font-medium ${ col.num ? "text-right" : "text-left" } ${col.key === "controller" ? "pl-3" : ""} ${ col.key === "act" ? "pr-3" : "" }`} > - {col.label || {col.hint}} + {col.label} ))} @@ -431,27 +440,25 @@ export function DockExecution({ symbol={currencySymbol} onOpen={(scope) => navigate(`/bots?scope=${scope}`)} /> + ) : row.kind === "agent" ? ( + toggle(row.id)} + onOpenAgent={onOpenAgent} + /> ) : ( - {row.kind === "agent" ? ( - toggle(row.id)} - onOpenAgent={onOpenAgent} - /> - ) : ( - toggle(row.id)} - onOpen={() => navigate(`/bots?scope=bot:${row.label}`)} - /> - )} + toggle(row.id)} + onOpen={() => navigate(`/bots?scope=bot:${row.label}`)} + /> )} @@ -569,79 +576,99 @@ function AgentRow({ else navigate(`/agents/${encodeURIComponent(slug)}`); }; - return ( -
-
- - - - {formatCurrencyPnl(row.totals.net, symbol)} - - - {formatCompactVolume(row.totals.volume, symbol)} - -
+ /** + * The agent's totals, each under the heading that names it. + * + * They used to float at the end of the name line as two bare figures — a + * signed one and a compact one — and nothing on screen said the first was + * net PnL and the second volume. In the controllers' own columns the heading + * says it, and the sum sits directly above the rows it adds up. + */ + const total = (value: number, what: string, pnl: boolean) => ( + + {pnl ? formatCurrencyPnl(value, symbol) : formatCompactVolume(value, symbol)} + + ); - {live && ( -
- {due !== null && ( + return ( + + {/* Controller, pair, exec and up: the name and its liveness line take + the columns an owner has no single value for. */} + +
+ +
- )} -
+ + {live && ( +
+ {due !== null && ( + + {tickCountdownLabel(due)} + + )} + {live.lastDid ? ( + + {live.lastDid.summary} + + ) : ( + live.lastSaid && ( + + {live.lastSaid} + + ) + )} +
+ )} + + {total(row.totals.volume, "volume traded", false)} + {total(row.totals.realized, "realized PnL", true)} + {total(row.totals.unrealized, "unrealized PnL", true)} + {total(row.totals.net, "net PnL", true)} + + ); }