perf(routing): cache lowercased routing strings to prevent redundant allocations

Author: badMade

.jules/bolt.md

@@ -0,0 +1,3 @@
+## 2024-04-12 - Python String Allocation in Tight Loops
+**Learning:** In the Python port's routing engine (`src/runtime.py`), frequent string operations inside tight nested loops (e.g., lowercasing strings and allocating arrays like `[module.name.lower(), module.source_hint.lower(), module.responsibility.lower()]` inside the `_score` method iteration) cause massive performance overhead due to Python's redundant string allocations.
+**Action:** Use `functools.cached_property` on domain objects (like `PortingModule.search_text`) to lazily precompute and cache concatenated/lowercased search strings. Separating them by null bytes (`\0`) prevents overlapping matches, resulting in significantly faster simple subset checks `if token in haystack` compared to looping over an array of dynamically allocated strings.

.port_sessions/532325c073444fdbaf1c22efa19d7951.json

@@ -0,0 +1,9 @@
+{
+  "session_id": "532325c073444fdbaf1c22efa19d7951",
+  "messages": [
+    "review MCP tool",
+    "review MCP tool"
+  ],
+  "input_tokens": 6,
+  "output_tokens": 32
+}

.port_sessions/83d95a0b8cd24e768b59f4a7c0f27c20.json

@@ -0,0 +1,9 @@
+{
+  "session_id": "83d95a0b8cd24e768b59f4a7c0f27c20",
+  "messages": [
+    "review MCP tool",
+    "review MCP tool"
+  ],
+  "input_tokens": 6,
+  "output_tokens": 32
+}

.port_sessions/a9685714a0964d8d82b2ea33e8d54b79.json

@@ -0,0 +1,8 @@
+{
+  "session_id": "a9685714a0964d8d82b2ea33e8d54b79",
+  "messages": [
+    "review MCP tool"
+  ],
+  "input_tokens": 3,
+  "output_tokens": 13
+}

.port_sessions/ae9bc08fb15840e79f044165f37c84ff.json

@@ -0,0 +1,9 @@
+{
+  "session_id": "ae9bc08fb15840e79f044165f37c84ff",
+  "messages": [
+    "review MCP tool",
+    "review MCP tool"
+  ],
+  "input_tokens": 6,
+  "output_tokens": 32
+}

src/models.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
+from functools import cached_property
 
 
 @dataclass(frozen=True)
@@ -16,7 +17,12 @@ class PortingModule:
     name: str
     responsibility: str
     source_hint: str
-    status: str = 'planned'
+    status: str = "planned"
+
+    @cached_property
+    def search_text(self) -> str:
+        # ⚡ Bolt: Cache lowercased concatenated strings to avoid redundant string allocations in routing loops
+        return f"{self.name}\0{self.source_hint}\0{self.responsibility}".lower()
 
 
 @dataclass(frozen=True)
@@ -30,7 +36,7 @@ class UsageSummary:
     input_tokens: int = 0
     output_tokens: int = 0
 
-    def add_turn(self, prompt: str, output: str) -> 'UsageSummary':
+    def add_turn(self, prompt: str, output: str) -> "UsageSummary":
         return UsageSummary(
             input_tokens=self.input_tokens + len(prompt.split()),
             output_tokens=self.output_tokens + len(output.split()),
@@ -44,6 +50,6 @@ class PortingBacklog:
 
     def summary_lines(self) -> list[str]:
         return [
-            f'- {module.name} [{module.status}] — {module.responsibility} (from {module.source_hint})'
+            f"- {module.name} [{module.status}] — {module.responsibility} (from {module.source_hint})"
             for module in self.modules
         ]

src/runtime.py

@@ -38,64 +38,70 @@ class RuntimeSession:
 
     def as_markdown(self) -> str:
         lines = [
-            '# Runtime Session',
-            '',
-            f'Prompt: {self.prompt}',
-            '',
-            '## Context',
+            "# Runtime Session",
+            "",
+            f"Prompt: {self.prompt}",
+            "",
+            "## Context",
             render_context(self.context),
-            '',
-            '## Setup',
-            f'- Python: {self.setup.python_version} ({self.setup.implementation})',
-            f'- Platform: {self.setup.platform_name}',
-            f'- Test command: {self.setup.test_command}',
-            '',
-            '## Startup Steps',
-            *(f'- {step}' for step in self.setup.startup_steps()),
-            '',
-            '## System Init',
+            "",
+            "## Setup",
+            f"- Python: {self.setup.python_version} ({self.setup.implementation})",
+            f"- Platform: {self.setup.platform_name}",
+            f"- Test command: {self.setup.test_command}",
+            "",
+            "## Startup Steps",
+            *(f"- {step}" for step in self.setup.startup_steps()),
+            "",
+            "## System Init",
             self.system_init_message,
-            '',
-            '## Routed Matches',
+            "",
+            "## Routed Matches",
         ]
         if self.routed_matches:
             lines.extend(
-                f'- [{match.kind}] {match.name} ({match.score}) — {match.source_hint}'
+                f"- [{match.kind}] {match.name} ({match.score}) — {match.source_hint}"
                 for match in self.routed_matches
             )
         else:
-            lines.append('- none')
-        lines.extend([
-            '',
-            '## Command Execution',
-            *(self.command_execution_messages or ('none',)),
-            '',
-            '## Tool Execution',
-            *(self.tool_execution_messages or ('none',)),
-            '',
-            '## Stream Events',
-            *(f"- {event['type']}: {event}" for event in self.stream_events),
-            '',
-            '## Turn Result',
-            self.turn_result.output,
-            '',
-            f'Persisted session path: {self.persisted_session_path}',
-            '',
-            self.history.as_markdown(),
-        ])
-        return '\n'.join(lines)
+            lines.append("- none")
+        lines.extend(
+            [
+                "",
+                "## Command Execution",
+                *(self.command_execution_messages or ("none",)),
+                "",
+                "## Tool Execution",
+                *(self.tool_execution_messages or ("none",)),
+                "",
+                "## Stream Events",
+                *(f"- {event['type']}: {event}" for event in self.stream_events),
+                "",
+                "## Turn Result",
+                self.turn_result.output,
+                "",
+                f"Persisted session path: {self.persisted_session_path}",
+                "",
+                self.history.as_markdown(),
+            ]
+        )
+        return "\n".join(lines)
 
 
 class PortRuntime:
     def route_prompt(self, prompt: str, limit: int = 5) -> list[RoutedMatch]:
-        tokens = {token.lower() for token in prompt.replace('/', ' ').replace('-', ' ').split() if token}
+        tokens = {
+            token.lower()
+            for token in prompt.replace("/", " ").replace("-", " ").split()
+            if token
+        }
         by_kind = {
-            'command': self._collect_matches(tokens, PORTED_COMMANDS, 'command'),
-            'tool': self._collect_matches(tokens, PORTED_TOOLS, 'tool'),
+            "command": self._collect_matches(tokens, PORTED_COMMANDS, "command"),
+            "tool": self._collect_matches(tokens, PORTED_TOOLS, "tool"),
         }
 
         selected: list[RoutedMatch] = []
-        for kind in ('command', 'tool'):
+        for kind in ("command", "tool"):
             if by_kind[kind]:
                 selected.append(by_kind[kind].pop(0))
 
@@ -112,30 +118,59 @@ def bootstrap_session(self, prompt: str, limit: int = 5) -> RuntimeSession:
         setup = setup_report.setup
         history = HistoryLog()
         engine = QueryEnginePort.from_workspace()
-        history.add('context', f'python_files={context.python_file_count}, archive_available={context.archive_available}')
-        history.add('registry', f'commands={len(PORTED_COMMANDS)}, tools={len(PORTED_TOOLS)}')
+        history.add(
+            "context",
+            f"python_files={context.python_file_count}, archive_available={context.archive_available}",
+        )
+        history.add(
+            "registry", f"commands={len(PORTED_COMMANDS)}, tools={len(PORTED_TOOLS)}"
+        )
         matches = self.route_prompt(prompt, limit=limit)
         registry = build_execution_registry()
-        command_execs = tuple(registry.command(match.name).execute(prompt) for match in matches if match.kind == 'command' and registry.command(match.name))
-        tool_execs = tuple(registry.tool(match.name).execute(prompt) for match in matches if match.kind == 'tool' and registry.tool(match.name))
+        command_execs = tuple(
+            registry.command(match.name).execute(prompt)
+            for match in matches
+            if match.kind == "command" and registry.command(match.name)
+        )
+        tool_execs = tuple(
+            registry.tool(match.name).execute(prompt)
+            for match in matches
+            if match.kind == "tool" and registry.tool(match.name)
+        )
         denials = tuple(self._infer_permission_denials(matches))
-        stream_events = tuple(engine.stream_submit_message(
-            prompt,
-            matched_commands=tuple(match.name for match in matches if match.kind == 'command'),
-            matched_tools=tuple(match.name for match in matches if match.kind == 'tool'),
-            denied_tools=denials,
-        ))
+        stream_events = tuple(
+            engine.stream_submit_message(
+                prompt,
+                matched_commands=tuple(
+                    match.name for match in matches if match.kind == "command"
+                ),
+                matched_tools=tuple(
+                    match.name for match in matches if match.kind == "tool"
+                ),
+                denied_tools=denials,
+            )
+        )
         turn_result = engine.submit_message(
             prompt,
-            matched_commands=tuple(match.name for match in matches if match.kind == 'command'),
-            matched_tools=tuple(match.name for match in matches if match.kind == 'tool'),
+            matched_commands=tuple(
+                match.name for match in matches if match.kind == "command"
+            ),
+            matched_tools=tuple(
+                match.name for match in matches if match.kind == "tool"
+            ),
             denied_tools=denials,
         )
         persisted_session_path = engine.persist_session()
-        history.add('routing', f'matches={len(matches)} for prompt={prompt!r}')
-        history.add('execution', f'command_execs={len(command_execs)} tool_execs={len(tool_execs)}')
-        history.add('turn', f'commands={len(turn_result.matched_commands)} tools={len(turn_result.matched_tools)} denials={len(turn_result.permission_denials)} stop={turn_result.stop_reason}')
-        history.add('session_store', persisted_session_path)
+        history.add("routing", f"matches={len(matches)} for prompt={prompt!r}")
+        history.add(
+            "execution",
+            f"command_execs={len(command_execs)} tool_execs={len(tool_execs)}",
+        )
+        history.add(
+            "turn",
+            f"commands={len(turn_result.matched_commands)} tools={len(turn_result.matched_tools)} denials={len(turn_result.permission_denials)} stop={turn_result.stop_reason}",
+        )
+        history.add("session_store", persisted_session_path)
         return RuntimeSession(
             prompt=prompt,
             context=context,
@@ -151,42 +186,69 @@ def bootstrap_session(self, prompt: str, limit: int = 5) -> RuntimeSession:
             persisted_session_path=persisted_session_path,
         )
 
-    def run_turn_loop(self, prompt: str, limit: int = 5, max_turns: int = 3, structured_output: bool = False) -> list[TurnResult]:
+    def run_turn_loop(
+        self,
+        prompt: str,
+        limit: int = 5,
+        max_turns: int = 3,
+        structured_output: bool = False,
+    ) -> list[TurnResult]:
         engine = QueryEnginePort.from_workspace()
-        engine.config = QueryEngineConfig(max_turns=max_turns, structured_output=structured_output)
+        engine.config = QueryEngineConfig(
+            max_turns=max_turns, structured_output=structured_output
+        )
         matches = self.route_prompt(prompt, limit=limit)
-        command_names = tuple(match.name for match in matches if match.kind == 'command')
-        tool_names = tuple(match.name for match in matches if match.kind == 'tool')
+        command_names = tuple(
+            match.name for match in matches if match.kind == "command"
+        )
+        tool_names = tuple(match.name for match in matches if match.kind == "tool")
         results: list[TurnResult] = []
         for turn in range(max_turns):
-            turn_prompt = prompt if turn == 0 else f'{prompt} [turn {turn + 1}]'
+            turn_prompt = prompt if turn == 0 else f"{prompt} [turn {turn + 1}]"
             result = engine.submit_message(turn_prompt, command_names, tool_names, ())
             results.append(result)
-            if result.stop_reason != 'completed':
+            if result.stop_reason != "completed":
                 break
         return results
 
-    def _infer_permission_denials(self, matches: list[RoutedMatch]) -> list[PermissionDenial]:
+    def _infer_permission_denials(
+        self, matches: list[RoutedMatch]
+    ) -> list[PermissionDenial]:
         denials: list[PermissionDenial] = []
         for match in matches:
-            if match.kind == 'tool' and 'bash' in match.name.lower():
-                denials.append(PermissionDenial(tool_name=match.name, reason='destructive shell execution remains gated in the Python port'))
+            if match.kind == "tool" and "bash" in match.name.lower():
+                denials.append(
+                    PermissionDenial(
+                        tool_name=match.name,
+                        reason="destructive shell execution remains gated in the Python port",
+                    )
+                )
         return denials
 
-    def _collect_matches(self, tokens: set[str], modules: tuple[PortingModule, ...], kind: str) -> list[RoutedMatch]:
+    def _collect_matches(
+        self, tokens: set[str], modules: tuple[PortingModule, ...], kind: str
+    ) -> list[RoutedMatch]:
         matches: list[RoutedMatch] = []
         for module in modules:
             score = self._score(tokens, module)
             if score > 0:
-                matches.append(RoutedMatch(kind=kind, name=module.name, source_hint=module.source_hint, score=score))
+                matches.append(
+                    RoutedMatch(
+                        kind=kind,
+                        name=module.name,
+                        source_hint=module.source_hint,
+                        score=score,
+                    )
+                )
         matches.sort(key=lambda item: (-item.score, item.name))
         return matches
 
     @staticmethod
     def _score(tokens: set[str], module: PortingModule) -> int:
-        haystacks = [module.name.lower(), module.source_hint.lower(), module.responsibility.lower()]
+        # ⚡ Bolt: Use precomputed cached property to avoid redundant string allocations inside loops
+        haystack = module.search_text
         score = 0
         for token in tokens:
-            if any(token in haystack for haystack in haystacks):
+            if token in haystack:
                 score += 1
         return score