diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fa1a19..2aeff67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,27 @@ All notable changes to this project are documented here. Format: [Keep a Changel ### Fixed +- **Chart loading and background traffic** (measured 6 Oct, one tab, the exchange mocked, headless Chromium; the + numbers are for localhost and grow with request latency). + - **The module graph is fetched in parallel.** Vela's workspace, vela-pinets, PineTS and ~140 small Zag / + floating-ui files were discovered one import level at a time — 17 sequential waves, about 1.3 s of a 1.8 s + boot. `index.html` now lists all 157 as ``, generated by + `tools/modulepreload.py` (`--write` after a vendor upgrade; the test suite runs `--check`, so a stale list + fails there). First canvas 1,798 → 1,586 ms (median of 5); with 80 ms added per request 3,372 → 2,844 ms. + The block must come after the import map: a module fetch that starts first freezes the map and the workspace + silently falls back to the bare chart (a first version did exactly that and looked 300 ms faster — it was + measured by checking that the workspace really loaded; a test pins the order). + - **The exchange connections are warmed early** (`preconnect` for api.binance.com, fapi.binance.com and the + icon host), so the TLS handshakes happen while the module graph arrives instead of after it. Not measurable + offline; it removes a round trip the first bars wait on. + - **The heartbeat read the chart's bars twice every 4 s**, and when the chart hands none over (the normal case + on this Vela) each read was a live fetch to the venue through `/api/bars`: 16 in 30 idle seconds. It reads + once; `chartBars()` shares one fetch between callers within 3 s (and between callers arriving while one is in + flight); the server keeps a clean answer for 1.5 s. 16 → 8 in 30 s, and the overlay, the agent and a Pine run + no longer each pay their own round trip. An empty, failed or partial answer is never kept; every caller gets + its own array. + - Held by `test_boot_speed.py` (17 tests, each fix mutation-checked). + - **Audit round 4, phase C: the four findings that were ours** (about 900 engine runs under Node; the engine-side findings are in the audit report, not here). - `docs/studies/pine/fvg-mitigation.pine` failed at its default inputs on any data without a gap yet diff --git a/console/backend/server.py b/console/backend/server.py index 40c6780..fceed7f 100755 --- a/console/backend/server.py +++ b/console/backend/server.py @@ -1137,10 +1137,37 @@ def _fetch_klines(symbol: str, interval: str, limit: int, end_time: int | None = return json.load(res) +# The same (symbol, interval, limit) asked for again within this many seconds is answered from memory. The page's +# heartbeat, the overlay, the agent and a Pine run all ask for the chart's bars, and each used to be a live round +# trip to the venue (measured 6 Oct: 16 in 30 idle seconds with one tab open). Only a clean answer is kept — a +# failed or partial one is never served twice. +BARS_TTL_S = 1.5 +_BARS_MEMO: dict = {} +_BARS_MEMO_LOCK = threading.Lock() + + def fetch_bars(symbol: str, interval: str, limit: int) -> dict: - """`limit` most recent bars, oldest first, paged back from now. Never raises: a failed page is an - answer — the bars gathered so far come back with `partial: true` and the reason in `error`.""" + """`limit` most recent bars, oldest first, paged back from now (the same ask within BARS_TTL_S is answered + from memory). Never raises: a failed page is an answer — the bars gathered so far come back with + `partial: true` and the reason in `error`.""" limit = max(1, min(BARS_MAX, int(limit))) + key = (symbol, interval, limit) + now = time.monotonic() + with _BARS_MEMO_LOCK: + hit = _BARS_MEMO.get(key) + if hit and now - hit[0] < BARS_TTL_S: + return json.loads(hit[1]) + out = _fetch_bars_uncached(symbol, interval, limit) + if not out.get("error") and out.get("bars"): + with _BARS_MEMO_LOCK: + if len(_BARS_MEMO) > 64: + for k in [k for k, v in _BARS_MEMO.items() if now - v[0] >= BARS_TTL_S] or list(_BARS_MEMO)[:32]: + _BARS_MEMO.pop(k, None) + _BARS_MEMO[key] = (time.monotonic(), json.dumps(out)) + return out + + +def _fetch_bars_uncached(symbol: str, interval: str, limit: int) -> dict: rows: list = [] error = None end_time = None diff --git a/console/backend/tests/test_boot_speed.py b/console/backend/tests/test_boot_speed.py new file mode 100644 index 0000000..34d0f22 --- /dev/null +++ b/console/backend/tests/test_boot_speed.py @@ -0,0 +1,226 @@ +"""Chart loading and background traffic (6 Oct 2026), measured and then held. + +What was found, with one tab open and the exchange mocked (localhost, headless Chromium): + + * the ES-module graph (workspace.js -> Vela, vela-pinets, PineTS, ~140 Zag files) is discovered one import level at + a time: 17 sequential waves, about 1.3 s of a 1.8 s boot -> a generated `` block; + * the first chart data waits for TLS to the exchange, which only started once the chart did -> `preconnect`; + * the 4-second heartbeat read the chart's bars TWICE, and when the chart hands none over each read was a live + round trip to the venue (16 in 30 idle seconds) -> one read, shared between callers, and cached on the server. + +The one trap, found by measuring (a first version "improved" boot by quietly breaking it): a module fetch that +starts before the import map is parsed FREEZES the map, every bare specifier ("pinets") then fails to resolve, and +the workspace falls back to the bare chart. The preload block must come after the import map. +""" +import json +import os +import re +import shutil +import subprocess +import sys +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +sys.path.insert(0, os.path.dirname(HERE)) +ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..")) +FRONT = os.path.join(ROOT, "console", "frontend") +NODE = shutil.which("node") + +import server as srv # noqa: E402 + + +def read(*parts): + with open(os.path.join(ROOT, *parts), encoding="utf-8") as fh: + return fh.read() + + +INDEX = read("console", "frontend", "index.html") +APP = read("console", "frontend", "app.js") +BRIDGE = read("console", "frontend", "chart-bridge.js") + + +class ModulePreload(unittest.TestCase): + def test_the_block_is_exactly_what_the_generator_derives(self): + done = subprocess.run([sys.executable, os.path.join(ROOT, "tools", "modulepreload.py"), "--check"], + capture_output=True, text=True) + self.assertEqual(done.returncode, 0, done.stderr or "tools/modulepreload.py --write") + + def test_the_block_comes_after_the_import_map(self): + """Before it, the browser freezes an empty map and the workspace silently falls back to the bare chart.""" + importmap = INDEX.index(' + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/console/frontend/vendor/VENDORING.md b/console/frontend/vendor/VENDORING.md index 74e3ce9..d78a4c2 100644 --- a/console/frontend/vendor/VENDORING.md +++ b/console/frontend/vendor/VENDORING.md @@ -58,3 +58,12 @@ Two gotchas, both learned the hard way: The pane answers that question itself now: the heartbeat carries `diag: {wsReady, wsError, cells, activeChart, consoleChart}` (`frontend/chart-bridge.js` → `backend/chart_bridge.py`), which is exactly how the missing `@zag-js/vanilla` specifier was found. + +## After a vendor upgrade + +`index.html` lists the page's whole ES-module graph as `` so the browser fetches it in +parallel (see CHANGELOG, 6 Oct). When a vendored family changes its files, regenerate the list: + + python3 tools/modulepreload.py --write + +`test_boot_speed.py` runs `--check`, so a stale list fails the suite instead of quietly slowing the boot. diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index bf82963..48b9d2a 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -46,7 +46,7 @@ native tools. Versions: Hermes Desktop 0.21.5, console `SERVER_VERSION = 1.0.0`. CI was red from 28 Sep (`1f9d340`) to 2 Oct (`8df6e2e` — a stale MCP test only CI could see); green since `ab3708b`, and -green on every ship below. **1017 backend tests** (+77 skipped without `fastmcp` / a real Edge Stats engine), 8 plugin contributions across 5 areas. +green on every ship below. **1034 backend tests** (+77 skipped without `fastmcp` / a real Edge Stats engine), 8 plugin contributions across 5 areas. Shipped since this file was written: the paper broker (Phase 5), replay (6/6b), the top-bar one-door consolidation, the drawer as the single Library surface, and Phase 7 — three agent doors into Vela (`chart_drawing` 76 types / `chart_view` settings / `chart_marks`), 54 MCP tools (8 of them `edgestats_*`). 4 Oct: Edge Stats — the optional local engine, the sheet behind the `⋯` menu, the eight tools (see CHANGELOG; real Binance / Dukascopy downloads were never run from the build machine). Also 4 Oct: the top row @@ -268,7 +268,7 @@ Working, with evidence: - Latency: symbol switch **686–698 ms** end to end, screenshot **35 ms**, `clear` **7 ms**, SSE push single-digit ms (transport is not the bottleneck; the chart engine fetch+render is). - Console health: `/api/health` ok, LuxAlgo MCP connected, 29 endpoints. -- 1017 backend tests (77 skipped bare; 1 with fastmcp + a real engine), plugin harness OK (8 contributions / 5 areas), CI green since `ab3708b` +- 1034 backend tests (77 skipped bare; 1 with fastmcp + a real engine), plugin harness OK (8 contributions / 5 areas), CI green since `ab3708b` (see the correction above — earlier runs were red for a week and nobody looked). Current live state (transient): chart on **BTCUSDT 1m** (dark), the Breakout Detector overlay restored diff --git a/tools/modulepreload.py b/tools/modulepreload.py new file mode 100644 index 0000000..b1e1995 --- /dev/null +++ b/tools/modulepreload.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Keep the `` block in console/frontend/index.html exact. + +The page's ES-module graph (workspace.js -> the vendored Vela, vela-pinets, PineTS and ~140 small Zag / +floating-ui files) is found one level at a time by the browser: each file has to arrive and be read before the +files IT imports are even requested. Measured 6 Oct: 17 sequential waves, about 1.3 s of a 1.8 s boot. Listing the +whole graph up front lets the browser fetch it in parallel (first canvas 1,792 ms -> 1,501 ms, median of 5, +localhost). + +The list is derived, never typed: this script walks the static `import ... from "x"`, `export ... from "x"`, +`import "x"` and `import("x")` string specifiers from workspace.js, resolves bare ones through the page's import +map, and prints / writes the block. The test suite runs `--check`, so a vendor upgrade that changes the graph fails +there instead of silently leaving a stale list. + + tools/modulepreload.py print the block + tools/modulepreload.py --write rewrite it in index.html + tools/modulepreload.py --check exit 1 when index.html's block is not exactly this +""" +import json +import re +import sys +from pathlib import Path + +FRONT = Path(__file__).resolve().parent.parent / "console" / "frontend" +INDEX = FRONT / "index.html" +ENTRY = "workspace.js" +START = "" +END = "" + +# `from "x"`, `from'x'`, `import "x"`, `import"x"`, `import("x")`, `import('x')` — also in minified code +SPECIFIER = re.compile(r"""(?:\bfrom\s*|\bimport\s*\(?\s*)(["'])([^"'\n]+)\1""") + + +def import_map() -> dict: + m = re.search(r'', INDEX.read_text(), re.S) + return json.loads(m.group(1))["imports"] if m else {} + + +def resolve(spec: str, importer: Path, imports: dict): + if re.match(r"^[a-z]+:", spec): + return None # a URL: not ours to preload + if spec.startswith((".", "/")): + target = (importer.parent / spec).resolve() if spec.startswith(".") else (FRONT / spec.lstrip("/")).resolve() + elif spec in imports: + target = (FRONT / imports[spec]).resolve() + else: + return None # a bare name the map does not know + return target if target.is_file() and FRONT.resolve() in target.parents else None + + +def graph() -> list: + imports = import_map() + entry = (FRONT / ENTRY).resolve() + seen, order, todo = set(), [], [entry] + while todo: + f = todo.pop() + if f in seen: + continue + seen.add(f) + if f != entry: # (vendor/vela/dist/workspace.js is a different file) + order.append(f) + if f.suffix not in (".js", ".mjs"): + continue + for _q, spec in SPECIFIER.findall(f.read_text(errors="replace")): + t = resolve(spec, f, imports) + if t is not None and t not in seen: + todo.append(t) + return sorted("./" + p.relative_to(FRONT.resolve()).as_posix() for p in order) + + +def block() -> str: + return "\n".join([START] + [f'' for h in graph()] + [END]) + + +def current(html: str): + m = re.search(re.escape(START) + r".*?" + re.escape(END), html, re.S) + return m.group(0) if m else None + + +def main(argv) -> int: + want = block() + html = INDEX.read_text() + have = current(html) + if "--check" in argv: + if have == want: + return 0 + print("index.html's modulepreload block is stale: run tools/modulepreload.py --write", file=sys.stderr) + return 1 + if "--write" in argv: + if have is None: + print("no modulepreload block in index.html (add the start / end markers first)", file=sys.stderr) + return 1 + INDEX.write_text(html.replace(have, want)) + print(f"wrote {want.count('