From 3289e4c8b6910da143817896f533b001738ffac1 Mon Sep 17 00:00:00 2001 From: Kaden McKeen Date: Wed, 20 May 2026 21:31:50 -0400 Subject: [PATCH] refactor: drop jupyter-compute from the Jupyter MCP integration `aexp install --with-jupyter` wired two near-duplicate Jupyter MCP servers into a consumer's `.mcp.json`: `jupyter` (uvx jupyter-mcp-server, MCP_SERVER mode, runtime-retargetable via `connect_to_jupyter`) and `jupyter-compute` (an npx mcp-remote proxy to a fixed cluster `/mcp` endpoint, JUPYTER_SERVER mode). `jupyter-compute` could not retarget to a different node without a config edit + MCP restart, which breaks the multi-node workflow the slash commands are built around. It was otherwise a near-duplicate of `jupyter` and a standing "which server do I use?" confusion surface. Reduce the integration to a single server, `jupyter`: - install.py: `_jupyter_mcp_entries()` emits only the `jupyter` entry; the additive merge no longer special-cases `jupyter-compute`. - Slash commands `/aexp-jupyter-iterate` and `/aexp-promote-nb`, `AGENTS.md`, and `docs/setup/jupyter-mcp.md` retargeted to the single-server `mcp__jupyter__*` tool family. - The two `jupyter-mcp-tools` UI tools are lost: `notebook_run-all-cells` was already 404-broken upstream; `notebook_get-selected-cell` is gone, so the affected commands now ask the user for the notebook/cell or use `aexp.jupyter.init().attached_notebooks`. The cluster-side `[jupyter]` pip extra and `aexp jupyter setup` extension recipe are intentionally unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 25 +++ README.md | 2 +- src/aexp/cli.py | 4 +- src/aexp/install.py | 83 +++----- .../slash_commands/aexp-jupyter-iterate.md | 84 ++++---- src/aexp/slash_commands/aexp-promote-nb.md | 38 ++-- src/aexp/vendor/limina/AGENTS.md | 22 +- .../vendor/limina/docs/setup/jupyter-mcp.md | 196 ++++++------------ tests/test_install.py | 45 ++-- 9 files changed, 206 insertions(+), 293 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de29254..d1b7491 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **The Jupyter MCP integration is now a single MCP server.** `aexp + install --with-jupyter` writes only the `jupyter` server entry to + `.mcp.json` (laptop-side `uvx jupyter-mcp-server` in MCP_SERVER mode, + runtime-retargetable to any node via `connect_to_jupyter`). The second + `jupyter-compute` server — an `npx mcp-remote` proxy to a cluster + `/mcp` endpoint (JUPYTER_SERVER mode) — is no longer emitted. It was a + near-duplicate of `jupyter`, could not retarget to a different node + without a `.mcp.json` edit + MCP restart, and was a standing + "which server do I use?" confusion surface. + - The `/aexp-jupyter-iterate` and `/aexp-promote-nb` slash commands, + `AGENTS.md`, and `docs/setup/jupyter-mcp.md` are updated to the + single-server tool family (`mcp__jupyter__*`). + - **Lost capability:** the two `jupyter-mcp-tools` UI-delegated tools. + `notebook_run-all-cells` was already 404-broken upstream; + `notebook_get-selected-cell` ("which cell is the user looking at") + is genuinely gone — the affected slash commands now ask the user for + the notebook/cell or use `aexp.jupyter.init().attached_notebooks`. + - A consumer `.mcp.json` written by an earlier `--with-jupyter` + install keeps its `jupyter-compute` entry (the merge is + additive-only and never deletes servers); remove it by hand for the + cleanup. The cluster-side `[jupyter]` extra and `aexp jupyter setup` + extension recipe are unchanged. + ## [0.4.0] - 2026-05-20 ### Added diff --git a/README.md b/README.md index 9c59427..560178a 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ The design bet: agents already know how to run experiments. What they need is a | **Slash commands** | Artifact creation: `/aexp-new-hypothesis`, `/aexp-new-experiment`, `/aexp-new-run`. Threads (forward-looking research concerns broader than a hypothesis): `/aexp-new-thread`, `/aexp-list-threads`, `/aexp-show-thread`, `/aexp-close-thread`. Finding creation (pick by what the finding cites): `/aexp-finding-from-run`, `/aexp-finding-from-batch`, `/aexp-finding-placeholder`. Read / inspect: `/aexp-show-run`, `/aexp-show-batch`, `/aexp-list-runs`, `/aexp-status`, `/aexp-validate`. Queue: `/aexp-queue-add`, `/aexp-queue-list`, `/aexp-queue-materialize`, `/aexp-queue-stop`. Notebook lifecycle (when `--with-jupyter` is configured): `/aexp-jupyter-iterate` (test loop), `/aexp-promote-nb` (promote working cells into a tracked-run script). Sandbox scaffolding: `/aexp-new-sandbox` (create an exploratory notebook subdir under `notebooks/_sandbox/`). 22 total. | | **CLI** | 22 verbs covering install, artifact creation (H/E/F/T + thread lifecycle), run lifecycle, batch queries, tracker binding, validation, offline sync, optional `jupyter-setup`, the `queue` subcommand group (add/list/remove/stop/clear/materialize/run) + `run-queued`, and sandbox scaffolding (`new-sandbox`). See `aexp --help` for the full list. Python API is a one-line `from aexp import ...`. | | **Typed JSON contracts** | Pydantic models (`RunLink`, `BatchSelector`, `Issue`, …) back the schema; MCP tools and CLI return the same shapes. | -| **Jupyter MCP integration** (optional, `[jupyter]` extra) | `aexp install --with-jupyter` adds `jupyter` and `jupyter-compute` MCP servers to `.mcp.json` so Claude can read/edit/execute cells in a remote JupyterLab through an existing SSH tunnel — no agent SSH required. `aexp jupyter-setup` applies the verified Jupyter Server extension state on the cluster (disable Datalayer experiments that conflict with the mainstream stack). After install, see `docs/setup/jupyter-mcp.md` for cluster-side recipe + investigation log. The `/aexp-jupyter-iterate` slash command guides the read → propose → execute loop. | +| **Jupyter MCP integration** (optional, `[jupyter]` extra) | `aexp install --with-jupyter` adds the `jupyter` MCP server to `.mcp.json` so Claude can read/edit/execute cells in a remote JupyterLab through an existing SSH tunnel — no agent SSH required. The target Jupyter is set per-session at runtime via `connect_to_jupyter`, so one entry retargets to any node. `aexp jupyter-setup` applies the verified Jupyter Server extension state on the cluster (disable Datalayer experiments that conflict with the mainstream stack). After install, see `docs/setup/jupyter-mcp.md` for cluster-side recipe + investigation log. The `/aexp-jupyter-iterate` slash command guides the read → propose → execute loop. | ### Exploratory surfaces diff --git a/src/aexp/cli.py b/src/aexp/cli.py index cf542b6..f0524e5 100644 --- a/src/aexp/cli.py +++ b/src/aexp/cli.py @@ -249,8 +249,8 @@ def install( False, "--with-jupyter", help=( - "Opt into the Jupyter MCP integration: writes `jupyter` and " - "`jupyter-compute` server entries to `.mcp.json`, sets " + "Opt into the Jupyter MCP integration: writes the `jupyter` " + "server entry to `.mcp.json`, sets " "`jupyter_enabled: true` (sticky) in the install marker, and " "ensures `docs/setup/jupyter-mcp.md` is vendored. Requires " "`pip install agentic-experiments[jupyter]` for the Python " diff --git a/src/aexp/install.py b/src/aexp/install.py index ad9523f..6dba766 100644 --- a/src/aexp/install.py +++ b/src/aexp/install.py @@ -481,11 +481,11 @@ def _merge_or_write_mcp_json( current interpreter instead — lets editable installs take effect on the MCP side (at the cost of a machine-specific ``.mcp.json``). - When ``with_jupyter=True``, also writes the ``jupyter`` and - ``jupyter-compute`` entries used by the Jupyter MCP integration. The - entries are *additive*: once written, subsequent installs without the - flag leave them in place (matching the "never delete user-defined - servers" pattern). To back out, the user edits ``.mcp.json`` by hand. + When ``with_jupyter=True``, also writes the ``jupyter`` entry used by + the Jupyter MCP integration. The entry is *additive*: once written, + subsequent installs without the flag leave it in place (matching the + "never delete user-defined servers" pattern). To back out, the user + edits ``.mcp.json`` by hand. """ rel = _display_relpath(dst) our_entries: dict[str, Any] = {"aexp": _build_mcp_server_entry(repo_root, dev=dev)} @@ -517,14 +517,12 @@ def _merge_or_write_mcp_json( merged.setdefault("mcpServers", {}) # Always refresh our own ``aexp`` entry; preserve any user-defined servers. merged["mcpServers"]["aexp"] = our_entries["aexp"] - # Jupyter entries: only ever ADD. If the user already has a `jupyter` / - # `jupyter-compute` block (either from a prior --with-jupyter install or - # from a manual setup) leave it alone — they may have hardcoded the - # Windows-stable token there, which we must not clobber. - if with_jupyter: - for key in ("jupyter", "jupyter-compute"): - if key not in merged["mcpServers"]: - merged["mcpServers"][key] = our_entries[key] + # Jupyter entry: only ever ADD. If the user already has a `jupyter` + # block (from a prior --with-jupyter install or a manual setup) leave + # it alone — they may have customized the URL/port or pinned a + # version, which we must not clobber. + if with_jupyter and "jupyter" not in merged["mcpServers"]: + merged["mcpServers"]["jupyter"] = our_entries["jupyter"] if merged == existing: return InstallAction("skipped_identical", rel) @@ -588,49 +586,25 @@ def _build_mcp_server_entry(repo_root: Path, *, dev: bool = False) -> dict[str, def _jupyter_mcp_entries() -> dict[str, Any]: - """MCP server entries for the Jupyter MCP integration. + """MCP server entry for the Jupyter MCP integration. - Two side-by-side servers, both reaching the same JupyterLab through the - user's existing SSH tunnel: + A single laptop-side server: - ``jupyter`` — laptop-side ``uvx jupyter-mcp-server`` running in - MCP_SERVER mode (stdio to Claude, HTTP+WS to remote Jupyter). The - token is provided per-session at runtime via the ``connect_to_jupyter`` - tool, so no token lives in this entry. - - ``jupyter-compute`` — laptop-side ``npx mcp-remote`` proxy bridging - Claude's stdio to the cluster's ``/mcp`` SSE endpoint, where - ``jupyter-mcp-server`` runs as a Jupyter Server extension - (JUPYTER_SERVER mode). Token is interpolated from the - ``JUPYTER_TOKEN`` env var by default. - - Default port is ``3618`` (matches the verified electricrag deployment). - Consumers using a different port edit ``.mcp.json`` post-install. - - On Windows, ``${JUPYTER_TOKEN}`` interpolation is fragile because - ``setx`` does not propagate to already-running processes (notably - Explorer, which spawns Start-Menu apps including Claude Desktop). The - documented fix is to hardcode the literal token in ``.mcp.json`` and - set the matching value in ``~/.jupyter/jupyter_server_config.py`` on - the cluster — see ``docs/setup/jupyter-mcp.md`` "Investigation log §4". - The install never auto-rewrites the literal token: token management - stays the consumer's responsibility. + MCP_SERVER mode (stdio to Claude, HTTP+WS to the remote Jupyter). + The target Jupyter URL + token are supplied per-session at runtime + via the ``connect_to_jupyter`` tool, so no token lives in this + entry and the *same* entry retargets to any node — open a tunnel on + a new local port, call ``connect_to_jupyter`` at the new URL, done. + No ``.mcp.json`` edit, no MCP restart. That runtime retargeting is + what makes the multi-node workflow (``/aexp-jupyter-connect`` / + ``/aexp-jupyter-discover``) work. """ return { "jupyter": { "command": "uvx", "args": ["jupyter-mcp-server"], }, - "jupyter-compute": { - "command": "npx", - "args": [ - "-y", - "mcp-remote", - "http://127.0.0.1:3618/mcp", - "--allow-http", - "--header", - "Authorization:token ${JUPYTER_TOKEN}", - ], - }, } @@ -841,14 +815,13 @@ def install_limina( override (e.g. dogfooding the consumer scaffold against the dev repo on purpose). with_jupyter : bool, optional - If ``True``, also write the ``jupyter`` and ``jupyter-compute`` - MCP server entries into ``.mcp.json``, vendor - ``docs/setup/jupyter-mcp.md`` into the consumer repo, and set - ``jupyter_enabled: true`` in the install marker. The marker bit - is sticky — once set, subsequent installs preserve it even if - ``with_jupyter=False``. The ``.mcp.json`` entries are additive: - existing user-defined ``jupyter`` / ``jupyter-compute`` blocks - are preserved (so a hardcoded Windows-stable token survives). + If ``True``, also write the ``jupyter`` MCP server entry into + ``.mcp.json``, vendor ``docs/setup/jupyter-mcp.md`` into the + consumer repo, and set ``jupyter_enabled: true`` in the install + marker. The marker bit is sticky — once set, subsequent installs + preserve it even if ``with_jupyter=False``. The ``.mcp.json`` + entry is additive: an existing user-defined ``jupyter`` block is + preserved (so a customized URL/port survives). See ``docs/setup/jupyter-mcp.md`` for the full setup recipe. Returns diff --git a/src/aexp/slash_commands/aexp-jupyter-iterate.md b/src/aexp/slash_commands/aexp-jupyter-iterate.md index 5823d4b..aeb9b54 100644 --- a/src/aexp/slash_commands/aexp-jupyter-iterate.md +++ b/src/aexp/slash_commands/aexp-jupyter-iterate.md @@ -2,18 +2,27 @@ description: "Iterate on a JupyterLab cell with the user via the Jupyter MCP bridge (read → propose → execute)." --- -Iterate on whatever cell the user is currently looking at in JupyterLab, -through the Jupyter MCP bridge. +Iterate on a notebook cell with the user, through the Jupyter MCP bridge. -> **Prerequisite.** This command requires the `mcp__jupyter-compute__*` -> tool family. Those tools come from `aexp install --with-jupyter` plus a -> JupyterLab process reachable through the user's SSH tunnel. If the -> tools are missing, run `/mcp` to inspect server status and consult -> `docs/setup/jupyter-mcp.md` for the cluster-side setup recipe. +> **Prerequisite.** This command requires the `mcp__jupyter__*` tool +> family — in particular `connect_to_jupyter`, `execute_code`, +> `read_cell`, and `execute_cell`. Those tools come from +> `aexp install --with-jupyter` plus a JupyterLab process reachable +> through the user's SSH tunnel. If the tools are missing, run `/mcp` to +> inspect server status and consult `docs/setup/jupyter-mcp.md` for the +> setup recipe. Run through these steps: -0. **Confirm session identity.** Before touching any cells, dispatch: +1. **Check tool availability.** Verify that `mcp__jupyter__execute_cell` + and `mcp__jupyter__read_cell` are present in your tool list. If not, + stop and report: + "Jupyter MCP integration not available in this session. Run + `aexp install --with-jupyter`, ensure the SSH tunnel to the cluster is + open, connect with `/aexp-jupyter-connect`, and restart Claude. See + `docs/setup/jupyter-mcp.md`." + +2. **Confirm session identity.** Before touching any cells, dispatch: ``` execute_code(code="from aexp.jupyter import init; import json; print(json.dumps(init().model_dump(), default=str))") ``` @@ -27,46 +36,43 @@ Run through these steps: shouldn't disturb. If anything mismatches — wrong SLURM job, wrong host, unexpected GPU - resident — STOP and ask. Do not proceed to step 1. + resident — STOP and ask. Do not proceed to step 3. To switch to a + different Jupyter, use `/aexp-jupyter-connect`. -1. **Check tool availability.** Verify that - `mcp__jupyter-compute__notebook_get-selected-cell` and - `mcp__jupyter-compute__execute_cell` are present in your tool list. If - not, stop and report: - "Jupyter MCP integration not available in this session. Run - `aexp install --with-jupyter`, ensure the SSH tunnel to the cluster is - open, and restart Claude Desktop. See `docs/setup/jupyter-mcp.md`." +3. **Identify the target notebook and cell.** This single-server setup + has no live "what is the user looking at" tool, so ask the user + directly: + "Which notebook should I work in, and which cell — give me the cell + index, or describe it (e.g. 'the training loop')?" + Cross-check the notebook name against the `attached_notebooks` list + from step 2. Open it with `use_notebook` if it isn't already open. -2. **Identify what the user is looking at.** Call - `mcp__jupyter-compute__notebook_get-selected-cell` to read the live UI - selection. Report the notebook path, cell index, and cell type, and - quote the source verbatim so the user can confirm you're targeting the - right cell. +4. **Locate and quote the cell.** Use `read_cell(cell_index=N)` for a + single cell, or `read_notebook` (brief mode) to find the cell the + user described. Report the notebook path, cell index, and cell type, + and quote the source verbatim so the user can confirm you're + targeting the right cell before any edit. -3. **Gather context if needed.** If the user's request references "the - cells above" or relies on prior state, use - `mcp__jupyter-compute__read_cell` on adjacent indices, or - `mcp__jupyter-compute__read_notebook` (brief mode) for an overview. - Don't dump the whole notebook unless asked. +5. **Gather context if needed.** If the user's request references "the + cells above" or relies on prior state, use `read_cell` on adjacent + indices, or `read_notebook` (brief mode) for an overview. Don't dump + the whole notebook unless asked. -4. **Propose a change.** Describe what you intend to modify and why. +6. **Propose a change.** Describe what you intend to modify and why. Do NOT make the edit until the user confirms. -5. **On approval, apply the edit.** Use: +7. **On approval, apply the edit.** Use: - `edit_cell_source` for surgical find/replace within one cell. - `overwrite_cell_source` for full replacement. - `insert_cell` to add a new cell. -6. **Execute the cell.** Call - `mcp__jupyter-compute__execute_cell(cell_index=N)` with a reasonable - timeout. Paste the actual stdout / errors verbatim — don't paraphrase. +8. **Execute the cell.** Call `execute_cell(cell_index=N)` with a + reasonable timeout. Paste the actual stdout / errors verbatim — don't + paraphrase. -7. **Iterate or wrap up.** If the cell errored, propose the next fix and - loop back to step 4. If it succeeded, ask the user whether to continue - or stop. +9. **Iterate or wrap up.** If the cell errored, propose the next fix and + loop back to step 6. If it succeeded, ask the user whether to + continue or stop. -> **Do NOT use** `notebook_run-all-cells` — it is exposed by the bridge -> but currently returns 404 (asymmetric upstream bug, see -> `docs/setup/jupyter-mcp.md` "Investigation log" §5). Loop -> `execute_cell(cell_index=i)` over indices instead when a multi-cell run -> is needed. +> **Multi-cell runs.** To run a span of cells, loop +> `execute_cell(cell_index=i)` over the indices in order. diff --git a/src/aexp/slash_commands/aexp-promote-nb.md b/src/aexp/slash_commands/aexp-promote-nb.md index fd084ab..0820fc8 100644 --- a/src/aexp/slash_commands/aexp-promote-nb.md +++ b/src/aexp/slash_commands/aexp-promote-nb.md @@ -13,11 +13,10 @@ committed experiment. > The notebook stays as the smoke-test record — it's not edited, only > read. Outputs land at `/experiments/E-.py`. -> **Prerequisite.** Best with `mcp__jupyter-compute__*` tools available +> **Prerequisite.** Best with the `mcp__jupyter__*` tools available > (from `aexp install --with-jupyter`). If they're not, you can still > promote cells from a `.ipynb` file on disk via the standard Read tool; -> you'll lose the live cell-selection convenience but everything else -> works. +> everything in this command works either way. > **Invocation note.** The examples below use `python -m aexp` directly. > If running from a Claude Code session where `python` does not resolve @@ -33,25 +32,26 @@ committed experiment. Run through these steps: -1. **Tool availability check.** Verify whether - `mcp__jupyter-compute__notebook_get-selected-cell` and - `mcp__jupyter-compute__read_cell` are present. If yes, use them in - the steps below. If not, ask the user for the path to the `.ipynb` - file on disk and read it via the standard Read tool — JupyterLab - notebooks are JSON; you can extract `cells[i].source` directly. +1. **Tool availability check.** Verify whether the `mcp__jupyter__*` + tools (`read_cell`, `read_notebook`, `use_notebook`) are present. If + yes, use them in the steps below. If not, ask the user for the path + to the `.ipynb` file on disk and read it via the standard Read tool — + JupyterLab notebooks are JSON; you can extract `cells[i].source` + directly. -2. **Identify the source notebook.** With the MCP bridge: call - `mcp__jupyter-compute__notebook_get-selected-cell` to anchor on the - user's current focus and report the notebook path back to them. Without - the bridge: ask the user for the notebook path explicitly. +2. **Identify the source notebook.** Ask the user for the notebook path + explicitly. With the MCP bridge, cross-check it against the open + notebooks (`aexp.jupyter.init().attached_notebooks`) and open it with + `use_notebook` if it isn't already open. Without the bridge, take the + path the user gives you. 3. **Identify the cell range to promote.** Ask: - "promote just the currently-selected cell, or a range? if a range, - give me indices (e.g., `4-12`) or describe the cells (e.g., 'from the - model definition through the training loop')." Read each target cell - (`read_cell(cell_index=N)` via MCP, or by indexing into - `cells[]` from the on-disk JSON). Quote the source verbatim back to - the user and confirm the selection before going further. + "which cells should I promote — give me indices (e.g., `4-12`) or + describe the cells (e.g., 'from the model definition through the + training loop')." Read each target cell (`read_cell(cell_index=N)` + via MCP, or by indexing into `cells[]` from the on-disk JSON). Quote + the source verbatim back to the user and confirm the selection before + going further. 4. **Identify the experiment.** Ask which `E###` this script is being promoted under. If the user is unsure, suggest checking diff --git a/src/aexp/vendor/limina/AGENTS.md b/src/aexp/vendor/limina/AGENTS.md index 4143c72..4bbb9ba 100644 --- a/src/aexp/vendor/limina/AGENTS.md +++ b/src/aexp/vendor/limina/AGENTS.md @@ -155,14 +155,14 @@ The active session is the default transport for updates. ## Working with Jupyter MCP (when configured) -If `mcp__jupyter-compute__*` tools are available, the user has set up -the Claude ↔ JupyterLab integration via `aexp install --with-jupyter` -plus a running JupyterLab on a remote node (SSH-tunneled). Prefer these -tools over git-based round-trips for notebook work: - -- **"What am I looking at?"** → `notebook_get-selected-cell` returns the - user's live UI selection (cell index + type + source). Use this when - the user says "this cell" or "what I have open." +If `mcp__jupyter__*` tools are available, the user has set up the +Claude ↔ JupyterLab integration via `aexp install --with-jupyter` plus a +running JupyterLab on a remote node (SSH-tunneled). Prefer these tools +over git-based round-trips for notebook work: + +- **Opening a notebook** → `use_notebook` before reading or executing + its cells. Ask the user which notebook and cell to work on — this + single-server setup has no live "what is the user looking at" tool. - **Reading context** → `read_cell` / `read_notebook` (brief or detailed mode) before proposing edits. - **Editing** → `edit_cell_source` for surgical find-and-replace within @@ -170,10 +170,8 @@ tools over git-based round-trips for notebook work: for additions. - **Executing** → `execute_cell` to run a cell and persist outputs to the notebook; `execute_code` to run kernel-direct Python without - saving (good for sanity checks). -- **DO NOT** use `notebook_run-all-cells` — exposed but currently - returns 404 (asymmetric upstream bug). Loop `execute_cell` over - indices for multi-cell runs. + saving (good for sanity checks). Loop `execute_cell` over indices for + multi-cell runs. - **`read_cell` / `read_notebook` outputs lag live state.** After an `execute_cell`, the cached output (and execution count) returned by `read_cell` may still reflect the *prior* run. Trust `execute_cell`'s diff --git a/src/aexp/vendor/limina/docs/setup/jupyter-mcp.md b/src/aexp/vendor/limina/docs/setup/jupyter-mcp.md index 9f45bb1..f5cb9ab 100644 --- a/src/aexp/vendor/limina/docs/setup/jupyter-mcp.md +++ b/src/aexp/vendor/limina/docs/setup/jupyter-mcp.md @@ -32,22 +32,19 @@ Fill in your values once and refer back to them in the rest of the doc: ## TL;DR -Two MCP servers run side-by-side on the laptop, both reaching the same -cluster JupyterLab through your SSH-tunneled port: +One MCP server runs on the laptop, reaching the cluster JupyterLab +through your SSH-tunneled port: - **`jupyter`** — laptop-side `uvx jupyter-mcp-server` (stdio transport, - MCP_SERVER mode). Per-session token via `connect_to_jupyter` runtime - call. -- **`jupyter-compute`** — laptop-side `npx mcp-remote` proxy talking to - the cluster's `/mcp` HTTP endpoint (JUPYTER_SERVER mode, - `jupyter-mcp-server` running as a Jupyter Server extension). Token via - `${JUPYTER_TOKEN}` env var (or hardcoded literal — recommended on - Windows; see "Stable token setup"). - -Both expose the same 17 core tools (read/edit/execute cells, etc.). -`jupyter-compute` adds 2 JupyterLab-UI-delegated tools -(`notebook_run-all-cells`, `notebook_get-selected-cell`) that work when -a JupyterLab browser tab is open. + MCP_SERVER mode). The target Jupyter URL + token are supplied + per-session at runtime via the `connect_to_jupyter` tool — nothing is + baked into `.mcp.json`, so the *same* entry retargets to any node: + open a tunnel on a free local port, call `connect_to_jupyter` at the + new URL, done — no config edit, no MCP restart. That runtime + retargeting is what makes the multi-node workflow + (`/aexp-jupyter-connect`, `/aexp-jupyter-discover`) work. + +It exposes ~17 core tools (read/edit/execute cells, kernel ops, etc.). ## ⚠️ The Datalayer extension disable list @@ -97,24 +94,21 @@ re-enable if needed. ``` laptop (has internet) │ - ├─ Claude ──stdio──→ uvx jupyter-mcp-server (MCP_SERVER mode, "jupyter") - │ │ - │ │ HTTP+WS via tunnel - │ ▼ - │ ssh -N -L :: ── login node ── compute node (no internet) - │ │ - │ │ jupyter lab - │ │ with jupyter-mcp-server as Jupyter extension - │ ▼ - │ - │ (--ServerApp.root_dir confines content manager) - │ - └─ Claude ──stdio──→ npx mcp-remote ──HTTP──→ /mcp endpoint (JUPYTER_SERVER mode, "jupyter-compute") - (same SSH tunnel, port ) + └─ Claude ──stdio──→ uvx jupyter-mcp-server (MCP_SERVER mode, "jupyter") + │ + │ HTTP+WS via tunnel + ▼ + ssh -N -L :: ── login node ── compute node (no internet) + │ + │ jupyter lab + ▼ + + (--ServerApp.root_dir confines content manager) ``` -Both modes ride the same SSH tunnel. Only the laptop has internet. The -compute node never reaches outside. +The MCP server runs on the laptop and reaches the remote Jupyter over +standard HTTP + kernel-WebSocket through the SSH tunnel. Only the laptop +has internet; the compute node never reaches outside. ## One-time cluster setup @@ -169,12 +163,10 @@ Expected (the canonical working configuration): ## One-time laptop setup -You need three things on the laptop: +You need two things on the laptop: -1. **`uv`** (for `uvx`) — runs the laptop-side MCP_SERVER mode entry -2. **Node.js** (for `npx`) — runs the `mcp-remote` proxy for - JUPYTER_SERVER mode -3. **`.mcp.json` entries** — written by `aexp install --with-jupyter` +1. **`uv`** (for `uvx`) — runs the laptop-side MCP server +2. **`.mcp.json` entry** — written by `aexp install --with-jupyter` ```powershell # Install uv if not already there @@ -183,18 +175,10 @@ pip install uv # Verify uvx can fetch jupyter-mcp-server (does NOT support --help cleanly, # but a non-error exit means it's installed) uvx jupyter-mcp-server --help # may exit with usage; that's fine - -# Install Node.js LTS from https://nodejs.org/ if you don't have it -node --version - -# Verify mcp-remote installs and runs (will error on the fake URL — that's expected) -npx -y mcp-remote http://nonexistent.local/mcp 2>&1 | Select-Object -First 5 -# Look for: "Non-HTTPS URLs are only allowed for localhost or when --allow-http" -# That confirms mcp-remote is functional. ``` -After `aexp install --with-jupyter` your `.mcp.json` will include both -servers: +After `aexp install --with-jupyter` your `.mcp.json` will include the +`jupyter` server: ```json { @@ -202,23 +186,14 @@ servers: "jupyter": { "command": "uvx", "args": ["jupyter-mcp-server"] - }, - "jupyter-compute": { - "command": "npx", - "args": [ - "-y", "mcp-remote", - "http://127.0.0.1:3618/mcp", - "--allow-http", - "--header", "Authorization:token ${JUPYTER_TOKEN}" - ] } } } ``` -The `${JUPYTER_TOKEN}` interpolation reads from your shell environment -when Claude Desktop spawns the proxy. **On Windows this is fragile** — -see "Stable token setup" below. +No token or URL is baked into the entry — the agent supplies them at +runtime via `connect_to_jupyter` (see "Per-session: connect from the +laptop" below). ## Per-session: launch JupyterLab on the cluster @@ -234,10 +209,6 @@ of line 1 changes per job; lines 2 and 3 are the same every time. ## Per-session: connect from the laptop -If you've done the **stable token setup** below, this is just two steps: -open the SSH tunnel and (if Claude Desktop isn't already running) launch -it. The token is hardcoded in `.mcp.json` and stable across all jobs. - ```powershell # Open the SSH tunnel ssh -N -L :: @ @@ -246,13 +217,26 @@ ssh -N -L :: @ as the file root. ``` -That's it. Open a fresh Claude Desktop session; both `mcp__jupyter__*` -and `mcp__jupyter-compute__*` tool families should appear. +Open a fresh Claude session; the `mcp__jupyter__*` tool family should +appear. Then point it at the running Jupyter — paste line 3 of the +launcher log, or just tell the agent: -### Stable token setup (one-time, do this!) +``` +Connect to jupyter at http://127.0.0.1: with token . +``` + +The agent calls `connect_to_jupyter(jupyter_url, jupyter_token)`; a +PostToolUse hook then surfaces a re-introspection directive it follows +to confirm the session landed where intended. To switch to a different +Jupyter mid-session, use `/aexp-jupyter-connect`. + +### Stable token setup (one-time, recommended) -If you don't want to setx + restart Claude Desktop every time the batch -launcher generates a new random token, set up a stable token once: +The MCP server takes the token at runtime via `connect_to_jupyter`, so +there is no token in `.mcp.json` to keep in sync. But a *stable* token +still helps: it means the connect prompt (line 3 of the launcher log) is +identical for every job, so you paste the same string each time instead +of copying a fresh random token. **On the cluster** (one-time): @@ -263,38 +247,18 @@ mkdir -p ~/.jupyter TOKEN=$(openssl rand -hex 32) echo "c.IdentityProvider.token = '$TOKEN'" > ~/.jupyter/jupyter_server_config.py chmod 600 ~/.jupyter/jupyter_server_config.py -echo "JUPYTER_TOKEN=$TOKEN" # copy this for the .mcp.json edit +echo "JUPYTER_TOKEN=$TOKEN" # this is the token you paste into the connect prompt ``` Then update your batch launcher script: - **Remove** any `JUPYTER_TOKEN="${JUPYTER_TOKEN:-$(openssl rand -hex 32)}"` line (no longer needed) - **Remove** `--IdentityProvider.token="$JUPYTER_TOKEN"` from the `jupyter lab` invocation (the config file provides it) -**On the laptop** (one-time): - -Edit `.mcp.json`'s `jupyter-compute` entry — replace `${JUPYTER_TOKEN}` -with the literal token from the cluster step. The hardcoded form is -**required on Windows** because `setx` doesn't propagate to -already-running processes (see "Investigation log §4"); on Linux/Mac -the env-var form works fine if you prefer. - -After this: every launcher job uses the same token. Hardcoded -`.mcp.json` matches. No `setx`, no Claude Desktop restarts for token -changes ever again. The token is only valid against your SSH-tunneled -localhost endpoint, so committing it is acceptable for private repos -(it requires your SSH key to actually exploit). - -In a fresh Claude Desktop session you can: - -- **For `jupyter` (MCP_SERVER mode):** paste line 3 verbatim — Claude - calls `connect_to_jupyter(jupyter_url, jupyter_token)` then can - read/edit cells. -- **For `jupyter-compute` (JUPYTER_SERVER mode):** the proxy - auto-authenticates; just call `mcp__jupyter-compute__list_files` - directly. No `connect_to_jupyter` needed. - -Both servers expose the same core tools. `jupyter-compute` adds the UI -tools (only meaningful with a JupyterLab browser tab open). +After this, every launcher job uses the same token, so the connect +prompt never changes — only the tunnel's compute-node host varies per +job. The token is only valid against your SSH-tunneled localhost +endpoint (exploiting it requires your SSH key), so a private repo can +keep the connect prompt in its launcher log. ## Smoke test @@ -394,9 +358,6 @@ additive follow-up — open an issue. | Manual notebook edits in browser tab stop persisting after Claude has edited cells | [datalayer/jupyter-mcp-server#146](https://github.com/datalayer/jupyter-mcp-server/issues/146) — Y.js sync corruption | Stop using MCP on that notebook; reload the tab. If this becomes recurrent, consider building a narrower custom Jupyter Server extension exposing only the four tools you actually need. | | `poetry add` hangs / silently bails / `Resolving dependencies... (0.0s)` | Broken `file://` path dep in pyproject.toml getting parsed before markers are evaluated ([poetry#9679](https://github.com/python-poetry/poetry/issues/9679)) | Don't add `file://` path deps with `sys_platform` markers. Move sibling editable installs to `poetry run pip install -e ...` outside pyproject.toml | | Claude tries to use `list_notebooks` before opening any | The tool only lists *opened* notebooks, not files | Tell it to use `list_files` to enumerate `.ipynb` files | -| Only `mcp__jupyter-compute__*` tools visible (laptop-side `mcp__jupyter__*` missing) | Probably `uvx jupyter-mcp-server` startup race in Claude Desktop | Restart Claude Desktop. Check Claude Desktop's MCP status panel for the `jupyter` server connection state. If persistently broken, drop the `jupyter` entry — `jupyter-compute` is functionally a superset. | -| Only `mcp__jupyter__*` visible (cluster-side `mcp__jupyter-compute__*` missing) | `mcp-remote` proxy failed to authenticate. Most likely cause: stale `JUPYTER_TOKEN` env var on Windows from a previous job. Confirm by running `npx -y mcp-remote http://127.0.0.1:/mcp --allow-http --header "Authorization:token $env:JUPYTER_TOKEN"` directly — if it returns HTTP 403, the token is stale. | `setx JUPYTER_TOKEN ""`, fresh PowerShell, restart Claude Desktop. Or migrate to a stable token via `~/.jupyter/jupyter_server_config.py` to avoid this entirely. | -| `notebook_run-all-cells` returns `404 Not Found`, but `notebook_get-selected-cell` works | Asymmetric route registration in `jupyter-mcp-tools` frontend bridge. Only `get-selected-cell` is reachable; `run-all-cells` 404s even though it's in the default `--allowed-jupyter-mcp-tools` list. | Workaround: loop `execute_cell(cell_index=i)` over indices, or trigger Run-All manually from the JupyterLab toolbar. The `get-selected-cell` UI bridge remains usable for "what's the user looking at?" workflows. | ## Investigation log — how we arrived at this configuration @@ -496,45 +457,11 @@ labextension was the only real culprit. **disable one at a time and re-test**, instead of disabling everything that *might* be at fault. -### Round 4: token rotation pain (Windows env-var propagation through Claude Desktop) - -**Symptom:** After every job-resubmit, the new job's random token is -different. Per-session ritual was: copy token → `setx JUPYTER_TOKEN` -→ close PowerShell → open fresh one → restart Claude Desktop. The "only -`mcp__jupyter__*` tools visible, no `mcp__jupyter-compute__*`" failure -mode hit three times. Direct `npx mcp-remote` test against the cluster -returned HTTP 403, confirming the env var was stale in Claude Desktop's -process. - -**Root cause:** Windows `setx` updates the persistent user environment -but already-running processes (notably Explorer, which spawns Start -Menu apps) retain their old environment until restarted. Claude Desktop -launched from the Start Menu inherits Explorer's stale env, not the -freshly-set value. - -**Fix:** Pin a single stable token. Hardcode in `.mcp.json` (replace -`${JUPYTER_TOKEN}` with literal value) AND hardcode the same value in -the launcher script. Now every job uses the same token, hardcoded -`.mcp.json` matches, no `setx` ever needed. The token's only attack -surface is the SSH-tunneled localhost endpoint, which requires the -user's SSH key — committing it to a private repo is acceptable. - -### Round 5: asymmetric UI bridge (`notebook_run-all-cells` 404s, `notebook_get-selected-cell` works) - -**Symptom:** Both tools are nominally JupyterLab-frontend-delegated and -both should be in the default `--allowed-jupyter-mcp-tools` list. But -`notebook_get-selected-cell` reaches the live frontend and returns the -actual selected cell, while `notebook_run-all-cells` returns HTTP 404 -from the server. - -**Status:** Unresolved upstream. Documented as a known quirk. Workaround: -loop `execute_cell(cell_index=i)` over indices for the same effect. - ### What this means for first-time setup If you're setting this up fresh on a new env and following the recipe at the top of this document (or running `aexp jupyter-setup`), you should -never see Rounds 1-4 — the recipe disables/enables the right things +never see Rounds 1-3 — the recipe disables/enables the right things from the start. But if you ever: - Hit the kernel-WS-hangs-silently symptom → Round 1 @@ -543,8 +470,6 @@ from the start. But if you ever: `@jupyter-ai-contrib/server-documents`), NOT `nbmodel` - Hit `execute_cell returns "nbmodel not found"` → Round 3 (re-enable `jupyter_server_nbmodel`) -- Hit token mismatches that survive `setx` + Claude Desktop restart → - Round 4 (hardcode the token) The matching troubleshooting-table entries above link to these rounds. @@ -610,9 +535,7 @@ snapshot. | Tool | Version | Notes | |---|---|---| -| `uv` | latest | Used by `uvx jupyter-mcp-server` for laptop-side MCP_SERVER mode | -| `node` | LTS | Required for `npx mcp-remote` (JUPYTER_SERVER mode proxy) | -| `mcp-remote` | (npx-fetched) | stdio↔HTTP proxy bridging Claude Desktop to the cluster `/mcp` endpoint | +| `uv` | latest | Used by `uvx jupyter-mcp-server` for the laptop-side MCP server | | `jupyter-mcp-server` | 1.0.2+ | fetched ephemerally by `uvx`; not permanently installed | ### Cluster server endpoint (when Jupyter is running) @@ -624,9 +547,6 @@ snapshot. | `ws://127.0.0.1:/api/kernels//channels` | Kernel WebSocket (used by MCP_SERVER mode kernel-client) | | `http://127.0.0.1:/api/collaboration/session/` | Y.js doc room session (PUT to register; required for cell-document ops) | | `http://127.0.0.1:/api/fileid/id` and `.../path` | Standard file-ID lookup (jupyter-server-fileid) | -| `http://127.0.0.1:/mcp` | JUPYTER_SERVER mode MCP SSE endpoint (used by `mcp-remote` proxy) | -| `http://127.0.0.1:/mcp/healthz` | MCP server health check | -| `http://127.0.0.1:/mcp/tools/list` | MCP tools list | ### Re-capture this snapshot later diff --git a/tests/test_install.py b/tests/test_install.py index 8c7dd16..0333f52 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -762,22 +762,20 @@ def test_install_dry_run_also_refuses_source_tree(fresh_git_repo: Path) -> None: def test_install_with_jupyter_writes_mcp_entries(fresh_git_repo: Path) -> None: - """--with-jupyter writes both jupyter and jupyter-compute entries to .mcp.json - alongside the existing aexp entry.""" + """--with-jupyter writes the jupyter entry to .mcp.json alongside the + existing aexp entry. The legacy jupyter-compute server is not emitted.""" install_limina(fresh_git_repo, with_jupyter=True, dev=True) mcp_json = json.loads((fresh_git_repo / ".mcp.json").read_text(encoding="utf-8")) servers = mcp_json["mcpServers"] assert "aexp" in servers assert "jupyter" in servers - assert "jupyter-compute" in servers assert servers["jupyter"]["command"] == "uvx" assert "jupyter-mcp-server" in servers["jupyter"]["args"] - assert servers["jupyter-compute"]["command"] == "npx" - assert "mcp-remote" in servers["jupyter-compute"]["args"] + assert "jupyter-compute" not in servers def test_install_without_jupyter_omits_mcp_entries(fresh_git_repo: Path) -> None: - """Default install (no --with-jupyter) does NOT write jupyter entries.""" + """Default install (no --with-jupyter) does NOT write the jupyter entry.""" install_limina(fresh_git_repo, dev=True) mcp_json = json.loads((fresh_git_repo / ".mcp.json").read_text(encoding="utf-8")) servers = mcp_json["mcpServers"] @@ -847,25 +845,19 @@ def test_install_with_jupyter_preserves_user_entries(fresh_git_repo: Path) -> No assert servers["my_custom"]["command"] == "echo" assert "aexp" in servers assert "jupyter" in servers - assert "jupyter-compute" in servers def test_install_with_jupyter_preserves_existing_jupyter_entry(fresh_git_repo: Path) -> None: - """If the user has hardcoded a Windows-stable token in jupyter-compute, - re-running --with-jupyter must not clobber it. + """If the user has customized their `jupyter` entry, re-running + --with-jupyter must not clobber it: the install only ever ADDS the + entry, never overwrites an existing one. """ custom = { "mcpServers": { - "jupyter-compute": { - "command": "npx", - "args": [ - "-y", - "mcp-remote", - "http://127.0.0.1:3618/mcp", - "--allow-http", - "--header", - "Authorization:token MY_HARDCODED_LITERAL_TOKEN", - ], + "jupyter": { + "command": "uvx", + "args": ["jupyter-mcp-server"], + "env": {"MY_CUSTOM_MARKER": "kept"}, } } } @@ -874,12 +866,11 @@ def test_install_with_jupyter_preserves_existing_jupyter_entry(fresh_git_repo: P servers = json.loads( (fresh_git_repo / ".mcp.json").read_text(encoding="utf-8") )["mcpServers"] - # The user's hardcoded token survives — install only ADDS, never overwrites - # an existing jupyter-compute entry. - args = servers["jupyter-compute"]["args"] - assert any("MY_HARDCODED_LITERAL_TOKEN" in a for a in args) - # And the missing jupyter entry was added. - assert "jupyter" in servers + # The user's customized jupyter entry survives untouched — our generated + # entry has no `env` block, so its presence proves we did not overwrite. + assert servers["jupyter"].get("env") == {"MY_CUSTOM_MARKER": "kept"} + # And the aexp entry was still written. + assert "aexp" in servers def test_install_with_jupyter_slash_command_always_present(fresh_git_repo: Path) -> None: @@ -893,7 +884,7 @@ def test_install_with_jupyter_slash_command_always_present(fresh_git_repo: Path) def test_install_writes_promote_nb_slash_command(fresh_git_repo: Path) -> None: """The /aexp-promote-nb slash command is installed during default install, and its body contains the load-bearing guardrails (refuses without an - experiment ID, references the jupyter-compute MCP family, refuses to + experiment ID, references the jupyter MCP family, refuses to invent a tracked_notebook_run API).""" install_limina(fresh_git_repo, dev=True) slash_cmd = fresh_git_repo / ".claude" / "commands" / "aexp-promote-nb.md" @@ -903,7 +894,7 @@ def test_install_writes_promote_nb_slash_command(fresh_git_repo: Path) -> None: assert body.startswith("---\n") assert "description:" in body.split("---", 2)[1] # Self-check guidance for tool availability — degrades gracefully without MCP. - assert "mcp__jupyter-compute__" in body + assert "mcp__jupyter__" in body # The experiment-required guardrail (without it, promotion lands code in # experiments/ that has no H/E chain — the failure mode this command # exists to prevent).