From a89cda45b524dce49ac640cc11d6d7580f376473 Mon Sep 17 00:00:00 2001 From: sameeeeeeep <191771413+sameeeeeeep@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:48:13 +0530 Subject: [PATCH 1/2] =?UTF-8?q?feat(harness):=20run-parity=20=E2=80=94=20t?= =?UTF-8?q?he=20interchangeability=20gate=20(same=20real=20flows=20per=20p?= =?UTF-8?q?rovider=20+=20the=20mixed=20cases)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex-parity slice 4a. The Codex integration was validated with only Codex registered; the first mixed run broke. This gate spins an isolated daemon (own RELAY_DIR/port, real gate/consent/broker MCP/models) and runs, once per signed-in provider with the app's grant restricted to that provider: a plain completion (implicit model must stay in grant), an agentic read through a broker MCP tool (tools ARE portable via the broker), and write consent approve-then-deny. Then the two mixed cases: a Claude-only app with the global default on Codex must stay in its grant (the 2026-09-07 Brandbrain bug), and a dual-granted app with a Claude-only tool must route its agentic turn to Claude, honour the Codex default for plain turns, and refuse an explicit Codex request by name. Routing is asserted from the daemon's own `done` result (`result.model` is the resolved model) — not inferred. A provider that isn't online is a SKIP, never a silent pass. `npm run try-parity`. First real run: 8 PASS / 0 FAIL on claude-code + codex. Also fixes the shared harness client: `stream()` used an async Promise executor, so a daemon-time refusal of the claude_stream REQUEST (before any stream existed) was an unhandled rejection that killed the whole harness process instead of rejecting the promise the caller could assert on. Co-Authored-By: Claude Fable 5.1 --- examples/harness/dev-extension.mjs | 19 ++-- examples/harness/run-parity.mjs | 171 +++++++++++++++++++++++++++++ package.json | 5 +- 3 files changed, 186 insertions(+), 9 deletions(-) create mode 100644 examples/harness/run-parity.mjs diff --git a/examples/harness/dev-extension.mjs b/examples/harness/dev-extension.mjs index 0abe362..a87d7cb 100644 --- a/examples/harness/dev-extension.mjs +++ b/examples/harness/dev-extension.mjs @@ -44,13 +44,18 @@ export function connectAsExtension({ port, token, origin, onConsent, onEvent }) request: (method, params) => rpc({ type: "request", origin, method, params, sentAt: Date.now() }).then((m) => { if (m.error) throw Object.assign(new Error(m.error.message), m.error); return m.result; }), control: (action, args) => rpc({ type: "control", action, args }), /** Stream a completion; calls onDelta for each delta; resolves the final result on 'done'. */ - stream: (params, onDelta) => new Promise(async (res, rej) => { - const { streamId } = await api.request("claude_stream", params); - streams.set(streamId, (d) => { - onDelta?.(d); - if (d.type === "done") { streams.delete(streamId); res(d.result); } - else if (d.type === "error") { streams.delete(streamId); rej(Object.assign(new Error(d.error.message), d.error)); } - }); + // A daemon-time refusal (gate/routing rejects the claude_stream REQUEST before any stream exists — + // e.g. "Codex can't run WebSearch for this app") must reject THIS promise. The old `async` executor + // swallowed that rejection, so Node killed the whole harness as an unhandled rejection instead of + // letting the caller assert on it (found by run-parity, 2026-09-07). + stream: (params, onDelta) => new Promise((res, rej) => { + api.request("claude_stream", params).then(({ streamId }) => { + streams.set(streamId, (d) => { + onDelta?.(d); + if (d.type === "done") { streams.delete(streamId); res(d.result); } + else if (d.type === "error") { streams.delete(streamId); rej(Object.assign(new Error(d.error.message), d.error)); } + }); + }).catch(rej); }), close: () => ws.close(), }; diff --git a/examples/harness/run-parity.mjs b/examples/harness/run-parity.mjs new file mode 100644 index 0000000..b913998 --- /dev/null +++ b/examples/harness/run-parity.mjs @@ -0,0 +1,171 @@ +#!/usr/bin/env node +/** + * run-parity — THE INTERCHANGEABILITY GATE (codex-parity slice 4, 2026-09-07). + * + * Why this exists: the Codex integration was validated with ONLY Codex registered. The first time both + * providers were signed in it broke on the first mixed case (a Claude-only app routed to the global Codex + * default and denied). "Interchangeable" is the MIXED matrix, so this runs the same REAL flows once per + * signed-in provider — through an isolated daemon (own RELAY_DIR/port), real gate, real consent, real + * broker MCP, real models — and then the two mixed cases that bit us. Routing is asserted from the + * daemon's own `done` result (`result.model` is the RESOLVED model), not inferred. + * + * node examples/harness/run-parity.mjs # every signed-in provider in PARITY_PROVIDERS + * PARITY_PROVIDERS=codex node examples/harness/run-parity.mjs + * + * A provider that isn't online is reported as SKIP for every cell — never a silent pass. Exit 1 on any FAIL. + * Needs the providers signed in on THIS Mac (it makes real calls); hosted CI runs the fixture tests instead. + */ +import { spawn } from "node:child_process"; +import { mkdtempSync, writeFileSync, readFileSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { connectAsExtension } from "./dev-extension.mjs"; + +const PORT = Number(process.env.PARITY_PORT ?? 8794); +const WANT = (process.env.PARITY_PROVIDERS ?? "claude-code,codex").split(",").map((s) => s.trim()).filter(Boolean); +const TURN_MS = Number(process.env.PARITY_TURN_MS ?? 150_000); + +const dir = mkdtempSync(join(tmpdir(), "relay-parity-")); +const testServer = resolve("packages/sidekick/spike/test-mcp-server.mjs"); +writeFileSync(join(dir, "mcp.json"), JSON.stringify({ servers: { test: { command: process.execPath, args: [testServer] } } })); +const setModelsJson = (obj) => writeFileSync(join(dir, "models.json"), JSON.stringify(obj)); +setModelsJson({ disabled: [] }); + +const daemon = spawn(process.execPath, [resolve("packages/sidekick/dist/index.js")], { + env: { ...process.env, RELAY_DIR: dir, RELAY_PORT: String(PORT) }, + stdio: ["ignore", "inherit", "inherit"], +}); +process.on("exit", () => daemon.kill("SIGKILL")); +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +async function token() { + const f = join(dir, "pairing-token"); + for (let i = 0; i < 80; i++) { if (existsSync(f)) return readFileSync(f, "utf8").trim(); await sleep(150); } + throw new Error("no pairing token — daemon did not start"); +} +const withTimeout = (p, label) => Promise.race([p, new Promise((_, rej) => setTimeout(() => rej(new Error(`${label}: no result within ${TURN_MS / 1000}s`)), TURN_MS))]); + +// Per-origin consent PLAN: which models the harness approves for that app (this is how a cell is pinned to +// ONE provider), and whether write consents are denied. The daemon classifies tools; we approve what it asks. +const plan = new Map(); +function onConsent(kind, body) { + const p = plan.get(body.origin) ?? {}; + if (kind === "consent:connect") { + const models = p.models ?? (body.models?.requested?.length ? body.models.requested : (body.models?.available ?? []).slice(0, 1)); + const tools = (body.tools ?? []).map((t) => ({ name: t.name, access: t.access })); + return { models, tools, budgets: { maxTokensPerDay: 500_000, maxCallsPerMin: 60 } }; + } + return !p.denyWrites; +} + +const rows = []; +const mark = (provider, check, status, detail = "") => { rows.push({ provider, check, status, detail }); console.log(` ${status === "PASS" ? "✅" : status === "SKIP" ? "⏭ " : "❌"} [${provider}] ${check}${detail ? " — " + detail : ""}`); }; +async function cell(provider, check, fn) { + try { const detail = await fn(); mark(provider, check, "PASS", detail ?? ""); } + catch (err) { mark(provider, check, "FAIL", String(err?.message ?? err).slice(0, 160)); } +} + +async function connect(tok, origin, models, extra = {}) { + plan.set(origin, { models, ...extra }); + const t0 = Date.now(); + for (;;) { + try { return await connectAsExtension({ port: PORT, token: tok, origin, onConsent }); } + catch (err) { if (Date.now() - t0 > 20_000) throw err; await sleep(250); } + } +} + +async function main() { + const tok = await token(); + console.log(`\nparity gate · daemon on :${PORT} · state ${dir}\n`); + + // ── discovery: which providers are actually online here, and which models are whose ── + const probe = await connect(tok, "https://probe.parity", []); + const caps = await probe.request("claude_capabilities", {}); + probe.close(); + const byBackend = {}; + for (const m of caps.modelInfo ?? []) (byBackend[m.backend] ??= []).push(m.id); + const online = new Set(caps.backends ?? []); + const providers = WANT.filter((p) => online.has(p) && byBackend[p]?.length); + for (const p of WANT.filter((p) => !providers.includes(p))) mark(p, "provider online", "SKIP", online.has(p) ? "no models advertised" : "not signed in / offline on this Mac"); + console.log(`providers under test: ${providers.join(", ") || "(none)"}\n`); + + // ── per-provider matrix: the same three real flows, app granted THIS provider only ── + for (const p of providers) { + const models = byBackend[p]; + console.log(`── ${p} · models ${models.join(", ")}`); + await cell(p, "plain completion (implicit model stays in grant)", async () => { + const app = await connect(tok, `https://chat.${p}.parity`, models); + await app.request("claude_connect", { reason: "parity chat", tools: [] }); + const res = await withTimeout(app.stream({ prompt: "In one sentence, what does a 'bring your own model' broker do?" }), "plain"); + app.close(); + if (!models.includes(res.model)) throw new Error(`routed to ${res.model}, expected one of ${models.join("/")}`); + if (!res.text?.trim()) throw new Error("empty reply"); + return `${res.model} · ${res.text.length}ch`; + }); + await cell(p, "agentic read via broker MCP tool (tools portable through the broker)", async () => { + const app = await connect(tok, `https://notes.${p}.parity`, models); + await app.request("claude_connect", { reason: "parity notes", tools: ["mcp__test__read_note"] }); + let proposed = false; + const res = await withTimeout(app.stream({ prompt: "Use the read_note tool to read the note with id 'groceries', then say in one sentence what it says.", agentic: true }, + (d) => { if (d.type === "tool_proposed") proposed = true; }), "agentic read"); + app.close(); + if (!models.includes(res.model)) throw new Error(`routed to ${res.model}`); + if (!proposed) throw new Error("model never proposed the tool"); + return `${res.model} · tool proposed`; + }); + await cell(p, "write consent: approve then deny (per-action gate)", async () => { + const app = await connect(tok, `https://outbox.${p}.parity`, models, { denyWrites: false }); + await app.request("claude_connect", { reason: "parity outbox", tools: ["mcp__test__read_note", "mcp__test__send_note"] }); + const results = []; + await withTimeout(app.stream({ prompt: "Use the send_note tool to send a note to 'bob' with body 'ship it'. Then confirm in one short sentence.", agentic: true }, + (d) => { if (d.type === "tool_result") results.push(d.result.ok); }), "approve send"); + plan.get(`https://outbox.${p}.parity`).denyWrites = true; + await withTimeout(app.stream({ prompt: "Use the send_note tool to send a note to 'alice' with body 'lunch at noon?'. Then say in one short sentence whether it was sent.", agentic: true }, + (d) => { if (d.type === "tool_result") results.push(d.result.ok); }), "deny send"); + app.close(); + if (!results.includes(true)) throw new Error("approved send never ran"); + if (!results.includes(false)) throw new Error("denied send was not blocked"); + return `ran=${results.filter(Boolean).length} blocked=${results.filter((r) => r === false).length}`; + }); + } + + // ── the MIXED cases — the ones a single-provider validation can never see ── + const claude = byBackend["claude-code"] ?? [], codex = byBackend["codex"] ?? []; + const mixed = providers.includes("claude-code") && providers.includes("codex"); + if (!mixed) { mark("mixed", "global Codex default vs Claude-only grant", "SKIP", "needs both providers online"); mark("mixed", "tool pre-flight (Claude-only tool routes to Claude)", "SKIP", "needs both providers online"); } + else { + setModelsJson({ disabled: [], defaultModel: codex[0] }); // the user's global default is Codex + await cell("mixed", `global Codex default (${codex[0]}) vs a Claude-only app`, async () => { + const app = await connect(tok, "https://claudeonly.parity", claude); + await app.request("claude_connect", { reason: "parity claude-only", tools: [] }); + const res = await withTimeout(app.stream({ prompt: "Say OK." }), "claude-only implicit"); + app.close(); + if (!claude.includes(res.model)) throw new Error(`routed OUTSIDE the grant to ${res.model} (the 2026-09-07 Brandbrain bug)`); + return `stayed in grant: ${res.model}`; + }); + await cell("mixed", "tool pre-flight: dual-granted app, Claude-only tool (WebSearch)", async () => { + const app = await connect(tok, "https://dual.parity", [...claude, ...codex]); + await app.request("claude_connect", { reason: "parity dual", tools: ["WebSearch"] }); + const plain = await withTimeout(app.stream({ prompt: "Say OK." }), "dual plain"); + if (!codex.includes(plain.model)) throw new Error(`plain turn should honour the Codex default, got ${plain.model}`); + const research = await withTimeout(app.stream({ prompt: "Search the web for today's date and reply with just the year.", agentic: true }), "dual agentic"); + if (!claude.includes(research.model)) throw new Error(`agentic turn should route to Claude (WebSearch), got ${research.model}`); + let refused = ""; + try { await withTimeout(app.stream({ prompt: "Search the web for anything.", agentic: true, model: codex[0] }), "explicit codex agentic"); } + catch (err) { refused = String(err?.message ?? err); } + app.close(); + if (!/can't run WebSearch/.test(refused)) throw new Error(`explicit Codex + WebSearch should be refused by name, got: ${refused || "(no error)"}`); + return `plain→${plain.model} · agentic→${research.model} · explicit Codex refused by name`; + }); + setModelsJson({ disabled: [] }); + } + + // ── report ── + const fails = rows.filter((r) => r.status === "FAIL").length, skips = rows.filter((r) => r.status === "SKIP").length; + console.log(`\n${"provider".padEnd(12)} ${"check".padEnd(64)} status`); + for (const r of rows) console.log(`${r.provider.padEnd(12)} ${r.check.slice(0, 64).padEnd(64)} ${r.status}`); + console.log(`\n${rows.length - fails - skips} PASS · ${fails} FAIL · ${skips} SKIP`); + daemon.kill("SIGKILL"); + process.exit(fails ? 1 : 0); +} + +main().catch((err) => { console.error("parity harness error:", err); daemon.kill("SIGKILL"); process.exit(1); }); diff --git a/package.json b/package.json index e0653f1..d64da2d 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "relay", "private": true, "version": "0.0.0", - "description": "Your private AI workspace. A local sidekick brokers your own model + MCP tools to any website through window.claude, under per-origin, out-of-band consent — your data stays yours.", + "description": "Your private AI workspace. A local sidekick brokers your own model + MCP tools to any website through window.claude, under per-origin, out-of-band consent \u2014 your data stays yours.", "workspaces": [ "packages/*", "examples/*" @@ -28,7 +28,8 @@ "daemon:restart": "npm run build -w @relay/sidekick && launchctl kickstart -k gui/$(id -u)/com.relay.sidekick && echo restarted", "runner": "node examples/runner/serve.mjs", "try-team-cloud": "npm run build -w @relay/protocol && npm run build -w @relay/sidekick && node examples/harness/run-team-cloud.mjs", - "try-compat": "npm run build -w @relay/protocol && npm run build -w @relay/sidekick && node examples/harness/run-compat.mjs" + "try-compat": "npm run build -w @relay/protocol && npm run build -w @relay/sidekick && node examples/harness/run-compat.mjs", + "try-parity": "npm run build -w @relay/protocol && npm run build -w @relay/sidekick && node examples/harness/run-parity.mjs" }, "devDependencies": { "esbuild": "^0.28.1", From 874f3c5253829bd91cd5c01c648efb93f319d4b1 Mon Sep 17 00:00:00 2001 From: sameeeeeeep <191771413+sameeeeeeep@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:48:13 +0530 Subject: [PATCH 2/2] =?UTF-8?q?ci:=20first=20workflow=20=E2=80=94=20builds?= =?UTF-8?q?,=20typecheck,=20fixture=20daemon=20suites,=20headless=20Swift?= =?UTF-8?q?=20tests=20(no=20sign-in=20needed)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex-parity slice 4b. There was no CI. A hosted runner has no Claude Code or Codex account, so this runs the half that needs none: protocol/sidekick/sdk builds, workspace typecheck, the fixture-backed daemon suites (routing incl. the mixed-provider and tool pre-flight cases, the Codex App Server contract, storage) on ubuntu, and the LauncherRouting + ConsentRouting headless tests on macos. The real-provider matrix is `npm run try-parity` on a signed-in Mac, now a P0 line in docs/GO-LIVE.md. Motivation on record: a red `tsc` shipped past a tests-only gate earlier today. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 38 ++++++++++++++++++++++++++++++++++++++ docs/GO-LIVE.md | 2 ++ 2 files changed, 40 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0363b3a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: ci + +# The no-sign-in half of the interchangeability gate (codex-parity slice 4). A hosted runner has no Claude +# Code or Codex account, so this runs everything that needs none: builds, typecheck, the fixture-backed +# daemon suites (routing incl. mixed-provider + tool pre-flight, Codex App Server contract, storage) and the +# headless Swift routing/consent tests. The real-provider matrix is `npm run try-parity` on a signed-in Mac +# (docs/GO-LIVE.md P0). This exists because a red `tsc` once shipped past a tests-only gate. + +on: + push: + branches: [main] + pull_request: + +jobs: + node: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + - run: npm ci + - name: build (protocol → sidekick → sdk) + run: npm run build -w @relay/protocol && npm run build -w @relay/sidekick && npm run build -w @relay/sdk + - name: typecheck + run: npm run typecheck + - name: daemon suites (fixtures, no provider sign-in) + run: node --test packages/sidekick/dist/backends/routing.test.js packages/sidekick/dist/backends/codex.test.js packages/sidekick/dist/storage/find.test.js + + swift-headless: + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - name: launcher routing + run: cd packages/menubar && swiftc -parse-as-library LauncherRouting.swift LauncherRouting.test.swift -o /tmp/rt && /tmp/rt + - name: consent routing (dual-grant pre-selection) + run: cd packages/menubar && swiftc -parse-as-library ConsentRouting.swift ConsentRouting.test.swift -o /tmp/ct && /tmp/ct diff --git a/docs/GO-LIVE.md b/docs/GO-LIVE.md index 7bc0ffe..18bf725 100644 --- a/docs/GO-LIVE.md +++ b/docs/GO-LIVE.md @@ -33,6 +33,8 @@ This resolves the landing-page and onboarding concerns at the root. → **Decisi ## P0 — blocks launch +- [ ] **Provider parity gate green** — `npm run try-parity` with Claude Code AND Codex signed in on this Mac (same real flows per provider + the mixed cases; exit 0). After any Codex CLI upgrade also run the revalidation in docs/CODEX.md. Hosted CI (`.github/workflows/ci.yml`) covers the no-sign-in half. + - ✅ **Autopilot defaults to ON — FIXED.** Daemon now seeds `routines-control.json {off:true}` at first boot (`registry.ts seedControlIfAbsent`) and `control()` treats an absent/unreadable file as OFF, so autonomous routines never run without an explicit opt-in. Existing settings preserved. Verified with a