diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0363b3a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: ci + +# The no-sign-in half of the interchangeability gate (codex-parity slice 4). A hosted runner has no Claude +# Code or Codex account, so this runs everything that needs none: builds, typecheck, the fixture-backed +# daemon suites (routing incl. mixed-provider + tool pre-flight, Codex App Server contract, storage) and the +# headless Swift routing/consent tests. The real-provider matrix is `npm run try-parity` on a signed-in Mac +# (docs/GO-LIVE.md P0). This exists because a red `tsc` once shipped past a tests-only gate. + +on: + push: + branches: [main] + pull_request: + +jobs: + node: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + - run: npm ci + - name: build (protocol → sidekick → sdk) + run: npm run build -w @relay/protocol && npm run build -w @relay/sidekick && npm run build -w @relay/sdk + - name: typecheck + run: npm run typecheck + - name: daemon suites (fixtures, no provider sign-in) + run: node --test packages/sidekick/dist/backends/routing.test.js packages/sidekick/dist/backends/codex.test.js packages/sidekick/dist/storage/find.test.js + + swift-headless: + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - name: launcher routing + run: cd packages/menubar && swiftc -parse-as-library LauncherRouting.swift LauncherRouting.test.swift -o /tmp/rt && /tmp/rt + - name: consent routing (dual-grant pre-selection) + run: cd packages/menubar && swiftc -parse-as-library ConsentRouting.swift ConsentRouting.test.swift -o /tmp/ct && /tmp/ct diff --git a/docs/GO-LIVE.md b/docs/GO-LIVE.md index 7bc0ffe..18bf725 100644 --- a/docs/GO-LIVE.md +++ b/docs/GO-LIVE.md @@ -33,6 +33,8 @@ This resolves the landing-page and onboarding concerns at the root. → **Decisi ## P0 — blocks launch +- [ ] **Provider parity gate green** — `npm run try-parity` with Claude Code AND Codex signed in on this Mac (same real flows per provider + the mixed cases; exit 0). After any Codex CLI upgrade also run the revalidation in docs/CODEX.md. Hosted CI (`.github/workflows/ci.yml`) covers the no-sign-in half. + - ✅ **Autopilot defaults to ON — FIXED.** Daemon now seeds `routines-control.json {off:true}` at first boot (`registry.ts seedControlIfAbsent`) and `control()` treats an absent/unreadable file as OFF, so autonomous routines never run without an explicit opt-in. Existing settings preserved. Verified with a diff --git a/examples/harness/dev-extension.mjs b/examples/harness/dev-extension.mjs index 0abe362..a87d7cb 100644 --- a/examples/harness/dev-extension.mjs +++ b/examples/harness/dev-extension.mjs @@ -44,13 +44,18 @@ export function connectAsExtension({ port, token, origin, onConsent, onEvent }) request: (method, params) => rpc({ type: "request", origin, method, params, sentAt: Date.now() }).then((m) => { if (m.error) throw Object.assign(new Error(m.error.message), m.error); return m.result; }), control: (action, args) => rpc({ type: "control", action, args }), /** Stream a completion; calls onDelta for each delta; resolves the final result on 'done'. */ - stream: (params, onDelta) => new Promise(async (res, rej) => { - const { streamId } = await api.request("claude_stream", params); - streams.set(streamId, (d) => { - onDelta?.(d); - if (d.type === "done") { streams.delete(streamId); res(d.result); } - else if (d.type === "error") { streams.delete(streamId); rej(Object.assign(new Error(d.error.message), d.error)); } - }); + // A daemon-time refusal (gate/routing rejects the claude_stream REQUEST before any stream exists — + // e.g. "Codex can't run WebSearch for this app") must reject THIS promise. The old `async` executor + // swallowed that rejection, so Node killed the whole harness as an unhandled rejection instead of + // letting the caller assert on it (found by run-parity, 2026-09-07). + stream: (params, onDelta) => new Promise((res, rej) => { + api.request("claude_stream", params).then(({ streamId }) => { + streams.set(streamId, (d) => { + onDelta?.(d); + if (d.type === "done") { streams.delete(streamId); res(d.result); } + else if (d.type === "error") { streams.delete(streamId); rej(Object.assign(new Error(d.error.message), d.error)); } + }); + }).catch(rej); }), close: () => ws.close(), }; diff --git a/examples/harness/run-parity.mjs b/examples/harness/run-parity.mjs new file mode 100644 index 0000000..b913998 --- /dev/null +++ b/examples/harness/run-parity.mjs @@ -0,0 +1,171 @@ +#!/usr/bin/env node +/** + * run-parity — THE INTERCHANGEABILITY GATE (codex-parity slice 4, 2026-09-07). + * + * Why this exists: the Codex integration was validated with ONLY Codex registered. The first time both + * providers were signed in it broke on the first mixed case (a Claude-only app routed to the global Codex + * default and denied). "Interchangeable" is the MIXED matrix, so this runs the same REAL flows once per + * signed-in provider — through an isolated daemon (own RELAY_DIR/port), real gate, real consent, real + * broker MCP, real models — and then the two mixed cases that bit us. Routing is asserted from the + * daemon's own `done` result (`result.model` is the RESOLVED model), not inferred. + * + * node examples/harness/run-parity.mjs # every signed-in provider in PARITY_PROVIDERS + * PARITY_PROVIDERS=codex node examples/harness/run-parity.mjs + * + * A provider that isn't online is reported as SKIP for every cell — never a silent pass. Exit 1 on any FAIL. + * Needs the providers signed in on THIS Mac (it makes real calls); hosted CI runs the fixture tests instead. + */ +import { spawn } from "node:child_process"; +import { mkdtempSync, writeFileSync, readFileSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { connectAsExtension } from "./dev-extension.mjs"; + +const PORT = Number(process.env.PARITY_PORT ?? 8794); +const WANT = (process.env.PARITY_PROVIDERS ?? "claude-code,codex").split(",").map((s) => s.trim()).filter(Boolean); +const TURN_MS = Number(process.env.PARITY_TURN_MS ?? 150_000); + +const dir = mkdtempSync(join(tmpdir(), "relay-parity-")); +const testServer = resolve("packages/sidekick/spike/test-mcp-server.mjs"); +writeFileSync(join(dir, "mcp.json"), JSON.stringify({ servers: { test: { command: process.execPath, args: [testServer] } } })); +const setModelsJson = (obj) => writeFileSync(join(dir, "models.json"), JSON.stringify(obj)); +setModelsJson({ disabled: [] }); + +const daemon = spawn(process.execPath, [resolve("packages/sidekick/dist/index.js")], { + env: { ...process.env, RELAY_DIR: dir, RELAY_PORT: String(PORT) }, + stdio: ["ignore", "inherit", "inherit"], +}); +process.on("exit", () => daemon.kill("SIGKILL")); +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +async function token() { + const f = join(dir, "pairing-token"); + for (let i = 0; i < 80; i++) { if (existsSync(f)) return readFileSync(f, "utf8").trim(); await sleep(150); } + throw new Error("no pairing token — daemon did not start"); +} +const withTimeout = (p, label) => Promise.race([p, new Promise((_, rej) => setTimeout(() => rej(new Error(`${label}: no result within ${TURN_MS / 1000}s`)), TURN_MS))]); + +// Per-origin consent PLAN: which models the harness approves for that app (this is how a cell is pinned to +// ONE provider), and whether write consents are denied. The daemon classifies tools; we approve what it asks. +const plan = new Map(); +function onConsent(kind, body) { + const p = plan.get(body.origin) ?? {}; + if (kind === "consent:connect") { + const models = p.models ?? (body.models?.requested?.length ? body.models.requested : (body.models?.available ?? []).slice(0, 1)); + const tools = (body.tools ?? []).map((t) => ({ name: t.name, access: t.access })); + return { models, tools, budgets: { maxTokensPerDay: 500_000, maxCallsPerMin: 60 } }; + } + return !p.denyWrites; +} + +const rows = []; +const mark = (provider, check, status, detail = "") => { rows.push({ provider, check, status, detail }); console.log(` ${status === "PASS" ? "✅" : status === "SKIP" ? "⏭ " : "❌"} [${provider}] ${check}${detail ? " — " + detail : ""}`); }; +async function cell(provider, check, fn) { + try { const detail = await fn(); mark(provider, check, "PASS", detail ?? ""); } + catch (err) { mark(provider, check, "FAIL", String(err?.message ?? err).slice(0, 160)); } +} + +async function connect(tok, origin, models, extra = {}) { + plan.set(origin, { models, ...extra }); + const t0 = Date.now(); + for (;;) { + try { return await connectAsExtension({ port: PORT, token: tok, origin, onConsent }); } + catch (err) { if (Date.now() - t0 > 20_000) throw err; await sleep(250); } + } +} + +async function main() { + const tok = await token(); + console.log(`\nparity gate · daemon on :${PORT} · state ${dir}\n`); + + // ── discovery: which providers are actually online here, and which models are whose ── + const probe = await connect(tok, "https://probe.parity", []); + const caps = await probe.request("claude_capabilities", {}); + probe.close(); + const byBackend = {}; + for (const m of caps.modelInfo ?? []) (byBackend[m.backend] ??= []).push(m.id); + const online = new Set(caps.backends ?? []); + const providers = WANT.filter((p) => online.has(p) && byBackend[p]?.length); + for (const p of WANT.filter((p) => !providers.includes(p))) mark(p, "provider online", "SKIP", online.has(p) ? "no models advertised" : "not signed in / offline on this Mac"); + console.log(`providers under test: ${providers.join(", ") || "(none)"}\n`); + + // ── per-provider matrix: the same three real flows, app granted THIS provider only ── + for (const p of providers) { + const models = byBackend[p]; + console.log(`── ${p} · models ${models.join(", ")}`); + await cell(p, "plain completion (implicit model stays in grant)", async () => { + const app = await connect(tok, `https://chat.${p}.parity`, models); + await app.request("claude_connect", { reason: "parity chat", tools: [] }); + const res = await withTimeout(app.stream({ prompt: "In one sentence, what does a 'bring your own model' broker do?" }), "plain"); + app.close(); + if (!models.includes(res.model)) throw new Error(`routed to ${res.model}, expected one of ${models.join("/")}`); + if (!res.text?.trim()) throw new Error("empty reply"); + return `${res.model} · ${res.text.length}ch`; + }); + await cell(p, "agentic read via broker MCP tool (tools portable through the broker)", async () => { + const app = await connect(tok, `https://notes.${p}.parity`, models); + await app.request("claude_connect", { reason: "parity notes", tools: ["mcp__test__read_note"] }); + let proposed = false; + const res = await withTimeout(app.stream({ prompt: "Use the read_note tool to read the note with id 'groceries', then say in one sentence what it says.", agentic: true }, + (d) => { if (d.type === "tool_proposed") proposed = true; }), "agentic read"); + app.close(); + if (!models.includes(res.model)) throw new Error(`routed to ${res.model}`); + if (!proposed) throw new Error("model never proposed the tool"); + return `${res.model} · tool proposed`; + }); + await cell(p, "write consent: approve then deny (per-action gate)", async () => { + const app = await connect(tok, `https://outbox.${p}.parity`, models, { denyWrites: false }); + await app.request("claude_connect", { reason: "parity outbox", tools: ["mcp__test__read_note", "mcp__test__send_note"] }); + const results = []; + await withTimeout(app.stream({ prompt: "Use the send_note tool to send a note to 'bob' with body 'ship it'. Then confirm in one short sentence.", agentic: true }, + (d) => { if (d.type === "tool_result") results.push(d.result.ok); }), "approve send"); + plan.get(`https://outbox.${p}.parity`).denyWrites = true; + await withTimeout(app.stream({ prompt: "Use the send_note tool to send a note to 'alice' with body 'lunch at noon?'. Then say in one short sentence whether it was sent.", agentic: true }, + (d) => { if (d.type === "tool_result") results.push(d.result.ok); }), "deny send"); + app.close(); + if (!results.includes(true)) throw new Error("approved send never ran"); + if (!results.includes(false)) throw new Error("denied send was not blocked"); + return `ran=${results.filter(Boolean).length} blocked=${results.filter((r) => r === false).length}`; + }); + } + + // ── the MIXED cases — the ones a single-provider validation can never see ── + const claude = byBackend["claude-code"] ?? [], codex = byBackend["codex"] ?? []; + const mixed = providers.includes("claude-code") && providers.includes("codex"); + if (!mixed) { mark("mixed", "global Codex default vs Claude-only grant", "SKIP", "needs both providers online"); mark("mixed", "tool pre-flight (Claude-only tool routes to Claude)", "SKIP", "needs both providers online"); } + else { + setModelsJson({ disabled: [], defaultModel: codex[0] }); // the user's global default is Codex + await cell("mixed", `global Codex default (${codex[0]}) vs a Claude-only app`, async () => { + const app = await connect(tok, "https://claudeonly.parity", claude); + await app.request("claude_connect", { reason: "parity claude-only", tools: [] }); + const res = await withTimeout(app.stream({ prompt: "Say OK." }), "claude-only implicit"); + app.close(); + if (!claude.includes(res.model)) throw new Error(`routed OUTSIDE the grant to ${res.model} (the 2026-09-07 Brandbrain bug)`); + return `stayed in grant: ${res.model}`; + }); + await cell("mixed", "tool pre-flight: dual-granted app, Claude-only tool (WebSearch)", async () => { + const app = await connect(tok, "https://dual.parity", [...claude, ...codex]); + await app.request("claude_connect", { reason: "parity dual", tools: ["WebSearch"] }); + const plain = await withTimeout(app.stream({ prompt: "Say OK." }), "dual plain"); + if (!codex.includes(plain.model)) throw new Error(`plain turn should honour the Codex default, got ${plain.model}`); + const research = await withTimeout(app.stream({ prompt: "Search the web for today's date and reply with just the year.", agentic: true }), "dual agentic"); + if (!claude.includes(research.model)) throw new Error(`agentic turn should route to Claude (WebSearch), got ${research.model}`); + let refused = ""; + try { await withTimeout(app.stream({ prompt: "Search the web for anything.", agentic: true, model: codex[0] }), "explicit codex agentic"); } + catch (err) { refused = String(err?.message ?? err); } + app.close(); + if (!/can't run WebSearch/.test(refused)) throw new Error(`explicit Codex + WebSearch should be refused by name, got: ${refused || "(no error)"}`); + return `plain→${plain.model} · agentic→${research.model} · explicit Codex refused by name`; + }); + setModelsJson({ disabled: [] }); + } + + // ── report ── + const fails = rows.filter((r) => r.status === "FAIL").length, skips = rows.filter((r) => r.status === "SKIP").length; + console.log(`\n${"provider".padEnd(12)} ${"check".padEnd(64)} status`); + for (const r of rows) console.log(`${r.provider.padEnd(12)} ${r.check.slice(0, 64).padEnd(64)} ${r.status}`); + console.log(`\n${rows.length - fails - skips} PASS · ${fails} FAIL · ${skips} SKIP`); + daemon.kill("SIGKILL"); + process.exit(fails ? 1 : 0); +} + +main().catch((err) => { console.error("parity harness error:", err); daemon.kill("SIGKILL"); process.exit(1); }); diff --git a/package.json b/package.json index e0653f1..d64da2d 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "relay", "private": true, "version": "0.0.0", - "description": "Your private AI workspace. A local sidekick brokers your own model + MCP tools to any website through window.claude, under per-origin, out-of-band consent — your data stays yours.", + "description": "Your private AI workspace. A local sidekick brokers your own model + MCP tools to any website through window.claude, under per-origin, out-of-band consent \u2014 your data stays yours.", "workspaces": [ "packages/*", "examples/*" @@ -28,7 +28,8 @@ "daemon:restart": "npm run build -w @relay/sidekick && launchctl kickstart -k gui/$(id -u)/com.relay.sidekick && echo restarted", "runner": "node examples/runner/serve.mjs", "try-team-cloud": "npm run build -w @relay/protocol && npm run build -w @relay/sidekick && node examples/harness/run-team-cloud.mjs", - "try-compat": "npm run build -w @relay/protocol && npm run build -w @relay/sidekick && node examples/harness/run-compat.mjs" + "try-compat": "npm run build -w @relay/protocol && npm run build -w @relay/sidekick && node examples/harness/run-compat.mjs", + "try-parity": "npm run build -w @relay/protocol && npm run build -w @relay/sidekick && node examples/harness/run-parity.mjs" }, "devDependencies": { "esbuild": "^0.28.1",