From dbef530fbd5dce01735c34ad61e9a8cd5e206039 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Fri, 25 Sep 2026 22:34:44 -0700 Subject: [PATCH] test(coordination): add real HTTP producer and consumer fixtures --- evals/coordination-fixtures/README.md | 32 +++++++++++++++++++ evals/coordination-fixtures/api/README.md | 8 +++++ evals/coordination-fixtures/api/package.json | 1 + evals/coordination-fixtures/api/server.mjs | 18 +++++++++++ .../coordination-fixtures/api/server.test.mjs | 13 ++++++++ evals/coordination-fixtures/client/README.md | 6 ++++ evals/coordination-fixtures/client/client.mjs | 9 ++++++ .../client/client.test.mjs | 15 +++++++++ .../coordination-fixtures/client/package.json | 1 + evals/coordination-fixtures/verify-pair.mjs | 18 +++++++++++ 10 files changed, 121 insertions(+) create mode 100644 evals/coordination-fixtures/README.md create mode 100644 evals/coordination-fixtures/api/README.md create mode 100644 evals/coordination-fixtures/api/package.json create mode 100644 evals/coordination-fixtures/api/server.mjs create mode 100644 evals/coordination-fixtures/api/server.test.mjs create mode 100644 evals/coordination-fixtures/client/README.md create mode 100644 evals/coordination-fixtures/client/client.mjs create mode 100644 evals/coordination-fixtures/client/client.test.mjs create mode 100644 evals/coordination-fixtures/client/package.json create mode 100644 evals/coordination-fixtures/verify-pair.mjs diff --git a/evals/coordination-fixtures/README.md b/evals/coordination-fixtures/README.md new file mode 100644 index 0000000..34b6b1c --- /dev/null +++ b/evals/coordination-fixtures/README.md @@ -0,0 +1,32 @@ +# Real producer/consumer coordination fixture + +Use separate disposable repositories containing `api/` and `client/`. Both have +an honest, dependency-free `npm test` suite and a working HTTP relationship. +The client is an API consumer, not a documentation-only repository. + +Parent task: rename the arithmetic API's `POST /add` endpoint to `POST /sum`, +remove the old endpoint, and update the client to call the new endpoint. Preserve +the response shape, exported server/client functions, validation and error +handling; update each declared suite to the new contract. + +Before launch, run both suites and: + +```sh +node evals/coordination-fixtures/verify-pair.mjs API_CHECKOUT CLIENT_CHECKOUT /add +``` + +Keep `verify-pair.mjs` outside both repositories. After API and client changes, +check out the exact API anchor and exact client commit into separate directories, +run their declared suites, then run the same command with `/sum`. It exercises +actual HTTP, positive and fractional/negative operands, the renamed endpoint, +and refusal of the removed endpoint. Record both commit hashes and outputs. + +Ship's built-in coordination check remains a static compatibility scan. The +external pair test is additional evidence and must not be described as execution +performed by that scan. An uncertain verdict remains uncertain until resolved +through the product's normal decision path. No automatic merge is implied by +this fixture; retain each approval and its evidence. + +Fixture self-check: the original pair passes `/add` and fails `/sum`; changing +only the API to `/sum` still fails; changing the matching client path as well +passes `/sum`. These are grader sensitivity controls, not a live Ship receipt. diff --git a/evals/coordination-fixtures/api/README.md b/evals/coordination-fixtures/api/README.md new file mode 100644 index 0000000..04df654 --- /dev/null +++ b/evals/coordination-fixtures/api/README.md @@ -0,0 +1,8 @@ +# Arithmetic API fixture + +Node 22+, no dependencies. `npm test` runs the real HTTP contract. + +`server()` from `server.mjs` creates an unbound HTTP server. `POST /add` accepts +JSON `{a: number, b: number}` and returns `{result: number}`. Invalid operands +return 400; unknown endpoints return 404. Bind to a loopback ephemeral port in +tests. Keep the exported server factory and response shape stable during a rename. diff --git a/evals/coordination-fixtures/api/package.json b/evals/coordination-fixtures/api/package.json new file mode 100644 index 0000000..c1f36aa --- /dev/null +++ b/evals/coordination-fixtures/api/package.json @@ -0,0 +1 @@ +{"name":"coordination-api-fixture","private":true,"type":"module","scripts":{"test":"node --test server.test.mjs"}} diff --git a/evals/coordination-fixtures/api/server.mjs b/evals/coordination-fixtures/api/server.mjs new file mode 100644 index 0000000..340b569 --- /dev/null +++ b/evals/coordination-fixtures/api/server.mjs @@ -0,0 +1,18 @@ +import { createServer } from 'node:http'; +export function server() { + return createServer(async (req, res) => { + if (req.method !== 'POST' || req.url !== '/add') { + res.writeHead(404).end(); return; + } + let text = ''; + for await (const chunk of req) { + text += chunk; + if (text.length > 1024) { res.writeHead(413).end(); return; } + } + try { + const { a, b } = JSON.parse(text); + if (!Number.isFinite(a) || !Number.isFinite(b)) { res.writeHead(400).end(); return; } + res.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ result: a + b })); + } catch { res.writeHead(400).end(); } + }); +} diff --git a/evals/coordination-fixtures/api/server.test.mjs b/evals/coordination-fixtures/api/server.test.mjs new file mode 100644 index 0000000..bbb589f --- /dev/null +++ b/evals/coordination-fixtures/api/server.test.mjs @@ -0,0 +1,13 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { server } from './server.mjs'; + +test('Add HTTP contract', async (t) => { + const s = server(); await new Promise(r => s.listen(0, '127.0.0.1', r)); + t.after(() => { s.closeAllConnections(); s.close(); }); + const base = `http://127.0.0.1:${s.address().port}`; + const call = (body, path = '/add') => fetch(base + path, { method: 'POST', body: JSON.stringify(body) }); + assert.deepEqual(await (await call({ a: 2, b: 5 })).json(), { result: 7 }); + assert.equal((await call({ a: '2', b: 5 })).status, 400); + assert.equal((await call({ a: 2, b: 5 }, '/missing')).status, 404); +}); diff --git a/evals/coordination-fixtures/client/README.md b/evals/coordination-fixtures/client/README.md new file mode 100644 index 0000000..d61b64f --- /dev/null +++ b/evals/coordination-fixtures/client/README.md @@ -0,0 +1,6 @@ +# Arithmetic client fixture + +Node 22+, no dependencies. `npm test` verifies the client request contract and +error handling. `calculate(baseURL, a, b)` calls the API's `POST /add`, returning +the numeric `result`. The optional fourth argument is an injected fetch function. +Keep the export and arguments stable when updating to a renamed API endpoint. diff --git a/evals/coordination-fixtures/client/client.mjs b/evals/coordination-fixtures/client/client.mjs new file mode 100644 index 0000000..d5830d5 --- /dev/null +++ b/evals/coordination-fixtures/client/client.mjs @@ -0,0 +1,9 @@ +export async function calculate(baseURL, a, b, request = fetch) { + const response = await request(new URL('/add', baseURL), { + method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ a, b }), + }); + if (!response.ok) throw new Error(`Arithmetic API returned ${response.status}`); + const data = await response.json(); + if (!Number.isFinite(data.result)) throw new Error('Arithmetic API returned an invalid result'); + return data.result; +} diff --git a/evals/coordination-fixtures/client/client.test.mjs b/evals/coordination-fixtures/client/client.test.mjs new file mode 100644 index 0000000..0311745 --- /dev/null +++ b/evals/coordination-fixtures/client/client.test.mjs @@ -0,0 +1,15 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { calculate } from './client.mjs'; + +test('client sends the Add contract and returns its result', async () => { + const result = await calculate('http://api.example.test', 2, 5, async (url, options) => { + assert.equal(url.pathname, '/add'); assert.equal(options.method, 'POST'); + assert.deepEqual(JSON.parse(options.body), { a: 2, b: 5 }); + return Response.json({ result: 7 }); + }); + assert.equal(result, 7); +}); +test('client surfaces API refusal', async () => { + await assert.rejects(calculate('http://api.example.test', 2, 5, async () => new Response('', { status: 503 })), /503/); +}); diff --git a/evals/coordination-fixtures/client/package.json b/evals/coordination-fixtures/client/package.json new file mode 100644 index 0000000..ae84359 --- /dev/null +++ b/evals/coordination-fixtures/client/package.json @@ -0,0 +1 @@ +{"name":"coordination-client-fixture","private":true,"type":"module","scripts":{"test":"node --test client.test.mjs"}} diff --git a/evals/coordination-fixtures/verify-pair.mjs b/evals/coordination-fixtures/verify-pair.mjs new file mode 100644 index 0000000..4a38b61 --- /dev/null +++ b/evals/coordination-fixtures/verify-pair.mjs @@ -0,0 +1,18 @@ +// Independent held-out contract: run against the two worked checkout paths. +import assert from 'node:assert/strict'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +const [api, client, endpoint = '/sum'] = process.argv.slice(2); +if (!api || !client || !['/add', '/sum'].includes(endpoint)) throw new Error('usage: verify-pair.mjs API_DIR CLIENT_DIR [/add|/sum]'); +const { server } = await import(pathToFileURL(resolve(api, 'server.mjs'))); +const { calculate } = await import(pathToFileURL(resolve(client, 'client.mjs'))); +const s = server(); await new Promise(r => s.listen(0, '127.0.0.1', r)); +try { + const base = `http://127.0.0.1:${s.address().port}`; + for (const [a, b] of [[2, 5], [-4, 3], [0.5, 0.25]]) assert.equal(await calculate(base, a, b), a + b); + const response = await fetch(base + endpoint, { method: 'POST', body: JSON.stringify({ a: 4, b: 9 }) }); + assert.equal(response.status, 200); assert.deepEqual(await response.json(), { result: 13 }); + const removed = endpoint === '/sum' ? '/add' : '/sum'; + assert.equal((await fetch(base + removed, { method: 'POST', body: '{"a":1,"b":2}' })).status, 404); + console.log(JSON.stringify({ verified: true, endpoint, realHTTP: true })); +} finally { s.closeAllConnections(); await new Promise(r => s.close(r)); }