Add real API consumer fixtures for coordination acceptance

evals/coordination-fixtures/README.md

@@ -0,0 +1,32 @@
+# Real producer/consumer coordination fixture
+
+Use separate disposable repositories containing `api/` and `client/`. Both have
+an honest, dependency-free `npm test` suite and a working HTTP relationship.
+The client is an API consumer, not a documentation-only repository.
+
+Parent task: rename the arithmetic API's `POST /add` endpoint to `POST /sum`,
+remove the old endpoint, and update the client to call the new endpoint. Preserve
+the response shape, exported server/client functions, validation and error
+handling; update each declared suite to the new contract.
+
+Before launch, run both suites and:
+
+```sh
+node evals/coordination-fixtures/verify-pair.mjs API_CHECKOUT CLIENT_CHECKOUT /add
+```
+
+Keep `verify-pair.mjs` outside both repositories. After API and client changes,
+check out the exact API anchor and exact client commit into separate directories,
+run their declared suites, then run the same command with `/sum`. It exercises
+actual HTTP, positive and fractional/negative operands, the renamed endpoint,
+and refusal of the removed endpoint. Record both commit hashes and outputs.
+
+Ship's built-in coordination check remains a static compatibility scan. The
+external pair test is additional evidence and must not be described as execution
+performed by that scan. An uncertain verdict remains uncertain until resolved
+through the product's normal decision path. No automatic merge is implied by
+this fixture; retain each approval and its evidence.
+
+Fixture self-check: the original pair passes `/add` and fails `/sum`; changing
+only the API to `/sum` still fails; changing the matching client path as well
+passes `/sum`. These are grader sensitivity controls, not a live Ship receipt.

evals/coordination-fixtures/api/README.md

@@ -0,0 +1,8 @@
+# Arithmetic API fixture
+
+Node 22+, no dependencies. `npm test` runs the real HTTP contract.
+
+`server()` from `server.mjs` creates an unbound HTTP server. `POST /add` accepts
+JSON `{a: number, b: number}` and returns `{result: number}`. Invalid operands
+return 400; unknown endpoints return 404. Bind to a loopback ephemeral port in
+tests. Keep the exported server factory and response shape stable during a rename.

evals/coordination-fixtures/api/package.json

@@ -0,0 +1 @@
+{"name":"coordination-api-fixture","private":true,"type":"module","scripts":{"test":"node --test server.test.mjs"}}

evals/coordination-fixtures/api/server.mjs

@@ -0,0 +1,18 @@
+import { createServer } from 'node:http';
+export function server() {
+  return createServer(async (req, res) => {
+    if (req.method !== 'POST' || req.url !== '/add') {
+      res.writeHead(404).end(); return;
+    }
+    let text = '';
+    for await (const chunk of req) {
+      text += chunk;
+      if (text.length > 1024) { res.writeHead(413).end(); return; }
+    }
+    try {
+      const { a, b } = JSON.parse(text);
+      if (!Number.isFinite(a) || !Number.isFinite(b)) { res.writeHead(400).end(); return; }
+      res.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ result: a + b }));
+    } catch { res.writeHead(400).end(); }
+  });
+}

evals/coordination-fixtures/api/server.test.mjs

@@ -0,0 +1,13 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { server } from './server.mjs';
+
+test('Add HTTP contract', async (t) => {
+  const s = server(); await new Promise(r => s.listen(0, '127.0.0.1', r));
+  t.after(() => { s.closeAllConnections(); s.close(); });
+  const base = `http://127.0.0.1:${s.address().port}`;
+  const call = (body, path = '/add') => fetch(base + path, { method: 'POST', body: JSON.stringify(body) });
+  assert.deepEqual(await (await call({ a: 2, b: 5 })).json(), { result: 7 });
+  assert.equal((await call({ a: '2', b: 5 })).status, 400);
+  assert.equal((await call({ a: 2, b: 5 }, '/missing')).status, 404);
+});

evals/coordination-fixtures/client/README.md

@@ -0,0 +1,6 @@
+# Arithmetic client fixture
+
+Node 22+, no dependencies. `npm test` verifies the client request contract and
+error handling. `calculate(baseURL, a, b)` calls the API's `POST /add`, returning
+the numeric `result`. The optional fourth argument is an injected fetch function.
+Keep the export and arguments stable when updating to a renamed API endpoint.

evals/coordination-fixtures/client/client.mjs

@@ -0,0 +1,9 @@
+export async function calculate(baseURL, a, b, request = fetch) {
+  const response = await request(new URL('/add', baseURL), {
+    method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ a, b }),
+  });
+  if (!response.ok) throw new Error(`Arithmetic API returned ${response.status}`);
+  const data = await response.json();
+  if (!Number.isFinite(data.result)) throw new Error('Arithmetic API returned an invalid result');
+  return data.result;
+}

evals/coordination-fixtures/client/client.test.mjs

@@ -0,0 +1,15 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { calculate } from './client.mjs';
+
+test('client sends the Add contract and returns its result', async () => {
+  const result = await calculate('http://api.example.test', 2, 5, async (url, options) => {
+    assert.equal(url.pathname, '/add'); assert.equal(options.method, 'POST');
+    assert.deepEqual(JSON.parse(options.body), { a: 2, b: 5 });
+    return Response.json({ result: 7 });
+  });
+  assert.equal(result, 7);
+});
+test('client surfaces API refusal', async () => {
+  await assert.rejects(calculate('http://api.example.test', 2, 5, async () => new Response('', { status: 503 })), /503/);
+});

evals/coordination-fixtures/client/package.json

@@ -0,0 +1 @@
+{"name":"coordination-client-fixture","private":true,"type":"module","scripts":{"test":"node --test client.test.mjs"}}

evals/coordination-fixtures/verify-pair.mjs

@@ -0,0 +1,18 @@
+// Independent held-out contract: run against the two worked checkout paths.
+import assert from 'node:assert/strict';
+import { resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+const [api, client, endpoint = '/sum'] = process.argv.slice(2);
+if (!api || !client || !['/add', '/sum'].includes(endpoint)) throw new Error('usage: verify-pair.mjs API_DIR CLIENT_DIR [/add|/sum]');
+const { server } = await import(pathToFileURL(resolve(api, 'server.mjs')));
+const { calculate } = await import(pathToFileURL(resolve(client, 'client.mjs')));
+const s = server(); await new Promise(r => s.listen(0, '127.0.0.1', r));
+try {
+  const base = `http://127.0.0.1:${s.address().port}`;
+  for (const [a, b] of [[2, 5], [-4, 3], [0.5, 0.25]]) assert.equal(await calculate(base, a, b), a + b);
+  const response = await fetch(base + endpoint, { method: 'POST', body: JSON.stringify({ a: 4, b: 9 }) });
+  assert.equal(response.status, 200); assert.deepEqual(await response.json(), { result: 13 });
+  const removed = endpoint === '/sum' ? '/add' : '/sum';
+  assert.equal((await fetch(base + removed, { method: 'POST', body: '{"a":1,"b":2}' })).status, 404);
+  console.log(JSON.stringify({ verified: true, endpoint, realHTTP: true }));
+} finally { s.closeAllConnections(); await new Promise(r => s.close(r)); }