basou: Observe the response to an HTTP GET on a board, and move boards and records to version 3

CHANGELOG.md

@@ -3,6 +3,48 @@
 All notable changes to **basou** are recorded here. The project follows
 [Semantic Versioning](https://semver.org/spec/v2.0.0.html) starting with v0.1.0.
 
+## Unreleased
+
+### Added
+
+- **A board can observe the response to an HTTP GET (`http`, from
+  `board_version: 3`, experimental).** An `http` observation names an
+  `https://` URL, held to the same shape as a `page_version`'s, and may
+  declare what to read off the body: `words` (whether the body holds each,
+  as written), a `count_pattern` (how many times it matches) and a
+  `capture_pattern` (what its first group holds in the first match). It is
+  a GET only, with no method to declare. basou does not make it and sends
+  nothing: `basou board guide` gives the commands, a `curl` without `-f`
+  that prints the status of the last response, how many redirects led to
+  it and the URL it ended at, and, when the board reads the body, a `node`
+  line that reads it whole as UTF-8 (without a leading BOM, decoding
+  nothing else) and is given the patterns and words after `--`. The value
+  is `{ status, redirects, final_url }`, with `words` (true or false in the
+  order declared), `count` and `capture` as declared; a 4xx or 5xx is a
+  value, not an observation not made, and when the status is not 2xx what
+  the body would give is null. Both patterns are matched against the whole
+  body (`^` and `$` are its start and end), not line by line as a measure's
+  `pattern` is: `count_pattern` with the flags `gu`, `capture_pattern` with
+  `u`. A word or a pattern that is empty, holds a control character or a
+  `'`, or starts with `-` is refused, as is a word given twice, a
+  `count_pattern` that matches the empty string and a `capture_pattern`
+  with no capturing group; the guide's line stops, rather than count, at an
+  empty match the declaration could not see. The keys of the value are
+  basou's words, so `basou board record`'s check for the names of other
+  workspaces looks at what they hold, not at them.
+
+### Changed
+
+- **Boards are declared in version 3 and records written in version 3
+  (`board_version: 3`, `record_version: 3`, experimental).**
+  `basou board init` prints version 3. A board of version 1 or 2 is read as
+  before, and every record is now written as version 3, whatever the
+  version of its board; records of version 1 and 2 are still read,
+  compared with and drawn. A board of version 1 or 2 that uses what a later
+  version reads is told so on a line of its own (`observe[0].kind: 'http'
+  is read from board_version 3 on; this file is board_version 2`) for a
+  kind or a value as for a key: until now only a key was told so.
+
 ## 0.70.1 — 2026-10-17
 
 ### Fixed

packages/cli/src/commands/board.test.ts

@@ -934,7 +934,7 @@ describe("basou board record", () => {
     expect(await readdir(join(repo, "board", "records"))).toEqual([name]);
     const written = JSON.parse(await readFile(join(repo, result.record ?? ""), "utf8"));
     expect(written).toMatchObject({
-      record_version: 2,
+      record_version: 3,
       measure: { digest: input.measure_digest },
       judged_by: { model: "Claude Opus 5.5", self_reported: true },
       order_anomalies: [],
@@ -967,7 +967,7 @@ describe("basou board record", () => {
     const result = await doRunBoardRecord({}, fed(repo, { ...input, observed }));
     const written = JSON.parse(await readFile(join(repo, result.record ?? ""), "utf8"));
     expect(written).toMatchObject({
-      record_version: 2,
+      record_version: 3,
       measure: { board_version: 2 },
       observed,
       declaration: {
@@ -977,6 +977,50 @@ describe("basou board record", () => {
     });
   });
 
+  const http = {
+    key: "shop",
+    kind: "http",
+    url: "https://example.com/item",
+    words: ["in stock"],
+    count_pattern: "<loc>",
+  };
+
+  it("tells a version 2 board that observes over http which version reads it", async () => {
+    const board = JSON.parse(boardYaml([MD]));
+    board.board_version = 2;
+    board.observe = [http];
+    const repo = await workspace([{ path: ".", visibility: "private" }]);
+    await placeBoard(repo, JSON.stringify(board));
+    const { err } = capture();
+    await runBoardMeasure({}, ctx(repo));
+    expect(err.join("\n")).toContain(
+      "observe[0].kind: 'http' is read from board_version 3 on; this file is board_version 2",
+    );
+    expect(process.exitCode).toBe(1);
+  });
+
+  it("records a version 3 board that observes over http", async () => {
+    const board = JSON.parse(boardYaml([MD]));
+    board.board_version = 3;
+    board.observe = [http];
+    const { repo, input } = await judged(undefined, JSON.stringify(board));
+    const observed = {
+      shop: {
+        value: { status: 200, redirects: 0, final_url: http.url, words: [true], count: 3 },
+        observed_at: "2026-10-17T00:00:00Z",
+        source: http.url,
+      },
+    };
+    const result = await doRunBoardRecord({}, fed(repo, { ...input, observed }));
+    const written = JSON.parse(await readFile(join(repo, result.record ?? ""), "utf8"));
+    expect(written).toMatchObject({
+      record_version: 3,
+      measure: { board_version: 3 },
+      observed,
+      declaration: { observe: [http] },
+    });
+  });
+
   it("checks and measures but writes nothing with --dry-run, and reports as JSON", async () => {
     const { repo, input } = await judged();
     const { out } = capture();
@@ -1022,7 +1066,7 @@ describe("basou board record", () => {
     );
     expect(out.join("\n")).toBe(`Recorded ${result.record}\nNot compared with a previous record.`);
     await expect(readFile(join(deeper, result.record ?? ""), "utf8")).resolves.toContain(
-      '"record_version": 2',
+      '"record_version": 3',
     );
   });
 
@@ -1241,6 +1285,41 @@ describe("basou board record", () => {
       expect(process.exitCode).toBe(1);
     });
 
+    it("takes the keys of an http observation's value as words of the input's shape, and still looks at what they hold", async () => {
+      const { repo, input } = await judged();
+      const url = "https://example.com/item";
+      const at = { observed_at: "2026-10-17T00:00:00Z", source: url };
+      const value = {
+        status: 200,
+        redirects: 0,
+        final_url: url,
+        words: [true],
+        count: 3,
+        capture: "2.3",
+      };
+      capture();
+      for (const name of ["status", "redirects", "final_url", "words", "count", "capture"]) {
+        await registered(repo, name);
+        const http = {
+          ...input,
+          measure_digest: await measuredAgain(repo),
+          observed: { shop: { value, ...at } },
+        };
+        capture();
+        expect((await doRunBoardRecord({ dryRun: true }, fed(repo, http))).dry_run).toBe(true);
+      }
+      const name = await registered(repo);
+      const named = {
+        ...input,
+        measure_digest: await measuredAgain(repo),
+        observed: { shop: { value: { ...value, capture: `${name} 2.3` }, ...at } },
+      };
+      const { err } = capture();
+      await runBoardRecord({ dryRun: true }, fed(repo, named));
+      expect(err.join("\n")).toContain("  - observed (a named entry).value.capture");
+      expect(process.exitCode).toBe(1);
+    });
+
     it("reads a number in a value as the text it is written as", async () => {
       const { repo, input } = await judged();
       const name = await registered(repo, "424242");
@@ -1712,7 +1791,7 @@ describe("basou board init", () => {
     const parsed = parseBoardDeclaration(text, { manifestRepoPaths: [".", "../elsewhere"] });
     if (!parsed.ok) throw new Error(parsed.errors.join("\n"));
     expect(parsed.declaration.title).toBe("board-ws progress board");
-    expect(parsed.declaration.board_version).toBe(2);
+    expect(parsed.declaration.board_version).toBe(3);
     // No session yet: the effort starts today.
     expect(parsed.declaration.effort.start).toBe(
       new Intl.DateTimeFormat("en-CA", {

packages/cli/src/commands/board.ts

@@ -612,8 +612,9 @@ async function checkAndRecord(
 }
 
 // The keys of a record's input, which a location may name: the words of its
-// shape. Any other key is a name someone chose (an observation's, a lane's,
-// or one the input should not have), which may be the very name found.
+// shape, and those of an http observation's value, which the guide fixes.
+// Any other key is a name someone chose (an observation's, a lane's, or one
+// the input should not have), which may be the very name found.
 const INPUT_KEYS = new Set([
   "measure_digest",
   "observed",
@@ -637,6 +638,12 @@ const INPUT_KEYS = new Set([
   "model",
   "self_reported",
   "triggers",
+  "status",
+  "redirects",
+  "final_url",
+  "words",
+  "count",
+  "capture",
 ]);
 
 // The places whose values are held to a shape a refusal does not quote (a