From 8480a6ac01193f917c3004efc419c27684357e67 Mon Sep 17 00:00:00 2001 From: Takashi Matsuyama Date: Sat, 17 Oct 2026 08:35:30 +0900 Subject: [PATCH 1/2] basou: Observe the response to an HTTP GET on a board, and move boards and records to version 3 - board.yaml: board_version 3 adds the observation kind http: an https:// URL (held to the shape of a page_version's), and optionally words, a count_pattern (flags gu, matched against the whole body; one that matches the empty string is refused) and a capture_pattern (flag u, with at least one capturing group). GET only. A word or a pattern that is empty, holds a control character or a quote, or starts with '-' is refused, as is a word given twice. - A board of version 1 or 2 that uses what a later version reads is told so on a line of its own for a kind or a value as for a key: each problem is put down to the first version that reads that part as written, and a key, when none does, to the first version that takes it. - board guide: a curl without -f that prints the status, the redirects and the URL it ended at, and a node line that reads the body whole as UTF-8 (without a leading BOM), is given the patterns and words after --, and stops at an empty match. The value is { status, redirects, final_url } with words, count and capture as declared, null when the status is not 2xx. - board init prints board_version 3, and its comment names http. - Records are written as record_version 3, whatever the version of the board; versions 1 and 2 are still read and drawn. --- CHANGELOG.md | 40 +++ packages/cli/src/commands/board.test.ts | 52 +++- packages/core/src/board/declaration.test.ts | 177 +++++++++++- packages/core/src/board/declaration.ts | 282 ++++++++++++++++---- packages/core/src/board/guide.test.ts | 206 +++++++++++++- packages/core/src/board/guide.ts | 75 +++++- packages/core/src/board/init.test.ts | 9 +- packages/core/src/board/page.test.ts | 36 ++- packages/core/src/board/previous.test.ts | 4 +- packages/core/src/board/record.test.ts | 73 ++++- packages/core/src/board/record.ts | 9 +- packages/core/src/lib/view-strings.ts | 4 +- 12 files changed, 897 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39369657..94ed94ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,46 @@ All notable changes to **basou** are recorded here. The project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html) starting with v0.1.0. +## Unreleased + +### Added + +- **A board can observe the response to an HTTP GET (`http`, from + `board_version: 3`, experimental).** An `http` observation names an + `https://` URL, held to the same shape as a `page_version`'s, and may + declare what to read off the body: `words` (whether the body holds each, + as written), a `count_pattern` (how many times it matches) and a + `capture_pattern` (what its first group holds in the first match). It is + a GET only, with no method to declare. basou does not make it and sends + nothing: `basou board guide` gives the commands, a `curl` without `-f` + that prints the status of the last response, how many redirects led to + it and the URL it ended at, and, when the board reads the body, a `node` + line that reads it whole as UTF-8 (without a leading BOM, decoding + nothing else) and is given the patterns and words after `--`. The value + is `{ status, redirects, final_url }`, with `words` (true or false in the + order declared), `count` and `capture` as declared; a 4xx or 5xx is a + value, not an observation not made, and when the status is not 2xx what + the body would give is null. Both patterns are matched against the whole + body (`^` and `$` are its start and end), not line by line as a measure's + `pattern` is: `count_pattern` with the flags `gu`, `capture_pattern` with + `u`. A word or a pattern that is empty, holds a control character or a + `'`, or starts with `-` is refused, as is a word given twice, a + `count_pattern` that matches the empty string and a `capture_pattern` + with no capturing group; the guide's line stops, rather than count, at an + empty match the declaration could not see. + +### Changed + +- **Boards are declared in version 3 and records written in version 3 + (`board_version: 3`, `record_version: 3`, experimental).** + `basou board init` prints version 3. A board of version 1 or 2 is read as + before, and every record is now written as version 3, whatever the + version of its board; records of version 1 and 2 are still read, + compared with and drawn. A board of version 1 or 2 that uses what a later + version reads is told so on a line of its own (`observe[0].kind: 'http' + is read from board_version 3 on; this file is board_version 2`) for a + kind or a value as for a key: until now only a key was told so. + ## 0.70.1 — 2026-10-17 ### Fixed diff --git a/packages/cli/src/commands/board.test.ts b/packages/cli/src/commands/board.test.ts index aea92d39..046e0020 100644 --- a/packages/cli/src/commands/board.test.ts +++ b/packages/cli/src/commands/board.test.ts @@ -934,7 +934,7 @@ describe("basou board record", () => { expect(await readdir(join(repo, "board", "records"))).toEqual([name]); const written = JSON.parse(await readFile(join(repo, result.record ?? ""), "utf8")); expect(written).toMatchObject({ - record_version: 2, + record_version: 3, measure: { digest: input.measure_digest }, judged_by: { model: "Claude Opus 5.5", self_reported: true }, order_anomalies: [], @@ -967,7 +967,7 @@ describe("basou board record", () => { const result = await doRunBoardRecord({}, fed(repo, { ...input, observed })); const written = JSON.parse(await readFile(join(repo, result.record ?? ""), "utf8")); expect(written).toMatchObject({ - record_version: 2, + record_version: 3, measure: { board_version: 2 }, observed, declaration: { @@ -977,6 +977,50 @@ describe("basou board record", () => { }); }); + const http = { + key: "shop", + kind: "http", + url: "https://example.com/item", + words: ["in stock"], + count_pattern: "", + }; + + it("tells a version 2 board that observes over http which version reads it", async () => { + const board = JSON.parse(boardYaml([MD])); + board.board_version = 2; + board.observe = [http]; + const repo = await workspace([{ path: ".", visibility: "private" }]); + await placeBoard(repo, JSON.stringify(board)); + const { err } = capture(); + await runBoardMeasure({}, ctx(repo)); + expect(err.join("\n")).toContain( + "observe[0].kind: 'http' is read from board_version 3 on; this file is board_version 2", + ); + expect(process.exitCode).toBe(1); + }); + + it("records a version 3 board that observes over http", async () => { + const board = JSON.parse(boardYaml([MD])); + board.board_version = 3; + board.observe = [http]; + const { repo, input } = await judged(undefined, JSON.stringify(board)); + const observed = { + shop: { + value: { status: 200, redirects: 0, final_url: http.url, words: [true], count: 3 }, + observed_at: "2026-10-17T00:00:00Z", + source: http.url, + }, + }; + const result = await doRunBoardRecord({}, fed(repo, { ...input, observed })); + const written = JSON.parse(await readFile(join(repo, result.record ?? ""), "utf8")); + expect(written).toMatchObject({ + record_version: 3, + measure: { board_version: 3 }, + observed, + declaration: { observe: [http] }, + }); + }); + it("checks and measures but writes nothing with --dry-run, and reports as JSON", async () => { const { repo, input } = await judged(); const { out } = capture(); @@ -1022,7 +1066,7 @@ describe("basou board record", () => { ); expect(out.join("\n")).toBe(`Recorded ${result.record}\nNot compared with a previous record.`); await expect(readFile(join(deeper, result.record ?? ""), "utf8")).resolves.toContain( - '"record_version": 2', + '"record_version": 3', ); }); @@ -1712,7 +1756,7 @@ describe("basou board init", () => { const parsed = parseBoardDeclaration(text, { manifestRepoPaths: [".", "../elsewhere"] }); if (!parsed.ok) throw new Error(parsed.errors.join("\n")); expect(parsed.declaration.title).toBe("board-ws progress board"); - expect(parsed.declaration.board_version).toBe(2); + expect(parsed.declaration.board_version).toBe(3); // No session yet: the effort starts today. expect(parsed.declaration.effort.start).toBe( new Intl.DateTimeFormat("en-CA", { diff --git a/packages/core/src/board/declaration.test.ts b/packages/core/src/board/declaration.test.ts index 8ebd3fe5..c3b717d8 100644 --- a/packages/core/src/board/declaration.test.ts +++ b/packages/core/src/board/declaration.test.ts @@ -190,14 +190,14 @@ describe("parseBoardDeclaration: accepted declarations", () => { describe("parseBoardDeclaration: board_version", () => { it("stops at an unknown board_version, naming it, whatever else is wrong", () => { - expect(errorsOf(parse({ ...board(), board_version: 3, title: "", extra: 1 }))).toEqual([ - "board_version: this basou reads board_version 1 or 2, not 3", + expect(errorsOf(parse({ ...board(), board_version: 4, title: "", extra: 1 }))).toEqual([ + "board_version: this basou reads board_version 1, 2 or 3, not 4", ]); }); it("refuses a board_version that is missing or not a number", () => { const { board_version: _v, ...missing } = board(); - const reads = "board_version: must be a board version this basou reads (1 or 2)"; + const reads = "board_version: must be a board version this basou reads (1, 2 or 3)"; expect(errorsOf(parse(missing))).toEqual([reads]); expect(errorsOf(parse({ ...board(), board_version: "1" }))).toEqual([reads]); }); @@ -1133,11 +1133,11 @@ describe("parseBoardDeclaration: what the review found", () => { }); it("names an unknown board_version before a YAML warning", () => { - const plain = stringify({ ...board(), board_version: 3 }); + const plain = stringify({ ...board(), board_version: 4 }); expect(plain).toContain("title: A board\n"); - const text = plain.replace("title: A board\n", "title: !v3tag A board\n"); + const text = plain.replace("title: A board\n", "title: !v4tag A board\n"); expect(errorsOf(parseBoardDeclaration(text, { manifestRepoPaths: REPOS }))).toEqual([ - "board_version: this basou reads board_version 1 or 2, not 3", + "board_version: this basou reads board_version 1, 2 or 3, not 4", ]); }); @@ -1458,3 +1458,168 @@ describe("parseBoardDeclaration: board_version 2", () => { ]); }); }); + +// A version 3 board whose only observation is `observe`. +function withHttp(observe: Record): Record { + return { ...boardV2(), board_version: 3, observe: [{ key: "h", kind: "http", ...observe }] }; +} + +describe("parseBoardDeclaration: board_version 3 and http", () => { + const url = "https://example.com/shop/item-1"; + + it("accepts an http observation with what to read off the body, and one with none", () => { + const doc: Record = { ...boardV2(), board_version: 3 }; + const full = { + key: "shop_item", + kind: "http", + url, + words: ["in stock", "\u5728\u5eab\u3042\u308a", "a\\b", "x-y"], + count_pattern: "", + capture_pattern: "Version ([0-9.]+)", + }; + doc.observe = [...(doc.observe as unknown[]), full, { key: "bare", kind: "http", url }]; + const result = parse(doc); + if (!result.ok) throw new Error(result.errors.join("\n")); + expect(result.declaration.board_version).toBe(3); + expect(result.declaration.observe.slice(-2)).toEqual([ + full, + { key: "bare", kind: "http", url }, + ]); + }); + + it("reads a version 2 board as before", () => { + const result = parse(boardV2()); + if (!result.ok) throw new Error(result.errors.join("\n")); + expect(result.declaration.board_version).toBe(2); + }); + + it("refuses a method and any key it does not take, and holds the url to the shape of a page's", () => { + expect(errorsOf(parse(withHttp({ url, method: "HEAD", status: 200 })))).toEqual([ + "observe[0]: unknown keys 'method', 'status'", + ]); + expect(errorsOf(parse(withHttp({ url: "http://example.com/" })))).toEqual([ + "observe[0].url: must be an https:// URL", + ]); + expect(errorsOf(parse(withHttp({})))).toEqual([ + "observe[0].url: Invalid input: expected string, received undefined", + ]); + }); + + it("refuses a word list that is empty or not a list, and a word a command could misread", () => { + expect(errorsOf(parse(withHttp({ url, words: [] })))).toEqual([ + "observe[0].words: must list at least one word", + ]); + expect(errorsOf(parse(withHttp({ url, words: "in stock" })))).toEqual([ + "observe[0].words: Invalid input: expected array, received string", + ]); + expect( + errorsOf(parse(withHttp({ url, words: ["", " ", "a\tb", "a\u0085b", "-p", "it's", 1] }))), + ).toEqual([ + "observe[0].words[0]: must be a non-empty string", + "observe[0].words[1]: must be a non-empty string", + "observe[0].words[2]: must not contain control characters", + "observe[0].words[3]: must not contain control characters", + "observe[0].words[4]: must not start with '-'", + 'observe[0].words[5]: must not contain "\'"', + "observe[0].words[6]: Invalid input: expected string, received number", + ]); + }); + + it("refuses a word given twice, saying where it was first", () => { + expect(errorsOf(parse(withHttp({ url, words: ["a", "b", "a", "b", "a"] })))).toEqual([ + "observe[0].words[2]: duplicate word (first at words[0])", + "observe[0].words[3]: duplicate word (first at words[1])", + "observe[0].words[4]: duplicate word (first at words[0])", + ]); + }); + + it("refuses a count_pattern that does not compile with gu, matches the empty string or a command could misread", () => { + const at = (count_pattern: unknown) => errorsOf(parse(withHttp({ url, count_pattern }))); + const empty = + "observe[0].count_pattern: must not match the empty string: each match is counted"; + for (const pattern of ["a*", "x?", "^", "$", "(?:)", "a|", "[a-z]{0,3}"]) { + expect(at(pattern)).toEqual([empty]); + } + expect(at("")).toEqual(["observe[0].count_pattern: must be a non-empty string"]); + expect(at("-x")).toEqual(["observe[0].count_pattern: must not start with '-'"]); + expect(at("a\nb")).toEqual(["observe[0].count_pattern: must not contain control characters"]); + expect(at("content='x'")).toEqual([ + 'observe[0].count_pattern: must not contain "\'" (match one as \\x27)', + ]); + const [broken] = at("("); + expect(broken).toMatch( + /^observe\[0\]\.count_pattern: is not a valid regular expression with the 'gu' flags \(/, + ); + // An escape the u flag refuses, which a pattern without it would take. + expect(at("\\-")[0]).toMatch(/with the 'gu' flags/); + for (const pattern of ["", "^#", "end$", "content=\\x27x\\x27", "\\p{L}+"]) { + expect(parse(withHttp({ url, count_pattern: pattern })).ok).toBe(true); + } + }); + + it("refuses a capture_pattern with no capturing group, or one that does not compile with u", () => { + const at = (capture_pattern: unknown) => errorsOf(parse(withHttp({ url, capture_pattern }))); + const group = + "observe[0].capture_pattern: must have a capturing group: the value is what the first one holds"; + expect(at("Version [0-9.]+")).toEqual([group]); + expect(at("Version (?:[0-9.]+)")).toEqual([group]); + expect(at("-(x)")).toEqual(["observe[0].capture_pattern: must not start with '-'"]); + expect(at("(")[0]).toMatch(/with the 'u' flags \(/); + for (const pattern of ["Version ([0-9.]+)", "(?[0-9]+)", "a*(b)?", "^(.*)$"]) { + expect(parse(withHttp({ url, capture_pattern: pattern })).ok).toBe(true); + } + }); + + it("says that version 3 reads http when version 2 or 1 has it, apart from what no version reads", () => { + const doc = boardV2(); + doc.observe = [ + { key: "a", kind: "http", url, words: ["x"] }, + { key: "b", kind: "rss_feed", url }, + { key: "c", kind: "http", url: "http://example.com/" }, + ]; + expect(errorsOf(parse(doc))).toEqual([ + "observe[0].kind: 'http' is read from board_version 3 on; this file is board_version 2", + "observe[1].kind: Invalid discriminator value. Expected 'npm_version' | 'github_release' | 'github_open_issues' | 'github_open_prs' | 'github_ci' | 'page_version' | 'manual'", + "observe[2].kind: 'http' is read from board_version 3 on; this file is board_version 2", + ]); + // Version 1 reads no observe at all: the key is put down to the version that reads it as written. + const v1 = board(); + v1.observe = [{ key: "a", kind: "http", url }]; + expect(errorsOf(parse(v1))).toEqual([ + "(top level): 'observe' is read from board_version 3 on; this file is board_version 1", + ]); + v1.observe = [{ key: "a", kind: "npm_version", package: "x" }]; + entry(v1.stages, "02").look = ["x"]; + expect(errorsOf(parse(v1))).toEqual([ + "stages.02: 'look' is read from board_version 2 on; this file is board_version 1", + "(top level): 'observe' is read from board_version 2 on; this file is board_version 1", + ]); + }); + + it("does not put a problem down to a later version that refuses the same part", () => { + const doc = boardV2(); + doc.observe = [{ key: "a", kind: "http" }]; + // Version 3 takes the kind but not the entry without its url: still read from 3 on. + expect(errorsOf(parse(doc))).toEqual([ + "observe[0].kind: 'http' is read from board_version 3 on; this file is board_version 2", + ]); + const v1 = board(); + v1.observe = [{ key: "a", kind: "http" }]; + // No version reads the key as written (3 refuses the entry inside it): + // it is put down to the first version that takes the key. + expect(errorsOf(parse(v1))).toEqual([ + "(top level): 'observe' is read from board_version 2 on; this file is board_version 1", + ]); + // A key no shape takes inside it does not keep it from being taken. + v1.observe = [{ key: "a", kind: "manual", how: "ask", words: ["x"] }]; + expect(errorsOf(parse(v1))).toEqual([ + "(top level): 'observe' is read from board_version 2 on; this file is board_version 1", + ]); + v1.observe = "not a list"; + entry(v1.stages, "02").look = [""]; + expect(errorsOf(parse(v1))).toEqual([ + "stages.02: 'look' is read from board_version 2 on; this file is board_version 1", + "(top level): 'observe' is read from board_version 2 on; this file is board_version 1", + ]); + }); +}); diff --git a/packages/core/src/board/declaration.ts b/packages/core/src/board/declaration.ts index 31356bf7..bfe94cae 100644 --- a/packages/core/src/board/declaration.ts +++ b/packages/core/src/board/declaration.ts @@ -14,14 +14,15 @@ import { TaskStatusSchema } from "../schemas/task.schema.js"; * an error, so a typo is refused instead of silently ignored, and every * problem the reader can see is reported at once. * - * Every version ever written is read: 1, and 2, which adds what to observe + * Every version ever written is read: 1; 2, which adds what to observe * outside basou (`observe`) and what to look at when judging a stage - * (`stages..look` and `.notes`). + * (`stages..look` and `.notes`); and 3, which adds the response to an + * HTTP GET as an observation (`http`). */ -export const BOARD_VERSION = 2; +export const BOARD_VERSION = 3; /** The board versions this basou reads. A version is added here, never removed. */ -export const BOARD_VERSIONS = [1, 2] as const; +export const BOARD_VERSIONS = [1, 2, 3] as const; /** The stage ids of every board. Their order is fixed; a board sets only their meanings. */ export const BOARD_STAGE_IDS = ["01", "02", "03", "04", "05", "06"] as const; @@ -266,12 +267,26 @@ export const BOARD_OBSERVE_KINDS = [ "github_open_prs", "github_ci", "page_version", + "http", "manual", ] as const; /** The branch a `github_ci` observation looks at when it names none. */ export const BOARD_DEFAULT_CI_BRANCH = "main"; +/** How many redirects an `http` observation follows; one that needs more is not made. */ +export const BOARD_HTTP_MAX_REDIRECTS = 10; + +/** + * The flags an `http` observation's `count_pattern` is matched with: every + * match in the whole body, with no `m`, so `^` and `$` are the body's start + * and end. A measure's `pattern` is matched line by line instead. + */ +export const BOARD_HTTP_COUNT_FLAGS = "gu"; + +/** The flags an `http` observation's `capture_pattern` is matched with: the first match in the whole body. */ +export const BOARD_HTTP_CAPTURE_FLAGS = "u"; + // The values of an observation are put into the commands a judge runs, each // as one single-quoted shell word. Each is held to a shape that, quoted so, // cannot end its quotation (no "'") or, where a command takes it as an @@ -335,7 +350,61 @@ const httpsUrl = z.string().superRefine((s, ctx) => { if (problem !== undefined) ctx.addIssue({ code: "custom", message: problem }); }); -const observeSchema = z.discriminatedUnion("kind", [ +// Why a text cannot be given to a command as one argument, or undefined. It +// reaches the command after `--`, so it is never read as an option; a +// leading '-' is refused all the same, as for every value a command takes. +function argumentProblem(s: string, quoteHint = ""): string | undefined { + if (s.trim().length === 0) return "must be a non-empty string"; + if (hasControlCharacter(s)) return "must not contain control characters"; + if (s.startsWith("-")) return "must not start with '-'"; + if (s.includes("'")) return `must not contain "'"${quoteHint}`; + return undefined; +} + +// A word an `http` observation looks for in the body, as it is written. +const httpWord = z.string().superRefine((s, ctx) => { + const problem = argumentProblem(s); + if (problem !== undefined) ctx.addIssue({ code: "custom", message: problem }); +}); + +// Whether a pattern matches the empty string. +function matchesEmpty(pattern: string): boolean { + try { + return new RegExp(pattern, BOARD_HTTP_COUNT_FLAGS).test(""); + } catch { + return false; + } +} + +// A pattern an `http` observation matches against the whole body, with the +// flags given. "'" is refused as in every value of a command; a pattern can +// match one as \x27. +const httpPattern = (flags: string, check: (s: string) => string | undefined) => + z.string().superRefine((s, ctx) => { + let problem = argumentProblem(s, " (match one as \\x27)"); + if (problem === undefined) { + try { + new RegExp(s, flags); + problem = check(s); + } catch (error: unknown) { + const reason = error instanceof Error ? error.message : "invalid"; + problem = `is not a valid regular expression with the '${flags}' flags (${reason})`; + } + } + if (problem !== undefined) ctx.addIssue({ code: "custom", message: problem }); + }); + +const countPattern = httpPattern(BOARD_HTTP_COUNT_FLAGS, (s) => + matchesEmpty(s) ? "must not match the empty string: each match is counted" : undefined, +); + +const capturePattern = httpPattern(BOARD_HTTP_CAPTURE_FLAGS, (s) => + (captureGroups(s) ?? 0) === 0 + ? "must have a capturing group: the value is what the first one holds" + : undefined, +); + +const observeKinds = [ z.strictObject({ key: idText, kind: z.literal("npm_version"), package: npmPackage }), z.strictObject({ key: idText, kind: z.literal("github_release"), repo: githubRepo }), z.strictObject({ key: idText, kind: z.literal("github_open_issues"), repo: githubRepo }), @@ -348,8 +417,23 @@ const observeSchema = z.discriminatedUnion("kind", [ branch: branchName.default(BOARD_DEFAULT_CI_BRANCH), }), z.strictObject({ key: idText, kind: z.literal("page_version"), url: httpsUrl }), - z.strictObject({ key: idText, kind: z.literal("manual"), how: nonEmptyText }), -]); +] as const; + +const manualObserve = z.strictObject({ key: idText, kind: z.literal("manual"), how: nonEmptyText }); + +// GET only: a HEAD is answered differently by many servers, and is another command. +const httpObserve = z.strictObject({ + key: idText, + kind: z.literal("http"), + url: httpsUrl, + words: z.array(httpWord).min(1, { error: "must list at least one word" }).optional(), + count_pattern: countPattern.optional(), + capture_pattern: capturePattern.optional(), +}); + +const observeSchemaV2 = z.discriminatedUnion("kind", [...observeKinds, manualObserve]); + +const observeSchemaV3 = z.discriminatedUnion("kind", [...observeKinds, httpObserve, manualObserve]); const stageSchemaV1 = z.strictObject({ meaning: nonEmptyText }); @@ -362,8 +446,11 @@ const stageSchemaV2 = z.strictObject({ const stagesOf = (stage: S) => z.strictObject({ "01": stage, "02": stage, "03": stage, "04": stage, "05": stage, "06": stage }); +// The versions this basou reads, as a sentence says them: "1, 2 or 3". +const READ_VERSIONS = `${BOARD_VERSIONS.slice(0, -1).join(", ")} or ${BOARD_VERSIONS.at(-1)}`; + const boardVersionText = z.literal(BOARD_VERSIONS, { - error: `must be a board version this basou reads (${BOARD_VERSIONS.join(" or ")})`, + error: `must be a board version this basou reads (${READ_VERSIONS})`, }); // The keys every version has after its stages, in the order a problem with @@ -434,14 +521,23 @@ const boardSchemaV2 = z.strictObject({ title: nonEmptyText, stages: stagesOf(stageSchemaV2), ...boardFields, - observe: z.array(observeSchema).default([]), + observe: z.array(observeSchemaV2).default([]), +}); + +const boardSchemaV3 = z.strictObject({ + board_version: boardVersionText, + title: nonEmptyText, + stages: stagesOf(stageSchemaV2), + ...boardFields, + observe: z.array(observeSchemaV3).default([]), }); /** * A declaration as read, whatever its version: one of version 1 has no - * observations and no stage has `look` or `notes`. + * observations and no stage has `look` or `notes`, and one of version 2 + * observes nothing over `http`. */ -export type BoardDeclaration = z.output; +export type BoardDeclaration = z.output; export type BoardMeasure = BoardDeclaration["measures"][number]; export type BoardMeasureKind = BoardMeasure["kind"]; /** One entry of `observe`: what to observe outside basou, and the key it is recorded under. */ @@ -543,9 +639,7 @@ export function parseBoardDeclaration( if (typeof version === "number" && !(BOARD_VERSIONS as readonly number[]).includes(version)) { return { ok: false, - errors: [ - `board_version: this basou reads board_version ${BOARD_VERSIONS.join(" or ")}, not ${version}`, - ], + errors: [`board_version: this basou reads board_version ${READ_VERSIONS}, not ${version}`], }; } const yamlErrors = [ @@ -557,43 +651,128 @@ export function parseBoardDeclaration( const errors: string[] = []; // A file with no version, or one that is not a number, is held to the // newest shape. + const parsed = parseAs(version, raw); + if (!parsed.ok) { + errors.push( + ...(version === BOARD_VERSION || typeof version !== "number" + ? parsed.issues.map(formatIssue) + : formatOlderIssues(raw, version, parsed.issues)), + ); + } + errors.push(...crossCheck(raw, context)); + if (errors.length > 0 || !parsed.ok) return { ok: false, errors }; + return { ok: true, declaration: parsed.declaration }; +} + +// The schema of each version this basou reads. +function schemaOf(version: number) { + return version === 1 ? boardSchemaV1 : version === 2 ? boardSchemaV2 : boardSchemaV3; +} + +function parseAs( + version: unknown, + raw: unknown, +): { ok: true; declaration: BoardDeclaration } | { ok: false; issues: Issue[] } { if (version === 1) { const parsed = boardSchemaV1.safeParse(raw); - if (!parsed.success) { - for (const issue of parsed.error.issues) errors.push(...formatVersion1Issue(issue)); + return parsed.success + ? { ok: true, declaration: { ...parsed.data, observe: [] } } + : { ok: false, issues: parsed.error.issues }; + } + const parsed = (version === 2 ? boardSchemaV2 : boardSchemaV3).safeParse(raw); + return parsed.success + ? { ok: true, declaration: parsed.data } + : { ok: false, issues: parsed.error.issues }; +} + +// The problems of a file of an older version, where a later version reads +// what is written told apart from what no version reads: a key, a kind, a +// value of a form a later version takes. Each is put down to the first +// version that reads that part of the file as it is written, with no problem +// at it, inside it or around it; a key, when no version reads it so, to the +// first version that takes the key, whatever is wrong inside it. +function formatOlderIssues(raw: unknown, version: number, issues: readonly Issue[]): string[] { + const later = BOARD_VERSIONS.filter((v) => v > version).map((v) => ({ + version: v, + issues: schemaOf(v).safeParse(raw).error?.issues ?? [], + })); + const readsAsWritten = (path: readonly PropertyKey[]): number | undefined => + later.find((l) => !l.issues.some((issue) => issuePaths(issue).some((p) => related(p, path)))) + ?.version; + const takesKey = (path: readonly PropertyKey[]): number | undefined => + later.find( + (l) => + !l.issues.some((issue) => + issuePaths(issue).some( + (p) => + (p.length < path.length || + (p.length === path.length && issue.code === "unrecognized_keys")) && + related(p, path), + ), + ), + )?.version; + const readFrom = (path: readonly PropertyKey[], isKey: boolean): number | undefined => + readsAsWritten(path) ?? (isKey ? takesKey(path) : undefined); + const readFromLine = (path: readonly PropertyKey[], what: string, from: number) => + `${formatPath(path)}: ${what} read from board_version ${from} on; this file is board_version ${version}`; + const out: string[] = []; + for (const issue of issues) { + if (issue.code !== "unrecognized_keys") { + const from = readFrom(issue.path, false); + out.push( + from === undefined + ? formatIssue(issue) + : readFromLine(issue.path, `${writtenValue(valueAt(raw, issue.path))} is`, from), + ); + continue; + } + const others: string[] = []; + const byVersion = new Map(); + for (const key of issue.keys) { + const from = readFrom([...issue.path, key], true); + if (from === undefined) others.push(key); + else byVersion.set(from, [...(byVersion.get(from) ?? []), key]); + } + if (others.length > 0) out.push(formatIssue({ ...issue, keys: others })); + for (const [from, keys] of [...byVersion].sort(([a], [b]) => a - b)) { + const named = keys.map((k) => `'${k}'`).join(", "); + out.push(readFromLine(issue.path, `${named} ${keys.length === 1 ? "is" : "are"}`, from)); } - errors.push(...crossCheck(raw, context)); - if (errors.length > 0 || !parsed.success) return { ok: false, errors }; - return { ok: true, declaration: { ...parsed.data, observe: [] } }; } - const parsed = boardSchemaV2.safeParse(raw); - if (!parsed.success) { - for (const issue of parsed.error.issues) errors.push(formatIssue(issue)); + return out; +} + +// The parts of the file an issue is about: the key itself, for a key no +// shape takes. +function issuePaths(issue: Issue): (readonly PropertyKey[])[] { + return issue.code === "unrecognized_keys" + ? issue.keys.map((k) => [...issue.path, k]) + : [issue.path]; +} + +// Whether one path is the other or holds it. +function related(a: readonly PropertyKey[], b: readonly PropertyKey[]): boolean { + const n = Math.min(a.length, b.length); + for (let i = 0; i < n; i++) if (a[i] !== b[i]) return false; + return true; +} + +function valueAt(raw: unknown, path: readonly PropertyKey[]): unknown { + let value = raw; + for (const key of path) { + if (typeof value !== "object" || value === null || !Object.hasOwn(value, key)) { + return undefined; + } + value = (value as Record)[key]; } - errors.push(...crossCheck(raw, context)); - if (errors.length > 0 || !parsed.success) return { ok: false, errors }; - return { ok: true, declaration: parsed.data }; + return value; } -// The keys version 2 added, where it added them: a file of version 1 that -// has one is told so apart from the keys no version reads. -function formatVersion1Issue(issue: Issue): string[] { - if (issue.code !== "unrecognized_keys") return [formatIssue(issue)]; - const added = - issue.path.length === 0 - ? ["observe"] - : issue.path.length === 2 && issue.path[0] === "stages" - ? ["look", "notes"] - : []; - const later = issue.keys.filter((k) => added.includes(k)); - if (later.length === 0) return [formatIssue(issue)]; - const others = issue.keys.filter((k) => !added.includes(k)); - const out = others.length === 0 ? [] : [formatIssue({ ...issue, keys: others })]; - const named = later.map((k) => `'${k}'`).join(", "); - out.push( - `${formatPath(issue.path)}: ${named} ${later.length === 1 ? "is" : "are"} read from board_version 2 on; this file is board_version 1`, - ); - return out; +// A value as a problem names it: a short plain one as it is written. +function writtenValue(value: unknown): string { + return typeof value === "string" && /^[\x21-\x26\x28-\x7e]{1,40}$/.test(value) + ? `'${value}'` + : "what is written here"; } // The first line of a parser message, without the ':' that introduces the @@ -734,7 +913,20 @@ function crossCheck(raw: Record, context: BoardDeclarationConte const measureIds = idsOf("measures", "measure"); idsOf("ratios", "ratio"); // A version 1 file has no observations: its `observe` is refused as a key. - if (raw.board_version !== 1) idsOf("observe", "observation", "key"); + if (raw.board_version !== 1) { + idsOf("observe", "observation", "key"); + // The value holds one answer for each word, in the order declared. + for (const [i, o] of entries(raw, "observe") ?? []) { + if (o.kind !== "http" || !Array.isArray(o.words)) continue; + const seen = new Map(); + o.words.forEach((word: unknown, j) => { + if (typeof word !== "string") return; + const first = seen.get(word); + if (first === undefined) seen.set(word, j); + else errors.push(`observe[${i}].words[${j}]: duplicate word (first at words[${first}])`); + }); + } + } const repoPaths = new Set(context.manifestRepoPaths); const declared = diff --git a/packages/core/src/board/guide.test.ts b/packages/core/src/board/guide.test.ts index 408b6ac6..7780b233 100644 --- a/packages/core/src/board/guide.test.ts +++ b/packages/core/src/board/guide.test.ts @@ -1,4 +1,4 @@ -import { execFileSync, spawn } from "node:child_process"; +import { execFile, execFileSync, spawn } from "node:child_process"; import { chmod, mkdir, @@ -10,6 +10,8 @@ import { symlink, writeFile, } from "node:fs/promises"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; @@ -827,3 +829,205 @@ describe("boardGuide: the commands run", () => { } }); }); + +// The board of a guide's input: declared, with no record yet. +function declared(d: BoardDeclaration): BoardGuideInput["board"] { + return { status: "declared", declaration: d, recordCount: 0, previous: NO_PREVIOUS_RECORDS }; +} + +// A board of version 3 whose observations are `observe`. +function httpDeclaration(observe: Record[]): BoardDeclaration { + return declaration({ board_version: 3, observe }); +} + +const SHOP = "https://example.com/shop/item-1"; +const IN_STOCK = "\u5728\u5eab\u3042\u308a"; + +describe("boardGuide: an http observation", () => { + const full = { + key: "shop", + kind: "http", + url: SHOP, + words: ["in stock", IN_STOCK, "AT&T"], + count_pattern: "", + capture_pattern: "Version ([0-9.]+)", + }; + + it("gives a GET without -f, then the body read with the patterns and words after --, quoted", () => { + const guide = boardGuide(guideInput({ board: declared(httpDeclaration([full])) })); + const blocks = blocksIn(guide); + const get = blocks.find((b) => b.includes("curl ")) as string; + expect(get).toContain( + `curl -sS -L --max-redirs 10 --max-time 20 -o "$W/http-shop.body" -w '%{http_code} %{num_redirects} %{url_effective}\\n' '${SHOP}'; echo "curl exit=$?"`, + ); + expect(get).not.toMatch(/ -f|-fsS/); + const read = blocks.find((b) => b.includes("node -e")) as string; + expect(read).toMatch(/^cd .* && node -e '[^']+' -- /); + expect(read).toContain( + ` -- '' 'Version ([0-9.]+)' 'in stock' '${IN_STOCK}' 'AT&T' < "$W/http-shop.body"; echo "exit=$?"`, + ); + expect(read).toContain('readFileSync(0, "utf8")'); + expect(guide).toContain("#### `shop` (http)"); + expect(guide).toContain("When the status is 2xx, read the body."); + expect(guide).toContain("not line by line as a measure's `pattern` is"); + expect(guide).toContain( + `- \`words\`: whether the body holds each word, as true or false in the order the command gives them (\`in stock\`, \`${IN_STOCK}\`, \`AT&T\`)`, + ); + expect(guide).toContain( + "- `count`: how many times `` matches the body (0 when it does not)", + ); + expect(guide).toContain( + "- `capture`: what the first group of the first match of `Version ([0-9.]+)` holds, as a string (null when nothing matches)", + ); + expect(guide).toContain( + "When the status is not 2xx, do not read the body: give each of `words`, `count` and `capture` it declares as null", + ); + expect(guide).toContain("A 4xx or 5xx is a value, not an observation not made"); + expect(guide).toContain("more than 10 redirects: exit 47), or reading the body exits 1,"); + expect(guide).toContain(`Source: \`${SHOP}\`.`); + const observed = templateIn(guide).observed as Record; + expect(observed.shop?.source).toBe(SHOP); + }); + + it("gives '' for a pattern not declared, and no body read when nothing is read off the body", () => { + const guide = boardGuide( + guideInput({ + board: declared( + httpDeclaration([ + { key: "words_only", kind: "http", url: SHOP, words: ["x"] }, + { key: "bare", kind: "http", url: SHOP }, + ]), + ), + }), + ); + const blocks = blocksIn(guide); + expect(blocks.find((b) => b.includes('"$W/http-words_only.body"; echo'))).toContain( + ` -- '' '' 'x' < "$W/http-words_only.body"`, + ); + expect(blocks.some((b) => b.includes("http-bare.body") && b.includes("node -e"))).toBe(false); + const bare = guide.slice(guide.indexOf("#### `bare` (http)")); + expect(bare).toContain( + 'Value: `{ "status": , "redirects": , "final_url": }`', + ); + expect(bare.slice(0, bare.indexOf("### 4."))).not.toContain("2xx"); + }); +}); + +describe("boardGuide: an http observation, run", () => { + let dir: string; + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "basou-board-http-")); + }); + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + // Run a block of the guide in a shell, with W under dir, and no proxy. + const run = (script: string) => + new Promise<{ out: string; err: string }>((resolve) => { + execFile( + "sh", + ["-c", script], + { env: { ...process.env, TMPDIR: dir, NO_PROXY: "*", no_proxy: "*" }, encoding: "utf8" }, + (_error, out, err) => resolve({ out, err }), + ); + }); + + const blockOf = (guide: string, key: string, has: string) => + blocksIn(guide).find((b) => b.includes(`"$W/http-${key}.body"`) && b.includes(has)) as string; + + it("reads the words, the count and the capture off the whole body, decoded once and without its BOM", async () => { + const guide = boardGuide( + guideInput({ + anchor: dir, + board: declared( + httpDeclaration([ + { + key: "shop", + kind: "http", + url: SHOP, + words: [IN_STOCK, "sold out", "AT&T"], + count_pattern: "^|", + capture_pattern: "Version ([0-9.]+)", + }, + { key: "opt", kind: "http", url: SHOP, capture_pattern: "x(y)?" }, + ]), + ), + }), + ); + const work = boardGuideWorkDir(dir).replace(TMPDIR_WORD, dir); + await mkdir(work, { recursive: true }); + const read = blockOf(guide, "shop", "node -e"); + // The word starts one byte before 64 KiB (after the 3 bytes of the BOM + // and the 8 of ), so a chunk of a stream would cut it. + const body = `\ufeff${"a".repeat(65535 - 3 - 8)}${IN_STOCK} AT&T 12 Version 2.3 and Version 9.9`; + await writeFile(join(work, "http-shop.body"), body, "utf8"); + expect(await run(read)).toEqual({ + out: '{"words":[true,false,false],"count":3,"capture":"2.3"}\nexit=0\n', + err: "", + }); + await writeFile(join(work, "http-shop.body"), "nothing here", "utf8"); + expect((await run(read)).out).toBe( + '{"words":[false,false,false],"count":0,"capture":null}\nexit=0\n', + ); + // A group that took no part in the match gives null, as no match does. + await writeFile(join(work, "http-opt.body"), "x", "utf8"); + expect((await run(blockOf(guide, "opt", "node -e"))).out).toBe('{"capture":null}\nexit=0\n'); + }); + + it("stops rather than count an empty match the declaration could not see", async () => { + const guide = boardGuide( + guideInput({ + anchor: dir, + board: declared( + httpDeclaration([{ key: "edge", kind: "http", url: SHOP, count_pattern: "\\b" }]), + ), + }), + ); + const work = boardGuideWorkDir(dir).replace(TMPDIR_WORD, dir); + await mkdir(work, { recursive: true }); + await writeFile(join(work, "http-edge.body"), "a b", "utf8"); + expect(await run(blockOf(guide, "edge", "node -e"))).toEqual({ + out: "exit=1\n", + err: "count_pattern matched an empty string at 0\n", + }); + }); + + it("prints the status, the redirects and where it ended, a 404 with exit 0, and exit 47 past 10 redirects", async () => { + const server = createServer((req, res) => { + if (req.url === "/moved") { + res.writeHead(301, { location: "/item" }).end(); + } else if (req.url === "/item") { + res.writeHead(200, { "content-type": "text/html" }).end(IN_STOCK); + } else if (req.url?.startsWith("/loop")) { + res.writeHead(302, { location: `/loop${(req.url.length % 50) + 1}` }).end(); + } else { + res.writeHead(404).end("not here"); + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + try { + const guide = boardGuide( + guideInput({ + anchor: dir, + board: declared(httpDeclaration([{ key: "shop", kind: "http", url: SHOP }])), + }), + ); + const get = (path: string) => + run(blockOf(guide, "shop", "curl ").replace(`'${SHOP}'`, `'${origin}${path}'`)); + expect(await get("/moved")).toEqual({ + out: `200 1 ${origin}/item\ncurl exit=0\n`, + err: "", + }); + const work = boardGuideWorkDir(dir).replace(TMPDIR_WORD, dir); + expect(await readFile(join(work, "http-shop.body"), "utf8")).toBe(IN_STOCK); + expect((await get("/gone")).out).toBe(`404 0 ${origin}/gone\ncurl exit=0\n`); + expect(await readFile(join(work, "http-shop.body"), "utf8")).toBe("not here"); + const loop = await get("/loop"); + expect(loop.out).toMatch(/^302 10 .*\ncurl exit=47\n$/); + } finally { + await new Promise((resolve) => server.close(resolve)); + } + }); +}); diff --git a/packages/core/src/board/guide.ts b/packages/core/src/board/guide.ts index 6c2d90df..3be3bd2b 100644 --- a/packages/core/src/board/guide.ts +++ b/packages/core/src/board/guide.ts @@ -2,6 +2,9 @@ import { createHash } from "node:crypto"; import { decodeTime } from "ulid"; import { displayPath } from "../lib/display-path.js"; import { + BOARD_HTTP_CAPTURE_FLAGS, + BOARD_HTTP_COUNT_FLAGS, + BOARD_HTTP_MAX_REDIRECTS, BOARD_STAGE_IDS, type BoardDeclaration, type BoardObserve, @@ -445,7 +448,7 @@ function observeSteps( ): void { if (observe.length === 0) { line( - "This board declares nothing to observe outside basou. If a lane reaches its users through something outside the repos (a package registry, a release, a site, a production database), declare it under `observe` (board_version 2) through an axis review (step 2b).", + "This board declares nothing to observe outside basou. If a lane reaches its users through something outside the repos (a package registry, a release, a site, a production database), declare it under `observe` (board_version 2 on, and 3 for `http`) through an axis review (step 2b).", "", ); return; @@ -511,6 +514,9 @@ function observeSteps( "", ); break; + case "http": + httpSteps(o, go, line, block); + break; case "manual": line( `Only the operator can make this one: ${displayPath(o.how)}`, @@ -585,12 +591,79 @@ function sourceOf(o: BoardObserve): string { case "github_ci": return `gh run list --repo ${o.repo} --workflow ${o.workflow} --branch ${o.branch}`; case "page_version": + case "http": return o.url; case "manual": return "the operator"; } } +// What reads the values of an http observation off its body, on stdin. The +// patterns and the words come after `--`, so none is read as an option of +// node's: the count_pattern, the capture_pattern (each '' when not declared) +// and the words. The body is decoded only once all of it is read, so a +// character is never cut where a chunk ends, and a leading BOM is taken +// off; nothing else of it is decoded. An empty match would count positions, +// not matches: it stops instead. It prints the values as JSON. +const HTTP_BODY_SCRIPT = [ + "const [count, capture, ...words] = process.argv.slice(1);", + 'let t = require("fs").readFileSync(0, "utf8");', + "if (t.charCodeAt(0) === 0xfeff) t = t.slice(1);", + "const v = {};", + "if (words.length > 0) v.words = words.map((w) => t.includes(w));", + `if (count !== "") { let n = 0; for (const m of t.matchAll(new RegExp(count, "${BOARD_HTTP_COUNT_FLAGS}"))) { if (m[0] === "") { console.error("count_pattern matched an empty string at " + m.index); process.exit(1); } n++; } v.count = n; }`, + `if (capture !== "") { const m = new RegExp(capture, "${BOARD_HTTP_CAPTURE_FLAGS}").exec(t); v.capture = m === null || m[1] === undefined ? null : m[1]; }`, + "console.log(JSON.stringify(v));", +].join(" "); + +// How an http observation is made: a GET, then its body read when the +// observation declares what to read off it. +function httpSteps( + o: Extract, + go: string, + line: (...lines: string[]) => void, + block: (...lines: string[]) => void, +): void { + const body = `"$W/http-${o.key}.body"`; + const reads = [ + ...(o.words === undefined + ? [] + : [ + `\`words\`: whether the body holds each word, as true or false in the order the command gives them (${o.words.map((w) => `\`${displayPath(w)}\``).join(", ")})`, + ]), + ...(o.count_pattern === undefined + ? [] + : [ + `\`count\`: how many times \`${displayPath(o.count_pattern)}\` matches the body (0 when it does not)`, + ]), + ...(o.capture_pattern === undefined + ? [] + : [ + `\`capture\`: what the first group of the first match of \`${displayPath(o.capture_pattern)}\` holds, as a string (null when nothing matches)`, + ]), + ]; + line( + `A GET of ${o.url}, following up to ${BOARD_HTTP_MAX_REDIRECTS} redirects. It prints the status of the last response, how many redirects led to it and the URL it ended at (no \`-f\`: a 4xx or 5xx is printed and exits 0):`, + ); + block( + `${go} && curl -sS -L --max-redirs ${BOARD_HTTP_MAX_REDIRECTS} --max-time 20 -o ${body} -w '%{http_code} %{num_redirects} %{url_effective}\\n' ${shellWord(o.url)}; echo "curl exit=$?"`, + ); + if (reads.length > 0) { + line( + "When the status is 2xx, read the body. The words are found as written, character for character, and HTML entities are not decoded; the patterns are matched against the whole body at once, not line by line as a measure's `pattern` is, so `^` and `$` are the body's start and end:", + ); + const args = [o.count_pattern ?? "", o.capture_pattern ?? "", ...(o.words ?? [])]; + block( + `${go} && node -e ${shellWord(HTTP_BODY_SCRIPT)} -- ${args.map(shellWord).join(" ")} < ${body}; echo "exit=$?"`, + ); + line("It prints, as JSON:", "", ...reads.map((r) => `- ${r}`), ""); + } + line( + `Value: \`{ "status": , "redirects": , "final_url": ${reads.length > 0 ? ", " : ""} }\`, the numbers as numbers. A 4xx or 5xx is a value, not an observation not made: the status is what was observed.${reads.length > 0 ? " When the status is not 2xx, do not read the body: give each of `words`, `count` and `capture` it declares as null (an error page's text is not the page's)." : ""} When curl exits other than 0 (the name does not resolve, TLS fails, the time runs out, more than ${BOARD_HTTP_MAX_REDIRECTS} redirects: exit 47)${reads.length > 0 ? ", or reading the body exits 1," : ""} the observation is not made: a null with that as the error. Source: \`${sourceOf(o)}\`.`, + "", + ); +} + // The record's input to fill in: the previous record's cells beside each // (null where it has none), or no `previous` with no previous record. function templateOf(d: BoardDeclaration, previous: BoardPreviousRecords): string { diff --git a/packages/core/src/board/init.test.ts b/packages/core/src/board/init.test.ts index 739a8c37..7223132f 100644 --- a/packages/core/src/board/init.test.ts +++ b/packages/core/src/board/init.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { boardInitStrings } from "../lib/view-strings.js"; -import { parseBoardDeclaration } from "./declaration.js"; +import { BOARD_OBSERVE_KINDS, parseBoardDeclaration } from "./declaration.js"; import { BOARD_INIT_LANE_ID, boardInitText } from "./init.js"; const en = boardInitStrings("en"); @@ -19,7 +19,7 @@ describe("boardInitText", () => { const result = read(text); if (!result.ok) throw new Error(result.errors.join("\n")); const d = result.declaration; - expect(d.board_version).toBe(2); + expect(d.board_version).toBe(3); expect(d.title).toBe("shop progress board"); expect(Object.keys(d.stages)).toEqual(["01", "02", "03", "04", "05", "06"]); expect(d.stages["04"].meaning).toBe(en.stages["04"]); @@ -48,6 +48,11 @@ describe("boardInitText", () => { expect(text).toContain(`# ${words.comments.measures}\nmeasures: []\n`); } expect(en.comments.measures).toContain("One with a lane is drawn in that lane's section"); + // The comment on observe names every kind a board can declare, http among them. + for (const kind of BOARD_OBSERVE_KINDS) { + expect(en.comments.observe).toContain(kind); + expect(ja.comments.observe).toContain(kind); + } expect(ja.comments.measures).toContain("lane を付けたものは盤のページのそのレーンの節に描かれ"); }); diff --git a/packages/core/src/board/page.test.ts b/packages/core/src/board/page.test.ts index 5816dd8a..a821cdca 100644 --- a/packages/core/src/board/page.test.ts +++ b/packages/core/src/board/page.test.ts @@ -207,6 +207,38 @@ describe("boardPage", () => { expect(older.status).toBe("ok"); }); + it("draws a record of version 3, which holds an http observation, its value as it is", async () => { + await mkdir(records); + await place(A, record({ record_version: 2 })); + const v3 = record({ + record_version: 3, + observed: { + shop: { + value: { status: 404, redirects: 0, final_url: "https://example.com/x", words: [null] }, + observed_at: "2026-10-07T00:00:00Z", + source: "https://example.com/x", + }, + }, + }); + v3.declaration = { + ...v3.declaration, + observe: [{ key: "shop", kind: "http", url: "https://example.com/x", words: ["in stock"] }], + } as typeof v3.declaration; + await place(B, v3); + const page = await boardPage(records); + if (page.status !== "ok") throw new Error(page.why); + expect(page.id).toBe(B); + expect(page.board.summary.observed).toEqual([ + { + name: "shop", + value: { status: 404, redirects: 0, final_url: "https://example.com/x", words: [null] }, + observed_at: "2026-10-07T00:00:00Z", + source: "https://example.com/x", + }, + ]); + expect((await boardPage(records, A)).status).toBe("ok"); + }); + it("draws the heading, the tiles and the observations from the record", async () => { await mkdir(records); await place(A, record()); @@ -554,11 +586,11 @@ describe("boardPage", () => { older: null, newer: B, }); - await place(A, record({ record_version: 3 })); + await place(A, record({ record_version: 4 })); expect(await boardPage(records, A)).toMatchObject({ why: "unknown_version", id: A, - version: 3, + version: 4, }); await place(A, record({ prose: { summary: 1 } })); expect(await boardPage(records, A)).toMatchObject({ why: "not_a_record", id: A }); diff --git a/packages/core/src/board/previous.test.ts b/packages/core/src/board/previous.test.ts index 94314c9b..186a8254 100644 --- a/packages/core/src/board/previous.test.ts +++ b/packages/core/src/board/previous.test.ts @@ -112,10 +112,10 @@ describe("readPreviousRecords", () => { status: "unreadable", reason: `the record ${B} could not be read as JSON`, }); - await place(`${C}.json`, record({ record_version: 3 })); + await place(`${C}.json`, record({ record_version: 4 })); const unread = { status: "unreadable", - reason: `the record ${C} is of record_version 3, which this basou does not read`, + reason: `the record ${C} is of record_version 4, which this basou does not read`, }; expect(await readPreviousRecords(records)).toEqual({ last: unread, lastReview: unread }); await place(`${C}.json`, record({ cells: "all done" })); diff --git a/packages/core/src/board/record.test.ts b/packages/core/src/board/record.test.ts index 9efead7c..b90ef144 100644 --- a/packages/core/src/board/record.test.ts +++ b/packages/core/src/board/record.test.ts @@ -375,7 +375,7 @@ describe("parseRecordInput: a board that declares what to observe", () => { recordedAt: new Date("2026-10-08T00:00:00.000Z"), recordedWith: { basou: "0.0.0-test", build: null }, }); - expect(r.record_version).toBe(2); + expect(r.record_version).toBe(3); expect(r.declaration.observe).toEqual([ { key: "npm_cli", kind: "npm_version", package: "@scope/cli" }, { key: "users", kind: "manual", how: "ask the operator" }, @@ -385,6 +385,75 @@ describe("parseRecordInput: a board that declares what to observe", () => { }); }); +describe("buildRecord: a board of version 3 that observes over http", () => { + const url = "https://example.com/shop/item-1"; + const httpDeclaration = (): BoardDeclaration => { + const result = parseBoardDeclaration( + stringify({ + board_version: 3, + title: "Test board", + stages: Object.fromEntries(STAGES.map((id) => [id, { meaning: `stage ${id}` }])), + lanes: [ + { id: "core", name: "Core" }, + { id: "docs", name: "Docs" }, + ], + observe: [ + { + key: "shop", + kind: "http", + url, + words: ["in stock", "sold out"], + count_pattern: "", + capture_pattern: "Version ([0-9.]+)", + }, + { key: "gone", kind: "http", url, words: ["x"] }, + ], + axis: { version: 1, review_due_days: 60 }, + effort: { start: "2026-04-28", time_zone: "UTC" }, + }), + { manifestRepoPaths: [] }, + ); + if (!result.ok) throw new Error(result.errors.join("\n")); + return result.declaration; + }; + + it("takes the value of a GET as given, and keeps the observation as declared in a record of version 3", () => { + const at = "2026-10-17T00:00:00Z"; + const observed = { + shop: { + value: { + status: 200, + redirects: 1, + final_url: `${url}/`, + words: [true, false], + count: 42, + capture: "2.3", + }, + observed_at: at, + source: url, + }, + // Not 2xx: the status is the value, and what the body would give is null. + gone: { + value: { status: 404, redirects: 0, final_url: url, words: [null] }, + observed_at: at, + source: url, + }, + }; + const parsed = parseRecordInput(input({ observed }), httpDeclaration()); + if (!parsed.ok) throw new Error(parsed.errors.join("\n")); + const r = buildRecord({ + declaration: httpDeclaration(), + measurement: { digest: DIGEST } as BoardMeasurement, + recordInput: parsed.input, + recordedAt: new Date("2026-10-17T00:00:00.000Z"), + recordedWith: { basou: "0.0.0-test", build: null }, + }); + expect(r.record_version).toBe(3); + expect(r.declaration.observe).toEqual(httpDeclaration().observe); + expect(r.observed).toEqual(observed); + }); +}); + describe("orderAnomalies", () => { it("finds a stage not started, blocked or shelved before one done or begun, not counting unverified", () => { const parsed = parseRecordInput( @@ -458,7 +527,7 @@ describe("buildRecord and writeRecord", () => { effort: { start: "2026-04-28", time_zone: "UTC" }, observe: [], }); - expect(r.record_version).toBe(2); + expect(r.record_version).toBe(3); expect(r.recorded_at).toBe("2026-10-07T00:00:00.000Z"); const records = join(dir, "records"); const name = await writeRecord(records, r); diff --git a/packages/core/src/board/record.ts b/packages/core/src/board/record.ts index 596a0bef..e7d2bd7c 100644 --- a/packages/core/src/board/record.ts +++ b/packages/core/src/board/record.ts @@ -23,12 +23,15 @@ function isRecord(value: unknown): value is Record { * The version of a record's shape, the shape of the measurement it holds * included. Raised whenever that shape changes: 2 added the observations the * declaration names (`declaration.observe`) and what each stage is judged by - * (`look` and `notes` of `declaration.stages`). Every version is read. + * (`look` and `notes` of `declaration.stages`); 3 holds a declaration of + * board_version 3, which may observe an HTTP response (`http`). Every record + * is written in the newest version, whatever the version of the board it + * records, and every version is read. */ -export const BOARD_RECORD_VERSION = 2; +export const BOARD_RECORD_VERSION = 3; /** The record versions this basou reads. A version is added here, never removed. */ -export const BOARD_RECORD_VERSIONS = [1, 2] as const; +export const BOARD_RECORD_VERSIONS = [1, 2, 3] as const; /** The states a cell of the board can be in. */ export const BOARD_CELL_STATES = [ diff --git a/packages/core/src/lib/view-strings.ts b/packages/core/src/lib/view-strings.ts index 15792138..155abaa9 100644 --- a/packages/core/src/lib/view-strings.ts +++ b/packages/core/src/lib/view-strings.ts @@ -1450,7 +1450,7 @@ const BOARD_INIT_EN: BoardInitStrings = { measures: "What to count in the repos (file_count, dir_count, line_count, match_count, regex_capture, json_length, trail_count). One with a lane is drawn in that lane's section of the board page; one without is kept only in the records and their differences (and in a ratio that uses it).", observe: - "What to observe outside basou (npm_version, github_release, github_open_issues, github_open_prs, github_ci, page_version, manual). basou never makes these; the guide says how.", + "What to observe outside basou (npm_version, github_release, github_open_issues, github_open_prs, github_ci, page_version, http, manual). basou never makes these; the guide says how.", components: "Left empty: the first measure names every component it finds, and the axis review registers each with its lanes or '-' and a note.", axis: "Raise version when a lane is added or removed or a stage's meaning changes.", @@ -1487,7 +1487,7 @@ const BOARD_INIT_JA: BoardInitStrings = { measures: "repo の中で数えるもの（file_count・dir_count・line_count・match_count・regex_capture・json_length・trail_count）。lane を付けたものは盤のページのそのレーンの節に描かれ、付けないものは記録と差分（と、それを使う ratio）にだけ残る。", observe: - "basou の外で観測するもの（npm_version・github_release・github_open_issues・github_open_prs・github_ci・page_version・manual）。basou は取りに行かない。取り方は guide が出す。", + "basou の外で観測するもの（npm_version・github_release・github_open_issues・github_open_prs・github_ci・page_version・http・manual）。basou は取りに行かない。取り方は guide が出す。", components: "空のままでよい。最初の measure が見つけた構成要素を全部名指しし、軸の見直しで各要素をレーンに（数えないなら '-' と note で）登録する。", axis: "レーンの増減や段の意味を変えたら version を上げる。", From ba87675bb81e734eb1987178b75c1c5d83540ea2 Mon Sep 17 00:00:00 2001 From: Takashi Matsuyama Date: Sat, 17 Oct 2026 09:06:53 +0900 Subject: [PATCH 2/2] basou: Apply the review of the http observation: the value's keys, nulls off 2xx, a kind under an older key, and the guide's code spans - board record: the keys of an http observation's value (status, redirects, final_url, words, count, capture) are words of the input's shape, so the check for other workspaces' names no longer refuses a value written as the guide says when a registered workspace is named like one of them. What they hold is still looked at. - board guide: off 2xx, say "words": null and not a list of nulls, naming only what the board declares; capture is null also when its first group takes no part in the match; words and patterns are written as Markdown code that shows backquotes and spaces as they are. - A board of version 1 whose observe holds an http observation with a problem inside it is told version 3, not 2: a key is put down to a later version when what is wrong inside it is what that version reads. - Only an empty word or pattern is empty: one of spaces is taken. - The record version's comment says a record of version 3 may hold a declaration of version 3. - Tests for the u flag in the guide's line, case, a word at the body's start, a BOM inside the body, an http key that is not an id, a word given twice outside http, a file with no version, empty matches only under u or only in the empty text, and the guide's text with nothing observed. --- CHANGELOG.md | 4 +- packages/cli/src/commands/board.test.ts | 35 +++++++++++++++ packages/cli/src/commands/board.ts | 11 ++++- packages/core/src/board/declaration.test.ts | 47 ++++++++++++++++++--- packages/core/src/board/declaration.ts | 37 ++++++++++++---- packages/core/src/board/guide.test.ts | 45 +++++++++++++++++--- packages/core/src/board/guide.ts | 30 ++++++++++--- packages/core/src/board/page.test.ts | 4 +- packages/core/src/board/record.test.ts | 2 +- packages/core/src/board/record.ts | 2 +- 10 files changed, 187 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94ed94ca..29fb6775 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,9 @@ All notable changes to **basou** are recorded here. The project follows `'`, or starts with `-` is refused, as is a word given twice, a `count_pattern` that matches the empty string and a `capture_pattern` with no capturing group; the guide's line stops, rather than count, at an - empty match the declaration could not see. + empty match the declaration could not see. The keys of the value are + basou's words, so `basou board record`'s check for the names of other + workspaces looks at what they hold, not at them. ### Changed diff --git a/packages/cli/src/commands/board.test.ts b/packages/cli/src/commands/board.test.ts index 046e0020..6ca6b5a9 100644 --- a/packages/cli/src/commands/board.test.ts +++ b/packages/cli/src/commands/board.test.ts @@ -1285,6 +1285,41 @@ describe("basou board record", () => { expect(process.exitCode).toBe(1); }); + it("takes the keys of an http observation's value as words of the input's shape, and still looks at what they hold", async () => { + const { repo, input } = await judged(); + const url = "https://example.com/item"; + const at = { observed_at: "2026-10-17T00:00:00Z", source: url }; + const value = { + status: 200, + redirects: 0, + final_url: url, + words: [true], + count: 3, + capture: "2.3", + }; + capture(); + for (const name of ["status", "redirects", "final_url", "words", "count", "capture"]) { + await registered(repo, name); + const http = { + ...input, + measure_digest: await measuredAgain(repo), + observed: { shop: { value, ...at } }, + }; + capture(); + expect((await doRunBoardRecord({ dryRun: true }, fed(repo, http))).dry_run).toBe(true); + } + const name = await registered(repo); + const named = { + ...input, + measure_digest: await measuredAgain(repo), + observed: { shop: { value: { ...value, capture: `${name} 2.3` }, ...at } }, + }; + const { err } = capture(); + await runBoardRecord({ dryRun: true }, fed(repo, named)); + expect(err.join("\n")).toContain(" - observed (a named entry).value.capture"); + expect(process.exitCode).toBe(1); + }); + it("reads a number in a value as the text it is written as", async () => { const { repo, input } = await judged(); const name = await registered(repo, "424242"); diff --git a/packages/cli/src/commands/board.ts b/packages/cli/src/commands/board.ts index 160b1364..46af63b6 100644 --- a/packages/cli/src/commands/board.ts +++ b/packages/cli/src/commands/board.ts @@ -612,8 +612,9 @@ async function checkAndRecord( } // The keys of a record's input, which a location may name: the words of its -// shape. Any other key is a name someone chose (an observation's, a lane's, -// or one the input should not have), which may be the very name found. +// shape, and those of an http observation's value, which the guide fixes. +// Any other key is a name someone chose (an observation's, a lane's, or one +// the input should not have), which may be the very name found. const INPUT_KEYS = new Set([ "measure_digest", "observed", @@ -637,6 +638,12 @@ const INPUT_KEYS = new Set([ "model", "self_reported", "triggers", + "status", + "redirects", + "final_url", + "words", + "count", + "capture", ]); // The places whose values are held to a shape a refusal does not quote (a diff --git a/packages/core/src/board/declaration.test.ts b/packages/core/src/board/declaration.test.ts index c3b717d8..1b3334ea 100644 --- a/packages/core/src/board/declaration.test.ts +++ b/packages/core/src/board/declaration.test.ts @@ -1516,13 +1516,28 @@ describe("parseBoardDeclaration: board_version 3 and http", () => { errorsOf(parse(withHttp({ url, words: ["", " ", "a\tb", "a\u0085b", "-p", "it's", 1] }))), ).toEqual([ "observe[0].words[0]: must be a non-empty string", - "observe[0].words[1]: must be a non-empty string", "observe[0].words[2]: must not contain control characters", "observe[0].words[3]: must not contain control characters", "observe[0].words[4]: must not start with '-'", 'observe[0].words[5]: must not contain "\'"', "observe[0].words[6]: Invalid input: expected string, received number", ]); + // Only the empty text is empty, and a '-' after a space starts no option. + expect(parse(withHttp({ url, words: [" ", " -p", "In Stock"] })).ok).toBe(true); + }); + + it("refuses an http key that is not an id: the guide names a working file by it", () => { + for (const key of ["Shop", "a/b", "a:b", "-a", "a b"]) { + expect(errorsOf(parse(withHttp({ key, url })))).toEqual([ + "observe[0].key: must start with a lowercase letter and use only a-z, 0-9, '_' and '-'", + ]); + } + }); + + it("looks for a word given twice only in an http observation", () => { + const doc = withHttp({ url }); + doc.observe = [{ key: "m", kind: "manual", how: "ask", words: ["a", "a"] }]; + expect(errorsOf(parse(doc))).toEqual(["observe[0]: unknown key 'words'"]); }); it("refuses a word given twice, saying where it was first", () => { @@ -1537,9 +1552,12 @@ describe("parseBoardDeclaration: board_version 3 and http", () => { const at = (count_pattern: unknown) => errorsOf(parse(withHttp({ url, count_pattern }))); const empty = "observe[0].count_pattern: must not match the empty string: each match is counted"; - for (const pattern of ["a*", "x?", "^", "$", "(?:)", "a|", "[a-z]{0,3}"]) { + // \p{L}* matches the empty string only under the u flag, ^$ only in the empty text. + for (const pattern of ["a*", "x?", "^", "$", "(?:)", "a|", "[a-z]{0,3}", "\\p{L}*", "^$"]) { expect(at(pattern)).toEqual([empty]); } + // Only the empty text is empty: a pattern of a space counts spaces. + expect(parse(withHttp({ url, count_pattern: " " })).ok).toBe(true); expect(at("")).toEqual(["observe[0].count_pattern: must be a non-empty string"]); expect(at("-x")).toEqual(["observe[0].count_pattern: must not start with '-'"]); expect(at("a\nb")).toEqual(["observe[0].count_pattern: must not contain control characters"]); @@ -1596,6 +1614,13 @@ describe("parseBoardDeclaration: board_version 3 and http", () => { ]); }); + it("holds a file with no version, or one that is not a number, to the newest shape", () => { + const reads = "board_version: must be a board version this basou reads (1, 2 or 3)"; + for (const version of [undefined, "2", "3"]) { + expect(errorsOf(parse({ ...withHttp({ url }), board_version: version }))).toEqual([reads]); + } + }); + it("does not put a problem down to a later version that refuses the same part", () => { const doc = boardV2(); doc.observe = [{ key: "a", kind: "http" }]; @@ -1604,9 +1629,21 @@ describe("parseBoardDeclaration: board_version 3 and http", () => { "observe[0].kind: 'http' is read from board_version 3 on; this file is board_version 2", ]); const v1 = board(); - v1.observe = [{ key: "a", kind: "http" }]; - // No version reads the key as written (3 refuses the entry inside it): - // it is put down to the first version that takes the key. + // No version reads the key as written (3 refuses the entry inside it), but + // what version 2 refuses inside it is a kind version 3 reads: from 3 on. + for (const http of [ + { key: "a", kind: "http" }, + { key: "a", kind: "http", url: "http://example.com/" }, + { key: "a", kind: "http", url, words: ["-p"] }, + { key: "a", kind: "http", url, method: "GET" }, + ]) { + v1.observe = [http]; + expect(errorsOf(parse(v1))).toEqual([ + "(top level): 'observe' is read from board_version 3 on; this file is board_version 1", + ]); + } + // What every version refuses inside it: put down to the first version that takes the key. + v1.observe = [{ key: "a", kind: "manual" }]; expect(errorsOf(parse(v1))).toEqual([ "(top level): 'observe' is read from board_version 2 on; this file is board_version 1", ]); diff --git a/packages/core/src/board/declaration.ts b/packages/core/src/board/declaration.ts index bfe94cae..cfb8416e 100644 --- a/packages/core/src/board/declaration.ts +++ b/packages/core/src/board/declaration.ts @@ -354,7 +354,7 @@ const httpsUrl = z.string().superRefine((s, ctx) => { // reaches the command after `--`, so it is never read as an option; a // leading '-' is refused all the same, as for every value a command takes. function argumentProblem(s: string, quoteHint = ""): string | undefined { - if (s.trim().length === 0) return "must be a non-empty string"; + if (s.length === 0) return "must be a non-empty string"; if (hasControlCharacter(s)) return "must not contain control characters"; if (s.startsWith("-")) return "must not start with '-'"; if (s.includes("'")) return `must not contain "'"${quoteHint}`; @@ -689,19 +689,25 @@ function parseAs( // what is written told apart from what no version reads: a key, a kind, a // value of a form a later version takes. Each is put down to the first // version that reads that part of the file as it is written, with no problem -// at it, inside it or around it; a key, when no version reads it so, to the -// first version that takes the key, whatever is wrong inside it. +// at it, inside it or around it. A key no version reads so is put down to +// the first version that takes the key, whatever is wrong inside it, or to a +// later one when what is wrong inside it is what a later version reads (a +// kind of version 3 under a key of version 2). function formatOlderIssues(raw: unknown, version: number, issues: readonly Issue[]): string[] { const later = BOARD_VERSIONS.filter((v) => v > version).map((v) => ({ version: v, issues: schemaOf(v).safeParse(raw).error?.issues ?? [], })); - const readsAsWritten = (path: readonly PropertyKey[]): number | undefined => - later.find((l) => !l.issues.some((issue) => issuePaths(issue).some((p) => related(p, path)))) - ?.version; - const takesKey = (path: readonly PropertyKey[]): number | undefined => + const readsAsWritten = (path: readonly PropertyKey[], after = version): number | undefined => later.find( (l) => + l.version > after && + !l.issues.some((issue) => issuePaths(issue).some((p) => related(p, path))), + )?.version; + const takesKey = (path: readonly PropertyKey[], after = version): number | undefined => + later.find( + (l) => + l.version > after && !l.issues.some((issue) => issuePaths(issue).some( (p) => @@ -711,8 +717,23 @@ function formatOlderIssues(raw: unknown, version: number, issues: readonly Issue ), ), )?.version; + const keyFrom = (path: readonly PropertyKey[]): number | undefined => { + const taken = takesKey(path); + if (taken === undefined) return undefined; + let from = taken; + for (const issue of later.find((l) => l.version === taken)?.issues ?? []) { + for (const p of issuePaths(issue)) { + if (p.length <= path.length || !related(p, path)) continue; + const inside = + readsAsWritten(p, taken) ?? + (issue.code === "unrecognized_keys" ? takesKey(p, taken) : undefined); + if (inside !== undefined && inside > from) from = inside; + } + } + return from; + }; const readFrom = (path: readonly PropertyKey[], isKey: boolean): number | undefined => - readsAsWritten(path) ?? (isKey ? takesKey(path) : undefined); + readsAsWritten(path) ?? (isKey ? keyFrom(path) : undefined); const readFromLine = (path: readonly PropertyKey[], what: string, from: number) => `${formatPath(path)}: ${what} read from board_version ${from} on; this file is board_version ${version}`; const out: string[] = []; diff --git a/packages/core/src/board/guide.test.ts b/packages/core/src/board/guide.test.ts index 7780b233..e7a32bce 100644 --- a/packages/core/src/board/guide.test.ts +++ b/packages/core/src/board/guide.test.ts @@ -345,6 +345,7 @@ describe("boardGuide: a board declared", () => { }), ); expect(guide).toContain("This board declares nothing to observe outside basou."); + expect(guide).toContain("declare it under `observe` (board_version 2 on, and 3 for `http`)"); expect(templateIn(guide).observed).toEqual({}); }); @@ -877,10 +878,10 @@ describe("boardGuide: an http observation", () => { "- `count`: how many times `` matches the body (0 when it does not)", ); expect(guide).toContain( - "- `capture`: what the first group of the first match of `Version ([0-9.]+)` holds, as a string (null when nothing matches)", + "- `capture`: what the first group of the first match of `Version ([0-9.]+)` holds, as a string (null when nothing matches, or when the first group takes no part in the match)", ); expect(guide).toContain( - "When the status is not 2xx, do not read the body: give each of `words`, `count` and `capture` it declares as null", + 'When the status is not 2xx, do not read the body: give `"words": null, "count": null, "capture": null`, each a null and not a list of nulls', ); expect(guide).toContain("A 4xx or 5xx is a value, not an observation not made"); expect(guide).toContain("more than 10 redirects: exit 47), or reading the body exits 1,"); @@ -910,6 +911,29 @@ describe("boardGuide: an http observation", () => { 'Value: `{ "status": , "redirects": , "final_url": }`', ); expect(bare.slice(0, bare.indexOf("### 4."))).not.toContain("2xx"); + expect(guide).toContain('give `"words": null`, each a null and not a list of nulls'); + }); + + it("writes a word or a pattern as Markdown code that shows it as it is, backquotes and spaces too", () => { + const guide = boardGuide( + guideInput({ + board: declared( + httpDeclaration([ + { + key: "odd", + kind: "http", + url: SHOP, + words: ["a`b", " in stock ", "``x"], + count_pattern: "|`", + }, + ]), + ), + }), + ); + expect(guide).toContain( + "in the order the command gives them (``a`b``, ` in stock `, ``` ``x ```)", + ); + expect(guide).toContain("- `count`: how many times `` |` `` matches the body"); }); }); @@ -946,11 +970,16 @@ describe("boardGuide: an http observation, run", () => { key: "shop", kind: "http", url: SHOP, - words: [IN_STOCK, "sold out", "AT&T"], + // Found as written: not ignoring case, and at the body's start too. + words: [IN_STOCK, "sold out", "AT&T", "", "VERSION 2.3"], count_pattern: "^|", capture_pattern: "Version ([0-9.]+)", }, { key: "opt", kind: "http", url: SHOP, capture_pattern: "x(y)?" }, + // Under the u flag . is one character, not one half of a surrogate pair. + { key: "flags", kind: "http", url: SHOP, count_pattern: ".", capture_pattern: "(.)" }, + // Only a BOM at the start is taken off. + { key: "bom", kind: "http", url: SHOP, count_pattern: "\\uFEFF" }, ]), ), }), @@ -963,16 +992,22 @@ describe("boardGuide: an http observation, run", () => { const body = `\ufeff${"a".repeat(65535 - 3 - 8)}${IN_STOCK} AT&T 12 Version 2.3 and Version 9.9`; await writeFile(join(work, "http-shop.body"), body, "utf8"); expect(await run(read)).toEqual({ - out: '{"words":[true,false,false],"count":3,"capture":"2.3"}\nexit=0\n', + out: '{"words":[true,false,false,true,false],"count":3,"capture":"2.3"}\nexit=0\n', err: "", }); await writeFile(join(work, "http-shop.body"), "nothing here", "utf8"); expect((await run(read)).out).toBe( - '{"words":[false,false,false],"count":0,"capture":null}\nexit=0\n', + '{"words":[false,false,false,false,false],"count":0,"capture":null}\nexit=0\n', ); // A group that took no part in the match gives null, as no match does. await writeFile(join(work, "http-opt.body"), "x", "utf8"); expect((await run(blockOf(guide, "opt", "node -e"))).out).toBe('{"capture":null}\nexit=0\n'); + await writeFile(join(work, "http-flags.body"), "\u{1F600}", "utf8"); + expect((await run(blockOf(guide, "flags", "node -e"))).out).toBe( + '{"count":1,"capture":"\u{1F600}"}\nexit=0\n', + ); + await writeFile(join(work, "http-bom.body"), "\ufeffa\ufeffb", "utf8"); + expect((await run(blockOf(guide, "bom", "node -e"))).out).toBe('{"count":1}\nexit=0\n'); }); it("stops rather than count an empty match the declaration could not see", async () => { diff --git a/packages/core/src/board/guide.ts b/packages/core/src/board/guide.ts index 3be3bd2b..52b79220 100644 --- a/packages/core/src/board/guide.ts +++ b/packages/core/src/board/guide.ts @@ -616,6 +616,17 @@ const HTTP_BODY_SCRIPT = [ "console.log(JSON.stringify(v));", ].join(" "); +// A text as Markdown code, as it is: fenced by more backquotes than it +// holds in a row, and padded with a space when it starts or ends with a +// backquote or a space, which the fence would otherwise eat. +function codeSpan(text: string): string { + const shown = displayPath(text); + const longest = Math.max(0, ...(shown.match(/`+/g) ?? []).map((run) => run.length)); + const fence = "`".repeat(longest + 1); + const pad = /^[` ]|[` ]$/.test(shown) ? " " : ""; + return `${fence}${pad}${shown}${pad}${fence}`; +} + // How an http observation is made: a GET, then its body read when the // observation declares what to read off it. function httpSteps( @@ -629,17 +640,17 @@ function httpSteps( ...(o.words === undefined ? [] : [ - `\`words\`: whether the body holds each word, as true or false in the order the command gives them (${o.words.map((w) => `\`${displayPath(w)}\``).join(", ")})`, + `\`words\`: whether the body holds each word, as true or false in the order the command gives them (${o.words.map(codeSpan).join(", ")})`, ]), ...(o.count_pattern === undefined ? [] : [ - `\`count\`: how many times \`${displayPath(o.count_pattern)}\` matches the body (0 when it does not)`, + `\`count\`: how many times ${codeSpan(o.count_pattern)} matches the body (0 when it does not)`, ]), ...(o.capture_pattern === undefined ? [] : [ - `\`capture\`: what the first group of the first match of \`${displayPath(o.capture_pattern)}\` holds, as a string (null when nothing matches)`, + `\`capture\`: what the first group of the first match of ${codeSpan(o.capture_pattern)} holds, as a string (null when nothing matches, or when the first group takes no part in the match)`, ]), ]; line( @@ -658,8 +669,17 @@ function httpSteps( ); line("It prints, as JSON:", "", ...reads.map((r) => `- ${r}`), ""); } - line( - `Value: \`{ "status": , "redirects": , "final_url": ${reads.length > 0 ? ", " : ""} }\`, the numbers as numbers. A 4xx or 5xx is a value, not an observation not made: the status is what was observed.${reads.length > 0 ? " When the status is not 2xx, do not read the body: give each of `words`, `count` and `capture` it declares as null (an error page's text is not the page's)." : ""} When curl exits other than 0 (the name does not resolve, TLS fails, the time runs out, more than ${BOARD_HTTP_MAX_REDIRECTS} redirects: exit 47)${reads.length > 0 ? ", or reading the body exits 1," : ""} the observation is not made: a null with that as the error. Source: \`${sourceOf(o)}\`.`, + // What the body would give, as the value holds it when the status is not 2xx. + const nulls = (["words", "count", "capture"] as const) + .filter((k) => (k === "words" ? o.words : k === "count" ? o.count_pattern : o.capture_pattern)) + .map((k) => `"${k}": null`) + .join(", "); + const notRead = + nulls === "" + ? "" + : ` When the status is not 2xx, do not read the body: give \`${nulls}\`, each a null and not a list of nulls (an error page's text is not the page's).`; + line( + `Value: \`{ "status": , "redirects": , "final_url": ${reads.length > 0 ? ", " : ""} }\`, the numbers as numbers. A 4xx or 5xx is a value, not an observation not made: the status is what was observed.${notRead} When curl exits other than 0 (the name does not resolve, TLS fails, the time runs out, more than ${BOARD_HTTP_MAX_REDIRECTS} redirects: exit 47)${reads.length > 0 ? ", or reading the body exits 1," : ""} the observation is not made: a null with that as the error. Source: \`${sourceOf(o)}\`.`, "", ); } diff --git a/packages/core/src/board/page.test.ts b/packages/core/src/board/page.test.ts index a821cdca..a59972bf 100644 --- a/packages/core/src/board/page.test.ts +++ b/packages/core/src/board/page.test.ts @@ -214,7 +214,7 @@ describe("boardPage", () => { record_version: 3, observed: { shop: { - value: { status: 404, redirects: 0, final_url: "https://example.com/x", words: [null] }, + value: { status: 404, redirects: 0, final_url: "https://example.com/x", words: null }, observed_at: "2026-10-07T00:00:00Z", source: "https://example.com/x", }, @@ -231,7 +231,7 @@ describe("boardPage", () => { expect(page.board.summary.observed).toEqual([ { name: "shop", - value: { status: 404, redirects: 0, final_url: "https://example.com/x", words: [null] }, + value: { status: 404, redirects: 0, final_url: "https://example.com/x", words: null }, observed_at: "2026-10-07T00:00:00Z", source: "https://example.com/x", }, diff --git a/packages/core/src/board/record.test.ts b/packages/core/src/board/record.test.ts index b90ef144..0f0276a4 100644 --- a/packages/core/src/board/record.test.ts +++ b/packages/core/src/board/record.test.ts @@ -434,7 +434,7 @@ describe("buildRecord: a board of version 3 that observes over http", () => { }, // Not 2xx: the status is the value, and what the body would give is null. gone: { - value: { status: 404, redirects: 0, final_url: url, words: [null] }, + value: { status: 404, redirects: 0, final_url: url, words: null }, observed_at: at, source: url, }, diff --git a/packages/core/src/board/record.ts b/packages/core/src/board/record.ts index e7d2bd7c..fcb8d968 100644 --- a/packages/core/src/board/record.ts +++ b/packages/core/src/board/record.ts @@ -23,7 +23,7 @@ function isRecord(value: unknown): value is Record { * The version of a record's shape, the shape of the measurement it holds * included. Raised whenever that shape changes: 2 added the observations the * declaration names (`declaration.observe`) and what each stage is judged by - * (`look` and `notes` of `declaration.stages`); 3 holds a declaration of + * (`look` and `notes` of `declaration.stages`); 3 may hold a declaration of * board_version 3, which may observe an HTTP response (`http`). Every record * is written in the newest version, whatever the version of the board it * records, and every version is read.