diff --git a/.github/workflows/cribl-pack-test.yml b/.github/workflows/cribl-pack-test.yml index 52e1db9..c236aae 100644 --- a/.github/workflows/cribl-pack-test.yml +++ b/.github/workflows/cribl-pack-test.yml @@ -15,10 +15,15 @@ on: description: "Pack type — 'edge' or 'stream'." type: string required: true - cribl_version: - description: "cribl/cribl Docker image tag. Defaults to 'latest'." + cribl_versions: + description: >- + JSON array of {version, required} objects — one parallel test leg + per entry. `required: true` blocks merges on failure; `required: + false` is best-effort. Default: `latest` (N) plus the last patches + of N-1 and N-2 minors. Bump all three when `latest` rolls over to + a new minor. type: string - default: 'latest' + default: '[{"version":"latest","required":true},{"version":"4.17.1","required":false},{"version":"4.16.1","required":false}]' node_version: description: 'Node.js version for the test runner.' type: string @@ -56,14 +61,22 @@ jobs: run: ./scripts/validate-pack-structure.sh test: - name: Test pack pipelines (Vitest) + # One leg per entry in `cribl_versions`. Required legs block PR merges; + # optional legs run best-effort. `fail-fast: false` so every leg's result + # is visible regardless of sibling outcomes. + name: Test pack pipelines (Cribl ${{ matrix.cribl.version }}) runs-on: ubuntu-latest needs: validate + strategy: + fail-fast: false + matrix: + cribl: ${{ fromJSON(inputs.cribl_versions) }} + continue-on-error: ${{ !matrix.cribl.required }} env: PACK_TYPE: ${{ inputs.pack_type }} services: cribl: - image: cribl/cribl:${{ inputs.cribl_version }} + image: cribl/cribl:${{ matrix.cribl.version }} ports: - 9000:9000 - 10080:10080 diff --git a/README.md b/README.md index 3bf3fef..0d04fa8 100644 --- a/README.md +++ b/README.md @@ -15,20 +15,20 @@ that wraps the common day-to-day commands. ## Installation -Create a new pack repo from this template: - ```sh -gh repo create dryvist/cc-edge--io \ - --template dryvist/cc-edge-pack-template \ - --public --clone +GH_TOKEN= \ + gh repo create dryvist/cc-edge--io \ + --template dryvist/cc-edge-pack-template \ + --public --clone cd cc-edge--io make install # installs Node deps + git hooks ``` -`make install` requires `node` (20+) and `pnpm` (10+). Docker is only needed -when you run `make test`. See [`docs/development.md`](docs/development.md) for -installation paths. +Org-repo creation needs a token with `Administration: Read & write` on the +dryvist org (standard user PATs typically can't). `make install` needs Node +20+ and pnpm 10+. Docker is only needed for `make test`. See +[`docs/development.md`](docs/development.md) for the dev shell. ## After scaffolding @@ -70,7 +70,7 @@ version bump on every push to `main`; merge that PR to publish a release. See | Doc | What it covers | |---|---| | [`docs/development.md`](docs/development.md) | Local dev setup, Make targets, optional Nix shell | -| [`docs/test-harness.md`](docs/test-harness.md) | What gets tested, fixture conventions | +| [`docs/test-harness.md`](docs/test-harness.md) | What gets tested, fixture conventions, Cribl version matrix | | [`docs/file-boundary.md`](docs/file-boundary.md) | Generic vs pack-specific files (sync rules) | | [`docs/release-process.md`](docs/release-process.md) | release-please flow, version bump rules | | [`docs/validator-rules.md`](docs/validator-rules.md) | vct-cribl-pack-validator rules + how they're enforced | diff --git a/biome.jsonc b/biome.jsonc index 26f652a..9a65237 100644 --- a/biome.jsonc +++ b/biome.jsonc @@ -1,6 +1,6 @@ { // dryvist canonical Biome config (mirror of dryvist/.github/biome.jsonc). - "$schema": "https://biomejs.dev/schemas/2.3.6/schema.json", + "$schema": "https://biomejs.dev/schemas/2.4.13/schema.json", "vcs": { "enabled": true, "clientKind": "git", diff --git a/docs/test-harness.md b/docs/test-harness.md index 758c191..b0a97a4 100644 --- a/docs/test-harness.md +++ b/docs/test-harness.md @@ -10,10 +10,16 @@ convention — no TypeScript edits needed to add tests. | `routes.test.ts` (structure) | route.yml exists, every route has a pipeline, every referenced pipeline file exists, routes use `output: __group`, filters aren't statically falsy, no pipeline named `main` | | `routes.test.ts` (dynamic flow) | Per route: a synthetic event matching its filter triggers the named pipeline and isn't dropped (uses live Cribl) | | `pipelines.test.ts` | Per fixture: pipeline produces non-empty output; partial-match against `.expected.json` if present; required-fields assertion (`sourcetype`+`index` for Edge; `host`+`source`+`_time` for Stream) unless `.skip-required-fields` marker present | +| `tarball-parity.test.ts` | The whitelist in `tests/cribl-client.ts::PACK_ROOT_ENTRIES` (used by every test-time pack install) matches `INCLUDE=` in `scripts/build-crbl.sh` (used by every release). Catches drift before a release ships a tarball CI never validated. | +| `harness-teeth.test.ts` | Meta-tests: every assertion helper used by the suites above actually throws on its target failure mode. Pure unit-level; no Cribl required. | + +Adding a new assertion helper? Add a matching case to `harness-teeth.test.ts` +in the same PR — the `it()` names there are the source of truth for what each +guard catches. ## Fixture convention -``` +```text tests/fixtures//.json # input event(s) tests/fixtures//.expected.json # optional partial-match expected output tests/fixtures/.skip-required-fields # optional org-wide opt-out marker @@ -23,6 +29,16 @@ The generic `pipelines.test.ts` auto-discovers and parametrizes one Vitest case per `.json`. Add a fixture → tests run automatically. Remove a fixture → tests stop running. No code changes. +## Cribl version matrix + +Default: `latest` plus the last patches of the previous two minors (N / N-1 / +N-2). `latest` is required; older legs are best-effort. Shape lives in the +`cribl_versions` input default in `.github/workflows/cribl-pack-test.yml` — +edit the JSON there to bump, add, or remove versions. Each leg posts its own +status check (`Test pack pipelines (Cribl )`) so branch protection +can require `latest` specifically. When `latest` rolls over to a new minor, +bump both older entries forward one. + ## Required-fields assertion Edge packs are expected to set `sourcetype` and `index` (typically via an Eval diff --git a/package.json b/package.json index e86ba3d..18aca15 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cc-edge-demo-io", "version": "0.0.1", - "minLogStreamVersion": "4.17.0", + "minLogStreamVersion": "4.16.0", "author": "dryvist ", "description": "Demo passthrough pack — replace name/displayName/description/author when scaffolding from this template.", "displayName": "Cribl Edge Demo Pack", diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..13ca454 --- /dev/null +++ b/renovate.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["github>dryvist/.github"], + "customManagers": [ + { + "description": "Track Cribl version pins inside the multi-version test matrix default in .github/workflows/cribl-pack-test.yml. Captures any `\"version\":\"X.Y.Z\"` inside the cribl_versions JSON-array default; ignores `\"version\":\"latest\"` because the regex only matches semver triples. Each match becomes a docker dep tied to cribl/cribl, so Renovate opens a PR when a newer patch ships.", + "customType": "regex", + "managerFilePatterns": ["/\\.github/workflows/cribl-pack-test\\.yml$/"], + "matchStrings": [ + "\"version\"\\s*:\\s*\"(?\\d+\\.\\d+\\.\\d+)\"" + ], + "datasourceTemplate": "docker", + "depNameTemplate": "cribl/cribl" + } + ], + "packageRules": [ + { + "description": "Allow only patch bumps on cribl/cribl pins. The matrix shape is `latest` + last-patch-of-N-1 + last-patch-of-N-2; Renovate keeps each pinned minor on its newest patch. Minor/major shifts (when the N window rolls forward) stay manual — they're a deliberate compat-band decision, not a routine bump.", + "matchPackageNames": ["cribl/cribl"], + "matchUpdateTypes": ["minor", "major"], + "enabled": false + }, + { + "description": "Cribl is in the criblio/** trusted-org list per JacobPEvans renovate-presets, so patch bumps auto-merge after the inherited stabilization window.", + "matchPackageNames": ["cribl/cribl"], + "matchUpdateTypes": ["patch"], + "automerge": true + } + ] +} diff --git a/tests/cribl-client.ts b/tests/cribl-client.ts index 7f71751..4be3fbc 100644 --- a/tests/cribl-client.ts +++ b/tests/cribl-client.ts @@ -62,7 +62,10 @@ const REQUIRED_FIELDS: Record = { // as repo-level dev tooling (flake.nix, biome.jsonc, etc.) accumulates. // node-tar's create() recurses into the directory entries automatically and // emits proper directory headers (Cribl rejects tarballs missing them). -const PACK_ROOT_ENTRIES = new Set([ +// +// Exported so `tarball-parity.test.ts` can assert it stays in sync with the +// `INCLUDE=(...)` line in `scripts/build-crbl.sh` (the release-path whitelist). +export const PACK_ROOT_ENTRIES = new Set([ "data", "default", "package.json", @@ -400,11 +403,11 @@ export class CriblClient { for (const route of config.routes ?? []) { if (route.disabled === true) continue; const parsed = parseSimpleFilter(route.filter); - if (parsed === null) { - skippedFilters.push(route.filter ?? ""); + if (parsed.kind === "unsupported") { + skippedFilters.push(parsed.expression); continue; } - const [field, expectedValue] = parsed; + const { field, value: expectedValue } = parsed; const matches = events.some((event) => event[field] === expectedValue); if (matches) { const output = await this.runPipeline( diff --git a/tests/harness-teeth.test.ts b/tests/harness-teeth.test.ts new file mode 100644 index 0000000..6a86b59 --- /dev/null +++ b/tests/harness-teeth.test.ts @@ -0,0 +1,154 @@ +/** + * Harness teeth — meta-tests that prove the assertion helpers used by + * pipelines.test.ts actually catch the failure modes they claim to catch. + * + * GENERIC: copied verbatim from the template into every pack repo. + * + * These are pure unit tests over the assertion helpers — no Cribl required, + * no fixtures consumed. They exist so that a future refactor that silently + * breaks `assertPartialMatch` or `assertRequiredFields` (e.g. drops a guard, + * narrows a check, swaps a `!==` for `==`) fails CI on every pack that + * inherits this harness. Without these, "the tests pass" only tells us the + * fixtures we happened to write didn't fail; it tells us nothing about + * whether the assertion library can catch a real pipeline regression. + * + * Each test phrases a known-bad input and asserts the helper throws with a + * recognizable diagnostic. If you add a new assertion helper to + * `test-helpers.ts` or `cribl-client.ts`, add a teeth test here. + */ + +import { describe, expect, it } from "vitest"; +import { CriblClient, type CriblEvent } from "./cribl-client.js"; +import { parseSimpleFilter } from "./parse-filter.js"; +import { assertPartialMatch } from "./test-helpers.js"; + +describe("assertPartialMatch (teeth)", () => { + it("passes when actual matches expected exactly", () => { + expect(() => + assertPartialMatch([{ a: 1, b: "x" }], [{ a: 1, b: "x" }], "ctx"), + ).not.toThrow(); + }); + + it("passes when actual has extra keys beyond expected (partial-match contract)", () => { + expect(() => + assertPartialMatch( + [{ a: 1, b: "x", extra: true }], + [{ a: 1, b: "x" }], + "ctx", + ), + ).not.toThrow(); + }); + + it("throws when an expected key is missing from actual", () => { + expect(() => + assertPartialMatch([{ a: 1 }], [{ a: 1, b: "x" }], "fixture-a"), + ).toThrow(/expected key 'b' missing/); + }); + + it("throws when an expected value differs from actual", () => { + expect(() => + assertPartialMatch([{ a: 1 }], [{ a: 2 }], "fixture-b"), + ).toThrow(/key 'a'/); + }); + + it("throws on event-count mismatch (actual longer)", () => { + expect(() => + assertPartialMatch([{ a: 1 }, { a: 2 }], [{ a: 1 }], "fixture-c"), + ).toThrow(/produced 2 events, expected 1/); + }); + + it("throws on event-count mismatch (actual shorter)", () => { + expect(() => + assertPartialMatch([{ a: 1 }], [{ a: 1 }, { a: 2 }], "fixture-d"), + ).toThrow(/produced 1 events, expected 2/); + }); +}); + +describe("CriblClient.assertRequiredFields (teeth)", () => { + it("passes when edge required fields are present", () => { + const events: CriblEvent[] = [{ sourcetype: "s", index: "main" }]; + expect(() => + CriblClient.assertRequiredFields(events, "edge"), + ).not.toThrow(); + }); + + it("throws when edge sourcetype is missing", () => { + const events: CriblEvent[] = [{ index: "main" }]; + expect(() => CriblClient.assertRequiredFields(events, "edge")).toThrow( + /sourcetype/, + ); + }); + + it("throws when edge index is missing", () => { + const events: CriblEvent[] = [{ sourcetype: "s" }]; + expect(() => CriblClient.assertRequiredFields(events, "edge")).toThrow( + /index/, + ); + }); + + it("passes when stream required fields are present", () => { + const events: CriblEvent[] = [{ host: "h", source: "src", _time: 0 }]; + expect(() => + CriblClient.assertRequiredFields(events, "stream"), + ).not.toThrow(); + }); + + it("throws when stream host is missing", () => { + const events: CriblEvent[] = [{ source: "src", _time: 0 }]; + expect(() => CriblClient.assertRequiredFields(events, "stream")).toThrow( + /host/, + ); + }); + + it("throws when stream _time is missing", () => { + const events: CriblEvent[] = [{ host: "h", source: "src" }]; + expect(() => CriblClient.assertRequiredFields(events, "stream")).toThrow( + /_time/, + ); + }); + + it("reports the count of violating events", () => { + const events: CriblEvent[] = [ + { sourcetype: "s", index: "main" }, + { sourcetype: "s" }, + {}, + ]; + expect(() => CriblClient.assertRequiredFields(events, "edge")).toThrow( + /2\/3 event\(s\)/, + ); + }); +}); + +describe("parseSimpleFilter (teeth)", () => { + it("classifies canonical equality as simple", () => { + const parsed = parseSimpleFilter("datatype=='cribl-demo'"); + expect(parsed.kind).toBe("simple"); + if (parsed.kind === "simple") { + expect(parsed.field).toBe("datatype"); + expect(parsed.value).toBe("cribl-demo"); + } + }); + + it("classifies double-quoted equality as simple", () => { + const parsed = parseSimpleFilter('sourcetype=="json"'); + expect(parsed.kind).toBe("simple"); + }); + + it("classifies boolean expressions as unsupported (not silently simple)", () => { + const parsed = parseSimpleFilter("a=='x' && b=='y'"); + expect(parsed.kind).toBe("unsupported"); + if (parsed.kind === "unsupported") { + expect(parsed.expression).toBe("a=='x' && b=='y'"); + } + }); + + it("classifies function calls as unsupported", () => { + const parsed = parseSimpleFilter("includes(_raw, 'error')"); + expect(parsed.kind).toBe("unsupported"); + }); + + it("classifies null/undefined input as unsupported with empty expression", () => { + expect(parseSimpleFilter(null).kind).toBe("unsupported"); + expect(parseSimpleFilter(undefined).kind).toBe("unsupported"); + }); +}); diff --git a/tests/parse-filter.ts b/tests/parse-filter.ts index 73c872a..274c842 100644 --- a/tests/parse-filter.ts +++ b/tests/parse-filter.ts @@ -1,18 +1,36 @@ // Canonical Cribl filter shape we can auto-resolve: `==''`. // Anything more complex (boolean ops, function calls, regex) is out of scope -// for the local evaluator — callers should it.skip when this returns null. -const SIMPLE_FILTER_RE = /^\s*([A-Za-z_]\w*)\s*==\s*['"](.*?)['"]\s*$/; +// for the local evaluator. Callers receive an explicit `{kind: 'unsupported'}` +// so they can decide whether to fail loudly or fall back — silent skips are +// banned (they let coverage rot invisibly). +// +// The quote group `(['"])` is back-referenced via `\2` to require matching +// open/close quotes, and the value class `[^'"]*` rejects embedded quotes — +// without this, `(.*?)` would greedily span boolean expressions like +// `a=='x' && b=='y'` and silently classify them as simple. +const SIMPLE_FILTER_RE = /^\s*([A-Za-z_]\w*)\s*==\s*(['"])([^'"]*)\2\s*$/; + +export type ParsedFilter = + | { kind: "simple"; field: string; value: string } + | { kind: "unsupported"; expression: string }; /** - * Parse `==''` (or `==""`) into [field, value]. - * Returns null for any expression we cannot statically resolve. + * Classify a Cribl route filter expression. + * + * Returns `{kind: "simple", field, value}` for `==''` (or + * `==""`); `{kind: "unsupported", expression}` for anything else. + * + * Discriminated union (not nullable) so callers MUST handle the unsupported + * case explicitly — a missed branch becomes a TypeScript error rather than + * an implicit skip. */ export function parseSimpleFilter( expr: string | null | undefined, -): [string, string] | null { - const match = (expr ?? "").match(SIMPLE_FILTER_RE); - if (!match || match[1] === undefined || match[2] === undefined) { - return null; +): ParsedFilter { + const expression = expr ?? ""; + const match = expression.match(SIMPLE_FILTER_RE); + if (!match || match[1] === undefined || match[3] === undefined) { + return { kind: "unsupported", expression }; } - return [match[1], match[2]]; + return { kind: "simple", field: match[1], value: match[3] }; } diff --git a/tests/pipelines.test.ts b/tests/pipelines.test.ts index e3c9aad..a0900bc 100644 --- a/tests/pipelines.test.ts +++ b/tests/pipelines.test.ts @@ -24,7 +24,11 @@ import { existsSync, readdirSync, readFileSync } from "node:fs"; import * as path from "node:path"; import { describe, expect, it } from "vitest"; import { CriblClient, type CriblEvent } from "./cribl-client.js"; -import { getInstalledPackId, makeClient } from "./test-helpers.js"; +import { + assertPartialMatch, + getInstalledPackId, + makeClient, +} from "./test-helpers.js"; const FIXTURES = path.join(import.meta.dirname, "fixtures"); const SKIP_REQUIRED_FIELDS_MARKER = path.join( @@ -68,32 +72,6 @@ function loadEvents(filePath: string): CriblEvent[] { return Array.isArray(raw) ? (raw as CriblEvent[]) : [raw as CriblEvent]; } -function assertPartialMatch( - actual: CriblEvent[], - expected: CriblEvent[], - context: string, -): void { - expect( - actual.length, - `${context}: pipeline produced ${actual.length} events, expected ${expected.length}`, - ).toBe(expected.length); - for (let i = 0; i < expected.length; i++) { - const act = actual[i]; - const exp = expected[i]; - if (act === undefined || exp === undefined) continue; - for (const [key, expVal] of Object.entries(exp)) { - expect( - key in act, - `${context} event ${i}: expected key '${key}' missing from output`, - ).toBe(true); - expect( - act[key], - `${context} event ${i}, key '${key}': expected ${JSON.stringify(expVal)}, got ${JSON.stringify(act[key])}`, - ).toEqual(expVal); - } - } -} - const CASES = discoverCases(); const SKIP_REQUIRED_FIELDS = existsSync(SKIP_REQUIRED_FIELDS_MARKER); diff --git a/tests/routes.test.ts b/tests/routes.test.ts index 77ccd33..d40c02a 100644 --- a/tests/routes.test.ts +++ b/tests/routes.test.ts @@ -116,43 +116,47 @@ describe("route flow (dynamic)", () => { } for (const route of routesForFlow) { + const routeId = route.id ?? ""; const parsed = parseSimpleFilter(route.filter); - const skipReason = - parsed === null - ? `filter '${route.filter}' not auto-resolvable by the local matcher (expected \`==''\`)` - : null; - - it.skipIf(skipReason !== null)( - `${route.id ?? ""} routes a synthetic event to its pipeline${skipReason !== null ? ` [skip: ${skipReason}]` : ""}`, - async () => { - // parsed is guaranteed non-null here (test would be skipped otherwise), - // but TS doesn't know — re-derive in-test for type narrowing. - const reparsed = parseSimpleFilter(route.filter); - if (reparsed === null) throw new Error("unreachable: skipped above"); - const [field, value] = reparsed; - const event = { [field]: value, _raw: "{}", _time: Date.now() / 1000 }; - - const client = makeClient(); - const packId = getInstalledPackId(); - const sampleId = await client.saveSample(`route-flow-${route.id}`, [ - event, - ]); - try { - const result = await client.runRouteFlow(sampleId, [event], { - pack: packId, - }); - expect( - result.route.id, - `Expected route '${route.id}' to match the synthetic event, but route '${result.route.id}' matched first.`, - ).toBe(route.id); - expect( - result.events.length, - `Pipeline '${result.pipeline}' produced no output for the synthetic event matching route '${route.id}' — check the pipeline for unconditional Drop functions.`, - ).toBeGreaterThan(0); - } finally { - await client.deleteSample(sampleId); - } - }, - ); + + if (parsed.kind === "unsupported") { + // Loud failure (not silent skip) — the harness can't auto-resolve this + // filter, so the dynamic flow assertion would otherwise erode invisibly + // as packs ship more complex filters. Author must either simplify the + // route filter to `==''` or extend tests/parse-filter.ts + // to handle the new form. + it(`${routeId} route filter is auto-resolvable by the local matcher`, () => { + expect.fail( + `Route '${routeId}' filter '${parsed.expression}' is not the canonical \`==''\` shape. The dynamic-flow test cannot exercise it. Fix by (a) simplifying the route filter to the canonical shape, or (b) extending tests/parse-filter.ts to handle this form (and adding a teeth-test for the new shape).`, + ); + }); + continue; + } + + it(`${routeId} routes a synthetic event to its pipeline`, async () => { + const { field, value } = parsed; + const event = { [field]: value, _raw: "{}", _time: Date.now() / 1000 }; + + const client = makeClient(); + const packId = getInstalledPackId(); + const sampleId = await client.saveSample(`route-flow-${route.id}`, [ + event, + ]); + try { + const result = await client.runRouteFlow(sampleId, [event], { + pack: packId, + }); + expect( + result.route.id, + `Expected route '${route.id}' to match the synthetic event, but route '${result.route.id}' matched first.`, + ).toBe(route.id); + expect( + result.events.length, + `Pipeline '${result.pipeline}' produced no output for the synthetic event matching route '${route.id}' — check the pipeline for unconditional Drop functions.`, + ).toBeGreaterThan(0); + } finally { + await client.deleteSample(sampleId); + } + }); } }); diff --git a/tests/tarball-parity.test.ts b/tests/tarball-parity.test.ts new file mode 100644 index 0000000..4918f34 --- /dev/null +++ b/tests/tarball-parity.test.ts @@ -0,0 +1,135 @@ +/** + * Tarball parity — the test-path tarball builder and the release-path tarball + * builder must include the same top-level entries. + * + * GENERIC: copied verbatim from the template into every pack repo. + * + * Two parallel implementations of the .crbl whitelist live in this repo: + * - `tests/cribl-client.ts::PACK_ROOT_ENTRIES` (used in CI test job) + * - `scripts/build-crbl.sh::INCLUDE=(...)` (used in release workflow) + * + * If they drift, CI passes against one whitelist and ships a tarball with a + * different whitelist — exactly the failure mode "fixed in commit 452f029 + * (top-level whitelist for createPackTarball)" was meant to prevent. This + * test pins them together: it builds the tarball both ways from the live + * pack root and asserts the set of entries matches. + * + * No Cribl required. Runs as a plain unit test. + */ + +import { execFileSync } from "node:child_process"; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import * as path from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { CriblClient, PACK_ROOT, PACK_ROOT_ENTRIES } from "./cribl-client.js"; + +const BUILD_SCRIPT = path.join(PACK_ROOT, "scripts", "build-crbl.sh"); +const REPO_NAME = "tarball-parity-test"; +const TAG_NAME = "v0.0.0-parity"; +const RELEASE_TARBALL = `/tmp/${REPO_NAME}-${TAG_NAME}.crbl`; +const RELEASE_TARBALL_LATEST = `/tmp/${REPO_NAME}.crbl`; + +let tmpDir: string; +let testTarballPath: string; + +beforeAll(async () => { + tmpDir = mkdtempSync(path.join(tmpdir(), "tarball-parity-")); + + // Build the release-path tarball. + execFileSync(BUILD_SCRIPT, { + cwd: PACK_ROOT, + env: { ...process.env, REPO_NAME, TAG_NAME }, + stdio: "pipe", + }); + + // Build the test-path tarball (in-memory) and write it next to the release + // tarball so we can extract both the same way. + const buffer = await CriblClient.createPackTarball(PACK_ROOT); + testTarballPath = path.join(tmpDir, "test-path.crbl"); + writeFileSync(testTarballPath, buffer); +}); + +afterAll(() => { + for (const p of [tmpDir, RELEASE_TARBALL, RELEASE_TARBALL_LATEST]) { + if (existsSync(p)) rmSync(p, { recursive: true, force: true }); + } +}); + +function listTopLevelEntries(tarball: string): string[] { + // `tar -tzf` prints every entry; keep only the first path segment. + const out = execFileSync("tar", ["-tzf", tarball], { encoding: "utf8" }); + const top = new Set(); + for (const line of out.split("\n")) { + const trimmed = line.trim(); + if (trimmed.length === 0) continue; + const segment = trimmed.replace(/\/.*$/, "").replace(/\/$/, ""); + if (segment.length > 0) top.add(segment); + } + return Array.from(top).sort(); +} + +function parseIncludeFromShellScript(): string[] { + const script = readFileSync(BUILD_SCRIPT, "utf8"); + const match = script.match(/^INCLUDE=\(([^)]+)\)/m); + if (match === null || match[1] === undefined) { + throw new Error( + `Could not parse INCLUDE=(...) from ${BUILD_SCRIPT} — has the script format changed?`, + ); + } + return match[1] + .split(/\s+/) + .map((s) => s.trim()) + .filter((s) => s.length > 0) + .sort(); +} + +describe("tarball builder parity", () => { + it("release-path and test-path tarballs contain the same top-level entries", () => { + const releaseEntries = listTopLevelEntries(RELEASE_TARBALL); + const testEntries = listTopLevelEntries(testTarballPath); + expect(testEntries).toEqual(releaseEntries); + }); + + it("shell INCLUDE=(...) is a subset of PACK_ROOT_ENTRIES", () => { + // The shell whitelist conditionally adds LICENSE inside an `if` block, so + // we compare the static INCLUDE list against PACK_ROOT_ENTRIES rather than + // requiring exact equality. Every name in the shell list must appear in + // the TS set — drift in either direction fails. + const shellIncludes = parseIncludeFromShellScript(); + const tsWhitelist = Array.from(PACK_ROOT_ENTRIES).sort(); + for (const name of shellIncludes) { + expect( + tsWhitelist, + `shell INCLUDE=(...) contains '${name}' but PACK_ROOT_ENTRIES does not — drift detected`, + ).toContain(name); + } + }); + + it("PACK_ROOT_ENTRIES covers every conditional entry in build-crbl.sh", () => { + // Catches the inverse drift: build-crbl.sh's conditional `LICENSE` (or any + // future conditional addition) must be allow-listed in PACK_ROOT_ENTRIES. + const script = readFileSync(BUILD_SCRIPT, "utf8"); + const conditionalAdds = Array.from( + script.matchAll(/INCLUDE\+=\(([^)]+)\)/g), + (m) => (m[1] ?? "").trim(), + ); + for (const raw of conditionalAdds) { + // Skip the ADDITIONAL_FILES loop iteration — it expands a shell variable, + // not a baked-in conditional path. + if (/\$\{extra\}/.test(raw)) continue; + for (const name of raw.split(/\s+/).filter((s) => s.length > 0)) { + expect( + Array.from(PACK_ROOT_ENTRIES), + `build-crbl.sh conditionally adds '${name}' but PACK_ROOT_ENTRIES does not list it — drift detected`, + ).toContain(name); + } + } + }); +}); diff --git a/tests/test-helpers.ts b/tests/test-helpers.ts index a7701f5..0489fd6 100644 --- a/tests/test-helpers.ts +++ b/tests/test-helpers.ts @@ -1,13 +1,18 @@ /** - * Per-test-file Cribl client helper. + * Per-test-file Cribl client helper + shared assertion utilities. * * Vitest's globalSetup runs in a separate process, so we can't share a token * — each test file constructs its own CriblClient that re-authenticates on * first use. The pack itself is already installed (from globalSetup); the * pack id flows via env var. + * + * `assertPartialMatch` lives here (not inside pipelines.test.ts) so the + * harness-teeth meta-tests can exercise it in isolation without spinning up + * Cribl. */ -import { CriblClient } from "./cribl-client.js"; +import { expect } from "vitest"; +import { CriblClient, type CriblEvent } from "./cribl-client.js"; export function makeClient(): CriblClient { return new CriblClient({ @@ -30,3 +35,37 @@ export function getInstalledPackId(): string { } return id; } + +/** + * Partial-match assertion: every key in each `expected[i]` must be present in + * `actual[i]` with an equal value. Extra keys in `actual` are allowed. + * + * Throws (via vitest's `expect`) with a context-prefixed message on mismatch + * — the message names the offending event index and key so failures point + * straight at the pipeline output that diverged. + */ +export function assertPartialMatch( + actual: CriblEvent[], + expected: CriblEvent[], + context: string, +): void { + expect( + actual.length, + `${context}: pipeline produced ${actual.length} events, expected ${expected.length}`, + ).toBe(expected.length); + for (let i = 0; i < expected.length; i++) { + const act = actual[i]; + const exp = expected[i]; + if (act === undefined || exp === undefined) continue; + for (const [key, expVal] of Object.entries(exp)) { + expect( + key in act, + `${context} event ${i}: expected key '${key}' missing from output`, + ).toBe(true); + expect( + act[key], + `${context} event ${i}, key '${key}': expected ${JSON.stringify(expVal)}, got ${JSON.stringify(act[key])}`, + ).toEqual(expVal); + } + } +}