feat: validate end-to-end pack test flow

.github/workflows/cribl-pack-test.yml

@@ -15,10 +15,15 @@ on:
         description: "Pack type — 'edge' or 'stream'."
         type: string
         required: true
-      cribl_version:
-        description: "cribl/cribl Docker image tag. Defaults to 'latest'."
+      cribl_versions:
+        description: >-
+          JSON array of {version, required} objects — one parallel test leg
+          per entry. `required: true` blocks merges on failure; `required:
+          false` is best-effort. Default: `latest` (N) plus the last patches
+          of N-1 and N-2 minors. Bump all three when `latest` rolls over to
+          a new minor.
         type: string
-        default: 'latest'
+        default: '[{"version":"latest","required":true},{"version":"4.17.1","required":false},{"version":"4.16.1","required":false}]'
       node_version:
         description: 'Node.js version for the test runner.'
         type: string
@@ -56,14 +61,22 @@ jobs:
         run: ./scripts/validate-pack-structure.sh
 
   test:
-    name: Test pack pipelines (Vitest)
+    # One leg per entry in `cribl_versions`. Required legs block PR merges;
+    # optional legs run best-effort. `fail-fast: false` so every leg's result
+    # is visible regardless of sibling outcomes.
+    name: Test pack pipelines (Cribl ${{ matrix.cribl.version }})
     runs-on: ubuntu-latest
     needs: validate
+    strategy:
+      fail-fast: false
+      matrix:
+        cribl: ${{ fromJSON(inputs.cribl_versions) }}
+    continue-on-error: ${{ !matrix.cribl.required }}
     env:
       PACK_TYPE: ${{ inputs.pack_type }}
     services:
       cribl:
-        image: cribl/cribl:${{ inputs.cribl_version }}
+        image: cribl/cribl:${{ matrix.cribl.version }}
         ports:
           - 9000:9000
           - 10080:10080

README.md

@@ -15,20 +15,20 @@ that wraps the common day-to-day commands.
 
 ## Installation
 
-Create a new pack repo from this template:
-
 ```sh
-gh repo create dryvist/cc-edge-<source>-io \
-  --template dryvist/cc-edge-pack-template \
-  --public --clone
+GH_TOKEN=<org-admin-token> \
+  gh repo create dryvist/cc-edge-<source>-io \
+    --template dryvist/cc-edge-pack-template \
+    --public --clone
 
 cd cc-edge-<source>-io
 make install            # installs Node deps + git hooks
 ```
 
-`make install` requires `node` (20+) and `pnpm` (10+). Docker is only needed
-when you run `make test`. See [`docs/development.md`](docs/development.md) for
-installation paths.
+Org-repo creation needs a token with `Administration: Read & write` on the
+dryvist org (standard user PATs typically can't). `make install` needs Node
+20+ and pnpm 10+. Docker is only needed for `make test`. See
+[`docs/development.md`](docs/development.md) for the dev shell.
 
 ## After scaffolding
 
@@ -70,7 +70,7 @@ version bump on every push to `main`; merge that PR to publish a release. See
 | Doc | What it covers |
 |---|---|
 | [`docs/development.md`](docs/development.md) | Local dev setup, Make targets, optional Nix shell |
-| [`docs/test-harness.md`](docs/test-harness.md) | What gets tested, fixture conventions |
+| [`docs/test-harness.md`](docs/test-harness.md) | What gets tested, fixture conventions, Cribl version matrix |
 | [`docs/file-boundary.md`](docs/file-boundary.md) | Generic vs pack-specific files (sync rules) |
 | [`docs/release-process.md`](docs/release-process.md) | release-please flow, version bump rules |
 | [`docs/validator-rules.md`](docs/validator-rules.md) | vct-cribl-pack-validator rules + how they're enforced |

biome.jsonc

@@ -1,6 +1,6 @@
 {
   // dryvist canonical Biome config (mirror of dryvist/.github/biome.jsonc).
-  "$schema": "https://biomejs.dev/schemas/2.3.6/schema.json",
+  "$schema": "https://biomejs.dev/schemas/2.4.13/schema.json",
   "vcs": {
     "enabled": true,
     "clientKind": "git",

docs/test-harness.md

@@ -10,10 +10,16 @@ convention — no TypeScript edits needed to add tests.
 | `routes.test.ts` (structure) | route.yml exists, every route has a pipeline, every referenced pipeline file exists, routes use `output: __group`, filters aren't statically falsy, no pipeline named `main` |
 | `routes.test.ts` (dynamic flow) | Per route: a synthetic event matching its filter triggers the named pipeline and isn't dropped (uses live Cribl) |
 | `pipelines.test.ts` | Per fixture: pipeline produces non-empty output; partial-match against `<case>.expected.json` if present; required-fields assertion (`sourcetype`+`index` for Edge; `host`+`source`+`_time` for Stream) unless `.skip-required-fields` marker present |
+| `tarball-parity.test.ts` | The whitelist in `tests/cribl-client.ts::PACK_ROOT_ENTRIES` (used by every test-time pack install) matches `INCLUDE=` in `scripts/build-crbl.sh` (used by every release). Catches drift before a release ships a tarball CI never validated. |
+| `harness-teeth.test.ts` | Meta-tests: every assertion helper used by the suites above actually throws on its target failure mode. Pure unit-level; no Cribl required. |
+
+Adding a new assertion helper? Add a matching case to `harness-teeth.test.ts`
+in the same PR — the `it()` names there are the source of truth for what each
+guard catches.
 
 ## Fixture convention
 
-```
+```text
 tests/fixtures/<pipeline-name>/<case>.json           # input event(s)
 tests/fixtures/<pipeline-name>/<case>.expected.json  # optional partial-match expected output
 tests/fixtures/.skip-required-fields                 # optional org-wide opt-out marker
@@ -23,6 +29,16 @@ The generic `pipelines.test.ts` auto-discovers and parametrizes one Vitest case
 per `<case>.json`. Add a fixture → tests run automatically. Remove a fixture →
 tests stop running. No code changes.
 
+## Cribl version matrix
+
+Default: `latest` plus the last patches of the previous two minors (N / N-1 /
+N-2). `latest` is required; older legs are best-effort. Shape lives in the
+`cribl_versions` input default in `.github/workflows/cribl-pack-test.yml` —
+edit the JSON there to bump, add, or remove versions. Each leg posts its own
+status check (`Test pack pipelines (Cribl <version>)`) so branch protection
+can require `latest` specifically. When `latest` rolls over to a new minor,
+bump both older entries forward one.
+
 ## Required-fields assertion
 
 Edge packs are expected to set `sourcetype` and `index` (typically via an Eval

package.json

@@ -1,7 +1,7 @@
 {
   "name": "cc-edge-demo-io",
   "version": "0.0.1",
-  "minLogStreamVersion": "4.17.0",
+  "minLogStreamVersion": "4.16.0",
   "author": "dryvist <hello@dryvist.example>",
   "description": "Demo passthrough pack — replace name/displayName/description/author when scaffolding from this template.",
   "displayName": "Cribl Edge Demo Pack",

renovate.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": ["github>dryvist/.github"],
+  "customManagers": [
+    {
+      "description": "Track Cribl version pins inside the multi-version test matrix default in .github/workflows/cribl-pack-test.yml. Captures any `\"version\":\"X.Y.Z\"` inside the cribl_versions JSON-array default; ignores `\"version\":\"latest\"` because the regex only matches semver triples. Each match becomes a docker dep tied to cribl/cribl, so Renovate opens a PR when a newer patch ships.",
+      "customType": "regex",
+      "managerFilePatterns": ["/\\.github/workflows/cribl-pack-test\\.yml$/"],
+      "matchStrings": [
+        "\"version\"\\s*:\\s*\"(?<currentValue>\\d+\\.\\d+\\.\\d+)\""
+      ],
+      "datasourceTemplate": "docker",
+      "depNameTemplate": "cribl/cribl"
+    }
+  ],
+  "packageRules": [
+    {
+      "description": "Allow only patch bumps on cribl/cribl pins. The matrix shape is `latest` + last-patch-of-N-1 + last-patch-of-N-2; Renovate keeps each pinned minor on its newest patch. Minor/major shifts (when the N window rolls forward) stay manual — they're a deliberate compat-band decision, not a routine bump.",
+      "matchPackageNames": ["cribl/cribl"],
+      "matchUpdateTypes": ["minor", "major"],
+      "enabled": false
+    },
+    {
+      "description": "Cribl is in the criblio/** trusted-org list per JacobPEvans renovate-presets, so patch bumps auto-merge after the inherited stabilization window.",
+      "matchPackageNames": ["cribl/cribl"],
+      "matchUpdateTypes": ["patch"],
+      "automerge": true
+    }
+  ]
+}

tests/cribl-client.ts

@@ -62,7 +62,10 @@ const REQUIRED_FIELDS: Record<PackType, readonly string[]> = {
 // as repo-level dev tooling (flake.nix, biome.jsonc, etc.) accumulates.
 // node-tar's create() recurses into the directory entries automatically and
 // emits proper directory headers (Cribl rejects tarballs missing them).
-const PACK_ROOT_ENTRIES = new Set([
+//
+// Exported so `tarball-parity.test.ts` can assert it stays in sync with the
+// `INCLUDE=(...)` line in `scripts/build-crbl.sh` (the release-path whitelist).
+export const PACK_ROOT_ENTRIES = new Set([
   "data",
   "default",
   "package.json",
@@ -400,11 +403,11 @@ export class CriblClient {
     for (const route of config.routes ?? []) {
       if (route.disabled === true) continue;
       const parsed = parseSimpleFilter(route.filter);
-      if (parsed === null) {
-        skippedFilters.push(route.filter ?? "");
+      if (parsed.kind === "unsupported") {
+        skippedFilters.push(parsed.expression);
         continue;
       }
-      const [field, expectedValue] = parsed;
+      const { field, value: expectedValue } = parsed;
       const matches = events.some((event) => event[field] === expectedValue);
       if (matches) {
         const output = await this.runPipeline(

tests/harness-teeth.test.ts

@@ -0,0 +1,154 @@
+/**
+ * Harness teeth — meta-tests that prove the assertion helpers used by
+ * pipelines.test.ts actually catch the failure modes they claim to catch.
+ *
+ * GENERIC: copied verbatim from the template into every pack repo.
+ *
+ * These are pure unit tests over the assertion helpers — no Cribl required,
+ * no fixtures consumed. They exist so that a future refactor that silently
+ * breaks `assertPartialMatch` or `assertRequiredFields` (e.g. drops a guard,
+ * narrows a check, swaps a `!==` for `==`) fails CI on every pack that
+ * inherits this harness. Without these, "the tests pass" only tells us the
+ * fixtures we happened to write didn't fail; it tells us nothing about
+ * whether the assertion library can catch a real pipeline regression.
+ *
+ * Each test phrases a known-bad input and asserts the helper throws with a
+ * recognizable diagnostic. If you add a new assertion helper to
+ * `test-helpers.ts` or `cribl-client.ts`, add a teeth test here.
+ */
+
+import { describe, expect, it } from "vitest";
+import { CriblClient, type CriblEvent } from "./cribl-client.js";
+import { parseSimpleFilter } from "./parse-filter.js";
+import { assertPartialMatch } from "./test-helpers.js";
+
+describe("assertPartialMatch (teeth)", () => {
+  it("passes when actual matches expected exactly", () => {
+    expect(() =>
+      assertPartialMatch([{ a: 1, b: "x" }], [{ a: 1, b: "x" }], "ctx"),
+    ).not.toThrow();
+  });
+
+  it("passes when actual has extra keys beyond expected (partial-match contract)", () => {
+    expect(() =>
+      assertPartialMatch(
+        [{ a: 1, b: "x", extra: true }],
+        [{ a: 1, b: "x" }],
+        "ctx",
+      ),
+    ).not.toThrow();
+  });
+
+  it("throws when an expected key is missing from actual", () => {
+    expect(() =>
+      assertPartialMatch([{ a: 1 }], [{ a: 1, b: "x" }], "fixture-a"),
+    ).toThrow(/expected key 'b' missing/);
+  });
+
+  it("throws when an expected value differs from actual", () => {
+    expect(() =>
+      assertPartialMatch([{ a: 1 }], [{ a: 2 }], "fixture-b"),
+    ).toThrow(/key 'a'/);
+  });
+
+  it("throws on event-count mismatch (actual longer)", () => {
+    expect(() =>
+      assertPartialMatch([{ a: 1 }, { a: 2 }], [{ a: 1 }], "fixture-c"),
+    ).toThrow(/produced 2 events, expected 1/);
+  });
+
+  it("throws on event-count mismatch (actual shorter)", () => {
+    expect(() =>
+      assertPartialMatch([{ a: 1 }], [{ a: 1 }, { a: 2 }], "fixture-d"),
+    ).toThrow(/produced 1 events, expected 2/);
+  });
+});
+
+describe("CriblClient.assertRequiredFields (teeth)", () => {
+  it("passes when edge required fields are present", () => {
+    const events: CriblEvent[] = [{ sourcetype: "s", index: "main" }];
+    expect(() =>
+      CriblClient.assertRequiredFields(events, "edge"),
+    ).not.toThrow();
+  });
+
+  it("throws when edge sourcetype is missing", () => {
+    const events: CriblEvent[] = [{ index: "main" }];
+    expect(() => CriblClient.assertRequiredFields(events, "edge")).toThrow(
+      /sourcetype/,
+    );
+  });
+
+  it("throws when edge index is missing", () => {
+    const events: CriblEvent[] = [{ sourcetype: "s" }];
+    expect(() => CriblClient.assertRequiredFields(events, "edge")).toThrow(
+      /index/,
+    );
+  });
+
+  it("passes when stream required fields are present", () => {
+    const events: CriblEvent[] = [{ host: "h", source: "src", _time: 0 }];
+    expect(() =>
+      CriblClient.assertRequiredFields(events, "stream"),
+    ).not.toThrow();
+  });
+
+  it("throws when stream host is missing", () => {
+    const events: CriblEvent[] = [{ source: "src", _time: 0 }];
+    expect(() => CriblClient.assertRequiredFields(events, "stream")).toThrow(
+      /host/,
+    );
+  });
+
+  it("throws when stream _time is missing", () => {
+    const events: CriblEvent[] = [{ host: "h", source: "src" }];
+    expect(() => CriblClient.assertRequiredFields(events, "stream")).toThrow(
+      /_time/,
+    );
+  });
+
+  it("reports the count of violating events", () => {
+    const events: CriblEvent[] = [
+      { sourcetype: "s", index: "main" },
+      { sourcetype: "s" },
+      {},
+    ];
+    expect(() => CriblClient.assertRequiredFields(events, "edge")).toThrow(
+      /2\/3 event\(s\)/,
+    );
+  });
+});
+
+describe("parseSimpleFilter (teeth)", () => {
+  it("classifies canonical equality as simple", () => {
+    const parsed = parseSimpleFilter("datatype=='cribl-demo'");
+    expect(parsed.kind).toBe("simple");
+    if (parsed.kind === "simple") {
+      expect(parsed.field).toBe("datatype");
+      expect(parsed.value).toBe("cribl-demo");
+    }
+  });
+
+  it("classifies double-quoted equality as simple", () => {
+    const parsed = parseSimpleFilter('sourcetype=="json"');
+    expect(parsed.kind).toBe("simple");
+  });
+
+  it("classifies boolean expressions as unsupported (not silently simple)", () => {
+    const parsed = parseSimpleFilter("a=='x' && b=='y'");
+    expect(parsed.kind).toBe("unsupported");
+    if (parsed.kind === "unsupported") {
+      expect(parsed.expression).toBe("a=='x' && b=='y'");
+    }
+  });
+
+  it("classifies function calls as unsupported", () => {
+    const parsed = parseSimpleFilter("includes(_raw, 'error')");
+    expect(parsed.kind).toBe("unsupported");
+  });
+
+  it("classifies null/undefined input as unsupported with empty expression", () => {
+    expect(parseSimpleFilter(null).kind).toBe("unsupported");
+    expect(parseSimpleFilter(undefined).kind).toBe("unsupported");
+  });
+});

tests/parse-filter.ts

@@ -1,18 +1,36 @@
 // Canonical Cribl filter shape we can auto-resolve: `<field>=='<value>'`.
 // Anything more complex (boolean ops, function calls, regex) is out of scope
-// for the local evaluator — callers should it.skip when this returns null.
-const SIMPLE_FILTER_RE = /^\s*([A-Za-z_]\w*)\s*==\s*['"](.*?)['"]\s*$/;
+// for the local evaluator. Callers receive an explicit `{kind: 'unsupported'}`
+// so they can decide whether to fail loudly or fall back — silent skips are
+// banned (they let coverage rot invisibly).
+//
+// The quote group `(['"])` is back-referenced via `\2` to require matching
+// open/close quotes, and the value class `[^'"]*` rejects embedded quotes —
+// without this, `(.*?)` would greedily span boolean expressions like
+// `a=='x' && b=='y'` and silently classify them as simple.
+const SIMPLE_FILTER_RE = /^\s*([A-Za-z_]\w*)\s*==\s*(['"])([^'"]*)\2\s*$/;
+
+export type ParsedFilter =
+  | { kind: "simple"; field: string; value: string }
+  | { kind: "unsupported"; expression: string };
 
 /**
- * Parse `<field>=='<value>'` (or `=="<value>"`) into [field, value].
- * Returns null for any expression we cannot statically resolve.
+ * Classify a Cribl route filter expression.
+ *
+ * Returns `{kind: "simple", field, value}` for `<field>=='<value>'` (or
+ * `=="<value>"`); `{kind: "unsupported", expression}` for anything else.
+ *
+ * Discriminated union (not nullable) so callers MUST handle the unsupported
+ * case explicitly — a missed branch becomes a TypeScript error rather than
+ * an implicit skip.
  */
 export function parseSimpleFilter(
   expr: string | null | undefined,
-): [string, string] | null {
-  const match = (expr ?? "").match(SIMPLE_FILTER_RE);
-  if (!match || match[1] === undefined || match[2] === undefined) {
-    return null;
+): ParsedFilter {
+  const expression = expr ?? "";
+  const match = expression.match(SIMPLE_FILTER_RE);
+  if (!match || match[1] === undefined || match[3] === undefined) {
+    return { kind: "unsupported", expression };
   }
-  return [match[1], match[2]];
+  return { kind: "simple", field: match[1], value: match[3] };
 }