feat(core/reviews): parse kind:38000, materialize aggregates, rank by bayesian damping

packages/core/src/cache/schema.test.ts

@@ -1,3 +1,4 @@
+import Dexie from "dexie";
 import { afterEach, describe, expect, it } from "vitest";
 import { BitcoinmintsDB } from "./schema";
 
@@ -19,14 +20,16 @@ afterEach(async () => {
 });
 
 describe("BitcoinmintsDB schema", () => {
-  it("opens at version 2 with all 6 tables present", async () => {
+  it("opens at version 3 with all 6 tables present", async () => {
     const db = new BitcoinmintsDB(freshName());
     toDispose.push(db);
     await db.open();
 
-    // v2 adds the [kind+createdAt] compound index to announcements (used by
-    // restoreWatermarks for bounded .last() lookups per kind).
-    expect(db.verno).toBe(2);
+    // v3 renames mintAggregate's `bayesianRank` index → `bayesianScore` and
+    // adds `avgRating` so the ranking aggregator can sort by either without
+    // a full-table scan. v2 added the [kind+createdAt] compound index on
+    // announcements (used by scheduler.restoreWatermarks).
+    expect(db.verno).toBe(3);
     const names = db.tables.map((t) => t.name).sort();
     expect(names).toEqual(
       ["announcements", "mintAggregate", "mintInfo", "profiles", "relayLists", "reviews"].sort(),
@@ -74,8 +77,9 @@ describe("BitcoinmintsDB schema", () => {
     expect(indexNames("reviews")).toEqual(["createdAt", "d", "eventId", "k"]);
     // mintInfo secondary: fetchedAt, ok
     expect(indexNames("mintInfo")).toEqual(["fetchedAt", "ok"]);
-    // mintAggregate secondary: bayesianRank, updatedAt
-    expect(indexNames("mintAggregate")).toEqual(["bayesianRank", "updatedAt"]);
+    // mintAggregate secondary (v3): bayesianScore + avgRating (new) +
+    // updatedAt. `bayesianRank` from v1 is dropped in v3.
+    expect(indexNames("mintAggregate")).toEqual(["avgRating", "bayesianScore", "updatedAt"]);
   });
 
   it("starts empty", async () => {
@@ -90,4 +94,53 @@ describe("BitcoinmintsDB schema", () => {
     expect(await db.mintInfo.count()).toBe(0);
     expect(await db.mintAggregate.count()).toBe(0);
   });
+
+  it("v2 → v3 upgrade clears the mintAggregate table", async () => {
+    // A dev who opened the app at v2 has rows shaped
+    // `{d, averageRating, bayesianRank, updatedAt}` — the `averageRating`
+    // field renamed to `avgRating` in v3 and `bayesianRank` was dropped,
+    // so without a migration hook those rows fail every v3 query shape
+    // (the indexes point at fields the row doesn't have). The v3 upgrade
+    // wipes the table and lets it repopulate from live review traffic.
+    const name = freshName();
+    // Open a separate Dexie handle declaring only the first two schema
+    // versions so we can seed a v2-shape row before the BitcoinmintsDB
+    // handle (which declares v3 and its upgrade hook) ever touches it.
+    const v2 = new Dexie(name);
+    v2.version(1).stores({
+      announcements: "[pubkey+kind+d], eventId, kind, d, createdAt",
+      reviews: "[pubkey+kind+d], eventId, d, createdAt, k",
+      profiles: "pubkey, createdAt",
+      relayLists: "pubkey, createdAt",
+      mintInfo: "d, fetchedAt, ok",
+      mintAggregate: "d, bayesianRank, updatedAt",
+    });
+    v2.version(2).stores({
+      announcements: "[pubkey+kind+d], eventId, kind, d, createdAt, [kind+createdAt]",
+    });
+    await v2.open();
+    // Seed a v2-shape row — the pre-rename payload.
+    await v2.table("mintAggregate").put({
+      d: "5fe928ae0970844f3c5253d2e85a88788486edcbd96c070334a4a2d0d0154a77",
+      averageRating: 4,
+      bayesianRank: 4 * Math.log10(11),
+      updatedAt: 1_700_000_000,
+    });
+    expect(await v2.table("mintAggregate").count()).toBe(1);
+    v2.close();
+
+    // Reopen via BitcoinmintsDB (declares v3 + upgrade hook) — the
+    // upgrade callback should clear the mintAggregate table.
+    const v3 = new BitcoinmintsDB(name);
+    toDispose.push(v3);
+    await v3.open();
+    expect(v3.verno).toBe(3);
+    expect(await v3.mintAggregate.count()).toBe(0);
+    // And the v3 indexes are queryable — a live review upsert would
+    // repopulate via these.
+    const byScore = await v3.mintAggregate.orderBy("bayesianScore").toArray();
+    expect(byScore).toEqual([]);
+    const byAvg = await v3.mintAggregate.orderBy("avgRating").toArray();
+    expect(byAvg).toEqual([]);
+  });
 });

packages/core/src/cache/schema.ts

@@ -59,10 +59,23 @@ export type ReviewRow = {
   d: string;
   eventId: string;
   createdAt: number;
-  /** Pointer-kind tag: 38172 (Cashu) or 38173 (Fedimint). */
-  k?: number;
-  /** Parsed 0..5 rating. */
-  rating?: number;
+  /**
+   * Pointer-kind tag: 38172 (Cashu) or 38173 (Fedimint). Optional because
+   * in-the-wild events sometimes omit the `k` tag entirely; keep lenient.
+   */
+  k?: 38172 | 38173;
+  /**
+   * Mint URL(s) from optional `u` tags on the recommendation — display-only
+   * helper, does NOT participate in replaceable-event keying.
+   */
+  u?: string[];
+  /**
+   * Parsed rating in 1..5 inclusive, `null` when no rating could be extracted
+   * from tags or content. Explicit `null` rather than `undefined` to
+   * distinguish "no rating present" (which is a valid review state) from
+   * "field not yet populated".
+   */
+  rating: number | null;
   /** Freeform review text. */
   content: string;
   rawTags: string[][];
@@ -111,12 +124,39 @@ export type MintInfoRow = {
   lastError?: string;
 };
 
-/** Aggregated per-mint ranking row (populated in PR #5 — empty in PR #3). */
+/**
+ * Aggregated per-mint ranking row (populated in PR #5).
+ *
+ * Materialized from all `reviews` rows with a given `d`. Recomputed in the
+ * same Dexie transaction as the triggering review upsert so they never go
+ * out of sync. `bayesianScore` is the sort key — it damps low-count
+ * averages so a single 5★ review cannot outrank 4★×10 (see data-model-v1.md
+ * §13).
+ *
+ * Schema transition: v2 exposed `averageRating` + `bayesianRank` as
+ * optional-number fields; v3 tightens the contract so every row carries
+ * explicit values (`avgRating: number | null`, `bayesianScore: number`) and
+ * adds `ratedCount` — the count of reviews contributing to `avgRating`,
+ * distinct from `reviewCount` which counts all reviews including unrated.
+ * The index rename from `bayesianRank` → `bayesianScore` drives the v3 bump.
+ */
 export type MintAggregateRow = {
+  /** Primary key — the mint d-tag this aggregate is for. */
   d: string;
+  /** Count of ALL reviews for this mint (rated + unrated). */
   reviewCount: number;
-  averageRating?: number;
-  bayesianRank?: number;
+  /** Count of reviews with `rating != null` — the divisor of `avgRating`. */
+  ratedCount: number;
+  /** Mean across the `ratedCount` reviews, or `null` when `ratedCount===0`. */
+  avgRating: number | null;
+  /**
+   * Bayesian sort score — `avgRating * log10(ratedCount + 1)` when
+   * `avgRating != null`, else `0`. `log10(1)=0` so a single review gets
+   * `rating * log10(2) ≈ rating * 0.301`; 10 reviews get `rating * log10(11)
+   * ≈ rating * 1.041`. The damping makes low-count mints sort below
+   * higher-count mints of the same average.
+   */
+  bayesianScore: number;
   /** Epoch-ms timestamp of the aggregate recompute (used for CAS). */
   updatedAt: number;
 };
@@ -151,5 +191,26 @@ export class BitcoinmintsDB extends Dexie {
     this.version(2).stores({
       announcements: "[pubkey+kind+d], eventId, kind, d, createdAt, [kind+createdAt]",
     });
+    // v3: rename `bayesianRank` → `bayesianScore` on mintAggregate and add
+    // `avgRating` to the index set so sort-by-avg queries don't need a full
+    // table scan. This is the indexes materialized in PR #5's ranking
+    // aggregator. The prior `bayesianRank` index is dropped.
+    //
+    // Upgrade semantics: Dexie auto-migrates the SCHEMA (indexes) but does
+    // NOT transform existing row PAYLOADS. A dev with a local v2 IndexedDB
+    // would otherwise have rows shaped `{d, averageRating, bayesianRank,
+    // updatedAt}` — the `averageRating` field is `avgRating` in v3 and
+    // `bayesianRank` doesn't exist — which would fail every v3 query shape
+    // (the indexes point at fields the row doesn't have). Since there's no
+    // prod data yet and the aggregate is re-derived from reviews on the
+    // next review upsert, a clean wipe is the correct migration: clear
+    // `mintAggregate`, let it repopulate from live review traffic.
+    this.version(3)
+      .stores({
+        mintAggregate: "d, bayesianScore, avgRating, updatedAt",
+      })
+      .upgrade(async (tx) => {
+        await tx.table("mintAggregate").clear();
+      });
   }
 }

packages/core/src/cache/upsert.test.ts

@@ -112,8 +112,9 @@ function makeMintAggregate(over: Partial<MintAggregateRow> = {}): MintAggregateR
   return {
     d: D_XONLY,
     reviewCount: 5,
-    averageRating: 4.2,
-    bayesianRank: 3.8,
+    ratedCount: 5,
+    avgRating: 4.2,
+    bayesianScore: 3.8,
     updatedAt: 1_700_000_000,
     ...over,
   };
@@ -590,16 +591,16 @@ describe("upsertMintInfo", () => {
 describe("upsertMintAggregate", () => {
   it("inserts, replaces on newer updatedAt, rejects older", async () => {
     const db = await freshDB();
-    const older = makeMintAggregate({ updatedAt: 1000, bayesianRank: 1.0 });
-    const newer = makeMintAggregate({ updatedAt: 2000, bayesianRank: 4.5 });
+    const older = makeMintAggregate({ updatedAt: 1000, bayesianScore: 1.0 });
+    const newer = makeMintAggregate({ updatedAt: 2000, bayesianScore: 4.5 });
     const ancient = makeMintAggregate({ updatedAt: 500 });
 
     expect(await upsertMintAggregate(db, older)).toBe("inserted");
     expect(await upsertMintAggregate(db, newer)).toBe("replaced");
     expect(await upsertMintAggregate(db, ancient)).toBe("rejected-stale");
 
     const fetched = await db.mintAggregate.get(older.d);
-    expect(fetched?.bayesianRank).toBe(4.5);
+    expect(fetched?.bayesianScore).toBe(4.5);
     expect(fetched?.updatedAt).toBe(2000);
   });
 });

packages/core/src/cache/upsert.ts

@@ -15,14 +15,16 @@
  * Layer A gate for kind:38172: before writing an announcement we check
  * isValidCashuDTag(d). Invalid shapes (bot spam, non-hex garbage) are
  * returned as "rejected-invalid" and never hit the DB. kind:38173
- * (Fedimint) bypasses Layer A — federation IDs have a different shape
- * and their validator is a TODO-v1.1 concern.
+ * (Fedimint) uses a sibling shape gate (isValidFedimintDTag) — every real
+ * federation ID in the audit corpus is 64-char lowercase hex, so short /
+ * junk d-tags with `["k","38173"]` are still filtered at the same choke
+ * point as Cashu bot spam.
  *
  * mintInfo and mintAggregate aren't event-based, so their CAS predicate
  * is a monotonically-increasing timestamp: `fetchedAt` for mintInfo,
  * `updatedAt` for mintAggregate.
  */
-import { isValidCashuDTag } from "../nip87/dtag";
+import { isValidCashuDTag, isValidFedimintDTag } from "../nip87/dtag";
 import type {
   AnnouncementRow,
   BitcoinmintsDB,
@@ -56,14 +58,19 @@ function nextWins(
   return next.eventId > prev.eventId;
 }
 
-/** Upsert a kind:38172 or kind:38173 announcement with Layer A gating on 38172. */
+/** Upsert a kind:38172 or kind:38173 announcement with Layer A gating on both kinds. */
 export async function upsertAnnouncement(
   db: BitcoinmintsDB,
   row: AnnouncementRow,
 ): Promise<UpsertResult> {
-  // Layer A gate — reject invalid Cashu d-tag shapes before touching the DB.
-  // Fedimint (38173) bypasses: federation-id shape is TODO-v1.1.
-  if (row.kind === 38172 && !isValidCashuDTag(row.d)) {
+  // Layer A gate — reject invalid d-tag shapes before touching the DB.
+  // Cashu (38172) requires a 64- or 66-char secp256k1 pubkey shape;
+  // Fedimint (38173) requires a 64-char lowercase hex federation-id shape.
+  // A short/junk d-tag with `k=38173` slapped on is still bot spam and
+  // must be caught by the same firewall — don't free-pass by kind alone.
+  if (row.kind === 38173) {
+    if (!isValidFedimintDTag(row.d)) return "rejected-invalid";
+  } else if (!isValidCashuDTag(row.d)) {
     return "rejected-invalid";
   }
 
@@ -88,8 +95,41 @@ export async function upsertAnnouncement(
   });
 }
 
-/** Upsert a kind:38000 review. No Layer A gate — the `d` here points at a mint but isn't itself a Cashu pubkey owned by the reviewer. */
+/**
+ * Upsert a kind:38000 review with Layer A gating on the target `d` tag.
+ *
+ * The review's `d` points at a mint. When `k === 38172` (or `k` is absent,
+ * which is how most in-the-wild Cashu reviews shape), we apply the same
+ * Layer A d-regex gate that `upsertAnnouncement` uses — if the referenced
+ * mint pubkey isn't 64/66-char hex, the review is bot-spam pointing at
+ * bot-spam, returned as `rejected-invalid`. This is the firewall that
+ * keeps the 959 zero-d-tag bot spam events (per relay-strategy §4) from
+ * filtering up into the ranking aggregate.
+ *
+ * `k === 38173` (Fedimint) switches to the sibling `isValidFedimintDTag`
+ * shape gate — every real federation ID in the audit corpus is lowercase
+ * 64-char hex, so a short / junk d-tag with `k=38173` attached is still
+ * bot spam and must be caught by the same firewall.
+ *
+ * Note: this low-level upsert is the mechanical write. It does NOT
+ * materialize the `mintAggregate` row — the `reviews/` wrapper composes
+ * this with `recomputeAggregateInTx` inside a single transaction so the
+ * two stores stay in sync. Callers outside `reviews/` (integration tests,
+ * direct usage) can call this helper directly and will simply skip the
+ * aggregate materialization — safe but stale.
+ */
 export async function upsertReview(db: BitcoinmintsDB, row: ReviewRow): Promise<UpsertResult> {
+  // Layer A gate — reject invalid d-tag shapes before touching the DB.
+  // Reviews point at a target mint via `d`; the pointer-kind `k` selects
+  // which shape gate applies. No `k` tag → treat as Cashu (the default
+  // for in-the-wild events per rating-tag-research §3). Fedimint rows
+  // still get a sibling shape check (64-char hex federation id) so junk
+  // d-tags with `k=38173` slapped on don't free-pass the firewall.
+  if (row.k === 38173) {
+    if (!isValidFedimintDTag(row.d)) return "rejected-invalid";
+  } else if (!isValidCashuDTag(row.d)) {
+    return "rejected-invalid";
+  }
   return db.transaction("rw", db.reviews, async () => {
     const prev = await db.reviews.get([row.pubkey, row.kind, row.d]);
     if (!prev) {

packages/core/src/index.ts

@@ -2,5 +2,6 @@ export * from "./cache";
 export * from "./cashu";
 export * from "./nip87";
 export * from "./nostr";
+export * from "./reviews";
 
 export const VERSION = "0.0.0";