From ce843a4713c7a93c692878c136e89dda40f4463c Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 15 Sep 2026 23:25:46 -0700 Subject: [PATCH 01/54] adopt @civfix/shared 0.49.0 --- pnpm-lock.yaml | 14 +++++++------- services/api/package.json | 2 +- services/media-worker/package.json | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b29a9b23..bc839a3c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -65,8 +65,8 @@ importers: specifier: ^3.700.0 version: 3.1057.0 '@civfix/shared': - specifier: ^0.48.1 - version: 0.48.1 + specifier: ^0.49.0 + version: 0.49.0 '@fastify/cookie': specifier: ^11.0.1 version: 11.0.2 @@ -210,8 +210,8 @@ importers: specifier: workspace:* version: link:../api '@civfix/shared': - specifier: ^0.48.1 - version: 0.48.1 + specifier: ^0.49.0 + version: 0.49.0 '@sentry/node': specifier: ^10.73.0 version: 10.73.0(@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1)) @@ -414,8 +414,8 @@ packages: '@balena/dockerignore@1.0.2': resolution: {integrity: sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==} - '@civfix/shared@0.48.1': - resolution: {integrity: sha512-NXk+GK9YnsE0dzSDwyvyEjALTg1i0ylFTu7YKgSpzJZ5RNvSz6qx235sbuGIk7aKNLx0Bmu9SmOpmd92eTHcsQ==} + '@civfix/shared@0.49.0': + resolution: {integrity: sha512-xs6fB2p8bG26T5TRnKWKEQGq06zOm4DAjYiKgdHus6+CRifPlbpQaSIa2KcHxqaDyki3Dp/OwRxGhTjqQGlpgw==} engines: {node: '>=20'} '@derhuerst/http-basic@8.2.4': @@ -4440,7 +4440,7 @@ snapshots: '@balena/dockerignore@1.0.2': {} - '@civfix/shared@0.48.1': + '@civfix/shared@0.49.0': dependencies: zod: 3.25.76 diff --git a/services/api/package.json b/services/api/package.json index 66c962a5..ddb92928 100644 --- a/services/api/package.json +++ b/services/api/package.json @@ -56,7 +56,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.700.0", "@aws-sdk/s3-request-presigner": "^3.700.0", - "@civfix/shared": "^0.48.1", + "@civfix/shared": "^0.49.0", "@fastify/cookie": "^11.0.1", "@fastify/cors": "^10.0.1", "@fastify/helmet": "^12.0.1", diff --git a/services/media-worker/package.json b/services/media-worker/package.json index 82cc4730..fd34f73d 100644 --- a/services/media-worker/package.json +++ b/services/media-worker/package.json @@ -23,7 +23,7 @@ "@aws-sdk/client-s3": "^3.700.0", "@aws-sdk/s3-request-presigner": "^3.700.0", "@civfix/api": "workspace:*", - "@civfix/shared": "^0.48.1", + "@civfix/shared": "^0.49.0", "@sentry/node": "^10.73.0", "@smithy/node-http-handler": "4.7.5", "drizzle-orm": "^0.45.2", From 1959bef152f4e32c87d193aa1348b7527b3a9937 Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 15 Sep 2026 23:07:03 -0700 Subject: [PATCH 02/54] admin report list rows carry a presigned thumbnail --- .../admin/admin-report-repository.drizzle.ts | 26 +++++++- .../admin/admin-report-repository.memory.ts | 3 +- .../services/admin/admin-report-service.ts | 22 ++++++- .../src/services/admin/admin-report-types.ts | 13 ++++ services/api/test/unit/admin-reports.test.ts | 61 +++++++++++++++++++ services/api/test/unit/email-format.test.ts | 1 + 6 files changed, 121 insertions(+), 5 deletions(-) diff --git a/services/api/src/services/admin/admin-report-repository.drizzle.ts b/services/api/src/services/admin/admin-report-repository.drizzle.ts index b6d7412e..304abd4d 100644 --- a/services/api/src/services/admin/admin-report-repository.drizzle.ts +++ b/services/api/src/services/admin/admin-report-repository.drizzle.ts @@ -67,6 +67,10 @@ interface ReportRowSelect { lng: number confirmations: string has_photo: boolean + preview_id: string | null + preview_kind: "image" | "video" | null + preview_key: string | null + preview_thumb_key: string | null created_at: Date reference_code: string | null verification_verdict: "approved" | "rejected" | null @@ -106,6 +110,15 @@ function toRecord(r: ReportRowSelect): AdminReportRecord { lat: r.lat, lng: r.lng, hasPhoto: r.has_photo, + previewMedia: + r.preview_id !== null && r.preview_kind !== null && r.preview_key !== null + ? { + id: r.preview_id, + kind: r.preview_kind, + r2Key: r.preview_key, + thumbKey: r.preview_thumb_key, + } + : null, createdAt: r.created_at, referenceCode: r.reference_code, verificationVerdict: r.verification_verdict, @@ -135,6 +148,10 @@ function reportSelect( -- Kept as a stable admin DTO field (always 0 now) so the admin UI neighbors-confirmed row still parses. '0'::text AS confirmations, EXISTS (SELECT 1 FROM media_assets m WHERE m.report_id = r.id AND m.status = 'ready') AS has_photo, + pm.id AS preview_id, + pm.kind AS preview_kind, + pm.served_key AS preview_key, + pm.thumb_key AS preview_thumb_key, r.created_at, r.reference_code, r.verification_verdict, @@ -147,6 +164,13 @@ function reportSelect( LEFT JOIN jurisdictions j ON j.geoid = r.jurisdiction_geoid LEFT JOIN users u ON u.id = r.reporter_user_id LEFT JOIN user_moderation um ON um.user_id = r.reporter_user_id + LEFT JOIN LATERAL ( + SELECT pa.id, pa.kind, pa.served_key, pa.thumb_key + FROM media_assets pa + WHERE pa.report_id = r.id AND pa.status = 'ready' AND pa.served_key IS NOT NULL + ORDER BY (pa.kind = 'image') DESC, pa.created_at ASC, pa.id ASC + LIMIT 1 + ) pm ON TRUE WHERE r.deleted_at IS NULL ${extraWhere} ${orderLimit} @@ -396,7 +420,7 @@ export function makeDrizzleAdminReportRepository(sql: Sql): AdminReportRepositor SELECT id, kind, served_key AS r2_key, thumb_key FROM media_assets WHERE report_id = ${id} AND status = 'ready' AND served_key IS NOT NULL - ORDER BY created_at ASC + ORDER BY created_at ASC, id ASC LIMIT ${MEDIA_CAP} ` return rows.map((m) => ({ diff --git a/services/api/src/services/admin/admin-report-repository.memory.ts b/services/api/src/services/admin/admin-report-repository.memory.ts index ce1283fc..a3d184d2 100644 --- a/services/api/src/services/admin/admin-report-repository.memory.ts +++ b/services/api/src/services/admin/admin-report-repository.memory.ts @@ -20,7 +20,7 @@ import type { ReportTimelineItem, } from "@civfix/shared" import { mapOutreachStatus } from "./admin-report-repository.drizzle.js" -import { REPORT_VERIFIED_THRESHOLD } from "./admin-report-service.js" +import { pickPreviewMedia, REPORT_VERIFIED_THRESHOLD } from "./admin-report-service.js" import { STATUS_BUCKETS } from "./admin-report-status.js" export interface RecordedReportNotification { @@ -112,6 +112,7 @@ export class InMemoryAdminReportRepository implements AdminReportRepository { lat: input.lat ?? 0, lng: input.lng ?? 0, hasPhoto: input.hasPhoto ?? false, + previewMedia: pickPreviewMedia(input.media ?? []), createdAt: input.createdAt ?? this.now, referenceCode: input.referenceCode ?? null, verificationVerdict: input.verificationVerdict ?? null, diff --git a/services/api/src/services/admin/admin-report-service.ts b/services/api/src/services/admin/admin-report-service.ts index 622bcc64..7d882d09 100644 --- a/services/api/src/services/admin/admin-report-service.ts +++ b/services/api/src/services/admin/admin-report-service.ts @@ -31,6 +31,7 @@ import { MAX_PACKET_ATTACHMENT_BYTES, MAX_PACKET_TOTAL_BYTES, } from "./mail-format.js" +import { pickPreviewMedia, previewThumbnailUrl } from "./admin-report-types.js" import type { AdminReportRecord, AdminReportService, @@ -81,7 +82,18 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor await deps.reportChatEmitter.emit(event) } - function toListItem(record: AdminReportRecord, ref: Date): AdminReportListItemDTO { + async function previewMediaDTO(record: AdminReportRecord): Promise { + const m = record.previewMedia + if (m === null) return null + const { url, thumbUrl } = await presignMedia(m.r2Key, m.thumbKey) + return { id: m.id, kind: m.kind, url, thumbUrl: thumbUrl ?? null } + } + + function toListItem( + record: AdminReportRecord, + ref: Date, + preview: ReportMedia | null, + ): AdminReportListItemDTO { return { id: record.id, category: record.category, @@ -99,6 +111,7 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor coords: [record.lat, record.lng], address: record.address, hasPhoto: record.hasPhoto, + thumbnailUrl: previewThumbnailUrl(preview), } } @@ -140,7 +153,10 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor flagged: 0, }), ]) - return { items: records.map((r) => toListItem(r, ref)), nextCursor, counts } + const items = await mapWithLimit(records, PRESIGN_CONCURRENCY, async (r) => + toListItem(r, ref, await previewMediaDTO(r)), + ) + return { items, nextCursor, counts } }, async get(id: string): Promise { @@ -156,7 +172,6 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor : Promise.resolve(new Map()), deps.repo.getOutreach(id), ]) - const base = toListItem(record, ref) const city: ReportRouting = { dept: routing?.dept ?? "", place: routing?.place ?? record.place, @@ -167,6 +182,7 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor const { url, thumbUrl } = await presignMedia(m.r2Key, m.thumbKey) return { id: m.id, kind: m.kind, url, thumbUrl: thumbUrl ?? null } }) + const base = toListItem(record, ref, pickPreviewMedia(mediaDtos)) const linkedEvents: LinkedEventRef[] = (linkedEventsMap.get(id) ?? []).map(toLinkedEventRef) const outreachDTO: ReportOutreach = { status: outreach.status, diff --git a/services/api/src/services/admin/admin-report-types.ts b/services/api/src/services/admin/admin-report-types.ts index 9f425139..2c39ef57 100644 --- a/services/api/src/services/admin/admin-report-types.ts +++ b/services/api/src/services/admin/admin-report-types.ts @@ -6,6 +6,7 @@ import type { AdminReportListItemDTO, AdminReportStatus, ReportCategory, + ReportMedia, ReportOutreachStatus, ReportTimelineItem, } from "@civfix/shared" @@ -65,6 +66,7 @@ export interface AdminReportRecord { lat: number lng: number hasPhoto: boolean + previewMedia: AdminReportMediaRecord | null createdAt: Date referenceCode: string | null verificationVerdict: "approved" | "rejected" | null @@ -146,6 +148,17 @@ export interface AdminReportRepository { export const REPORT_VERIFIED_THRESHOLD = 2 +export function pickPreviewMedia( + media: readonly T[], +): T | null { + return media.find((m) => m.kind === "image") ?? media[0] ?? null +} + +export function previewThumbnailUrl(media: ReportMedia | null): string | null { + if (media === null) return null + return media.kind === "image" ? (media.thumbUrl ?? media.url) : (media.thumbUrl ?? null) +} + export interface AdminReportServiceDeps { repo: AdminReportRepository outboundMail: OutboundMailService diff --git a/services/api/test/unit/admin-reports.test.ts b/services/api/test/unit/admin-reports.test.ts index b52943b6..8d22dc8e 100644 --- a/services/api/test/unit/admin-reports.test.ts +++ b/services/api/test/unit/admin-reports.test.ts @@ -128,6 +128,66 @@ describe("admin reports list", () => { expect(row.submitted.abs).toContain("2026") }) + it("presigns a thumbnail for the first image, preferring the pipeline thumb over the full image", async () => { + const { repo, svc } = harness() + repo.seedReport({ + id: "11111111-1111-1111-1111-111111111111", + hasPhoto: true, + media: [ + { id: "m1", kind: "image", r2Key: "r2://photo.jpg", thumbKey: "r2://thumb.jpg" }, + { id: "m2", kind: "image", r2Key: "r2://second.jpg", thumbKey: null }, + ], + }) + + const page = await svc.list({}) + expect(page.items[0]!.thumbnailUrl).toBe("https://media.test/r2://thumb.jpg") + }) + + it("falls back to the served image when the pipeline made no thumbnail", async () => { + const { repo, svc } = harness() + repo.seedReport({ + id: "11111111-1111-1111-1111-111111111111", + hasPhoto: true, + media: [{ id: "m1", kind: "image", r2Key: "r2://photo.jpg", thumbKey: null }], + }) + + const page = await svc.list({}) + expect(page.items[0]!.thumbnailUrl).toBe("https://media.test/r2://photo.jpg") + }) + + it("prefers an image over an earlier video, and leaves a poster-less video with no thumbnail", async () => { + const { repo, svc } = harness() + repo.seedReport({ + id: "11111111-1111-1111-1111-111111111111", + hasPhoto: true, + media: [ + { id: "m1", kind: "video", r2Key: "r2://clip.mp4", thumbKey: null }, + { id: "m2", kind: "image", r2Key: "r2://photo.jpg", thumbKey: null }, + ], + }) + repo.seedReport({ + id: "22222222-2222-2222-2222-222222222222", + hasPhoto: true, + createdAt: hoursAgo(1), + media: [{ id: "m3", kind: "video", r2Key: "r2://clip.mp4", thumbKey: null }], + }) + + const page = await svc.list({}) + const byId = new Map(page.items.map((r) => [r.id, r.thumbnailUrl])) + expect(byId.get("11111111-1111-1111-1111-111111111111")).toBe( + "https://media.test/r2://photo.jpg", + ) + expect(byId.get("22222222-2222-2222-2222-222222222222")).toBeNull() + }) + + it("leaves thumbnailUrl null for a report with no media", async () => { + const { repo, svc } = harness() + repo.seedReport({ id: "11111111-1111-1111-1111-111111111111", hasPhoto: false }) + + const page = await svc.list({}) + expect(page.items[0]!.thumbnailUrl).toBeNull() + }) + it("projects a named reporter, and falls back to Anonymous for a report with no reporter", async () => { const { repo, svc } = harness() repo.seedReport({ @@ -282,6 +342,7 @@ describe("admin reports detail", () => { thumbUrl: "https://media.test/r2://thumb.jpg", }, ]) + expect(detail.thumbnailUrl).toBe("https://media.test/r2://thumb.jpg") expect(detail.timeline).toHaveLength(1) expect(detail.timeline[0]).toMatchObject({ what: "Report submitted", kind: "submit" }) }) diff --git a/services/api/test/unit/email-format.test.ts b/services/api/test/unit/email-format.test.ts index 3afbaf94..06ec535a 100644 --- a/services/api/test/unit/email-format.test.ts +++ b/services/api/test/unit/email-format.test.ts @@ -30,6 +30,7 @@ function reportRecord(overrides: Partial = {}): AdminReportRe lat: 39.5, lng: -98.35, hasPhoto: true, + previewMedia: null, createdAt: new Date("2026-01-01T00:00:00Z"), referenceCode: "ABC123", verificationVerdict: null, From 158642566906d39459b258fdb57381c03eb2fc88 Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 15 Sep 2026 23:33:24 -0700 Subject: [PATCH 03/54] posts.geom denormalization with partial GIST and keyset backfill --- services/api/drizzle/0172_posts_geom.sql | 63 +++++++++++++++++++ .../0173_posts_author_public_recent_idx.sql | 33 ++++++++++ services/api/package.json | 2 + .../api/src/db/backfill-post-geom-core.ts | 57 +++++++++++++++++ services/api/src/db/backfill-post-geom.ts | 14 +++++ services/api/src/db/schema/posts.ts | 9 ++- services/api/tsup.config.ts | 1 + 7 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 services/api/drizzle/0172_posts_geom.sql create mode 100644 services/api/drizzle/0173_posts_author_public_recent_idx.sql create mode 100644 services/api/src/db/backfill-post-geom-core.ts create mode 100644 services/api/src/db/backfill-post-geom.ts diff --git a/services/api/drizzle/0172_posts_geom.sql b/services/api/drizzle/0172_posts_geom.sql new file mode 100644 index 00000000..0b3f398f --- /dev/null +++ b/services/api/drizzle/0172_posts_geom.sql @@ -0,0 +1,63 @@ +-- ============================================================================= +-- 0172_posts_geom.sql +-- ----------------------------------------------------------------------------- +-- ISSUE #100 (ranked home feed): the ranker scores a "nearby" term, but `posts` +-- carried no geometry at all. A post's only location was indirect — through +-- posts.report_id -> reports.geom or posts.event_id -> cleanups.geom — so a +-- proximity pool would need two joins and could not use a KNN order at all. +-- +-- This denormalises that point onto the post row so the nearby candidate pool +-- is ONE bounded GIST/KNN scan (see feedCandidates in +-- services/post-repository.drizzle.ts). NULL for a post with no attachment. +-- +-- NOT A HOT TABLE: `posts` is absent from the hot-table list in +-- docs/out-of-band-indexes.md (users, reports, chat_messages, dm_messages, +-- media_assets, notifications, sessions), so the partial GIST is built inline +-- here rather than out of band. If `posts` has grown large by the time this +-- deploys, build it with CREATE INDEX CONCURRENTLY first — the IF NOT EXISTS +-- guard below then turns this statement into a no-op. +-- +-- NO BACKFILL HERE: one UPDATE over the whole table inside this file's single +-- transaction is a lock hazard. Existing rows are populated after the deploy is +-- healthy by `pnpm db:backfill:post-geom` (src/db/backfill-post-geom.ts), which +-- is keyset-paged, idempotent and safe to run while the API serves traffic. The +-- DO block raises a WARNING while rows remain unpopulated so a forgotten +-- backfill is loud instead of silent; the feed is correct without it, those +-- posts simply score no proximity term. +-- +-- PRIVACY: no new class of data. The value is a copy of a coordinate this +-- platform already publishes at full precision on the linked report or event +-- DTO. It is never populated from a client-supplied coordinate, carries no +-- EXIF, and is never returned to any client — it only orders the feed. Any +-- future location-coarsening decision must cover this column too +-- (docs/location-coarsening-assessment.md). +-- +-- CANONICAL DDL: hand-authored source of truth. Mirror: schema/posts.ts. +-- +-- Conventions: additive ADD COLUMN IF NOT EXISTS; one concern per file; one +-- transaction per file. Forward-only, no down. +-- +-- Ordering rules: requires 0051_social_posts.sql (posts). +-- ============================================================================= + +ALTER TABLE posts ADD COLUMN IF NOT EXISTS geom geometry(Point, 4326); + +COMMENT ON COLUMN posts.geom IS + 'Denormalised from the linked report or event at insert time; NULL for a post with no attachment. Feed-ranking proximity only - never a user-supplied coordinate, never served to clients. Backfill: pnpm db:backfill:post-geom.'; + +CREATE INDEX IF NOT EXISTS posts_geom_gist + ON posts USING gist (geom) + WHERE geom IS NOT NULL AND deleted_at IS NULL AND visibility = 'public'; + +DO $$ +DECLARE + pending bigint; +BEGIN + SELECT count(*) INTO pending + FROM posts + WHERE geom IS NULL AND (report_id IS NOT NULL OR event_id IS NOT NULL); + + IF pending > 0 THEN + RAISE WARNING 'posts.geom is unpopulated for % attached post(s) - run pnpm db:backfill:post-geom after this deploy is healthy; those posts score no feed proximity term until then', pending; + END IF; +END $$; diff --git a/services/api/drizzle/0173_posts_author_public_recent_idx.sql b/services/api/drizzle/0173_posts_author_public_recent_idx.sql new file mode 100644 index 00000000..2c7d14d7 --- /dev/null +++ b/services/api/drizzle/0173_posts_author_public_recent_idx.sql @@ -0,0 +1,33 @@ +-- ============================================================================= +-- 0173_posts_author_public_recent_idx.sql +-- ----------------------------------------------------------------------------- +-- ISSUE #100 (ranked home feed): the ranked feed's in-network candidate pool +-- selects top-level PUBLIC posts by the viewer plus everyone the viewer +-- follows, newest-first, bounded by LIMIT. +-- +-- Neither existing index serves that shape: +-- posts_author_created_idx (0051) - author + created_at, but not partial on +-- reply_to_id / visibility, and no id tiebreak +-- posts_toplevel_recent_idx (0071) - the right ordering, but not author-keyed, +-- so an author-scoped pool scans it whole +-- +-- This is the author-keyed twin of 0071: the same live top-level keyset, +-- narrowed to public rows and led by author_id so the IN (followees) probe stops +-- at the pool LIMIT instead of sorting. +-- +-- NOT A HOT TABLE: `posts` is absent from the hot-table list in +-- docs/out-of-band-indexes.md, so this builds inline. If `posts` has grown large +-- by the time this deploys, build it with CREATE INDEX CONCURRENTLY first and +-- the IF NOT EXISTS guard turns this into a no-op. +-- +-- CANONICAL DDL: hand-authored source of truth. Mirror: schema/posts.ts. +-- +-- Conventions: one concern per file; one transaction per file. Forward-only, +-- no down. +-- +-- Ordering rules: requires 0051_social_posts.sql. +-- ============================================================================= + +CREATE INDEX IF NOT EXISTS posts_author_public_recent_idx + ON posts (author_id, created_at DESC, id DESC) + WHERE deleted_at IS NULL AND reply_to_id IS NULL AND visibility = 'public'; diff --git a/services/api/package.json b/services/api/package.json index ddb92928..9ff931db 100644 --- a/services/api/package.json +++ b/services/api/package.json @@ -32,6 +32,7 @@ "start:backfill:served-key": "node dist/db/backfill-served-key.js", "start:backfill:user-activity": "node dist/db/backfill-user-activity.js", "start:backfill:signup-seats": "node dist/db/backfill-signup-seats.js", + "start:backfill:post-geom": "node dist/db/backfill-post-geom.js", "start:seed:demo": "node dist/db/seed-demo-la.js", "typecheck": "tsc --noEmit", "lint": "eslint .", @@ -48,6 +49,7 @@ "db:backfill:served-key": "tsx src/db/backfill-served-key.ts", "db:backfill:user-activity": "tsx src/db/backfill-user-activity.ts", "db:backfill:signup-seats": "tsx src/db/backfill-signup-seats.ts", + "db:backfill:post-geom": "tsx src/db/backfill-post-geom.ts", "db:certificate:revoke": "tsx scripts/revoke-certificate.ts", "db:prepare-boundaries": "tsx scripts/prepare-boundaries.ts", "db:boundaries:refresh": "tsx scripts/refresh-boundaries.ts", diff --git a/services/api/src/db/backfill-post-geom-core.ts b/services/api/src/db/backfill-post-geom-core.ts new file mode 100644 index 00000000..1b1c6913 --- /dev/null +++ b/services/api/src/db/backfill-post-geom-core.ts @@ -0,0 +1,57 @@ +import type postgres from "postgres" +import type { Sql } from "./client.js" + +type SqlFragment = postgres.Fragment + +export const POST_GEOM_BACKFILL_BATCH = 1000 + +export async function backfillPostGeom( + sql: Sql, + opts: { batchSize?: number; log?: (message: string) => void } = {}, +): Promise<{ scanned: number; filled: number }> { + const batchSize = opts.batchSize ?? POST_GEOM_BACKFILL_BATCH + const log = opts.log ?? ((message: string) => console.log(`backfill-post-geom: ${message}`)) + let scanned = 0 + let filled = 0 + let cursor: string | null = null + + for (;;) { + const cursorFilter: SqlFragment = cursor === null ? sql`` : sql`AND id > ${cursor}` + const page = await sql<{ id: string }[]>` + SELECT id + FROM posts + WHERE geom IS NULL + AND (report_id IS NOT NULL OR event_id IS NOT NULL) + ${cursorFilter} + ORDER BY id + LIMIT ${batchSize} + ` + if (page.length === 0) break + const ids = page.map((r) => r.id) + + const updated = await sql<{ id: string }[]>` + WITH candidates AS ( + SELECT s.id, COALESCE(r.geom, c.geom) AS geom + FROM posts s + LEFT JOIN reports r ON r.id = s.report_id + LEFT JOIN cleanups c ON c.id = s.event_id + WHERE s.id IN ${sql(ids)} + AND s.geom IS NULL + ) + UPDATE posts p + SET geom = cand.geom + FROM candidates cand + WHERE p.id = cand.id + AND p.geom IS NULL + AND cand.geom IS NOT NULL + RETURNING p.id + ` + + scanned += page.length + filled += updated.length + cursor = page[page.length - 1]!.id + log(`page of ${page.length} (filled ${updated.length}); running scanned=${scanned}, filled=${filled}`) + } + + return { scanned, filled } +} diff --git a/services/api/src/db/backfill-post-geom.ts b/services/api/src/db/backfill-post-geom.ts new file mode 100644 index 00000000..b1216626 --- /dev/null +++ b/services/api/src/db/backfill-post-geom.ts @@ -0,0 +1,14 @@ + +import { runDbCli, runIfMain } from "./cli.js" +import { backfillPostGeom } from "./backfill-post-geom-core.js" + +export { backfillPostGeom } from "./backfill-post-geom-core.js" + +async function main(): Promise { + await runDbCli(async (_db, sql) => { + const { scanned, filled } = await backfillPostGeom(sql) + console.log(`backfill-post-geom: done — scanned=${scanned} filled=${filled}`) + }) +} + +runIfMain(import.meta.url, "backfill-post-geom", main) diff --git a/services/api/src/db/schema/posts.ts b/services/api/src/db/schema/posts.ts index d4d81a06..14e3c903 100644 --- a/services/api/src/db/schema/posts.ts +++ b/services/api/src/db/schema/posts.ts @@ -14,7 +14,7 @@ import { cleanups } from "./cleanups.js" import { organizations } from "./organizations.js" import { reports } from "./reports.js" import { users } from "./users.js" -import type { POST_KIND_VALUES, REPORT_VISIBILITY_VALUES } from "./types.js" +import { geometry, type POST_KIND_VALUES, type REPORT_VISIBILITY_VALUES } from "./types.js" type PostKind = (typeof POST_KIND_VALUES)[number] type PostVisibility = (typeof REPORT_VISIBILITY_VALUES)[number] @@ -45,6 +45,7 @@ export const posts = pgTable( organizationId: uuid("organization_id").references(() => organizations.id, { onDelete: "set null", }), + geom: geometry("geom", { subtype: "Point", srid: 4326 }), likeCount: integer("like_count").notNull().default(0), repostCount: integer("repost_count").notNull().default(0), replyCount: integer("reply_count").notNull().default(0), @@ -75,6 +76,12 @@ export const posts = pgTable( index("posts_toplevel_recent_idx") .on(t.createdAt.desc(), t.id.desc()) .where(sql`deleted_at IS NULL AND reply_to_id IS NULL`), + index("posts_author_public_recent_idx") + .on(t.authorId, t.createdAt.desc(), t.id.desc()) + .where(sql`deleted_at IS NULL AND reply_to_id IS NULL AND visibility = 'public'`), + index("posts_geom_gist") + .using("gist", t.geom) + .where(sql`geom IS NOT NULL AND deleted_at IS NULL AND visibility = 'public'`), index("posts_organization_created_idx") .on(t.organizationId, t.createdAt.desc()) .where(sql`${t.organizationId} IS NOT NULL AND deleted_at IS NULL`), diff --git a/services/api/tsup.config.ts b/services/api/tsup.config.ts index 83f80076..8fa1b24a 100644 --- a/services/api/tsup.config.ts +++ b/services/api/tsup.config.ts @@ -13,6 +13,7 @@ export default defineConfig({ "src/db/backfill-served-key.ts", "src/db/backfill-user-activity.ts", "src/db/backfill-signup-seats.ts", + "src/db/backfill-post-geom.ts", // db/seed-demo-la.ts: the LA demo-community seeder, emitted so an operator can run // `node dist/db/seed-demo-la.js [--yes|--purge --yes]` INSIDE the api container (which already // carries DATABASE_URL) WITHOUT tsx or handling credentials. NEVER wired into the deploy From 02a5c53459d109fcbc7690564d47991fcc0a2cd4 Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 15 Sep 2026 23:44:20 -0700 Subject: [PATCH 04/54] FEED_RANKING env var validated by the contract schema --- services/api/.env.example | 18 ++++++++++++++++++ services/api/src/env.ts | 27 +++++++++++++++++++++++++++ services/api/src/env/types.ts | 3 +++ 3 files changed, 48 insertions(+) diff --git a/services/api/.env.example b/services/api/.env.example index e66352ee..76d26541 100644 --- a/services/api/.env.example +++ b/services/api/.env.example @@ -99,6 +99,24 @@ HOME_REGION_LAT= # [OPT] home-region latitude, -90..90 HOME_REGION_LNG= # [OPT] home-region longitude, -180..180 (default -118.2437) HOME_REGION_RADIUS_KM= # [OPT] home-region accuracy radius in km, > 0 (default 40) +# ===== ranked home feed (GET /feed/home) ===== +# FEED_RANKING is a JSON object of ranking overrides, validated at boot against the contract's +# FeedRankingConfigSchema (@civfix/shared). EVERY key is optional and defaulted, so leaving this +# blank yields the full default profile below - nothing breaks when it is unset. The schema is +# .strict(): a typo'd key or an out-of-range value is a NAMED boot failure, never a silently +# ignored knob. Override only what you are tuning, e.g. +# FEED_RANKING={"halfLifeHours":12,"followWeight":70} +# Changing a weight is a redeploy (edit the SOPS dotenv, merge to main for staging), never a +# client release. +# +# Defaults: baseWeight 10, followWeight 100, selfWeight 60, mentionWeight 40, nearbyWeight 50, +# nearbyRadiusKm 40, orgVerifiedWeight 30, attachEventWeight 30, attachReportWeight 30, +# imageWeight 5, likeWeight 3, replyWeight 8, repostWeight 5, halfLifeHours 36, decayFloor 0.15, +# diversityFloor 0.25, diversityDecay 0.5, seenDiscount 0.7, minScore 12, minPageItems 5, +# candidateWindowDays 30, candidateCap 400, clockBucketSeconds 60, snapshotTtlSeconds 180, +# servedTtlSeconds 900, viewerFanoutMax 500, newPostFanoutMax 1000 +FEED_RANKING= # [OPT] JSON feed-ranking overrides (default: the profile above) + # ===== mailer: OCI SMTP (bypassed by USE_FAKE_MAILER) ===== OCI_EMAIL_SMTP_HOST= # [BOOT] unless USE_FAKE_MAILER - OCI Email Delivery SMTP host OCI_EMAIL_SMTP_PORT=587 # [BOOT] unless USE_FAKE_MAILER - SMTP port (default 587) diff --git a/services/api/src/env.ts b/services/api/src/env.ts index 4750133f..b21edaa2 100644 --- a/services/api/src/env.ts +++ b/services/api/src/env.ts @@ -1,5 +1,7 @@ import { z } from "zod" +import { DEFAULT_FEED_RANKING, FeedRankingConfigSchema } from "@civfix/shared" +import type { FeedRankingConfig } from "@civfix/shared" import type { Env } from "./env/types.js" import { assertOutboundSendPolicy } from "./services/admin/outbound-send-policy.js" import { loadCommsEnv } from "./env/comms-env.js" @@ -161,6 +163,27 @@ export function loadEnv(source: NodeJS.ProcessEnv = process.env): Env { return fallback } + function reqFeedRanking(key: string): FeedRankingConfig { + const raw = (source[key] ?? "").trim() + if (raw.length === 0) return DEFAULT_FEED_RANKING + let decoded: unknown + try { + decoded = JSON.parse(raw) + } catch { + errors.push(`${key}: must be a JSON object of feed-ranking overrides`) + return DEFAULT_FEED_RANKING + } + const parsed = FeedRankingConfigSchema.safeParse(decoded) + if (!parsed.success) { + for (const issue of parsed.error.issues) { + const path = issue.path.length > 0 ? `.${issue.path.join(".")}` : "" + errors.push(`${key}${path}: ${issue.message}`) + } + return DEFAULT_FEED_RANKING + } + return parsed.data + } + function reqCron(key: string, fallback: string): string { const value = (source[key] ?? "").trim() || fallback if (!isCronish(value)) { @@ -249,6 +272,8 @@ export function loadEnv(source: NodeJS.ProcessEnv = process.env): Env { { min: 0, max: 20_000, exclusiveMin: true }, ) + const FEED_RANKING = reqFeedRanking("FEED_RANKING") + const LOCAL_STORAGE_DIR = (source.LOCAL_STORAGE_DIR ?? "").trim() const usesLocalStorage = LOCAL_STORAGE_DIR.length > 0 if (isProd && usesLocalStorage) { @@ -384,6 +409,8 @@ export function loadEnv(source: NodeJS.ProcessEnv = process.env): Env { HOME_REGION_LNG, HOME_REGION_RADIUS_KM, + FEED_RANKING, + CENSUS_GEOCODER_URL: (source.CENSUS_GEOCODER_URL ?? "").trim() || "https://geocoding.geo.census.gov/geocoder/geographies/coordinates", diff --git a/services/api/src/env/types.ts b/services/api/src/env/types.ts index 817eb2ff..6944407b 100644 --- a/services/api/src/env/types.ts +++ b/services/api/src/env/types.ts @@ -1,3 +1,4 @@ +import type { FeedRankingConfig } from "@civfix/shared" import type { TrustProxyValue } from "./parsers.js" import type { CommsEnv } from "./comms-env.js" import type { RegistrationEnv } from "./registration-env.js" @@ -34,6 +35,8 @@ export interface Env extends CommsEnv, RegistrationEnv { HOME_REGION_LNG: number HOME_REGION_RADIUS_KM: number + FEED_RANKING: FeedRankingConfig + CENSUS_GEOCODER_URL: string CENSUS_GEOCODER_TIMEOUT_MS: number From 761204b5c0903709a487c10701b02601eeb4e22e Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 15 Sep 2026 23:44:20 -0700 Subject: [PATCH 05/54] ranked home feed: scorer, candidate SQL, redis presence, realtime fanout, feed counts endpoint --- ...2026-07-24-full-backend-security-review.md | 7 +- services/api/src/auth/cache.ts | 9 + services/api/src/di.ts | 19 ++ services/api/src/routes/posts.routes.ts | 42 ++- services/api/src/services/feed-presence.ts | 143 ++++++++++ services/api/src/services/feed-ranking.ts | 122 +++++++++ .../src/services/post-repository.drizzle.ts | 259 +++++++++++++++++- services/api/src/services/post-service.ts | 238 ++++++++++++++-- .../api/test/unit/admin-guard-cache.test.ts | 3 + services/api/test/unit/auth-session.test.ts | 3 + services/api/test/unit/host-analytics.test.ts | 1 + services/api/test/unit/post-service.test.ts | 100 ++++++- services/api/test/unit/posts-routes.test.ts | 58 +++- .../api/test/unit/rate-limit-plugin.test.ts | 7 +- services/api/test/unit/route-coverage.test.ts | 4 +- 15 files changed, 976 insertions(+), 39 deletions(-) create mode 100644 services/api/src/services/feed-presence.ts create mode 100644 services/api/src/services/feed-ranking.ts diff --git a/docs/security/2026-07-24-full-backend-security-review.md b/docs/security/2026-07-24-full-backend-security-review.md index da97f5ad..5bebede8 100644 --- a/docs/security/2026-07-24-full-backend-security-review.md +++ b/docs/security/2026-07-24-full-backend-security-review.md @@ -322,7 +322,7 @@ Everything an operator has to do by hand, in order, plus the two infra facts and node dist/db/migrate.js # == pnpm --filter @civfix/api db:migrate ``` -`drizzle/` holds **154 files**, `0000_extensions.sql` … `0171_legal_documents_2026_09_16.sql`. The nine rows +`drizzle/` holds **156 files**, `0000_extensions.sql` … `0173_posts_author_public_recent_idx.sql`. The nine rows below are exactly what this change set adds — `0052`–`0060`, contiguous, no gaps — and everything from `0000` through `0051_social_posts.sql` predates it. (`0060` arrived later than the rest, with the feed redesign; it is listed here because this table is the single operator runbook. `0061`–`0064` arrived @@ -559,6 +559,11 @@ foreign key into `organizations` from `0105`. | `0171_legal_documents_2026_09_16.sql` | DATA only, no DDL (the Drizzle mirror is unchanged): INSERTs the four `2026-09-16` `legal_documents` rows - `terms`, `privacy`, `cookies`, `subprocessors` - mirroring `LEGAL_DOCUMENTS` in `@civfix/shared` 0.48.1 byte for byte. Removing platform-processed donations rewrote all four public documents (payment/refund/application-fee sections out of the terms, Stripe out of the privacy notice and the subprocessor list, checkout entries out of the cookie notice), and different text is a different document, so each gets a new version and a new sha256 and every person is asked to accept again. `ON CONFLICT (type, version) DO NOTHING`, so a re-run writes nothing. The `2026-09-06` rows are left in place exactly as 0151's header requires - publishing is an INSERT, never an UPDATE, because `consent_records_document_fk` points every consent already collected at them - and the three retired types (`donations`, `org_donation_agreement`, `donation_disclosure`) are deliberately not re-versioned: nothing renders them any more and the type CHECK still accepts their existing rows. Four rows into a cold table: no index build, no lock of consequence | Registration and onboarding 500 on every consent: `assertConsentVersionsCurrent` demands `2026-09-16` from the shared package, the client sends it, and the `consent_records` INSERT then violates `consent_records_document_fk` because no `legal_documents` row carries that version. Nobody can create an account on the new image | +| `0172_posts_geom.sql` | adds `posts.geom` (nullable `geometry(Point,4326)`, no default, so a catalog-only ADD COLUMN with no table rewrite) plus the partial GIST `posts_geom_gist` over live public rows, denormalising the linked report's or event's point onto the post so the ranked home feed's proximity pool is one bounded KNN scan instead of two joins. `posts` is NOT on the hot-table list in `docs/out-of-band-indexes.md`, so the index builds inline; build it with `CREATE INDEX CONCURRENTLY` first if `posts` has grown, and the `IF NOT EXISTS` guard makes the migration a no-op. No backfill in the file - a single UPDATE over the whole table inside one transaction is a lock hazard - so a DO block RAISEs a WARNING while attached rows are still unpopulated and `node dist/db/backfill-post-geom.js` (keyset-paged, idempotent, safe under live traffic) fills them after the deploy is healthy. PRIVACY: no new class of data, a copy of a coordinate already published at full precision on the linked report or event, never client-supplied, never served to clients | Nothing breaks: `p.geom IS NULL` simply yields a NULL `distance_km`, the ranker scores no proximity term, and the feed falls back to the in-network and recent-public pools. Skipping only the BACKFILL means existing attached posts score no proximity until it is run | + +| `0173_posts_author_public_recent_idx.sql` | adds the partial index `posts_author_public_recent_idx (author_id, created_at DESC, id DESC) WHERE deleted_at IS NULL AND reply_to_id IS NULL AND visibility = 'public'` - the author-keyed twin of `0071_posts_toplevel_recent_idx`, backing the ranked feed's in-network candidate pool (self + followees). Neither `posts_author_created_idx` (0051, not partial on reply_to_id/visibility, no id tiebreak) nor `posts_toplevel_recent_idx` (0071, not author-keyed) serves that shape. Same not-a-hot-table reasoning as 0172 | The in-network pool sorts instead of stopping at its LIMIT. Correct, slower; no functional change | + + **Deferred to the NEXT release** (expand/contract, `docs/migrations-expand-contract.md`): 0.43.0 stops every code path from reading `user_verification` but does NOT drop it — a `DROP TABLE` in the same release would make the previous api color raise `relation does not exist` on the feed, the diff --git a/services/api/src/auth/cache.ts b/services/api/src/auth/cache.ts index 52a8be83..f0156b7b 100644 --- a/services/api/src/auth/cache.ts +++ b/services/api/src/auth/cache.ts @@ -11,6 +11,7 @@ export interface CacheClient { sadd(key: string, ...members: string[]): Promise srem(key: string, ...members: string[]): Promise smembers(key: string): Promise + scard(key: string): Promise expire(key: string, ttlSeconds: number): Promise } @@ -115,6 +116,10 @@ export class InMemoryCacheClient implements CacheClient { return Promise.resolve([...(this.liveSet(key)?.members ?? [])]) } + scard(key: string): Promise { + return Promise.resolve(this.liveSet(key)?.members.size ?? 0) + } + expire(key: string, ttlSeconds: number): Promise { const at = this.clock() + ttlSeconds * 1000 const setEntry = this.liveSet(key) @@ -178,6 +183,10 @@ export class RedisCacheClient implements CacheClient { return this.redis.smembers(key) } + async scard(key: string): Promise { + return this.redis.scard(key) + } + async expire(key: string, ttlSeconds: number): Promise { await this.redis.expire(key, Math.max(1, Math.ceil(ttlSeconds))) } diff --git a/services/api/src/di.ts b/services/api/src/di.ts index 1ff2061b..cda4206d 100644 --- a/services/api/src/di.ts +++ b/services/api/src/di.ts @@ -70,6 +70,7 @@ import { type PostRepository, } from "./services/post-repository.drizzle.js" import { makePostService, type PostService } from "./services/post-service.js" +import { makeFeedPresence, type FeedPresence } from "./services/feed-presence.js" import { makeNotificationService, type NotificationService, @@ -126,6 +127,7 @@ export interface Container { getAffiliationLoader(): AffiliationLoader getPostRepo(): PostRepository getPostService(): PostService + getFeedPresence(): FeedPresence getNotificationService(logger?: NotificationLogger): NotificationService getCounterStore(): CounterStore getCache(): CacheClient @@ -217,6 +219,18 @@ export function buildContainer(env: Env): Container { return postRepo } + let feedPresence: FeedPresence | undefined + function getFeedPresence(): FeedPresence { + if (!feedPresence) { + feedPresence = makeFeedPresence({ + ...(env.REDIS_URL.length > 0 ? { cache: getCache() } : {}), + config: env.FEED_RANKING, + ...(serverLogger !== undefined ? { logger: serverLogger } : {}), + }) + } + return feedPresence + } + let postService: PostService | undefined function getPostService(): PostService { if (!postService) { @@ -225,6 +239,9 @@ export function buildContainer(env: Env): Container { sql: getDb().sql, notifier: getNotificationService(), isBlockedEitherWay: (a: string, b: string) => getBlocksRepo().isBlockedEitherWay(a, b), + feedRanking: env.FEED_RANKING, + feedPresence: getFeedPresence(), + userChannel: getUserChannel(), }) } return postService @@ -476,6 +493,7 @@ export function buildContainer(env: Env): Container { redis = undefined redisCounters = undefined cacheClient = undefined + feedPresence = undefined redisByteMeter = undefined } if (dbHandle) { @@ -523,6 +541,7 @@ export function buildContainer(env: Env): Container { getAffiliationLoader, getPostRepo, getPostService, + getFeedPresence, getNotificationService, getCounterStore, getCache, diff --git a/services/api/src/routes/posts.routes.ts b/services/api/src/routes/posts.routes.ts index dcc6f82c..dbc7f0e1 100644 --- a/services/api/src/routes/posts.routes.ts +++ b/services/api/src/routes/posts.routes.ts @@ -1,16 +1,27 @@ -import { HomeFeedQuerySchema, IdSchema, PaginationQuerySchema, PostComposeInputSchema } from "@civfix/shared" +import { + FeedCountsRequestSchema, + HomeFeedQuerySchema, + IdSchema, + PaginationQuerySchema, + PostComposeInputSchema, +} from "@civfix/shared" import { z } from "zod" -import type { FastifyInstance } from "fastify" +import type { FastifyInstance, FastifyRequest } from "fastify" import { perIdentity } from "../plugins/rate-limit.js" import type { Container } from "../di.js" import { requireAuth } from "../auth/context.js" +import { cfGeoFromTrustedEdge } from "../abuse/gps-sanity.js" +import { approximateLocationFor } from "../services/geo-approximate.js" +import type { FeedViewerLocation } from "../services/post-service.js" import { NIL_VIEWER_ID } from "../services/post-repository.drizzle.js" import { route } from "../versioning/route.js" import { parse } from "./_validate.js" const PostIdParamsSchema = z.object({ id: IdSchema }).strict() +export const FEED_COUNTS_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute" }) + export const CREATE_POST_RATE_LIMIT = perIdentity({ max: 120, timeWindow: "1 minute" }) export const POST_INTERACTION_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute" }) @@ -86,14 +97,39 @@ export async function registerPostRoutes(app: FastifyInstance, container: Contai reply.status(200).send(await service().unsavePost(id, userId)) }) + const viewerLocationOf = (request: FastifyRequest): FeedViewerLocation => { + const env = container.env + const approximate = approximateLocationFor( + { headers: request.headers, trusted: cfGeoFromTrustedEdge(request) }, + { lat: env.HOME_REGION_LAT, lng: env.HOME_REGION_LNG, radiusKm: env.HOME_REGION_RADIUS_KM }, + ) + return { lat: approximate.lat, lng: approximate.lng } + } + route(app, "homeFeed", async (request, reply) => { const userId = request.auth.userId const query = parse(HomeFeedQuerySchema, request.query) + const location = viewerLocationOf(request) reply .status(200) - .send(userId ? await service().homeFeed(userId, query) : await service().publicFeed(query)) + .send( + userId + ? await service().homeFeed(userId, query, location) + : await service().publicFeed(query, location), + ) }) + route( + app, + "getFeedCounts", + { config: { rateLimit: FEED_COUNTS_RATE_LIMIT } }, + async (request, reply) => { + const userId = requireAuth(request) + const { postIds } = parse(FeedCountsRequestSchema, request.body) + reply.status(200).send(await service().getFeedCounts(postIds, userId)) + }, + ) + route(app, "listUserPosts", async (request, reply) => { const userId = request.auth.userId ?? NIL_VIEWER_ID const { id } = parse(PostIdParamsSchema, request.params) diff --git a/services/api/src/services/feed-presence.ts b/services/api/src/services/feed-presence.ts new file mode 100644 index 00000000..87df1175 --- /dev/null +++ b/services/api/src/services/feed-presence.ts @@ -0,0 +1,143 @@ +import type { FeedRankingConfig } from "@civfix/shared" +import type { RankedCandidate } from "./feed-ranking.js" + +export interface FeedPresenceCache { + get(key: string): Promise + set(key: string, value: string, ttlSeconds: number): Promise + sadd(key: string, ...members: string[]): Promise + smembers(key: string): Promise + scard(key: string): Promise + expire(key: string, ttlSeconds: number): Promise +} + +export interface FeedPresenceDeps { + cache?: FeedPresenceCache + config: FeedRankingConfig + logger?: { warn(obj: unknown, msg?: string): void; debug?(obj: unknown, msg?: string): void } +} + +export interface FeedSnapshotEntry { + id: string + score: number +} + +export interface FeedPresence { + readSnapshot(userId: string, filter: string): Promise + writeSnapshot(userId: string, filter: string, ranked: readonly RankedCandidate[]): Promise + seenBy(userId: string, postIds: readonly string[]): Promise> + recordServed(userId: string, postIds: readonly string[]): Promise + viewersOf(postId: string): Promise +} + +export function snapshotKey(userId: string, filter: string): string { + return `feed:rank:v1:${userId}:${filter}` +} + +export function servedKey(userId: string): string { + return `feed:served:v1:${userId}` +} + +export function viewersKey(postId: string): string { + return `feed:viewers:v1:${postId}` +} + +function isSnapshotEntry(value: unknown): value is [string, number] { + return ( + Array.isArray(value) && + value.length === 2 && + typeof value[0] === "string" && + typeof value[1] === "number" && + Number.isFinite(value[1]) + ) +} + +export function makeFeedPresence(deps: FeedPresenceDeps): FeedPresence { + const cfg = deps.config + + async function guarded(fallback: T, label: string, run: () => Promise): Promise { + if (deps.cache === undefined) return fallback + try { + return await run() + } catch (err) { + deps.logger?.warn({ err }, `feed presence: ${label} failed (ignored)`) + return fallback + } + } + + return { + readSnapshot(userId: string, filter: string): Promise { + return guarded(null, "snapshot read", async () => { + const raw = await deps.cache!.get(snapshotKey(userId, filter)) + if (raw === null) return null + const parsed: unknown = JSON.parse(raw) + if (!Array.isArray(parsed)) return null + const entries: FeedSnapshotEntry[] = [] + for (const value of parsed) { + if (!isSnapshotEntry(value)) return null + entries.push({ id: value[0], score: value[1] }) + } + return entries + }) + }, + + writeSnapshot( + userId: string, + filter: string, + ranked: readonly RankedCandidate[], + ): Promise { + if (cfg.snapshotTtlSeconds <= 0) return Promise.resolve() + return guarded(undefined, "snapshot write", async () => { + const payload = ranked + .slice(0, cfg.candidateCap) + .map((entry) => [entry.id, entry.score] as const) + await deps.cache!.set( + snapshotKey(userId, filter), + JSON.stringify(payload), + cfg.snapshotTtlSeconds, + ) + }) + }, + + seenBy(userId: string, postIds: readonly string[]): Promise> { + if (postIds.length === 0) return Promise.resolve(new Set()) + return guarded(new Set(), "served-set read", async () => { + const members = await deps.cache!.smembers(servedKey(userId)) + const served = new Set(members) + return new Set(postIds.filter((id) => served.has(id))) + }) + }, + + recordServed(userId: string, postIds: readonly string[]): Promise { + if (postIds.length === 0) return Promise.resolve() + const unique = [...new Set(postIds)] + return guarded(undefined, "served-set write", async () => { + const cache = deps.cache! + await cache.sadd(servedKey(userId), ...unique) + await cache.expire(servedKey(userId), cfg.servedTtlSeconds) + for (const postId of unique) { + const key = viewersKey(postId) + if ((await cache.scard(key)) >= cfg.viewerFanoutMax) continue + await cache.sadd(key, userId) + await cache.expire(key, cfg.servedTtlSeconds) + } + }) + }, + + viewersOf(postId: string): Promise { + return guarded([], "viewer index read", async () => { + const cache = deps.cache! + const key = viewersKey(postId) + const size = await cache.scard(key) + if (size === 0) return [] + if (size > cfg.viewerFanoutMax) { + deps.logger?.debug?.( + { postId, size, cap: cfg.viewerFanoutMax }, + "feed presence: viewer fanout skipped (over cap)", + ) + return [] + } + return cache.smembers(key) + }) + }, + } +} diff --git a/services/api/src/services/feed-ranking.ts b/services/api/src/services/feed-ranking.ts new file mode 100644 index 00000000..7e9f5f56 --- /dev/null +++ b/services/api/src/services/feed-ranking.ts @@ -0,0 +1,122 @@ +import { quantizeFeedScore, type FeedRankingConfig } from "@civfix/shared" + +export interface FeedCandidate { + id: string + authorId: string + createdAtMs: number + likeCount: number + replyCount: number + repostCount: number + hasMedia: boolean + hasReport: boolean + hasLiveEvent: boolean + authorFollowed: boolean + authorIsViewer: boolean + viewerMentioned: boolean + authorOrgVerified: boolean + distanceKm: number | null + alreadySeen: boolean +} + +export interface RankedCandidate { + id: string + authorId: string + score: number +} + +const MS_PER_HOUR = 3_600_000 + +export function quantizeClock(nowMs: number, clockBucketSeconds: number): number { + const bucketMs = Math.max(1, Math.trunc(clockBucketSeconds)) * 1000 + return Math.floor(nowMs / bucketMs) * bucketMs +} + +export function proximity(distanceKm: number | null, radiusKm: number): number { + if (distanceKm === null || !Number.isFinite(distanceKm)) return 0 + if (radiusKm <= 0) return 0 + return Math.max(0, 1 - Math.max(0, distanceKm) / radiusKm) +} + +export function recency(ageHours: number, cfg: FeedRankingConfig): number { + const age = Math.max(0, ageHours) + const decayed = 0.5 ** (age / cfg.halfLifeHours) + return cfg.decayFloor + (1 - cfg.decayFloor) * decayed +} + +export function diversityMultiplier(authorRank: number, cfg: FeedRankingConfig): number { + const rank = Math.max(0, Math.trunc(authorRank)) + return cfg.diversityFloor + (1 - cfg.diversityFloor) * cfg.diversityDecay ** rank +} + +function logScale(count: number): number { + return Math.log1p(Math.max(0, count)) +} + +export function rawScore(candidate: FeedCandidate, cfg: FeedRankingConfig): number { + return ( + cfg.baseWeight + + (candidate.authorFollowed ? cfg.followWeight : 0) + + (candidate.authorIsViewer ? cfg.selfWeight : 0) + + (candidate.viewerMentioned ? cfg.mentionWeight : 0) + + (candidate.authorOrgVerified ? cfg.orgVerifiedWeight : 0) + + (candidate.hasLiveEvent ? cfg.attachEventWeight : 0) + + (candidate.hasReport ? cfg.attachReportWeight : 0) + + (candidate.hasMedia ? cfg.imageWeight : 0) + + cfg.nearbyWeight * proximity(candidate.distanceKm, cfg.nearbyRadiusKm) + + cfg.likeWeight * logScale(candidate.likeCount) + + cfg.replyWeight * logScale(candidate.replyCount) + + cfg.repostWeight * logScale(candidate.repostCount) + ) +} + +export function scoreCandidate( + candidate: FeedCandidate, + cfg: FeedRankingConfig, + nowBucketMs: number, +): number { + const ageHours = Math.max(0, (nowBucketMs - candidate.createdAtMs) / MS_PER_HOUR) + const seen = candidate.alreadySeen ? cfg.seenDiscount : 1 + return Math.max(0, rawScore(candidate, cfg)) * recency(ageHours, cfg) * seen +} + +function byScoreThenId(a: RankedCandidate, b: RankedCandidate): number { + if (a.score !== b.score) return b.score - a.score + return a.id < b.id ? 1 : a.id > b.id ? -1 : 0 +} + +export function rankCandidates( + candidates: readonly FeedCandidate[], + cfg: FeedRankingConfig, + nowMs: number, +): RankedCandidate[] { + const nowBucketMs = quantizeClock(nowMs, cfg.clockBucketSeconds) + const base = candidates.map((candidate) => ({ + id: candidate.id, + authorId: candidate.authorId, + score: quantizeFeedScore(scoreCandidate(candidate, cfg, nowBucketMs)), + })) + base.sort(byScoreThenId) + + const seenPerAuthor = new Map() + const discounted = base.map((entry) => { + const rank = seenPerAuthor.get(entry.authorId) ?? 0 + seenPerAuthor.set(entry.authorId, rank + 1) + return { + id: entry.id, + authorId: entry.authorId, + score: quantizeFeedScore(entry.score * diversityMultiplier(rank, cfg)), + } + }) + discounted.sort(byScoreThenId) + return discounted +} + +export function applyCutoff( + ranked: readonly RankedCandidate[], + cfg: FeedRankingConfig, + isFirstPage: boolean, +): RankedCandidate[] { + const kept = ranked.filter((entry) => entry.score >= cfg.minScore) + if (isFirstPage && kept.length < cfg.minPageItems) return [...ranked] + return kept +} diff --git a/services/api/src/services/post-repository.drizzle.ts b/services/api/src/services/post-repository.drizzle.ts index 675c71b1..e1a87ae5 100644 --- a/services/api/src/services/post-repository.drizzle.ts +++ b/services/api/src/services/post-repository.drizzle.ts @@ -97,6 +97,58 @@ export interface HomeFeedArgs extends PostListArgs { filter: "all" | "events" | "fixes" } +export type FeedFilter = "all" | "events" | "fixes" + +export const FEED_IN_NETWORK_POOL = 250 +export const FEED_NEARBY_POOL = 150 +export const FEED_RECENT_POOL = 150 + +export const FEED_GEOM_INDEX = "posts_geom_gist" +export const FEED_AUTHOR_INDEX = "posts_author_public_recent_idx" + +const KM_PER_DEGREE_LAT = 111.32 + +const MIN_LATITUDE_COSINE = 0.25 + +export function nearbyRadiusDegrees(radiusKm: number): number { + return Math.min(90, radiusKm / KM_PER_DEGREE_LAT / MIN_LATITUDE_COSINE) +} + +export interface FeedCandidateArgs { + viewerId: string + filter: FeedFilter + fallbackLat: number | null + fallbackLng: number | null + windowDays: number + radiusKm: number + candidateCap: number +} + +export interface FeedCandidateRow { + id: string + author_id: string + created_at: Date + like_count: number + reply_count: number + repost_count: number + has_report: boolean + has_live_event: boolean + has_media: boolean + author_followed: boolean + author_is_viewer: boolean + viewer_mentioned: boolean + author_org_verified: boolean + distance_km: number | null +} + +export interface FeedCountsRow { + id: string + like_count: number + repost_count: number + reply_count: number + save_count: number +} + export interface PublicFeedArgs { filter: "all" | "events" | "fixes" cursor: string | null @@ -121,8 +173,12 @@ export interface PostRepository { unrepost(postId: string, userId: string): Promise<{ targetId: string; removed: boolean }> getPostDTO(id: string, viewerId: string): Promise - homeFeed(args: HomeFeedArgs): Promise + homeFeedChronological(args: HomeFeedArgs): Promise publicFeed(args: PublicFeedArgs): Promise + feedCandidates(args: FeedCandidateArgs): Promise + hydrateByIds(ids: readonly string[], viewerId: string): Promise + followerIdsOf(authorId: string, limit: number): Promise + readableCounts(postIds: readonly string[], viewerId: string): Promise listReplies(postId: string, args: ReplyListArgs): Promise listUserPosts(authorId: string, args: PostListArgs): Promise listSaves(args: PostListArgs): Promise @@ -276,6 +332,153 @@ export async function tombstonePostInTx(tx: Queryable, postId: string): Promise< return true } +function feedFilterClause(sql: Queryable, filter: FeedFilter): postgres.Fragment { + if (filter === "events") return sql`AND p.event_id IS NOT NULL` + if (filter === "fixes") { + return sql`AND EXISTS (SELECT 1 FROM reports fr WHERE fr.id = p.report_id AND fr.status = 'resolved')` + } + return sql`` +} + +export function feedCandidatesStatement( + sql: Queryable, + args: FeedCandidateArgs, +): postgres.Fragment { + const viewerId = args.viewerId + const filterClause = feedFilterClause(sql, args.filter) + const eligible = (): postgres.Fragment => sql` + p.deleted_at IS NULL + AND p.reply_to_id IS NULL + AND p.visibility = 'public' + AND p.created_at >= (SELECT since FROM eligible_window) + ${filterClause} + ` + return sql` + WITH viewer_point AS ( + SELECT p.geom FROM ( + SELECT r.geom, r.created_at FROM reports r + WHERE r.reporter_user_id = ${viewerId} AND r.deleted_at IS NULL + UNION ALL + SELECT c.geom, c.created_at FROM cleanups c + WHERE c.organizer_user_id = ${viewerId} + UNION ALL + SELECT c.geom, c.created_at + FROM cleanups c JOIN cleanup_members m ON m.cleanup_id = c.id + WHERE m.user_id = ${viewerId} + ) p + ORDER BY p.created_at DESC NULLS LAST + LIMIT 1 + ), + point AS ( + SELECT COALESCE( + (SELECT geom FROM viewer_point), + ST_SetSRID( + ST_MakePoint(${args.fallbackLng}::double precision, ${args.fallbackLat}::double precision), + 4326 + ) + ) AS geom + ), + eligible_window AS ( + SELECT (now() - ${args.windowDays}::int * interval '1 day') AS since + ), + in_network AS ( + SELECT p.id + FROM posts p + WHERE ${eligible()} + AND ( + p.author_id = ${viewerId} + OR p.author_id IN (SELECT followee_id FROM follows_people WHERE follower_id = ${viewerId}) + ) + ORDER BY p.created_at DESC, p.id DESC + LIMIT ${FEED_IN_NETWORK_POOL} + ), + nearby AS ( + SELECT n.id + FROM point vp + CROSS JOIN LATERAL ( + SELECT p.id + FROM posts p + WHERE ${eligible()} + AND p.geom IS NOT NULL + AND ST_DWithin(p.geom, vp.geom, ${nearbyRadiusDegrees(args.radiusKm)}) + ORDER BY p.geom <-> vp.geom + LIMIT ${FEED_NEARBY_POOL} + ) n + ), + recent_public AS ( + SELECT p.id + FROM posts p + WHERE ${eligible()} + ORDER BY p.created_at DESC, p.id DESC + LIMIT ${FEED_RECENT_POOL} + ), + pool AS ( + SELECT id FROM in_network + UNION SELECT id FROM nearby + UNION SELECT id FROM recent_public + ) + SELECT + p.id, + p.author_id, + p.created_at, + p.like_count, + p.reply_count, + p.repost_count, + (p.report_id IS NOT NULL) AS has_report, + (p.event_id IS NOT NULL AND ev.ends_at >= now()) AS has_live_event, + EXISTS ( + SELECT 1 FROM media_assets ma + WHERE ma.post_id = p.id AND ma.status = 'ready' AND ma.served_key IS NOT NULL + ) AS has_media, + EXISTS ( + SELECT 1 FROM follows_people f + WHERE f.follower_id = ${viewerId} AND f.followee_id = p.author_id + ) AS author_followed, + (p.author_id = ${viewerId}) AS author_is_viewer, + EXISTS ( + SELECT 1 FROM post_mentions pm + WHERE pm.post_id = p.id AND pm.mentioned_user_id = ${viewerId} + ) AS viewer_mentioned, + COALESCE(po.verified_status = 'verified', aff.verified, false) AS author_org_verified, + CASE WHEN p.geom IS NULL THEN NULL + ELSE ST_Distance(p.geom::geography, (SELECT geom FROM point)::geography) / 1000.0 + END AS distance_km + FROM posts p + JOIN pool ON pool.id = p.id + LEFT JOIN cleanups ev ON ev.id = p.event_id + LEFT JOIN organizations po + ON po.id = p.organization_id AND po.deleted_at IS NULL AND po.suspended_at IS NULL + LEFT JOIN LATERAL ( + SELECT (o.verified_status = 'verified') AS verified + FROM organization_members m + JOIN organizations o ON o.id = m.organization_id + JOIN users au ON au.id = m.user_id + WHERE m.user_id = p.author_id + AND au.deleted_at IS NULL + AND o.deleted_at IS NULL + AND o.suspended_at IS NULL + ORDER BY COALESCE(o.id = au.primary_organization_id, false) DESC, m.joined_at ASC, o.id ASC + LIMIT 1 + ) aff ON true + WHERE NOT EXISTS ( + SELECT 1 FROM user_blocks b + WHERE (b.blocker_id = ${viewerId} AND b.blocked_id = p.author_id) + OR (b.blocker_id = p.author_id AND b.blocked_id = ${viewerId}) + ) + LIMIT ${args.candidateCap} + ` +} + +export async function explainFeedCandidates( + sql: Queryable, + args: FeedCandidateArgs, +): Promise { + const rows = await sql[]>` + EXPLAIN (COSTS OFF, VERBOSE) ${feedCandidatesStatement(sql, args)} + ` + return rows.map((r) => Object.values(r)[0] ?? "").join("\n") +} + export function makeDrizzlePostRepository(sql: Sql, deps: PostRepoDeps): PostRepository { async function loadAffiliations( userIds: string[], @@ -848,11 +1051,15 @@ export function makeDrizzlePostRepository(sql: Sql, deps: PostRepoDeps): PostRep const inserted = await tx<{ id: string }[]>` INSERT INTO posts ( author_id, kind, body, reply_to_id, thread_root_id, repost_of_id, event_id, report_id, - organization_id + organization_id, geom ) VALUES ( ${args.authorId}, ${args.kind}, ${args.body}, ${replyToId}, ${threadRootId}, - ${repostOfId}, ${args.eventId}, ${args.reportId}, ${args.organizationId ?? null} + ${repostOfId}, ${args.eventId}, ${args.reportId}, ${args.organizationId ?? null}, + COALESCE( + (SELECT r.geom FROM reports r WHERE r.id = ${args.reportId}), + (SELECT c.geom FROM cleanups c WHERE c.id = ${args.eventId}) + ) ) RETURNING id ` @@ -990,7 +1197,51 @@ export function makeDrizzlePostRepository(sql: Sql, deps: PostRepoDeps): PostRep return hydrated[0] ?? null }, - async homeFeed(args: HomeFeedArgs): Promise { + async feedCandidates(args: FeedCandidateArgs): Promise { + return sql`${feedCandidatesStatement(sql, args)}` + }, + + async hydrateByIds(ids: readonly string[], viewerId: string): Promise { + if (ids.length === 0) return [] + const rows = await sql` + SELECT ${postColumns(sql)} + FROM posts p + WHERE p.id = ANY(${[...ids]}::uuid[]) + AND p.deleted_at IS NULL + AND p.visibility = 'public' + ` + const order = new Map(ids.map((id, index) => [id, index])) + rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0)) + return hydrate(rows, viewerId) + }, + + async followerIdsOf(authorId: string, limit: number): Promise { + if (limit <= 0) return [] + const rows = await sql<{ follower_id: string }[]>` + SELECT follower_id FROM follows_people + WHERE followee_id = ${authorId} + LIMIT ${limit} + ` + return rows.map((r) => r.follower_id) + }, + + async readableCounts(postIds: readonly string[], viewerId: string): Promise { + if (postIds.length === 0) return [] + return sql` + SELECT p.id, p.like_count, p.repost_count, p.reply_count, p.save_count + FROM posts p + WHERE p.id = ANY(${[...postIds]}::uuid[]) + AND p.deleted_at IS NULL + AND p.visibility = 'public' + AND NOT EXISTS ( + SELECT 1 FROM user_blocks b + WHERE (b.blocker_id = ${viewerId} AND b.blocked_id = p.author_id) + OR (b.blocker_id = p.author_id AND b.blocked_id = ${viewerId}) + ) + ` + }, + + async homeFeedChronological(args: HomeFeedArgs): Promise { const cursor = parseTimeCursor(args.cursor) const cursorFilter = cursor !== null ? sql`AND (p.created_at, p.id) < (${cursor.at}, ${cursor.id}::uuid)` : sql`` diff --git a/services/api/src/services/post-service.ts b/services/api/src/services/post-service.ts index 675a5045..d5e55698 100644 --- a/services/api/src/services/post-service.ts +++ b/services/api/src/services/post-service.ts @@ -1,28 +1,62 @@ -import { AppError } from "@civfix/shared" -import type { HomeFeedQuery, PaginationQuery, PostComposeInput, PostDTO } from "@civfix/shared" +import { + AppError, + DEFAULT_FEED_RANKING, + formatFeedScoreCursor, + isAfterFeedScoreCursor, + parseFeedScoreCursor, +} from "@civfix/shared" +import type { + FeedCountsResponse, + FeedRankingConfig, + HomeFeedQuery, + PaginationQuery, + PostComposeInput, + PostDTO, +} from "@civfix/shared" +import type { UserChannel } from "@civfix/shared/interfaces" import type { Sql } from "../db/client.js" import { assertNoSlur } from "../abuse/slur-filter.js" import { resolveMentionTargets } from "./mention-resolver.drizzle.js" +import { parseTimeCursor } from "../db/cursor-helpers.js" import { + NIL_VIEWER_ID, POSTS_DEFAULT_LIMIT, + type FeedCandidateRow, type FeedPage, type PostBrief, type PostRepository, type RepliesPage, } from "./post-repository.drizzle.js" +import { applyCutoff, rankCandidates, type FeedCandidate } from "./feed-ranking.js" +import type { FeedPresence } from "./feed-presence.js" import type { PostNotifier } from "./notification-service.js" function isVisible(brief: PostBrief): boolean { return brief.deletedAt === null && brief.visibility === "public" } +function isLegacyTimeCursor(cursor: string | null | undefined): boolean { + if (cursor === null || cursor === undefined || cursor === "") return false + if (parseFeedScoreCursor(cursor) !== null) return false + return parseTimeCursor(cursor) !== null +} + +export interface FeedViewerLocation { + lat: number + lng: number +} + export interface PostServiceDeps { repo: PostRepository sql: Sql notifier?: PostNotifier isBlockedEitherWay?: (a: string, b: string) => Promise logger?: { warn(obj: unknown, msg: string): void } + feedRanking?: FeedRankingConfig + feedPresence?: FeedPresence + userChannel?: UserChannel + now?: () => number } export interface PostService { @@ -36,14 +70,139 @@ export interface PostService { unsavePost(id: string, viewerId: string): Promise repostPost(id: string, viewerId: string): Promise unrepostPost(id: string, viewerId: string): Promise - homeFeed(viewerId: string, query: HomeFeedQuery): Promise - publicFeed(query: HomeFeedQuery): Promise + homeFeed( + viewerId: string, + query: HomeFeedQuery, + location?: FeedViewerLocation, + ): Promise + publicFeed(query: HomeFeedQuery, location?: FeedViewerLocation): Promise + getFeedCounts(postIds: readonly string[], viewerId: string): Promise listUserPosts(authorId: string, viewerId: string, pagination: PaginationQuery): Promise listSaves(viewerId: string, pagination: PaginationQuery): Promise } export function makePostService(deps: PostServiceDeps): PostService { const isBlocked = deps.isBlockedEitherWay ?? (() => Promise.resolve(false)) + const feedConfig = deps.feedRanking ?? DEFAULT_FEED_RANKING + const nowMs = deps.now ?? (() => Date.now()) + + function fanout(recipients: readonly string[], topic: "feed" | "feed_counts", id: string): void { + const channel = deps.userChannel + if (channel === undefined || recipients.length === 0) return + void channel.publishToUsers(recipients, { topic, id }).catch((err: unknown) => { + deps.logger?.warn({ err, topic, id }, "post: realtime feed fanout failed (suppressed)") + }) + } + + async function announceNewPost(postId: string, authorId: string): Promise { + if (deps.userChannel === undefined) return + try { + const followers = await deps.repo.followerIdsOf(authorId, feedConfig.newPostFanoutMax) + fanout(followers, "feed", postId) + } catch (err) { + deps.logger?.warn({ err, postId }, "post: new-post fanout lookup failed (suppressed)") + } + } + + async function announceCountChange(postId: string, actorId: string): Promise { + if (deps.userChannel === undefined || deps.feedPresence === undefined) return + try { + const viewers = await deps.feedPresence.viewersOf(postId) + fanout( + viewers.filter((viewerId) => viewerId !== actorId), + "feed_counts", + postId, + ) + } catch (err) { + deps.logger?.warn({ err, postId }, "post: count fanout lookup failed (suppressed)") + } + } + + function toCandidate(row: FeedCandidateRow, seen: ReadonlySet): FeedCandidate { + return { + id: row.id, + authorId: row.author_id, + createdAtMs: row.created_at.getTime(), + likeCount: Number(row.like_count), + replyCount: Number(row.reply_count), + repostCount: Number(row.repost_count), + hasMedia: row.has_media, + hasReport: row.has_report, + hasLiveEvent: row.has_live_event, + authorFollowed: row.author_followed, + authorIsViewer: row.author_is_viewer, + viewerMentioned: row.viewer_mentioned, + authorOrgVerified: row.author_org_verified, + distanceKm: row.distance_km === null ? null : Number(row.distance_km), + alreadySeen: seen.has(row.id), + } + } + + async function rankedFeed( + viewerId: string, + query: HomeFeedQuery, + location: FeedViewerLocation | undefined, + ): Promise { + const limit = query.limit ?? POSTS_DEFAULT_LIMIT + const cursor = parseFeedScoreCursor(query.cursor) + const isFirstPage = cursor === null + + const rows = await deps.repo.feedCandidates({ + viewerId, + filter: query.filter, + fallbackLat: location?.lat ?? null, + fallbackLng: location?.lng ?? null, + windowDays: feedConfig.candidateWindowDays, + radiusKm: feedConfig.nearbyRadiusKm, + candidateCap: feedConfig.candidateCap, + }) + + const seen = + deps.feedPresence === undefined + ? new Set() + : await deps.feedPresence.seenBy( + viewerId, + rows.map((r) => r.id), + ) + + const ranked = rankCandidates( + rows.map((row) => toCandidate(row, seen)), + feedConfig, + nowMs(), + ) + const kept = applyCutoff(ranked, feedConfig, isFirstPage) + const after = + cursor === null + ? kept + : kept.filter((entry) => + isAfterFeedScoreCursor({ score: entry.score, postId: entry.id }, cursor), + ) + + const pageEntries = after.slice(0, limit) + const hasMore = after.length > pageEntries.length + const last = pageEntries[pageEntries.length - 1] + const nextCursor = + hasMore && last !== undefined + ? formatFeedScoreCursor({ score: last.score, postId: last.id }) + : null + + const items = await deps.repo.hydrateByIds( + pageEntries.map((entry) => entry.id), + viewerId, + ) + + if (deps.feedPresence !== undefined) { + if (isFirstPage) { + void deps.feedPresence.writeSnapshot(viewerId, query.filter, ranked) + } + void deps.feedPresence.recordServed( + viewerId, + items.map((item) => item.id), + ) + } + + return { items, nextCursor } + } async function safeNotify(fn: () => Promise): Promise { if (!deps.notifier) return @@ -158,6 +317,13 @@ export function makePostService(deps: PostServiceDeps): PostService { } } + if (kind === "post" || kind === "quote") { + await announceNewPost(postId, authorId) + } + if (replyParentAuthor !== null) { + await announceCountChange(input.replyToId!, authorId) + } + return hydrateOrThrow(postId, authorId) }, @@ -191,6 +357,7 @@ export function makePostService(deps: PostServiceDeps): PostService { async likePost(id: string, viewerId: string): Promise { const subject = await requireReadable(id, viewerId) const created = await deps.repo.like(subject.id, viewerId) + if (created) await announceCountChange(subject.id, viewerId) if (created && (await shouldNotify(viewerId, subject.authorId))) { const actorName = await deps.repo.actorNameOf(viewerId) await safeNotify(() => @@ -202,7 +369,8 @@ export function makePostService(deps: PostServiceDeps): PostService { async unlikePost(id: string, viewerId: string): Promise { const subject = await requireReadable(id, viewerId) - await deps.repo.unlike(subject.id, viewerId) + const removed = await deps.repo.unlike(subject.id, viewerId) + if (removed) await announceCountChange(subject.id, viewerId) return hydrateOrThrow(id, viewerId) }, @@ -224,6 +392,7 @@ export function makePostService(deps: PostServiceDeps): PostService { throw AppError.validation({ id: "You cannot repost your own post." }) } const { targetId, created } = await deps.repo.repost(id, viewerId) + if (created) await announceCountChange(targetId, viewerId) if (created) { const targetBrief = await deps.repo.getPostBrief(targetId) if (targetBrief && (await shouldNotify(viewerId, targetBrief.authorId))) { @@ -242,25 +411,56 @@ export function makePostService(deps: PostServiceDeps): PostService { async unrepostPost(id: string, viewerId: string): Promise { await requireReadable(id, viewerId) - const { targetId } = await deps.repo.unrepost(id, viewerId) + const { targetId, removed } = await deps.repo.unrepost(id, viewerId) + if (removed) await announceCountChange(targetId, viewerId) return hydrateOrThrow(targetId, viewerId) }, - async homeFeed(viewerId: string, query: HomeFeedQuery): Promise { - return deps.repo.homeFeed({ - viewerId, - filter: query.filter, - cursor: query.cursor ?? null, - limit: query.limit ?? POSTS_DEFAULT_LIMIT, - }) + async homeFeed( + viewerId: string, + query: HomeFeedQuery, + location?: FeedViewerLocation, + ): Promise { + if (isLegacyTimeCursor(query.cursor)) { + return deps.repo.homeFeedChronological({ + viewerId, + filter: query.filter, + cursor: query.cursor ?? null, + limit: query.limit ?? POSTS_DEFAULT_LIMIT, + }) + } + return rankedFeed(viewerId, query, location) }, - async publicFeed(query: HomeFeedQuery): Promise { - return deps.repo.publicFeed({ - filter: query.filter, - cursor: query.cursor ?? null, - limit: query.limit ?? POSTS_DEFAULT_LIMIT, - }) + async publicFeed(query: HomeFeedQuery, location?: FeedViewerLocation): Promise { + if (isLegacyTimeCursor(query.cursor)) { + return deps.repo.publicFeed({ + filter: query.filter, + cursor: query.cursor ?? null, + limit: query.limit ?? POSTS_DEFAULT_LIMIT, + }) + } + return rankedFeed(NIL_VIEWER_ID, query, location) + }, + + async getFeedCounts( + postIds: readonly string[], + viewerId: string, + ): Promise { + const unique = [...new Set(postIds)] + if (unique.length === 0) return { items: [] } + const rows = await deps.repo.readableCounts(unique, viewerId) + return { + items: rows.map((row) => ({ + id: row.id, + counts: { + likes: Number(row.like_count), + reposts: Number(row.repost_count), + replies: Number(row.reply_count), + saves: Number(row.save_count), + }, + })), + } }, async listUserPosts( diff --git a/services/api/test/unit/admin-guard-cache.test.ts b/services/api/test/unit/admin-guard-cache.test.ts index 93a38ee7..6805b786 100644 --- a/services/api/test/unit/admin-guard-cache.test.ts +++ b/services/api/test/unit/admin-guard-cache.test.ts @@ -67,6 +67,9 @@ class SpyCache implements CacheClient { smembers(key: string): Promise { return this.inner.smembers(key) } + scard(key: string): Promise { + return this.inner.scard(key) + } expire(key: string, ttlSeconds: number): Promise { return this.inner.expire(key, ttlSeconds) } diff --git a/services/api/test/unit/auth-session.test.ts b/services/api/test/unit/auth-session.test.ts index ce81360c..e20f4bb7 100644 --- a/services/api/test/unit/auth-session.test.ts +++ b/services/api/test/unit/auth-session.test.ts @@ -349,6 +349,9 @@ class FlakyDelCache implements CacheClient { smembers(key: string): Promise { return this.inner.smembers(key) } + scard(key: string): Promise { + return this.inner.scard(key) + } expire(key: string, ttlSeconds: number): Promise { return this.inner.expire(key, ttlSeconds) } diff --git a/services/api/test/unit/host-analytics.test.ts b/services/api/test/unit/host-analytics.test.ts index 0428ae6a..6d99590f 100644 --- a/services/api/test/unit/host-analytics.test.ts +++ b/services/api/test/unit/host-analytics.test.ts @@ -568,6 +568,7 @@ describe("host analytics cache", () => { sadd: () => Promise.resolve(0), srem: () => Promise.resolve(0), smembers: () => Promise.resolve([]), + scard: () => Promise.resolve(0), expire: () => Promise.resolve(), } const cache = makeHostAnalyticsCache({ cache: broken, ttlSeconds: 60 }) diff --git a/services/api/test/unit/post-service.test.ts b/services/api/test/unit/post-service.test.ts index 90473b1e..808e2e76 100644 --- a/services/api/test/unit/post-service.test.ts +++ b/services/api/test/unit/post-service.test.ts @@ -1,19 +1,23 @@ import { describe, expect, it } from "vitest" -import { AppError } from "@civfix/shared" +import { AppError, DEFAULT_FEED_RANKING } from "@civfix/shared" import type { PostComposeInput, PostDTO } from "@civfix/shared" import type { Sql } from "../../src/db/client.js" import { makePostService } from "../../src/services/post-service.js" import { makeFakeSql } from "../helpers/fake-sql.js" import { makeDrizzlePostRepository, + NIL_VIEWER_ID, type CreatePostArgs, + type FeedCandidateArgs, type PostBrief, type PostRepository, type ReplyListArgs, } from "../../src/services/post-repository.drizzle.js" import type { PostNotifier } from "../../src/services/notification-service.js" +const LEGACY_CURSOR_ID = "33333333-3333-3333-3333-333333333333" + const throwingSql = (() => { throw new Error("sql must not be called in these unit paths") }) as unknown as Sql @@ -81,8 +85,12 @@ function fakeRepo(cfg: FakeConfig = {}): PostRepository & { created: CreatePostA repost: (id) => Promise.resolve({ targetId: id, created: true }), unrepost: (id) => Promise.resolve({ targetId: id, removed: true }), getPostDTO: (id) => Promise.resolve(dto(id)), - homeFeed: () => Promise.resolve({ items: [], nextCursor: null }), + homeFeedChronological: () => Promise.resolve({ items: [], nextCursor: null }), publicFeed: () => Promise.resolve({ items: [], nextCursor: null }), + feedCandidates: () => Promise.resolve([]), + hydrateByIds: () => Promise.resolve([]), + followerIdsOf: () => Promise.resolve([]), + readableCounts: () => Promise.resolve([]), listReplies: () => Promise.resolve({ items: [], nextCursor: null, authorReplies: [] }), listUserPosts: () => Promise.resolve({ items: [], nextCursor: null }), listSaves: () => Promise.resolve({ items: [], nextCursor: null }), @@ -134,17 +142,77 @@ describe("PostService validation + authorization", () => { }) it("publicFeed serves the global feed with no viewer (you don't need an account to read)", async () => { - const calls: Array<{ filter: string; cursor: string | null; limit: number }> = [] + const calls: Array<{ viewerId: string; filter: string; candidateCap: number }> = [] const repo = { ...fakeRepo(), + feedCandidates: (args: FeedCandidateArgs) => { + calls.push({ + viewerId: args.viewerId, + filter: args.filter, + candidateCap: args.candidateCap, + }) + return Promise.resolve([]) + }, + } + const svc = makePostService({ repo, sql: throwingSql }) + await svc.publicFeed({ filter: "all" }) + expect(calls).toEqual([ + { viewerId: NIL_VIEWER_ID, filter: "all", candidateCap: DEFAULT_FEED_RANKING.candidateCap }, + ]) + }) + + it("routes a legacy ISO cursor to the untouched chronological query (the rollback lever)", async () => { + const ranked: string[] = [] + const chronological: Array<{ cursor: string | null; limit: number }> = [] + const repo = { + ...fakeRepo(), + feedCandidates: (args: FeedCandidateArgs) => { + ranked.push(args.viewerId) + return Promise.resolve([]) + }, publicFeed: (args: { filter: "all" | "events" | "fixes"; cursor: string | null; limit: number }) => { - calls.push(args) + chronological.push({ cursor: args.cursor, limit: args.limit }) + return Promise.resolve({ items: [], nextCursor: null }) + }, + homeFeedChronological: (args: { cursor: string | null; limit: number }) => { + chronological.push({ cursor: args.cursor, limit: args.limit }) return Promise.resolve({ items: [], nextCursor: null }) }, } const svc = makePostService({ repo, sql: throwingSql }) - await svc.publicFeed({ filter: "all" }) - expect(calls).toEqual([{ filter: "all", cursor: null, limit: 20 }]) + const legacy = `2026-09-14T12:00:00.000Z|${LEGACY_CURSOR_ID}` + + await svc.publicFeed({ filter: "all", cursor: legacy }) + await svc.homeFeed("viewer", { filter: "all", cursor: legacy }) + + expect(ranked).toEqual([]) + expect(chronological).toEqual([ + { cursor: legacy, limit: 20 }, + { cursor: legacy, limit: 20 }, + ]) + }) + + it("routes a score cursor and a garbage cursor to the ranked path", async () => { + const ranked: string[] = [] + const chronological: string[] = [] + const repo = { + ...fakeRepo(), + feedCandidates: (args: FeedCandidateArgs) => { + ranked.push(args.viewerId) + return Promise.resolve([]) + }, + publicFeed: () => { + chronological.push("publicFeed") + return Promise.resolve({ items: [], nextCursor: null }) + }, + } + const svc = makePostService({ repo, sql: throwingSql }) + + await svc.publicFeed({ filter: "all", cursor: `115.900000|${LEGACY_CURSOR_ID}` }) + await svc.publicFeed({ filter: "all", cursor: "not-a-cursor" }) + + expect(chronological).toEqual([]) + expect(ranked).toEqual([NIL_VIEWER_ID, NIL_VIEWER_ID]) }) it("rejects an attached event the author does not host/attend (403)", async () => { @@ -825,11 +893,26 @@ describe("PostRepository read paths all exclude non-public posts", () => { it("filters every post-listing query on visibility, not just the public feed", async () => { const cases: Array<[string, (r: ReturnType) => Promise]> = [ ["getPostDTO", (r) => r.getPostDTO(SUBJECT, VIEWER)], - ["homeFeed", (r) => r.homeFeed({ ...args, filter: "all" })], + ["homeFeedChronological", (r) => r.homeFeedChronological({ ...args, filter: "all" })], ["publicFeed", (r) => r.publicFeed({ filter: "all", cursor: null, limit: 10 })], ["listReplies", (r) => r.listReplies(SUBJECT, { ...args, focalAuthorId: VIEWER })], ["listUserPosts", (r) => r.listUserPosts(SUBJECT, args)], ["listSaves", (r) => r.listSaves(args)], + ["hydrateByIds", (r) => r.hydrateByIds([SUBJECT], VIEWER)], + ["readableCounts", (r) => r.readableCounts([SUBJECT], VIEWER)], + [ + "feedCandidates", + (r) => + r.feedCandidates({ + viewerId: VIEWER, + filter: "all", + fallbackLat: null, + fallbackLng: null, + windowDays: 30, + radiusKm: 40, + candidateCap: 400, + }), + ], ] for (const [name, run] of cases) { const stmt = await emitted(run) @@ -905,9 +988,10 @@ describe("PostRepository organization hydration", () => { const args = { viewerId: VIEWER, cursor: null, limit: 10 } const cases: Array<[string, (r: ReturnType) => Promise]> = [ ["getPostDTO", (r) => r.getPostDTO(POST, VIEWER)], - ["homeFeed", (r) => r.homeFeed({ ...args, filter: "all" })], + ["homeFeedChronological", (r) => r.homeFeedChronological({ ...args, filter: "all" })], ["publicFeed", (r) => r.publicFeed({ filter: "all", cursor: null, limit: 10 })], ["listReplies", (r) => r.listReplies(POST, { ...args, focalAuthorId: AUTHOR })], + ["hydrateByIds", (r) => r.hydrateByIds([POST], VIEWER)], ["listUserPosts", (r) => r.listUserPosts(AUTHOR, args)], ["listSaves", (r) => r.listSaves(args)], ] diff --git a/services/api/test/unit/posts-routes.test.ts b/services/api/test/unit/posts-routes.test.ts index ff502fbf..6b5373e6 100644 --- a/services/api/test/unit/posts-routes.test.ts +++ b/services/api/test/unit/posts-routes.test.ts @@ -30,6 +30,9 @@ import { paginate, parseTimeCursor } from "../../src/db/cursor-helpers.js" import { makePostService } from "../../src/services/post-service.js" import type { CreatePostArgs, + FeedCandidateArgs, + FeedCandidateRow, + FeedCountsRow, FeedPage, PostBrief, PostListArgs, @@ -291,10 +294,63 @@ class InMemoryPostRepository implements PostRepository { }) } - homeFeed(args: { viewerId: string; filter: "all" | "events" | "fixes"; cursor: string | null; limit: number }): Promise { + homeFeedChronological(args: { viewerId: string; filter: "all" | "events" | "fixes"; cursor: string | null; limit: number }): Promise { return Promise.resolve(this.page(this.byFilter(args.filter), args.viewerId, args)) } + feedCandidates(args: FeedCandidateArgs): Promise { + const rows = this.byFilter(args.filter).map((p) => ({ + id: p.id, + author_id: p.authorId, + created_at: p.createdAt, + like_count: p.likes.size, + reply_count: [...this.posts.values()].filter( + (r) => r.replyToId === p.id && r.deletedAt === null, + ).length, + repost_count: p.reposts.size, + has_report: p.reportId !== null, + has_live_event: p.eventId !== null, + has_media: false, + author_followed: false, + author_is_viewer: p.authorId === args.viewerId, + viewer_mentioned: p.mentionedUserIds.includes(args.viewerId), + author_org_verified: false, + distance_km: null, + })) + return Promise.resolve(rows.slice(0, args.candidateCap)) + } + + hydrateByIds(ids: readonly string[], viewerId: string): Promise { + const out: PostDTO[] = [] + for (const id of ids) { + const p = this.live(id) + if (p) out.push(this.dto(p, viewerId)) + } + return Promise.resolve(out) + } + + followerIdsOf(): Promise { + return Promise.resolve([]) + } + + readableCounts(postIds: readonly string[]): Promise { + const out: FeedCountsRow[] = [] + for (const id of postIds) { + const p = this.live(id) + if (!p) continue + out.push({ + id: p.id, + like_count: p.likes.size, + repost_count: p.reposts.size, + reply_count: [...this.posts.values()].filter( + (r) => r.replyToId === p.id && r.deletedAt === null, + ).length, + save_count: p.saves.size, + }) + } + return Promise.resolve(out) + } + publicFeed(args: { filter: "all" | "events" | "fixes"; cursor: string | null; limit: number }): Promise { // NIL viewer: every viewer flag must come back false for a signed-out reader. return Promise.resolve(this.page(this.byFilter(args.filter), "", args)) diff --git a/services/api/test/unit/rate-limit-plugin.test.ts b/services/api/test/unit/rate-limit-plugin.test.ts index e07f01bb..9dc677f7 100644 --- a/services/api/test/unit/rate-limit-plugin.test.ts +++ b/services/api/test/unit/rate-limit-plugin.test.ts @@ -56,7 +56,11 @@ import { GUEST_RSVP_CANCEL_RATE_LIMIT, GUEST_LIST_RATE_LIMIT, } from "../../src/routes/guest-rsvp.routes.js" -import { CREATE_POST_RATE_LIMIT, POST_INTERACTION_RATE_LIMIT } from "../../src/routes/posts.routes.js" +import { + CREATE_POST_RATE_LIMIT, + FEED_COUNTS_RATE_LIMIT, + POST_INTERACTION_RATE_LIMIT, +} from "../../src/routes/posts.routes.js" import { CLEANUP_MEMBERSHIP_RATE_LIMIT, EVENT_ICS_RATE_LIMIT, @@ -453,6 +457,7 @@ const IDENTITY_SCOPED_LIMITS = { ROUTE_REPORT_RATE_LIMIT, THREAD_READ_RATE_LIMIT, POST_INTERACTION_RATE_LIMIT, + FEED_COUNTS_RATE_LIMIT, PUSH_TOKEN_RATE_LIMIT, ADMIN_OUTBOUND_MAIL_RATE_LIMIT, FOLLOW_RATE_LIMIT, diff --git a/services/api/test/unit/route-coverage.test.ts b/services/api/test/unit/route-coverage.test.ts index bf9ceafc..7761a3a4 100644 --- a/services/api/test/unit/route-coverage.test.ts +++ b/services/api/test/unit/route-coverage.test.ts @@ -229,8 +229,8 @@ describe("route-coverage: every shared endpoint is registered (offline boot smok }) } - it("covers ALL 318 endpoints in the registry (no endpoint skipped)", () => { - expect(Object.keys(endpoints).length).toBe(318) + it("covers ALL 319 endpoints in the registry (no endpoint skipped)", () => { + expect(Object.keys(endpoints).length).toBe(319) }) it("the discriminator is not vacuous: a bogus path IS detected as route-missing", async () => { From 7d75fbc7727efde0d1b2cbf76a771561fc5a7f32 Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 15 Sep 2026 23:51:05 -0700 Subject: [PATCH 06/54] tests for the ranked feed: scorer, presence, candidate SQL, counts authz, env parsing --- .../test/integration/feed-ranked-pg.test.ts | 333 +++++++++++++++ .../api/test/unit/feed-candidates-sql.test.ts | 167 ++++++++ .../api/test/unit/feed-counts-service.test.ts | 99 +++++ services/api/test/unit/feed-presence.test.ts | 201 +++++++++ .../api/test/unit/feed-ranked-service.test.ts | 361 +++++++++++++++++ .../api/test/unit/feed-ranking-env.test.ts | 61 +++ services/api/test/unit/feed-ranking.test.ts | 380 ++++++++++++++++++ 7 files changed, 1602 insertions(+) create mode 100644 services/api/test/integration/feed-ranked-pg.test.ts create mode 100644 services/api/test/unit/feed-candidates-sql.test.ts create mode 100644 services/api/test/unit/feed-counts-service.test.ts create mode 100644 services/api/test/unit/feed-presence.test.ts create mode 100644 services/api/test/unit/feed-ranked-service.test.ts create mode 100644 services/api/test/unit/feed-ranking-env.test.ts create mode 100644 services/api/test/unit/feed-ranking.test.ts diff --git a/services/api/test/integration/feed-ranked-pg.test.ts b/services/api/test/integration/feed-ranked-pg.test.ts new file mode 100644 index 00000000..62de234f --- /dev/null +++ b/services/api/test/integration/feed-ranked-pg.test.ts @@ -0,0 +1,333 @@ + +import { afterAll, beforeAll, describe, expect, it } from "vitest" +import { DEFAULT_FEED_RANKING } from "@civfix/shared" +import { withPg, type PgHarness, testHandle } from "../helpers/pg.js" +import { + explainFeedCandidates, + makeDrizzlePostRepository, + type FeedCandidateArgs, + type PostRepository, +} from "../../src/services/post-repository.drizzle.js" +import { makePostService, type PostService } from "../../src/services/post-service.js" + +const pg = await withPg() + +const echoPresign = (r2Key: string, thumbKey: string | null) => + Promise.resolve( + thumbKey === null ? { url: `m://${r2Key}` } : { url: `m://${r2Key}`, thumbUrl: `m://${thumbKey}` }, + ) +const echoAvatar = (k: string) => Promise.resolve(`m://${k}`) + +const LA = { lat: 34.0522, lng: -118.2437 } + +describe.skipIf(!pg)("ranked home feed: candidate SQL (integration)", () => { + let h: PgHarness + + function makeRepo(): PostRepository { + return makeDrizzlePostRepository(h.sql, { + presignMedia: echoPresign, + presignAvatar: echoAvatar, + }) + } + + function makeService(repo: PostRepository = makeRepo()): PostService { + return makePostService({ repo, sql: h.sql, feedRanking: DEFAULT_FEED_RANKING }) + } + + function args(over: Partial = {}): FeedCandidateArgs { + return { + viewerId: "00000000-0000-0000-0000-000000000000", + filter: "all", + fallbackLat: LA.lat, + fallbackLng: LA.lng, + windowDays: DEFAULT_FEED_RANKING.candidateWindowDays, + radiusKm: DEFAULT_FEED_RANKING.nearbyRadiusKm, + candidateCap: DEFAULT_FEED_RANKING.candidateCap, + ...over, + } + } + + beforeAll(() => { + h = pg as PgHarness + }) + + afterAll(async () => { + await h.teardown() + }) + + async function newUser(name: string): Promise { + const [u] = await h.sql<{ id: string }[]>` + INSERT INTO users (display_name, handle) VALUES (${name}, ${testHandle()}) RETURNING id + ` + return u!.id + } + + async function follow(follower: string, followee: string): Promise { + await h.sql` + INSERT INTO follows_people (follower_id, followee_id) VALUES (${follower}, ${followee}) + ON CONFLICT DO NOTHING + ` + } + + async function block(blocker: string, blocked: string): Promise { + await h.sql` + INSERT INTO user_blocks (blocker_id, blocked_id) VALUES (${blocker}, ${blocked}) + ON CONFLICT DO NOTHING + ` + } + + async function setGeom(postId: string, lat: number, lng: number): Promise { + await h.sql` + UPDATE posts SET geom = ST_SetSRID(ST_MakePoint(${lng}, ${lat}), 4326) WHERE id = ${postId} + ` + } + + it("returns the in-network pool for a viewer who follows someone", async () => { + const viewer = await newUser("Net Viewer") + const followee = await newUser("Net Followee") + await follow(viewer, followee) + const svc = makeService() + const post = await svc.createPost( + { kind: "post", body: "in network", mediaUploadIds: [], mentionedUserIds: [] }, + followee, + ) + + const rows = await makeRepo().feedCandidates(args({ viewerId: viewer })) + const row = rows.find((r) => r.id === post.id) + expect(row).toBeDefined() + expect(row!.author_followed).toBe(true) + expect(row!.author_is_viewer).toBe(false) + }) + + it("marks the viewer's own post and excludes replies from the pool", async () => { + const viewer = await newUser("Self Viewer") + const svc = makeService() + const post = await svc.createPost( + { kind: "post", body: "mine", mediaUploadIds: [], mentionedUserIds: [] }, + viewer, + ) + const reply = await svc.createPost( + { kind: "post", body: "a reply", replyToId: post.id, mediaUploadIds: [], mentionedUserIds: [] }, + viewer, + ) + + const rows = await makeRepo().feedCandidates(args({ viewerId: viewer })) + expect(rows.find((r) => r.id === post.id)!.author_is_viewer).toBe(true) + expect(rows.find((r) => r.id === reply.id)).toBeUndefined() + }) + + it("excludes a blocked pair in BOTH directions", async () => { + const viewer = await newUser("Block Viewer") + const other = await newUser("Block Other") + const svc = makeService() + const theirPost = await svc.createPost( + { kind: "post", body: "theirs", mediaUploadIds: [], mentionedUserIds: [] }, + other, + ) + + await block(viewer, other) + let rows = await makeRepo().feedCandidates(args({ viewerId: viewer })) + expect(rows.find((r) => r.id === theirPost.id)).toBeUndefined() + + await h.sql`DELETE FROM user_blocks WHERE blocker_id = ${viewer} AND blocked_id = ${other}` + await block(other, viewer) + rows = await makeRepo().feedCandidates(args({ viewerId: viewer })) + expect(rows.find((r) => r.id === theirPost.id)).toBeUndefined() + }) + + it("excludes a soft-deleted post and a non-public one", async () => { + const viewer = await newUser("Vis Viewer") + const author = await newUser("Vis Author") + const svc = makeService() + const deleted = await svc.createPost( + { kind: "post", body: "gone", mediaUploadIds: [], mentionedUserIds: [] }, + author, + ) + const hidden = await svc.createPost( + { kind: "post", body: "hidden", mediaUploadIds: [], mentionedUserIds: [] }, + author, + ) + await svc.deletePost(deleted.id, author) + await h.sql`UPDATE posts SET visibility = 'hidden' WHERE id = ${hidden.id}` + + const rows = await makeRepo().feedCandidates(args({ viewerId: viewer })) + expect(rows.find((r) => r.id === deleted.id)).toBeUndefined() + expect(rows.find((r) => r.id === hidden.id)).toBeUndefined() + }) + + it("computes distance_km against the viewer point and NULL without geometry", async () => { + const viewer = await newUser("Geo Viewer") + const author = await newUser("Geo Author") + const svc = makeService() + const near = await svc.createPost( + { kind: "post", body: "near", mediaUploadIds: [], mentionedUserIds: [] }, + author, + ) + const nowhere = await svc.createPost( + { kind: "post", body: "nowhere", mediaUploadIds: [], mentionedUserIds: [] }, + author, + ) + await setGeom(near.id, LA.lat + 0.05, LA.lng) + + const rows = await makeRepo().feedCandidates(args({ viewerId: viewer })) + const nearRow = rows.find((r) => r.id === near.id)! + expect(Number(nearRow.distance_km)).toBeGreaterThan(4) + expect(Number(nearRow.distance_km)).toBeLessThan(7) + expect(rows.find((r) => r.id === nowhere.id)!.distance_km).toBeNull() + }) + + it("leaves distance NULL for every post when the caller has no location at all", async () => { + const viewer = await newUser("NoGeo Viewer") + const author = await newUser("NoGeo Author") + const svc = makeService() + const post = await svc.createPost( + { kind: "post", body: "somewhere", mediaUploadIds: [], mentionedUserIds: [] }, + author, + ) + await setGeom(post.id, LA.lat, LA.lng) + + const rows = await makeRepo().feedCandidates( + args({ viewerId: viewer, fallbackLat: null, fallbackLng: null }), + ) + expect(rows.find((r) => r.id === post.id)!.distance_km).toBeNull() + }) + + it("denormalises geom from the linked report at insert time", async () => { + const author = await newUser("Report Author") + const [report] = await h.sql<{ id: string }[]>` + INSERT INTO reports (reporter_user_id, category, type, description, geom, geom_source, status, visibility) + VALUES ( + ${author}, 'graffiti', 'graffiti', 'tag', + ST_SetSRID(ST_MakePoint(${LA.lng}, ${LA.lat}), 4326), 'manual', 'published', 'public' + ) + RETURNING id + ` + const svc = makeService() + const post = await svc.createPost( + { + kind: "post", + body: "about this report", + reportId: report!.id, + mediaUploadIds: [], + mentionedUserIds: [], + }, + author, + ) + + const [row] = await h.sql<{ lat: number; lng: number }[]>` + SELECT ST_Y(geom) AS lat, ST_X(geom) AS lng FROM posts WHERE id = ${post.id} + ` + expect(Number(row!.lat)).toBeCloseTo(LA.lat, 5) + expect(Number(row!.lng)).toBeCloseTo(LA.lng, 5) + }) + + it("respects the candidate window, excluding an older post", async () => { + const viewer = await newUser("Window Viewer") + const author = await newUser("Window Author") + const svc = makeService() + const old = await svc.createPost( + { kind: "post", body: "ancient", mediaUploadIds: [], mentionedUserIds: [] }, + author, + ) + await h.sql`UPDATE posts SET created_at = now() - interval '90 days' WHERE id = ${old.id}` + + const rows = await makeRepo().feedCandidates(args({ viewerId: viewer, windowDays: 30 })) + expect(rows.find((r) => r.id === old.id)).toBeUndefined() + }) + + it("honours the candidate cap", async () => { + const viewer = await newUser("Cap Viewer") + const author = await newUser("Cap Author") + const svc = makeService() + for (let i = 0; i < 6; i += 1) { + await svc.createPost( + { kind: "post", body: `cap ${i}`, mediaUploadIds: [], mentionedUserIds: [] }, + author, + ) + } + const rows = await makeRepo().feedCandidates(args({ viewerId: viewer, candidateCap: 3 })) + expect(rows.length).toBeLessThanOrEqual(3) + }) + + it("plans the nearby pool as an index scan, not a sequential scan", async () => { + const viewer = await newUser("Explain Viewer") + const plan = await explainFeedCandidates(h.sql, args({ viewerId: viewer })) + expect(plan).toContain("posts_geom_gist") + expect(plan).not.toMatch(/Seq Scan on posts/) + }) + + it("pages the ranked feed over every seeded post exactly once", async () => { + const viewer = await newUser("Page Viewer") + const author = await newUser("Page Author") + await follow(viewer, author) + const svc = makeService() + const created: string[] = [] + for (let i = 0; i < 25; i += 1) { + const post = await svc.createPost( + { kind: "post", body: `page ${i}`, mediaUploadIds: [], mentionedUserIds: [] }, + author, + ) + created.push(post.id) + } + + const seen: string[] = [] + let cursor: string | undefined + for (let page = 0; page < 6; page += 1) { + const result = await svc.homeFeed( + viewer, + { filter: "all", limit: 10, ...(cursor === undefined ? {} : { cursor }) }, + LA, + ) + seen.push(...result.items.map((item) => item.id)) + if (result.nextCursor === null) break + cursor = result.nextCursor + } + + const mine = seen.filter((id) => created.includes(id)) + expect(new Set(mine).size).toBe(mine.length) + expect(new Set(mine).size).toBe(created.length) + }) + + it("serves a legacy ISO cursor through the untouched chronological query", async () => { + const viewer = await newUser("Legacy Viewer") + const author = await newUser("Legacy Author") + await follow(viewer, author) + const svc = makeService() + for (let i = 0; i < 3; i += 1) { + await svc.createPost( + { kind: "post", body: `legacy ${i}`, mediaUploadIds: [], mentionedUserIds: [] }, + author, + ) + } + + const legacyCursor = `${new Date().toISOString()}|ffffffff-ffff-ffff-ffff-ffffffffffff` + const page = await svc.homeFeed(viewer, { filter: "all", limit: 10, cursor: legacyCursor }, LA) + expect(page.items.length).toBeGreaterThan(0) + }) + + it("returns counts only for posts the caller may read", async () => { + const viewer = await newUser("Counts Viewer") + const author = await newUser("Counts Author") + const blocker = await newUser("Counts Blocker") + const svc = makeService() + const readable = await svc.createPost( + { kind: "post", body: "readable", mediaUploadIds: [], mentionedUserIds: [] }, + author, + ) + const blocked = await svc.createPost( + { kind: "post", body: "blocked", mediaUploadIds: [], mentionedUserIds: [] }, + blocker, + ) + const hidden = await svc.createPost( + { kind: "post", body: "hidden", mediaUploadIds: [], mentionedUserIds: [] }, + author, + ) + await h.sql`UPDATE posts SET visibility = 'hidden' WHERE id = ${hidden.id}` + await block(blocker, viewer) + await svc.likePost(readable.id, viewer) + + const result = await svc.getFeedCounts([readable.id, blocked.id, hidden.id], viewer) + expect(result.items.map((item) => item.id)).toEqual([readable.id]) + expect(result.items[0]!.counts.likes).toBe(1) + }) +}) diff --git a/services/api/test/unit/feed-candidates-sql.test.ts b/services/api/test/unit/feed-candidates-sql.test.ts new file mode 100644 index 00000000..c97a1fec --- /dev/null +++ b/services/api/test/unit/feed-candidates-sql.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, it } from "vitest" +import { makeFakeSql } from "../helpers/fake-sql.js" +import type { Sql } from "../../src/db/client.js" +import { + FEED_IN_NETWORK_POOL, + FEED_NEARBY_POOL, + FEED_RECENT_POOL, + makeDrizzlePostRepository, + nearbyRadiusDegrees, + type FeedCandidateArgs, +} from "../../src/services/post-repository.drizzle.js" + +const VIEWER = "11111111-1111-1111-1111-111111111111" + +const ARGS: FeedCandidateArgs = { + viewerId: VIEWER, + filter: "all", + fallbackLat: 34.0522, + fallbackLng: -118.2437, + windowDays: 30, + radiusKm: 40, + candidateCap: 400, +} + +async function emitted(over: Partial = {}) { + const fake = makeFakeSql() + const repo = makeDrizzlePostRepository(fake.sql as unknown as Sql, { + presignMedia: () => Promise.resolve({ url: "u" }), + presignAvatar: () => Promise.resolve("a"), + }) + await repo.feedCandidates({ ...ARGS, ...over }) + expect(fake.statements, "feedCandidates must be exactly ONE statement (no N+1)").toHaveLength(1) + return fake.statements[0]! +} + +describe("feed candidate SQL: one bounded statement", () => { + it("issues exactly one statement per feed request", async () => { + await emitted() + }) + + it("parameterizes every value (no interpolated literals)", async () => { + const stmt = await emitted() + expect(stmt.values).toContain(VIEWER) + expect(stmt.values).toContain(34.0522) + expect(stmt.values).toContain(-118.2437) + expect(stmt.sql).not.toContain(VIEWER) + }) + + it("bounds all three pools and the final result", async () => { + const stmt = await emitted() + expect(stmt.values).toContain(FEED_IN_NETWORK_POOL) + expect(stmt.values).toContain(FEED_NEARBY_POOL) + expect(stmt.values).toContain(FEED_RECENT_POOL) + expect(stmt.values).toContain(400) + const limits = stmt.sql.match(/LIMIT/g) ?? [] + expect(limits.length).toBeGreaterThanOrEqual(5) + }) + + it("bounds the scan by the candidate window, never the whole table", async () => { + const stmt = await emitted() + expect(stmt.sql).toContain("eligible_window") + expect(stmt.sql).toMatch(/created_at >= \(SELECT since FROM eligible_window\)/) + expect(stmt.values).toContain(30) + }) +}) + +describe("feed candidate SQL: security invariants", () => { + it("excludes non-public and soft-deleted posts in every pool", async () => { + const stmt = await emitted() + const visibility = stmt.sql.match(/p\.visibility = 'public'/g) ?? [] + const deleted = stmt.sql.match(/p\.deleted_at IS NULL/g) ?? [] + expect(visibility.length).toBeGreaterThanOrEqual(3) + expect(deleted.length).toBeGreaterThanOrEqual(3) + }) + + it("excludes replies from the timeline in every pool", async () => { + const stmt = await emitted() + const replies = stmt.sql.match(/p\.reply_to_id IS NULL/g) ?? [] + expect(replies.length).toBeGreaterThanOrEqual(3) + }) + + it("applies the block exclusion symmetrically, once over the union", async () => { + const stmt = await emitted() + expect(stmt.sql).toContain("FROM user_blocks b") + expect(stmt.sql).toMatch(/b\.blocker_id = \? AND b\.blocked_id = p\.author_id/) + expect(stmt.sql).toMatch(/b\.blocker_id = p\.author_id AND b\.blocked_id = \?/) + expect(stmt.sql.match(/FROM user_blocks b/g)).toHaveLength(1) + }) + + it("never counts a suspended or deleted organization as verified", async () => { + const stmt = await emitted() + expect(stmt.sql).toMatch(/po\.deleted_at IS NULL AND po\.suspended_at IS NULL/) + expect(stmt.sql).toContain("o.deleted_at IS NULL") + expect(stmt.sql).toContain("o.suspended_at IS NULL") + expect(stmt.sql).toContain("au.deleted_at IS NULL") + }) + + it("reproduces the affiliation.ts primary-org ordering exactly", async () => { + const stmt = await emitted() + expect(stmt.sql).toMatch( + /ORDER BY COALESCE\(o\.id = au\.primary_organization_id, false\) DESC, m\.joined_at ASC, o\.id ASC/, + ) + }) + + it("counts only media that will actually render", async () => { + const stmt = await emitted() + expect(stmt.sql).toMatch(/ma\.status = 'ready' AND ma\.served_key IS NOT NULL/) + }) + + it("treats an ended event as not live", async () => { + const stmt = await emitted() + expect(stmt.sql).toMatch(/p\.event_id IS NOT NULL AND ev\.ends_at >= now\(\)/) + }) +}) + +describe("feed candidate SQL: nearby pool uses the GIST index", () => { + it("is a CROSS JOIN LATERAL with a KNN order, so the LIMIT stops the scan", async () => { + const stmt = await emitted() + expect(stmt.sql).toContain("CROSS JOIN LATERAL") + expect(stmt.sql).toMatch(/ORDER BY p\.geom <-> vp\.geom/) + expect(stmt.sql).toContain("ST_DWithin(p.geom, vp.geom") + }) + + it("passes a degree radius padded for longitude convergence", async () => { + const stmt = await emitted() + expect(stmt.values).toContain(nearbyRadiusDegrees(40)) + expect(nearbyRadiusDegrees(40)).toBeGreaterThan(40 / 111.32) + }) + + it("falls back to the caller's approximate point when the viewer has no history", async () => { + const stmt = await emitted() + expect(stmt.sql).toContain("viewer_point") + expect(stmt.sql).toMatch(/COALESCE\(\s*\(SELECT geom FROM viewer_point\)/) + expect(stmt.sql).toContain("ST_SetSRID") + }) + + it("casts the fallback coordinates so a NULL point is typed, not an error", async () => { + const stmt = await emitted({ fallbackLat: null, fallbackLng: null }) + expect(stmt.sql).toContain("::double precision") + expect(stmt.values).toContain(null) + }) + + it("returns a NULL distance for a post with no geometry", async () => { + const stmt = await emitted() + expect(stmt.sql).toMatch(/CASE WHEN p\.geom IS NULL THEN NULL/) + }) +}) + +describe("feed candidate SQL: filter tab", () => { + it("adds no filter clause for the all tab", async () => { + const stmt = await emitted({ filter: "all" }) + expect(stmt.sql).not.toContain("p.event_id IS NOT NULL\n") + expect(stmt.sql).not.toContain("fr.status = 'resolved'") + }) + + it("narrows the pools themselves for the events tab", async () => { + const stmt = await emitted({ filter: "events" }) + const clauses = stmt.sql.match(/AND p\.event_id IS NOT NULL/g) ?? [] + expect(clauses.length).toBeGreaterThanOrEqual(3) + }) + + it("narrows the pools themselves for the fixes tab", async () => { + const stmt = await emitted({ filter: "fixes" }) + const clauses = stmt.sql.match(/fr\.status = 'resolved'/g) ?? [] + expect(clauses.length).toBeGreaterThanOrEqual(3) + }) +}) diff --git a/services/api/test/unit/feed-counts-service.test.ts b/services/api/test/unit/feed-counts-service.test.ts new file mode 100644 index 00000000..d46ff8d6 --- /dev/null +++ b/services/api/test/unit/feed-counts-service.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from "vitest" +import type { Sql } from "../../src/db/client.js" +import { makePostService } from "../../src/services/post-service.js" +import type { + FeedCountsRow, + PostRepository, +} from "../../src/services/post-repository.drizzle.js" + +const VIEWER = "11111111-1111-1111-1111-111111111111" +const PUBLIC_POST = "22222222-2222-2222-2222-222222222222" +const HIDDEN_POST = "33333333-3333-3333-3333-333333333333" +const BLOCKED_POST = "44444444-4444-4444-4444-444444444444" + +const throwingSql = (() => { + throw new Error("sql must not be called in these unit paths") +}) as unknown as Sql + +function countsRepo(readable: Set): { + repo: PostRepository + calls: Array<{ postIds: readonly string[]; viewerId: string }> +} { + const calls: Array<{ postIds: readonly string[]; viewerId: string }> = [] + const repo = { + readableCounts: (postIds: readonly string[], viewerId: string): Promise => { + calls.push({ postIds, viewerId }) + return Promise.resolve( + postIds + .filter((id) => readable.has(id)) + .map((id) => ({ + id, + like_count: 3, + repost_count: 2, + reply_count: 1, + save_count: 4, + })), + ) + }, + } as unknown as PostRepository + return { repo, calls } +} + +describe("getFeedCounts", () => { + it("returns counts for a post the caller may read", async () => { + const { repo } = countsRepo(new Set([PUBLIC_POST])) + const svc = makePostService({ repo, sql: throwingSql }) + const result = await svc.getFeedCounts([PUBLIC_POST], VIEWER) + expect(result).toEqual({ + items: [{ id: PUBLIC_POST, counts: { likes: 3, reposts: 2, replies: 1, saves: 4 } }], + }) + }) + + it("omits a hidden or deleted post rather than erroring, so it cannot probe existence", async () => { + const { repo } = countsRepo(new Set([PUBLIC_POST])) + const svc = makePostService({ repo, sql: throwingSql }) + const result = await svc.getFeedCounts([PUBLIC_POST, HIDDEN_POST], VIEWER) + expect(result.items.map((item) => item.id)).toEqual([PUBLIC_POST]) + }) + + it("omits a post whose author blocks the caller (no IDOR either direction)", async () => { + const { repo } = countsRepo(new Set([PUBLIC_POST])) + const svc = makePostService({ repo, sql: throwingSql }) + const result = await svc.getFeedCounts([BLOCKED_POST, PUBLIC_POST], VIEWER) + expect(result.items.map((item) => item.id)).toEqual([PUBLIC_POST]) + }) + + it("returns an empty list for entirely unreadable ids, never a 404", async () => { + const { repo } = countsRepo(new Set()) + const svc = makePostService({ repo, sql: throwingSql }) + await expect(svc.getFeedCounts([HIDDEN_POST], VIEWER)).resolves.toEqual({ items: [] }) + }) + + it("scopes the read to the calling viewer, never an id from the body", async () => { + const { repo, calls } = countsRepo(new Set([PUBLIC_POST])) + const svc = makePostService({ repo, sql: throwingSql }) + await svc.getFeedCounts([PUBLIC_POST], VIEWER) + expect(calls).toEqual([{ postIds: [PUBLIC_POST], viewerId: VIEWER }]) + }) + + it("deduplicates ids so a repeated id cannot multiply the query cost", async () => { + const { repo, calls } = countsRepo(new Set([PUBLIC_POST])) + const svc = makePostService({ repo, sql: throwingSql }) + await svc.getFeedCounts([PUBLIC_POST, PUBLIC_POST, PUBLIC_POST], VIEWER) + expect(calls[0]!.postIds).toEqual([PUBLIC_POST]) + }) + + it("issues exactly one repository read (no N+1)", async () => { + const { repo, calls } = countsRepo(new Set([PUBLIC_POST, HIDDEN_POST, BLOCKED_POST])) + const svc = makePostService({ repo, sql: throwingSql }) + await svc.getFeedCounts([PUBLIC_POST, HIDDEN_POST, BLOCKED_POST], VIEWER) + expect(calls).toHaveLength(1) + }) + + it("short-circuits an empty request without touching the repository", async () => { + const { repo, calls } = countsRepo(new Set()) + const svc = makePostService({ repo, sql: throwingSql }) + await expect(svc.getFeedCounts([], VIEWER)).resolves.toEqual({ items: [] }) + expect(calls).toHaveLength(0) + }) +}) diff --git a/services/api/test/unit/feed-presence.test.ts b/services/api/test/unit/feed-presence.test.ts new file mode 100644 index 00000000..aeb5d228 --- /dev/null +++ b/services/api/test/unit/feed-presence.test.ts @@ -0,0 +1,201 @@ +import { describe, expect, it } from "vitest" +import { DEFAULT_FEED_RANKING } from "@civfix/shared" +import { InMemoryCacheClient } from "../../src/auth/cache.js" +import { + makeFeedPresence, + servedKey, + snapshotKey, + viewersKey, + type FeedPresenceCache, +} from "../../src/services/feed-presence.js" + +const VIEWER = "11111111-1111-1111-1111-111111111111" +const OTHER = "22222222-2222-2222-2222-222222222222" +const POST = "33333333-3333-3333-3333-333333333333" + +const CFG = DEFAULT_FEED_RANKING + +function present(over: Partial = {}) { + const cache = new InMemoryCacheClient(() => Date.now()) + return { + cache, + presence: makeFeedPresence({ cache, config: { ...CFG, ...over } }), + } +} + +describe("feed presence: key convention", () => { + it("namespaces every key with the house lowercase colon :v1 prefix", () => { + expect(snapshotKey(VIEWER, "all")).toBe(`feed:rank:v1:${VIEWER}:all`) + expect(servedKey(VIEWER)).toBe(`feed:served:v1:${VIEWER}`) + expect(viewersKey(POST)).toBe(`feed:viewers:v1:${POST}`) + }) + + it("keys the snapshot per filter, so the events tab cannot serve the all tab's page", () => { + expect(snapshotKey(VIEWER, "events")).not.toBe(snapshotKey(VIEWER, "all")) + }) +}) + +describe("feed presence: snapshot", () => { + it("round-trips a ranked set", async () => { + const { presence } = present() + await presence.writeSnapshot(VIEWER, "all", [ + { id: POST, authorId: "a", score: 115.9 }, + { id: OTHER, authorId: "b", score: 13.125 }, + ]) + expect(await presence.readSnapshot(VIEWER, "all")).toEqual([ + { id: POST, score: 115.9 }, + { id: OTHER, score: 13.125 }, + ]) + }) + + it("caps the stored snapshot at candidateCap", async () => { + const { cache, presence } = present({ candidateCap: 50 }) + const ranked = Array.from({ length: 400 }, (_, i) => ({ + id: `${i}`, + authorId: "a", + score: 400 - i, + })) + await presence.writeSnapshot(VIEWER, "all", ranked) + const raw = await cache.get(snapshotKey(VIEWER, "all")) + expect(JSON.parse(raw!)).toHaveLength(50) + }) + + it("returns null rather than throwing on a corrupt payload", async () => { + const { cache, presence } = present() + await cache.set(snapshotKey(VIEWER, "all"), "{not json", 60) + expect(await presence.readSnapshot(VIEWER, "all")).toBeNull() + }) + + it("returns null on a well-formed but wrongly-shaped payload", async () => { + const { cache, presence } = present() + await cache.set(snapshotKey(VIEWER, "all"), JSON.stringify([["id", "not-a-number"]]), 60) + expect(await presence.readSnapshot(VIEWER, "all")).toBeNull() + }) + + it("writes nothing when the TTL knob is zero", async () => { + const { cache, presence } = present({ snapshotTtlSeconds: 0 }) + await presence.writeSnapshot(VIEWER, "all", [{ id: POST, authorId: "a", score: 1 }]) + expect(await cache.get(snapshotKey(VIEWER, "all"))).toBeNull() + }) +}) + +describe("feed presence: served set and the seen signal", () => { + it("reports only the ids this viewer was actually served", async () => { + const { presence } = present() + await presence.recordServed(VIEWER, [POST]) + const seen = await presence.seenBy(VIEWER, [POST, OTHER]) + expect([...seen]).toEqual([POST]) + }) + + it("keeps served sets per viewer", async () => { + const { presence } = present() + await presence.recordServed(VIEWER, [POST]) + expect([...(await presence.seenBy(OTHER, [POST]))]).toEqual([]) + }) + + it("puts a TTL on both the served set and the viewer index (no unbounded growth)", async () => { + const expiries: Array<[string, number]> = [] + const inner = new InMemoryCacheClient(() => Date.now()) + const recording: FeedPresenceCache = { + get: (k) => inner.get(k), + set: (k, v, ttl) => inner.set(k, v, ttl), + sadd: (k, ...m) => inner.sadd(k, ...m), + smembers: (k) => inner.smembers(k), + scard: (k) => inner.scard(k), + expire: (k, ttl) => { + expiries.push([k, ttl]) + return inner.expire(k, ttl) + }, + } + const presence = makeFeedPresence({ cache: recording, config: CFG }) + await presence.recordServed(VIEWER, [POST, OTHER]) + + expect(expiries).toEqual([ + [servedKey(VIEWER), CFG.servedTtlSeconds], + [viewersKey(POST), CFG.servedTtlSeconds], + [viewersKey(OTHER), CFG.servedTtlSeconds], + ]) + }) + + it("is a no-op for an empty page", async () => { + const { cache, presence } = present() + await presence.recordServed(VIEWER, []) + expect(await cache.scard(servedKey(VIEWER))).toBe(0) + }) +}) + +describe("feed presence: viewer reverse index is bounded", () => { + it("lists the viewers currently served a post", async () => { + const { presence } = present() + await presence.recordServed(VIEWER, [POST]) + await presence.recordServed(OTHER, [POST]) + expect((await presence.viewersOf(POST)).sort()).toEqual([VIEWER, OTHER].sort()) + }) + + it("stops adding viewers once the cap is reached", async () => { + const { cache, presence } = present({ viewerFanoutMax: 3 }) + for (let i = 0; i < 10; i += 1) { + await presence.recordServed(`viewer-${i}`, [POST]) + } + expect(await cache.scard(viewersKey(POST))).toBe(3) + }) + + it("returns nothing for a post with no live viewers", async () => { + const { presence } = present() + expect(await presence.viewersOf(POST)).toEqual([]) + }) + + it("skips the fanout entirely when the set is over the cap, without reading it", async () => { + let smembersCalls = 0 + const cache: FeedPresenceCache = { + get: () => Promise.resolve(null), + set: () => Promise.resolve(), + sadd: () => Promise.resolve(1), + smembers: () => { + smembersCalls += 1 + return Promise.resolve([]) + }, + scard: () => Promise.resolve(50_000), + expire: () => Promise.resolve(), + } + const presence = makeFeedPresence({ cache, config: { ...CFG, viewerFanoutMax: 500 } }) + expect(await presence.viewersOf(POST)).toEqual([]) + expect(smembersCalls).toBe(0) + }) +}) + +describe("feed presence: degrades instead of failing the request", () => { + const exploding: FeedPresenceCache = { + get: () => Promise.reject(new Error("redis down")), + set: () => Promise.reject(new Error("redis down")), + sadd: () => Promise.reject(new Error("redis down")), + smembers: () => Promise.reject(new Error("redis down")), + scard: () => Promise.reject(new Error("redis down")), + expire: () => Promise.reject(new Error("redis down")), + } + + it("swallows every Redis failure and returns the safe fallback", async () => { + const warnings: unknown[] = [] + const presence = makeFeedPresence({ + cache: exploding, + config: CFG, + logger: { warn: (obj) => warnings.push(obj) }, + }) + expect(await presence.readSnapshot(VIEWER, "all")).toBeNull() + await expect( + presence.writeSnapshot(VIEWER, "all", [{ id: POST, authorId: "a", score: 1 }]), + ).resolves.toBeUndefined() + expect([...(await presence.seenBy(VIEWER, [POST]))]).toEqual([]) + await expect(presence.recordServed(VIEWER, [POST])).resolves.toBeUndefined() + expect(await presence.viewersOf(POST)).toEqual([]) + expect(warnings.length).toBeGreaterThan(0) + }) + + it("is inert with no cache at all (fake/no-Redis mode)", async () => { + const presence = makeFeedPresence({ config: CFG }) + expect(await presence.readSnapshot(VIEWER, "all")).toBeNull() + expect([...(await presence.seenBy(VIEWER, [POST]))]).toEqual([]) + expect(await presence.viewersOf(POST)).toEqual([]) + await expect(presence.recordServed(VIEWER, [POST])).resolves.toBeUndefined() + }) +}) diff --git a/services/api/test/unit/feed-ranked-service.test.ts b/services/api/test/unit/feed-ranked-service.test.ts new file mode 100644 index 00000000..8f421f3e --- /dev/null +++ b/services/api/test/unit/feed-ranked-service.test.ts @@ -0,0 +1,361 @@ +import { describe, expect, it } from "vitest" +import { DEFAULT_FEED_RANKING, parseFeedScoreCursor } from "@civfix/shared" +import type { PostDTO, UserSignal } from "@civfix/shared" +import type { UserChannel } from "@civfix/shared/interfaces" +import type { Sql } from "../../src/db/client.js" +import { makePostService, type PostService } from "../../src/services/post-service.js" +import { makeFeedPresence } from "../../src/services/feed-presence.js" +import { InMemoryCacheClient } from "../../src/auth/cache.js" +import type { + FeedCandidateRow, + PostBrief, + PostRepository, +} from "../../src/services/post-repository.drizzle.js" + +const VIEWER = "11111111-1111-1111-1111-111111111111" +const AUTHOR = "22222222-2222-2222-2222-222222222222" +const POST = "33333333-3333-3333-3333-333333333333" +const NOW = Date.UTC(2026, 8, 15, 12, 0, 0) + +const throwingSql = (() => { + throw new Error("sql must not be called in these unit paths") +}) as unknown as Sql + +function uuidAt(index: number): string { + return `${String(index).padStart(8, "0")}-0000-0000-0000-000000000000` +} + +function candidateRow(index: number, over: Partial = {}): FeedCandidateRow { + return { + id: uuidAt(index), + author_id: `author-${index}`, + created_at: new Date(NOW - index * 3_600_000), + like_count: index, + reply_count: 0, + repost_count: 0, + has_report: false, + has_live_event: false, + has_media: false, + author_followed: true, + author_is_viewer: false, + viewer_mentioned: false, + author_org_verified: false, + distance_km: null, + ...over, + } +} + +function dto(id: string): PostDTO { + return { + id, + author: { + id: AUTHOR, + name: "Author", + handle: "author", + bio: null, + avatar: ["#000000", "#111111"], + followers: 0, + following: 0, + isFollowing: false, + }, + kind: "post", + body: "hi", + createdAt: new Date(NOW).toISOString(), + editedAt: null, + counts: { likes: 0, reposts: 0, replies: 0, saves: 0 }, + viewer: { liked: false, reposted: false, saved: false }, + media: [], + mentions: [], + event: null, + report: null, + repostOf: null, + replyToId: null, + threadRootId: null, + } +} + +function brief(over: Partial = {}): PostBrief { + return { + id: POST, + authorId: AUTHOR, + kind: "post", + replyToId: null, + repostOfId: null, + deletedAt: null, + visibility: "public", + ...over, + } +} + +function repoOver(over: Partial): PostRepository { + return { + getPostBrief: () => Promise.resolve(brief()), + actorNameOf: () => Promise.resolve("Actor"), + canPostAsOrganization: () => Promise.resolve(true), + isEventMember: () => Promise.resolve(true), + isReportAttachable: () => Promise.resolve(true), + createPost: () => Promise.resolve(POST), + softDeletePost: () => Promise.resolve(), + like: () => Promise.resolve(true), + unlike: () => Promise.resolve(true), + save: () => Promise.resolve(true), + unsave: () => Promise.resolve(true), + repost: (id: string) => Promise.resolve({ targetId: id, created: true }), + unrepost: (id: string) => Promise.resolve({ targetId: id, removed: true }), + getPostDTO: (id: string) => Promise.resolve(dto(id)), + homeFeedChronological: () => Promise.resolve({ items: [], nextCursor: null }), + publicFeed: () => Promise.resolve({ items: [], nextCursor: null }), + feedCandidates: () => Promise.resolve([]), + hydrateByIds: (ids: readonly string[]) => Promise.resolve(ids.map(dto)), + followerIdsOf: () => Promise.resolve([]), + readableCounts: () => Promise.resolve([]), + listReplies: () => Promise.resolve({ items: [], nextCursor: null, authorReplies: [] }), + listUserPosts: () => Promise.resolve({ items: [], nextCursor: null }), + listSaves: () => Promise.resolve({ items: [], nextCursor: null }), + ...over, + } as PostRepository +} + +function recordingChannel(): { channel: UserChannel; sent: Array<{ users: string[]; signal: UserSignal }> } { + const sent: Array<{ users: string[]; signal: UserSignal }> = [] + const channel = { + subscribeUser: () => Promise.resolve(() => Promise.resolve()), + publishToUser: (userId: string, signal: UserSignal) => { + sent.push({ users: [userId], signal }) + return Promise.resolve() + }, + publishToUsers: (userIds: readonly string[], signal: UserSignal) => { + sent.push({ users: [...userIds], signal }) + return Promise.resolve() + }, + } as unknown as UserChannel + return { channel, sent } +} + +describe("ranked feed: cursor continuation covers every item exactly once", () => { + const rows = Array.from({ length: 60 }, (_, i) => candidateRow(i + 1)) + + function service(): PostService { + return makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + now: () => NOW, + }) + } + + it("pages 1 -> 2 -> 3 with no duplicates and no gaps", async () => { + const svc = service() + const seen: string[] = [] + let cursor: string | undefined + let pages = 0 + + for (;;) { + const page = await svc.homeFeed(VIEWER, { + filter: "all", + limit: 20, + ...(cursor === undefined ? {} : { cursor }), + }) + seen.push(...page.items.map((item) => item.id)) + pages += 1 + if (page.nextCursor === null) break + cursor = page.nextCursor + expect(pages).toBeLessThan(10) + } + + expect(pages).toBe(3) + expect(seen).toHaveLength(60) + expect(new Set(seen).size).toBe(60) + }) + + it("emits a parseable score cursor whose score matches the last served item", async () => { + const svc = service() + const page = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + const cursor = parseFeedScoreCursor(page.nextCursor) + expect(cursor).not.toBeNull() + expect(cursor!.postId).toBe(page.items[page.items.length - 1]!.id) + }) + + it("returns nextCursor null once the ranked set is exhausted", async () => { + const svc = service() + let cursor: string | undefined + let last: string | null = "seed" + for (let i = 0; i < 3; i += 1) { + const page: { items: PostDTO[]; nextCursor: string | null } = await svc.homeFeed(VIEWER, { + filter: "all", + limit: 20, + ...(cursor === undefined ? {} : { cursor }), + }) + last = page.nextCursor + if (page.nextCursor !== null) cursor = page.nextCursor + } + expect(last).toBeNull() + }) + + it("keeps the continuation strictly after the cursor under the repository ordering", async () => { + const svc = service() + const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + const second = await svc.homeFeed(VIEWER, { + filter: "all", + limit: 20, + cursor: first.nextCursor!, + }) + const firstIds = new Set(first.items.map((item) => item.id)) + expect(second.items.some((item) => firstIds.has(item.id))).toBe(false) + }) +}) + +describe("ranked feed: served set feeds the seen discount", () => { + it("records the served page so the next refresh demotes what was already read", async () => { + const cache = new InMemoryCacheClient(() => Date.now()) + const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + const rows = Array.from({ length: 5 }, (_, i) => candidateRow(i + 1)) + const svc = makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: presence, + now: () => NOW, + }) + + const page = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + await Promise.resolve() + const served = await presence.seenBy( + VIEWER, + rows.map((r) => r.id), + ) + expect(served.size).toBe(page.items.length) + }) +}) + +describe("ranked feed: realtime fanout never blocks the request path", () => { + it("publishes topic feed to the author's followers on a new top-level post", async () => { + const { channel, sent } = recordingChannel() + const svc = makePostService({ + repo: repoOver({ followerIdsOf: () => Promise.resolve(["f1", "f2"]) }), + sql: throwingSql, + userChannel: channel, + }) + await svc.createPost( + { kind: "post", body: "hello", mediaUploadIds: [], mentionedUserIds: [] } as never, + AUTHOR, + ) + expect(sent).toEqual([{ users: ["f1", "f2"], signal: { topic: "feed", id: POST } }]) + }) + + it("publishes topic feed_counts to the post's live viewers on a like", async () => { + const { channel, sent } = recordingChannel() + const cache = new InMemoryCacheClient(() => Date.now()) + const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + await presence.recordServed("reader-1", [POST]) + await presence.recordServed("reader-2", [POST]) + + const svc = makePostService({ + repo: repoOver({}), + sql: throwingSql, + userChannel: channel, + feedPresence: presence, + }) + await svc.likePost(POST, VIEWER) + + expect(sent).toHaveLength(1) + expect(sent[0]!.signal).toEqual({ topic: "feed_counts", id: POST }) + expect(sent[0]!.users.sort()).toEqual(["reader-1", "reader-2"]) + }) + + it("excludes the acting user, whose own client already patched optimistically", async () => { + const { channel, sent } = recordingChannel() + const cache = new InMemoryCacheClient(() => Date.now()) + const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + await presence.recordServed(VIEWER, [POST]) + await presence.recordServed("reader-1", [POST]) + + const svc = makePostService({ + repo: repoOver({}), + sql: throwingSql, + userChannel: channel, + feedPresence: presence, + }) + await svc.likePost(POST, VIEWER) + + expect(sent[0]!.users).toEqual(["reader-1"]) + }) + + it("does not publish when the interaction was a no-op (double like)", async () => { + const { channel, sent } = recordingChannel() + const cache = new InMemoryCacheClient(() => Date.now()) + const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + await presence.recordServed("reader-1", [POST]) + + const svc = makePostService({ + repo: repoOver({ like: () => Promise.resolve(false) }), + sql: throwingSql, + userChannel: channel, + feedPresence: presence, + }) + await svc.likePost(POST, VIEWER) + expect(sent).toHaveLength(0) + }) + + it("a rejecting channel does not fail likePost", async () => { + const warnings: unknown[] = [] + const cache = new InMemoryCacheClient(() => Date.now()) + const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + await presence.recordServed("reader-1", [POST]) + const channel = { + subscribeUser: () => Promise.resolve(() => Promise.resolve()), + publishToUser: () => Promise.reject(new Error("redis down")), + publishToUsers: () => Promise.reject(new Error("redis down")), + } as unknown as UserChannel + + const svc = makePostService({ + repo: repoOver({}), + sql: throwingSql, + userChannel: channel, + feedPresence: presence, + logger: { warn: (obj) => warnings.push(obj) }, + }) + + await expect(svc.likePost(POST, VIEWER)).resolves.toMatchObject({ id: POST }) + await Promise.resolve() + expect(warnings.length).toBeGreaterThan(0) + }) + + it("a throwing follower lookup does not fail createPost", async () => { + const { channel, sent } = recordingChannel() + const svc = makePostService({ + repo: repoOver({ followerIdsOf: () => Promise.reject(new Error("db down")) }), + sql: throwingSql, + userChannel: channel, + logger: { warn: () => {} }, + }) + await expect( + svc.createPost( + { kind: "post", body: "hello", mediaUploadIds: [], mentionedUserIds: [] } as never, + AUTHOR, + ), + ).resolves.toMatchObject({ id: POST }) + expect(sent).toHaveLength(0) + }) + + it("publishes nothing at all in fake mode (no user channel wired)", async () => { + const cache = new InMemoryCacheClient(() => Date.now()) + const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + await presence.recordServed("reader-1", [POST]) + let followerLookups = 0 + const svc = makePostService({ + repo: repoOver({ + followerIdsOf: () => { + followerLookups += 1 + return Promise.resolve(["f1"]) + }, + }), + sql: throwingSql, + feedPresence: presence, + }) + await svc.likePost(POST, VIEWER) + await svc.createPost( + { kind: "post", body: "hello", mediaUploadIds: [], mentionedUserIds: [] } as never, + AUTHOR, + ) + expect(followerLookups).toBe(0) + }) +}) diff --git a/services/api/test/unit/feed-ranking-env.test.ts b/services/api/test/unit/feed-ranking-env.test.ts new file mode 100644 index 00000000..76b5cb92 --- /dev/null +++ b/services/api/test/unit/feed-ranking-env.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest" +import { DEFAULT_FEED_RANKING } from "@civfix/shared" +import { loadEnv } from "../../src/env.js" + +function load(feedRanking?: string) { + return loadEnv({ + NODE_ENV: "test", + ...(feedRanking === undefined ? {} : { FEED_RANKING: feedRanking }), + }) +} + +describe("FEED_RANKING env var", () => { + it("falls back to the full default profile when unset", () => { + expect(load().FEED_RANKING).toEqual(DEFAULT_FEED_RANKING) + }) + + it("treats blank and whitespace as unset, so a commented-out SOPS line is safe", () => { + expect(load("").FEED_RANKING).toEqual(DEFAULT_FEED_RANKING) + expect(load(" ").FEED_RANKING).toEqual(DEFAULT_FEED_RANKING) + }) + + it("merges a partial override onto the defaults", () => { + const env = load('{"halfLifeHours":12,"followWeight":70}') + expect(env.FEED_RANKING.halfLifeHours).toBe(12) + expect(env.FEED_RANKING.followWeight).toBe(70) + expect(env.FEED_RANKING.minScore).toBe(DEFAULT_FEED_RANKING.minScore) + }) + + it("accepts a full profile", () => { + const env = load(JSON.stringify(DEFAULT_FEED_RANKING)) + expect(env.FEED_RANKING).toEqual(DEFAULT_FEED_RANKING) + }) + + it("refuses to boot on malformed JSON, naming the variable", () => { + expect(() => load("{not json")).toThrow(/FEED_RANKING: must be a JSON object/) + }) + + it("refuses to boot on a typo'd knob rather than silently ignoring it", () => { + expect(() => load('{"folowWeight":70}')).toThrow(/FEED_RANKING/) + }) + + it("refuses to boot on an out-of-range value, naming the knob", () => { + expect(() => load('{"decayFloor":1.5}')).toThrow(/FEED_RANKING\.decayFloor/) + }) + + it("refuses a JSON scalar where an object is required", () => { + expect(() => load("42")).toThrow(/FEED_RANKING/) + expect(() => load('"forty-two"')).toThrow(/FEED_RANKING/) + }) + + it("reports every bad knob at once, in the one-throw boot list", () => { + try { + load('{"decayFloor":1.5,"halfLifeHours":0}') + expect.unreachable("loadEnv should have thrown") + } catch (err) { + const message = (err as Error).message + expect(message).toContain("FEED_RANKING.decayFloor") + expect(message).toContain("FEED_RANKING.halfLifeHours") + } + }) +}) diff --git a/services/api/test/unit/feed-ranking.test.ts b/services/api/test/unit/feed-ranking.test.ts new file mode 100644 index 00000000..76de8162 --- /dev/null +++ b/services/api/test/unit/feed-ranking.test.ts @@ -0,0 +1,380 @@ +import { describe, expect, it } from "vitest" +import { DEFAULT_FEED_RANKING, FeedRankingConfigSchema } from "@civfix/shared" +import type { FeedRankingConfig } from "@civfix/shared" +import { + applyCutoff, + diversityMultiplier, + proximity, + quantizeClock, + rankCandidates, + rawScore, + recency, + scoreCandidate, + type FeedCandidate, +} from "../../src/services/feed-ranking.js" + +const NOW = Date.UTC(2026, 8, 15, 12, 0, 0) +const HOUR = 3_600_000 + +const CFG = DEFAULT_FEED_RANKING + +function candidate(over: Partial = {}): FeedCandidate { + return { + id: "11111111-1111-1111-1111-111111111111", + authorId: "author-a", + createdAtMs: NOW, + likeCount: 0, + replyCount: 0, + repostCount: 0, + hasMedia: false, + hasReport: false, + hasLiveEvent: false, + authorFollowed: false, + authorIsViewer: false, + viewerMentioned: false, + authorOrgVerified: false, + distanceKm: null, + alreadySeen: false, + ...over, + } +} + +describe("feed ranking: each weight term in isolation", () => { + const base = rawScore(candidate(), CFG) + + it("starts every eligible post at the base weight", () => { + expect(base).toBe(CFG.baseWeight) + }) + + const flags: Array<[keyof FeedCandidate, keyof FeedRankingConfig]> = [ + ["authorFollowed", "followWeight"], + ["authorIsViewer", "selfWeight"], + ["viewerMentioned", "mentionWeight"], + ["authorOrgVerified", "orgVerifiedWeight"], + ["hasLiveEvent", "attachEventWeight"], + ["hasReport", "attachReportWeight"], + ["hasMedia", "imageWeight"], + ] + + for (const [flag, weight] of flags) { + it(`adds exactly ${weight} when ${flag} is set`, () => { + const score = rawScore(candidate({ [flag]: true } as Partial), CFG) + expect(score - base).toBeCloseTo(CFG[weight] as number, 10) + }) + } + + it("scores an ended event as no event at all", () => { + expect(rawScore(candidate({ hasLiveEvent: false }), CFG)).toBe(base) + }) +}) + +describe("feed ranking: proximity", () => { + it("is 1 at the viewer's own point and 0 at the radius", () => { + expect(proximity(0, 40)).toBe(1) + expect(proximity(40, 40)).toBe(0) + }) + + it("falls off linearly, with no discontinuity at the edge", () => { + expect(proximity(20, 40)).toBeCloseTo(0.5, 10) + expect(proximity(39.9, 40)).toBeCloseTo(0.0025, 10) + }) + + it("never goes negative beyond the radius", () => { + expect(proximity(4000, 40)).toBe(0) + }) + + it("is zero when either the post or the viewer has no point", () => { + expect(proximity(null, 40)).toBe(0) + }) +}) + +describe("feed ranking: recency decay", () => { + it("is 1 at age zero", () => { + expect(recency(0, CFG)).toBeCloseTo(1, 10) + }) + + it("is halfway to the floor at one half-life", () => { + const expected = CFG.decayFloor + (1 - CFG.decayFloor) * 0.5 + expect(recency(CFG.halfLifeHours, CFG)).toBeCloseTo(expected, 10) + }) + + it("decreases monotonically and never crosses the floor", () => { + let previous = recency(0, CFG) + for (const hours of [1, 6, 24, 36, 72, 24 * 7, 24 * 30]) { + const value = recency(hours, CFG) + expect(value).toBeLessThan(previous) + expect(value).toBeGreaterThan(CFG.decayFloor) + previous = value + } + expect(recency(24 * 365, CFG)).toBeGreaterThanOrEqual(CFG.decayFloor) + }) + + it("approaches but never reaches zero, so an old unresolved report is demoted not erased", () => { + expect(recency(24 * 3650, CFG)).toBeGreaterThanOrEqual(CFG.decayFloor) + }) + + it("treats a future-dated post as brand new rather than boosting it", () => { + expect(recency(-100, CFG)).toBeCloseTo(1, 10) + }) +}) + +describe("feed ranking: engagement is log-scaled, not stepped", () => { + const withLikes = (likes: number) => rawScore(candidate({ likeCount: likes }), CFG) + + it("has diminishing returns: 0 to 10 likes is worth more than 10 to 50", () => { + expect(withLikes(10) - withLikes(0)).toBeGreaterThan(withLikes(50) - withLikes(10)) + }) + + it("has no cliff between 49 and 50 likes", () => { + expect(withLikes(50) - withLikes(49)).toBeLessThan(0.1) + }) + + it("keeps rewarding a runaway post, slowly", () => { + expect(withLikes(5000)).toBeGreaterThan(withLikes(200)) + }) + + it("weights a reply above a repost above a like", () => { + expect(CFG.replyWeight).toBeGreaterThan(CFG.repostWeight) + expect(CFG.repostWeight).toBeGreaterThan(CFG.likeWeight) + }) +}) + +describe("feed ranking: author diversity", () => { + it("reproduces the Twitter DiversityDiscountProvider curve", () => { + expect(diversityMultiplier(0, CFG)).toBeCloseTo(1, 10) + expect(diversityMultiplier(1, CFG)).toBeCloseTo(0.625, 10) + expect(diversityMultiplier(2, CFG)).toBeCloseTo(0.4375, 10) + }) + + it("never discounts below the floor", () => { + expect(diversityMultiplier(10, CFG)).toBeGreaterThanOrEqual(CFG.diversityFloor) + expect(diversityMultiplier(100, CFG)).toBeCloseTo(CFG.diversityFloor, 10) + }) + + it("stops one enthusiastic poster from taking the whole page", () => { + const flood = Array.from({ length: 5 }, (_, i) => + candidate({ + id: `a${i}`, + authorId: "flooder", + authorFollowed: true, + likeCount: 50, + createdAtMs: NOW - i * 60_000, + }), + ) + const other = candidate({ id: "b0", authorId: "someone-else", authorFollowed: true }) + + const undiscounted = [...flood, other] + .map((c) => ({ id: c.id, score: scoreCandidate(c, CFG, NOW) })) + .sort((a, b) => b.score - a.score) + expect(undiscounted.findIndex((entry) => entry.id === "b0")).toBe(flood.length) + + const ranked = rankCandidates([...flood, other], CFG, NOW) + expect(ranked.findIndex((entry) => entry.id === "b0")).toBeLessThan(flood.length) + }) + + it("applies the discount per author, not globally", () => { + const ranked = rankCandidates( + [ + candidate({ id: "a1", authorId: "a" }), + candidate({ id: "b1", authorId: "b" }), + candidate({ id: "c1", authorId: "c" }), + ], + CFG, + NOW, + ) + const scores = new Set(ranked.map((entry) => entry.score)) + expect(scores.size).toBe(1) + }) +}) + +describe("feed ranking: seen discount", () => { + it("demotes a post the viewer was already served", () => { + const fresh = scoreCandidate(candidate(), CFG, NOW) + const seen = scoreCandidate(candidate({ alreadySeen: true }), CFG, NOW) + expect(seen).toBeCloseTo(fresh * CFG.seenDiscount, 10) + }) +}) + +describe("feed ranking: clock quantisation keeps a cursor stable", () => { + it("buckets two instants inside the same minute to the same value", () => { + const a = quantizeClock(NOW + 1_000, CFG.clockBucketSeconds) + const b = quantizeClock(NOW + 59_000, CFG.clockBucketSeconds) + expect(a).toBe(b) + }) + + it("produces byte-identical scores 59 seconds apart", () => { + const rows = [candidate({ id: "a", createdAtMs: NOW - 5 * HOUR, likeCount: 7 })] + const first = rankCandidates(rows, CFG, NOW + 1_000) + const second = rankCandidates(rows, CFG, NOW + 59_000) + expect(second).toEqual(first) + }) + + it("does move the score once the bucket rolls over", () => { + const rows = [candidate({ id: "a", createdAtMs: NOW - 5 * HOUR })] + const inBucket = rankCandidates(rows, CFG, NOW) + const nextBucket = rankCandidates(rows, CFG, NOW + 61_000) + expect(nextBucket[0]!.score).toBeLessThan(inBucket[0]!.score) + }) +}) + +describe("feed ranking: the worked examples from the design", () => { + const at = (hoursAgo: number) => NOW - hoursAgo * HOUR + + it("followed author, 2 h old, 3 likes, 1 reply scores ~115.9", () => { + const score = scoreCandidate( + candidate({ authorFollowed: true, createdAtMs: at(2), likeCount: 3, replyCount: 1 }), + CFG, + NOW, + ) + expect(score).toBeCloseTo(115.9, 1) + }) + + it("stranger 5 km away, verified org, report, image, 12 likes, 3 d old scores ~45.8", () => { + const score = scoreCandidate( + candidate({ + createdAtMs: at(72), + distanceKm: 5, + authorOrgVerified: true, + hasReport: true, + hasMedia: true, + likeCount: 12, + }), + CFG, + NOW, + ) + expect(score).toBeCloseTo(45.8, 1) + }) + + it("stranger, no location, 2 likes, 1 h old scores ~13.1 and clears the cutoff", () => { + const score = scoreCandidate(candidate({ createdAtMs: at(1), likeCount: 2 }), CFG, NOW) + expect(score).toBeCloseTo(13.1, 1) + expect(score).toBeGreaterThanOrEqual(CFG.minScore) + }) + + it("the same post at 30 days old scores ~2.0 and falls below the cutoff", () => { + const score = scoreCandidate( + candidate({ createdAtMs: at(24 * 30), likeCount: 2 }), + CFG, + NOW, + ) + expect(score).toBeCloseTo(2.0, 1) + expect(score).toBeLessThan(CFG.minScore) + }) + + it("stranger, no location, no engagement, 1 h old scores ~9.9, below the cutoff", () => { + const score = scoreCandidate(candidate({ createdAtMs: at(1) }), CFG, NOW) + expect(score).toBeCloseTo(9.84, 2) + expect(score).toBeLessThan(CFG.minScore) + }) +}) + +describe("feed ranking: ordering", () => { + it("orders by score descending with a stable id tiebreak", () => { + const ranked = rankCandidates( + [ + candidate({ id: "aaaaaaaa-0000-0000-0000-000000000000", authorId: "x" }), + candidate({ id: "cccccccc-0000-0000-0000-000000000000", authorId: "y" }), + candidate({ id: "bbbbbbbb-0000-0000-0000-000000000000", authorId: "z" }), + ], + CFG, + NOW, + ) + expect(ranked.map((entry) => entry.id)).toEqual([ + "cccccccc-0000-0000-0000-000000000000", + "bbbbbbbb-0000-0000-0000-000000000000", + "aaaaaaaa-0000-0000-0000-000000000000", + ]) + }) + + it("is deterministic for the same inputs and clock", () => { + const rows = Array.from({ length: 30 }, (_, i) => + candidate({ + id: `${i}`.padStart(8, "0") + "-0000-0000-0000-000000000000", + authorId: `author-${i % 4}`, + createdAtMs: NOW - i * HOUR, + likeCount: i, + }), + ) + expect(rankCandidates(rows, CFG, NOW)).toEqual(rankCandidates(rows, CFG, NOW)) + }) + + it("never emits a negative score", () => { + const hostile: FeedRankingConfig = { ...CFG, baseWeight: 0 } + const score = scoreCandidate(candidate({ createdAtMs: NOW - 24 * 365 * HOUR }), hostile, NOW) + expect(score).toBeGreaterThanOrEqual(0) + }) +}) + +describe("feed ranking: cutoff and cold start", () => { + const weak = () => + Array.from({ length: 3 }, (_, i) => + candidate({ + id: `${i}`.padStart(8, "0") + "-0000-0000-0000-000000000000", + authorId: `author-${i}`, + createdAtMs: NOW - 24 * 30 * HOUR, + }), + ) + + it("suspends the cutoff on page 1 when too little clears it", () => { + const ranked = rankCandidates(weak(), CFG, NOW) + expect(ranked.every((entry) => entry.score < CFG.minScore)).toBe(true) + expect(applyCutoff(ranked, CFG, true)).toHaveLength(3) + }) + + it("never suspends the cutoff on a later page: running out of feed is normal", () => { + const ranked = rankCandidates(weak(), CFG, NOW) + expect(applyCutoff(ranked, CFG, false)).toHaveLength(0) + }) + + it("applies the cutoff normally once page 1 has enough strong items", () => { + const strong = Array.from({ length: 6 }, (_, i) => + candidate({ + id: `s${i}`, + authorId: `author-${i}`, + authorFollowed: true, + createdAtMs: NOW, + }), + ) + const ranked = rankCandidates([...strong, ...weak()], CFG, NOW) + const kept = applyCutoff(ranked, CFG, true) + expect(kept).toHaveLength(6) + expect(kept.every((entry) => entry.score >= CFG.minScore)).toBe(true) + }) + + it("gives a brand-new account with no follows and no location a non-empty page 1", () => { + const strangers = Array.from({ length: 3 }, (_, i) => + candidate({ id: `n${i}`, authorId: `author-${i}`, createdAtMs: NOW - HOUR }), + ) + const ranked = rankCandidates(strangers, CFG, NOW) + expect(applyCutoff(ranked, CFG, true).length).toBeGreaterThan(0) + }) +}) + +describe("FeedRankingConfigSchema", () => { + it("fills every knob from an empty object", () => { + const parsed = FeedRankingConfigSchema.parse({}) + expect(parsed).toEqual(DEFAULT_FEED_RANKING) + expect(Object.keys(parsed)).toHaveLength(27) + }) + + it("merges a partial override onto the defaults", () => { + const parsed = FeedRankingConfigSchema.parse({ halfLifeHours: 12, followWeight: 70 }) + expect(parsed.halfLifeHours).toBe(12) + expect(parsed.followWeight).toBe(70) + expect(parsed.minScore).toBe(DEFAULT_FEED_RANKING.minScore) + }) + + it("rejects an unknown key rather than silently ignoring the knob", () => { + expect(FeedRankingConfigSchema.safeParse({ folowWeight: 70 }).success).toBe(false) + }) + + it("rejects an out-of-range value", () => { + expect(FeedRankingConfigSchema.safeParse({ decayFloor: 1.5 }).success).toBe(false) + expect(FeedRankingConfigSchema.safeParse({ halfLifeHours: 0 }).success).toBe(false) + expect(FeedRankingConfigSchema.safeParse({ candidateCap: 5000 }).success).toBe(false) + }) + + it("rejects a non-integer where the knob is a count", () => { + expect(FeedRankingConfigSchema.safeParse({ minPageItems: 2.5 }).success).toBe(false) + }) +}) From 107e6a1a1373d3f2d1ccb618a2838cff945f1091 Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 15 Sep 2026 23:53:48 -0700 Subject: [PATCH 07/54] serve cursor pages from the ranked snapshot and re-check blocks on hydrate --- .../src/services/post-repository.drizzle.ts | 5 ++ services/api/src/services/post-service.ts | 72 ++++++++++++------ .../api/test/unit/feed-ranked-service.test.ts | 75 +++++++++++++++++++ 3 files changed, 130 insertions(+), 22 deletions(-) diff --git a/services/api/src/services/post-repository.drizzle.ts b/services/api/src/services/post-repository.drizzle.ts index e1a87ae5..731ee5b7 100644 --- a/services/api/src/services/post-repository.drizzle.ts +++ b/services/api/src/services/post-repository.drizzle.ts @@ -1209,6 +1209,11 @@ export function makeDrizzlePostRepository(sql: Sql, deps: PostRepoDeps): PostRep WHERE p.id = ANY(${[...ids]}::uuid[]) AND p.deleted_at IS NULL AND p.visibility = 'public' + AND NOT EXISTS ( + SELECT 1 FROM user_blocks b + WHERE (b.blocker_id = ${viewerId} AND b.blocked_id = p.author_id) + OR (b.blocker_id = p.author_id AND b.blocked_id = ${viewerId}) + ) ` const order = new Map(ids.map((id, index) => [id, index])) rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0)) diff --git a/services/api/src/services/post-service.ts b/services/api/src/services/post-service.ts index d5e55698..2889455e 100644 --- a/services/api/src/services/post-service.ts +++ b/services/api/src/services/post-service.ts @@ -138,6 +138,46 @@ export function makePostService(deps: PostServiceDeps): PostService { } } + async function pageFrom( + viewerId: string, + entries: readonly { id: string; score: number }[], + limit: number, + ): Promise { + const pageEntries = entries.slice(0, limit) + const hasMore = entries.length > pageEntries.length + const last = pageEntries[pageEntries.length - 1] + const nextCursor = + hasMore && last !== undefined + ? formatFeedScoreCursor({ score: last.score, postId: last.id }) + : null + + const items = await deps.repo.hydrateByIds( + pageEntries.map((entry) => entry.id), + viewerId, + ) + + if (deps.feedPresence !== undefined) { + void deps.feedPresence.recordServed( + viewerId, + items.map((item) => item.id), + ) + } + + return { items, nextCursor } + } + + function continueFromSnapshot( + viewerId: string, + snapshot: readonly { id: string; score: number }[], + cursor: { score: number; postId: string }, + limit: number, + ): Promise { + const after = snapshot.filter((entry) => + isAfterFeedScoreCursor({ score: entry.score, postId: entry.id }, cursor), + ) + return pageFrom(viewerId, after, limit) + } + async function rankedFeed( viewerId: string, query: HomeFeedQuery, @@ -147,6 +187,13 @@ export function makePostService(deps: PostServiceDeps): PostService { const cursor = parseFeedScoreCursor(query.cursor) const isFirstPage = cursor === null + if (cursor !== null && deps.feedPresence !== undefined) { + const snapshot = await deps.feedPresence.readSnapshot(viewerId, query.filter) + if (snapshot !== null) { + return continueFromSnapshot(viewerId, snapshot, cursor, limit) + } + } + const rows = await deps.repo.feedCandidates({ viewerId, filter: query.filter, @@ -178,30 +225,11 @@ export function makePostService(deps: PostServiceDeps): PostService { isAfterFeedScoreCursor({ score: entry.score, postId: entry.id }, cursor), ) - const pageEntries = after.slice(0, limit) - const hasMore = after.length > pageEntries.length - const last = pageEntries[pageEntries.length - 1] - const nextCursor = - hasMore && last !== undefined - ? formatFeedScoreCursor({ score: last.score, postId: last.id }) - : null - - const items = await deps.repo.hydrateByIds( - pageEntries.map((entry) => entry.id), - viewerId, - ) - - if (deps.feedPresence !== undefined) { - if (isFirstPage) { - void deps.feedPresence.writeSnapshot(viewerId, query.filter, ranked) - } - void deps.feedPresence.recordServed( - viewerId, - items.map((item) => item.id), - ) + if (isFirstPage && deps.feedPresence !== undefined) { + void deps.feedPresence.writeSnapshot(viewerId, query.filter, ranked) } - return { items, nextCursor } + return pageFrom(viewerId, after, limit) } async function safeNotify(fn: () => Promise): Promise { diff --git a/services/api/test/unit/feed-ranked-service.test.ts b/services/api/test/unit/feed-ranked-service.test.ts index 8f421f3e..e7c8cf27 100644 --- a/services/api/test/unit/feed-ranked-service.test.ts +++ b/services/api/test/unit/feed-ranked-service.test.ts @@ -204,6 +204,81 @@ describe("ranked feed: cursor continuation covers every item exactly once", () = }) }) +describe("ranked feed: warm snapshot continues a cursor without re-ranking", () => { + const rows = Array.from({ length: 60 }, (_, i) => candidateRow(i + 1)) + + function harness() { + const cache = new InMemoryCacheClient(() => Date.now()) + const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + let candidateQueries = 0 + const svc = makePostService({ + repo: repoOver({ + feedCandidates: () => { + candidateQueries += 1 + return Promise.resolve(rows) + }, + }), + sql: throwingSql, + feedPresence: presence, + now: () => NOW, + }) + return { svc, presence, candidateQueries: () => candidateQueries } + } + + it("re-ranks on page 1 and slices the snapshot on page 2", async () => { + const { svc, candidateQueries } = harness() + const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + expect(candidateQueries()).toBe(1) + + const second = await svc.homeFeed(VIEWER, { + filter: "all", + limit: 20, + cursor: first.nextCursor!, + }) + expect(candidateQueries()).toBe(1) + expect(second.items).toHaveLength(20) + + const firstIds = new Set(first.items.map((item) => item.id)) + expect(second.items.some((item) => firstIds.has(item.id))).toBe(false) + }) + + it("produces the same page whether the snapshot is warm or cold", async () => { + const warm = harness() + const first = await warm.svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + const fromSnapshot = await warm.svc.homeFeed(VIEWER, { + filter: "all", + limit: 20, + cursor: first.nextCursor!, + }) + + const cold = harness() + await cold.svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + await cold.presence.readSnapshot(VIEWER, "all") + const coldPresence = makeFeedPresence({ config: DEFAULT_FEED_RANKING }) + const coldSvc = makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: coldPresence, + now: () => NOW, + }) + const fromRerank = await coldSvc.homeFeed(VIEWER, { + filter: "all", + limit: 20, + cursor: first.nextCursor!, + }) + + expect(fromSnapshot.items.map((i) => i.id)).toEqual(fromRerank.items.map((i) => i.id)) + expect(fromSnapshot.nextCursor).toBe(fromRerank.nextCursor) + }) + + it("keys the snapshot per filter, so the events tab does not slice the all tab's page", async () => { + const { svc, candidateQueries } = harness() + const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + await svc.homeFeed(VIEWER, { filter: "events", limit: 20, cursor: first.nextCursor! }) + expect(candidateQueries()).toBe(2) + }) +}) + describe("ranked feed: served set feeds the seen discount", () => { it("records the served page so the next refresh demotes what was already read", async () => { const cache = new InMemoryCacheClient(() => Date.now()) From 51e23d99279ba32059eb83f3d2d0e56ef901f9ae Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 15 Sep 2026 23:53:48 -0700 Subject: [PATCH 08/54] document the posts.geom backfill and its coarsening implication --- docs/location-coarsening-assessment.md | 18 ++++++++++ docs/out-of-band-indexes.md | 47 ++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/docs/location-coarsening-assessment.md b/docs/location-coarsening-assessment.md index 08aa8fa3..b9ca5f18 100644 --- a/docs/location-coarsening-assessment.md +++ b/docs/location-coarsening-assessment.md @@ -22,6 +22,24 @@ The owner's own detail/list (`mine: true`) and the create-time idempotency snapshot also carry full precision; routing to a jurisdiction uses the precise `geom`. +**A second stored copy exists as of migration 0172 (issue #100).** `posts.geom` +denormalises the linked report's or event's point onto the post row so the +ranked home feed's proximity pool is one bounded KNN scan. It is NOT a new +class of data and NOT a new exposure: the value is copied server-side from a +coordinate this platform already publishes at full precision on the linked +report or event, it is never populated from a client-supplied coordinate, and +it is never projected into any DTO — it only orders the feed, and the feed +emits `PostDTO`, which carries no post-level coordinate at all. The only +derived value that leaves the server is the ranking score. + +Consequence for any future coarsening decision: rounding the projected +`lat`/`lng` would NOT cover `posts.geom`, which would keep ordering the feed at +full precision. That is almost certainly the desired behaviour (proximity +ranking is the point), but it must be a stated decision rather than an +oversight, and a policy that requires coarsening at REST — not just in +transit — has to cover this column, `reports.geom`, `cleanups.geom` and +`users.last_activity_geom` together. + ## Is a backend-only coarsening possible without a shared-contract change? **Technically yes.** `ReportDTO.lat/lng` and `ReportPinDTO.lat/lng` are diff --git a/docs/out-of-band-indexes.md b/docs/out-of-band-indexes.md index 8e9c88ba..49308026 100644 --- a/docs/out-of-band-indexes.md +++ b/docs/out-of-band-indexes.md @@ -104,6 +104,53 @@ statement through the exported `explainSuggestFollows`. The offline half — that the emitted SQL really is a `CROSS JOIN LATERAL` and not a same-level cross join — is `test/unit/social-suggest-sql.test.ts`. +### `posts.geom` backfill (migration 0172, issue #100) — data, not an index + +`posts` is NOT a hot table, so `posts_geom_gist` and +`posts_author_public_recent_idx` are built inline by migrations 0172 and 0173 +and need nothing here. What DOES need an out-of-band run is the **backfill**: +0172 adds the column but deliberately populates no rows, because one `UPDATE` +over the whole table inside the migration's single transaction is a lock +hazard. The migration RAISEs a `WARNING` naming how many attached posts are +still unpopulated. + +Run it once the deploy is healthy — keyset-paged, idempotent, safe to re-run +and safe while the API serves traffic: + +```sh +sudo -n docker exec compose-api-1 node dist/db/backfill-post-geom.js +``` + +Nothing breaks without it: a `NULL` `posts.geom` yields a `NULL` `distance_km`, +the ranker simply scores no proximity term for that post, and the in-network +and recent-public pools still fill the feed. Only the *nearby* pool is degraded. + +Verify: + +```sql +SELECT count(*) FROM posts + WHERE geom IS NULL AND (report_id IS NOT NULL OR event_id IS NOT NULL); +``` + +must return `0`. If `posts` has grown large enough that an inline +`CREATE INDEX` would be disruptive, build both indexes with `CONCURRENTLY` +BEFORE deploying — the migrations' `IF NOT EXISTS` guards then no-op: + +```sql +CREATE INDEX CONCURRENTLY IF NOT EXISTS posts_geom_gist + ON posts USING gist (geom) + WHERE geom IS NOT NULL AND deleted_at IS NULL AND visibility = 'public'; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS posts_author_public_recent_idx + ON posts (author_id, created_at DESC, id DESC) + WHERE deleted_at IS NULL AND reply_to_id IS NULL AND visibility = 'public'; +``` + +The nearby pool's plan is asserted by +`test/integration/feed-ranked-pg.test.ts`, which `EXPLAIN`s the real statement +through the exported `explainFeedCandidates` and requires +`posts_geom_gist` with no `Seq Scan on posts`. + ### `media_assets_orphan_sweep_idx` (migration 0098, audit H13) Back the hourly orphan sweep's candidate scan (`findOrphans` /`deleteOrphan` in From a02e686527b622a3301ad08550ffb1b4ec61eaf5 Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 15 Sep 2026 23:56:52 -0700 Subject: [PATCH 09/54] batch the served-set redis writes instead of one round trip per post --- services/api/src/services/feed-presence.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/services/api/src/services/feed-presence.ts b/services/api/src/services/feed-presence.ts index 87df1175..c9366c22 100644 --- a/services/api/src/services/feed-presence.ts +++ b/services/api/src/services/feed-presence.ts @@ -114,12 +114,13 @@ export function makeFeedPresence(deps: FeedPresenceDeps): FeedPresence { const cache = deps.cache! await cache.sadd(servedKey(userId), ...unique) await cache.expire(servedKey(userId), cfg.servedTtlSeconds) - for (const postId of unique) { - const key = viewersKey(postId) - if ((await cache.scard(key)) >= cfg.viewerFanoutMax) continue - await cache.sadd(key, userId) - await cache.expire(key, cfg.servedTtlSeconds) - } + + const sizes = await Promise.all(unique.map((postId) => cache.scard(viewersKey(postId)))) + const admitted = unique.filter((_, i) => (sizes[i] ?? 0) < cfg.viewerFanoutMax) + await Promise.all(admitted.map((postId) => cache.sadd(viewersKey(postId), userId))) + await Promise.all( + admitted.map((postId) => cache.expire(viewersKey(postId), cfg.servedTtlSeconds)), + ) }) }, From 11144440904043fdd1b17a3dbecaae909d49ddd2 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 00:13:55 -0700 Subject: [PATCH 10/54] paginate the ranked feed from the snapshot only, never a re-ranked clock --- docs/out-of-band-indexes.md | 34 +++++-- services/api/drizzle/0172_posts_geom.sql | 17 ++-- services/api/src/db/schema/posts.ts | 4 +- services/api/src/di.ts | 2 - services/api/src/routes/posts.routes.ts | 4 +- .../admin/admin-report-repository.drizzle.ts | 2 +- services/api/src/services/feed-presence.ts | 10 ++- .../src/services/post-repository.drizzle.ts | 32 ++++--- services/api/src/services/post-service.ts | 76 +++++++++------- .../test/integration/feed-ranked-pg.test.ts | 12 ++- .../api/test/unit/feed-candidates-sql.test.ts | 10 ++- .../api/test/unit/feed-ranked-service.test.ts | 89 ++++++++++++++----- services/api/test/unit/post-service.test.ts | 4 +- .../api/test/unit/rate-limit-plugin.test.ts | 2 + 14 files changed, 205 insertions(+), 93 deletions(-) diff --git a/docs/out-of-band-indexes.md b/docs/out-of-band-indexes.md index 49308026..3d9e29fd 100644 --- a/docs/out-of-band-indexes.md +++ b/docs/out-of-band-indexes.md @@ -111,8 +111,8 @@ cross join — is `test/unit/social-suggest-sql.test.ts`. and need nothing here. What DOES need an out-of-band run is the **backfill**: 0172 adds the column but deliberately populates no rows, because one `UPDATE` over the whole table inside the migration's single transaction is a lock -hazard. The migration RAISEs a `WARNING` naming how many attached posts are -still unpopulated. +hazard. The migration RAISEs a `WARNING` when any backfillable post is still +unpopulated. Run it once the deploy is healthy — keyset-paged, idempotent, safe to re-run and safe while the API serves traffic: @@ -128,18 +128,36 @@ and recent-public pools still fill the feed. Only the *nearby* pool is degraded. Verify: ```sql -SELECT count(*) FROM posts - WHERE geom IS NULL AND (report_id IS NOT NULL OR event_id IS NOT NULL); +SELECT count(*) + FROM posts p + LEFT JOIN reports r ON r.id = p.report_id + LEFT JOIN cleanups c ON c.id = p.event_id + WHERE p.geom IS NULL AND COALESCE(r.geom, c.geom) IS NOT NULL; ``` -must return `0`. If `posts` has grown large enough that an inline -`CREATE INDEX` would be disruptive, build both indexes with `CONCURRENTLY` -BEFORE deploying — the migrations' `IF NOT EXISTS` guards then no-op: +must return `0`. (The `COALESCE(...) IS NOT NULL` term matters: a post linked to +a row whose own `geom` is null is not backfillable, and counting it would make +this check permanently unsatisfiable. The migration's `DO` block uses the same +predicate.) + +**`posts.geom` is an insert-time snapshot, not a live mirror.** It is written +once by `createPost` and never updated: there is no trigger and no relocation +hook, and the backfill only touches rows where `geom IS NULL`. If a report or +cleanup is later moved to a new coordinate, every post already linked to it +keeps ranking against the OLD point indefinitely. That is acceptable for feed +proximity (the post was about the place as it was), but it is a deliberate +property, not an oversight — if live tracking is ever wanted, the relocation +paths must update the derived posts explicitly. + +If `posts` has grown large enough that an inline `CREATE INDEX` would be +disruptive, build both indexes with `CONCURRENTLY` BEFORE deploying — the +migrations' `IF NOT EXISTS` guards then no-op: ```sql CREATE INDEX CONCURRENTLY IF NOT EXISTS posts_geom_gist ON posts USING gist (geom) - WHERE geom IS NOT NULL AND deleted_at IS NULL AND visibility = 'public'; + WHERE geom IS NOT NULL AND deleted_at IS NULL AND reply_to_id IS NULL + AND visibility = 'public'; CREATE INDEX CONCURRENTLY IF NOT EXISTS posts_author_public_recent_idx ON posts (author_id, created_at DESC, id DESC) diff --git a/services/api/drizzle/0172_posts_geom.sql b/services/api/drizzle/0172_posts_geom.sql index 0b3f398f..b04c94f9 100644 --- a/services/api/drizzle/0172_posts_geom.sql +++ b/services/api/drizzle/0172_posts_geom.sql @@ -47,17 +47,18 @@ COMMENT ON COLUMN posts.geom IS CREATE INDEX IF NOT EXISTS posts_geom_gist ON posts USING gist (geom) - WHERE geom IS NOT NULL AND deleted_at IS NULL AND visibility = 'public'; + WHERE geom IS NOT NULL AND deleted_at IS NULL AND reply_to_id IS NULL AND visibility = 'public'; DO $$ -DECLARE - pending bigint; BEGIN - SELECT count(*) INTO pending - FROM posts - WHERE geom IS NULL AND (report_id IS NOT NULL OR event_id IS NOT NULL); + PERFORM 1 + FROM posts p + LEFT JOIN reports r ON r.id = p.report_id + LEFT JOIN cleanups c ON c.id = p.event_id + WHERE p.geom IS NULL AND COALESCE(r.geom, c.geom) IS NOT NULL + LIMIT 1; - IF pending > 0 THEN - RAISE WARNING 'posts.geom is unpopulated for % attached post(s) - run pnpm db:backfill:post-geom after this deploy is healthy; those posts score no feed proximity term until then', pending; + IF FOUND THEN + RAISE WARNING 'posts.geom is unpopulated for at least one attached post - run pnpm db:backfill:post-geom after this deploy is healthy; those posts score no feed proximity term until then'; END IF; END $$; diff --git a/services/api/src/db/schema/posts.ts b/services/api/src/db/schema/posts.ts index 14e3c903..13217fa5 100644 --- a/services/api/src/db/schema/posts.ts +++ b/services/api/src/db/schema/posts.ts @@ -81,7 +81,9 @@ export const posts = pgTable( .where(sql`deleted_at IS NULL AND reply_to_id IS NULL AND visibility = 'public'`), index("posts_geom_gist") .using("gist", t.geom) - .where(sql`geom IS NOT NULL AND deleted_at IS NULL AND visibility = 'public'`), + .where( + sql`geom IS NOT NULL AND deleted_at IS NULL AND reply_to_id IS NULL AND visibility = 'public'`, + ), index("posts_organization_created_idx") .on(t.organizationId, t.createdAt.desc()) .where(sql`${t.organizationId} IS NOT NULL AND deleted_at IS NULL`), diff --git a/services/api/src/di.ts b/services/api/src/di.ts index cda4206d..0770cbc4 100644 --- a/services/api/src/di.ts +++ b/services/api/src/di.ts @@ -127,7 +127,6 @@ export interface Container { getAffiliationLoader(): AffiliationLoader getPostRepo(): PostRepository getPostService(): PostService - getFeedPresence(): FeedPresence getNotificationService(logger?: NotificationLogger): NotificationService getCounterStore(): CounterStore getCache(): CacheClient @@ -541,7 +540,6 @@ export function buildContainer(env: Env): Container { getAffiliationLoader, getPostRepo, getPostService, - getFeedPresence, getNotificationService, getCounterStore, getCache, diff --git a/services/api/src/routes/posts.routes.ts b/services/api/src/routes/posts.routes.ts index dbc7f0e1..0cd2b3cf 100644 --- a/services/api/src/routes/posts.routes.ts +++ b/services/api/src/routes/posts.routes.ts @@ -22,6 +22,8 @@ const PostIdParamsSchema = z.object({ id: IdSchema }).strict() export const FEED_COUNTS_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute" }) +export const HOME_FEED_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute" }) + export const CREATE_POST_RATE_LIMIT = perIdentity({ max: 120, timeWindow: "1 minute" }) export const POST_INTERACTION_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute" }) @@ -106,7 +108,7 @@ export async function registerPostRoutes(app: FastifyInstance, container: Contai return { lat: approximate.lat, lng: approximate.lng } } - route(app, "homeFeed", async (request, reply) => { + route(app, "homeFeed", { config: { rateLimit: HOME_FEED_RATE_LIMIT } }, async (request, reply) => { const userId = request.auth.userId const query = parse(HomeFeedQuerySchema, request.query) const location = viewerLocationOf(request) diff --git a/services/api/src/services/admin/admin-report-repository.drizzle.ts b/services/api/src/services/admin/admin-report-repository.drizzle.ts index 304abd4d..adbcd71b 100644 --- a/services/api/src/services/admin/admin-report-repository.drizzle.ts +++ b/services/api/src/services/admin/admin-report-repository.drizzle.ts @@ -147,7 +147,7 @@ function reportSelect( -- confirmations was the report_follows count; that table was dropped with the discussion system. -- Kept as a stable admin DTO field (always 0 now) so the admin UI neighbors-confirmed row still parses. '0'::text AS confirmations, - EXISTS (SELECT 1 FROM media_assets m WHERE m.report_id = r.id AND m.status = 'ready') AS has_photo, + EXISTS (SELECT 1 FROM media_assets m WHERE m.report_id = r.id AND m.status = 'ready' AND m.served_key IS NOT NULL) AS has_photo, pm.id AS preview_id, pm.kind AS preview_kind, pm.served_key AS preview_key, diff --git a/services/api/src/services/feed-presence.ts b/services/api/src/services/feed-presence.ts index c9366c22..44288933 100644 --- a/services/api/src/services/feed-presence.ts +++ b/services/api/src/services/feed-presence.ts @@ -24,6 +24,7 @@ export interface FeedSnapshotEntry { export interface FeedPresence { readSnapshot(userId: string, filter: string): Promise writeSnapshot(userId: string, filter: string, ranked: readonly RankedCandidate[]): Promise + touchSnapshot(userId: string, filter: string): Promise seenBy(userId: string, postIds: readonly string[]): Promise> recordServed(userId: string, postIds: readonly string[]): Promise viewersOf(postId: string): Promise @@ -98,6 +99,13 @@ export function makeFeedPresence(deps: FeedPresenceDeps): FeedPresence { }) }, + touchSnapshot(userId: string, filter: string): Promise { + if (cfg.snapshotTtlSeconds <= 0) return Promise.resolve() + return guarded(undefined, "snapshot touch", async () => { + await deps.cache!.expire(snapshotKey(userId, filter), cfg.snapshotTtlSeconds) + }) + }, + seenBy(userId: string, postIds: readonly string[]): Promise> { if (postIds.length === 0) return Promise.resolve(new Set()) return guarded(new Set(), "served-set read", async () => { @@ -130,7 +138,7 @@ export function makeFeedPresence(deps: FeedPresenceDeps): FeedPresence { const key = viewersKey(postId) const size = await cache.scard(key) if (size === 0) return [] - if (size > cfg.viewerFanoutMax) { + if (size >= cfg.viewerFanoutMax) { deps.logger?.debug?.( { postId, size, cap: cfg.viewerFanoutMax }, "feed presence: viewer fanout skipped (over cap)", diff --git a/services/api/src/services/post-repository.drizzle.ts b/services/api/src/services/post-repository.drizzle.ts index 731ee5b7..ce606e0d 100644 --- a/services/api/src/services/post-repository.drizzle.ts +++ b/services/api/src/services/post-repository.drizzle.ts @@ -103,9 +103,6 @@ export const FEED_IN_NETWORK_POOL = 250 export const FEED_NEARBY_POOL = 150 export const FEED_RECENT_POOL = 150 -export const FEED_GEOM_INDEX = "posts_geom_gist" -export const FEED_AUTHOR_INDEX = "posts_author_public_recent_idx" - const KM_PER_DEGREE_LAT = 111.32 const MIN_LATITUDE_COSINE = 0.25 @@ -351,6 +348,11 @@ export function feedCandidatesStatement( AND p.reply_to_id IS NULL AND p.visibility = 'public' AND p.created_at >= (SELECT since FROM eligible_window) + AND NOT EXISTS ( + SELECT 1 FROM user_blocks b + WHERE (b.blocker_id = ${viewerId} AND b.blocked_id = p.author_id) + OR (b.blocker_id = p.author_id AND b.blocked_id = ${viewerId}) + ) ${filterClause} ` return sql` @@ -413,9 +415,12 @@ export function feedCandidatesStatement( LIMIT ${FEED_RECENT_POOL} ), pool AS ( - SELECT id FROM in_network - UNION SELECT id FROM nearby - UNION SELECT id FROM recent_public + SELECT DISTINCT ON (id) id, source FROM ( + SELECT id, 0 AS source FROM in_network + UNION ALL SELECT id, 1 AS source FROM nearby + UNION ALL SELECT id, 2 AS source FROM recent_public + ) merged + ORDER BY id, source ) SELECT p.id, @@ -460,11 +465,7 @@ export function feedCandidatesStatement( ORDER BY COALESCE(o.id = au.primary_organization_id, false) DESC, m.joined_at ASC, o.id ASC LIMIT 1 ) aff ON true - WHERE NOT EXISTS ( - SELECT 1 FROM user_blocks b - WHERE (b.blocker_id = ${viewerId} AND b.blocked_id = p.author_id) - OR (b.blocker_id = p.author_id AND b.blocked_id = ${viewerId}) - ) + ORDER BY pool.source, p.created_at DESC, p.id DESC LIMIT ${args.candidateCap} ` } @@ -1223,8 +1224,13 @@ export function makeDrizzlePostRepository(sql: Sql, deps: PostRepoDeps): PostRep async followerIdsOf(authorId: string, limit: number): Promise { if (limit <= 0) return [] const rows = await sql<{ follower_id: string }[]>` - SELECT follower_id FROM follows_people - WHERE followee_id = ${authorId} + SELECT f.follower_id FROM follows_people f + WHERE f.followee_id = ${authorId} + AND NOT EXISTS ( + SELECT 1 FROM user_blocks b + WHERE (b.blocker_id = f.follower_id AND b.blocked_id = ${authorId}) + OR (b.blocker_id = ${authorId} AND b.blocked_id = f.follower_id) + ) LIMIT ${limit} ` return rows.map((r) => r.follower_id) diff --git a/services/api/src/services/post-service.ts b/services/api/src/services/post-service.ts index 2889455e..adca3b39 100644 --- a/services/api/src/services/post-service.ts +++ b/services/api/src/services/post-service.ts @@ -29,6 +29,7 @@ import { type RepliesPage, } from "./post-repository.drizzle.js" import { applyCutoff, rankCandidates, type FeedCandidate } from "./feed-ranking.js" +import { mapWithLimit } from "./media-presign.js" import type { FeedPresence } from "./feed-presence.js" import type { PostNotifier } from "./notification-service.js" @@ -42,6 +43,15 @@ function isLegacyTimeCursor(cursor: string | null | undefined): boolean { return parseTimeCursor(cursor) !== null } +export const FEED_FANOUT_CONCURRENCY = 16 + +export const FEED_DISTANCE_RESOLUTION_KM = 1 + +function coarseDistanceKm(distanceKm: number | null): number | null { + if (distanceKm === null || !Number.isFinite(distanceKm)) return null + return Math.round(distanceKm / FEED_DISTANCE_RESOLUTION_KM) * FEED_DISTANCE_RESOLUTION_KM +} + export interface FeedViewerLocation { lat: number lng: number @@ -89,7 +99,9 @@ export function makePostService(deps: PostServiceDeps): PostService { function fanout(recipients: readonly string[], topic: "feed" | "feed_counts", id: string): void { const channel = deps.userChannel if (channel === undefined || recipients.length === 0) return - void channel.publishToUsers(recipients, { topic, id }).catch((err: unknown) => { + void mapWithLimit([...recipients], FEED_FANOUT_CONCURRENCY, (userId) => + channel.publishToUser(userId, { topic, id }), + ).catch((err: unknown) => { deps.logger?.warn({ err, topic, id }, "post: realtime feed fanout failed (suppressed)") }) } @@ -133,7 +145,7 @@ export function makePostService(deps: PostServiceDeps): PostService { authorIsViewer: row.author_is_viewer, viewerMentioned: row.viewer_mentioned, authorOrgVerified: row.author_org_verified, - distanceKm: row.distance_km === null ? null : Number(row.distance_km), + distanceKm: coarseDistanceKm(row.distance_km === null ? null : Number(row.distance_km)), alreadySeen: seen.has(row.id), } } @@ -147,7 +159,7 @@ export function makePostService(deps: PostServiceDeps): PostService { const hasMore = entries.length > pageEntries.length const last = pageEntries[pageEntries.length - 1] const nextCursor = - hasMore && last !== undefined + hasMore && last !== undefined && deps.feedPresence !== undefined && viewerId !== NIL_VIEWER_ID ? formatFeedScoreCursor({ score: last.score, postId: last.id }) : null @@ -156,22 +168,34 @@ export function makePostService(deps: PostServiceDeps): PostService { viewerId, ) - if (deps.feedPresence !== undefined) { - void deps.feedPresence.recordServed( - viewerId, - items.map((item) => item.id), - ) + if (deps.feedPresence !== undefined && viewerId !== NIL_VIEWER_ID) { + void deps.feedPresence + .recordServed( + viewerId, + items.map((item) => item.id), + ) + .catch((err: unknown) => { + deps.logger?.warn({ err }, "post: feed served-set write failed (suppressed)") + }) } return { items, nextCursor } } - function continueFromSnapshot( + async function continueRankedPage( viewerId: string, - snapshot: readonly { id: string; score: number }[], + filter: string, cursor: { score: number; postId: string }, limit: number, ): Promise { + if (deps.feedPresence === undefined || viewerId === NIL_VIEWER_ID) { + return { items: [], nextCursor: null } + } + const snapshot = await deps.feedPresence.readSnapshot(viewerId, filter) + if (snapshot === null) return { items: [], nextCursor: null } + void deps.feedPresence.touchSnapshot(viewerId, filter).catch((err: unknown) => { + deps.logger?.warn({ err }, "post: feed snapshot touch failed (suppressed)") + }) const after = snapshot.filter((entry) => isAfterFeedScoreCursor({ score: entry.score, postId: entry.id }, cursor), ) @@ -185,13 +209,9 @@ export function makePostService(deps: PostServiceDeps): PostService { ): Promise { const limit = query.limit ?? POSTS_DEFAULT_LIMIT const cursor = parseFeedScoreCursor(query.cursor) - const isFirstPage = cursor === null - if (cursor !== null && deps.feedPresence !== undefined) { - const snapshot = await deps.feedPresence.readSnapshot(viewerId, query.filter) - if (snapshot !== null) { - return continueFromSnapshot(viewerId, snapshot, cursor, limit) - } + if (cursor !== null) { + return continueRankedPage(viewerId, query.filter, cursor, limit) } const rows = await deps.repo.feedCandidates({ @@ -205,7 +225,7 @@ export function makePostService(deps: PostServiceDeps): PostService { }) const seen = - deps.feedPresence === undefined + deps.feedPresence === undefined || viewerId === NIL_VIEWER_ID ? new Set() : await deps.feedPresence.seenBy( viewerId, @@ -217,19 +237,15 @@ export function makePostService(deps: PostServiceDeps): PostService { feedConfig, nowMs(), ) - const kept = applyCutoff(ranked, feedConfig, isFirstPage) - const after = - cursor === null - ? kept - : kept.filter((entry) => - isAfterFeedScoreCursor({ score: entry.score, postId: entry.id }, cursor), - ) + const kept = applyCutoff(ranked, feedConfig, true) - if (isFirstPage && deps.feedPresence !== undefined) { - void deps.feedPresence.writeSnapshot(viewerId, query.filter, ranked) + if (deps.feedPresence !== undefined && viewerId !== NIL_VIEWER_ID) { + void deps.feedPresence.writeSnapshot(viewerId, query.filter, kept).catch((err: unknown) => { + deps.logger?.warn({ err }, "post: feed snapshot write failed (suppressed)") + }) } - return pageFrom(viewerId, after, limit) + return pageFrom(viewerId, kept, limit) } async function safeNotify(fn: () => Promise): Promise { @@ -397,8 +413,7 @@ export function makePostService(deps: PostServiceDeps): PostService { async unlikePost(id: string, viewerId: string): Promise { const subject = await requireReadable(id, viewerId) - const removed = await deps.repo.unlike(subject.id, viewerId) - if (removed) await announceCountChange(subject.id, viewerId) + await deps.repo.unlike(subject.id, viewerId) return hydrateOrThrow(id, viewerId) }, @@ -439,8 +454,7 @@ export function makePostService(deps: PostServiceDeps): PostService { async unrepostPost(id: string, viewerId: string): Promise { await requireReadable(id, viewerId) - const { targetId, removed } = await deps.repo.unrepost(id, viewerId) - if (removed) await announceCountChange(targetId, viewerId) + const { targetId } = await deps.repo.unrepost(id, viewerId) return hydrateOrThrow(targetId, viewerId) }, diff --git a/services/api/test/integration/feed-ranked-pg.test.ts b/services/api/test/integration/feed-ranked-pg.test.ts index 62de234f..e5ea38ae 100644 --- a/services/api/test/integration/feed-ranked-pg.test.ts +++ b/services/api/test/integration/feed-ranked-pg.test.ts @@ -2,6 +2,8 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest" import { DEFAULT_FEED_RANKING } from "@civfix/shared" import { withPg, type PgHarness, testHandle } from "../helpers/pg.js" +import { makeFeedPresence } from "../../src/services/feed-presence.js" +import { InMemoryCacheClient } from "../../src/auth/cache.js" import { explainFeedCandidates, makeDrizzlePostRepository, @@ -31,7 +33,15 @@ describe.skipIf(!pg)("ranked home feed: candidate SQL (integration)", () => { } function makeService(repo: PostRepository = makeRepo()): PostService { - return makePostService({ repo, sql: h.sql, feedRanking: DEFAULT_FEED_RANKING }) + return makePostService({ + repo, + sql: h.sql, + feedRanking: DEFAULT_FEED_RANKING, + feedPresence: makeFeedPresence({ + cache: new InMemoryCacheClient(() => Date.now()), + config: DEFAULT_FEED_RANKING, + }), + }) } function args(over: Partial = {}): FeedCandidateArgs { diff --git a/services/api/test/unit/feed-candidates-sql.test.ts b/services/api/test/unit/feed-candidates-sql.test.ts index c97a1fec..cc0705a6 100644 --- a/services/api/test/unit/feed-candidates-sql.test.ts +++ b/services/api/test/unit/feed-candidates-sql.test.ts @@ -79,12 +79,18 @@ describe("feed candidate SQL: security invariants", () => { expect(replies.length).toBeGreaterThanOrEqual(3) }) - it("applies the block exclusion symmetrically, once over the union", async () => { + it("applies the block exclusion symmetrically, inside every pool", async () => { const stmt = await emitted() expect(stmt.sql).toContain("FROM user_blocks b") expect(stmt.sql).toMatch(/b\.blocker_id = \? AND b\.blocked_id = p\.author_id/) expect(stmt.sql).toMatch(/b\.blocker_id = p\.author_id AND b\.blocked_id = \?/) - expect(stmt.sql.match(/FROM user_blocks b/g)).toHaveLength(1) + expect(stmt.sql.match(/FROM user_blocks b/g)?.length).toBeGreaterThanOrEqual(3) + }) + + it("cuts the candidate pool deterministically, in-network first", async () => { + const stmt = await emitted() + expect(stmt.sql).toMatch(/ORDER BY pool\.source, p\.created_at DESC, p\.id DESC/) + expect(stmt.sql).toContain("DISTINCT ON (id)") }) it("never counts a suspended or deleted organization as verified", async () => { diff --git a/services/api/test/unit/feed-ranked-service.test.ts b/services/api/test/unit/feed-ranked-service.test.ts index e7c8cf27..bd3d21d6 100644 --- a/services/api/test/unit/feed-ranked-service.test.ts +++ b/services/api/test/unit/feed-ranked-service.test.ts @@ -136,9 +136,11 @@ describe("ranked feed: cursor continuation covers every item exactly once", () = const rows = Array.from({ length: 60 }, (_, i) => candidateRow(i + 1)) function service(): PostService { + const cache = new InMemoryCacheClient(() => Date.now()) return makePostService({ repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), sql: throwingSql, + feedPresence: makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }), now: () => NOW, }) } @@ -242,40 +244,83 @@ describe("ranked feed: warm snapshot continues a cursor without re-ranking", () expect(second.items.some((item) => firstIds.has(item.id))).toBe(false) }) - it("produces the same page whether the snapshot is warm or cold", async () => { - const warm = harness() - const first = await warm.svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) - const fromSnapshot = await warm.svc.homeFeed(VIEWER, { + it("ends the feed rather than re-ranking when the snapshot has expired", async () => { + const { svc } = harness() + const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + + const expired = makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: makeFeedPresence({ + cache: new InMemoryCacheClient(() => Date.now()), + config: DEFAULT_FEED_RANKING, + }), + now: () => NOW, + }) + const page = await expired.homeFeed(VIEWER, { filter: "all", limit: 20, cursor: first.nextCursor!, }) - const cold = harness() - await cold.svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) - await cold.presence.readSnapshot(VIEWER, "all") - const coldPresence = makeFeedPresence({ config: DEFAULT_FEED_RANKING }) - const coldSvc = makePostService({ + expect(page.items).toEqual([]) + expect(page.nextCursor).toBeNull() + }) + + it("re-ranking a moved clock never re-serves a page-1 item", async () => { + const { svc } = harness() + const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + + const later = makePostService({ repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), sql: throwingSql, - feedPresence: coldPresence, - now: () => NOW, + feedPresence: makeFeedPresence({ + cache: new InMemoryCacheClient(() => Date.now()), + config: DEFAULT_FEED_RANKING, + }), + now: () => NOW + 10 * 60_000, }) - const fromRerank = await coldSvc.homeFeed(VIEWER, { + const page = await later.homeFeed(VIEWER, { filter: "all", limit: 20, cursor: first.nextCursor!, }) - expect(fromSnapshot.items.map((i) => i.id)).toEqual(fromRerank.items.map((i) => i.id)) - expect(fromSnapshot.nextCursor).toBe(fromRerank.nextCursor) + const firstIds = new Set(first.items.map((item) => item.id)) + expect(page.items.some((item) => firstIds.has(item.id))).toBe(false) }) it("keys the snapshot per filter, so the events tab does not slice the all tab's page", async () => { - const { svc, candidateQueries } = harness() + const { svc } = harness() + const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + const events = await svc.homeFeed(VIEWER, { + filter: "events", + limit: 20, + cursor: first.nextCursor!, + }) + expect(events.items).toEqual([]) + }) + + it("keeps an active scroll alive by refreshing the snapshot TTL", async () => { + const touched: string[] = [] + const cache = new InMemoryCacheClient(() => Date.now()) + const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + const spied = { + ...presence, + touchSnapshot: (userId: string, filter: string) => { + touched.push(`${userId}:${filter}`) + return presence.touchSnapshot(userId, filter) + }, + } + const svc = makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: spied, + now: () => NOW, + }) const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) - await svc.homeFeed(VIEWER, { filter: "events", limit: 20, cursor: first.nextCursor! }) - expect(candidateQueries()).toBe(2) + await svc.homeFeed(VIEWER, { filter: "all", limit: 20, cursor: first.nextCursor! }) + expect(touched).toEqual([`${VIEWER}:all`]) }) }) @@ -313,7 +358,8 @@ describe("ranked feed: realtime fanout never blocks the request path", () => { { kind: "post", body: "hello", mediaUploadIds: [], mentionedUserIds: [] } as never, AUTHOR, ) - expect(sent).toEqual([{ users: ["f1", "f2"], signal: { topic: "feed", id: POST } }]) + expect(sent.flatMap((s) => s.users)).toEqual(["f1", "f2"]) + expect(sent.every((s) => s.signal.topic === "feed" && s.signal.id === POST)).toBe(true) }) it("publishes topic feed_counts to the post's live viewers on a like", async () => { @@ -331,9 +377,8 @@ describe("ranked feed: realtime fanout never blocks the request path", () => { }) await svc.likePost(POST, VIEWER) - expect(sent).toHaveLength(1) - expect(sent[0]!.signal).toEqual({ topic: "feed_counts", id: POST }) - expect(sent[0]!.users.sort()).toEqual(["reader-1", "reader-2"]) + expect(sent.every((s) => s.signal.topic === "feed_counts" && s.signal.id === POST)).toBe(true) + expect(sent.flatMap((s) => s.users).sort()).toEqual(["reader-1", "reader-2"]) }) it("excludes the acting user, whose own client already patched optimistically", async () => { @@ -351,7 +396,7 @@ describe("ranked feed: realtime fanout never blocks the request path", () => { }) await svc.likePost(POST, VIEWER) - expect(sent[0]!.users).toEqual(["reader-1"]) + expect(sent.flatMap((s) => s.users)).toEqual(["reader-1"]) }) it("does not publish when the interaction was a no-op (double like)", async () => { diff --git a/services/api/test/unit/post-service.test.ts b/services/api/test/unit/post-service.test.ts index 808e2e76..1d0beebf 100644 --- a/services/api/test/unit/post-service.test.ts +++ b/services/api/test/unit/post-service.test.ts @@ -192,7 +192,7 @@ describe("PostService validation + authorization", () => { ]) }) - it("routes a score cursor and a garbage cursor to the ranked path", async () => { + it("never routes a score cursor or garbage to the chronological query", async () => { const ranked: string[] = [] const chronological: string[] = [] const repo = { @@ -212,7 +212,7 @@ describe("PostService validation + authorization", () => { await svc.publicFeed({ filter: "all", cursor: "not-a-cursor" }) expect(chronological).toEqual([]) - expect(ranked).toEqual([NIL_VIEWER_ID, NIL_VIEWER_ID]) + expect(ranked).toEqual([NIL_VIEWER_ID]) }) it("rejects an attached event the author does not host/attend (403)", async () => { diff --git a/services/api/test/unit/rate-limit-plugin.test.ts b/services/api/test/unit/rate-limit-plugin.test.ts index 9dc677f7..65db9560 100644 --- a/services/api/test/unit/rate-limit-plugin.test.ts +++ b/services/api/test/unit/rate-limit-plugin.test.ts @@ -59,6 +59,7 @@ import { import { CREATE_POST_RATE_LIMIT, FEED_COUNTS_RATE_LIMIT, + HOME_FEED_RATE_LIMIT, POST_INTERACTION_RATE_LIMIT, } from "../../src/routes/posts.routes.js" import { @@ -458,6 +459,7 @@ const IDENTITY_SCOPED_LIMITS = { THREAD_READ_RATE_LIMIT, POST_INTERACTION_RATE_LIMIT, FEED_COUNTS_RATE_LIMIT, + HOME_FEED_RATE_LIMIT, PUSH_TOKEN_RATE_LIMIT, ADMIN_OUTBOUND_MAIL_RATE_LIMIT, FOLLOW_RATE_LIMIT, From e976360fe483e56d2d18a4418f6a44902c1b01f9 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 00:45:18 -0700 Subject: [PATCH 11/54] probe the feed served set with SMISMEMBER and expire its keys from creation --- services/api/src/auth/cache.ts | 25 ++++++ services/api/src/services/feed-presence.ts | 15 ++-- .../api/test/unit/admin-guard-cache.test.ts | 6 ++ services/api/test/unit/auth-session.test.ts | 6 ++ services/api/test/unit/feed-presence.test.ts | 78 ++++++++++++++++--- services/api/test/unit/host-analytics.test.ts | 2 + 6 files changed, 116 insertions(+), 16 deletions(-) diff --git a/services/api/src/auth/cache.ts b/services/api/src/auth/cache.ts index f0156b7b..49088d46 100644 --- a/services/api/src/auth/cache.ts +++ b/services/api/src/auth/cache.ts @@ -11,8 +11,10 @@ export interface CacheClient { sadd(key: string, ...members: string[]): Promise srem(key: string, ...members: string[]): Promise smembers(key: string): Promise + smismember(key: string, members: readonly string[]): Promise scard(key: string): Promise expire(key: string, ttlSeconds: number): Promise + expireNx(key: string, ttlSeconds: number): Promise } export type Clock = () => number @@ -116,6 +118,11 @@ export class InMemoryCacheClient implements CacheClient { return Promise.resolve([...(this.liveSet(key)?.members ?? [])]) } + smismember(key: string, members: readonly string[]): Promise { + const entry = this.liveSet(key) + return Promise.resolve(members.map((member) => (entry?.members.has(member) ? 1 : 0))) + } + scard(key: string): Promise { return Promise.resolve(this.liveSet(key)?.members.size ?? 0) } @@ -129,6 +136,15 @@ export class InMemoryCacheClient implements CacheClient { return Promise.resolve() } + expireNx(key: string, ttlSeconds: number): Promise { + const at = this.clock() + ttlSeconds * 1000 + const setEntry = this.liveSet(key) + if (setEntry && !Number.isFinite(setEntry.expiresAtMs)) setEntry.expiresAtMs = at + const entry = this.live(key) + if (entry && !Number.isFinite(entry.expiresAtMs)) entry.expiresAtMs = at + return Promise.resolve() + } + size(): number { return this.store.size } @@ -183,6 +199,11 @@ export class RedisCacheClient implements CacheClient { return this.redis.smembers(key) } + async smismember(key: string, members: readonly string[]): Promise { + if (members.length === 0) return [] + return this.redis.smismember(key, [...members]) + } + async scard(key: string): Promise { return this.redis.scard(key) } @@ -190,4 +211,8 @@ export class RedisCacheClient implements CacheClient { async expire(key: string, ttlSeconds: number): Promise { await this.redis.expire(key, Math.max(1, Math.ceil(ttlSeconds))) } + + async expireNx(key: string, ttlSeconds: number): Promise { + await this.redis.expire(key, Math.max(1, Math.ceil(ttlSeconds)), "NX") + } } diff --git a/services/api/src/services/feed-presence.ts b/services/api/src/services/feed-presence.ts index 44288933..16bb3d1d 100644 --- a/services/api/src/services/feed-presence.ts +++ b/services/api/src/services/feed-presence.ts @@ -6,7 +6,9 @@ export interface FeedPresenceCache { set(key: string, value: string, ttlSeconds: number): Promise sadd(key: string, ...members: string[]): Promise smembers(key: string): Promise + smismember(key: string, members: readonly string[]): Promise scard(key: string): Promise + expireNx(key: string, ttlSeconds: number): Promise expire(key: string, ttlSeconds: number): Promise } @@ -22,6 +24,7 @@ export interface FeedSnapshotEntry { } export interface FeedPresence { + readonly snapshotsAvailable: boolean readSnapshot(userId: string, filter: string): Promise writeSnapshot(userId: string, filter: string, ranked: readonly RankedCandidate[]): Promise touchSnapshot(userId: string, filter: string): Promise @@ -66,6 +69,8 @@ export function makeFeedPresence(deps: FeedPresenceDeps): FeedPresence { } return { + snapshotsAvailable: deps.cache !== undefined && cfg.snapshotTtlSeconds > 0, + readSnapshot(userId: string, filter: string): Promise { return guarded(null, "snapshot read", async () => { const raw = await deps.cache!.get(snapshotKey(userId, filter)) @@ -108,10 +113,10 @@ export function makeFeedPresence(deps: FeedPresenceDeps): FeedPresence { seenBy(userId: string, postIds: readonly string[]): Promise> { if (postIds.length === 0) return Promise.resolve(new Set()) + const unique = [...new Set(postIds)] return guarded(new Set(), "served-set read", async () => { - const members = await deps.cache!.smembers(servedKey(userId)) - const served = new Set(members) - return new Set(postIds.filter((id) => served.has(id))) + const flags = await deps.cache!.smismember(servedKey(userId), unique) + return new Set(unique.filter((_, i) => flags[i] === 1)) }) }, @@ -121,13 +126,13 @@ export function makeFeedPresence(deps: FeedPresenceDeps): FeedPresence { return guarded(undefined, "served-set write", async () => { const cache = deps.cache! await cache.sadd(servedKey(userId), ...unique) - await cache.expire(servedKey(userId), cfg.servedTtlSeconds) + await cache.expireNx(servedKey(userId), cfg.servedTtlSeconds) const sizes = await Promise.all(unique.map((postId) => cache.scard(viewersKey(postId)))) const admitted = unique.filter((_, i) => (sizes[i] ?? 0) < cfg.viewerFanoutMax) await Promise.all(admitted.map((postId) => cache.sadd(viewersKey(postId), userId))) await Promise.all( - admitted.map((postId) => cache.expire(viewersKey(postId), cfg.servedTtlSeconds)), + admitted.map((postId) => cache.expireNx(viewersKey(postId), cfg.servedTtlSeconds)), ) }) }, diff --git a/services/api/test/unit/admin-guard-cache.test.ts b/services/api/test/unit/admin-guard-cache.test.ts index 6805b786..a1b5ebfb 100644 --- a/services/api/test/unit/admin-guard-cache.test.ts +++ b/services/api/test/unit/admin-guard-cache.test.ts @@ -67,12 +67,18 @@ class SpyCache implements CacheClient { smembers(key: string): Promise { return this.inner.smembers(key) } + smismember(key: string, members: readonly string[]): Promise { + return this.inner.smismember(key, members) + } scard(key: string): Promise { return this.inner.scard(key) } expire(key: string, ttlSeconds: number): Promise { return this.inner.expire(key, ttlSeconds) } + expireNx(key: string, ttlSeconds: number): Promise { + return this.inner.expireNx(key, ttlSeconds) + } readonly sets: Array<{ key: string; value: string; ttlSeconds: number }> = [] failGet = false failSet = false diff --git a/services/api/test/unit/auth-session.test.ts b/services/api/test/unit/auth-session.test.ts index e20f4bb7..900d22ed 100644 --- a/services/api/test/unit/auth-session.test.ts +++ b/services/api/test/unit/auth-session.test.ts @@ -349,12 +349,18 @@ class FlakyDelCache implements CacheClient { smembers(key: string): Promise { return this.inner.smembers(key) } + smismember(key: string, members: readonly string[]): Promise { + return this.inner.smismember(key, members) + } scard(key: string): Promise { return this.inner.scard(key) } expire(key: string, ttlSeconds: number): Promise { return this.inner.expire(key, ttlSeconds) } + expireNx(key: string, ttlSeconds: number): Promise { + return this.inner.expireNx(key, ttlSeconds) + } errors: unknown[] = [] constructor(private readonly inner: InMemoryCacheClient) {} get(key: string): Promise { diff --git a/services/api/test/unit/feed-presence.test.ts b/services/api/test/unit/feed-presence.test.ts index aeb5d228..586d670b 100644 --- a/services/api/test/unit/feed-presence.test.ts +++ b/services/api/test/unit/feed-presence.test.ts @@ -72,6 +72,12 @@ describe("feed presence: snapshot", () => { expect(await presence.readSnapshot(VIEWER, "all")).toBeNull() }) + it("advertises snapshots only when a cache is wired and the TTL knob is positive", () => { + expect(present().presence.snapshotsAvailable).toBe(true) + expect(present({ snapshotTtlSeconds: 0 }).presence.snapshotsAvailable).toBe(false) + expect(makeFeedPresence({ config: CFG }).snapshotsAvailable).toBe(false) + }) + it("writes nothing when the TTL knob is zero", async () => { const { cache, presence } = present({ snapshotTtlSeconds: 0 }) await presence.writeSnapshot(VIEWER, "all", [{ id: POST, authorId: "a", score: 1 }]) @@ -93,21 +99,30 @@ describe("feed presence: served set and the seen signal", () => { expect([...(await presence.seenBy(OTHER, [POST]))]).toEqual([]) }) - it("puts a TTL on both the served set and the viewer index (no unbounded growth)", async () => { + function recordingCache(): { cache: FeedPresenceCache; expiries: Array<[string, number]> } { const expiries: Array<[string, number]> = [] const inner = new InMemoryCacheClient(() => Date.now()) - const recording: FeedPresenceCache = { - get: (k) => inner.get(k), - set: (k, v, ttl) => inner.set(k, v, ttl), - sadd: (k, ...m) => inner.sadd(k, ...m), - smembers: (k) => inner.smembers(k), - scard: (k) => inner.scard(k), - expire: (k, ttl) => { - expiries.push([k, ttl]) - return inner.expire(k, ttl) + return { + expiries, + cache: { + get: (k) => inner.get(k), + set: (k, v, ttl) => inner.set(k, v, ttl), + sadd: (k, ...m) => inner.sadd(k, ...m), + smembers: (k) => inner.smembers(k), + smismember: (k, m) => inner.smismember(k, m), + scard: (k) => inner.scard(k), + expire: (k, ttl) => inner.expire(k, ttl), + expireNx: (k, ttl) => { + expiries.push([k, ttl]) + return inner.expireNx(k, ttl) + }, }, } - const presence = makeFeedPresence({ cache: recording, config: CFG }) + } + + it("puts a TTL on both the served set and the viewer index (no unbounded growth)", async () => { + const { cache, expiries } = recordingCache() + const presence = makeFeedPresence({ cache, config: CFG }) await presence.recordServed(VIEWER, [POST, OTHER]) expect(expiries).toEqual([ @@ -117,6 +132,43 @@ describe("feed presence: served set and the seen signal", () => { ]) }) + it("never refreshes the served-set TTL, so an active reader's key still expires", async () => { + const clock = { now: 1_000_000 } + const cache = new InMemoryCacheClient(() => clock.now) + const presence = makeFeedPresence({ cache, config: CFG }) + + await presence.recordServed(VIEWER, [POST]) + clock.now += (CFG.servedTtlSeconds - 1) * 1000 + await presence.recordServed(VIEWER, [OTHER]) + clock.now += 2000 + + expect(await cache.scard(servedKey(VIEWER))).toBe(0) + expect(await cache.scard(viewersKey(POST))).toBe(0) + }) + + it("reads the seen signal with a membership probe over the candidate ids, not a full set read", async () => { + let smembersCalls = 0 + const inner = new InMemoryCacheClient(() => Date.now()) + const cache: FeedPresenceCache = { + get: (k) => inner.get(k), + set: (k, v, ttl) => inner.set(k, v, ttl), + sadd: (k, ...m) => inner.sadd(k, ...m), + smembers: (k) => { + smembersCalls += 1 + return inner.smembers(k) + }, + smismember: (k, m) => inner.smismember(k, m), + scard: (k) => inner.scard(k), + expire: (k, ttl) => inner.expire(k, ttl), + expireNx: (k, ttl) => inner.expireNx(k, ttl), + } + const presence = makeFeedPresence({ cache, config: CFG }) + await presence.recordServed(VIEWER, [POST]) + + expect([...(await presence.seenBy(VIEWER, [POST, OTHER]))]).toEqual([POST]) + expect(smembersCalls).toBe(0) + }) + it("is a no-op for an empty page", async () => { const { cache, presence } = present() await presence.recordServed(VIEWER, []) @@ -155,8 +207,10 @@ describe("feed presence: viewer reverse index is bounded", () => { smembersCalls += 1 return Promise.resolve([]) }, + smismember: () => Promise.resolve([]), scard: () => Promise.resolve(50_000), expire: () => Promise.resolve(), + expireNx: () => Promise.resolve(), } const presence = makeFeedPresence({ cache, config: { ...CFG, viewerFanoutMax: 500 } }) expect(await presence.viewersOf(POST)).toEqual([]) @@ -170,8 +224,10 @@ describe("feed presence: degrades instead of failing the request", () => { set: () => Promise.reject(new Error("redis down")), sadd: () => Promise.reject(new Error("redis down")), smembers: () => Promise.reject(new Error("redis down")), + smismember: () => Promise.reject(new Error("redis down")), scard: () => Promise.reject(new Error("redis down")), expire: () => Promise.reject(new Error("redis down")), + expireNx: () => Promise.reject(new Error("redis down")), } it("swallows every Redis failure and returns the safe fallback", async () => { diff --git a/services/api/test/unit/host-analytics.test.ts b/services/api/test/unit/host-analytics.test.ts index 6d99590f..7c306fae 100644 --- a/services/api/test/unit/host-analytics.test.ts +++ b/services/api/test/unit/host-analytics.test.ts @@ -568,8 +568,10 @@ describe("host analytics cache", () => { sadd: () => Promise.resolve(0), srem: () => Promise.resolve(0), smembers: () => Promise.resolve([]), + smismember: () => Promise.resolve([]), scard: () => Promise.resolve(0), expire: () => Promise.resolve(), + expireNx: () => Promise.resolve(), } const cache = makeHostAnalyticsCache({ cache: broken, ttlSeconds: 60 }) expect(await cache.getOrSet("k", () => Promise.resolve(42))).toBe(42) From da543be24753e026ac339c4d12745d8a78468cd6 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 00:45:18 -0700 Subject: [PATCH 12/54] continue the ranked feed by re-ranking when no snapshot backs the cursor --- services/api/src/services/post-service.ts | 143 ++++++--- .../api/test/unit/feed-ranked-service.test.ts | 296 +++++++++++++++++- services/api/test/unit/post-service.test.ts | 2 +- 3 files changed, 382 insertions(+), 59 deletions(-) diff --git a/services/api/src/services/post-service.ts b/services/api/src/services/post-service.ts index adca3b39..99626039 100644 --- a/services/api/src/services/post-service.ts +++ b/services/api/src/services/post-service.ts @@ -28,9 +28,14 @@ import { type PostRepository, type RepliesPage, } from "./post-repository.drizzle.js" -import { applyCutoff, rankCandidates, type FeedCandidate } from "./feed-ranking.js" +import { + applyCutoff, + rankCandidates, + type FeedCandidate, + type RankedCandidate, +} from "./feed-ranking.js" import { mapWithLimit } from "./media-presign.js" -import type { FeedPresence } from "./feed-presence.js" +import type { FeedPresence, FeedSnapshotEntry } from "./feed-presence.js" import type { PostNotifier } from "./notification-service.js" function isVisible(brief: PostBrief): boolean { @@ -116,6 +121,12 @@ export function makePostService(deps: PostServiceDeps): PostService { } } + function presenceFor(viewerId: string): FeedPresence | undefined { + if (deps.feedPresence === undefined) return undefined + if (viewerId === NIL_VIEWER_ID) return undefined + return deps.feedPresence + } + async function announceCountChange(postId: string, actorId: string): Promise { if (deps.userChannel === undefined || deps.feedPresence === undefined) return try { @@ -150,16 +161,27 @@ export function makePostService(deps: PostServiceDeps): PostService { } } + function sliceAfterCursor( + entries: readonly FeedSnapshotEntry[], + cursor: { score: number; postId: string }, + ): FeedSnapshotEntry[] { + const anchor = entries.findIndex((entry) => entry.id === cursor.postId) + if (anchor !== -1) return entries.slice(anchor + 1) + return entries.filter((entry) => + isAfterFeedScoreCursor({ score: entry.score, postId: entry.id }, cursor), + ) + } + async function pageFrom( viewerId: string, - entries: readonly { id: string; score: number }[], + entries: readonly FeedSnapshotEntry[], limit: number, ): Promise { const pageEntries = entries.slice(0, limit) const hasMore = entries.length > pageEntries.length const last = pageEntries[pageEntries.length - 1] const nextCursor = - hasMore && last !== undefined && deps.feedPresence !== undefined && viewerId !== NIL_VIEWER_ID + hasMore && last !== undefined ? formatFeedScoreCursor({ score: last.score, postId: last.id }) : null @@ -168,8 +190,9 @@ export function makePostService(deps: PostServiceDeps): PostService { viewerId, ) - if (deps.feedPresence !== undefined && viewerId !== NIL_VIEWER_ID) { - void deps.feedPresence + const presence = presenceFor(viewerId) + if (presence !== undefined) { + void presence .recordServed( viewerId, items.map((item) => item.id), @@ -182,38 +205,12 @@ export function makePostService(deps: PostServiceDeps): PostService { return { items, nextCursor } } - async function continueRankedPage( - viewerId: string, - filter: string, - cursor: { score: number; postId: string }, - limit: number, - ): Promise { - if (deps.feedPresence === undefined || viewerId === NIL_VIEWER_ID) { - return { items: [], nextCursor: null } - } - const snapshot = await deps.feedPresence.readSnapshot(viewerId, filter) - if (snapshot === null) return { items: [], nextCursor: null } - void deps.feedPresence.touchSnapshot(viewerId, filter).catch((err: unknown) => { - deps.logger?.warn({ err }, "post: feed snapshot touch failed (suppressed)") - }) - const after = snapshot.filter((entry) => - isAfterFeedScoreCursor({ score: entry.score, postId: entry.id }, cursor), - ) - return pageFrom(viewerId, after, limit) - } - - async function rankedFeed( + async function rankCandidateSet( viewerId: string, query: HomeFeedQuery, location: FeedViewerLocation | undefined, - ): Promise { - const limit = query.limit ?? POSTS_DEFAULT_LIMIT - const cursor = parseFeedScoreCursor(query.cursor) - - if (cursor !== null) { - return continueRankedPage(viewerId, query.filter, cursor, limit) - } - + isFirstPage: boolean, + ): Promise<{ page: RankedCandidate[]; durable: RankedCandidate[] }> { const rows = await deps.repo.feedCandidates({ viewerId, filter: query.filter, @@ -224,10 +221,11 @@ export function makePostService(deps: PostServiceDeps): PostService { candidateCap: feedConfig.candidateCap, }) + const presence = presenceFor(viewerId) const seen = - deps.feedPresence === undefined || viewerId === NIL_VIEWER_ID + presence === undefined || !isFirstPage ? new Set() - : await deps.feedPresence.seenBy( + : await presence.seenBy( viewerId, rows.map((r) => r.id), ) @@ -237,15 +235,62 @@ export function makePostService(deps: PostServiceDeps): PostService { feedConfig, nowMs(), ) - const kept = applyCutoff(ranked, feedConfig, true) + return { + page: applyCutoff(ranked, feedConfig, true), + durable: applyCutoff(ranked, feedConfig, false), + } + } - if (deps.feedPresence !== undefined && viewerId !== NIL_VIEWER_ID) { - void deps.feedPresence.writeSnapshot(viewerId, query.filter, kept).catch((err: unknown) => { - deps.logger?.warn({ err }, "post: feed snapshot write failed (suppressed)") - }) + function persistSnapshot( + viewerId: string, + filter: string, + durable: readonly RankedCandidate[], + ): void { + const presence = presenceFor(viewerId) + if (presence === undefined || !presence.snapshotsAvailable) return + void presence.writeSnapshot(viewerId, filter, durable).catch((err: unknown) => { + deps.logger?.warn({ err }, "post: feed snapshot write failed (suppressed)") + }) + } + + async function continueRankedPage( + viewerId: string, + query: HomeFeedQuery, + location: FeedViewerLocation | undefined, + cursor: { score: number; postId: string }, + limit: number, + ): Promise { + const presence = presenceFor(viewerId) + if (presence !== undefined && presence.snapshotsAvailable) { + const snapshot = await presence.readSnapshot(viewerId, query.filter) + if (snapshot !== null) { + void presence.touchSnapshot(viewerId, query.filter).catch((err: unknown) => { + deps.logger?.warn({ err }, "post: feed snapshot touch failed (suppressed)") + }) + return pageFrom(viewerId, sliceAfterCursor(snapshot, cursor), limit) + } + } + + const { durable } = await rankCandidateSet(viewerId, query, location, false) + persistSnapshot(viewerId, query.filter, durable) + return pageFrom(viewerId, sliceAfterCursor(durable, cursor), limit) + } + + async function rankedFeed( + viewerId: string, + query: HomeFeedQuery, + location: FeedViewerLocation | undefined, + ): Promise { + const limit = query.limit ?? POSTS_DEFAULT_LIMIT + const cursor = parseFeedScoreCursor(query.cursor) + + if (cursor !== null) { + return continueRankedPage(viewerId, query, location, cursor, limit) } - return pageFrom(viewerId, kept, limit) + const { page, durable } = await rankCandidateSet(viewerId, query, location, true) + persistSnapshot(viewerId, query.filter, durable) + return pageFrom(viewerId, page, limit) } async function safeNotify(fn: () => Promise): Promise { @@ -303,9 +348,11 @@ export function makePostService(deps: PostServiceDeps): PostService { const kind = input.replyToId !== undefined ? "reply" : input.repostOfId !== undefined ? "quote" : "post" + let replyParentId: string | null = null let replyParentAuthor: string | null = null if (input.replyToId !== undefined) { const parent = await requireReadable(input.replyToId, authorId) + replyParentId = parent.id replyParentAuthor = parent.authorId } let quoteTargetAuthor: string | null = null @@ -364,8 +411,8 @@ export function makePostService(deps: PostServiceDeps): PostService { if (kind === "post" || kind === "quote") { await announceNewPost(postId, authorId) } - if (replyParentAuthor !== null) { - await announceCountChange(input.replyToId!, authorId) + if (replyParentId !== null) { + await announceCountChange(replyParentId, authorId) } return hydrateOrThrow(postId, authorId) @@ -413,7 +460,8 @@ export function makePostService(deps: PostServiceDeps): PostService { async unlikePost(id: string, viewerId: string): Promise { const subject = await requireReadable(id, viewerId) - await deps.repo.unlike(subject.id, viewerId) + const removed = await deps.repo.unlike(subject.id, viewerId) + if (removed) await announceCountChange(subject.id, viewerId) return hydrateOrThrow(id, viewerId) }, @@ -454,7 +502,8 @@ export function makePostService(deps: PostServiceDeps): PostService { async unrepostPost(id: string, viewerId: string): Promise { await requireReadable(id, viewerId) - const { targetId } = await deps.repo.unrepost(id, viewerId) + const { targetId, removed } = await deps.repo.unrepost(id, viewerId) + if (removed) await announceCountChange(targetId, viewerId) return hydrateOrThrow(targetId, viewerId) }, diff --git a/services/api/test/unit/feed-ranked-service.test.ts b/services/api/test/unit/feed-ranked-service.test.ts index bd3d21d6..a9a6f51c 100644 --- a/services/api/test/unit/feed-ranked-service.test.ts +++ b/services/api/test/unit/feed-ranked-service.test.ts @@ -6,6 +6,7 @@ import type { Sql } from "../../src/db/client.js" import { makePostService, type PostService } from "../../src/services/post-service.js" import { makeFeedPresence } from "../../src/services/feed-presence.js" import { InMemoryCacheClient } from "../../src/auth/cache.js" +import { NIL_VIEWER_ID } from "../../src/services/post-repository.drizzle.js" import type { FeedCandidateRow, PostBrief, @@ -244,7 +245,7 @@ describe("ranked feed: warm snapshot continues a cursor without re-ranking", () expect(second.items.some((item) => firstIds.has(item.id))).toBe(false) }) - it("ends the feed rather than re-ranking when the snapshot has expired", async () => { + it("re-ranks rather than ending the feed when the snapshot has expired under a dwelling reader", async () => { const { svc } = harness() const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) @@ -263,8 +264,45 @@ describe("ranked feed: warm snapshot continues a cursor without re-ranking", () cursor: first.nextCursor!, }) - expect(page.items).toEqual([]) - expect(page.nextCursor).toBeNull() + const firstIds = new Set(first.items.map((item) => item.id)) + expect(page.items).toHaveLength(20) + expect(page.items.some((item) => firstIds.has(item.id))).toBe(false) + expect(page.nextCursor).not.toBeNull() + }) + + it("rebuilds the snapshot on the re-ranked continuation so the next page is served from cache", async () => { + const { svc } = harness() + const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + + const cache = new InMemoryCacheClient(() => Date.now()) + const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + let candidateQueries = 0 + const expired = makePostService({ + repo: repoOver({ + feedCandidates: () => { + candidateQueries += 1 + return Promise.resolve(rows) + }, + }), + sql: throwingSql, + feedPresence: presence, + now: () => NOW, + }) + + const second = await expired.homeFeed(VIEWER, { + filter: "all", + limit: 20, + cursor: first.nextCursor!, + }) + await new Promise((resolve) => setTimeout(resolve, 0)) + const third = await expired.homeFeed(VIEWER, { + filter: "all", + limit: 20, + cursor: second.nextCursor!, + }) + + expect(candidateQueries).toBe(1) + expect(third.items).toHaveLength(20) }) it("re-ranking a moved clock never re-serves a page-1 item", async () => { @@ -290,15 +328,25 @@ describe("ranked feed: warm snapshot continues a cursor without re-ranking", () expect(page.items.some((item) => firstIds.has(item.id))).toBe(false) }) - it("keys the snapshot per filter, so the events tab does not slice the all tab's page", async () => { - const { svc } = harness() - const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) - const events = await svc.homeFeed(VIEWER, { - filter: "events", - limit: 20, - cursor: first.nextCursor!, + it("keys the snapshot per filter, so the events tab re-ranks instead of slicing the all tab's page", async () => { + const filters: string[] = [] + const cache = new InMemoryCacheClient(() => Date.now()) + const svc = makePostService({ + repo: repoOver({ + feedCandidates: (feedArgs: { filter: string }) => { + filters.push(feedArgs.filter) + return Promise.resolve(rows) + }, + }), + sql: throwingSql, + feedPresence: makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }), + now: () => NOW, }) - expect(events.items).toEqual([]) + + const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + await svc.homeFeed(VIEWER, { filter: "events", limit: 20, cursor: first.nextCursor! }) + + expect(filters).toEqual(["all", "events"]) }) it("keeps an active scroll alive by refreshing the snapshot TTL", async () => { @@ -324,6 +372,138 @@ describe("ranked feed: warm snapshot continues a cursor without re-ranking", () }) }) +describe("ranked feed: no Redis is a slower feed, not a one-page feed", () => { + const rows = Array.from({ length: 60 }, (_, i) => candidateRow(i + 1)) + + it.each([ + ["no cache wired at all", makeFeedPresence({ config: DEFAULT_FEED_RANKING })], + [ + "snapshots switched off by the TTL knob", + makeFeedPresence({ + cache: new InMemoryCacheClient(() => Date.now()), + config: { ...DEFAULT_FEED_RANKING, snapshotTtlSeconds: 0 }, + }), + ], + ])("pages an authenticated feed past page 1 with %s", async (_label, presence) => { + expect(presence.snapshotsAvailable).toBe(false) + const svc = makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: presence, + now: () => NOW, + }) + + const seen: string[] = [] + let cursor: string | undefined + for (let page = 0; page < 5; page += 1) { + const result = await svc.homeFeed(VIEWER, { + filter: "all", + limit: 20, + ...(cursor === undefined ? {} : { cursor }), + }) + seen.push(...result.items.map((item) => item.id)) + if (result.nextCursor === null) break + cursor = result.nextCursor + } + + expect(seen).toHaveLength(60) + expect(new Set(seen).size).toBe(60) + }) +}) + +describe("ranked feed: a signed-out reader keeps scrolling", () => { + const rows = Array.from({ length: 60 }, (_, i) => candidateRow(i + 1, { author_followed: false })) + + function guestService(presence?: ReturnType): PostService { + return makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + ...(presence === undefined ? {} : { feedPresence: presence }), + now: () => NOW, + }) + } + + it("pages the anonymous feed past page 1 and terminates without duplicates", async () => { + const svc = guestService() + const seen: string[] = [] + let cursor: string | undefined + let pages = 0 + + for (;;) { + const page = await svc.publicFeed({ + filter: "all", + limit: 20, + ...(cursor === undefined ? {} : { cursor }), + }) + seen.push(...page.items.map((item) => item.id)) + pages += 1 + if (page.nextCursor === null) break + cursor = page.nextCursor + expect(pages).toBeLessThan(10) + } + + expect(pages).toBeGreaterThan(1) + expect(seen.length).toBeGreaterThan(20) + expect(new Set(seen).size).toBe(seen.length) + }) + + it("keeps a guest continuation stable across a clock bucket boundary", async () => { + const first = await guestService().publicFeed({ filter: "all", limit: 20 }) + const later = makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + now: () => NOW + 5 * 60_000, + }) + const second = await later.publicFeed({ filter: "all", limit: 20, cursor: first.nextCursor! }) + + const firstIds = new Set(first.items.map((item) => item.id)) + expect(second.items.length).toBeGreaterThan(0) + expect(second.items.some((item) => firstIds.has(item.id))).toBe(false) + }) + + it("never writes a guest into the shared presence keys", async () => { + const cache = new InMemoryCacheClient(() => Date.now()) + const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + const page = await guestService(presence).publicFeed({ filter: "all", limit: 20 }) + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(page.items).toHaveLength(20) + expect(await presence.readSnapshot(NIL_VIEWER_ID, "all")).toBeNull() + expect(await presence.viewersOf(page.items[0]!.id)).toEqual([]) + }) +}) + +describe("ranked feed: the cold-start fallback is one request, not one session", () => { + const rows = Array.from({ length: 40 }, (_, i) => + candidateRow(i + 1, { author_followed: false, like_count: 0, created_at: new Date(NOW) }), + ) + + it("serves an uncut first page but never lets the cutoff stay suspended on page 2", async () => { + const cache = new InMemoryCacheClient(() => Date.now()) + const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + const svc = makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: presence, + now: () => NOW, + }) + + const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(first.items).toHaveLength(20) + expect(await presence.readSnapshot(VIEWER, "all")).toEqual([]) + + const second = await svc.homeFeed(VIEWER, { + filter: "all", + limit: 20, + cursor: first.nextCursor!, + }) + expect(second.items).toEqual([]) + expect(second.nextCursor).toBeNull() + }) +}) + describe("ranked feed: served set feeds the seen discount", () => { it("records the served page so the next refresh demotes what was already read", async () => { const cache = new InMemoryCacheClient(() => Date.now()) @@ -344,6 +524,35 @@ describe("ranked feed: served set feeds the seen discount", () => { ) expect(served.size).toBe(page.items.length) }) + + it("never applies the discount to a re-ranked continuation, which would reshuffle the cursor", async () => { + const cache = new InMemoryCacheClient(() => Date.now()) + const base = makeFeedPresence({ + cache, + config: { ...DEFAULT_FEED_RANKING, snapshotTtlSeconds: 0 }, + }) + const lookups: string[] = [] + const presence = { + ...base, + seenBy: (userId: string, postIds: readonly string[]) => { + lookups.push(userId) + return base.seenBy(userId, postIds) + }, + } + const rows = Array.from({ length: 60 }, (_, i) => candidateRow(i + 1)) + const svc = makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: presence, + now: () => NOW, + }) + + const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + await new Promise((resolve) => setTimeout(resolve, 0)) + await svc.homeFeed(VIEWER, { filter: "all", limit: 20, cursor: first.nextCursor! }) + + expect(lookups).toEqual([VIEWER]) + }) }) describe("ranked feed: realtime fanout never blocks the request path", () => { @@ -399,6 +608,71 @@ describe("ranked feed: realtime fanout never blocks the request path", () => { expect(sent.flatMap((s) => s.users)).toEqual(["reader-1"]) }) + it("publishes feed_counts on an unlike and an unrepost, so decrements reach viewers", async () => { + const { channel, sent } = recordingChannel() + const cache = new InMemoryCacheClient(() => Date.now()) + const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + await presence.recordServed("reader-1", [POST]) + + const svc = makePostService({ + repo: repoOver({}), + sql: throwingSql, + userChannel: channel, + feedPresence: presence, + }) + await svc.unlikePost(POST, VIEWER) + await svc.unrepostPost(POST, VIEWER) + + expect(sent).toHaveLength(2) + expect(sent.every((s) => s.signal.topic === "feed_counts" && s.signal.id === POST)).toBe(true) + }) + + it("does not publish an unlike that removed nothing", async () => { + const { channel, sent } = recordingChannel() + const cache = new InMemoryCacheClient(() => Date.now()) + const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + await presence.recordServed("reader-1", [POST]) + + const svc = makePostService({ + repo: repoOver({ unlike: () => Promise.resolve(false) }), + sql: throwingSql, + userChannel: channel, + feedPresence: presence, + }) + await svc.unlikePost(POST, VIEWER) + expect(sent).toHaveLength(0) + }) + + it("announces the reply count against the resolved parent, not the repost that was replied to", async () => { + const { channel, sent } = recordingChannel() + const ORIGINAL = "44444444-4444-4444-4444-444444444444" + const REPOST = "55555555-5555-5555-5555-555555555555" + const cache = new InMemoryCacheClient(() => Date.now()) + const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + await presence.recordServed("reader-1", [ORIGINAL, REPOST]) + + const svc = makePostService({ + repo: repoOver({ + getPostBrief: (id: string) => + Promise.resolve( + id === REPOST + ? brief({ id: REPOST, kind: "repost", repostOfId: ORIGINAL }) + : brief({ id: ORIGINAL }), + ), + }), + sql: throwingSql, + userChannel: channel, + feedPresence: presence, + }) + + await svc.createPost( + { kind: "post", body: "reply", replyToId: REPOST, mediaUploadIds: [], mentionedUserIds: [] } as never, + VIEWER, + ) + + expect(sent.map((s) => s.signal.id)).toEqual([ORIGINAL]) + }) + it("does not publish when the interaction was a no-op (double like)", async () => { const { channel, sent } = recordingChannel() const cache = new InMemoryCacheClient(() => Date.now()) diff --git a/services/api/test/unit/post-service.test.ts b/services/api/test/unit/post-service.test.ts index 1d0beebf..80711e29 100644 --- a/services/api/test/unit/post-service.test.ts +++ b/services/api/test/unit/post-service.test.ts @@ -212,7 +212,7 @@ describe("PostService validation + authorization", () => { await svc.publicFeed({ filter: "all", cursor: "not-a-cursor" }) expect(chronological).toEqual([]) - expect(ranked).toEqual([NIL_VIEWER_ID]) + expect(ranked).toEqual([NIL_VIEWER_ID, NIL_VIEWER_ID]) }) it("rejects an attached event the author does not host/attend (403)", async () => { From a92cc5c6a226d9996954d82411ca76825c3ec93e Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 00:45:18 -0700 Subject: [PATCH 13/54] skip the feed fanout lookups when the user channel is the fake --- services/api/src/di.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/api/src/di.ts b/services/api/src/di.ts index 0770cbc4..aaf4b1e5 100644 --- a/services/api/src/di.ts +++ b/services/api/src/di.ts @@ -240,7 +240,7 @@ export function buildContainer(env: Env): Container { isBlockedEitherWay: (a: string, b: string) => getBlocksRepo().isBlockedEitherWay(a, b), feedRanking: env.FEED_RANKING, feedPresence: getFeedPresence(), - userChannel: getUserChannel(), + ...(env.USE_FAKE_USER_CHANNEL ? {} : { userChannel: getUserChannel() }), }) } return postService From 975fac93156173267ef0a5dd4a97433be318d2c2 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 00:45:18 -0700 Subject: [PATCH 14/54] plan the feed explain assertion the way suggest-follows does --- .../src/services/post-repository.drizzle.ts | 2 ++ .../test/integration/feed-ranked-pg.test.ts | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/services/api/src/services/post-repository.drizzle.ts b/services/api/src/services/post-repository.drizzle.ts index ce606e0d..00f0c71c 100644 --- a/services/api/src/services/post-repository.drizzle.ts +++ b/services/api/src/services/post-repository.drizzle.ts @@ -35,6 +35,8 @@ export const POSTS_DEFAULT_LIMIT = 20 export const NIL_VIEWER_ID = "00000000-0000-0000-0000-000000000000" +export const FEED_NEARBY_INDEX = "posts_geom_gist" + function postColumns(sql: Queryable): postgres.Fragment { return sql` p.id, p.author_id, p.kind, p.body, p.reply_to_id, p.thread_root_id, p.repost_of_id, diff --git a/services/api/test/integration/feed-ranked-pg.test.ts b/services/api/test/integration/feed-ranked-pg.test.ts index e5ea38ae..0e2b1e48 100644 --- a/services/api/test/integration/feed-ranked-pg.test.ts +++ b/services/api/test/integration/feed-ranked-pg.test.ts @@ -7,6 +7,7 @@ import { InMemoryCacheClient } from "../../src/auth/cache.js" import { explainFeedCandidates, makeDrizzlePostRepository, + FEED_NEARBY_INDEX, type FeedCandidateArgs, type PostRepository, } from "../../src/services/post-repository.drizzle.js" @@ -259,11 +260,21 @@ describe.skipIf(!pg)("ranked home feed: candidate SQL (integration)", () => { expect(rows.length).toBeLessThanOrEqual(3) }) - it("plans the nearby pool as an index scan, not a sequential scan", async () => { + async function explainWithIndexPlan(viewerId: string): Promise { + await h.sql`ANALYZE posts` + await h.sql`ANALYZE follows_people` + return h.sql.begin(async (tx) => { + await tx`SET LOCAL enable_seqscan = off` + await tx`SET LOCAL enable_bitmapscan = off` + return explainFeedCandidates(tx, args({ viewerId })) + }) + } + + it("has an index path available for the nearby pool, and never falls back to a posts seq scan", async () => { const viewer = await newUser("Explain Viewer") - const plan = await explainFeedCandidates(h.sql, args({ viewerId: viewer })) - expect(plan).toContain("posts_geom_gist") - expect(plan).not.toMatch(/Seq Scan on posts/) + const plan = await explainWithIndexPlan(viewer) + expect(plan, `${FEED_NEARBY_INDEX} missing from:\n${plan}`).toContain(FEED_NEARBY_INDEX) + expect(plan, plan).not.toMatch(/Seq Scan on posts/) }) it("pages the ranked feed over every seeded post exactly once", async () => { From 5ed1b7e98f6faa4033a9c16308ced4a47ac24e31 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 00:54:51 -0700 Subject: [PATCH 15/54] wait a full tick for the fire-and-forget served-set write in the feed test --- services/api/test/unit/feed-ranked-service.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/api/test/unit/feed-ranked-service.test.ts b/services/api/test/unit/feed-ranked-service.test.ts index a9a6f51c..8fc1f99a 100644 --- a/services/api/test/unit/feed-ranked-service.test.ts +++ b/services/api/test/unit/feed-ranked-service.test.ts @@ -517,7 +517,7 @@ describe("ranked feed: served set feeds the seen discount", () => { }) const page = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) - await Promise.resolve() + await new Promise((resolve) => setTimeout(resolve, 0)) const served = await presence.seenBy( VIEWER, rows.map((r) => r.id), From 808a99293437c6cee884a45321f4306d7c878ca2 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 01:32:22 -0700 Subject: [PATCH 16/54] integration fixtures: idempotency key and served key --- services/api/test/integration/admin-reports.test.ts | 4 ++-- services/api/test/integration/feed-ranked-pg.test.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/api/test/integration/admin-reports.test.ts b/services/api/test/integration/admin-reports.test.ts index 69b49575..5fd91b5d 100644 --- a/services/api/test/integration/admin-reports.test.ts +++ b/services/api/test/integration/admin-reports.test.ts @@ -98,8 +98,8 @@ describe.skipIf(!pg)("admin report repository (integration: real schema)", () => const reporter = await insertUser(h, { name: "Jane", handle: "jane", emailVerified: true }) const id = await insertReport(h, { reporterId: reporter, title: "Overflowing bin" }) await h.sql` - INSERT INTO media_assets (report_id, upload_id, kind, r2_key, status) - VALUES (${id}, gen_random_uuid(), 'image', 'k/photo.jpg', 'ready') + INSERT INTO media_assets (report_id, upload_id, kind, r2_key, served_key, status) + VALUES (${id}, gen_random_uuid(), 'image', 'k/photo.jpg', 'k/photo.serve.jpg', 'ready') ` const { records } = await repo.listReports({ q: null, diff --git a/services/api/test/integration/feed-ranked-pg.test.ts b/services/api/test/integration/feed-ranked-pg.test.ts index 0e2b1e48..e2cde504 100644 --- a/services/api/test/integration/feed-ranked-pg.test.ts +++ b/services/api/test/integration/feed-ranked-pg.test.ts @@ -206,9 +206,9 @@ describe.skipIf(!pg)("ranked home feed: candidate SQL (integration)", () => { it("denormalises geom from the linked report at insert time", async () => { const author = await newUser("Report Author") const [report] = await h.sql<{ id: string }[]>` - INSERT INTO reports (reporter_user_id, category, type, description, geom, geom_source, status, visibility) + INSERT INTO reports (reporter_user_id, idempotency_key, category, type, description, geom, geom_source, status, visibility) VALUES ( - ${author}, 'graffiti', 'graffiti', 'tag', + ${author}, gen_random_uuid(), 'graffiti', 'graffiti', 'tag', ST_SetSRID(ST_MakePoint(${LA.lng}, ${LA.lat}), 4326), 'manual', 'published', 'public' ) RETURNING id From 3723ad0d7d5177641a4561749d124c17b7ca0de0 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 01:42:32 -0700 Subject: [PATCH 17/54] geom fixture carries h3_cell --- services/api/test/integration/feed-ranked-pg.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/api/test/integration/feed-ranked-pg.test.ts b/services/api/test/integration/feed-ranked-pg.test.ts index e2cde504..bd8337b5 100644 --- a/services/api/test/integration/feed-ranked-pg.test.ts +++ b/services/api/test/integration/feed-ranked-pg.test.ts @@ -206,10 +206,10 @@ describe.skipIf(!pg)("ranked home feed: candidate SQL (integration)", () => { it("denormalises geom from the linked report at insert time", async () => { const author = await newUser("Report Author") const [report] = await h.sql<{ id: string }[]>` - INSERT INTO reports (reporter_user_id, idempotency_key, category, type, description, geom, geom_source, status, visibility) + INSERT INTO reports (reporter_user_id, idempotency_key, category, type, description, geom, geom_source, status, visibility, h3_cell) VALUES ( ${author}, gen_random_uuid(), 'graffiti', 'graffiti', 'tag', - ST_SetSRID(ST_MakePoint(${LA.lng}, ${LA.lat}), 4326), 'manual', 'published', 'public' + ST_SetSRID(ST_MakePoint(${LA.lng}, ${LA.lat}), 4326), 'manual', 'published', 'public', 'h0' ) RETURNING id ` From 5887e8ec9d320509de877b1abe360ff41f9e42cc Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 16:09:00 -0700 Subject: [PATCH 18/54] adopt @civfix/shared 0.50.0 --- pnpm-lock.yaml | 14 +++++++------- services/api/package.json | 2 +- services/media-worker/package.json | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bc839a3c..85b9e598 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -65,8 +65,8 @@ importers: specifier: ^3.700.0 version: 3.1057.0 '@civfix/shared': - specifier: ^0.49.0 - version: 0.49.0 + specifier: ^0.50.0 + version: 0.50.0 '@fastify/cookie': specifier: ^11.0.1 version: 11.0.2 @@ -210,8 +210,8 @@ importers: specifier: workspace:* version: link:../api '@civfix/shared': - specifier: ^0.49.0 - version: 0.49.0 + specifier: ^0.50.0 + version: 0.50.0 '@sentry/node': specifier: ^10.73.0 version: 10.73.0(@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1)) @@ -414,8 +414,8 @@ packages: '@balena/dockerignore@1.0.2': resolution: {integrity: sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==} - '@civfix/shared@0.49.0': - resolution: {integrity: sha512-xs6fB2p8bG26T5TRnKWKEQGq06zOm4DAjYiKgdHus6+CRifPlbpQaSIa2KcHxqaDyki3Dp/OwRxGhTjqQGlpgw==} + '@civfix/shared@0.50.0': + resolution: {integrity: sha512-bteKU/bE5lKmoQvPWFgCVKBLCxzFqXVapLWg2+46FgbS9yEugGPv9zJQ88O+zUinv3na0Cvmzj+jLlN6FfyyYw==} engines: {node: '>=20'} '@derhuerst/http-basic@8.2.4': @@ -4440,7 +4440,7 @@ snapshots: '@balena/dockerignore@1.0.2': {} - '@civfix/shared@0.49.0': + '@civfix/shared@0.50.0': dependencies: zod: 3.25.76 diff --git a/services/api/package.json b/services/api/package.json index 9ff931db..4da180d6 100644 --- a/services/api/package.json +++ b/services/api/package.json @@ -58,7 +58,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.700.0", "@aws-sdk/s3-request-presigner": "^3.700.0", - "@civfix/shared": "^0.49.0", + "@civfix/shared": "^0.50.0", "@fastify/cookie": "^11.0.1", "@fastify/cors": "^10.0.1", "@fastify/helmet": "^12.0.1", diff --git a/services/media-worker/package.json b/services/media-worker/package.json index fd34f73d..cbc0a037 100644 --- a/services/media-worker/package.json +++ b/services/media-worker/package.json @@ -23,7 +23,7 @@ "@aws-sdk/client-s3": "^3.700.0", "@aws-sdk/s3-request-presigner": "^3.700.0", "@civfix/api": "workspace:*", - "@civfix/shared": "^0.49.0", + "@civfix/shared": "^0.50.0", "@sentry/node": "^10.73.0", "@smithy/node-http-handler": "4.7.5", "drizzle-orm": "^0.45.2", From d2dd194917eec71bc19d632a66872e00d5855fb3 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 16:09:00 -0700 Subject: [PATCH 19/54] chat attachments serve a finalized-but-validating asset --- .../src/services/chat-attachments.drizzle.ts | 4 +- .../services/message-attachments.drizzle.ts | 9 +- .../unit/chat-attachments-servable.test.ts | 163 ++++++++++++++++++ 3 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 services/api/test/unit/chat-attachments-servable.test.ts diff --git a/services/api/src/services/chat-attachments.drizzle.ts b/services/api/src/services/chat-attachments.drizzle.ts index 02df8bb8..7b33ebd4 100644 --- a/services/api/src/services/chat-attachments.drizzle.ts +++ b/services/api/src/services/chat-attachments.drizzle.ts @@ -1,7 +1,7 @@ import type { Queryable } from "../db/client.js" import type { MediaDTO } from "@civfix/shared" import type { PresignMedia } from "./media-presign.js" -import { loadReadyAttachmentsFor, makeAttachmentRepo } from "./message-attachments.drizzle.js" +import { loadServableAttachmentsFor, makeAttachmentRepo } from "./message-attachments.drizzle.js" export function attachChatMedia( sql: Queryable, @@ -17,5 +17,5 @@ export function loadChatAttachments( messageIds: string[], presign: PresignMedia, ): Promise> { - return loadReadyAttachmentsFor(sql, "chat_message_id", messageIds, presign) + return loadServableAttachmentsFor(sql, "chat_message_id", messageIds, presign) } diff --git a/services/api/src/services/message-attachments.drizzle.ts b/services/api/src/services/message-attachments.drizzle.ts index bd8dd06c..2634a40e 100644 --- a/services/api/src/services/message-attachments.drizzle.ts +++ b/services/api/src/services/message-attachments.drizzle.ts @@ -1,6 +1,7 @@ import type { Queryable } from "../db/client.js" import type { MediaDTO, MediaKind, MediaStatus } from "@civfix/shared" import { mapWithLimit, PRESIGN_CONCURRENCY, type PresignMedia } from "./media-presign.js" +import { servableMediaFilter, servedKeyExpr } from "./media-served-key.js" export type MessageMediaColumn = "chat_message_id" @@ -50,7 +51,7 @@ export function makeAttachmentRepo(column: MessageMediaColumn): MessageAttachmen } } -export async function loadReadyAttachmentsFor( +export async function loadServableAttachmentsFor( tag: Queryable, column: MessageMediaColumn, messageIds: string[], @@ -59,12 +60,12 @@ export async function loadReadyAttachmentsFor( const byMessage = new Map() if (messageIds.length === 0) return byMessage const rows = await tag` - SELECT id, ${tag(column)} AS message_id, kind, codec, served_key AS r2_key, + SELECT id, ${tag(column)} AS message_id, kind, codec, + ${servedKeyExpr(tag, "media_assets")} AS r2_key, thumb_key, status, width, height FROM media_assets WHERE ${tag(column)} IN ${tag(messageIds)} - AND status = 'ready' - AND served_key IS NOT NULL + AND ${servableMediaFilter(tag, "media_assets")} ORDER BY created_at ASC ` const projected = await mapWithLimit(rows, PRESIGN_CONCURRENCY, async (r) => { diff --git a/services/api/test/unit/chat-attachments-servable.test.ts b/services/api/test/unit/chat-attachments-servable.test.ts new file mode 100644 index 00000000..8206cca5 --- /dev/null +++ b/services/api/test/unit/chat-attachments-servable.test.ts @@ -0,0 +1,163 @@ +/** + * ISSUE #122 ("photo send in messages is bugged"). + * + * `attach()` deliberately ACCEPTS a finalized-but-still-`validating` asset, because that is the only + * state an asset is ever in at send time: `finalizeMedia` returns `{ status: "validating" }` and hands + * the flip to `ready` to the media worker. The READ used to require `status = 'ready' AND served_key IS + * NOT NULL`, so the WS ack and every history page came back with `attachments: []` and the client threw + * its optimistic local attachment away on reconcile — a photo-only message rendered as an empty row. + * + * The reports path solved this long ago with `servedKeyExpr` / `servableMediaFilter` + * (report-repository.drizzle.ts). These tests pin that chat now uses the same two fragments. + */ + +import { describe, expect, it } from "vitest" +import { makeFakeSql } from "../helpers/fake-sql.js" +import type { Queryable } from "../../src/db/client.js" +import { + loadServableAttachmentsFor, + makeAttachmentRepo, +} from "../../src/services/message-attachments.drizzle.js" + +const MESSAGE = "11111111-1111-4111-8111-111111111111" +const OTHER_MESSAGE = "22222222-2222-4222-8222-222222222222" + +const PRESIGN = (r2Key: string, thumbKey: string | null) => + Promise.resolve({ url: `memory://${r2Key}`, ...(thumbKey !== null ? { thumbUrl: `memory://${thumbKey}` } : {}) }) + +function row(over: Record = {}) { + return { + id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + message_id: MESSAGE, + kind: "photo", + codec: null, + r2_key: "chat/one.jpg", + thumb_key: null, + status: "validating", + width: 100, + height: 200, + ...over, + } +} + +describe("loadServableAttachmentsFor", () => { + it("selects the servable key expression, not the bare served_key column", async () => { + const fake = makeFakeSql([{ match: /FROM media_assets/, rows: [] }]) + + await loadServableAttachmentsFor( + fake.sql as unknown as Queryable, + "chat_message_id", + [MESSAGE], + PRESIGN, + ) + + const statement = fake.statements.at(-1)! + expect(statement.sql).toMatch(/COALESCE\(media_assets\.served_key/) + expect(statement.sql).toMatch(/media_assets\.r2_key/) + }) + + it("no longer filters on status = 'ready': a validating asset is servable", async () => { + const fake = makeFakeSql([{ match: /FROM media_assets/, rows: [] }]) + + await loadServableAttachmentsFor( + fake.sql as unknown as Queryable, + "chat_message_id", + [MESSAGE], + PRESIGN, + ) + + const statement = fake.statements.at(-1)! + expect(statement.sql).not.toMatch(/status = 'ready'/) + expect(statement.sql).toMatch(/media_assets\.status <> 'ready' OR media_assets\.served_key IS NOT NULL/) + }) + + it("returns the still-validating attachment with its status intact so the client can render it", async () => { + const fake = makeFakeSql([{ match: /FROM media_assets/, rows: [row()] }]) + + const byMessage = await loadServableAttachmentsFor( + fake.sql as unknown as Queryable, + "chat_message_id", + [MESSAGE], + PRESIGN, + ) + + expect(byMessage.get(MESSAGE)).toEqual([ + { + id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + kind: "photo", + codec: null, + url: "memory://chat/one.jpg", + width: 100, + height: 200, + status: "validating", + }, + ]) + }) + + it("groups several messages' attachments by message id", async () => { + const fake = makeFakeSql([ + { + match: /FROM media_assets/, + rows: [ + row(), + row({ id: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", message_id: OTHER_MESSAGE, status: "ready" }), + ], + }, + ]) + + const byMessage = await loadServableAttachmentsFor( + fake.sql as unknown as Queryable, + "chat_message_id", + [MESSAGE, OTHER_MESSAGE], + PRESIGN, + ) + + expect(byMessage.get(MESSAGE)).toHaveLength(1) + expect(byMessage.get(OTHER_MESSAGE)?.[0]?.status).toBe("ready") + }) + + it("short-circuits on an empty id list without touching the database", async () => { + const fake = makeFakeSql([]) + + const byMessage = await loadServableAttachmentsFor( + fake.sql as unknown as Queryable, + "chat_message_id", + [], + PRESIGN, + ) + + expect(byMessage.size).toBe(0) + expect(fake.statements).toHaveLength(0) + }) +}) + +describe("makeAttachmentRepo().attach", () => { + it("still claims a finalized validating asset — the read now matches what the claim accepts", async () => { + const fake = makeFakeSql([{ match: /UPDATE media_assets/, rows: [] }]) + + await makeAttachmentRepo("chat_message_id").attach( + fake.sql as unknown as Queryable, + MESSAGE, + ["cccccccc-cccc-4ccc-8ccc-cccccccccccc"], + new Date("2026-09-16T00:00:00.000Z"), + ) + + const statement = fake.statements.at(-1)! + expect(statement.sql).toMatch( + /status = 'ready' OR \(status = 'validating' AND finalized_at IS NOT NULL\)/, + ) + }) + + it("does nothing when no upload ids were sent", async () => { + const fake = makeFakeSql([]) + + await makeAttachmentRepo("chat_message_id").attach( + fake.sql as unknown as Queryable, + MESSAGE, + [], + new Date(), + ) + + expect(fake.statements).toHaveLength(0) + }) +}) From 305f287c1b1b2b3f451f96a58b1e6879a881d162 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 16:09:07 -0700 Subject: [PATCH 20/54] migration 0174: announcement broadcast kind, listing index, scrub exemption --- ...2026-07-24-full-backend-security-review.md | 4 +- .../api/drizzle/0174_event_announcements.sql | 58 +++++++++++++++++++ services/api/src/db/schema/broadcasts.ts | 5 +- services/api/src/db/schema/types-broadcast.ts | 1 + 4 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 services/api/drizzle/0174_event_announcements.sql diff --git a/docs/security/2026-07-24-full-backend-security-review.md b/docs/security/2026-07-24-full-backend-security-review.md index 5bebede8..d5c94fb1 100644 --- a/docs/security/2026-07-24-full-backend-security-review.md +++ b/docs/security/2026-07-24-full-backend-security-review.md @@ -322,7 +322,7 @@ Everything an operator has to do by hand, in order, plus the two infra facts and node dist/db/migrate.js # == pnpm --filter @civfix/api db:migrate ``` -`drizzle/` holds **156 files**, `0000_extensions.sql` … `0173_posts_author_public_recent_idx.sql`. The nine rows +`drizzle/` holds **157 files**, `0000_extensions.sql` … `0174_event_announcements.sql`. The nine rows below are exactly what this change set adds — `0052`–`0060`, contiguous, no gaps — and everything from `0000` through `0051_social_posts.sql` predates it. (`0060` arrived later than the rest, with the feed redesign; it is listed here because this table is the single operator runbook. `0061`–`0064` arrived @@ -563,6 +563,8 @@ foreign key into `organizations` from `0105`. | `0173_posts_author_public_recent_idx.sql` | adds the partial index `posts_author_public_recent_idx (author_id, created_at DESC, id DESC) WHERE deleted_at IS NULL AND reply_to_id IS NULL AND visibility = 'public'` - the author-keyed twin of `0071_posts_toplevel_recent_idx`, backing the ranked feed's in-network candidate pool (self + followees). Neither `posts_author_created_idx` (0051, not partial on reply_to_id/visibility, no id tiebreak) nor `posts_toplevel_recent_idx` (0071, not author-keyed) serves that shape. Same not-a-hot-table reasoning as 0172 | The in-network pool sorts instead of stopping at its LIMIT. Correct, slower; no functional change | +| `0174_event_announcements.sql` | widens `broadcasts_kind_check` to accept the new `announcement` kind (event announcements ride the existing host-broadcast pipeline rather than a parallel table), adds the partial `broadcasts_announcement_public_idx (cleanup_id, created_at DESC, id DESC) WHERE kind = 'announcement'` that backs the public per-event announcements list, and re-creates `broadcasts_scrub_idx` with `kind <> 'announcement'` so the retention scrub can never NULL an announcement body - an announcement is permanent public event content rendered on the event page, not a one-shot email. `broadcasts` is NOT on the hot-table list in `docs/out-of-band-indexes.md`, so both indexes build inline; `IF NOT EXISTS` makes an out-of-band CONCURRENTLY build a no-op | Every `createEventAnnouncement` fails on the CHECK constraint. Applying the CHECK but not the scrub-index swap still works (the repository query carries the same predicate), it just scans more rows | + **Deferred to the NEXT release** (expand/contract, `docs/migrations-expand-contract.md`): 0.43.0 stops every code path from reading `user_verification` but does NOT drop it — a `DROP TABLE` in the diff --git a/services/api/drizzle/0174_event_announcements.sql b/services/api/drizzle/0174_event_announcements.sql new file mode 100644 index 00000000..7cfa90cd --- /dev/null +++ b/services/api/drizzle/0174_event_announcements.sql @@ -0,0 +1,58 @@ +-- ============================================================================= +-- 0174_event_announcements.sql +-- ----------------------------------------------------------------------------- +-- Event announcements ride the existing host-broadcast pipeline as a new +-- broadcasts.kind rather than a parallel table: segment resolution, delivery +-- fan-out, suppression, unsubscribes and counting already exist and are tested. +-- +-- Three concerns, all on `broadcasts`: +-- +-- 1. kind CHECK widening. `broadcasts_kind_check` (0130) enumerates the seven +-- kinds that existed then; 'announcement' has to join them or every insert +-- fails. Dropped and recreated because a CHECK cannot be widened in place. +-- +-- 2. Public listing index. listEventAnnouncements reads an event's sent +-- announcements newest-first with a keyset cursor. The existing +-- broadcasts_cleanup_created_idx has the right ordering but spans every +-- kind, so an event whose hosts also send reminders/confirmations scans +-- rows the announcements list can never return. This partial twin is +-- keyed the same way and carries only announcement rows. +-- +-- 3. Scrub exemption. The retention lane +-- (broadcast-repository.scrubBroadcastContent) NULLs subject/body_md on +-- finished broadcasts past the cutoff -- correct for a one-shot email, +-- wrong for an announcement, which is permanent PUBLIC event content +-- rendered on the event page forever. The scrub's driving index gets the +-- matching `kind <> 'announcement'` predicate so exempt rows are not even +-- candidates, and the repository query carries the same predicate. +-- +-- NOT A HOT TABLE: `broadcasts` is absent from the hot-table list in +-- docs/out-of-band-indexes.md, so these build inline. IF NOT EXISTS keeps the +-- file a no-op if an index was built out of band with CONCURRENTLY first. +-- +-- CANONICAL DDL: hand-authored source of truth. Mirror: schema/broadcasts.ts. +-- Enum mirrors: schema/types-broadcast.ts (BROADCAST_KIND_VALUES) and shared +-- BroadcastKindSchema -- 'announcement' is APPENDED LAST in all three. +-- +-- Conventions: one concern per file; one transaction per file. Forward-only, +-- no down. +-- +-- Ordering rules: requires 0130_broadcasts.sql. +-- LOCK ORDER: broadcasts +-- ============================================================================= + +ALTER TABLE broadcasts DROP CONSTRAINT IF EXISTS broadcasts_kind_check; + +ALTER TABLE broadcasts ADD CONSTRAINT broadcasts_kind_check CHECK (kind IN ( + 'host_broadcast','confirmation','waitlist_promoted','reminder', + 'event_updated','event_cancelled','thank_you','announcement')); + +CREATE INDEX IF NOT EXISTS broadcasts_announcement_public_idx + ON broadcasts (cleanup_id, created_at DESC, id DESC) + WHERE kind = 'announcement'; + +DROP INDEX IF EXISTS broadcasts_scrub_idx; + +CREATE INDEX IF NOT EXISTS broadcasts_scrub_idx + ON broadcasts (finished_at) + WHERE content_scrubbed_at IS NULL AND body_md IS NOT NULL AND kind <> 'announcement'; diff --git a/services/api/src/db/schema/broadcasts.ts b/services/api/src/db/schema/broadcasts.ts index 35170a76..c6ee2449 100644 --- a/services/api/src/db/schema/broadcasts.ts +++ b/services/api/src/db/schema/broadcasts.ts @@ -69,9 +69,12 @@ export const broadcasts = pgTable( uniqueIndex("broadcasts_cancellation_uidx") .on(t.cleanupId) .where(sql`kind = 'event_cancelled'`), + index("broadcasts_announcement_public_idx") + .on(t.cleanupId, t.createdAt.desc(), t.id.desc()) + .where(sql`kind = 'announcement'`), index("broadcasts_scrub_idx") .on(t.finishedAt) - .where(sql`content_scrubbed_at IS NULL AND body_md IS NOT NULL`), + .where(sql`content_scrubbed_at IS NULL AND body_md IS NOT NULL AND kind <> 'announcement'`), index("broadcasts_admin_log_idx").on(t.createdAt.desc(), t.id.desc()), ], ) diff --git a/services/api/src/db/schema/types-broadcast.ts b/services/api/src/db/schema/types-broadcast.ts index a7357f62..668ff7df 100644 --- a/services/api/src/db/schema/types-broadcast.ts +++ b/services/api/src/db/schema/types-broadcast.ts @@ -6,6 +6,7 @@ export const BROADCAST_KIND_VALUES = [ "event_updated", "event_cancelled", "thank_you", + "announcement", ] as const export const BROADCAST_STATUS_VALUES = [ From 3f08f2a321b15aa289bd6c99bffddec82d1ac1d0 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 16:09:07 -0700 Subject: [PATCH 21/54] event announcements on the existing broadcast pipeline --- .../src/routes/host/announcements.routes.ts | 156 ++++++++ services/api/src/routes/host/index.ts | 2 + .../host/announcement-repository.drizzle.ts | 105 +++++ .../src/services/host/announcement-service.ts | 223 +++++++++++ .../src/services/host/broadcast-pipeline.ts | 24 +- .../host/broadcast-repository.drizzle.ts | 33 ++ .../host/broadcast-repository.memory.ts | 36 ++ .../src/services/host/broadcast-repository.ts | 8 + .../api/src/services/host/broadcast-types.ts | 3 + .../unit/host/announcement-pipeline.test.ts | 152 ++++++++ .../unit/host/announcement-service.test.ts | 365 ++++++++++++++++++ 11 files changed, 1105 insertions(+), 2 deletions(-) create mode 100644 services/api/src/routes/host/announcements.routes.ts create mode 100644 services/api/src/services/host/announcement-repository.drizzle.ts create mode 100644 services/api/src/services/host/announcement-service.ts create mode 100644 services/api/test/unit/host/announcement-pipeline.test.ts create mode 100644 services/api/test/unit/host/announcement-service.test.ts diff --git a/services/api/src/routes/host/announcements.routes.ts b/services/api/src/routes/host/announcements.routes.ts new file mode 100644 index 00000000..1c24a1f7 --- /dev/null +++ b/services/api/src/routes/host/announcements.routes.ts @@ -0,0 +1,156 @@ +import { + CreateEventAnnouncementRequestSchema, + GetEventAnnouncementRequestSchema, + ListEventAnnouncementsRequestSchema, + type AnnouncementDTO, + type ListEventAnnouncementsResponse, +} from "@civfix/shared" +import { can } from "@civfix/shared/host" +import type { FastifyInstance, FastifyRequest } from "fastify" +import type { Container } from "../../di.js" +import { requireAuth } from "../../auth/context.js" +import { perIdentity } from "../../plugins/rate-limit.js" +import { route } from "../../versioning/route.js" +import { parse, trimTextFields } from "../_validate.js" +import { requireCapability, resolveVisibleStanding } from "../../services/host/authz.js" +import { writeAudit } from "../../services/admin/audit.js" +import { + BroadcastCapError, + capError, +} from "../../services/host/broadcast-service.js" +import { makeCommsRuntime } from "../../services/host/comms-wiring.js" +import type { CommsRuntime } from "../../services/host/comms-wiring.js" +import type { + AnnouncementProjection, + AnnouncementService, +} from "../../services/host/announcement-service.js" + +export const ANNOUNCEMENT_CREATE_RATE_LIMIT = perIdentity({ max: 10, timeWindow: "1 minute" }) +export const ANNOUNCEMENT_READ_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute" }) + +export interface AnnouncementOverrides { + runtime: CommsRuntime +} + +declare module "fastify" { + interface FastifyInstance { + announcementOverrides?: AnnouncementOverrides + } +} + +function mergeParams(request: FastifyRequest): Record { + const params = (request.params ?? {}) as Record + const body = (request.body ?? {}) as Record + return { ...body, ...params } +} + +function mergeQuery(request: FastifyRequest): Record { + const params = (request.params ?? {}) as Record + const query = (request.query ?? {}) as Record + return { ...query, ...params } +} + +export async function registerHostAnnouncementRoutes( + app: FastifyInstance, + container: Container, +): Promise { + const csrfProtect = container.csrf.protect + let cached: CommsRuntime | undefined + + function runtime(): CommsRuntime { + const override = app.announcementOverrides + if (override) return override.runtime + return (cached ??= makeCommsRuntime(container, app.log)) + } + + function service(): AnnouncementService { + return runtime().announcements + } + + async function projectionFor( + cleanupId: string, + userId: string | null, + ): Promise { + const resolution = await resolveVisibleStanding(container.getDb().sql, cleanupId, userId) + return { host: can(resolution.standing, "broadcast") } + } + + route( + app, + "createEventAnnouncement", + { preHandler: csrfProtect, config: { rateLimit: ANNOUNCEMENT_CREATE_RATE_LIMIT } }, + async (request, reply) => { + const userId = requireAuth(request) + const body = parse( + trimTextFields(CreateEventAnnouncementRequestSchema, "title", "bodyMd"), + mergeParams(request), + ) + await requireCapability(container.getDb().sql, body.id, userId, "broadcast") + let payload: AnnouncementDTO + try { + payload = await service().create(body.id, userId, body) + } catch (err) { + if (err instanceof BroadcastCapError) throw capError(err.kind) + throw err + } + await audit(container, "event.announcement_sent", userId, payload.id, { + cleanupId: body.id, + audience: body.audience.kind, + }) + reply.status(200).send(payload) + }, + ) + + route( + app, + "listEventAnnouncements", + { config: { rateLimit: ANNOUNCEMENT_READ_RATE_LIMIT } }, + async (request, reply) => { + const userId = request.auth?.userId ?? null + const query = parse(ListEventAnnouncementsRequestSchema, mergeQuery(request)) + const projection = await projectionFor(query.id, userId) + const payload: ListEventAnnouncementsResponse = await service().list( + query.id, + query, + projection, + ) + reply.status(200).send(payload) + }, + ) + + route( + app, + "getEventAnnouncement", + { config: { rateLimit: ANNOUNCEMENT_READ_RATE_LIMIT } }, + async (request, reply) => { + const userId = request.auth?.userId ?? null + const params = parse(GetEventAnnouncementRequestSchema, mergeParams(request)) + const projection = await projectionFor(params.id, userId) + const payload: AnnouncementDTO = await service().get( + params.id, + params.announcementId, + projection, + ) + reply.status(200).send(payload) + }, + ) +} + +async function audit( + container: Container, + action: string, + actorId: string, + announcementId: string, + meta: Record, +): Promise { + try { + await writeAudit(container.getDb().sql, { + action, + actorId, + target: `broadcast:${announcementId}`, + meta, + }) + } catch { + return + } +} diff --git a/services/api/src/routes/host/index.ts b/services/api/src/routes/host/index.ts index 145618e5..fbd3cdd9 100644 --- a/services/api/src/routes/host/index.ts +++ b/services/api/src/routes/host/index.ts @@ -2,6 +2,7 @@ import type { FastifyInstance } from "fastify" import type { Container } from "../../di.js" import { makeHostRouteContext } from "./_host-routes.js" import { registerHostAnalyticsRoutes } from "./analytics.routes.js" +import { registerHostAnnouncementRoutes } from "./announcements.routes.js" import { registerHostBroadcastRoutes } from "./broadcasts.routes.js" import { registerHostCheckinRoutes } from "./checkin.routes.js" import { registerHostExportRoutes } from "./exports.routes.js" @@ -38,6 +39,7 @@ export async function registerHostCommsRoutes( container: Container, ): Promise { await registerHostBroadcastRoutes(app, container) + await registerHostAnnouncementRoutes(app, container) await registerHostAnalyticsRoutes(app, container) await registerHostExportRoutes(app, container) } diff --git a/services/api/src/services/host/announcement-repository.drizzle.ts b/services/api/src/services/host/announcement-repository.drizzle.ts new file mode 100644 index 00000000..c139f1ff --- /dev/null +++ b/services/api/src/services/host/announcement-repository.drizzle.ts @@ -0,0 +1,105 @@ +import type { OrganizationRefDTO, PersonDTO } from "@civfix/shared" +import type { Sql } from "../../db/client.js" +import { PRESIGN_CONCURRENCY, mapWithLimit } from "../media-presign.js" +import { presentIds } from "../present-ids.js" +import { publicAuthorIdentity } from "../public-author.js" +import { servedKeyExpr } from "../media-served-key.js" + +export type AnnouncementImagePresigner = (r2Key: string) => Promise + +export interface AnnouncementIdentityRepository { + authorsFor(userIds: readonly (string | null)[]): Promise> + organizationFor(cleanupId: string): Promise +} + +interface AuthorRowSelect { + id: string + display_name: string | null + handle: string | null + bio: string | null + avatar_r2_key: string | null + avatar_url: string | null + deleted_at: Date | null +} + +interface OrganizationRowSelect { + id: string + slug: string + name: string + verified_status: string + verified_kind: OrganizationRefDTO["verifiedKind"] + logo_key: string | null +} + +export function makeDrizzleAnnouncementIdentityRepository( + sql: Sql, + presign: AnnouncementImagePresigner, +): AnnouncementIdentityRepository { + return { + async authorsFor(userIds) { + const out = new Map() + const ids = presentIds(userIds) + if (ids.length === 0) return out + const rows = await sql` + SELECT u.id, u.display_name, u.handle, u.bio, + ${servedKeyExpr(sql, "am")} AS avatar_r2_key, + u.avatar_url, u.deleted_at + FROM users u + LEFT JOIN media_assets am ON am.id = u.avatar_media_id + WHERE u.id = ANY(${ids}::uuid[]) + ` + const resolved = await mapWithLimit(rows, PRESIGN_CONCURRENCY, async (row) => { + const avatarUrl = + row.deleted_at === null && row.avatar_r2_key !== null + ? await presign(row.avatar_r2_key) + : row.avatar_url + const identity = publicAuthorIdentity({ + id: row.id, + displayName: row.display_name, + handle: row.handle, + avatarUrl, + deletedAt: row.deleted_at, + }) + const person: PersonDTO = { + id: row.id, + name: identity.name, + handle: identity.handle, + bio: identity.deleted ? null : row.bio, + avatar: identity.avatar, + ...(identity.avatarUrl !== undefined ? { avatarUrl: identity.avatarUrl } : {}), + followers: 0, + following: 0, + isFollowing: false, + ...(identity.deleted ? { deleted: true } : {}), + } + return person + }) + for (const person of resolved) out.set(person.id, person) + return out + }, + + async organizationFor(cleanupId) { + const rows = await sql` + SELECT o.id, o.slug, o.name, o.verified_status, o.verified_kind, + ${servedKeyExpr(sql, "am")} AS logo_key + FROM cleanups c + JOIN organizations o ON o.id = c.organization_id + LEFT JOIN media_assets am ON am.id = o.logo_media_id + WHERE c.id = ${cleanupId} + AND o.deleted_at IS NULL + LIMIT 1 + ` + const row = rows[0] + if (row === undefined) return null + const logoUrl = row.logo_key === null ? null : await presign(row.logo_key) + return { + id: row.id, + slug: row.slug, + name: row.name, + logoUrl, + verified: row.verified_status === "verified", + verifiedKind: row.verified_kind, + } + }, + } +} diff --git a/services/api/src/services/host/announcement-service.ts b/services/api/src/services/host/announcement-service.ts new file mode 100644 index 00000000..c85baace --- /dev/null +++ b/services/api/src/services/host/announcement-service.ts @@ -0,0 +1,223 @@ +import { + ANNOUNCEMENT_BROADCAST_KIND, + ANNOUNCEMENT_CHANNELS, + AppError, + MAX_EVENT_ANNOUNCEMENTS_PER_DAY, + announcementAudienceToSegment, +} from "@civfix/shared" +import type { + AnnouncementAudience, + AnnouncementDTO, + BroadcastChannel, + CreateEventAnnouncementRequest, + ListEventAnnouncementsRequest, + ListEventAnnouncementsResponse, + PersonDTO, +} from "@civfix/shared" +import type { FastifyBaseLogger } from "fastify" +import { encodeTimeCursor, parseTimeCursor } from "../../db/cursor-helpers.js" +import type { BroadcastRepository } from "./broadcast-repository.js" +import type { BroadcastRecord } from "./broadcast-types.js" +import { broadcastLinkWarnings } from "./broadcast-render.js" +import { announcementPath } from "./broadcast-pipeline.js" +import type { BroadcastConfig, BroadcastService } from "./broadcast-service.js" +import type { AnnouncementIdentityRepository } from "./announcement-repository.drizzle.js" + +export const ANNOUNCEMENT_DEFAULT_LIMIT = 20 +export const ANNOUNCEMENT_MAX_LIMIT = 50 +export const ANNOUNCEMENT_WINDOW_MS = 24 * 60 * 60 * 1000 +export const ANNOUNCEMENT_CTA_LABEL = "View announcement" + +export interface AnnouncementServiceDeps { + repo: BroadcastRepository + identities: AnnouncementIdentityRepository + broadcasts: BroadcastService + config: Pick + logger?: Pick + now?: () => Date +} + +export interface AnnouncementProjection { + host: boolean +} + +export interface AnnouncementService { + create( + cleanupId: string, + actorId: string, + body: CreateEventAnnouncementRequest, + ): Promise + list( + cleanupId: string, + query: ListEventAnnouncementsRequest, + projection: AnnouncementProjection, + ): Promise + get( + cleanupId: string, + announcementId: string, + projection: AnnouncementProjection, + ): Promise +} + +export function announcementRateLimited(): AppError { + return AppError.rateLimited( + "Announcement limit reached for today — use the group chat for ongoing discussion.", + ) +} + +export function announcementTitleOf(title: string | null | undefined, eventTitle: string): string { + const trimmed = (title ?? "").trim() + return trimmed.length > 0 ? trimmed : `Announcement · ${eventTitle}` +} + +export function makeAnnouncementService(deps: AnnouncementServiceDeps): AnnouncementService { + const now = deps.now ?? (() => new Date()) + const { repo, config } = deps + + function notFound(): AppError { + return AppError.notFound("Announcement not found") + } + + function audienceOf(record: BroadcastRecord): AnnouncementAudience | null { + const segment = record.segment + if (segment === null) return null + if ( + segment.kind === "all_registered" || + segment.kind === "checked_in" || + segment.kind === "not_checked_in" || + segment.kind === "waitlist" || + segment.kind === "slots" + ) { + return segment + } + return null + } + + function toDTO( + record: BroadcastRecord, + author: PersonDTO | null, + authorOrg: AnnouncementDTO["authorOrg"], + projection: AnnouncementProjection, + ): AnnouncementDTO { + const base: AnnouncementDTO = { + id: record.id, + cleanupId: record.cleanupId, + status: record.status, + author, + authorOrg: authorOrg ?? null, + title: record.subject, + bodyMd: record.bodyMd ?? "", + sentAt: (record.finishedAt ?? record.startedAt)?.toISOString() ?? null, + createdAt: record.createdAt.toISOString(), + } + if (!projection.host) return base + return { + ...base, + audience: audienceOf(record), + recipientCount: record.recipientCount, + sentCount: record.sentCount, + failedCount: record.failedCount, + } + } + + async function hydrate( + cleanupId: string, + records: readonly BroadcastRecord[], + projection: AnnouncementProjection, + ): Promise { + if (records.length === 0) return [] + const [authors, organization] = await Promise.all([ + deps.identities.authorsFor(records.map((record) => record.createdBy)), + deps.identities.organizationFor(cleanupId), + ]) + return records.map((record) => + toDTO( + record, + record.createdBy === null ? null : (authors.get(record.createdBy) ?? null), + organization, + projection, + ), + ) + } + + async function assertUnderDailyCap(cleanupId: string): Promise { + const since = new Date(now().getTime() - ANNOUNCEMENT_WINDOW_MS) + const used = await repo.countAnnouncementsSince(cleanupId, since) + if (used >= MAX_EVENT_ANNOUNCEMENTS_PER_DAY) throw announcementRateLimited() + } + + function ctaFor(cleanupId: string, announcementId: string): string | null { + const url = `${config.webBaseUrl}${announcementPath(cleanupId, announcementId)}` + return broadcastLinkWarnings(url, config.linkAllowedHosts).length === 0 ? url : null + } + + return { + async create(cleanupId, actorId, body) { + await deps.broadcasts.assertComposeAllowed(cleanupId, actorId) + await assertUnderDailyCap(cleanupId) + + const event = await repo.eventContext(cleanupId) + if (event === null) throw AppError.notFound("Cleanup not found") + + const draft = await repo.create({ + cleanupId, + createdBy: actorId, + kind: ANNOUNCEMENT_BROADCAST_KIND, + subject: announcementTitleOf(body.title, event.title), + bodyMd: body.bodyMd, + segment: announcementAudienceToSegment(body.audience), + channels: [...ANNOUNCEMENT_CHANNELS] as BroadcastChannel[], + status: "draft", + chunkSize: config.chunkSize, + }) + + try { + const cta = ctaFor(cleanupId, draft.id) + if (cta !== null) { + await repo.updateDraft(cleanupId, draft.id, { + ctaLabel: ANNOUNCEMENT_CTA_LABEL, + ctaUrl: cta, + }) + } + await deps.broadcasts.send(cleanupId, actorId, draft.id) + } catch (err) { + await repo.deleteDraft(cleanupId, draft.id).catch(() => false) + throw err + } + + const sent = await repo.findForEvent(cleanupId, draft.id) + if (sent === null) throw notFound() + const [dto] = await hydrate(cleanupId, [sent], { host: true }) + if (dto === undefined) throw notFound() + return dto + }, + + async list(cleanupId, query, projection) { + const limit = Math.min(query.limit ?? ANNOUNCEMENT_DEFAULT_LIMIT, ANNOUNCEMENT_MAX_LIMIT) + const cursor = parseTimeCursor(query.cursor, { direction: "desc" }) + const rows = await repo.listAnnouncements({ + cleanupId, + cursor: cursor === null ? null : { createdAt: cursor.at, id: cursor.id }, + limit: limit + 1, + }) + const page = rows.slice(0, limit) + const last = page.at(-1) + const nextCursor = + rows.length > limit && last !== undefined + ? encodeTimeCursor({ at: last.createdAt, id: last.id }) + : null + return { items: await hydrate(cleanupId, page, projection), nextCursor } + }, + + async get(cleanupId, announcementId, projection) { + const record = await repo.findForEvent(cleanupId, announcementId) + if (record === null || record.kind !== ANNOUNCEMENT_BROADCAST_KIND) throw notFound() + if (!projection.host && record.status !== "sending" && record.status !== "sent") { + throw notFound() + } + const [dto] = await hydrate(cleanupId, [record], projection) + if (dto === undefined) throw notFound() + return dto + }, + } +} diff --git a/services/api/src/services/host/broadcast-pipeline.ts b/services/api/src/services/host/broadcast-pipeline.ts index 84a33487..ec87d639 100644 --- a/services/api/src/services/host/broadcast-pipeline.ts +++ b/services/api/src/services/host/broadcast-pipeline.ts @@ -1,3 +1,4 @@ +import { ANNOUNCEMENT_BROADCAST_KIND } from "@civfix/shared" import type { BroadcastKind, DeliveryFailureKind, NotificationType } from "@civfix/shared" import type { BroadcastVarValues } from "@civfix/shared/host" import type { Mailer } from "@civfix/shared/interfaces" @@ -61,6 +62,16 @@ export const KIND_NOTIFICATION_TYPE: Record = { event_cancelled: "cleanup_cancelled", confirmation: "system", waitlist_promoted: "system", + announcement: "event_broadcast", +} + +export const BUDGETED_KINDS: ReadonlySet = new Set([ + "host_broadcast", + ANNOUNCEMENT_BROADCAST_KIND, +]) + +export function announcementPath(cleanupId: string, announcementId: string): string { + return `/cleanups/${cleanupId}/announcements/${announcementId}` } export const PER_RECIPIENT_VARS = ["first_name", "ticket_type"] as const @@ -137,6 +148,15 @@ export interface PlanOutcome { chunks?: number } +export function notificationLink( + record: Pick, + event: Pick, +): string { + return record.kind === ANNOUNCEMENT_BROADCAST_KIND + ? announcementPath(record.cleanupId, record.id) + : eventPath(event.pageSlug, record.cleanupId) +} + export function usesVar(text: string, name: string): boolean { return text.includes(`{${name}}`) } @@ -266,7 +286,7 @@ export function makeBroadcastPipeline(deps: BroadcastPipelineDeps) { } const reservedNow = await repo.markPlanned(record.id, { recipientCount, plannedAt: now() }) - if (reservedNow && record.createdBy !== null && record.kind === "host_broadcast") { + if (reservedNow && record.createdBy !== null && BUDGETED_KINDS.has(record.kind)) { const withinBudget = await deps.service.reserveRecipientBudget(record.createdBy, recipientCount) if (!withinBudget) { await repo.suppressRemaining(record.id, "cap") @@ -469,7 +489,7 @@ export function makeBroadcastPipeline(deps: BroadcastPipelineDeps) { type, title: group.rendering.inAppTitle, body: group.rendering.inAppBody, - link: eventPath(event.pageSlug, record.cleanupId), + link: notificationLink(record, event), push: mode, }) } catch (err) { diff --git a/services/api/src/services/host/broadcast-repository.drizzle.ts b/services/api/src/services/host/broadcast-repository.drizzle.ts index 1b670e27..c8122296 100644 --- a/services/api/src/services/host/broadcast-repository.drizzle.ts +++ b/services/api/src/services/host/broadcast-repository.drizzle.ts @@ -1,3 +1,4 @@ +import { ANNOUNCEMENT_BROADCAST_KIND } from "@civfix/shared" import type { BroadcastChannel, BroadcastKind, @@ -9,6 +10,7 @@ import type { Sql } from "../../db/client.js" import { listGuestAudiencePage, listMemberAudiencePage } from "./broadcast-audience-sql.js" import type { AdminBroadcastListQuery, + AnnouncementListQuery, AudiencePageQuery, BroadcastListQuery, BroadcastRepository, @@ -34,6 +36,7 @@ import type { HostMessagingState, MemberContact, } from "./broadcast-types.js" +import { ANNOUNCEMENT_VISIBLE_STATUSES } from "./broadcast-types.js" interface BroadcastRowSelect { id: string @@ -194,6 +197,35 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { return rows.map(toRecord) }, + async listAnnouncements(query: AnnouncementListQuery): Promise { + const cursorFilter = + query.cursor !== null + ? sql`AND (created_at, id) < (${query.cursor.createdAt}, ${query.cursor.id})` + : sql`` + const rows = await sql` + SELECT id, cleanup_id, created_by, kind, reminder_offset_min, status, subject, body_md, + cta_label, cta_url, segment, channels, reply_to, scheduled_at, planned_at, started_at, + finished_at, chunk_size, chunk_count, recipient_count, sent_count, failed_count, + suppressed_count, content_scrubbed_at, created_at, updated_at FROM broadcasts + WHERE cleanup_id = ${query.cleanupId} + AND kind = ${ANNOUNCEMENT_BROADCAST_KIND} + AND status = ANY(${[...ANNOUNCEMENT_VISIBLE_STATUSES]}::text[]) + ${cursorFilter} + ORDER BY created_at DESC, id DESC + LIMIT ${query.limit}` + return rows.map(toRecord) + }, + + async countAnnouncementsSince(cleanupId: string, since: Date): Promise { + const rows = await sql<{ n: number }[]>` + SELECT count(*)::int AS n FROM broadcasts + WHERE cleanup_id = ${cleanupId} + AND kind = ${ANNOUNCEMENT_BROADCAST_KIND} + AND status <> 'draft' + AND created_at >= ${since}` + return rows[0]?.n ?? 0 + }, + async listAdmin(query: AdminBroadcastListQuery): Promise { const statusFilter = query.status !== undefined ? sql`AND b.status = ${query.status}` : sql`` const kindFilter = query.kind !== undefined ? sql`AND b.kind = ${query.kind}` : sql`` @@ -947,6 +979,7 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { SELECT id FROM broadcasts WHERE content_scrubbed_at IS NULL AND body_md IS NOT NULL + AND kind <> ${ANNOUNCEMENT_BROADCAST_KIND} AND finished_at IS NOT NULL AND finished_at < ${cutoff} ORDER BY finished_at diff --git a/services/api/src/services/host/broadcast-repository.memory.ts b/services/api/src/services/host/broadcast-repository.memory.ts index b9c3ae84..843fe4dd 100644 --- a/services/api/src/services/host/broadcast-repository.memory.ts +++ b/services/api/src/services/host/broadcast-repository.memory.ts @@ -1,7 +1,9 @@ import { randomUUID } from "node:crypto" +import { ANNOUNCEMENT_BROADCAST_KIND } from "@civfix/shared" import type { BroadcastKind, BroadcastStatus } from "@civfix/shared" import type { AdminBroadcastListQuery, + AnnouncementListQuery, AudiencePageQuery, BroadcastListQuery, BroadcastRepository, @@ -28,6 +30,7 @@ import type { MemberContact, } from "./broadcast-types.js" import { + ANNOUNCEMENT_VISIBLE_STATUSES, CRITICAL_BROADCAST_KINDS, HOST_COMPOSED_BROADCAST_KINDS, } from "./broadcast-types.js" @@ -120,6 +123,12 @@ export class InMemoryBroadcastRepository implements BroadcastRepository { this.guests.set(cleanupId, rows.map((row) => ({ registered: true, ...row }))) } + forceCreatedAt(broadcastId: string, at: Date): void { + const found = this.broadcasts.get(broadcastId) + if (found === undefined) return + this.broadcasts.set(broadcastId, { ...found, createdAt: at }) + } + forceUpdatedAt(broadcastId: string, at: Date): void { const found = this.broadcasts.get(broadcastId) if (found === undefined) return @@ -196,6 +205,32 @@ export class InMemoryBroadcastRepository implements BroadcastRepository { return Promise.resolve(rows.slice(0, query.limit)) } + listAnnouncements(query: AnnouncementListQuery): Promise { + const rows = [...this.broadcasts.values()] + .filter((b) => b.cleanupId === query.cleanupId) + .filter((b) => b.kind === ANNOUNCEMENT_BROADCAST_KIND) + .filter((b) => ANNOUNCEMENT_VISIBLE_STATUSES.includes(b.status)) + .filter( + (b) => + query.cursor === null || + b.createdAt.getTime() < query.cursor.createdAt.getTime() || + (b.createdAt.getTime() === query.cursor.createdAt.getTime() && b.id < query.cursor.id), + ) + .sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime() || (a.id < b.id ? 1 : -1)) + return Promise.resolve(rows.slice(0, query.limit)) + } + + countAnnouncementsSince(cleanupId: string, since: Date): Promise { + const rows = [...this.broadcasts.values()].filter( + (b) => + b.cleanupId === cleanupId && + b.kind === ANNOUNCEMENT_BROADCAST_KIND && + b.status !== "draft" && + b.createdAt.getTime() >= since.getTime(), + ) + return Promise.resolve(rows.length) + } + listAdmin(query: AdminBroadcastListQuery): Promise { const rows = [...this.broadcasts.values()] .filter((b) => query.status === undefined || b.status === query.status) @@ -695,6 +730,7 @@ export class InMemoryBroadcastRepository implements BroadcastRepository { for (const [id, row] of this.broadcasts) { if (n >= batchSize) break if (row.contentScrubbedAt !== null || row.bodyMd === null) continue + if (row.kind === ANNOUNCEMENT_BROADCAST_KIND) continue if (row.finishedAt === null || row.finishedAt.getTime() >= cutoff.getTime()) continue this.broadcasts.set(id, { ...row, diff --git a/services/api/src/services/host/broadcast-repository.ts b/services/api/src/services/host/broadcast-repository.ts index ce4492af..71a6977d 100644 --- a/services/api/src/services/host/broadcast-repository.ts +++ b/services/api/src/services/host/broadcast-repository.ts @@ -56,6 +56,12 @@ export interface AdminBroadcastListQuery { limit: number } +export interface AnnouncementListQuery { + cleanupId: string + cursor: { createdAt: Date; id: string } | null + limit: number +} + export interface AudiencePageQuery { cleanupId: string segment: BroadcastSegment @@ -72,6 +78,8 @@ export interface BroadcastRepository { findById(broadcastId: string): Promise findForEvent(cleanupId: string, broadcastId: string): Promise list(query: BroadcastListQuery): Promise + listAnnouncements(query: AnnouncementListQuery): Promise + countAnnouncementsSince(cleanupId: string, since: Date): Promise listAdmin(query: AdminBroadcastListQuery): Promise listAdminHosts(params: AdminHostListParams): Promise diff --git a/services/api/src/services/host/broadcast-types.ts b/services/api/src/services/host/broadcast-types.ts index 9ea56f06..80f84995 100644 --- a/services/api/src/services/host/broadcast-types.ts +++ b/services/api/src/services/host/broadcast-types.ts @@ -184,7 +184,10 @@ export const CRITICAL_BROADCAST_KINDS: ReadonlySet = new Set = new Set([ "host_broadcast", "thank_you", + "announcement", ]) diff --git a/services/api/test/unit/host/announcement-pipeline.test.ts b/services/api/test/unit/host/announcement-pipeline.test.ts new file mode 100644 index 00000000..35975d04 --- /dev/null +++ b/services/api/test/unit/host/announcement-pipeline.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from "vitest" +import { ANNOUNCEMENT_BROADCAST_KIND } from "@civfix/shared" +import { makeFakeSql } from "../../helpers/fake-sql.js" +import type { Sql } from "../../../src/db/client.js" +import { makeDrizzleBroadcastRepository } from "../../../src/services/host/broadcast-repository.drizzle.js" +import { InMemoryBroadcastRepository } from "../../../src/services/host/broadcast-repository.memory.js" +import { + KIND_NOTIFICATION_TYPE, + announcementPath, + notificationLink, +} from "../../../src/services/host/broadcast-pipeline.js" +import { + CRITICAL_BROADCAST_KINDS, + HOST_COMPOSED_BROADCAST_KINDS, +} from "../../../src/services/host/broadcast-types.js" + +const EVENT = "00000000-0000-0000-0000-0000000000ee" +const ANNOUNCEMENT = "00000000-0000-0000-0000-0000000000a1" + +describe("announcement notification routing", () => { + it("emits exactly the path the mobile deep-link allowlist accepts", () => { + expect(announcementPath(EVENT, ANNOUNCEMENT)).toBe( + `/cleanups/${EVENT}/announcements/${ANNOUNCEMENT}`, + ) + }) + + it("links an announcement to itself, not to the event page", () => { + const link = notificationLink( + { id: ANNOUNCEMENT, kind: ANNOUNCEMENT_BROADCAST_KIND, cleanupId: EVENT }, + { pageSlug: "beach-cleanup" }, + ) + expect(link).toBe(`/cleanups/${EVENT}/announcements/${ANNOUNCEMENT}`) + }) + + it("leaves every other broadcast kind pointing at the event page", () => { + expect( + notificationLink( + { id: ANNOUNCEMENT, kind: "host_broadcast", cleanupId: EVENT }, + { pageSlug: "beach-cleanup" }, + ), + ).toBe("/e/beach-cleanup") + }) + + it("reuses the event_broadcast notification type so the existing prefs plumbing applies", () => { + expect(KIND_NOTIFICATION_TYPE[ANNOUNCEMENT_BROADCAST_KIND]).toBe("event_broadcast") + }) +}) + +describe("announcement audience gates", () => { + it("is host-composed, so the hostBroadcasts preference opts a recipient out", () => { + expect(HOST_COMPOSED_BROADCAST_KINDS.has(ANNOUNCEMENT_BROADCAST_KIND)).toBe(true) + }) + + it("is not critical, so unsubscribes and per-event mutes are honored", () => { + expect(CRITICAL_BROADCAST_KINDS.has(ANNOUNCEMENT_BROADCAST_KIND)).toBe(false) + }) +}) + +describe("retention: announcements are permanent public content", () => { + it("the in-memory scrub skips announcements and still scrubs a host broadcast", async () => { + const repo = new InMemoryBroadcastRepository() + const finished = new Date("2026-01-01T00:00:00Z") + const announcement = await repo.create({ + cleanupId: EVENT, + createdBy: null, + kind: ANNOUNCEMENT_BROADCAST_KIND, + subject: "Pizza is here", + bodyMd: "Meet at the pavilion.", + segment: { kind: "all_registered" }, + channels: ["inapp", "push", "email"], + }) + const broadcast = await repo.create({ + cleanupId: EVENT, + createdBy: null, + kind: "host_broadcast", + subject: "Bring gloves", + bodyMd: "See you there.", + segment: { kind: "all_registered" }, + channels: ["email"], + }) + await repo.transition(announcement.id, ["draft"], "sent", { finishedAt: finished }) + await repo.transition(broadcast.id, ["draft"], "sent", { finishedAt: finished }) + + const scrubbed = await repo.scrubBroadcastContent(new Date("2026-06-01T00:00:00Z"), 100) + + expect(scrubbed).toBe(1) + expect((await repo.findById(announcement.id))?.bodyMd).toBe("Meet at the pavilion.") + expect((await repo.findById(broadcast.id))?.bodyMd).toBeNull() + }) + + it("the drizzle scrub statement carries the announcement exemption predicate", async () => { + const fake = makeFakeSql([{ match: /UPDATE broadcasts/, rows: [] }]) + + await makeDrizzleBroadcastRepository(fake.sql as unknown as Sql).scrubBroadcastContent( + new Date("2026-06-01T00:00:00Z"), + 100, + ) + + const statement = fake.statements.at(-1)! + expect(statement.sql).toMatch(/kind <> \?/) + expect(statement.values).toContain(ANNOUNCEMENT_BROADCAST_KIND) + }) +}) + +describe("announcement listing SQL", () => { + it("filters to the announcement kind and the publicly visible statuses, newest first", async () => { + const fake = makeFakeSql([{ match: /FROM broadcasts/, rows: [] }]) + + await makeDrizzleBroadcastRepository(fake.sql as unknown as Sql).listAnnouncements({ + cleanupId: EVENT, + cursor: null, + limit: 21, + }) + + const statement = fake.statements.at(-1)! + expect(statement.sql).toMatch(/kind = \?/) + expect(statement.sql).toMatch(/status = ANY\(\?::text\[\]\)/) + expect(statement.sql).toMatch(/ORDER BY created_at DESC, id DESC/) + expect(statement.values).toContain(ANNOUNCEMENT_BROADCAST_KIND) + expect(statement.values).toContain(21) + }) + + it("keyset-pages on (created_at, id) when a cursor is supplied", async () => { + const fake = makeFakeSql([{ match: /FROM broadcasts/, rows: [] }]) + const at = new Date("2026-01-20T12:00:00Z") + + await makeDrizzleBroadcastRepository(fake.sql as unknown as Sql).listAnnouncements({ + cleanupId: EVENT, + cursor: { createdAt: at, id: ANNOUNCEMENT }, + limit: 21, + }) + + const statement = fake.statements.at(-1)! + expect(statement.sql).toMatch(/\(created_at, id\) < \(\?, \?\)/) + expect(statement.values).toContain(ANNOUNCEMENT) + }) + + it("counts only non-draft announcements inside the rolling window", async () => { + const fake = makeFakeSql([{ match: /count\(\*\)::int AS n FROM broadcasts/, rows: [{ n: 3 }] }]) + const since = new Date("2026-01-19T12:00:00Z") + + const used = await makeDrizzleBroadcastRepository( + fake.sql as unknown as Sql, + ).countAnnouncementsSince(EVENT, since) + + expect(used).toBe(3) + const statement = fake.statements.at(-1)! + expect(statement.sql).toMatch(/status <> 'draft'/) + expect(statement.sql).toMatch(/created_at >= \?/) + expect(statement.values).toContain(since) + }) +}) diff --git a/services/api/test/unit/host/announcement-service.test.ts b/services/api/test/unit/host/announcement-service.test.ts new file mode 100644 index 00000000..695a611e --- /dev/null +++ b/services/api/test/unit/host/announcement-service.test.ts @@ -0,0 +1,365 @@ +import { beforeEach, describe, expect, it } from "vitest" +import { FakeMailer } from "@civfix/shared/fakes" +import { + ANNOUNCEMENT_BROADCAST_KIND, + MAX_EVENT_ANNOUNCEMENTS_PER_DAY, + type OrganizationRefDTO, + type PersonDTO, +} from "@civfix/shared" +import { InMemoryCounterStore } from "../../../src/abuse/counter-store.js" +import { InMemoryBroadcastRepository } from "../../../src/services/host/broadcast-repository.memory.js" +import { + makeBroadcastService, + type BroadcastConfig, +} from "../../../src/services/host/broadcast-service.js" +import { + announcementTitleOf, + makeAnnouncementService, + type AnnouncementService, +} from "../../../src/services/host/announcement-service.js" +import type { AnnouncementIdentityRepository } from "../../../src/services/host/announcement-repository.drizzle.js" +import type { EventBroadcastContext } from "../../../src/services/host/broadcast-types.js" + +const EVENT = "00000000-0000-0000-0000-0000000000ee" +const HOST = "00000000-0000-0000-0000-0000000000aa" +const SLOT = "00000000-0000-0000-0000-0000000000s1".replace("s1", "0f1") + +const CONFIG: BroadcastConfig = { + killSwitch: false, + perEventPerDay: 50, + recipientsPerDay: 2000, + cooldownSec: 0, + minAccountAgeHours: 24, + maxRecipients: 5000, + chunkSize: 200, + emailConcurrency: 2, + emailRatePerSec: 1000, + linkAllowedHosts: [], + mailFromEvents: "events@civfix.org", + unsubscribeSigningKey: "unsubscribe-signing-key-for-tests-0123456789", + webBaseUrl: "https://civfix.org", + apiBaseUrl: "https://api.civfix.org", + eventUpdatePerEventPerHour: 3, +} + +const EVENT_CONTEXT: EventBroadcastContext = { + cleanupId: EVENT, + title: "Beach Cleanup", + pageSlug: "beach-cleanup", + scheduledAt: new Date("2026-02-01T17:00:00Z"), + endsAt: null, + timezone: "America/Los_Angeles", + address: "Ocean Ave", + status: "upcoming", + organizerUserId: HOST, + organizationSuspended: false, + replyTo: null, + replyToVerified: false, +} + +const AUTHOR: PersonDTO = { + id: HOST, + name: "Hana Host", + handle: "hana", + bio: null, + avatar: ["#000000", "#ffffff"], + followers: 0, + following: 0, + isFollowing: false, +} + +const ORG: OrganizationRefDTO = { + id: "00000000-0000-0000-0000-0000000000cc", + slug: "ballona", + name: "Ballona Creek Trust", + logoUrl: null, + verified: true, + verifiedKind: null, +} + +function identities(org: OrganizationRefDTO | null = ORG): AnnouncementIdentityRepository { + return { + authorsFor: (ids) => + Promise.resolve(new Map(ids.filter((id) => id === HOST).map(() => [HOST, AUTHOR]))), + organizationFor: () => Promise.resolve(org), + } +} + +interface Harness { + repo: InMemoryBroadcastRepository + plans: string[] + service: AnnouncementService + clock: { at: Date } +} + +function harness(over: { org?: OrganizationRefDTO | null } = {}): Harness { + const repo = new InMemoryBroadcastRepository() + repo.seedEvent(EVENT_CONTEXT) + repo.seedHost(HOST, { accountCreatedAt: new Date("2020-01-01T00:00:00Z") }) + repo.seedMembers(EVENT, [{ userId: HOST }]) + repo.seedGuests(EVENT, []) + const clock = { at: new Date("2026-01-20T12:00:00.000Z") } + const plans: string[] = [] + const broadcasts = makeBroadcastService({ + repo, + counters: new InMemoryCounterStore(() => clock.at.getTime()), + config: CONFIG, + mailer: new FakeMailer(), + enqueuePlan: (broadcastId) => { + plans.push(broadcastId) + return Promise.resolve() + }, + now: () => clock.at, + }) + const service = makeAnnouncementService({ + repo, + identities: identities(over.org === undefined ? ORG : over.org), + broadcasts, + config: CONFIG, + now: () => clock.at, + }) + return { repo, plans, service, clock } +} + +let h: Harness + +beforeEach(() => { + h = harness() +}) + +describe("announcementTitleOf", () => { + it("keeps a host-written title", () => { + expect(announcementTitleOf(" Pizza is here ", "Beach Cleanup")).toBe("Pizza is here") + }) + + it("derives one from the event when the host left it blank", () => { + expect(announcementTitleOf(null, "Beach Cleanup")).toBe("Announcement · Beach Cleanup") + expect(announcementTitleOf(" ", "Beach Cleanup")).toBe("Announcement · Beach Cleanup") + }) +}) + +describe("createEventAnnouncement", () => { + it("writes an announcement broadcast, sends it immediately, and enqueues the existing plan job", async () => { + const dto = await h.service.create(EVENT, HOST, { + id: EVENT, + title: "Pizza is here", + bodyMd: "Meet at the pavilion.", + audience: { kind: "all_registered" }, + }) + + const record = await h.repo.findForEvent(EVENT, dto.id) + expect(record?.kind).toBe(ANNOUNCEMENT_BROADCAST_KIND) + expect(record?.status).toBe("sending") + expect(record?.channels).toEqual(["inapp", "push", "email"]) + expect(h.plans).toEqual([dto.id]) + expect(dto.title).toBe("Pizza is here") + expect(dto.bodyMd).toBe("Meet at the pavilion.") + }) + + it("maps every audience option straight onto the existing broadcast segment union", async () => { + const audiences = [ + { kind: "all_registered" as const }, + { kind: "checked_in" as const }, + { kind: "not_checked_in" as const }, + { kind: "waitlist" as const }, + { kind: "slots" as const, ids: [SLOT] }, + ] + for (const audience of audiences) { + const dto = await h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: "Body", + audience, + }) + const record = await h.repo.findForEvent(EVENT, dto.id) + expect(record?.segment).toEqual(audience) + expect(dto.audience).toEqual(audience) + } + }) + + it("derives the subject from the event title when none was given", async () => { + const dto = await h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: "Body", + audience: { kind: "all_registered" }, + }) + expect(dto.title).toBe("Announcement · Beach Cleanup") + }) + + it("attaches the announcement deep link as the email CTA", async () => { + const dto = await h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: "Body", + audience: { kind: "all_registered" }, + }) + const record = await h.repo.findForEvent(EVENT, dto.id) + expect(record?.ctaUrl).toBe(`https://civfix.org/cleanups/${EVENT}/announcements/${dto.id}`) + expect(record?.ctaLabel).toBe("View announcement") + }) + + it(`rate-limits at ${MAX_EVENT_ANNOUNCEMENTS_PER_DAY} per event per rolling 24h`, async () => { + for (let i = 0; i < MAX_EVENT_ANNOUNCEMENTS_PER_DAY; i += 1) { + await h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: `Body ${i}`, + audience: { kind: "all_registered" }, + }) + } + + await expect( + h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: "One too many", + audience: { kind: "all_registered" }, + }), + ).rejects.toMatchObject({ code: "RATE_LIMITED" }) + }) + + it("lets the host announce again once the 24h window has rolled past", async () => { + for (let i = 0; i < MAX_EVENT_ANNOUNCEMENTS_PER_DAY; i += 1) { + const dto = await h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: `Body ${i}`, + audience: { kind: "all_registered" }, + }) + h.repo.forceCreatedAt(dto.id, new Date("2026-01-19T00:00:00.000Z")) + } + + await expect( + h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: "A new day", + audience: { kind: "all_registered" }, + }), + ).resolves.toMatchObject({ bodyMd: "A new day" }) + }) + + it("leaves no orphan draft behind when the send is refused", async () => { + const suspended = harness() + suspended.repo.seedHost(HOST, { + accountCreatedAt: new Date("2020-01-01T00:00:00Z"), + suspended: true, + }) + + await expect( + suspended.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: "Body", + audience: { kind: "all_registered" }, + }), + ).rejects.toBeTruthy() + + const { items } = await suspended.service.list(EVENT, { id: EVENT }, { host: true }) + expect(items).toHaveLength(0) + expect(await suspended.repo.countAnnouncementsSince(EVENT, new Date(0))).toBe(0) + }) +}) + +describe("announcement projections", () => { + async function seeded(): Promise { + const dto = await h.service.create(EVENT, HOST, { + id: EVENT, + title: "Pizza is here", + bodyMd: "Meet at the pavilion.", + audience: { kind: "checked_in" }, + }) + return dto.id + } + + it("gives a host the audience and the delivery counts", async () => { + const id = await seeded() + const dto = await h.service.get(EVENT, id, { host: true }) + expect(dto.audience).toEqual({ kind: "checked_in" }) + expect(dto).toMatchObject({ recipientCount: 0, sentCount: 0, failedCount: 0 }) + }) + + it("omits audience and counts entirely from the public projection", async () => { + const id = await seeded() + const dto = await h.service.get(EVENT, id, { host: false }) + expect(dto).not.toHaveProperty("audience") + expect(dto).not.toHaveProperty("recipientCount") + expect(dto).not.toHaveProperty("sentCount") + expect(dto).not.toHaveProperty("failedCount") + expect(dto.bodyMd).toBe("Meet at the pavilion.") + }) + + it("hydrates the author and the hosting organization on both projections", async () => { + const id = await seeded() + for (const host of [true, false]) { + const dto = await h.service.get(EVENT, id, { host }) + expect(dto.author?.handle).toBe("hana") + expect(dto.authorOrg?.slug).toBe("ballona") + } + }) + + it("returns a null org for an event nobody hosts as an organization", async () => { + const solo = harness({ org: null }) + const dto = await solo.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: "Body", + audience: { kind: "all_registered" }, + }) + expect(dto.authorOrg).toBeNull() + }) +}) + +describe("listEventAnnouncements", () => { + it("returns announcements newest-first and never other broadcast kinds", async () => { + await h.repo.create({ + cleanupId: EVENT, + createdBy: HOST, + kind: "host_broadcast", + subject: "Not an announcement", + bodyMd: "Body", + segment: { kind: "all_registered" }, + channels: ["email"], + status: "sent", + }) + const first = await h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: "First", + audience: { kind: "all_registered" }, + }) + h.repo.forceCreatedAt(first.id, new Date("2026-01-20T12:00:00.000Z")) + const second = await h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: "Second", + audience: { kind: "all_registered" }, + }) + h.repo.forceCreatedAt(second.id, new Date("2026-01-20T13:00:00.000Z")) + + const { items, nextCursor } = await h.service.list(EVENT, { id: EVENT }, { host: true }) + expect(items.map((item) => item.id)).toEqual([second.id, first.id]) + expect(nextCursor).toBeNull() + }) + + it("pages with a cursor and stops without one on the last page", async () => { + const ids: string[] = [] + for (let i = 0; i < 3; i += 1) { + const dto = await h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: `Body ${i}`, + audience: { kind: "all_registered" }, + }) + h.repo.forceCreatedAt(dto.id, new Date(Date.UTC(2026, 0, 20, 12 + i))) + ids.push(dto.id) + } + + const page1 = await h.service.list(EVENT, { id: EVENT, limit: 2 }, { host: true }) + expect(page1.items).toHaveLength(2) + expect(page1.nextCursor).not.toBeNull() + + const page2 = await h.service.list( + EVENT, + { id: EVENT, limit: 2, cursor: page1.nextCursor as string }, + { host: true }, + ) + expect(page2.items.map((item) => item.id)).toEqual([ids[0]]) + expect(page2.nextCursor).toBeNull() + }) + + it("404s an announcement id that belongs to a different event", async () => { + await expect( + h.service.get(EVENT, "00000000-0000-0000-0000-00000000dead", { host: true }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }) + }) +}) From a16ba2c30d26357760e8aafa098ea60f388f2009 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 16:09:14 -0700 Subject: [PATCH 22/54] consolidated event analytics endpoint --- .../api/src/routes/host/analytics.routes.ts | 28 ++ .../host/analytics-repository.drizzle.ts | 5 +- .../api/src/services/host/comms-wiring.ts | 41 +- .../event-analytics-repository.drizzle.ts | 206 +++++++++ .../services/host/event-analytics-service.ts | 400 ++++++++++++++++++ services/api/test/unit/host-analytics.test.ts | 1 + services/api/test/unit/host-insights.test.ts | 1 + .../unit/host/event-analytics-service.test.ts | 368 ++++++++++++++++ .../host/insights-invalidation-wiring.test.ts | 1 + 9 files changed, 1049 insertions(+), 2 deletions(-) create mode 100644 services/api/src/services/host/event-analytics-repository.drizzle.ts create mode 100644 services/api/src/services/host/event-analytics-service.ts create mode 100644 services/api/test/unit/host/event-analytics-service.test.ts diff --git a/services/api/src/routes/host/analytics.routes.ts b/services/api/src/routes/host/analytics.routes.ts index 511f6861..4cd27377 100644 --- a/services/api/src/routes/host/analytics.routes.ts +++ b/services/api/src/routes/host/analytics.routes.ts @@ -1,6 +1,7 @@ import { AppError, EventAnalyticsRequestSchema, + GetEventAnalyticsRequestSchema, GetEventInsightsRequestSchema, HostedEventsAnalyticsRequestSchema, type AnalyticsRange, @@ -21,12 +22,14 @@ import { import { makeCommsRuntime } from "../../services/host/comms-wiring.js" import type { CommsRuntime } from "../../services/host/comms-wiring.js" import type { AnalyticsService } from "../../services/host/analytics-service.js" +import type { EventAnalyticsService } from "../../services/host/event-analytics-service.js" import type { InsightsService } from "../../services/host/insights-service.js" export const ANALYTICS_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute" }) export interface HostAnalyticsOverrides { analytics: AnalyticsService + eventAnalytics: EventAnalyticsService insights: InsightsService } @@ -54,6 +57,12 @@ export async function registerHostAnalyticsRoutes( return (cached ??= makeCommsRuntime(container, app.log)).analytics } + function eventAnalytics(): EventAnalyticsService { + const override = app.hostAnalyticsOverrides + if (override) return override.eventAnalytics + return (cached ??= makeCommsRuntime(container, app.log)).eventAnalytics + } + function insights(): InsightsService { const override = app.hostAnalyticsOverrides if (override) return override.insights @@ -75,6 +84,25 @@ export async function registerHostAnalyticsRoutes( return { cleanupId: query.id, range: query.range ?? "30d", viewerScope } } + route(app, "getEventAnalytics", { config: { rateLimit: ANALYTICS_RATE_LIMIT } }, async (request, reply) => { + const userId = requireAuth(request) + const query = parse(GetEventAnalyticsRequestSchema, mergeQuery(request)) + const resolution = await requireCapability( + container.getDb().sql, + query.id, + userId, + "view_analytics", + ) + const viewerScope = `${resolution.standing.eventRole ?? "none"}:${resolution.standing.orgRole ?? "none"}` + reply.status(200).send( + await eventAnalytics().analytics(query.id, query.scope ?? "full", { + userId, + organizationId: resolution.organizationId, + viewerScope, + }), + ) + }) + route(app, "eventAnalyticsOverview", { config: { rateLimit: ANALYTICS_RATE_LIMIT } }, async (request, reply) => { const scope = await eventScope(request) reply.status(200).send(await analytics().overview(scope.cleanupId, scope.range, scope.viewerScope)) diff --git a/services/api/src/services/host/analytics-repository.drizzle.ts b/services/api/src/services/host/analytics-repository.drizzle.ts index 5acf8a75..7531e15f 100644 --- a/services/api/src/services/host/analytics-repository.drizzle.ts +++ b/services/api/src/services/host/analytics-repository.drizzle.ts @@ -19,6 +19,7 @@ export interface EventKpiRow { export interface EventClockRecord { status: CleanupStatus + createdAt: Date scheduledAt: Date endsAt: Date | null completedAt: Date | null @@ -28,6 +29,7 @@ export interface EventClockRecord { interface EventClockRowSelect { status: CleanupStatus + created_at: Date scheduled_at: Date ends_at: Date | null completed_at: Date | null @@ -36,7 +38,7 @@ interface EventClockRowSelect { } function eventClockColumns(tag: Sql) { - return tag`${cleanupStatusExpr(tag)} AS status, c.scheduled_at, c.ends_at, c.completed_at, c.registration_closes_at, c.timezone` + return tag`${cleanupStatusExpr(tag)} AS status, c.created_at, c.scheduled_at, c.ends_at, c.completed_at, c.registration_closes_at, c.timezone` } export interface SeatTrendPoint { @@ -356,6 +358,7 @@ export function makeDrizzleAnalyticsRepository(sql: Sql): AnalyticsRepository { if (row === undefined) return null return { status: row.status, + createdAt: row.created_at, scheduledAt: row.scheduled_at, endsAt: row.ends_at, completedAt: row.completed_at, diff --git a/services/api/src/services/host/comms-wiring.ts b/services/api/src/services/host/comms-wiring.ts index b246076e..6da06254 100644 --- a/services/api/src/services/host/comms-wiring.ts +++ b/services/api/src/services/host/comms-wiring.ts @@ -6,6 +6,14 @@ import { writeAudit } from "../admin/audit.js" import { makeRouteNotificationService } from "../route-notifier.js" import { makeDrizzleAnalyticsRepository } from "./analytics-repository.drizzle.js" import { makeAnalyticsService, type AnalyticsService } from "./analytics-service.js" +import { makeDrizzleEventAnalyticsRepository } from "./event-analytics-repository.drizzle.js" +import { + makeEventAnalyticsService, + type EventAnalyticsService, +} from "./event-analytics-service.js" +import { makeDrizzleAnnouncementIdentityRepository } from "./announcement-repository.drizzle.js" +import { makeAnnouncementService, type AnnouncementService } from "./announcement-service.js" +import { MEDIA_GET_URL_TTL_SEC } from "../media-intake-service.js" import { makeInsightsService, type InsightsService } from "./insights-service.js" import { makeDrizzleHostRegistrationRepository } from "./registration-repository.drizzle.js" import { makeHostAnalyticsCache } from "./host-analytics-cache.js" @@ -39,10 +47,12 @@ export interface CommsRuntime { repo: BroadcastRepository metricsRepo: MetricsRepository broadcasts: BroadcastService + announcements: AnnouncementService pipeline: BroadcastPipeline lanes: BroadcastLanes metrics: MetricsService analytics: AnalyticsService + eventAnalytics: EventAnalyticsService insights: InsightsService exports: HostExportService } @@ -122,6 +132,16 @@ export function makeCommsRuntime(container: Container, logger?: CommsLogger): Co ...(logger !== undefined ? { logger } : {}), }) + const announcements = makeAnnouncementService({ + repo, + identities: makeDrizzleAnnouncementIdentityRepository(sql, (key) => + container.storage.presignGet(key, MEDIA_GET_URL_TTL_SEC), + ), + broadcasts, + config, + ...(logger !== undefined ? { logger } : {}), + }) + const pipeline = makeBroadcastPipeline({ repo, service: broadcasts, @@ -186,6 +206,13 @@ export function makeCommsRuntime(container: Container, logger?: CommsLogger): Co cache: analyticsCache, }) + const eventAnalytics = makeEventAnalyticsService({ + analytics: analyticsRepo, + events: makeDrizzleEventAnalyticsRepository(sql), + metrics: metricsRepo, + cache: analyticsCache, + }) + const insights = makeInsightsService({ analytics: analyticsRepo, registrations: makeDrizzleHostRegistrationRepository(sql), @@ -207,5 +234,17 @@ export function makeCommsRuntime(container: Container, logger?: CommsLogger): Co ...(logger !== undefined ? { logger } : {}), }) - return { repo, metricsRepo, broadcasts, pipeline, lanes, metrics, analytics, insights, exports } + return { + repo, + metricsRepo, + broadcasts, + announcements, + pipeline, + lanes, + metrics, + analytics, + eventAnalytics, + insights, + exports, + } } diff --git a/services/api/src/services/host/event-analytics-repository.drizzle.ts b/services/api/src/services/host/event-analytics-repository.drizzle.ts new file mode 100644 index 00000000..cf8bee59 --- /dev/null +++ b/services/api/src/services/host/event-analytics-repository.drizzle.ts @@ -0,0 +1,206 @@ +import type { KeyCount } from "@civfix/shared/host" +import type { Sql } from "../../db/client.js" + +export const EVENT_ANALYTICS_SLOT_ROW_LIMIT = 50 + +export interface EventAnalyticsFacts { + walkUps: number + reportsLinked: number + reportsResolved: number + postsCreated: number +} + +export const ZERO_EVENT_ANALYTICS_FACTS: EventAnalyticsFacts = Object.freeze({ + walkUps: 0, + reportsLinked: 0, + reportsResolved: 0, + postsCreated: 0, +}) + +export interface EventComparisonMedians { + sampleSize: number + signups: number | null + checkInRate: number | null + hoursPerVolunteer: number | null + fillRate: number | null +} + +export interface EventAnalyticsRepository { + previousCompletedEventIds(args: { + userId: string + organizationId: string | null + excludeCleanupId: string + limit: number + }): Promise + facts(cleanupId: string): Promise + registrationsBySlot(cleanupId: string): Promise + reportStatuses(cleanupId: string): Promise + hoursBuckets(cleanupId: string): Promise + comparisonMedians(cleanupIds: readonly string[]): Promise +} + +export function makeDrizzleEventAnalyticsRepository(sql: Sql): EventAnalyticsRepository { + return { + async previousCompletedEventIds(args) { + const orgFilter = + args.organizationId !== null + ? sql`AND c.organization_id = ${args.organizationId}` + : sql`` + const rows = await sql<{ id: string }[]>` + SELECT c.id + FROM cleanups c + WHERE c.completed_at IS NOT NULL + AND c.id <> ${args.excludeCleanupId} + AND (c.organizer_user_id = ${args.userId} + OR EXISTS ( + SELECT 1 FROM cleanup_members m + WHERE m.cleanup_id = c.id AND m.user_id = ${args.userId} + AND m.role IN ('organizer','cohost','coordinator')) + OR EXISTS ( + SELECT 1 FROM organization_members om + WHERE om.organization_id = c.organization_id AND om.user_id = ${args.userId} + AND om.role IN ('owner','admin'))) + ${orgFilter} + ORDER BY c.completed_at DESC + LIMIT ${args.limit}` + return rows.map((row) => row.id) + }, + + async facts(cleanupId) { + const rows = await sql< + { + walk_ups: number + reports_linked: number + reports_resolved: number + posts_created: number + }[] + >` + SELECT + (SELECT count(*) FROM cleanup_registrations r + WHERE r.cleanup_id = ${cleanupId} + AND r.status = 'registered' + AND r.source = 'walkup')::int AS walk_ups, + (SELECT count(*) FROM cleanup_reports cr + WHERE cr.cleanup_id = ${cleanupId})::int AS reports_linked, + (SELECT count(*) FROM cleanup_reports cr + JOIN reports rep ON rep.id = cr.report_id + WHERE cr.cleanup_id = ${cleanupId} + AND rep.status = 'resolved')::int AS reports_resolved, + (SELECT count(*) FROM posts p + WHERE p.event_id = ${cleanupId} + AND p.deleted_at IS NULL + AND p.reply_to_id IS NULL)::int AS posts_created` + const row = rows[0] + if (row === undefined) return { ...ZERO_EVENT_ANALYTICS_FACTS } + return { + walkUps: row.walk_ups, + reportsLinked: row.reports_linked, + reportsResolved: row.reports_resolved, + postsCreated: row.posts_created, + } + }, + + async registrationsBySlot(cleanupId) { + const rows = await sql<{ key: string; n: number }[]>` + SELECT sl.title AS key, count(*)::int AS n + FROM cleanup_slot_claims sc + JOIN cleanup_slots sl ON sl.id = sc.slot_id + WHERE sc.cleanup_id = ${cleanupId} + GROUP BY sl.title + ORDER BY count(*) DESC, sl.title ASC + LIMIT ${EVENT_ANALYTICS_SLOT_ROW_LIMIT}` + return rows.map((row) => ({ key: row.key, count: row.n })) + }, + + async reportStatuses(cleanupId) { + const rows = await sql<{ key: string; n: number }[]>` + SELECT rep.status AS key, count(*)::int AS n + FROM cleanup_reports cr + JOIN reports rep ON rep.id = cr.report_id + WHERE cr.cleanup_id = ${cleanupId} + GROUP BY rep.status + ORDER BY count(*) DESC, rep.status ASC + LIMIT ${EVENT_ANALYTICS_SLOT_ROW_LIMIT}` + return rows.map((row) => ({ key: row.key, count: row.n })) + }, + + async hoursBuckets(cleanupId) { + const rows = await sql<{ key: string; n: number }[]>` + WITH per_person AS ( + SELECT h.user_id, sum(h.hours)::float8 AS hours + FROM volunteer_hours h + WHERE h.cleanup_id = ${cleanupId} + AND h.source = 'event' + AND h.voided_at IS NULL + GROUP BY h.user_id + ) + SELECT CASE + WHEN hours < 1 THEN '0-1h' + WHEN hours < 2 THEN '1-2h' + WHEN hours < 4 THEN '2-4h' + ELSE '4h+' + END AS key, + count(*)::int AS n + FROM per_person + GROUP BY 1` + return rows.map((row) => ({ key: row.key, count: row.n })) + }, + + async comparisonMedians(cleanupIds) { + if (cleanupIds.length === 0) { + return { + sampleSize: 0, + signups: null, + checkInRate: null, + hoursPerVolunteer: null, + fillRate: null, + } + } + const rows = await sql< + { + sample_size: number + signups: number | null + check_in_rate: number | null + hours_per_volunteer: number | null + fill_rate: number | null + }[] + >` + WITH per_event AS ( + SELECT + c.id, + (SELECT count(*) FROM cleanup_registrations r + WHERE r.cleanup_id = c.id AND r.status = 'registered')::float8 AS signups, + (SELECT count(*) FROM cleanup_registration_seats s + WHERE s.cleanup_id = c.id AND s.checked_in_at IS NOT NULL)::float8 AS checked_in, + (SELECT COALESCE(sum(h.hours), 0) FROM volunteer_hours h + WHERE h.cleanup_id = c.id AND h.source = 'event' AND h.voided_at IS NULL)::float8 + AS hours, + (SELECT count(DISTINCT h.user_id) FROM volunteer_hours h + WHERE h.cleanup_id = c.id AND h.source = 'event' AND h.voided_at IS NULL)::float8 + AS volunteers, + c.capacity::float8 AS capacity + FROM cleanups c + WHERE c.id = ANY(${[...cleanupIds]}::uuid[]) + ) + SELECT + count(*)::int AS sample_size, + percentile_cont(0.5) WITHIN GROUP (ORDER BY signups)::float8 AS signups, + percentile_cont(0.5) WITHIN GROUP ( + ORDER BY CASE WHEN signups > 0 THEN checked_in / signups END)::float8 AS check_in_rate, + percentile_cont(0.5) WITHIN GROUP ( + ORDER BY CASE WHEN volunteers > 0 THEN hours / volunteers END)::float8 + AS hours_per_volunteer, + percentile_cont(0.5) WITHIN GROUP ( + ORDER BY CASE WHEN capacity > 0 THEN signups / capacity END)::float8 AS fill_rate + FROM per_event` + const row = rows[0] + return { + sampleSize: row?.sample_size ?? 0, + signups: row?.signups ?? null, + checkInRate: row?.check_in_rate ?? null, + hoursPerVolunteer: row?.hours_per_volunteer ?? null, + fillRate: row?.fill_rate ?? null, + } + }, + } +} diff --git a/services/api/src/services/host/event-analytics-service.ts b/services/api/src/services/host/event-analytics-service.ts new file mode 100644 index 00000000..d58b1501 --- /dev/null +++ b/services/api/src/services/host/event-analytics-service.ts @@ -0,0 +1,400 @@ +import { + ANALYTICS_SUPPRESSION_K, + AppError, + EVENT_ANALYTICS_CARD_SERIES_POINTS, + EVENT_ANALYTICS_CARD_SLOT_ROWS, + EVENT_ANALYTICS_COMPARISON_MIN_EVENTS, + EVENT_ANALYTICS_COMPARISON_WINDOW, + MAX_EVENT_ANALYTICS_ARRIVAL_BUCKETS, + MAX_EVENT_ANALYTICS_PANEL_ROWS, + MAX_EVENT_ANALYTICS_SERIES_POINTS, + type BreakdownRow, + type EventAnalyticsPhase, + type EventAnalyticsScope, + type FunnelStep, + type GetEventAnalyticsResponse, + type Panel, + type SeriesPoint, + type SuppressedRate, +} from "@civfix/shared" +import { + breakdown, + dailySeries, + eventPhase, + funnel, + seriesClosure, + suppressCount, + suppressRate, + type DayCount, + type DayRange, + type DerivedBreakdownRow, + type DerivedPanel, + type KeyCount, + type SeriesClosure, +} from "@civfix/shared/host" +import type { + AnalyticsRepository, + EventClockRecord, +} from "./analytics-repository.drizzle.js" +import type { EventAnalyticsRepository } from "./event-analytics-repository.drizzle.js" +import type { MetricRow, MetricsRepository } from "./metrics-repository.drizzle.js" +import { hostAnalyticsCacheKey, type HostAnalyticsCache } from "./host-analytics-cache.js" +import { eventDayKey } from "./event-day.js" +import { DEFAULT_EVENT_TIME_ZONE } from "./event-fields.js" +import { ARRIVAL_SAMPLE_LIMIT } from "./analytics-service.js" + +export const EVENT_ANALYTICS_LIFECYCLE_DAYS = MAX_EVENT_ANALYTICS_SERIES_POINTS +export const EVENT_ANALYTICS_ARCHIVE_DAYS = 30 +export const EVENT_ANALYTICS_ARRIVAL_BUCKET_MIN = 15 +export const EVENT_ANALYTICS_DELTA_DAYS = 7 +export const DAY_MS = 86_400_000 + +export interface EventAnalyticsServiceDeps { + analytics: AnalyticsRepository + events: EventAnalyticsRepository + metrics: MetricsRepository + cache: HostAnalyticsCache + now?: () => Date +} + +export interface EventAnalyticsViewer { + userId: string + organizationId: string | null + viewerScope: string +} + +export interface EventAnalyticsService { + analytics( + cleanupId: string, + scope: EventAnalyticsScope, + viewer: EventAnalyticsViewer, + ): Promise +} + +function shiftDayKey(day: string, deltaDays: number): string { + const [year, month, date] = day.split("-").map(Number) + const shifted = new Date( + Date.UTC(year ?? 1970, (month ?? 1) - 1, date ?? 1) + deltaDays * DAY_MS, + ) + return shifted.toISOString().slice(0, 10) +} + +function emptyRate(): SuppressedRate { + return { value: null, numerator: null, denominator: null, suppressed: true } +} + +function toRate(numerator: number, denominator: number): SuppressedRate { + const ratio = suppressRate(numerator, denominator) + return { + value: ratio.value, + numerator: ratio.suppressed ? null : numerator, + denominator: ratio.suppressed ? null : denominator, + suppressed: ratio.suppressed, + } +} + +function toSeries( + points: readonly { day: string; value: number | null; suppressed: boolean }[], +): SeriesPoint[] { + return points.map((point) => ({ + day: point.day, + value: point.value, + suppressed: point.suppressed, + })) +} + +function toPanel(panel: DerivedPanel): Panel { + return { + panelSuppressed: panel.panelSuppressed, + rows: panel.rows.slice(0, MAX_EVENT_ANALYTICS_PANEL_ROWS).map( + (row): BreakdownRow => ({ + key: row.key, + label: row.key, + value: row.value, + suppressed: row.suppressed, + }), + ), + } +} + +function toFunnelSteps( + panel: DerivedPanel<{ key: string; value: number | null; suppressed: boolean }>, +): FunnelStep[] { + return panel.rows.map((row) => ({ + step: row.key, + label: row.key, + value: row.value, + suppressed: row.suppressed, + })) +} + +function sumMetric(rows: readonly MetricRow[], metric: string): number { + return rows.filter((row) => row.metric === metric).reduce((acc, row) => acc + row.value, 0) +} + +function seriesOf(rows: readonly MetricRow[], metric: string): DayCount[] { + const byDay = new Map() + for (const row of rows) { + if (row.metric !== metric) continue + byDay.set(row.day, (byDay.get(row.day) ?? 0) + row.value) + } + return [...byDay].map(([day, count]) => ({ day, count })) +} + +function tailSum(points: readonly { value: number | null }[], days: number): number | null { + const tail = points.slice(-days) + let total = 0 + for (const point of tail) { + if (point.value === null) return null + total += point.value + } + return total +} + +function closureAllowsTotal(closure: SeriesClosure): boolean { + return closure.panelSuppressed || closure.totalPublishable +} + +export function analyticsPhaseOf(clock: EventClockRecord, at: Date): EventAnalyticsPhase { + const phase = eventPhase( + { + status: clock.status, + scheduledAt: clock.scheduledAt.toISOString(), + endsAt: clock.endsAt?.toISOString() ?? null, + completedAt: clock.completedAt?.toISOString() ?? null, + }, + at.getTime(), + ) + if (phase === "live") return "day_of" + if (phase !== "ended") return "upcoming" + const endedAt = clock.completedAt ?? clock.endsAt ?? clock.scheduledAt + return at.getTime() - endedAt.getTime() > EVENT_ANALYTICS_ARCHIVE_DAYS * DAY_MS + ? "archived" + : "completed" +} + +export function arrivalBuckets(offsets: readonly number[]): SeriesPoint[] { + const byBucket = new Map() + for (const offset of offsets) { + const bucket = Math.floor(offset / EVENT_ANALYTICS_ARRIVAL_BUCKET_MIN) + byBucket.set(bucket, (byBucket.get(bucket) ?? 0) + 1) + } + return [...byBucket] + .sort((a, b) => a[0] - b[0]) + .slice(0, MAX_EVENT_ANALYTICS_ARRIVAL_BUCKETS) + .map(([bucket, count]) => ({ + day: String(bucket * EVENT_ANALYTICS_ARRIVAL_BUCKET_MIN), + value: count, + suppressed: false, + })) +} + +export function makeEventAnalyticsService( + deps: EventAnalyticsServiceDeps, +): EventAnalyticsService { + const now = deps.now ?? (() => new Date()) + + function lifecycleWindow(clock: EventClockRecord, timezone: string): DayRange { + const to = eventDayKey(now(), timezone) + const from = eventDayKey(clock.createdAt, timezone) + const earliest = shiftDayKey(to, 1 - EVENT_ANALYTICS_LIFECYCLE_DAYS) + return { from: from < earliest ? earliest : from, to } + } + + async function comparisonOf( + cleanupId: string, + viewer: EventAnalyticsViewer, + ): Promise { + const ids = await deps.events.previousCompletedEventIds({ + userId: viewer.userId, + organizationId: viewer.organizationId, + excludeCleanupId: cleanupId, + limit: EVENT_ANALYTICS_COMPARISON_WINDOW, + }) + if (ids.length < EVENT_ANALYTICS_COMPARISON_MIN_EVENTS) return null + const medians = await deps.events.comparisonMedians(ids) + return { + sampleSize: medians.sampleSize, + medians: { + signups: medians.signups, + checkInRate: medians.checkInRate, + hoursPerVolunteer: medians.hoursPerVolunteer, + fillRate: medians.fillRate, + }, + } + } + + async function compute( + cleanupId: string, + scope: EventAnalyticsScope, + viewer: EventAnalyticsViewer, + clock: EventClockRecord, + phase: EventAnalyticsPhase, + at: Date, + ): Promise { + const timezone = clock.timezone ?? DEFAULT_EVENT_TIME_ZONE + const window = lifecycleWindow(clock, timezone) + const full = scope === "full" + + const [ + kpis, + facts, + metricRows, + registrationDays, + cancellationDays, + offsets, + waitlist, + hours, + bySource, + registrationsBySlot, + checkinsBySlot, + hoursBuckets, + reportStatuses, + comparison, + ] = await Promise.all([ + deps.analytics.eventKpis(cleanupId), + deps.events.facts(cleanupId), + deps.metrics.read(cleanupId, ["page_views", "donation_clicks"], window.from, window.to), + deps.analytics.registrationsByDay(cleanupId, timezone, window.from, window.to), + deps.analytics.cancellationsByDay(cleanupId, timezone, window.from, window.to), + deps.analytics.arrivalOffsets(cleanupId, ARRIVAL_SAMPLE_LIMIT), + deps.analytics.waitlistConversion(cleanupId), + deps.analytics.eventHoursTotals(cleanupId), + deps.analytics.registrationsBySource(cleanupId), + deps.events.registrationsBySlot(cleanupId), + full ? deps.analytics.checkinsBySlot(cleanupId) : Promise.resolve([]), + full ? deps.events.hoursBuckets(cleanupId) : Promise.resolve([]), + full ? deps.events.reportStatuses(cleanupId) : Promise.resolve([]), + full ? comparisonOf(cleanupId, viewer) : Promise.resolve(null), + ]) + + const closure = seriesClosure(registrationDays, window, { suppressPoints: true }) + const registeredPublishable = closureAllowsTotal(closure) + const cancellations = dailySeries(cancellationDays, window, { suppressPoints: true }) + const viewsSeries = dailySeries(seriesOf(metricRows, "page_views"), window) + const pageViews = sumMetric(metricRows, "page_views") + const donationClicks = sumMetric(metricRows, "donation_clicks") + + const steps: KeyCount[] = [ + { key: "page_views", count: pageViews }, + { key: "signups", count: kpis.registered }, + { key: "checked_in", count: kpis.checkedIn }, + { key: "logged_hours", count: hours.attendeesCredited }, + ] + + const cumulative = toSeries(closure.cumulative) + const daily = toSeries(closure.daily) + const viewsDaily = toSeries(viewsSeries.points) + const arrivals = arrivalBuckets(offsets) + + const response: GetEventAnalyticsResponse = { + generatedAt: at.toISOString(), + k: ANALYTICS_SUPPRESSION_K, + scope, + phase, + lifecycle: { + createdAt: clock.createdAt.toISOString(), + startAt: clock.scheduledAt.toISOString(), + endAt: clock.endsAt?.toISOString() ?? null, + completedAt: clock.completedAt?.toISOString() ?? null, + }, + kpis: { + signups: registeredPublishable ? suppressCount(kpis.registered).value : null, + capacity: kpis.capacity, + waitlisted: suppressCount(kpis.waitlisted).value, + cancelled: suppressCount(kpis.cancelled).value, + checkedIn: suppressCount(kpis.checkedIn).value, + noShow: suppressCount(kpis.noShow).value, + walkUps: suppressCount(facts.walkUps).value, + pageViews: suppressCount(pageViews).value, + uniqueViewers: null, + shares: null, + donationClicks: suppressCount(donationClicks).value, + hoursTotal: hours.credited, + hoursVolunteers: hours.attendeesCredited, + reportsLinked: facts.reportsLinked, + reportsResolved: facts.reportsResolved, + postsCreated: facts.postsCreated, + }, + rates: { + checkIn: registeredPublishable ? toRate(kpis.checkedIn, kpis.registered) : emptyRate(), + noShow: registeredPublishable ? toRate(kpis.noShow, kpis.registered) : emptyRate(), + fill: + kpis.capacity === null || kpis.capacity === 0 || !registeredPublishable + ? emptyRate() + : toRate(kpis.registered, kpis.capacity), + viewToSignup: + pageViews === 0 || !registeredPublishable + ? emptyRate() + : toRate(kpis.registered, pageViews), + waitlistConversion: toRate(waitlist.promoted, waitlist.joined), + }, + deltas: { + signups7d: tailSum(daily, EVENT_ANALYTICS_DELTA_DAYS), + views7d: tailSum(viewsDaily, EVENT_ANALYTICS_DELTA_DAYS), + }, + signups: { + cumulative: full ? cumulative : cumulative.slice(-EVENT_ANALYTICS_CARD_SERIES_POINTS), + daily: full ? daily : [], + cancellations: full ? toSeries(cancellations.points) : [], + bySlot: cardSlotPanel(registrationsBySlot, registeredPublishable, full), + ...(full + ? { + bySource: toPanel( + breakdown( + bySource.map((row) => ({ key: row.source, count: row.seats })), + { totalPublishable: registeredPublishable }, + ), + ), + } + : {}), + }, + reach: { + viewsDaily: full ? viewsDaily : viewsDaily.slice(-EVENT_ANALYTICS_CARD_SERIES_POINTS), + funnel: full ? toFunnelSteps(funnel(steps)) : [], + }, + eventDay: { + arrivals: full ? arrivals : arrivals.slice(-EVENT_ANALYTICS_CARD_SERIES_POINTS), + ...(full ? { bySlot: toPanel(breakdown(checkinsBySlot)) } : {}), + }, + impact: full + ? { + hoursBuckets: toPanel(breakdown(hoursBuckets)), + reportStatuses: toPanel(breakdown(reportStatuses)), + } + : {}, + comparison, + } + return response + } + + function cardSlotPanel( + rows: readonly KeyCount[], + registeredPublishable: boolean, + full: boolean, + ): Panel { + const panel = toPanel(breakdown([...rows], { totalPublishable: registeredPublishable })) + if (full) return panel + return { ...panel, rows: panel.rows.slice(0, EVENT_ANALYTICS_CARD_SLOT_ROWS) } + } + + return { + async analytics(cleanupId, scope, viewer) { + const clock = await deps.analytics.eventClock(cleanupId) + if (clock === null) throw AppError.notFound("Cleanup not found") + const at = now() + const phase = analyticsPhaseOf(clock, at) + const generation = await deps.cache.generationOf(cleanupId) + return deps.cache.getOrSet( + hostAnalyticsCacheKey({ + endpoint: "event-analytics", + scope: cleanupId, + range: `${scope}:${phase}`, + viewerScope: `${viewer.viewerScope}:${viewer.userId}`, + generation, + }), + () => compute(cleanupId, scope, viewer, clock, phase, at), + ) + }, + } +} diff --git a/services/api/test/unit/host-analytics.test.ts b/services/api/test/unit/host-analytics.test.ts index 7c306fae..96ec591b 100644 --- a/services/api/test/unit/host-analytics.test.ts +++ b/services/api/test/unit/host-analytics.test.ts @@ -54,6 +54,7 @@ function analyticsRepo(overrides: Partial = {}): AnalyticsR eventClock: () => Promise.resolve({ status: "active" as const, + createdAt: new Date("2026-02-01T17:00:00Z"), scheduledAt: new Date("2026-02-14T17:00:00Z"), endsAt: null, completedAt: null, diff --git a/services/api/test/unit/host-insights.test.ts b/services/api/test/unit/host-insights.test.ts index 9d51c906..4a0685f0 100644 --- a/services/api/test/unit/host-insights.test.ts +++ b/services/api/test/unit/host-insights.test.ts @@ -52,6 +52,7 @@ function seatDrafts(partySize: number): SeatDraft[] { function clockRecord(overrides: Partial = {}): EventClockRecord { return { status: "upcoming", + createdAt: STARTS_AT, scheduledAt: STARTS_AT, endsAt: null, completedAt: null, diff --git a/services/api/test/unit/host/event-analytics-service.test.ts b/services/api/test/unit/host/event-analytics-service.test.ts new file mode 100644 index 00000000..f9f93bbd --- /dev/null +++ b/services/api/test/unit/host/event-analytics-service.test.ts @@ -0,0 +1,368 @@ +import { describe, expect, it } from "vitest" +import { + ANALYTICS_SUPPRESSION_K, + EVENT_ANALYTICS_CARD_SLOT_ROWS, + EVENT_ANALYTICS_COMPARISON_MIN_EVENTS, + GetEventAnalyticsResponseSchema, +} from "@civfix/shared" +import { InMemoryCacheClient } from "../../../src/auth/cache.js" +import { makeHostAnalyticsCache } from "../../../src/services/host/host-analytics-cache.js" +import { + analyticsPhaseOf, + arrivalBuckets, + makeEventAnalyticsService, +} from "../../../src/services/host/event-analytics-service.js" +import type { + AnalyticsRepository, + EventClockRecord, +} from "../../../src/services/host/analytics-repository.drizzle.js" +import type { EventAnalyticsRepository } from "../../../src/services/host/event-analytics-repository.drizzle.js" +import type { MetricsRepository } from "../../../src/services/host/metrics-repository.drizzle.js" + +const EVENT = "00000000-0000-0000-0000-0000000000ee" +const HOST = "00000000-0000-0000-0000-0000000000aa" +const NOW = new Date("2026-02-20T00:00:00Z") + +const VIEWER = { userId: HOST, organizationId: null, viewerScope: "organizer:none" } + +function clock(over: Partial = {}): EventClockRecord { + return { ...baseClock(), ...over } +} + +function baseClock(): EventClockRecord { + return { + status: "done", + createdAt: new Date("2026-02-01T00:00:00Z"), + scheduledAt: new Date("2026-02-14T17:00:00Z"), + endsAt: new Date("2026-02-14T21:00:00Z"), + completedAt: new Date("2026-02-14T21:30:00Z"), + registrationClosesAt: null, + timezone: "UTC", + } +} + +function analyticsRepo(over: Partial = {}): AnalyticsRepository { + return { + eventKpis: () => + Promise.resolve({ + registered: 20, + checkedIn: 12, + waitlisted: 3, + cancelled: 2, + noShow: 1, + capacity: 40, + }), + registrationsByDay: () => + Promise.resolve([ + { day: "2026-02-10", count: 8 }, + { day: "2026-02-18", count: 12 }, + ]), + cancellationsByDay: () => Promise.resolve([{ day: "2026-02-11", count: 2 }]), + registrationsByTicketType: () => Promise.resolve([]), + registrationsByAudience: () => Promise.resolve([]), + checkinsByTicketType: () => Promise.resolve([]), + checkinsBySlot: () => Promise.resolve([{ key: "Trash pickup", count: 9 }]), + arrivalOffsets: () => Promise.resolve([0, 3, 14, 16, 31, 32, 33]), + waitlistConversion: () => Promise.resolve({ promoted: 2, joined: 3 }), + hostedEventIds: () => Promise.resolve([EVENT]), + portfolioTotals: () => + Promise.resolve({ + events: 0, + registrations: 0, + checkIns: 0, + uniqueAttendees: 0, + repeatAttendees: 0, + }), + portfolioByEvent: () => Promise.resolve([]), + portfolioDayTime: () => Promise.resolve([]), + broadcastsSent: () => Promise.resolve(0), + eventClock: () => Promise.resolve(clock()), + seatTrend: () => Promise.resolve([]), + registrationsBySource: () => + Promise.resolve([ + { source: "self" as const, seats: 16 }, + { source: "walkup" as const, seats: 9 }, + ]), + broadcastsForEvent: () => Promise.resolve([]), + eventHoursTotals: () => + Promise.resolve({ credited: 30, attendeesCredited: 10, attendeesCheckedIn: 12 }), + topVolunteers: () => Promise.resolve([]), + hoursTotals: () => Promise.resolve({ credited: 0, volunteersCredited: 0 }), + returningAttendees: () => Promise.resolve({ seats: 0, ofRegistered: 0 }), + ...over, + } +} + +function eventsRepo(over: Partial = {}): EventAnalyticsRepository { + return { + previousCompletedEventIds: () => Promise.resolve([]), + facts: () => + Promise.resolve({ walkUps: 4, reportsLinked: 3, reportsResolved: 1, postsCreated: 7 }), + registrationsBySlot: () => + Promise.resolve([ + { key: "Trash pickup", count: 10 }, + { key: "Sorting", count: 7 }, + { key: "Greeting", count: 6 }, + { key: "Cleanup", count: 5 }, + ]), + reportStatuses: () => Promise.resolve([{ key: "resolved", count: 1 }]), + hoursBuckets: () => Promise.resolve([{ key: "2-4h", count: 10 }]), + comparisonMedians: () => + Promise.resolve({ + sampleSize: 4, + signups: 18, + checkInRate: 0.55, + hoursPerVolunteer: 2.5, + fillRate: 0.4, + }), + ...over, + } +} + +function metricsRepo(rows: Array<{ day: string; metric: string; bucket: string; value: number }>) { + return { + read: () => Promise.resolve(rows), + readMany: () => Promise.resolve([]), + eventTimezone: () => Promise.resolve("UTC"), + } as unknown as MetricsRepository +} + +const METRIC_ROWS = [ + { day: "2026-02-10", metric: "page_views", bucket: "", value: 100 }, + { day: "2026-02-18", metric: "page_views", bucket: "", value: 60 }, + { day: "2026-02-18", metric: "donation_clicks", bucket: "", value: 9 }, +] + +function service( + over: { + analytics?: Partial + events?: Partial + metrics?: Array<{ day: string; metric: string; bucket: string; value: number }> + } = {}, +) { + return makeEventAnalyticsService({ + analytics: analyticsRepo(over.analytics), + events: eventsRepo(over.events), + metrics: metricsRepo(over.metrics ?? METRIC_ROWS), + cache: makeHostAnalyticsCache({ + cache: new InMemoryCacheClient(() => NOW.getTime()), + ttlSeconds: 60, + }), + now: () => NOW, + }) +} + +describe("analyticsPhaseOf", () => { + it("reads an unfinished event as upcoming", () => { + expect( + analyticsPhaseOf( + clock({ status: "upcoming", completedAt: null, endsAt: new Date("2026-03-01T00:00:00Z"), scheduledAt: new Date("2026-03-01T00:00:00Z") }), + NOW, + ), + ).toBe("upcoming") + }) + + it("reads a recently finished event as completed and an old one as archived", () => { + expect(analyticsPhaseOf(clock(), NOW)).toBe("completed") + expect(analyticsPhaseOf(clock(), new Date("2026-05-01T00:00:00Z"))).toBe("archived") + }) +}) + +describe("arrivalBuckets", () => { + it("buckets check-in offsets into 15-minute windows keyed by minute-of-offset", () => { + expect(arrivalBuckets([0, 3, 14, 16, 31, 32, 33])).toEqual([ + { day: "0", value: 3, suppressed: false }, + { day: "15", value: 1, suppressed: false }, + { day: "30", value: 3, suppressed: false }, + ]) + }) + + it("returns nothing for an event nobody checked into", () => { + expect(arrivalBuckets([])).toEqual([]) + }) +}) + +describe("getEventAnalytics", () => { + it("returns a payload the shared response schema accepts", async () => { + const payload = await service().analytics(EVENT, "full", VIEWER) + expect(() => GetEventAnalyticsResponseSchema.parse(payload)).not.toThrow() + expect(payload.k).toBe(ANALYTICS_SUPPRESSION_K) + expect(payload.scope).toBe("full") + expect(payload.phase).toBe("completed") + }) + + it("composes the KPIs from the existing aggregates plus event_metrics_daily", async () => { + const payload = await service().analytics(EVENT, "full", VIEWER) + expect(payload.kpis).toMatchObject({ + signups: 20, + capacity: 40, + checkedIn: 12, + pageViews: 160, + donationClicks: 9, + hoursTotal: 30, + hoursVolunteers: 10, + reportsLinked: 3, + reportsResolved: 1, + postsCreated: 7, + }) + }) + + it("leaves the metrics that need tracking we do not have as null", async () => { + const payload = await service().analytics(EVENT, "full", VIEWER) + expect(payload.kpis.uniqueViewers).toBeNull() + expect(payload.kpis.shares).toBeNull() + }) + + it(`suppresses a head count under k=${ANALYTICS_SUPPRESSION_K} but publishes one at or above it`, async () => { + const payload = await service().analytics(EVENT, "full", VIEWER) + expect(payload.kpis.walkUps).toBeNull() + expect(payload.kpis.waitlisted).toBeNull() + expect(payload.kpis.checkedIn).toBe(12) + }) + + it("derives every rate against its own denominator", async () => { + const payload = await service().analytics(EVENT, "full", VIEWER) + expect(payload.rates.checkIn).toMatchObject({ numerator: 12, denominator: 20 }) + expect(payload.rates.fill).toMatchObject({ numerator: 20, denominator: 40 }) + expect(payload.rates.viewToSignup).toMatchObject({ numerator: 20, denominator: 160 }) + }) + + it("suppresses rates whose denominator is under k and reports raw totals honestly", async () => { + const small = service({ + analytics: { + eventKpis: () => + Promise.resolve({ + registered: 3, + checkedIn: 1, + waitlisted: 0, + cancelled: 0, + noShow: 0, + capacity: 4, + }), + }, + }) + + const payload = await small.analytics(EVENT, "full", VIEWER) + expect(payload.rates.checkIn.suppressed).toBe(true) + expect(payload.rates.checkIn.value).toBeNull() + expect(payload.rates.fill.suppressed).toBe(true) + expect(payload.kpis.capacity).toBe(4) + }) + + it("survives an event with no data at all and still parses", async () => { + const empty = service({ + analytics: { + eventKpis: () => + Promise.resolve({ + registered: 0, + checkedIn: 0, + waitlisted: 0, + cancelled: 0, + noShow: 0, + capacity: null, + }), + registrationsByDay: () => Promise.resolve([]), + cancellationsByDay: () => Promise.resolve([]), + arrivalOffsets: () => Promise.resolve([]), + checkinsBySlot: () => Promise.resolve([]), + waitlistConversion: () => Promise.resolve({ promoted: 0, joined: 0 }), + registrationsBySource: () => Promise.resolve([]), + eventHoursTotals: () => + Promise.resolve({ credited: 0, attendeesCredited: 0, attendeesCheckedIn: 0 }), + }, + events: { + facts: () => + Promise.resolve({ + walkUps: 0, + reportsLinked: 0, + reportsResolved: 0, + postsCreated: 0, + }), + registrationsBySlot: () => Promise.resolve([]), + reportStatuses: () => Promise.resolve([]), + hoursBuckets: () => Promise.resolve([]), + }, + metrics: [], + }) + + const payload = await empty.analytics(EVENT, "full", VIEWER) + expect(() => GetEventAnalyticsResponseSchema.parse(payload)).not.toThrow() + expect(payload.eventDay.arrivals).toEqual([]) + expect(payload.rates.viewToSignup.suppressed).toBe(true) + expect(payload.kpis.hoursTotal).toBe(0) + }) + + it("trims the card scope to the panels a dashboard carousel needs", async () => { + const payload = await service().analytics(EVENT, "card", VIEWER) + expect(payload.scope).toBe("card") + expect(payload.signups.daily).toEqual([]) + expect(payload.signups.cancellations).toEqual([]) + expect(payload.signups.bySource).toBeUndefined() + expect(payload.reach.funnel).toEqual([]) + expect(payload.eventDay.bySlot).toBeUndefined() + expect(payload.impact).toEqual({}) + expect(payload.comparison).toBeNull() + expect(payload.signups.bySlot?.rows.length).toBeLessThanOrEqual(EVENT_ANALYTICS_CARD_SLOT_ROWS) + }) + + it("keeps the full scope's funnel and breakdown panels", async () => { + const payload = await service().analytics(EVENT, "full", VIEWER) + expect(payload.reach.funnel.map((step) => step.step)).toEqual([ + "page_views", + "signups", + "checked_in", + "logged_hours", + ]) + expect(payload.signups.bySource?.rows.map((row) => row.key)).toEqual(["self", "walkup"]) + expect(payload.impact.hoursBuckets?.rows).toHaveLength(1) + expect(payload.eventDay.bySlot?.rows).toHaveLength(1) + }) + + it("returns the host's own median comparison once there is enough history", async () => { + const withHistory = service({ + events: { + previousCompletedEventIds: () => + Promise.resolve(["a", "b", "c", "d"].map((c) => c.repeat(8))), + }, + }) + const payload = await withHistory.analytics(EVENT, "full", VIEWER) + expect(payload.comparison).toMatchObject({ + sampleSize: 4, + medians: { signups: 18, checkInRate: 0.55 }, + }) + }) + + it(`returns no comparison under ${EVENT_ANALYTICS_COMPARISON_MIN_EVENTS} previous events`, async () => { + const thin = service({ + events: { previousCompletedEventIds: () => Promise.resolve(["aaaaaaaa", "bbbbbbbb"]) }, + }) + expect((await thin.analytics(EVENT, "full", VIEWER)).comparison).toBeNull() + }) + + it("404s an event that does not exist", async () => { + const missing = service({ analytics: { eventClock: () => Promise.resolve(null) } }) + await expect(missing.analytics(EVENT, "full", VIEWER)).rejects.toMatchObject({ + code: "NOT_FOUND", + }) + }) + + it("serves the second identical request from the shared host-analytics cache", async () => { + let calls = 0 + const counted = service({ + events: { + facts: () => { + calls += 1 + return Promise.resolve({ + walkUps: 0, + reportsLinked: 0, + reportsResolved: 0, + postsCreated: 0, + }) + }, + }, + }) + await counted.analytics(EVENT, "full", VIEWER) + await counted.analytics(EVENT, "full", VIEWER) + expect(calls).toBe(1) + }) +}) diff --git a/services/api/test/unit/host/insights-invalidation-wiring.test.ts b/services/api/test/unit/host/insights-invalidation-wiring.test.ts index 54ffd2d8..69789b73 100644 --- a/services/api/test/unit/host/insights-invalidation-wiring.test.ts +++ b/services/api/test/unit/host/insights-invalidation-wiring.test.ts @@ -43,6 +43,7 @@ function analyticsStub(): InsightsServiceDeps["analytics"] { eventClock: () => Promise.resolve({ status: "upcoming" as const, + createdAt: STARTS_AT, scheduledAt: STARTS_AT, endsAt: null, completedAt: null, From 9939590677c7a2c83aa4e41e2ba5fb88cc111f97 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 16:09:14 -0700 Subject: [PATCH 23/54] guard the last owner-or-admin seat in an organization --- .../host/organization-repository.drizzle.ts | 22 ++++++ .../host/organization-repository.memory.ts | 12 +++ .../host/organization-repository.types.ts | 4 +- .../src/services/host/organization-service.ts | 12 +++ .../unit/host/organization-service.test.ts | 76 +++++++++++++++++++ 5 files changed, 124 insertions(+), 2 deletions(-) diff --git a/services/api/src/services/host/organization-repository.drizzle.ts b/services/api/src/services/host/organization-repository.drizzle.ts index 650b2a02..fc362fb8 100644 --- a/services/api/src/services/host/organization-repository.drizzle.ts +++ b/services/api/src/services/host/organization-repository.drizzle.ts @@ -342,6 +342,14 @@ async function claimVerificationDocumentsInTx( } } +async function countAdminSeats(tx: Queryable, organizationId: string): Promise { + const rows = await tx<{ n: number }[]>` + SELECT count(*)::int AS n FROM organization_members + WHERE organization_id = ${organizationId} AND role IN ('owner','admin') + ` + return rows[0]?.n ?? 0 +} + export function makeDrizzleOrganizationRepository(sql: Sql): OrganizationRepository { async function readById( tag: Queryable, @@ -733,6 +741,10 @@ export function makeDrizzleOrganizationRepository(sql: Sql): OrganizationReposit if (existing === undefined) return "not_member" if (existing.role === "owner") return "owner" if (existing.role === args.role) return "updated" + if (existing.role === "admin" && args.role === "member") { + const seats = await countAdminSeats(tx, args.organizationId) + if (seats <= 1) return "last_admin" + } await tx` UPDATE organization_members SET role = ${args.role} WHERE organization_id = ${args.organizationId} AND user_id = ${args.userId} @@ -757,6 +769,16 @@ export function makeDrizzleOrganizationRepository(sql: Sql): OrganizationReposit await tx` SELECT id FROM users WHERE id = ${args.userId} LIMIT 1 FOR UPDATE ` + const target = await tx<{ role: OrganizationMemberRole }[]>` + SELECT role FROM organization_members + WHERE organization_id = ${args.organizationId} AND user_id = ${args.userId} + LIMIT 1 + FOR UPDATE + ` + if (target[0]?.role === "admin") { + const seats = await countAdminSeats(tx, args.organizationId) + if (seats <= 1) return "last_admin" + } const removed = await tx<{ role: OrganizationMemberRole }[]>` DELETE FROM organization_members WHERE organization_id = ${args.organizationId} diff --git a/services/api/src/services/host/organization-repository.memory.ts b/services/api/src/services/host/organization-repository.memory.ts index ded5129f..2ad7216a 100644 --- a/services/api/src/services/host/organization-repository.memory.ts +++ b/services/api/src/services/host/organization-repository.memory.ts @@ -449,6 +449,12 @@ export class InMemoryOrganizationRepository implements OrganizationRepository { return Promise.resolve("added") } + private countAdminSeats(organizationId: string): number { + return this.members.filter( + (m) => m.organizationId === organizationId && (m.role === "owner" || m.role === "admin"), + ).length + } + setMemberRoleTx(args: { organizationId: string userId: string @@ -460,6 +466,9 @@ export class InMemoryOrganizationRepository implements OrganizationRepository { ) if (member === undefined) return Promise.resolve("not_member") if (member.role === "owner") return Promise.resolve("owner") + if (member.role === "admin" && args.role === "member") { + if (this.countAdminSeats(args.organizationId) <= 1) return Promise.resolve("last_admin") + } if (member.role !== args.role) { member.role = args.role this.audits.push({ @@ -484,6 +493,9 @@ export class InMemoryOrganizationRepository implements OrganizationRepository { const member = this.members[index] if (member === undefined) return Promise.resolve("not_member") if (member.role === "owner") return Promise.resolve("owner") + if (member.role === "admin" && this.countAdminSeats(args.organizationId) <= 1) { + return Promise.resolve("last_admin") + } this.members.splice(index, 1) this.audits.push({ actorId: args.actorId, diff --git a/services/api/src/services/host/organization-repository.types.ts b/services/api/src/services/host/organization-repository.types.ts index 5c2ca8e9..73aa5079 100644 --- a/services/api/src/services/host/organization-repository.types.ts +++ b/services/api/src/services/host/organization-repository.types.ts @@ -273,9 +273,9 @@ export type AcceptOrganizationInviteOutcome = export type AddOrganizationMemberOutcome = "added" | "already_member" | "user_not_found" -export type RemoveOrganizationMemberOutcome = "removed" | "not_member" | "owner" +export type RemoveOrganizationMemberOutcome = "removed" | "not_member" | "owner" | "last_admin" -export type SetOrganizationMemberRoleOutcome = "updated" | "not_member" | "owner" +export type SetOrganizationMemberRoleOutcome = "updated" | "not_member" | "owner" | "last_admin" export type DecideOrgVerificationOutcome = "decided" | "not_found" | "no_application" diff --git a/services/api/src/services/host/organization-service.ts b/services/api/src/services/host/organization-service.ts index f76aa643..1597fded 100644 --- a/services/api/src/services/host/organization-service.ts +++ b/services/api/src/services/host/organization-service.ts @@ -79,6 +79,15 @@ export const MY_ORG_INVITES_CAP = 20 export const ADMIN_ORGS_DEFAULT_LIMIT = 25 +export const ORG_LAST_ADMIN_CODE = "ORG_LAST_ADMIN" + +export function lastAdminError(): AppError { + return AppError.validation( + { userId: ORG_LAST_ADMIN_CODE }, + "An organization needs at least one admin.", + ) +} + const fallbackCounters = new InMemoryCounterStore() export interface OrganizationLogoPresigner { @@ -906,6 +915,7 @@ export function makeOrganizationService(deps: OrganizationServiceDeps): Organiza if (outcome === "owner") { throw AppError.forbidden("The owner's role can't be changed.") } + if (outcome === "last_admin") throw lastAdminError() return { ok: true } }, @@ -927,6 +937,7 @@ export function makeOrganizationService(deps: OrganizationServiceDeps): Organiza : "The owner can't be removed from their own organization.", ) } + if (outcome === "last_admin") throw lastAdminError() return { ok: true } }, @@ -1209,6 +1220,7 @@ export function makeOrganizationService(deps: OrganizationServiceDeps): Organiza "The owner can't be removed. Transfer ownership to another member first.", ) } + if (outcome === "last_admin") throw lastAdminError() return { ok: true } }, diff --git a/services/api/test/unit/host/organization-service.test.ts b/services/api/test/unit/host/organization-service.test.ts index 563f7d9a..bc19b127 100644 --- a/services/api/test/unit/host/organization-service.test.ts +++ b/services/api/test/unit/host/organization-service.test.ts @@ -7,6 +7,7 @@ import { makeFakeSql, type SqlHandler } from "../../helpers/fake-sql.js" import type { Sql } from "../../../src/db/client.js" import { makeOrganizationService, + ORG_LAST_ADMIN_CODE, ORGS_CREATED_PER_DAY, type OrganizationService, } from "../../../src/services/host/organization-service.js" @@ -401,6 +402,81 @@ describe("membership", () => { }) }) +describe("last-admin guard", () => { + async function seeded(): Promise { + const dto = await service.createOrganization(base(), OWNER) + await service.inviteMember(dto.id, OWNER, { + identifierKind: "handle", + identifier: "adam", + role: "admin", + }) + await service.inviteMember(dto.id, OWNER, { + identifierKind: "handle", + identifier: "mel", + role: "member", + }) + return dto.id + } + + /** The owner seat is the structural guard, so orphan the org first to reach the residual case. */ + function dropOwnerSeat(id: string): void { + const index = repo.members.findIndex((m) => m.organizationId === id && m.role === "owner") + repo.members.splice(index, 1) + } + + it("the owner seat alone already makes the invariant unreachable through the normal paths", async () => { + const id = await seeded() + await service.setMemberRole(id, OWNER, ADMIN, "member") + expect( + repo.members.filter((m) => m.organizationId === id && m.role !== "member"), + ).toHaveLength(1) + await expect(service.removeMember(id, OWNER, OWNER)).rejects.toMatchObject({ + code: "FORBIDDEN", + }) + }) + + it("refuses to remove the last owner-or-admin seat of an orphaned organization", async () => { + const id = await seeded() + dropOwnerSeat(id) + + await expect(service.removeMember(id, ADMIN, ADMIN)).rejects.toMatchObject({ + code: "VALIDATION", + fields: { userId: ORG_LAST_ADMIN_CODE }, + }) + expect(repo.members.some((m) => m.organizationId === id && m.userId === ADMIN)).toBe(true) + }) + + it("allows the removal once a second admin seat exists", async () => { + const id = await seeded() + dropOwnerSeat(id) + await repo.setMemberRoleTx({ organizationId: id, userId: MEMBER, role: "admin", actorId: ADMIN }) + + await expect(service.removeMember(id, ADMIN, ADMIN)).resolves.toEqual({ ok: true }) + }) + + it("refuses to demote the last owner-or-admin seat at the repository boundary", async () => { + const id = await seeded() + dropOwnerSeat(id) + + await expect( + repo.setMemberRoleTx({ organizationId: id, userId: ADMIN, role: "member", actorId: ADMIN }), + ).resolves.toBe("last_admin") + expect(repo.members.find((m) => m.organizationId === id && m.userId === ADMIN)?.role).toBe( + "admin", + ) + }) + + it("lets an admin be demoted while another owner-or-admin seat remains", async () => { + const id = await seeded() + dropOwnerSeat(id) + await repo.setMemberRoleTx({ organizationId: id, userId: MEMBER, role: "admin", actorId: ADMIN }) + + await expect( + repo.setMemberRoleTx({ organizationId: id, userId: ADMIN, role: "member", actorId: ADMIN }), + ).resolves.toBe("updated") + }) +}) + describe("verification", () => { async function seeded(): Promise { const dto = await service.createOrganization(base(), OWNER) From 2d65b6058b5954e2b6222f1845b6e7e271cd6bf6 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 16:09:14 -0700 Subject: [PATCH 24/54] wire the four new endpoints into the coverage and rate-limit gates --- services/api/test/unit/rate-limit-plugin.test.ts | 6 ++++++ services/api/test/unit/route-coverage.test.ts | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/services/api/test/unit/rate-limit-plugin.test.ts b/services/api/test/unit/rate-limit-plugin.test.ts index 65db9560..5381a379 100644 --- a/services/api/test/unit/rate-limit-plugin.test.ts +++ b/services/api/test/unit/rate-limit-plugin.test.ts @@ -102,6 +102,10 @@ import { BROADCAST_SEND_RATE_LIMIT, BROADCAST_READ_RATE_LIMIT, } from "../../src/routes/host/broadcasts.routes.js" +import { + ANNOUNCEMENT_CREATE_RATE_LIMIT, + ANNOUNCEMENT_READ_RATE_LIMIT, +} from "../../src/routes/host/announcements.routes.js" import { ANALYTICS_RATE_LIMIT } from "../../src/routes/host/analytics.routes.js" import { EXPORT_REQUEST_RATE_LIMIT, @@ -493,6 +497,8 @@ const IDENTITY_SCOPED_LIMITS = { BROADCAST_WRITE_RATE_LIMIT, BROADCAST_SEND_RATE_LIMIT, BROADCAST_READ_RATE_LIMIT, + ANNOUNCEMENT_CREATE_RATE_LIMIT, + ANNOUNCEMENT_READ_RATE_LIMIT, ANALYTICS_RATE_LIMIT, EXPORT_REQUEST_RATE_LIMIT, EXPORT_READ_RATE_LIMIT, diff --git a/services/api/test/unit/route-coverage.test.ts b/services/api/test/unit/route-coverage.test.ts index 7761a3a4..eabea9ec 100644 --- a/services/api/test/unit/route-coverage.test.ts +++ b/services/api/test/unit/route-coverage.test.ts @@ -229,8 +229,8 @@ describe("route-coverage: every shared endpoint is registered (offline boot smok }) } - it("covers ALL 319 endpoints in the registry (no endpoint skipped)", () => { - expect(Object.keys(endpoints).length).toBe(319) + it("covers ALL 323 endpoints in the registry (no endpoint skipped)", () => { + expect(Object.keys(endpoints).length).toBe(323) }) it("the discriminator is not vacuous: a bogus path IS detected as route-missing", async () => { From a987fd7cc6e8eccbde8b848d9d4ebc14369ae343 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 16:58:59 -0700 Subject: [PATCH 25/54] quarantine held and rejected media behind a servable whitelist --- .../admin/moderation-repository.drizzle.ts | 6 +- services/api/src/services/media-served-key.ts | 10 +- services/api/test/helpers/sql-predicate.ts | 17 +++ .../unit/chat-attachments-servable.test.ts | 53 ++++++++- .../unit/media-servable-quarantine.test.ts | 111 ++++++++++++++++++ 5 files changed, 190 insertions(+), 7 deletions(-) create mode 100644 services/api/test/helpers/sql-predicate.ts create mode 100644 services/api/test/unit/media-servable-quarantine.test.ts diff --git a/services/api/src/services/admin/moderation-repository.drizzle.ts b/services/api/src/services/admin/moderation-repository.drizzle.ts index 99009766..996e109d 100644 --- a/services/api/src/services/admin/moderation-repository.drizzle.ts +++ b/services/api/src/services/admin/moderation-repository.drizzle.ts @@ -7,7 +7,7 @@ import { ADMIN_CATEGORIES } from "./category-counts.js" import { ilikeAnyOf, type SqlFragment } from "./sql-fragments.js" import { tombstonePostInTx } from "../post-repository.drizzle.js" import { assertTargetIsNotOperatorRole } from "../../auth/operator-target.js" -import { servedKeyExpr, servableMediaFilter } from "../media-served-key.js" +import { moderationMediaKeyExpr, moderationMediaFilter } from "../media-served-key.js" import { type CreateModerationItemInput, type ListModerationArgs, @@ -256,10 +256,10 @@ async function loadMedia( ): Promise { if (subjectType !== "report") return [] const rows = await sql` - SELECT id, kind, ${servedKeyExpr(sql, "media_assets")} AS r2_key, thumb_key + SELECT id, kind, ${moderationMediaKeyExpr(sql, "media_assets")} AS r2_key, thumb_key FROM media_assets WHERE report_id = ${subjectId} - AND ${servableMediaFilter(sql, "media_assets")} + AND ${moderationMediaFilter(sql, "media_assets")} ORDER BY created_at ASC ` return rows.map((m) => ({ diff --git a/services/api/src/services/media-served-key.ts b/services/api/src/services/media-served-key.ts index 96006ccb..453096f9 100644 --- a/services/api/src/services/media-served-key.ts +++ b/services/api/src/services/media-served-key.ts @@ -7,9 +7,17 @@ type SqlFragment = postgres.Fragment export type MediaAlias = "m" | "am" | "ma" | "a" | "media_assets" export function servedKeyExpr(sql: Queryable, alias: MediaAlias): SqlFragment { - return sql`COALESCE(${sql(alias)}.served_key, CASE WHEN ${sql(alias)}.status <> 'ready' THEN ${sql(alias)}.r2_key END)` + return sql`COALESCE(${sql(alias)}.served_key, CASE WHEN ${sql(alias)}.status = 'validating' THEN ${sql(alias)}.r2_key END)` } export function servableMediaFilter(sql: Queryable, alias: MediaAlias): SqlFragment { + return sql`(${sql(alias)}.status = 'validating' OR (${sql(alias)}.status = 'ready' AND ${sql(alias)}.served_key IS NOT NULL))` +} + +export function moderationMediaKeyExpr(sql: Queryable, alias: MediaAlias): SqlFragment { + return sql`COALESCE(${sql(alias)}.served_key, CASE WHEN ${sql(alias)}.status <> 'ready' THEN ${sql(alias)}.r2_key END)` +} + +export function moderationMediaFilter(sql: Queryable, alias: MediaAlias): SqlFragment { return sql`(${sql(alias)}.status <> 'ready' OR ${sql(alias)}.served_key IS NOT NULL)` } diff --git a/services/api/test/helpers/sql-predicate.ts b/services/api/test/helpers/sql-predicate.ts new file mode 100644 index 00000000..2ad71d99 --- /dev/null +++ b/services/api/test/helpers/sql-predicate.ts @@ -0,0 +1,17 @@ + +export type PredicateRow = Record + +export function evalSqlPredicate(text: string, row: PredicateRow): boolean { + const js = text + .replace(/\b[a-z_]+\.([a-z_]+)\b/g, (_all, column: string) => `row[${JSON.stringify(column)}]`) + .replace(/\s+IS NOT NULL\b/g, " !== null") + .replace(/\s+IS NULL\b/g, " === null") + .replace(/<>/g, "!==") + .replace(/(?=])=(?!=)/g, "===") + .replace(/\bOR\b/g, "||") + .replace(/\bAND\b/g, "&&") + const compiled = new Function("row", `"use strict"; return Boolean(${js})`) as ( + candidate: PredicateRow, + ) => boolean + return compiled(row) +} diff --git a/services/api/test/unit/chat-attachments-servable.test.ts b/services/api/test/unit/chat-attachments-servable.test.ts index 8206cca5..a2f7ed84 100644 --- a/services/api/test/unit/chat-attachments-servable.test.ts +++ b/services/api/test/unit/chat-attachments-servable.test.ts @@ -13,6 +13,7 @@ import { describe, expect, it } from "vitest" import { makeFakeSql } from "../helpers/fake-sql.js" +import { evalSqlPredicate } from "../helpers/sql-predicate.js" import type { Queryable } from "../../src/db/client.js" import { loadServableAttachmentsFor, @@ -56,7 +57,7 @@ describe("loadServableAttachmentsFor", () => { expect(statement.sql).toMatch(/media_assets\.r2_key/) }) - it("no longer filters on status = 'ready': a validating asset is servable", async () => { + it("admits a validating asset without admitting every non-ready status", async () => { const fake = makeFakeSql([{ match: /FROM media_assets/, rows: [] }]) await loadServableAttachmentsFor( @@ -67,8 +68,54 @@ describe("loadServableAttachmentsFor", () => { ) const statement = fake.statements.at(-1)! - expect(statement.sql).not.toMatch(/status = 'ready'/) - expect(statement.sql).toMatch(/media_assets\.status <> 'ready' OR media_assets\.served_key IS NOT NULL/) + expect(statement.sql).toMatch( + /media_assets\.status = 'validating' OR \(media_assets\.status = 'ready' AND media_assets\.served_key IS NOT NULL\)/, + ) + expect(statement.sql).not.toMatch(/status <> 'ready'/) + }) + + it("quarantines held and rejected attachments and serves validating and ready ones", async () => { + const fake = makeFakeSql([{ match: /FROM media_assets/, rows: [] }]) + + await loadServableAttachmentsFor( + fake.sql as unknown as Queryable, + "chat_message_id", + [MESSAGE], + PRESIGN, + ) + + const predicate = /AND (\(media_assets\.status = 'validating'[^\n]*?IS NOT NULL\)\))/.exec( + fake.statements.at(-1)!.sql, + )?.[1] + expect(predicate).toBeDefined() + const servable = (status: string, servedKey: string | null): boolean => + evalSqlPredicate(predicate as string, { + status, + served_key: servedKey, + r2_key: "chat/one.jpg", + }) + + expect(servable("validating", null)).toBe(true) + expect(servable("ready", "chat/one.jpg.served")).toBe(true) + expect(servable("ready", null)).toBe(false) + expect(servable("held", "chat/one.jpg.served")).toBe(false) + expect(servable("held", null)).toBe(false) + expect(servable("rejected", null)).toBe(false) + }) + + it("falls back to the raw upload key only while the asset is validating", async () => { + const fake = makeFakeSql([{ match: /FROM media_assets/, rows: [] }]) + + await loadServableAttachmentsFor( + fake.sql as unknown as Queryable, + "chat_message_id", + [MESSAGE], + PRESIGN, + ) + + expect(fake.statements.at(-1)!.sql).toMatch( + /COALESCE\(media_assets\.served_key, CASE WHEN media_assets\.status = 'validating' THEN media_assets\.r2_key END\)/, + ) }) it("returns the still-validating attachment with its status intact so the client can render it", async () => { diff --git a/services/api/test/unit/media-servable-quarantine.test.ts b/services/api/test/unit/media-servable-quarantine.test.ts new file mode 100644 index 00000000..e1b25d0e --- /dev/null +++ b/services/api/test/unit/media-servable-quarantine.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest" +import { makeFakeSql } from "../helpers/fake-sql.js" +import { evalSqlPredicate, type PredicateRow } from "../helpers/sql-predicate.js" +import type { Queryable } from "../../src/db/client.js" +import { + moderationMediaFilter, + moderationMediaKeyExpr, + servableMediaFilter, + servedKeyExpr, +} from "../../src/services/media-served-key.js" +import { makeDrizzleReportRepository } from "../../src/services/report-repository.drizzle.js" +import type { Sql } from "../../src/db/client.js" + +const R2 = "reports/original.jpg" +const SERVED = "reports/original.jpg.served" + +async function fragmentSql(build: (sql: Queryable) => unknown): Promise { + const fake = makeFakeSql([]) + await fake.sql`${build(fake.sql as unknown as Queryable)}` + return fake.statements[0]?.sql ?? "" +} + +function asset(status: string, servedKey: string | null): PredicateRow { + return { status, served_key: servedKey, r2_key: R2 } +} + +const CASES: { label: string; row: PredicateRow; servable: boolean; moderatable: boolean }[] = [ + { label: "validating, not yet processed", row: asset("validating", null), servable: true, moderatable: true }, + { label: "validating, processed early", row: asset("validating", SERVED), servable: true, moderatable: true }, + { label: "ready with a served copy", row: asset("ready", SERVED), servable: true, moderatable: true }, + { label: "ready without a served copy", row: asset("ready", null), servable: false, moderatable: false }, + { label: "held with a served copy", row: asset("held", SERVED), servable: false, moderatable: true }, + { label: "held without a served copy", row: asset("held", null), servable: false, moderatable: true }, + { label: "rejected", row: asset("rejected", null), servable: false, moderatable: true }, +] + +describe("servableMediaFilter", () => { + it("is a whitelist, not a status <> 'ready' blacklist", async () => { + const text = await fragmentSql((sql) => servableMediaFilter(sql, "media_assets")) + expect(text).toBe( + "(media_assets.status = 'validating' OR (media_assets.status = 'ready' AND media_assets.served_key IS NOT NULL))", + ) + expect(text).not.toMatch(/status <> 'ready'/) + }) + + for (const c of CASES) { + it(`${c.servable ? "serves" : "quarantines"} an asset that is ${c.label}`, async () => { + const text = await fragmentSql((sql) => servableMediaFilter(sql, "media_assets")) + expect(evalSqlPredicate(text, c.row)).toBe(c.servable) + }) + } +}) + +describe("servedKeyExpr", () => { + it("falls back to the raw upload key only while the asset is validating", async () => { + const text = await fragmentSql((sql) => servedKeyExpr(sql, "media_assets")) + expect(text).toBe( + "COALESCE(media_assets.served_key, CASE WHEN media_assets.status = 'validating' THEN media_assets.r2_key END)", + ) + expect(text).not.toMatch(/status <> 'ready'/) + }) +}) + +describe("moderation media fragments", () => { + it("keep the operator surface able to view held and rejected assets", async () => { + const text = await fragmentSql((sql) => moderationMediaFilter(sql, "media_assets")) + for (const c of CASES) { + expect({ label: c.label, visible: evalSqlPredicate(text, c.row) }).toEqual({ + label: c.label, + visible: c.moderatable, + }) + } + }) + + it("still resolve a key for a held asset that never got a served copy", async () => { + const text = await fragmentSql((sql) => moderationMediaKeyExpr(sql, "media_assets")) + expect(text).toBe( + "COALESCE(media_assets.served_key, CASE WHEN media_assets.status <> 'ready' THEN media_assets.r2_key END)", + ) + }) +}) + +describe("report media reads", () => { + it("select through the whitelist filter and the validating-only fallback", async () => { + const fake = makeFakeSql([{ match: /FROM media_assets/, rows: [] }]) + const repo = makeDrizzleReportRepository(fake.sql as unknown as Sql) + + await repo.findMediaForReport("33333333-3333-4333-8333-333333333333", true) + + const statement = fake.statements.at(-1)! + expect(statement.sql).toMatch( + /media_assets\.status = 'validating' OR \(media_assets\.status = 'ready' AND media_assets\.served_key IS NOT NULL\)/, + ) + expect(statement.sql).not.toMatch(/status <> 'ready'/) + expect(statement.sql).toMatch( + /COALESCE\(media_assets\.served_key, CASE WHEN media_assets\.status = 'validating' THEN media_assets\.r2_key END\)/, + ) + }) + + it("applies the same filter to the batched multi-report read", async () => { + const fake = makeFakeSql([{ match: /FROM media_assets/, rows: [] }]) + const repo = makeDrizzleReportRepository(fake.sql as unknown as Sql) + + await repo.findMediaForReports(["33333333-3333-4333-8333-333333333333"], true) + + const statement = fake.statements.at(-1)! + expect(statement.sql).toMatch( + /media_assets\.status = 'validating' OR \(media_assets\.status = 'ready' AND media_assets\.served_key IS NOT NULL\)/, + ) + }) +}) From 6fc041578df26733dfa383ee917e14881a6c813e Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 16:58:59 -0700 Subject: [PATCH 26/54] announcement caps, org seat lock, analytics cohort query --- .../src/services/host/announcement-service.ts | 2 +- .../src/services/host/broadcast-service.ts | 74 ++++++++++----- .../event-analytics-repository.drizzle.ts | 48 ++++++---- .../host/organization-repository.drizzle.ts | 6 ++ .../host/analytics-repository-sql.test.ts | 45 +++++++++ .../unit/host/announcement-service.test.ts | 91 ++++++++++++++++++- .../unit/host/organization-service.test.ts | 45 +++++++++ 7 files changed, 269 insertions(+), 42 deletions(-) diff --git a/services/api/src/services/host/announcement-service.ts b/services/api/src/services/host/announcement-service.ts index c85baace..ecf9a05e 100644 --- a/services/api/src/services/host/announcement-service.ts +++ b/services/api/src/services/host/announcement-service.ts @@ -179,7 +179,7 @@ export function makeAnnouncementService(deps: AnnouncementServiceDeps): Announce ctaUrl: cta, }) } - await deps.broadcasts.send(cleanupId, actorId, draft.id) + await deps.broadcasts.sendAnnouncement(cleanupId, actorId, draft.id) } catch (err) { await repo.deleteDraft(cleanupId, draft.id).catch(() => false) throw err diff --git a/services/api/src/services/host/broadcast-service.ts b/services/api/src/services/host/broadcast-service.ts index 004a14b8..493b2b05 100644 --- a/services/api/src/services/host/broadcast-service.ts +++ b/services/api/src/services/host/broadcast-service.ts @@ -141,6 +141,10 @@ export function toBroadcastDTO(record: BroadcastRecord): BroadcastDTO { } } +export interface BroadcastSendOptions { + skipEventSendCounters?: boolean +} + export interface BroadcastService { list(cleanupId: string, query: ListEventBroadcastsRequest): Promise<{ items: BroadcastDTO[] @@ -164,7 +168,13 @@ export interface BroadcastService { body: PreviewEventBroadcastRequest, ): Promise testSend(cleanupId: string, actorId: string, broadcastId: string): Promise<{ ok: true }> - send(cleanupId: string, actorId: string, broadcastId: string): Promise + send( + cleanupId: string, + actorId: string, + broadcastId: string, + options?: BroadcastSendOptions, + ): Promise + sendAnnouncement(cleanupId: string, actorId: string, broadcastId: string): Promise schedule( cleanupId: string, actorId: string, @@ -276,6 +286,36 @@ export function makeBroadcastService(deps: BroadcastServiceDeps): BroadcastServi return event } + async function sendBroadcast( + cleanupId: string, + actorId: string, + broadcastId: string, + skipEventSendCounters: boolean, + ): Promise { + await guardHost(actorId) + const record = await requireDraft(cleanupId, broadcastId) + if (record.subject === null || record.bodyMd === null) { + throw AppError.conflict("That message has no content to send.") + } + assertContent(record.subject, record.bodyMd, record.ctaUrl) + const event = await requireEventOrgNotSuspended(cleanupId) + const moved = await repo.transition(broadcastId, ["draft", "scheduled"], "sending", { + startedAt: now(), + replyTo: event.replyToVerified ? event.replyTo : null, + }) + if (moved === null) throw AppError.conflict("That message is already sending.") + if (!skipEventSendCounters) { + try { + await reserveSendCounters(cleanupId, actorId) + } catch (err) { + await repo.transition(broadcastId, ["sending"], record.status, { startedAt: null }) + throw err + } + } + await deps.enqueuePlan(broadcastId) + return toBroadcastDTO(moved) + } + return { async list(cleanupId, query) { const limit = query.limit ?? BROADCAST_DEFAULT_LIMIT @@ -441,27 +481,17 @@ export function makeBroadcastService(deps: BroadcastServiceDeps): BroadcastServi return { ok: true } }, - async send(cleanupId, actorId, broadcastId) { - await guardHost(actorId) - const record = await requireDraft(cleanupId, broadcastId) - if (record.subject === null || record.bodyMd === null) { - throw AppError.conflict("That message has no content to send.") - } - assertContent(record.subject, record.bodyMd, record.ctaUrl) - const event = await requireEventOrgNotSuspended(cleanupId) - const moved = await repo.transition(broadcastId, ["draft", "scheduled"], "sending", { - startedAt: now(), - replyTo: event.replyToVerified ? event.replyTo : null, - }) - if (moved === null) throw AppError.conflict("That message is already sending.") - try { - await reserveSendCounters(cleanupId, actorId) - } catch (err) { - await repo.transition(broadcastId, ["sending"], record.status, { startedAt: null }) - throw err - } - await deps.enqueuePlan(broadcastId) - return toBroadcastDTO(moved) + async send(cleanupId, actorId, broadcastId, options) { + return sendBroadcast( + cleanupId, + actorId, + broadcastId, + options?.skipEventSendCounters === true, + ) + }, + + async sendAnnouncement(cleanupId, actorId, broadcastId) { + return sendBroadcast(cleanupId, actorId, broadcastId, true) }, async schedule(cleanupId, actorId, broadcastId, scheduledAt) { diff --git a/services/api/src/services/host/event-analytics-repository.drizzle.ts b/services/api/src/services/host/event-analytics-repository.drizzle.ts index cf8bee59..d02e384f 100644 --- a/services/api/src/services/host/event-analytics-repository.drizzle.ts +++ b/services/api/src/services/host/event-analytics-repository.drizzle.ts @@ -1,6 +1,9 @@ +import type postgres from "postgres" import type { KeyCount } from "@civfix/shared/host" import type { Sql } from "../../db/client.js" +type Fragment = postgres.Fragment + export const EVENT_ANALYTICS_SLOT_ROW_LIMIT = 50 export interface EventAnalyticsFacts { @@ -42,26 +45,39 @@ export interface EventAnalyticsRepository { export function makeDrizzleEventAnalyticsRepository(sql: Sql): EventAnalyticsRepository { return { async previousCompletedEventIds(args) { - const orgFilter = + const orgFilter = (): Fragment => args.organizationId !== null ? sql`AND c.organization_id = ${args.organizationId}` : sql`` const rows = await sql<{ id: string }[]>` - SELECT c.id - FROM cleanups c - WHERE c.completed_at IS NOT NULL - AND c.id <> ${args.excludeCleanupId} - AND (c.organizer_user_id = ${args.userId} - OR EXISTS ( - SELECT 1 FROM cleanup_members m - WHERE m.cleanup_id = c.id AND m.user_id = ${args.userId} - AND m.role IN ('organizer','cohost','coordinator')) - OR EXISTS ( - SELECT 1 FROM organization_members om - WHERE om.organization_id = c.organization_id AND om.user_id = ${args.userId} - AND om.role IN ('owner','admin'))) - ${orgFilter} - ORDER BY c.completed_at DESC + WITH hosted AS ( + SELECT c.id, c.completed_at + FROM cleanups c + WHERE c.organizer_user_id = ${args.userId} + AND c.completed_at IS NOT NULL + AND c.id <> ${args.excludeCleanupId} + ${orgFilter()} + UNION + SELECT c.id, c.completed_at + FROM cleanup_members m + JOIN cleanups c ON c.id = m.cleanup_id + WHERE m.user_id = ${args.userId} + AND m.role IN ('organizer','cohost','coordinator') + AND c.completed_at IS NOT NULL + AND c.id <> ${args.excludeCleanupId} + ${orgFilter()} + UNION + SELECT c.id, c.completed_at + FROM organization_members om + JOIN cleanups c ON c.organization_id = om.organization_id + WHERE om.user_id = ${args.userId} + AND om.role IN ('owner','admin') + AND c.completed_at IS NOT NULL + AND c.id <> ${args.excludeCleanupId} + ${orgFilter()} + ) + SELECT id FROM hosted + ORDER BY completed_at DESC LIMIT ${args.limit}` return rows.map((row) => row.id) }, diff --git a/services/api/src/services/host/organization-repository.drizzle.ts b/services/api/src/services/host/organization-repository.drizzle.ts index fc362fb8..a7bf9cb3 100644 --- a/services/api/src/services/host/organization-repository.drizzle.ts +++ b/services/api/src/services/host/organization-repository.drizzle.ts @@ -731,6 +731,9 @@ export function makeDrizzleOrganizationRepository(sql: Sql): OrganizationReposit actorId: string }): Promise { return sql.begin(async (tx) => { + await tx` + SELECT id FROM organizations WHERE id = ${args.organizationId} LIMIT 1 FOR UPDATE + ` const current = await tx<{ role: OrganizationMemberRole }[]>` SELECT role FROM organization_members WHERE organization_id = ${args.organizationId} AND user_id = ${args.userId} @@ -766,6 +769,9 @@ export function makeDrizzleOrganizationRepository(sql: Sql): OrganizationReposit reason?: string }): Promise { return sql.begin(async (tx) => { + await tx` + SELECT id FROM organizations WHERE id = ${args.organizationId} LIMIT 1 FOR UPDATE + ` await tx` SELECT id FROM users WHERE id = ${args.userId} LIMIT 1 FOR UPDATE ` diff --git a/services/api/test/unit/host/analytics-repository-sql.test.ts b/services/api/test/unit/host/analytics-repository-sql.test.ts index ebcedb4c..06d972f7 100644 --- a/services/api/test/unit/host/analytics-repository-sql.test.ts +++ b/services/api/test/unit/host/analytics-repository-sql.test.ts @@ -5,6 +5,7 @@ import { INSIGHTS_TREND_LIMIT, makeDrizzleAnalyticsRepository, } from "../../../src/services/host/analytics-repository.drizzle.js" +import { makeDrizzleEventAnalyticsRepository } from "../../../src/services/host/event-analytics-repository.drizzle.js" import { MAX_INSIGHTS_TOP_VOLUNTEERS } from "@civfix/shared" import type { Sql } from "../../../src/db/client.js" @@ -114,3 +115,47 @@ describe("#110 host hours SQL", () => { expect(fake.statements).toEqual([]) }) }) + +describe("event analytics comparison cohort SQL", () => { + const USER = "dddddddd-dddd-4ddd-8ddd-dddddddddddd" + const ORG = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee" + + it("drives each membership branch from an indexed column instead of a correlated EXISTS", async () => { + const stmt = await emitted((sql) => + makeDrizzleEventAnalyticsRepository(sql).previousCompletedEventIds({ + userId: USER, + organizationId: null, + excludeCleanupId: EVENT, + limit: 12, + }), + ) + const text = squash(stmt.sql) + + expect(text).not.toContain("EXISTS") + expect(text).toContain("FROM cleanups c WHERE c.organizer_user_id = ?") + expect(text).toContain("FROM cleanup_members m JOIN cleanups c ON c.id = m.cleanup_id") + expect(text).toContain( + "FROM organization_members om JOIN cleanups c ON c.organization_id = om.organization_id", + ) + expect(text.match(/UNION/g)).toHaveLength(2) + expect(text).toContain("SELECT id FROM hosted ORDER BY completed_at DESC LIMIT ?") + expect(stmt.values).toContain(12) + }) + + it("applies the organization filter to every branch", async () => { + const stmt = await emitted((sql) => + makeDrizzleEventAnalyticsRepository(sql).previousCompletedEventIds({ + userId: USER, + organizationId: ORG, + excludeCleanupId: EVENT, + limit: 12, + }), + ) + const text = squash(stmt.sql) + + expect(text.match(/AND c\.organization_id = \?/g)).toHaveLength(3) + expect(text.match(/AND c\.completed_at IS NOT NULL/g)).toHaveLength(3) + expect(text.match(/AND c\.id <> \?/g)).toHaveLength(3) + expect(stmt.values.filter((v) => v === ORG)).toHaveLength(3) + }) +}) diff --git a/services/api/test/unit/host/announcement-service.test.ts b/services/api/test/unit/host/announcement-service.test.ts index 695a611e..fb7716da 100644 --- a/services/api/test/unit/host/announcement-service.test.ts +++ b/services/api/test/unit/host/announcement-service.test.ts @@ -9,8 +9,10 @@ import { import { InMemoryCounterStore } from "../../../src/abuse/counter-store.js" import { InMemoryBroadcastRepository } from "../../../src/services/host/broadcast-repository.memory.js" import { + BroadcastCapError, makeBroadcastService, type BroadcastConfig, + type BroadcastService, } from "../../../src/services/host/broadcast-service.js" import { announcementTitleOf, @@ -26,9 +28,9 @@ const SLOT = "00000000-0000-0000-0000-0000000000s1".replace("s1", "0f1") const CONFIG: BroadcastConfig = { killSwitch: false, - perEventPerDay: 50, + perEventPerDay: 3, recipientsPerDay: 2000, - cooldownSec: 0, + cooldownSec: 900, minAccountAgeHours: 24, maxRecipients: 5000, chunkSize: 200, @@ -89,6 +91,7 @@ interface Harness { repo: InMemoryBroadcastRepository plans: string[] service: AnnouncementService + broadcasts: BroadcastService clock: { at: Date } } @@ -118,7 +121,30 @@ function harness(over: { org?: OrganizationRefDTO | null } = {}): Harness { config: CONFIG, now: () => clock.at, }) - return { repo, plans, service, clock } + return { repo, plans, service, broadcasts, clock } +} + +async function sendRegularBroadcast(h: Harness, subject: string): Promise { + const draft = await h.repo.create({ + cleanupId: EVENT, + createdBy: HOST, + kind: "host_broadcast", + subject, + bodyMd: "Body", + segment: { kind: "all_registered" }, + channels: ["email"], + status: "draft", + }) + await h.broadcasts.send(EVENT, HOST, draft.id) +} + +async function capKindOf(run: () => Promise): Promise { + try { + await run() + return "none" + } catch (err) { + return err instanceof BroadcastCapError ? err.kind : `other:${String(err)}` + } } let h: Harness @@ -233,6 +259,65 @@ describe("createEventAnnouncement", () => { ).resolves.toMatchObject({ bodyMd: "A new day" }) }) + it("is not blocked by the regular-broadcast cooldown", async () => { + await h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: "First", + audience: { kind: "all_registered" }, + }) + + await expect( + h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: "Second, minutes later", + audience: { kind: "all_registered" }, + }), + ).resolves.toMatchObject({ bodyMd: "Second, minutes later" }) + }) + + it("does not consume the event's regular-broadcast daily slots", async () => { + for (let i = 0; i < CONFIG.perEventPerDay + 2; i += 1) { + await h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: `Body ${i}`, + audience: { kind: "all_registered" }, + }) + } + + expect(await capKindOf(() => sendRegularBroadcast(h, "Still allowed"))).toBe("none") + }) + + it("leaves the regular-broadcast cooldown and per-event cap intact for host broadcasts", async () => { + await h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: "Announcement first", + audience: { kind: "all_registered" }, + }) + + expect(await capKindOf(() => sendRegularBroadcast(h, "One"))).toBe("none") + expect(await capKindOf(() => sendRegularBroadcast(h, "Two"))).toBe("cooldown") + }) + + it("still refuses a host broadcast that exhausts the per-event daily cap", async () => { + const noCooldown = harness() + for (let i = 0; i < CONFIG.perEventPerDay; i += 1) { + noCooldown.clock.at = new Date(Date.UTC(2026, 0, 20, 12 + i)) + expect(await capKindOf(() => sendRegularBroadcast(noCooldown, `Body ${i}`))).toBe("none") + } + noCooldown.clock.at = new Date(Date.UTC(2026, 0, 20, 12 + CONFIG.perEventPerDay)) + + expect(await capKindOf(() => sendRegularBroadcast(noCooldown, "One too many"))).toBe( + "per_event_per_day", + ) + await expect( + noCooldown.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: "Announcements are a separate lane", + audience: { kind: "all_registered" }, + }), + ).resolves.toMatchObject({ bodyMd: "Announcements are a separate lane" }) + }) + it("leaves no orphan draft behind when the send is refused", async () => { const suspended = harness() suspended.repo.seedHost(HOST, { diff --git a/services/api/test/unit/host/organization-service.test.ts b/services/api/test/unit/host/organization-service.test.ts index bc19b127..2fb39e33 100644 --- a/services/api/test/unit/host/organization-service.test.ts +++ b/services/api/test/unit/host/organization-service.test.ts @@ -475,6 +475,51 @@ describe("last-admin guard", () => { repo.setMemberRoleTx({ organizationId: id, userId: ADMIN, role: "member", actorId: ADMIN }), ).resolves.toBe("updated") }) + + const ORG = "66666666-6666-4666-8666-666666666666" + + function seatHandlers(): SqlHandler[] { + return [ + { match: /SELECT role FROM organization_members/, rows: [{ role: "admin" }] }, + { match: /count\(\*\)::int AS n\s+FROM organization_members/, rows: [{ n: 1 }] }, + ] + } + + function lockIndex(statements: { sql: string }[]): number { + return statements.findIndex((s) => /FROM organizations WHERE id = \?.*FOR UPDATE/s.test(s.sql)) + } + + function countIndex(statements: { sql: string }[]): number { + return statements.findIndex((s) => /count\(\*\)::int AS n\s+FROM organization_members/.test(s.sql)) + } + + it("serializes a demotion on the organizations row before it counts the seats", async () => { + const fake = makeFakeSql(seatHandlers()) + + const outcome = await makeDrizzleOrganizationRepository( + fake.sql as unknown as Sql, + ).setMemberRoleTx({ organizationId: ORG, userId: ADMIN, role: "member", actorId: OWNER }) + + expect(outcome).toBe("last_admin") + const lock = lockIndex(fake.statements) + const count = countIndex(fake.statements) + expect(lock).toBe(0) + expect(count).toBeGreaterThan(lock) + }) + + it("serializes a removal on the organizations row before it counts the seats", async () => { + const fake = makeFakeSql(seatHandlers()) + + const outcome = await makeDrizzleOrganizationRepository( + fake.sql as unknown as Sql, + ).removeMemberTx({ organizationId: ORG, userId: ADMIN, actorId: OWNER }) + + expect(outcome).toBe("last_admin") + const lock = lockIndex(fake.statements) + const count = countIndex(fake.statements) + expect(lock).toBe(0) + expect(count).toBeGreaterThan(lock) + }) }) describe("verification", () => { From 30a6fb7ee996688098024c4405da861301817d5b Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 18:57:01 -0700 Subject: [PATCH 27/54] adopt the @civfix/shared 0.51.0 address contract Moves both services onto ^0.51.0, which carries the address-resolution surface: AddressPrecision / EventAddressSource / ReportAddressSource, CleanupDTO.addressSource, ReportDTO.addrSource + addrPrecision, the resolveAddress endpoint definition, and the geocodePointKey helper the new cache keys on. Co-Authored-By: Claude Opus 5 (1M context) --- pnpm-lock.yaml | 14 +++++++------- services/api/package.json | 2 +- services/media-worker/package.json | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 85b9e598..26ca0308 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -65,8 +65,8 @@ importers: specifier: ^3.700.0 version: 3.1057.0 '@civfix/shared': - specifier: ^0.50.0 - version: 0.50.0 + specifier: ^0.51.0 + version: 0.51.0 '@fastify/cookie': specifier: ^11.0.1 version: 11.0.2 @@ -210,8 +210,8 @@ importers: specifier: workspace:* version: link:../api '@civfix/shared': - specifier: ^0.50.0 - version: 0.50.0 + specifier: ^0.51.0 + version: 0.51.0 '@sentry/node': specifier: ^10.73.0 version: 10.73.0(@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1)) @@ -414,8 +414,8 @@ packages: '@balena/dockerignore@1.0.2': resolution: {integrity: sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==} - '@civfix/shared@0.50.0': - resolution: {integrity: sha512-bteKU/bE5lKmoQvPWFgCVKBLCxzFqXVapLWg2+46FgbS9yEugGPv9zJQ88O+zUinv3na0Cvmzj+jLlN6FfyyYw==} + '@civfix/shared@0.51.0': + resolution: {integrity: sha512-bWzOhGZITr8G6I0XkvXP5id+EymKgP+TSBWcXBjXmc80/c2Q1NY+i8HatCsz/RgaqV4+DIoLxm85YX6QECoYbA==} engines: {node: '>=20'} '@derhuerst/http-basic@8.2.4': @@ -4440,7 +4440,7 @@ snapshots: '@balena/dockerignore@1.0.2': {} - '@civfix/shared@0.50.0': + '@civfix/shared@0.51.0': dependencies: zod: 3.25.76 diff --git a/services/api/package.json b/services/api/package.json index 4da180d6..d117730d 100644 --- a/services/api/package.json +++ b/services/api/package.json @@ -58,7 +58,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.700.0", "@aws-sdk/s3-request-presigner": "^3.700.0", - "@civfix/shared": "^0.50.0", + "@civfix/shared": "^0.51.0", "@fastify/cookie": "^11.0.1", "@fastify/cors": "^10.0.1", "@fastify/helmet": "^12.0.1", diff --git a/services/media-worker/package.json b/services/media-worker/package.json index cbc0a037..2a1a73fe 100644 --- a/services/media-worker/package.json +++ b/services/media-worker/package.json @@ -23,7 +23,7 @@ "@aws-sdk/client-s3": "^3.700.0", "@aws-sdk/s3-request-presigner": "^3.700.0", "@civfix/api": "workspace:*", - "@civfix/shared": "^0.50.0", + "@civfix/shared": "^0.51.0", "@sentry/node": "^10.73.0", "@smithy/node-http-handler": "4.7.5", "drizzle-orm": "^0.45.2", From 2003e1af4f9deb3ba919e0a08ddf77a9eaead928 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 18:57:11 -0700 Subject: [PATCH 28/54] migration 0175: geocode cache + address provenance columns Three concerns, one theme - a pin now yields an honest address line, and every stored address records where it came from: - geocode_cache, keyed by the shared 5-decimal point key so the client's debounced preview and the create that follows it collapse onto one row. TTL is applied on read (180 days for a hit, 15 minutes for a negative entry) rather than by a cron, so a provider outage cannot poison a point for half a year. - cleanups.address_source, backfilled to 'manual' wherever an address already exists - a host typed it, which is the closest truth. Events with no address get nothing: verification is the feature and the server cannot verify on a host's behalf. - reports.addr_source + addr_precision, so a landmark line can render as "Near X" instead of posing as a postal address. Not backfilled: for a legacy row there is no way to tell the reporter's text from the server's snapshot. No new index on a hot table - both reports columns are plain row columns. The CHECKs are NOT VALID (new columns hold by construction) and are recorded in the runbook's outstanding-VALIDATE set. Deviation from the design sketch: the cache column is address_precision, not precision, and address/address_precision are nullable to carry a negative entry. Co-Authored-By: Claude Opus 5 (1M context) --- ...2026-07-24-full-backend-security-review.md | 10 +- .../api/drizzle/0175_address_resolution.sql | 160 ++++++++++++++++++ services/api/src/db/schema/cleanups.ts | 3 + services/api/src/db/schema/geocode_cache.ts | 25 +++ services/api/src/db/schema/index.ts | 1 + services/api/src/db/schema/reports.ts | 5 + 6 files changed, 200 insertions(+), 4 deletions(-) create mode 100644 services/api/drizzle/0175_address_resolution.sql create mode 100644 services/api/src/db/schema/geocode_cache.ts diff --git a/docs/security/2026-07-24-full-backend-security-review.md b/docs/security/2026-07-24-full-backend-security-review.md index d5c94fb1..fb5e1073 100644 --- a/docs/security/2026-07-24-full-backend-security-review.md +++ b/docs/security/2026-07-24-full-backend-security-review.md @@ -322,7 +322,7 @@ Everything an operator has to do by hand, in order, plus the two infra facts and node dist/db/migrate.js # == pnpm --filter @civfix/api db:migrate ``` -`drizzle/` holds **157 files**, `0000_extensions.sql` … `0174_event_announcements.sql`. The nine rows +`drizzle/` holds **158 files**, `0000_extensions.sql` … `0175_address_resolution.sql`. The nine rows below are exactly what this change set adds — `0052`–`0060`, contiguous, no gaps — and everything from `0000` through `0051_social_posts.sql` predates it. (`0060` arrived later than the rest, with the feed redesign; it is listed here because this table is the single operator runbook. `0061`–`0064` arrived @@ -564,6 +564,7 @@ foreign key into `organizations` from `0105`. | `0173_posts_author_public_recent_idx.sql` | adds the partial index `posts_author_public_recent_idx (author_id, created_at DESC, id DESC) WHERE deleted_at IS NULL AND reply_to_id IS NULL AND visibility = 'public'` - the author-keyed twin of `0071_posts_toplevel_recent_idx`, backing the ranked feed's in-network candidate pool (self + followees). Neither `posts_author_created_idx` (0051, not partial on reply_to_id/visibility, no id tiebreak) nor `posts_toplevel_recent_idx` (0071, not author-keyed) serves that shape. Same not-a-hot-table reasoning as 0172 | The in-network pool sorts instead of stopping at its LIMIT. Correct, slower; no functional change | | `0174_event_announcements.sql` | widens `broadcasts_kind_check` to accept the new `announcement` kind (event announcements ride the existing host-broadcast pipeline rather than a parallel table), adds the partial `broadcasts_announcement_public_idx (cleanup_id, created_at DESC, id DESC) WHERE kind = 'announcement'` that backs the public per-event announcements list, and re-creates `broadcasts_scrub_idx` with `kind <> 'announcement'` so the retention scrub can never NULL an announcement body - an announcement is permanent public event content rendered on the event page, not a one-shot email. `broadcasts` is NOT on the hot-table list in `docs/out-of-band-indexes.md`, so both indexes build inline; `IF NOT EXISTS` makes an out-of-band CONCURRENTLY build a no-op | Every `createEventAnnouncement` fails on the CHECK constraint. Applying the CHECK but not the scrub-index swap still works (the repository query carries the same predicate), it just scans more rows | +| `0175_address_resolution.sql` | the address-resolution overhaul: creates `geocode_cache` (a read-through cache of reverse geocodes keyed by the shared 5-decimal point key, with its `geocode_cache_resolved_at_idx`; derived public data only, no user/report/event reference, TTL applied on read rather than by a cron), adds `cleanups.address_source` (resolved | edited | manual) and backfills it to `manual` for every event that already has an address, and adds `reports.addr_source` + `reports.addr_precision`. All three CHECK constraints are `NOT VALID` (new columns, so they hold by construction) and are listed in the outstanding-VALIDATE set below. No new index on a hot table: both `reports` columns are plain row columns | `POST /map/resolve-address` and every create path still answer, but uncached (the cache swallows its own errors) and with no provenance persisted; a new client's event create fails on the missing `cleanups.address_source` column | **Deferred to the NEXT release** (expand/contract, `docs/migrations-expand-contract.md`): 0.43.0 @@ -593,9 +594,10 @@ either way — `org_verifications` documents are claimed with it. **Deferred to a later release, out of band** (record them in the expand/contract ledger): the six `NOT VALID` CHECKs `0107` adds on `cleanups` and `0110`'s `media_assets_purpose_expanded` still need `ALTER TABLE … VALIDATE CONSTRAINT`, and so do `0164`'s -`users_primary_organization_fk` and `posts_organization_fk`. They are correct as `NOT VALID` — the -constraint is enforced for every new row — and validating takes a scan that does not belong in a -deploy transaction. +`users_primary_organization_fk` and `posts_organization_fk`, and `0175`'s +`cleanups_address_source_chk`, `reports_addr_source_chk` and `reports_addr_precision_chk`. They are +correct as `NOT VALID` — the constraint is enforced for every new row — and validating takes a scan +that does not belong in a deploy transaction. | `0096_cleanup_guests.sql` | guest event RSVP (contract 0.38.0): `cleanup_guests` (event-scoped, contact-bearing attendance rows; SHA-256 manage-token hash only; partial unique on the active `(cleanup_id, contact_key)`), `guest_otps` (event-scoped one-time codes, hash only), `sms_opt_outs` (STOP suppression list, kept indefinitely). Three brand-new empty tables; no existing table touched | `POST /v1/cleanups/:id/guest-rsvp/*` and `GET /v1/cleanups/:id/guests` 500 on a missing relation, and `going` cannot include guests | | `0097_media_assets_served_key.sql` | adds `media_assets.served_key` (audit C1): the worker-owned key the processed object is published to, so the client-writable upload key is never served after `ready`. Catalog-only nullable `ADD COLUMN` on a hot table. **Post-deploy:** run `node dist/db/backfill-served-key.js` immediately and again ~20 min later; pre-existing `ready` media reads as not-found until it completes | Every existing photo 404s until the backfill runs; new uploads work | | `0098_sweep_predicate_indexes.sql` | idempotent guard that warns while `media_assets_orphan_sweep_idx` is missing (hot table; build it out of band with `CREATE INDEX CONCURRENTLY`, see `docs/out-of-band-indexes.md`) plus two small inline `email_otps` indexes (`expires_at`, partial `consumed_at`) backing the OTP retention lane (audit H13/M) | Both sweeps sequential-scan; correctness unaffected | diff --git a/services/api/drizzle/0175_address_resolution.sql b/services/api/drizzle/0175_address_resolution.sql new file mode 100644 index 00000000..6cfba082 --- /dev/null +++ b/services/api/drizzle/0175_address_resolution.sql @@ -0,0 +1,160 @@ +-- ============================================================================= +-- 0175_address_resolution.sql +-- ----------------------------------------------------------------------------- +-- The address-resolution overhaul. Three concerns, one theme: a pin now yields a +-- HONEST address line, and every stored address carries where it came from. +-- +-- 1. `geocode_cache` (new). Reverse geocodes are stable per point, and the +-- same point is resolved several times over one creation flow (the client +-- previews it while the pin settles, then the create path resolves it +-- again server-side). This read-through cache collapses all of that onto +-- one row keyed by the SHARED 5-decimal point key +-- (@civfix/shared `geocodePointKey`, ~1.1 m), which is also the key the +-- client rounds its preview query on -- so a pin fine-tune hits one row +-- rather than minting a new provider call per micro-drag. +-- +-- TTLs are enforced in SQL-adjacent code (services/geocode-cache.ts), not +-- by a cron: a row past its TTL is simply read as a MISS and overwritten by +-- the fresh resolve. 180 days for a resolved address; 15 minutes when the +-- resolve produced nothing. The short negative TTL is the compromise the +-- design spec asks for from both ends: caching nulls at all is what stops a +-- dragged pin from hammering a provider that is currently down, and +-- expiring them in minutes is what stops a provider OUTAGE from poisoning +-- those points for half a year. +-- +-- DEVIATION from the design spec's sketch, deliberate: the column is +-- `address_precision`, not `precision` (`precision` is a Postgres keyword; +-- non-reserved, so it would parse, but this table is read by hand during +-- incidents and there is no upside to the ambiguity), and `address` / +-- `address_precision` are NULLABLE to carry a negative entry. +-- +-- NOT user data. Every row is a public coordinate -> public address line, +-- derived from a third-party geocoder. It holds no reference to the user, +-- report or event that caused the lookup, so it is outside the erasure lane +-- by construction (docs/erasure-behavior.md). +-- +-- 2. `cleanups.address_source`. Events were NEVER reverse-geocoded: the column +-- held whatever the host typed into "name the spot", or NULL. New clients +-- now confirm a resolved line, so the column needs provenance -- +-- 'resolved' (shown to the host unchanged), 'edited' (host amended the +-- resolved line) or 'manual' (host typed it because resolution failed or +-- was locality-grade). Backfilled to 'manual' for every existing event that +-- has an address, which is the closest truth: a host typed it. +-- +-- 3. `reports.addr_source` + `reports.addr_precision`. Reports already +-- snapshot an address at creation, either the reporter's own text or the +-- server's reverse geocode, and nothing recorded WHICH. 'user' vs +-- 'resolved' plus the precision rung the geocoder actually reached lets the +-- display layer be honest (a `landmark` line renders with a "Near " prefix +-- instead of posing as a postal address) and leaves a future +-- location-coarsening policy implementable as a display rule +-- (docs/location-coarsening-assessment.md) rather than a re-geocode. +-- +-- NO NEW INDEX ON A HOT TABLE. `reports` IS on the hot-table list in +-- docs/out-of-band-indexes.md; both of its new columns are plain row columns +-- read only alongside the row itself, so nothing here needs an index. The one +-- index created below is on `geocode_cache`, a table this file creates empty. +-- +-- CHECK constraints are added NOT VALID (`reports` is hot; `cleanups` follows +-- the same shape for consistency, as 0170 did). Every existing row has NULL in +-- these columns -- the columns are created by this file -- so each constraint +-- holds by construction from the instant it exists, and every INSERT/UPDATE is +-- checked from now on. +-- +-- STILL OUTSTANDING (a later release, out of band, SHARE UPDATE EXCLUSIVE): +-- ALTER TABLE cleanups VALIDATE CONSTRAINT cleanups_address_source_chk; +-- ALTER TABLE reports VALIDATE CONSTRAINT reports_addr_source_chk; +-- ALTER TABLE reports VALIDATE CONSTRAINT reports_addr_precision_chk; +-- +-- CANONICAL DDL: hand-authored source of truth. Mirrors: schema/geocode_cache.ts +-- (new), schema/cleanups.ts, schema/reports.ts. Enum mirrors: shared +-- AddressPrecisionSchema / EventAddressSourceSchema / ReportAddressSourceSchema. +-- +-- Conventions: additive ADD COLUMN IF NOT EXISTS; one concern per file; one +-- transaction per file. Forward-only, no down. +-- +-- Ordering rules: requires 0001_core.sql (reports, cleanups). +-- ============================================================================= + +CREATE TABLE IF NOT EXISTS geocode_cache ( + point_key text PRIMARY KEY, + address text, + address_precision text, + city_state_label text NOT NULL DEFAULT '', + provider text, + resolved_at timestamptz NOT NULL DEFAULT now() +); + +COMMENT ON TABLE geocode_cache IS + 'Read-through cache of reverse geocodes, keyed by the shared 5-decimal point key (@civfix/shared geocodePointKey). Derived public data only - no user, report or event reference. Rows past their TTL are read as misses and overwritten in place; see services/geocode-cache.ts.'; + +COMMENT ON COLUMN geocode_cache.address IS + 'The resolved one-line address, or NULL for a negative entry (the providers returned nothing). A landmark line is stored RAW - the "Near " prefix is a localized display concern.'; + +COMMENT ON COLUMN geocode_cache.address_precision IS + 'How far down the ladder the resolve got: street | intersection | landmark | locality. NULL on a negative entry. Named address_precision, not precision, to keep the column unambiguous in hand-written incident queries.'; + +COMMENT ON COLUMN geocode_cache.provider IS + 'Which adapter answered: mapbox | photon | tiger. NULL on a negative entry. Diagnostics only - never served to a client.'; + +CREATE INDEX IF NOT EXISTS geocode_cache_resolved_at_idx ON geocode_cache (resolved_at); + +ALTER TABLE cleanups ADD COLUMN IF NOT EXISTS address_source text; + +COMMENT ON COLUMN cleanups.address_source IS + 'Provenance of cleanups.address: resolved (host confirmed the reverse-geocoded line) | edited (host amended it) | manual (host typed it). NULL only for legacy rows with no address at all.'; + +ALTER TABLE reports ADD COLUMN IF NOT EXISTS addr_source text; +ALTER TABLE reports ADD COLUMN IF NOT EXISTS addr_precision text; + +COMMENT ON COLUMN reports.addr_source IS + 'Provenance of reports.addr: user (the reporter typed it) | resolved (the server reverse-geocoded the pin at creation). NULL for legacy rows and for reports with no address.'; + +COMMENT ON COLUMN reports.addr_precision IS + 'Ladder rung the geocoder reached for a resolved addr: street | intersection | landmark | locality. Always NULL when addr_source is user - the reporter''s own text carries no provider precision.'; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'cleanups_address_source_chk' AND conrelid = 'cleanups'::regclass + ) THEN + ALTER TABLE cleanups + ADD CONSTRAINT cleanups_address_source_chk + CHECK (address_source IS NULL OR address_source IN ('resolved', 'edited', 'manual')) NOT VALID; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'reports_addr_source_chk' AND conrelid = 'reports'::regclass + ) THEN + ALTER TABLE reports + ADD CONSTRAINT reports_addr_source_chk + CHECK (addr_source IS NULL OR addr_source IN ('user', 'resolved')) NOT VALID; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'reports_addr_precision_chk' AND conrelid = 'reports'::regclass + ) THEN + ALTER TABLE reports + ADD CONSTRAINT reports_addr_precision_chk + CHECK ( + addr_precision IS NULL + OR addr_precision IN ('street', 'intersection', 'landmark', 'locality') + ) NOT VALID; + END IF; +END $$; + +-- Backfill, inline and deliberately: `cleanups` is small (one row per event, not +-- per interaction) and is absent from the hot-table list, so this single UPDATE +-- inside the runner's transaction is not the lock hazard the same statement +-- would be on `reports` or `posts`. Idempotent via the IS NULL predicate. +-- +-- Existing events with address IS NULL get NOTHING here on purpose: host +-- verification is the whole feature, and the server cannot verify on a host's +-- behalf. Those events keep the client's "Meeting point" fallback and the host +-- is nudged to add one. Likewise `reports.addr_source` is NOT backfilled - for a +-- legacy row there is no way to tell the reporter's text from the server's +-- snapshot, and guessing would be worse than NULL. +UPDATE cleanups SET address_source = 'manual' WHERE address IS NOT NULL AND address_source IS NULL; diff --git a/services/api/src/db/schema/cleanups.ts b/services/api/src/db/schema/cleanups.ts index 512a59a5..d5323fe1 100644 --- a/services/api/src/db/schema/cleanups.ts +++ b/services/api/src/db/schema/cleanups.ts @@ -21,6 +21,7 @@ import { type EVENT_KIND_VALUES, } from "./types.js" import type { EVENT_VISIBILITY_VALUES } from "./types-host.js" +import type { EventAddressSource } from "@civfix/shared" type CleanupType = (typeof CLEANUP_TYPE_VALUES)[number] type CleanupStatus = (typeof CLEANUP_STATUS_VALUES)[number] @@ -45,6 +46,8 @@ export const cleanups = pgTable( status: text("status").$type().notNull(), bring: text("bring").array(), address: text("address"), + /** 0175: where `address` came from - resolved | edited | manual. NULL only for addressless legacy rows. */ + addressSource: text("address_source").$type(), capacity: integer("capacity"), bags: integer("bags").notNull().default(0), referenceCode: text("reference_code"), diff --git a/services/api/src/db/schema/geocode_cache.ts b/services/api/src/db/schema/geocode_cache.ts new file mode 100644 index 00000000..d2be7079 --- /dev/null +++ b/services/api/src/db/schema/geocode_cache.ts @@ -0,0 +1,25 @@ +import { index, pgTable, text, timestamp } from "drizzle-orm/pg-core" +import type { AddressPrecision } from "@civfix/shared" + +/** + * Mirror of drizzle/0175_address_resolution.sql. Read-through cache of reverse geocodes keyed by the + * shared 5-decimal point key (@civfix/shared `geocodePointKey`). A row with a NULL `address` is a + * NEGATIVE entry: the providers answered nothing, and it expires on a much shorter TTL than a hit so a + * provider outage cannot poison the point. TTL is applied on read (services/geocode-cache.ts), never by + * a cron. + */ +export const geocodeCache = pgTable( + "geocode_cache", + { + pointKey: text("point_key").primaryKey(), + address: text("address"), + addressPrecision: text("address_precision").$type(), + cityStateLabel: text("city_state_label").notNull().default(""), + provider: text("provider"), + resolvedAt: timestamp("resolved_at", { withTimezone: true }).notNull().defaultNow(), + }, + (t) => [index("geocode_cache_resolved_at_idx").on(t.resolvedAt)], +) + +export type GeocodeCacheRow = typeof geocodeCache.$inferSelect +export type NewGeocodeCacheRow = typeof geocodeCache.$inferInsert diff --git a/services/api/src/db/schema/index.ts b/services/api/src/db/schema/index.ts index 05bae25d..f5fb09d1 100644 --- a/services/api/src/db/schema/index.ts +++ b/services/api/src/db/schema/index.ts @@ -8,6 +8,7 @@ export * from "./sessions.js" export * from "./jurisdictions.js" export * from "./boundary_vintage.js" +export * from "./geocode_cache.js" export * from "./reports.js" export * from "./media.js" export * from "./timeline.js" diff --git a/services/api/src/db/schema/reports.ts b/services/api/src/db/schema/reports.ts index 52ac327a..7f1b7cc1 100644 --- a/services/api/src/db/schema/reports.ts +++ b/services/api/src/db/schema/reports.ts @@ -1,6 +1,7 @@ import { sql } from "drizzle-orm" import { index, pgTable, text, timestamp, uniqueIndex, uuid } from "drizzle-orm/pg-core" +import type { AddressPrecision, ReportAddressSource } from "@civfix/shared" import { jurisdictions } from "./jurisdictions.js" import { users } from "./users.js" import { @@ -35,6 +36,10 @@ export const reports = pgTable( title: text("title"), description: text("description"), addr: text("addr"), + /** 0175: where `addr` came from. NULL for legacy rows and for reports with no address. */ + addrSource: text("addr_source").$type(), + /** 0175: the ladder rung a RESOLVED addr reached. Always NULL when addrSource is 'user'. */ + addrPrecision: text("addr_precision").$type(), status: text("status").$type().notNull(), visibility: text("visibility").$type().notNull().default("public"), h3Cell: text("h3_cell").notNull(), From 024fdb3969210f27ea31d7fc37c1cc6c9d146e58 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 18:57:23 -0700 Subject: [PATCH 29/54] structured reverse geocoding: a precision ladder, behind a cache The reverse seam collapsed every answer to one string, so a caller could not tell "123 Main St" from "Los Angeles" and the creation flows had to treat both the same. It now returns { line, precision, provider }, and the adapters compose the ladder the data actually supports: - Photon takes one request (limit=5, radius=50m) and walks it: exact house number -> cross street composed from two distinct named roads within 40m -> the bare street (never an invented number) -> a named public feature within 60m. A residential building or place=house is skipped outright: those carry occupant names often enough that surfacing one would publish a private residence label next to a pin. - Mapbox claims street only with a house number and intersection for a named street, and returns null for anything coarser rather than short-circuiting the chain with a city name. - Neither adapter ever claims `locality`. That rung belongs to the local TIGER geocoder, which is authoritative for it and needs no network. A landmark line is stored RAW - the "Near " prefix is a localized display concern. makeAddressResolver composes the chain, the TIGER label and the read-through cache into the one shape every caller takes, and never throws: a provider that blows up, a cache that is unreachable or a TIGER query that fails each cost one rung, never a failed submit. chainReverse is now generic over the answer shape; its budget-sharing scheduling is untouched. Co-Authored-By: Claude Opus 5 (1M context) --- .../api/src/adapters/reverse-geocode.chain.ts | 37 +++- .../src/adapters/reverse-geocode.mapbox.ts | 56 +++++- .../src/adapters/reverse-geocode.photon.ts | 190 ++++++++++++++++-- services/api/src/services/address-resolver.ts | 142 +++++++++++++ services/api/src/services/geocode-cache.ts | 126 ++++++++++++ .../api/src/services/route-geo-helpers.ts | 25 ++- ...pters-reverse-geocode-chain-budget.test.ts | 11 +- .../test/unit/reverse-geocode-chain.test.ts | 13 +- .../test/unit/reverse-geocode-mapbox.test.ts | 64 +++++- .../test/unit/reverse-geocode-photon.test.ts | 180 ++++++++++++++++- 10 files changed, 795 insertions(+), 49 deletions(-) create mode 100644 services/api/src/services/address-resolver.ts create mode 100644 services/api/src/services/geocode-cache.ts diff --git a/services/api/src/adapters/reverse-geocode.chain.ts b/services/api/src/adapters/reverse-geocode.chain.ts index 8004d238..c280fc88 100644 --- a/services/api/src/adapters/reverse-geocode.chain.ts +++ b/services/api/src/adapters/reverse-geocode.chain.ts @@ -1,5 +1,24 @@ -/** A street-level reverse geocoder: coords -> one-line address, or null. Photon + Mapbox conform. */ -export type ReverseGeocode = (lat: number, lng: number) => Promise +import type { AddressPrecision } from "@civfix/shared" + +/** Any "answer a question about a point, or don't" provider. Chainable by chainReverse. */ +export type PointResolver = (lat: number, lng: number) => Promise + +/** + * One rung of the precision ladder, as the provider actually reached it. `line` is the display address + * WITHOUT any localized decoration: a `landmark` result carries the bare feature name plus locality + * ("Vista Hermosa Park, Los Angeles, CA") and the "Near " prefix is added by the UI, which is the only + * layer that knows the viewer's language. Never claim a rung the data does not support - a street with + * no house number is `intersection`, not `street`. + */ +export interface ReverseResult { + line: string + precision: AddressPrecision + /** Which adapter answered. Diagnostics and cache bookkeeping only; never served to a client. */ + provider: string +} + +/** A street-level reverse geocoder: coords -> a structured address rung, or null. Photon + Mapbox conform. */ +export type ReverseGeocode = PointResolver /** * Overall budget for the whole chain. Each provider promises "never delays a submit" with its own ~4s @@ -28,9 +47,11 @@ const CHAIN_BUDGET_MS = 5000 * A provider abandoned at its share keeps running (its own timeout ends it); its late answer is discarded * and a late rejection is swallowed, so it can never surface as an unhandled rejection. */ -export function chainReverse(...providers: Array): ReverseGeocode { - const active = providers.filter((p): p is ReverseGeocode => typeof p === "function") - return async (lat: number, lng: number): Promise => { +export function chainReverse( + ...providers: Array | null | undefined> +): PointResolver { + const active = providers.filter((p): p is PointResolver => typeof p === "function") + return async (lat: number, lng: number): Promise => { if (active.length === 0) return null const deadline = Date.now() + CHAIN_BUDGET_MS @@ -51,12 +72,12 @@ export function chainReverse(...providers: Array( + provider: PointResolver, lat: number, lng: number, timeoutMs: number, -): Promise { +): Promise { const attempt = (async () => provider(lat, lng))().catch(() => null) let timer: ReturnType | undefined const budget = new Promise((resolve) => { diff --git a/services/api/src/adapters/reverse-geocode.mapbox.ts b/services/api/src/adapters/reverse-geocode.mapbox.ts index 677ebd2d..fdccfaba 100644 --- a/services/api/src/adapters/reverse-geocode.mapbox.ts +++ b/services/api/src/adapters/reverse-geocode.mapbox.ts @@ -1,17 +1,38 @@ -import type { ReverseGeocode } from "./reverse-geocode.chain.js" +/** + * Mapbox reverse geocoder - the OPT-IN primary. Selected only when MAPBOX_TOKEN is configured (di.ts); + * with no token the chain is Photon-only and everything still works, one ladder rung lower on average. + * Mapbox interpolates rooftop/parcel addresses where OSM simply has no house number, which is the single + * biggest quality lever available here, but it is a spend decision, so nothing requires it. + * + * Same contract as every provider on this seam: never throws, never blocks, null on any failure. + * + * PRECISION. Mapbox will happily answer a coordinate in the middle of nowhere with the city name. That + * answer is worse than useless HERE: returning it would claim a rung this adapter did not reach and, far + * worse, short-circuit the rest of the chain with the least specific line available. So this adapter + * returns only what it can prove - + * + * street a house number is present (context.address.address_number, or a name that starts + * with one) "123 Main St, Inglewood, CA" + * intersection a named street but no number "Main St, Inglewood, CA" + * + * - and hands anything coarser to the next provider, ending at the local TIGER `locality` label. + */ + +import type { AddressPrecision } from "@civfix/shared" +import type { ReverseGeocode, ReverseResult } from "./reverse-geocode.chain.js" import { fetchJsonOrNull } from "./http-fetch.js" const MAPBOX_REVERSE_URL = "https://api.mapbox.com/search/geocode/v6/reverse" const DEFAULT_TIMEOUT_MS = 4000 interface MapboxReverseContext { - address?: { name?: string } + address?: { name?: string; address_number?: string; street_name?: string } street?: { name?: string } place?: { name?: string } region?: { name?: string; region_code?: string } country?: { name?: string; country_code?: string } } -interface MapboxReverseProps { +export interface MapboxReverseProps { name?: string full_address?: string place_formatted?: string @@ -39,6 +60,27 @@ export function formatMapboxReverse(p: MapboxReverseProps): string | null { return [primary, ...tail].join(", ") } +/** A leading house number is the only thing that earns the `street` rung. */ +function startsWithHouseNumber(name: string | undefined): boolean { + return name !== undefined && /^\d/.test(name.trim()) +} + +/** + * The rung this feature proves, or null when it proves nothing better than a locality (which this + * adapter never claims - the chain's last provider owns that rung). + */ +export function mapboxPrecision(p: MapboxReverseProps): AddressPrecision | null { + const ctx = p.context ?? {} + const hasNumber = + (ctx.address?.address_number?.trim().length ?? 0) > 0 || + startsWithHouseNumber(ctx.address?.name) || + startsWithHouseNumber(p.name) + if (hasNumber) return "street" + const named = ctx.street?.name ?? ctx.address?.street_name ?? null + if (named !== null && named.trim().length > 0) return "intersection" + return null +} + export function redactMapboxToken(url: string): string { return url.replace(/([?&]access_token=)[^&#\s]*/gi, "$1[redacted]") } @@ -48,7 +90,7 @@ export function makeMapboxReverseGeocode(opts: MapboxReverseOptions): ReverseGeo const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS // Left undefined when not injected so the helper resolves globalThis.fetch at CALL time. const doFetch = opts.fetchImpl - return async (lat: number, lng: number): Promise => { + return async (lat: number, lng: number): Promise => { if (!Number.isFinite(lat) || !Number.isFinite(lng)) return null let url: string try { @@ -69,6 +111,10 @@ export function makeMapboxReverseGeocode(opts: MapboxReverseOptions): ReverseGeo init: { headers: { Accept: "application/json" } }, }) const props = data?.features?.[0]?.properties - return props ? formatMapboxReverse(props) : null + if (!props) return null + const precision = mapboxPrecision(props) + if (precision === null) return null + const line = formatMapboxReverse(props) + return line === null ? null : { line, precision, provider: "mapbox" } } } diff --git a/services/api/src/adapters/reverse-geocode.photon.ts b/services/api/src/adapters/reverse-geocode.photon.ts index 26bedeca..cfc28da4 100644 --- a/services/api/src/adapters/reverse-geocode.photon.ts +++ b/services/api/src/adapters/reverse-geocode.photon.ts @@ -1,22 +1,62 @@ /** * Best-effort street-level reverse geocoder backed by the public Photon instance (the same service the - * clients use for FORWARD address autocomplete). Turns a report's pin into a one-line human address - * ("123 Imperial Hwy, Inglewood, CA") so the report list/detail show a location label even when the - * reporter never typed an address. + * clients use for FORWARD address autocomplete). Turns a pin into a one-line human address + * ("123 Imperial Hwy, Inglewood, CA") so a report or event shows a real location instead of "City, ST". * * Contract: NEVER throws and NEVER blocks. Any failure (network, HTTP error, timeout, no result, bad * coords) resolves to `null` so the caller falls back (to the local City, ST label) or simply stores no * address - report creation must not depend on an external geocoder being reachable. + * + * PRECISION LADDER. Photon/OSM chronically lacks house numbers, so a single "nearest feature" answer is + * either exact or wildly overconfident. One request (limit=5, radius=50 m) buys the whole ladder: + * + * street house number + street on one feature "123 Main St, Inglewood, CA" + * intersection two distinct named roads within ~40 m "Main St & 5th Ave, Inglewood, CA" + * (one road only -> the bare street) "Main St, Inglewood, CA" + * landmark a named POI within ~60 m and no road "Vista Hermosa Park, Los Angeles, CA" + * + * and `locality` is never produced here - that rung belongs to the local TIGER geocoder, which is + * authoritative for it and needs no network. Returning a city name from THIS adapter would also + * short-circuit the chain's remaining providers with the worst answer available. + * + * The landmark rung carries a hard guard: a residential building or `place=house` feature is skipped + * outright. Those carry occupant or family names in OSM often enough that surfacing one would publish a + * private residence label next to a public pin, and the line degrades to the road or to locality with no + * loss the reader can even perceive. */ +import type { AddressPrecision } from "@civfix/shared" +import type { ReverseGeocode, ReverseResult } from "./reverse-geocode.chain.js" import { fetchJsonOrNull } from "./http-fetch.js" /** Public Photon reverse endpoint. Self-host + override via the factory `url` for higher volume. */ const PHOTON_REVERSE_URL = "https://photon.komoot.io/reverse" /** Cap the geocode so a slow/hung Photon never delays a report submit. */ const DEFAULT_TIMEOUT_MS = 4000 +/** Features per request. Enough to see a cross street and a POI beside the nearest hit; still one call. */ +const REVERSE_LIMIT = 5 +/** Photon's search radius, in KILOMETRES. 50 m bounds the whole candidate set to "at this pin". */ +const REVERSE_RADIUS_KM = 0.05 +/** Two roads only compose an intersection when BOTH are essentially at the pin. */ +const INTERSECTION_RADIUS_M = 40 +/** A landmark may sit a little further out - a park or plaza entrance is not its centroid. */ +const LANDMARK_RADIUS_M = 60 + +const EARTH_RADIUS_M = 6_371_000 + +/** OSM keys whose named features are real public landmarks worth naming beside a pin. */ +const LANDMARK_KEYS = new Set([ + "amenity", + "leisure", + "tourism", + "shop", + "historic", + "natural", + "railway", + "aeroway", +]) -interface PhotonReverseProps { +export interface PhotonReverseProps { name?: string housenumber?: string street?: string @@ -24,25 +64,133 @@ interface PhotonReverseProps { district?: string state?: string country?: string + osm_key?: string + osm_value?: string +} + +interface PhotonFeature { + properties?: PhotonReverseProps + geometry?: { coordinates?: [number, number] } +} + +/** Metres between two WGS84 points (haversine). Only ever used at sub-kilometre scale. */ +export function distanceMeters( + a: { lat: number; lng: number }, + b: { lat: number; lng: number }, +): number { + const toRad = (d: number): number => (d * Math.PI) / 180 + const dLat = toRad(b.lat - a.lat) + const dLng = toRad(b.lng - a.lng) + const s = + Math.sin(dLat / 2) ** 2 + + Math.cos(toRad(a.lat)) * Math.cos(toRad(b.lat)) * Math.sin(dLng / 2) ** 2 + return 2 * EARTH_RADIUS_M * Math.asin(Math.min(1, Math.sqrt(s))) +} + +/** The ", City, ST" tail, skipping whatever is already the primary and dropping a US country. */ +function localityTail(p: PhotonReverseProps, primary: string): string[] { + return [ + p.city && p.city !== primary ? p.city : null, + p.state, + p.country && p.country !== "United States" && p.country !== "United States of America" + ? p.country + : null, + ].filter((v): v is string => !!v) } /** * Format a Photon reverse feature into a single human address line. Prefers a " " line, * else the place `name`/district, then appends the city/state (and country when it is not the US) - * skipping any part already used as the primary. Returns null when nothing usable is present. + * + * Kept exported and unchanged in shape: it is the formatter for a SINGLE feature, which the ladder below + * composes with. It deliberately makes no precision claim of its own. */ export function formatPhotonReverse(p: PhotonReverseProps): string | null { const street = [p.housenumber, p.street].filter(Boolean).join(" ").trim() const primary = street || p.name || p.district || p.city || "" if (!primary) return null - const tail = [ - p.city && p.city !== primary ? p.city : null, - p.state, - p.country && p.country !== "United States" && p.country !== "United States of America" - ? p.country - : null, - ].filter((v): v is string => !!v) - return [primary, ...tail].join(", ") + return [primary, ...localityTail(p, primary)].join(", ") +} + +/** True when a feature's `name` could be a private residence label rather than a public landmark. */ +export function isResidentialName(p: PhotonReverseProps): boolean { + if (p.osm_key === "building") return true + if (p.osm_key === "place" && (p.osm_value === "house" || p.osm_value === "farm")) return true + return false +} + +function namedRoad(p: PhotonReverseProps): string | null { + if (p.osm_key !== "highway") return null + const name = p.street ?? p.name ?? null + return name && name.trim().length > 0 ? name.trim() : null +} + +function landmarkName(p: PhotonReverseProps): string | null { + if (isResidentialName(p)) return null + if (p.osm_key === undefined || !LANDMARK_KEYS.has(p.osm_key)) return null + const name = p.name?.trim() + return name && name.length > 0 ? name : null +} + +interface Candidate { + props: PhotonReverseProps + meters: number +} + +/** + * Compose the ladder from one Photon response. Exported so the rungs and their guards are unit-testable + * without a fetch: every branch here is a product decision, not plumbing. + */ +export function composePhotonReverse( + features: PhotonFeature[], + at: { lat: number; lng: number }, +): { line: string; precision: AddressPrecision } | null { + const candidates: Candidate[] = [] + for (const f of features) { + const props = f.properties + if (!props) continue + const coords = f.geometry?.coordinates + const meters = + Array.isArray(coords) && coords.length === 2 && coords.every((n) => Number.isFinite(n)) + ? distanceMeters(at, { lng: coords[0], lat: coords[1] }) + : Number.POSITIVE_INFINITY + candidates.push({ props, meters }) + } + candidates.sort((a, b) => a.meters - b.meters) + + const exact = candidates.find( + (c) => !!c.props.housenumber?.trim() && !!c.props.street?.trim(), + ) + if (exact) { + const line = formatPhotonReverse(exact.props) + if (line) return { line, precision: "street" } + } + + const roads: string[] = [] + let roadProps: PhotonReverseProps | null = null + for (const c of candidates) { + if (c.meters > INTERSECTION_RADIUS_M) continue + const road = namedRoad(c.props) + if (road === null || roads.includes(road)) continue + roads.push(road) + roadProps ??= c.props + } + // A house number is what separates `street` from `intersection`; the pin sits ON a road either way, so + // even a single named road is a materially better answer than the city - it just never claims a number. + if (roadProps !== null && roads[0] !== undefined) { + const primary = roads.length >= 2 ? `${roads[0]} & ${roads[1]}` : roads[0] + return { line: [primary, ...localityTail(roadProps, primary)].join(", "), precision: "intersection" } + } + + for (const c of candidates) { + if (c.meters > LANDMARK_RADIUS_M) continue + const name = landmarkName(c.props) + if (name === null) continue + return { line: [name, ...localityTail(c.props, name)].join(", "), precision: "landmark" } + } + + return null } export interface PhotonReverseOptions { @@ -55,18 +203,16 @@ export interface PhotonReverseOptions { } /** - * Build a `(lat, lng) => Promise` reverse geocoder. Returns null on ANY failure so it is - * safe to call in the report-create hot path (the caller treats null as "no address"). + * Build a reverse geocoder returning the structured ladder result, or null on ANY failure so it is safe + * to call in the report-create hot path (the caller treats null as "no address"). */ -export function makePhotonReverseGeocode( - opts: PhotonReverseOptions = {}, -): (lat: number, lng: number) => Promise { +export function makePhotonReverseGeocode(opts: PhotonReverseOptions = {}): ReverseGeocode { const baseUrl = opts.url ?? PHOTON_REVERSE_URL const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS // Left undefined when not injected so the helper resolves globalThis.fetch at CALL time. const doFetch = opts.fetchImpl - return async (lat: number, lng: number): Promise => { + return async (lat: number, lng: number): Promise => { if (!Number.isFinite(lat) || !Number.isFinite(lng)) return null let url: string try { @@ -74,6 +220,8 @@ export function makePhotonReverseGeocode( u.searchParams.set("lat", String(lat)) u.searchParams.set("lon", String(lng)) u.searchParams.set("lang", "en") + u.searchParams.set("limit", String(REVERSE_LIMIT)) + u.searchParams.set("radius", String(REVERSE_RADIUS_KM)) url = u.toString() } catch { // A misconfigured base URL must not throw out of a best-effort geocode. @@ -81,12 +229,12 @@ export function makePhotonReverseGeocode( } // Network / abort / non-2xx / parse failure all degrade to null (the caller falls back or stores no // address); redirect:"error" (the helper's default) keeps a MITM Photon from 30x-ing us internally. - const data = await fetchJsonOrNull<{ features?: { properties?: PhotonReverseProps }[] }>(url, { + const data = await fetchJsonOrNull<{ features?: PhotonFeature[] }>(url, { timeoutMs, ...(doFetch !== undefined ? { fetchImpl: doFetch } : {}), init: { headers: { Accept: "application/json" } }, }) - const props = data?.features?.[0]?.properties - return props ? formatPhotonReverse(props) : null + const composed = composePhotonReverse(data?.features ?? [], { lat, lng }) + return composed === null ? null : { ...composed, provider: "photon" } } } diff --git a/services/api/src/services/address-resolver.ts b/services/api/src/services/address-resolver.ts new file mode 100644 index 00000000..50da4b06 --- /dev/null +++ b/services/api/src/services/address-resolver.ts @@ -0,0 +1,142 @@ +/** + * The one place a coordinate becomes an address. + * + * Before this existed there were two answers to "what is at this point" and they disagreed: reports got + * the street-level chain collapsed to a bare string, events got nothing at all, and the only + * client-callable endpoint returned TIGER's "City, ST". This composes all of it once, honestly, and + * hands every caller the same shape. + * + * THE LADDER, top rung first: + * + * street | intersection | landmark whatever the provider chain proved (Mapbox when configured, then + * Photon). The adapters never claim a rung they did not reach. + * locality the local TIGER "City, ST" label. Always computed, because the + * contract promises `cityStateLabel` on every response and the + * creation flows show it as a hint even when the ladder came up + * empty. Callers treat this rung as NOT LOCATED + * (@civfix/shared `isLocatedPrecision`). + * null nothing at all: outside TIGER coverage with a chain miss. + * + * A `landmark` line is stored and returned RAW ("Vista Hermosa Park, Los Angeles, CA"). The "Near " + * prefix is added by the display layer, which is the only one that knows the viewer's language. + * + * NEVER BLOCKS, NEVER THROWS. Both halves are best-effort by seam contract, and this is called on the + * report-create path where a geocoder outage must not cost a filing. A provider that throws, a cache + * that is unreachable, a TIGER query that fails - each degrades one rung, never into an error. + */ + +import { + geocodePointKey, + MAX_REPORT_ADDR_LENGTH, + type AddressPrecision, + type ReportAddressSource, +} from "@civfix/shared" +import type { Geocoder } from "@civfix/shared" +import type { ReverseGeocode } from "../adapters/reverse-geocode.chain.js" +import { NO_GEOCODE_CACHE, type GeocodeCache } from "./geocode-cache.js" + +export interface ResolvedAddress { + /** The display line, or null when nothing resolved. Landmarks are raw - no "Near " prefix. */ + address: string | null + precision: AddressPrecision | null + /** ALWAYS a string ("" when even TIGER has nothing) - the contract promises it unconditionally. */ + cityStateLabel: string +} + +/** Coords -> the resolved address. The dep every service and route takes. */ +export type AddressResolver = (lat: number, lng: number) => Promise + +export interface AddressResolverDeps { + streetReverseGeocode: ReverseGeocode + geocoder: Pick + /** Defaults to no caching, which is correct for a process with no database. */ + cache?: GeocodeCache +} + +const NOT_RESOLVED: ResolvedAddress = { address: null, precision: null, cityStateLabel: "" } + +async function orNull(p: Promise): Promise { + try { + return await p + } catch { + return null + } +} + +export function makeAddressResolver(deps: AddressResolverDeps): AddressResolver { + const cache = deps.cache ?? NO_GEOCODE_CACHE + + return async (lat: number, lng: number): Promise => { + if (!Number.isFinite(lat) || !Number.isFinite(lng)) return NOT_RESOLVED + + const pointKey = geocodePointKey({ lat, lng }) + const cached = await cache.read(pointKey) + if (cached !== null) { + return { + address: cached.address, + precision: cached.precision, + cityStateLabel: cached.cityStateLabel, + } + } + + // Raced, not sequenced: the TIGER label is a local PostGIS query and the chain is a network call, so + // the pair costs what the chain alone costs. The label is needed either way - as the response's + // unconditional cityStateLabel, and as the `locality` rung when the chain misses. + const [hit, label] = await Promise.all([ + orNull(deps.streetReverseGeocode(lat, lng)), + orNull(Promise.resolve(deps.geocoder.cityStateLabel(lat, lng))), + ]) + + const cityStateLabel = label ?? "" + const resolved: ResolvedAddress = + hit !== null + ? { address: hit.line, precision: hit.precision, cityStateLabel } + : cityStateLabel.length > 0 + ? { address: cityStateLabel, precision: "locality", cityStateLabel } + : { address: null, precision: null, cityStateLabel } + + await cache.write(pointKey, { + address: resolved.address, + precision: resolved.precision, + cityStateLabel, + provider: hit?.provider ?? (resolved.precision === "locality" ? "tiger" : null), + }) + + return resolved + } +} + +/** What a report row records about its address: the text, where it came from, and how exact it is. */ +export interface ReportAddressWrite { + addr: string | null + addrSource: ReportAddressSource | null + addrPrecision: AddressPrecision | null +} + +/** + * Decide a report's address provenance at creation. Shared by the signed-in and anonymous paths so the + * two can never drift - they already write the same column and are read by the same DTO. + * + * The reporter's own text always wins and is marked 'user' with NO precision: a provider rung would be a + * claim about text no provider produced. Otherwise the server's resolve is stored as 'resolved' with the + * rung it actually reached - INCLUDING `locality`, which is the behavior reports have always had (a + * "City, ST" line beat an empty one at the curb) and is now merely labelled honestly instead of being + * indistinguishable from a street address. + */ +export function addressProvenance( + suppliedAddr: string, + resolved: ResolvedAddress | null, +): ReportAddressWrite { + if (suppliedAddr.length > 0) { + return { addr: suppliedAddr, addrSource: "user", addrPrecision: null } + } + if (resolved === null || resolved.address === null) { + return { addr: null, addrSource: null, addrPrecision: null } + } + // The wire cap never applied to a server-derived line; clamp rather than overflow the column. + return { + addr: resolved.address.slice(0, MAX_REPORT_ADDR_LENGTH), + addrSource: "resolved", + addrPrecision: resolved.precision, + } +} diff --git a/services/api/src/services/geocode-cache.ts b/services/api/src/services/geocode-cache.ts new file mode 100644 index 00000000..652bf3ed --- /dev/null +++ b/services/api/src/services/geocode-cache.ts @@ -0,0 +1,126 @@ +/** + * Read-through cache for reverse geocodes (`geocode_cache`, migration 0175). + * + * WHY. A reverse geocode is stable per point and the SAME point is resolved several times over one + * creation flow: the client previews it while the host drags the pin (debounced, rounded to 5 decimals), + * then the create path resolves it again server-side. Keying on the shared `geocodePointKey` - the same + * 5-decimal rounding the client's preview query uses - collapses all of that onto one row, so a pin + * fine-tune is a cache hit rather than a fresh provider call, and the create that follows a preview + * costs zero provider calls. + * + * TTL IS APPLIED ON READ, never by a cron: a row past its TTL is reported as a miss and overwritten in + * place by the fresh resolve. Two TTLs, and the difference matters: + * + * - POSITIVE (180 days). Addresses do not move. + * - NEGATIVE (15 minutes). A null IS cached, briefly: that is what stops a dragged pin over a + * genuinely unaddressable point - or a provider that is down right now - from re-firing the whole + * chain per micro-drag. Expiring it in minutes is what stops a provider OUTAGE from poisoning those + * points for half a year, which is the failure mode the design spec called out. + * + * EVERY DB ERROR IS SWALLOWED. The cache is an optimisation on a seam whose entire contract is "best + * effort, never blocks a submit". A cache table that is missing, locked or unreachable must degrade to + * "no cache", never to a failed report. + */ + +import type { AddressPrecision } from "@civfix/shared" +import type { Queryable } from "../db/client.js" + +/** A resolved address line past this age is re-resolved (and the row overwritten) on next read. */ +export const GEOCODE_CACHE_TTL_MS = 180 * 24 * 60 * 60 * 1000 +/** A negative entry ("the providers had nothing") is trusted for minutes, not months. */ +export const GEOCODE_CACHE_NEGATIVE_TTL_MS = 15 * 60 * 1000 + +export interface GeocodeCacheEntry { + address: string | null + precision: AddressPrecision | null + cityStateLabel: string + provider: string | null +} + +export interface GeocodeCache { + read(pointKey: string): Promise + write(pointKey: string, entry: GeocodeCacheEntry): Promise +} + +interface GeocodeCacheRowSelect { + address: string | null + address_precision: AddressPrecision | null + city_state_label: string | null + provider: string | null + resolved_at: Date +} + +/** True while the row may still be served. Negative rows (no address) expire far sooner. */ +export function isFreshEntry(row: { address: string | null; resolvedAt: Date }, now: Date): boolean { + const ttl = row.address === null ? GEOCODE_CACHE_NEGATIVE_TTL_MS : GEOCODE_CACHE_TTL_MS + return now.getTime() - row.resolvedAt.getTime() < ttl +} + +export interface GeocodeCacheOptions { + /** + * Resolve the connection lazily and per call. It THROWS when the process has no database (the fully + * faked offline server), which is exactly why every use here sits inside a try/catch. + */ + getSql: () => Queryable + now?: () => Date +} + +export function makeGeocodeCache(opts: GeocodeCacheOptions): GeocodeCache { + const now = opts.now ?? ((): Date => new Date()) + return { + async read(pointKey: string): Promise { + try { + const sql = opts.getSql() + const rows = await sql` + SELECT address, address_precision, city_state_label, provider, resolved_at + FROM geocode_cache + WHERE point_key = ${pointKey} + LIMIT 1 + ` + const row = rows[0] + if (row === undefined) return null + if (!isFreshEntry({ address: row.address, resolvedAt: row.resolved_at }, now())) return null + return { + address: row.address, + precision: row.address_precision, + cityStateLabel: row.city_state_label ?? "", + provider: row.provider, + } + } catch { + return null + } + }, + + async write(pointKey: string, entry: GeocodeCacheEntry): Promise { + try { + const sql = opts.getSql() + await sql` + INSERT INTO geocode_cache ( + point_key, address, address_precision, city_state_label, provider, resolved_at + ) VALUES ( + ${pointKey}, + ${entry.address}, + ${entry.precision}, + ${entry.cityStateLabel}, + ${entry.provider}, + ${now()} + ) + ON CONFLICT (point_key) DO UPDATE SET + address = EXCLUDED.address, + address_precision = EXCLUDED.address_precision, + city_state_label = EXCLUDED.city_state_label, + provider = EXCLUDED.provider, + resolved_at = EXCLUDED.resolved_at + ` + } catch { + // See the header: a cache write can never be the reason a submit fails. + } + }, + } +} + +/** A cache that never hits and never stores - the wiring for a process with no database. */ +export const NO_GEOCODE_CACHE: GeocodeCache = { + read: async () => null, + write: async () => undefined, +} diff --git a/services/api/src/services/route-geo-helpers.ts b/services/api/src/services/route-geo-helpers.ts index 4b4cf5fd..80c7dae2 100644 --- a/services/api/src/services/route-geo-helpers.ts +++ b/services/api/src/services/route-geo-helpers.ts @@ -17,8 +17,10 @@ import { type JurisdictionLookup, } from "../adapters/jurisdiction-lookup.census.js" import { makeJurisdictionService, type JurisdictionService } from "./jurisdiction-service.js" +import { makeAddressResolver, type AddressResolver } from "./address-resolver.js" +import { makeGeocodeCache } from "./geocode-cache.js" -/** Coords -> geoid (or null outside coverage) / one-line address (or null). Matches the service deps. */ +/** Coords -> the owning jurisdiction's geoid, or null outside coverage. Matches the service deps. */ export type PointToString = (lat: number, lng: number) => Promise /** @@ -80,10 +82,21 @@ export function makeGeoidResolver(container: Container): PointToString { } /** - * reverseGeocode dep: street-level first (Mapbox/Photon chain), falling back to the local "City, ST" - * label. Best-effort by contract — null leaves the address empty and never blocks a submit. + * resolveAddress dep: the full precision ladder (Mapbox/Photon chain, then the local TIGER "City, ST" + * label) behind the read-through `geocode_cache`, so the preview a client just ran and the create that + * follows it cost ONE provider call between them. + * + * Built per call like every other helper here: `container.getDb()` is only ever touched INSIDE the + * returned closure (mounting a plugin must not open a connection), and the cache swallows the throw a + * DB-less process raises, degrading to an uncached resolve rather than an error. + * + * Best-effort by contract — a null address leaves the field empty and never blocks a submit. */ -export function makeReverseGeocoder(container: Container): PointToString { - return async (lat, lng) => - (await container.streetReverseGeocode(lat, lng)) ?? container.geocoder.cityStateLabel(lat, lng) +export function makeCachedAddressResolver(container: Container): AddressResolver { + return makeAddressResolver({ + streetReverseGeocode: container.streetReverseGeocode, + geocoder: container.geocoder, + cache: makeGeocodeCache({ getSql: () => container.getDb().sql }), + }) } + diff --git a/services/api/test/unit/adapters-reverse-geocode-chain-budget.test.ts b/services/api/test/unit/adapters-reverse-geocode-chain-budget.test.ts index 81b650a3..4f96fb82 100644 --- a/services/api/test/unit/adapters-reverse-geocode-chain-budget.test.ts +++ b/services/api/test/unit/adapters-reverse-geocode-chain-budget.test.ts @@ -14,10 +14,13 @@ */ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest" -import { chainReverse, type ReverseGeocode } from "../../src/adapters/reverse-geocode.chain.js" +import { chainReverse, type PointResolver } from "../../src/adapters/reverse-geocode.chain.js" + +/** The budget is scheduling, not shape: a string answer keeps these assertions about the clock. */ +type StringResolver = PointResolver /** A provider that never answers — the stalled-vendor case. Records when it was called. */ -function hangs(calls: number[]): ReverseGeocode { +function hangs(calls: number[]): StringResolver { return () => { calls.push(Date.now()) return new Promise(() => {}) @@ -25,7 +28,7 @@ function hangs(calls: number[]): ReverseGeocode { } /** A provider that answers `value` after `afterMs`. Records when it was called. */ -function answersAfter(afterMs: number, value: string | null, calls: number[]): ReverseGeocode { +function answersAfter(afterMs: number, value: string | null, calls: number[]): StringResolver { return () => { calls.push(Date.now()) return new Promise((resolve) => { @@ -97,7 +100,7 @@ describe("chainReverse budget", () => { } process.on("unhandledRejection", onUnhandled) try { - const late: ReverseGeocode = () => + const late: StringResolver = () => new Promise((_resolve, reject) => { setTimeout(() => reject(new Error("vendor blew up after we gave up")), 3_000) }) diff --git a/services/api/test/unit/reverse-geocode-chain.test.ts b/services/api/test/unit/reverse-geocode-chain.test.ts index 290c56b6..b047810e 100644 --- a/services/api/test/unit/reverse-geocode-chain.test.ts +++ b/services/api/test/unit/reverse-geocode-chain.test.ts @@ -1,11 +1,18 @@ import { describe, it, expect } from "vitest" -import { chainReverse, type ReverseGeocode } from "../../src/adapters/reverse-geocode.chain.js" +import { chainReverse, type PointResolver } from "../../src/adapters/reverse-geocode.chain.js" + +/** + * chainReverse is generic over the answer shape - the real seam carries a structured + * { line, precision, provider }, but the ORDERING contract under test here is about which provider + * answers first, not what it answers with, so a bare string keeps the assertions readable. + */ +type StringResolver = PointResolver const ok = - (label: string): ReverseGeocode => + (label: string): StringResolver => async () => label -const nul: ReverseGeocode = async () => null +const nul: StringResolver = async () => null describe("chainReverse", () => { it("returns the first non-null result", async () => { diff --git a/services/api/test/unit/reverse-geocode-mapbox.test.ts b/services/api/test/unit/reverse-geocode-mapbox.test.ts index 9303ffce..d977fc0b 100644 --- a/services/api/test/unit/reverse-geocode-mapbox.test.ts +++ b/services/api/test/unit/reverse-geocode-mapbox.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from "vitest" import { makeMapboxReverseGeocode, formatMapboxReverse, + mapboxPrecision, redactMapboxToken, } from "../../src/adapters/reverse-geocode.mapbox.js" @@ -71,7 +72,11 @@ describe("makeMapboxReverseGeocode", () => { context: { place: { name: "Springfield" }, region: { region_code: "IL" }, country: { country_code: "us" } }, }), }) - expect(await geocode(39.8, -89.6)).toBe("1 Main St, Springfield, IL") + expect(await geocode(39.8, -89.6)).toEqual({ + line: "1 Main St, Springfield, IL", + precision: "street", + provider: "mapbox", + }) }) it("returns null on non-ok HTTP", async () => { const geocode = makeMapboxReverseGeocode({ @@ -121,3 +126,60 @@ describe("makeMapboxReverseGeocode", () => { expect(opts?.redirect).toBe("error") }) }) + +/** + * Mapbox is the OPT-IN primary, so what it REFUSES to answer matters as much as what it answers: a + * place-only hit must fall through to Photon and then to the local TIGER locality rung rather than + * short-circuit the chain with the vaguest line available. + */ +describe("mapboxPrecision", () => { + it("claims street only with a house number", () => { + expect(mapboxPrecision({ context: { address: { address_number: "123", name: "123 Main St" } } })).toBe( + "street", + ) + expect(mapboxPrecision({ name: "123 Main St" })).toBe("street") + }) + + it("claims intersection for a named street with no number", () => { + expect(mapboxPrecision({ name: "Main St", context: { street: { name: "Main St" } } })).toBe( + "intersection", + ) + }) + + it("claims NOTHING for a place-only hit, so the chain keeps going", () => { + expect(mapboxPrecision({ name: "Los Angeles", context: { place: { name: "Los Angeles" } } })).toBeNull() + expect(mapboxPrecision({})).toBeNull() + }) +}) + +describe("makeMapboxReverseGeocode precision ladder", () => { + it("returns intersection (no fabricated number) for a street-only feature", async () => { + const geocode = makeMapboxReverseGeocode({ + token: "pk.test", + fetchImpl: okFetch({ + name: "Main St", + context: { + street: { name: "Main St" }, + place: { name: "Springfield" }, + region: { region_code: "IL" }, + }, + }), + }) + expect(await geocode(39.8, -89.6)).toEqual({ + line: "Main St, Springfield, IL", + precision: "intersection", + provider: "mapbox", + }) + }) + + it("returns null rather than a locality line", async () => { + const geocode = makeMapboxReverseGeocode({ + token: "pk.test", + fetchImpl: okFetch({ + name: "Springfield", + context: { place: { name: "Springfield" }, region: { region_code: "IL" } }, + }), + }) + expect(await geocode(39.8, -89.6)).toBeNull() + }) +}) diff --git a/services/api/test/unit/reverse-geocode-photon.test.ts b/services/api/test/unit/reverse-geocode-photon.test.ts index 7c97b0fc..ef73c018 100644 --- a/services/api/test/unit/reverse-geocode-photon.test.ts +++ b/services/api/test/unit/reverse-geocode-photon.test.ts @@ -1,5 +1,8 @@ import { describe, it, expect } from "vitest" import { + composePhotonReverse, + distanceMeters, + isResidentialName, makePhotonReverseGeocode, formatPhotonReverse, } from "../../src/adapters/reverse-geocode.photon.js" @@ -46,7 +49,11 @@ describe("makePhotonReverseGeocode", () => { const geocode = makePhotonReverseGeocode({ fetchImpl: okFetch({ housenumber: "1", street: "Main St", city: "Springfield", state: "IL" }), }) - expect(await geocode(39.8, -89.6)).toBe("1 Main St, Springfield, IL") + expect(await geocode(39.8, -89.6)).toEqual({ + line: "1 Main St, Springfield, IL", + precision: "street", + provider: "photon", + }) }) it("returns null on a non-ok HTTP response", async () => { @@ -77,3 +84,174 @@ describe("makePhotonReverseGeocode", () => { expect(called).toBe(false) }) }) + +/** + * The precision ladder (composePhotonReverse). Every rung here is a product decision about how much the + * platform is allowed to CLAIM from OSM data that chronically lacks house numbers, so each one is + * pinned: the exact hit, the cross-street composition, the single-road degrade, the landmark, the + * residential guard that must never surface an occupant's name, and the refusal to answer at all rather + * than return a city (the local TIGER geocoder owns the locality rung). + */ + +const PIN = { lat: 34.05, lng: -118.25 } + +/** A feature at an offset in metres roughly north of the pin. */ +function featureAt(props: Record, northMeters: number): { + properties: Record + geometry: { coordinates: [number, number] } +} { + const dLat = northMeters / 111_320 + return { properties: props, geometry: { coordinates: [PIN.lng, PIN.lat + dLat] } } +} + +describe("distanceMeters", () => { + it("measures a small northward offset to within a metre", () => { + expect(distanceMeters(PIN, { lat: PIN.lat + 100 / 111_320, lng: PIN.lng })).toBeCloseTo(100, 0) + }) + + it("is zero for the same point", () => { + expect(distanceMeters(PIN, PIN)).toBe(0) + }) +}) + +describe("isResidentialName", () => { + it("flags a building or a place=house, which can carry an occupant name", () => { + expect(isResidentialName({ osm_key: "building", osm_value: "residential", name: "The Smiths" })).toBe(true) + expect(isResidentialName({ osm_key: "place", osm_value: "house", name: "Rose Cottage" })).toBe(true) + }) + + it("does not flag a public amenity", () => { + expect(isResidentialName({ osm_key: "leisure", osm_value: "park", name: "Vista Hermosa Park" })).toBe(false) + }) +}) + +describe("composePhotonReverse ladder", () => { + it("street: a house number plus a street wins outright", () => { + const composed = composePhotonReverse( + [ + featureAt({ osm_key: "leisure", name: "Some Park", city: "Los Angeles", state: "CA" }, 5), + featureAt( + { housenumber: "123", street: "Main St", city: "Inglewood", state: "CA", osm_key: "place" }, + 12, + ), + ], + PIN, + ) + expect(composed).toEqual({ line: "123 Main St, Inglewood, CA", precision: "street" }) + }) + + it("intersection: two distinct named roads within 40 m compose a cross street", () => { + const composed = composePhotonReverse( + [ + featureAt({ osm_key: "highway", street: "Main St", city: "Inglewood", state: "CA" }, 8), + featureAt({ osm_key: "highway", street: "5th Ave", city: "Inglewood", state: "CA" }, 20), + ], + PIN, + ) + expect(composed).toEqual({ line: "Main St & 5th Ave, Inglewood, CA", precision: "intersection" }) + }) + + it("intersection: one road alone degrades to the bare street, never an invented number", () => { + const composed = composePhotonReverse( + [featureAt({ osm_key: "highway", street: "Main St", city: "Inglewood", state: "CA" }, 8)], + PIN, + ) + expect(composed).toEqual({ line: "Main St, Inglewood, CA", precision: "intersection" }) + }) + + it("intersection: the same road name twice is ONE road, not a crossing with itself", () => { + const composed = composePhotonReverse( + [ + featureAt({ osm_key: "highway", street: "Main St", city: "Inglewood", state: "CA" }, 5), + featureAt({ osm_key: "highway", street: "Main St", city: "Inglewood", state: "CA" }, 22), + ], + PIN, + ) + expect(composed).toEqual({ line: "Main St, Inglewood, CA", precision: "intersection" }) + }) + + it("intersection: a road beyond 40 m does not join the composition", () => { + const composed = composePhotonReverse( + [ + featureAt({ osm_key: "highway", street: "Main St", city: "Inglewood", state: "CA" }, 10), + featureAt({ osm_key: "highway", street: "Far Blvd", city: "Inglewood", state: "CA" }, 55), + ], + PIN, + ) + expect(composed).toEqual({ line: "Main St, Inglewood, CA", precision: "intersection" }) + }) + + it("landmark: a named public feature within 60 m, stored RAW (no 'Near ' prefix)", () => { + const composed = composePhotonReverse( + [ + featureAt( + { + osm_key: "leisure", + osm_value: "park", + name: "Vista Hermosa Park", + city: "Los Angeles", + state: "CA", + }, + 45, + ), + ], + PIN, + ) + expect(composed).toEqual({ + line: "Vista Hermosa Park, Los Angeles, CA", + precision: "landmark", + }) + }) + + it("landmark guard: a residential building's name is NEVER surfaced", () => { + const composed = composePhotonReverse( + [ + featureAt( + { + osm_key: "building", + osm_value: "residential", + name: "The Hendersons", + city: "Los Angeles", + state: "CA", + }, + 5, + ), + ], + PIN, + ) + expect(composed).toBeNull() + }) + + it("landmark: a feature beyond 60 m is out of range", () => { + const composed = composePhotonReverse( + [featureAt({ osm_key: "amenity", name: "Library", city: "Los Angeles", state: "CA" }, 90)], + PIN, + ) + expect(composed).toBeNull() + }) + + it("never claims locality: a city-only feature yields nothing (TIGER owns that rung)", () => { + const composed = composePhotonReverse( + [featureAt({ osm_key: "place", osm_value: "city", name: "Los Angeles", state: "CA" }, 5)], + PIN, + ) + expect(composed).toBeNull() + }) + + it("returns null for an empty feature set", () => { + expect(composePhotonReverse([], PIN)).toBeNull() + }) +}) + +describe("makePhotonReverseGeocode request shape", () => { + it("asks for several features within a 50 m radius so the ladder has candidates", async () => { + let seen = "" + const fetchImpl = (async (u: string) => { + seen = u + return { ok: true, json: async () => ({ features: [] }) } as unknown as Response + }) as unknown as typeof fetch + await makePhotonReverseGeocode({ fetchImpl })(34.05, -118.25) + expect(seen).toContain("limit=5") + expect(seen).toContain("radius=0.05") + }) +}) From d3c5dabd17b73fa8f821070195756dd354a0131d Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 18:57:34 -0700 Subject: [PATCH 30/54] POST /map/resolve-address, plus address provenance on events and reports The endpoint the creation flows call once a pin settles. reverse-label stays exactly as it is for deployed clients - it answers only TIGER's "City, ST", which is why pin previews were vague; this one walks the whole ladder and says which rung it reached, with cityStateLabel always populated as the hint line. Same GEOCODER_RATE_LIMIT bucket, and the cache means a dragged pin and the create that follows it share one provider call. Events (cleanup-service). addressSource is the client-version discriminator, because `address` has to stay optional on the wire: - present -> a new client. It resolved the pin, showed the line to the host and published with it on screen, which IS the confirmation, so the server only refuses a blank one (trimmed, min 3). - absent -> an old client. Text it sent is the host's own "name the spot" and is stored 'manual'; nothing at all falls back to the compat shim, which resolves the pin and stores 'resolved' - but only at a located rung, since "Los Angeles, CA" is not a meeting address. A duplicated event copies address and source as a pair, and the DTO serves addressSource. Reports keep the never-block-filing property exactly: the reporter's own text is 'user' with no precision, a server resolve is 'resolved' with the rung it reached (including locality, which reports have always stored and is now merely labelled honestly), and a miss leaves all three NULL. One shared helper for the signed-in and anonymous paths so they cannot drift. addr also joins the slur sweep, which events have had on `address` since forever. Route coverage 323 -> 324. Co-Authored-By: Claude Opus 5 (1M context) --- services/api/src/routes/anon.routes.ts | 7 +- services/api/src/routes/cleanups.routes.ts | 6 +- services/api/src/routes/map.routes.ts | 33 +++++- services/api/src/routes/reports.routes.ts | 13 ++- .../src/services/anon-repository.drizzle.ts | 5 +- services/api/src/services/anon-service.ts | 23 ++-- services/api/src/services/cleanup-dto.ts | 1 + .../services/cleanup-repository.drizzle.ts | 6 +- .../src/services/cleanup-repository.types.ts | 4 + services/api/src/services/cleanup-service.ts | 104 +++++++++++++++++- services/api/src/services/cleanup-sql.ts | 4 + .../src/services/report-repository.drizzle.ts | 5 +- services/api/src/services/report-service.ts | 21 +++- .../api/src/services/report-service.types.ts | 14 ++- services/api/src/services/report-sql.ts | 15 ++- .../src/services/social-repository.drizzle.ts | 4 + services/api/test/helpers/cleanups.ts | 5 + services/api/test/helpers/reports.ts | 4 + services/api/test/helpers/social.ts | 1 + .../test/integration/cleanup-slots-pg.test.ts | 1 + .../integration/host-orgs-team-pg.test.ts | 2 + .../api/test/unit/guest-rsvp-service.test.ts | 1 + services/api/test/unit/report-routes.test.ts | 30 +++-- services/api/test/unit/route-coverage.test.ts | 4 +- 24 files changed, 276 insertions(+), 37 deletions(-) diff --git a/services/api/src/routes/anon.routes.ts b/services/api/src/routes/anon.routes.ts index ab4fae83..4da423db 100644 --- a/services/api/src/routes/anon.routes.ts +++ b/services/api/src/routes/anon.routes.ts @@ -15,7 +15,10 @@ import { ANON_TOKEN_TTL_SECONDS } from "../abuse/anon-token.js" import { cfGeoFromTrustedEdge } from "../abuse/gps-sanity.js" import { makeAnonService, type AnonService } from "../services/anon-service.js" import { makeDrizzleAnonReportRepository } from "../services/anon-repository.drizzle.js" -import { makeGeoidResolver, makeReverseGeocoder } from "../services/route-geo-helpers.js" +import { + makeCachedAddressResolver, + makeGeoidResolver, +} from "../services/route-geo-helpers.js" import { resolveJurisdictionCode } from "../db/reference-code.js" import { route } from "../versioning/route.js" import { parse, trimTextFields } from "./_validate.js" @@ -82,7 +85,7 @@ export async function registerAnonRoutes( anonTokenSigningKey: container.env.ANON_TOKEN_SIGNING_KEY, resolveJurisdictionGeoid: makeGeoidResolver(container), resolveJurisdictionCode: (geoid) => resolveJurisdictionCode(sql, geoid), - reverseGeocode: makeReverseGeocoder(container), + resolveAddress: makeCachedAddressResolver(container), raiseAbuseFlag: async (subjectType, subjectId, reason) => { await sql` INSERT INTO abuse_flags (subject_type, subject_id, reason, source) diff --git a/services/api/src/routes/cleanups.routes.ts b/services/api/src/routes/cleanups.routes.ts index 25ff7fd3..487caa31 100644 --- a/services/api/src/routes/cleanups.routes.ts +++ b/services/api/src/routes/cleanups.routes.ts @@ -54,7 +54,10 @@ import { } from "../services/chat-repository.drizzle.js" import { makePrivateMediaPresigner } from "../services/media-presign.js" import { buildIcs } from "@civfix/shared/ics" -import { makeGeoidResolver } from "../services/route-geo-helpers.js" +import { + makeCachedAddressResolver, + makeGeoidResolver, +} from "../services/route-geo-helpers.js" import { resolveJurisdictionCode } from "../db/reference-code.js" import { makeOutboundMailService } from "../services/admin/outbound-mail-service.js" import { makeDrizzleMailRepository } from "../services/admin/mail-repository.drizzle.js" @@ -194,6 +197,7 @@ export function makeContainerCleanupService( ? {} : { resolveJurisdictionGeoid: makeGeoidResolver(container), + resolveAddress: makeCachedAddressResolver(container), resolveJurisdictionCode: (geoid: string | null) => resolveJurisdictionCode(container.getDb().sql, geoid), outboundMail: makeOutboundMailService({ diff --git a/services/api/src/routes/map.routes.ts b/services/api/src/routes/map.routes.ts index 24322232..05ec4fec 100644 --- a/services/api/src/routes/map.routes.ts +++ b/services/api/src/routes/map.routes.ts @@ -1,5 +1,6 @@ import { + ResolveAddressRequestSchema, ResolveJurisdictionRequestSchema, ReverseLabelRequestSchema, SuggestPlacesRequestSchema, @@ -8,6 +9,7 @@ import { AppError, type TileInfoResponse, type JurisdictionDTO, + type ResolveAddressResponse, type ReverseLabelResponse, type SuggestPlacesResponse, type MapCleanupsResponse, @@ -17,7 +19,10 @@ import { suggestAddresses } from "@civfix/shared/geocode" import { z } from "zod" import type { FastifyInstance } from "fastify" import type { Container } from "../di.js" -import { makeRouteJurisdictionService } from "../services/route-geo-helpers.js" +import { + makeCachedAddressResolver, + makeRouteJurisdictionService, +} from "../services/route-geo-helpers.js" import { makeCleanupMapRepository, MAP_CLEANUPS_LIMIT, @@ -111,6 +116,32 @@ export async function registerMapRoutes(app: FastifyInstance, container: Contain }, ) + /** + * The street-level preview the creation flows call once a pin settles. `reverseLabel` above stays + * exactly as it is for already-deployed clients — it answers only TIGER's "City, ST", which is why + * pin previews were vague; this one walks the whole ladder and says which rung it reached, so the UI + * can prefill a confirmed address (street/intersection/landmark) or ask the host to type one + * (locality/null) without ever guessing. `cityStateLabel` is populated either way as the hint line. + * + * Same rate-limit bucket as the other geocoder surfaces, and the read-through cache means a dragged + * pin and the create that follows it share one provider call. + */ + route( + app, + "resolveAddress", + { config: { rateLimit: GEOCODER_RATE_LIMIT } }, + async (request, reply) => { + const { lat, lng } = parse(ResolveAddressRequestSchema, request.body) + const resolved = await makeCachedAddressResolver(container)(lat, lng) + const payload: ResolveAddressResponse = { + address: resolved.address, + precision: resolved.precision, + cityStateLabel: resolved.cityStateLabel, + } + reply.status(200).send(payload) + }, + ) + route( app, "suggest", diff --git a/services/api/src/routes/reports.routes.ts b/services/api/src/routes/reports.routes.ts index e6f5e3d2..5858542e 100644 --- a/services/api/src/routes/reports.routes.ts +++ b/services/api/src/routes/reports.routes.ts @@ -22,7 +22,10 @@ import type { FastifyInstance, FastifyRequest } from "fastify" import type { Container } from "../di.js" import type { Sql } from "../db/client.js" import { requireAuth } from "../auth/context.js" -import { makeGeoidResolver, makeReverseGeocoder } from "../services/route-geo-helpers.js" +import { + makeCachedAddressResolver, + makeGeoidResolver, +} from "../services/route-geo-helpers.js" import { makeMediaPresigner, makePrivateMediaPresigner } from "../services/media-presign.js" import { perIdentity } from "../plugins/rate-limit.js" import { @@ -52,7 +55,7 @@ export interface ReportServiceOverrides { repo: ReportRepository resolveJurisdictionGeoid?: ReportServiceDeps["resolveJurisdictionGeoid"] resolveJurisdictionCode?: ReportServiceDeps["resolveJurisdictionCode"] - reverseGeocode?: ReportServiceDeps["reverseGeocode"] + resolveAddress?: ReportServiceDeps["resolveAddress"] presignMedia?: ReportServiceDeps["presignMedia"] presignPrivateMedia?: ReportServiceDeps["presignPrivateMedia"] loadLinkedEventsForReports?: ReportServiceDeps["loadLinkedEventsForReports"] @@ -184,7 +187,9 @@ export async function registerReportRoutes( presignMedia: overrides.presignMedia ?? makeMediaPresigner(container.storage), presignPrivateMedia: overrides.presignPrivateMedia ?? makePrivateMediaPresigner(container.storage), - ...(overrides.reverseGeocode !== undefined ? { reverseGeocode: overrides.reverseGeocode } : {}), + ...(overrides.resolveAddress !== undefined + ? { resolveAddress: overrides.resolveAddress } + : {}), ...(overrides.loadLinkedEventsForReports !== undefined ? { loadLinkedEventsForReports: overrides.loadLinkedEventsForReports } : {}), @@ -274,7 +279,7 @@ export async function registerReportRoutes( }, resolveJurisdictionGeoid: makeGeoidResolver(container), resolveJurisdictionCode: (geoid) => resolveJurisdictionCode(sql, geoid), - reverseGeocode: makeReverseGeocoder(container), + resolveAddress: makeCachedAddressResolver(container), presignMedia: makeMediaPresigner(container.storage), presignPrivateMedia: makePrivateMediaPresigner(container.storage), jobs: container.jobs, diff --git a/services/api/src/services/anon-repository.drizzle.ts b/services/api/src/services/anon-repository.drizzle.ts index e5696916..688c74ac 100644 --- a/services/api/src/services/anon-repository.drizzle.ts +++ b/services/api/src/services/anon-repository.drizzle.ts @@ -170,7 +170,8 @@ export function makeDrizzleAnonReportRepository(sql: Sql): AnonReportRepository await tx` INSERT INTO reports ( id, reporter_user_id, anon_session_id, idempotency_key, geom, geom_source, - jurisdiction_geoid, category, type, title, description, addr, status, visibility, h3_cell, + jurisdiction_geoid, category, type, title, description, addr, addr_source, addr_precision, + status, visibility, h3_cell, claim_code, claim_code_hash, reference_code, published_at ) VALUES ( ${args.reportId}, @@ -185,6 +186,8 @@ export function makeDrizzleAnonReportRepository(sql: Sql): AnonReportRepository ${args.title}, ${args.description}, ${args.addr}, + ${args.addrSource}, + ${args.addrPrecision}, ${"held"}, ${"public"}, ${args.h3Cell}, diff --git a/services/api/src/services/anon-service.ts b/services/api/src/services/anon-service.ts index fd39e4bf..6250b3fa 100644 --- a/services/api/src/services/anon-service.ts +++ b/services/api/src/services/anon-service.ts @@ -29,6 +29,8 @@ import { } from "../abuse/anon-token.js" import { generateToken, constantTimeStringEqual, sha256Hex } from "../auth/crypto.js" import { UNKNOWN_JURCODE } from "../db/reference-code.js" +import { addressProvenance, type AddressResolver } from "./address-resolver.js" +import type { AddressPrecision, ReportAddressSource } from "@civfix/shared" export const ANON_REPORT_CREATE_SCOPE = "anon_report_create" @@ -52,6 +54,8 @@ export interface CreateAnonReportTxArgs { title: string | null description: string | null addr: string | null + addrSource: ReportAddressSource | null + addrPrecision: AddressPrecision | null h3Cell: string mediaUploadIds: string[] claimCodeHash: string @@ -94,7 +98,8 @@ export interface AnonServiceDeps { anonTokenSigningKey: string resolveJurisdictionGeoid: (lat: number, lng: number) => Promise resolveJurisdictionCode?: (geoid: string | null) => Promise - reverseGeocode?: (lat: number, lng: number) => Promise + /** Structured twin of the signed-in path's dep - same resolver, same cache, same provenance rules. */ + resolveAddress?: AddressResolver raiseAbuseFlag?: (subjectType: "report" | "anon_token", subjectId: string, reason: AnonAbuseReason) => Promise enqueueMediaChecks?: (reportId: string, mediaUploadIds: string[]) => Promise newId?: () => string @@ -171,6 +176,7 @@ export function makeAnonService(deps: AnonServiceDeps): AnonService { assertNoSlur(input.title ?? null, "title") assertNoSlur(input.description ?? null, "description") + assertNoSlur(input.addr ?? null, "addr") await enforceIpRateLimit(ctx.ip, { counters: deps.counters }) @@ -187,12 +193,12 @@ export function makeAnonService(deps: AnonServiceDeps): AnonService { const { record: tokenRow, issuedToken } = await ensureAnonToken(presentedToken, tokenDeps) - const suppliedAddr = input.addr?.trim() - const [jurisdictionGeoid, geocodedAddr] = await Promise.all([ + const suppliedAddr = input.addr?.trim() ?? "" + const [jurisdictionGeoid, resolvedAddr] = await Promise.all([ deps.resolveJurisdictionGeoid(input.lat, input.lng), - suppliedAddr || !deps.reverseGeocode + suppliedAddr.length > 0 || !deps.resolveAddress ? Promise.resolve(null) - : deps.reverseGeocode(input.lat, input.lng), + : deps.resolveAddress(input.lat, input.lng), ]) const jurCode = deps.resolveJurisdictionCode !== undefined @@ -202,7 +208,8 @@ export function makeAnonService(deps: AnonServiceDeps): AnonService { const reportId = newId() const claimCode = newClaimCode() const claimCodeHash = await sha256Hex(claimCode) - const addr = suppliedAddr ? suppliedAddr : geocodedAddr + // Identical provenance rules to the signed-in path, by construction (one shared helper). + const addressWrite = addressProvenance(suppliedAddr, resolvedAddr) const responseSnapshot: AnonReportResponse = { reportId, @@ -223,7 +230,9 @@ export function makeAnonService(deps: AnonServiceDeps): AnonService { type: input.type, title: input.title ?? null, description: input.description ?? null, - addr, + addr: addressWrite.addr, + addrSource: addressWrite.addrSource, + addrPrecision: addressWrite.addrPrecision, h3Cell, mediaUploadIds: input.mediaUploadIds, claimCodeHash, diff --git a/services/api/src/services/cleanup-dto.ts b/services/api/src/services/cleanup-dto.ts index 805cfd44..023f7ae4 100644 --- a/services/api/src/services/cleanup-dto.ts +++ b/services/api/src/services/cleanup-dto.ts @@ -121,6 +121,7 @@ export function toCleanupDTO( ...(myRole !== null ? { myRole } : {}), bring: record.bring ?? [], address: record.address, + addressSource: record.addressSource, ...(record.dist !== null ? { dist: record.dist } : {}), ...(record.jurisdictionGeoid !== null ? { jurisdictionGeoid: record.jurisdictionGeoid } : {}), ...(record.referenceCode !== null ? { referenceCode: record.referenceCode } : {}), diff --git a/services/api/src/services/cleanup-repository.drizzle.ts b/services/api/src/services/cleanup-repository.drizzle.ts index 908f9436..a235277a 100644 --- a/services/api/src/services/cleanup-repository.drizzle.ts +++ b/services/api/src/services/cleanup-repository.drizzle.ts @@ -356,7 +356,7 @@ export function makeDrizzleCleanupRepository(sql: Sql): CleanupRepository { await tx` INSERT INTO cleanups ( id, organizer_user_id, type, event_kind, title, description, geom, scheduled_at, - status, bring, address, jurisdiction_geoid, reference_code, + status, bring, address, address_source, jurisdiction_geoid, reference_code, ends_at, timezone, visibility, cover_media_id, gallery_media_ids, donation_url, page_slug, registration_opens_at, registration_closes_at, organization_id, reminder_offsets_min, host_reply_to @@ -372,6 +372,7 @@ export function makeDrizzleCleanupRepository(sql: Sql): CleanupRepository { ${args.status}, ${args.bring as unknown as string[] | null}, ${args.address}, + ${args.addressSource}, ${args.jurisdictionGeoid}, ${referenceCode}, ${args.host.endsAt}, @@ -450,6 +451,9 @@ export function makeDrizzleCleanupRepository(sql: Sql): CleanupRepository { sets.push(sql`geom = ST_SetSRID(ST_MakePoint(${patch.lng}, ${patch.lat}), 4326)`) } if (patch.address !== undefined) sets.push(sql`address = ${patch.address}`) + if (patch.addressSource !== undefined) { + sets.push(sql`address_source = ${patch.addressSource}`) + } if (patch.bring !== undefined) { sets.push(sql`bring = ${patch.bring as unknown as string[] | null}`) } diff --git a/services/api/src/services/cleanup-repository.types.ts b/services/api/src/services/cleanup-repository.types.ts index b00c2663..d9d56ae2 100644 --- a/services/api/src/services/cleanup-repository.types.ts +++ b/services/api/src/services/cleanup-repository.types.ts @@ -2,6 +2,7 @@ import type { CleanupMemberRole, CleanupStatus, CleanupType, + EventAddressSource, EventKind, EventVisibility, OrganizationMemberRole, @@ -56,6 +57,7 @@ export interface CleanupRecord extends CleanupHostFields { status: CleanupStatus bring: string[] | null address: string | null + addressSource: EventAddressSource | null capacity: number | null jurisdictionGeoid: string | null referenceCode: string | null @@ -200,6 +202,7 @@ export interface CreateCleanupTxArgs { status: CleanupStatus bring: string[] | null address: string | null + addressSource: EventAddressSource | null jurisdictionGeoid: string | null jurCode: number linkedReportIds: string[] @@ -223,6 +226,7 @@ export interface UpdateCleanupPatch extends EventHostWrite { lat?: number lng?: number address?: string | null + addressSource?: EventAddressSource | null bring?: string[] | null jurisdictionGeoid?: string | null } diff --git a/services/api/src/services/cleanup-service.ts b/services/api/src/services/cleanup-service.ts index a6d47c7b..a808e9a5 100644 --- a/services/api/src/services/cleanup-service.ts +++ b/services/api/src/services/cleanup-service.ts @@ -4,6 +4,7 @@ import { AppError, ErrorCode, MAX_BRING_ITEMS, + MAX_EVENT_ADDRESS_LENGTH, MAX_EVENT_SLOTS, MIN_SLOT_DURATION_MINUTES, } from "@civfix/shared" @@ -16,6 +17,7 @@ import type { CleanupDTO, CreateCleanupRequest, DuplicateCleanupRequest, + EventAddressSource, EventKind, EventSlotDTO, EventSlotInput, @@ -29,6 +31,8 @@ import type { UpdateCleanupRequest, } from "@civfix/shared" import type { Jobs } from "@civfix/shared/interfaces" +import { isLocatedPrecision } from "@civfix/shared" +import type { AddressResolver } from "./address-resolver.js" import type { NotificationService } from "./notification-service.js" import type { MessageKey } from "../i18n/messages/en.js" import { EVENT_HOURS_MEMBER_CAP } from "./volunteer-hours-service.js" @@ -156,6 +160,12 @@ function isUuid(value: string): boolean { const EVENT_CLOSED_MESSAGE = "This event is closed." +/** + * Floor for a host-confirmed event address. Long enough to reject the accidental keystroke and the + * lone punctuation mark, short enough to allow a genuinely terse one ("Pier 3"). + */ +const MIN_EVENT_ADDRESS_LENGTH = 3 + function assertScheduledAtNotBackdated(next: string | undefined, stored: Date): void { if (next === undefined) return const nextMs = Date.parse(next) @@ -226,6 +236,12 @@ export interface CleanupServiceDeps { presignThumb?: (thumbKey: string) => Promise resolveJurisdictionGeoid?: (lat: number, lng: number) => Promise resolveJurisdictionCode?: (geoid: string | null) => Promise + /** + * Only ever called for an OLD client (one that sends no `addressSource`) that also sent no address — + * the compat shim in resolveEventAddress. A new client always confirms its own address with the host, + * which is the whole point of the feature; the server never resolves one behind a host's back. + */ + resolveAddress?: AddressResolver outboundMail?: OutboundMailService notifier?: Pick attendeeNotifier?: { eventCancelled(cleanupId: string, reason: string | null): Promise } @@ -518,6 +534,75 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { for (const item of input.bring ?? []) assertNoSlur(item, "bring") } + /** + * The event address, with its provenance, for a create or an update. + * + * `addressSource` is the CLIENT-VERSION discriminator, and it has to be: `address` itself stays + * optional on the wire so the TestFlight build in someone's pocket keeps working. + * + * addressSource PRESENT -> a new client. It resolved the pin, showed the line to the host, and the + * host published with it on screen. That is the confirmation, so the + * server only has to refuse a blank one — a client that sends a source + * without an address has a bug, and storing it would produce an event + * whose address is "verified" and empty. + * addressSource ABSENT -> an old client. Whatever it sent in `address` is the host's own "name the + * spot" text, so it is 'manual' (the same call migration 0175 makes for + * existing rows). If it sent nothing, the shim resolves the pin and stores + * 'resolved' — unverified, but an event with a street line beats an event + * with "Meeting point", and only while old clients are still in the wild. + * + * The shim stores NOTHING when the ladder only reached `locality`: "Los Angeles, CA" is not a meeting + * address, and writing it would dress up a non-answer as a host-provided one. + */ + async function resolveEventAddress(input: { + address?: string | undefined + addressSource?: EventAddressSource | undefined + lat: number + lng: number + }): Promise<{ address: string | null; addressSource: EventAddressSource | null }> { + if (input.addressSource !== undefined) { + return { address: assertConfirmedAddress(input.address), addressSource: input.addressSource } + } + const typed = input.address?.trim() ?? "" + if (typed.length > 0) return { address: typed, addressSource: "manual" } + if (deps.resolveAddress === undefined) return { address: null, addressSource: null } + const resolved = await deps.resolveAddress(input.lat, input.lng) + if (resolved.address === null || !isLocatedPrecision(resolved.precision)) { + return { address: null, addressSource: null } + } + // The wire cap never applied to a server-derived line; clamp rather than overflow the column. + return { + address: resolved.address.slice(0, MAX_EVENT_ADDRESS_LENGTH), + addressSource: "resolved", + } + } + + /** A new client that names a source must carry a real line with it. */ + function assertConfirmedAddress(address: string | undefined): string { + const trimmed = address?.trim() ?? "" + if (trimmed.length < MIN_EVENT_ADDRESS_LENGTH) { + throw AppError.validation({ + address: `must be at least ${MIN_EVENT_ADDRESS_LENGTH} characters`, + }) + } + return trimmed + } + + /** The update-path twin of resolveEventAddress: same rules, but every field stays optional. */ + function eventAddressPatch(patch: { + address?: string | undefined + addressSource?: EventAddressSource | undefined + }): { address?: string | null; addressSource?: EventAddressSource | null } { + if (patch.addressSource !== undefined) { + return { address: assertConfirmedAddress(patch.address), addressSource: patch.addressSource } + } + if (patch.address === undefined) return {} + const trimmed = patch.address.trim() + return trimmed.length > 0 + ? { address: trimmed, addressSource: "manual" } + : { address: null, addressSource: null } + } + function clampBring(bring: T): T { if (bring !== null && bring !== undefined && bring.length > MAX_BRING_ITEMS) { throw AppError.validation({ bring: `at most ${MAX_BRING_ITEMS} items may be listed` }) @@ -876,6 +961,9 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { throw AppError.validation({ linkedReportIds: "only cleanup events can link reports" }) } await assertReportsLinkable(linkedReportIds) + // Before any budget or host work: a malformed address is a client bug, and rejecting it early keeps + // the compat shim's geocode off the path of a request that was never going to be written. + const addressWrite = await resolveEventAddress(input) if (input.endsAt === null) throw AppError.validation({ endsAt: "required" }) if (input.slots !== undefined && input.slots.length === 0) { throw AppError.validation({ slots: EVENT_NEEDS_A_SLOT_MESSAGE }) @@ -921,7 +1009,8 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { scheduledAt, status: "upcoming", bring: input.bring ?? null, - address: input.address ?? null, + address: addressWrite.address, + addressSource: addressWrite.addressSource, jurisdictionGeoid, jurCode, linkedReportIds, @@ -996,7 +1085,14 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { lng: source.lng, scheduledAt: scheduledAt.toISOString(), ...(source.bring !== null ? { bring: source.bring } : {}), - ...(source.address !== null ? { address: source.address } : {}), + // A duplicate is the same place at a new time, so the host's verified address and its provenance + // both carry over verbatim. Copied as a PAIR: an address with no source would be re-read as an + // old client's payload and demoted to 'manual', and a source with no address cannot exist. + ...(source.address !== null && source.addressSource !== null + ? { address: source.address, addressSource: source.addressSource } + : source.address !== null + ? { address: source.address } + : {}), slots: slots.length > 0 ? slots.map((slot) => ({ @@ -1134,6 +1230,8 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { } } + const addressPatch = eventAddressPatch(patch) + const movedTo = patch.lat !== undefined && patch.lng !== undefined ? { lat: patch.lat, lng: patch.lng } @@ -1159,7 +1257,7 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { ...(patch.scheduledAt !== undefined ? { scheduledAt: new Date(patch.scheduledAt) } : {}), ...(patch.lat !== undefined ? { lat: patch.lat } : {}), ...(patch.lng !== undefined ? { lng: patch.lng } : {}), - ...(patch.address !== undefined ? { address: patch.address } : {}), + ...addressPatch, ...(patch.bring !== undefined ? { bring: patch.bring } : {}), ...(reresolvedGeoid !== undefined ? { jurisdictionGeoid: reresolvedGeoid } : {}), } diff --git a/services/api/src/services/cleanup-sql.ts b/services/api/src/services/cleanup-sql.ts index 8bb881de..1fe948ed 100644 --- a/services/api/src/services/cleanup-sql.ts +++ b/services/api/src/services/cleanup-sql.ts @@ -10,6 +10,7 @@ import type { CleanupMemberRole, CleanupStatus, CleanupType, + EventAddressSource, EventKind, EventVisibility, OrgVerificationKind, @@ -47,6 +48,7 @@ export interface CleanupRowSelect { status: CleanupStatus bring: string[] | null address: string | null + address_source: EventAddressSource | null jurisdiction_geoid: string | null reference_code: string | null created_at: Date @@ -129,6 +131,7 @@ export function toRecord(r: CleanupRowSelect): CleanupRecord { status: r.status, bring: r.bring, address: r.address, + addressSource: r.address_source, jurisdictionGeoid: r.jurisdiction_geoid, referenceCode: r.reference_code, createdAt: r.created_at, @@ -174,6 +177,7 @@ export function cleanupColumns(sql: Queryable, near: NearPoint | null) { ${cleanupStatusExpr(sql)} AS status, c.bring, c.address, + c.address_source, c.jurisdiction_geoid, c.reference_code, c.created_at, diff --git a/services/api/src/services/report-repository.drizzle.ts b/services/api/src/services/report-repository.drizzle.ts index 6c48b5cf..ac2f17ea 100644 --- a/services/api/src/services/report-repository.drizzle.ts +++ b/services/api/src/services/report-repository.drizzle.ts @@ -156,7 +156,8 @@ export function makeDrizzleReportRepository(sql: Sql): ReportRepository { await tx` INSERT INTO reports ( id, reporter_user_id, idempotency_key, geom, geom_source, jurisdiction_geoid, - category, type, title, description, addr, status, visibility, h3_cell, reference_code, + category, type, title, description, addr, addr_source, addr_precision, status, + visibility, h3_cell, reference_code, published_at ) VALUES ( ${args.reportId}, @@ -170,6 +171,8 @@ export function makeDrizzleReportRepository(sql: Sql): ReportRepository { ${args.title}, ${args.description}, ${args.addr}, + ${args.addrSource}, + ${args.addrPrecision}, ${args.status}, ${args.visibility}, ${args.h3Cell}, diff --git a/services/api/src/services/report-service.ts b/services/api/src/services/report-service.ts index 15016bfa..cea0cb7b 100644 --- a/services/api/src/services/report-service.ts +++ b/services/api/src/services/report-service.ts @@ -31,6 +31,7 @@ import { type UnsignedReportPin, } from "./report-clustering.js" import { isPubliclyVisibleStatus } from "./report-visibility.js" +import { addressProvenance } from "./address-resolver.js" import { REPORT_AUTOFORWARD_JOB, REPORT_CREATE_SCOPE, @@ -142,6 +143,8 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { ...(record.title !== null ? { title: record.title } : {}), ...(record.description !== null ? { description: record.description } : {}), ...(record.addr !== null ? { addr: record.addr } : {}), + ...(record.addrSource !== null ? { addrSource: record.addrSource } : {}), + ...(record.addrPrecision !== null ? { addrPrecision: record.addrPrecision } : {}), status: record.status, visibility: record.visibility, lat: record.lat, @@ -209,6 +212,9 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { assertNoSlur(input.title ?? null, "title") assertNoSlur(input.description ?? null, "description") + // Events have slur-checked `address` since forever; reports never checked `addr`, and it is the + // same class of public free text rendered on the same surfaces. + assertNoSlur(input.addr ?? null, "addr") const existing = await deps.repo.findIdempotentSnapshot( input.idempotencyKey, @@ -219,10 +225,11 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { const category = REPORT_TYPE_TO_CATEGORY[input.type] - const wantsReverse = !input.addr?.trim() && deps.reverseGeocode !== undefined + const suppliedAddr = input.addr?.trim() ?? "" + const wantsReverse = suppliedAddr.length === 0 && deps.resolveAddress !== undefined const [jurisdictionGeoid, reversed] = await Promise.all([ deps.resolveJurisdictionGeoid(input.lat, input.lng), - wantsReverse ? deps.reverseGeocode!(input.lat, input.lng) : Promise.resolve(null), + wantsReverse ? deps.resolveAddress!(input.lat, input.lng) : Promise.resolve(null), ]) const jurCode = deps.resolveJurisdictionCode !== undefined @@ -231,7 +238,11 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { const h3Cell = reportH3Cell(input.lat, input.lng) const publishedAt = now() const reportId = newId() - const addr = input.addr?.trim() ? input.addr.trim() : reversed + // Provenance, snapshotted with the address itself. 'user' text carries no provider precision by + // definition; a server resolve carries exactly the rung the ladder reached, so a `landmark` line + // can be rendered as "Near X" rather than posing as a postal address. A resolve that came back + // empty leaves all three NULL - filing is never blocked on any of this. + const addressWrite = addressProvenance(suppliedAddr, reversed) const result = await deps.repo.createReportTx({ reportId, @@ -246,7 +257,9 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { type: input.type, title: input.title ?? null, description: input.description ?? null, - addr, + addr: addressWrite.addr, + addrSource: addressWrite.addrSource, + addrPrecision: addressWrite.addrPrecision, status: "published", visibility: "public", h3Cell, diff --git a/services/api/src/services/report-service.types.ts b/services/api/src/services/report-service.types.ts index c334f4e8..e7d65340 100644 --- a/services/api/src/services/report-service.types.ts +++ b/services/api/src/services/report-service.types.ts @@ -1,4 +1,5 @@ import type { + AddressPrecision, CreateReportRequest, GeomSource, ListMyReportsResponse, @@ -6,6 +7,7 @@ import type { PaginationQuery, ReportCategory, ReportClusterResponse, + ReportAddressSource, ReportDTO, ReportStatus, ReportType, @@ -14,6 +16,7 @@ import type { import type { Jobs } from "@civfix/shared/interfaces" import type { LinkedEventView } from "./cleanup-service.js" import type { ReportChatSystemEmitter } from "./report-timeline-event.js" +import type { AddressResolver } from "./address-resolver.js" export const REPORT_CREATE_SCOPE = "report_create" @@ -68,6 +71,8 @@ export interface ReportRecord { title: string | null description: string | null addr: string | null + addrSource: ReportAddressSource | null + addrPrecision: AddressPrecision | null status: ReportStatus visibility: ReportVisibility lat: number @@ -109,6 +114,8 @@ export interface CreateReportTxArgs { title: string | null description: string | null addr: string | null + addrSource: ReportAddressSource | null + addrPrecision: AddressPrecision | null status: ReportStatus visibility: ReportVisibility h3Cell: string @@ -204,7 +211,12 @@ export interface ReportServiceDeps { repo: ReportRepository resolveJurisdictionGeoid: (lat: number, lng: number) => Promise resolveJurisdictionCode?: (geoid: string | null) => Promise - reverseGeocode?: (lat: number, lng: number) => Promise + /** + * Reverse-resolve the pin when the reporter typed no address. Structured now, so the snapshot can + * record the ladder rung it came from and the display layer can be honest about a landmark line. + * Still best-effort: omitted, throwing or empty, the report is filed either way. + */ + resolveAddress?: AddressResolver presignMedia: ( r2Key: string, thumbKey: string | null, diff --git a/services/api/src/services/report-sql.ts b/services/api/src/services/report-sql.ts index 1d7c00f6..4756177d 100644 --- a/services/api/src/services/report-sql.ts +++ b/services/api/src/services/report-sql.ts @@ -1,6 +1,12 @@ import type postgres from "postgres" import type { Queryable } from "../db/client.js" -import type { ReportCategory, ReportStatus, ReportType } from "@civfix/shared" +import type { + AddressPrecision, + ReportAddressSource, + ReportCategory, + ReportStatus, + ReportType, +} from "@civfix/shared" import { PUBLIC_REPORT_STATUSES } from "./report-visibility.js" import type { ReportMapPoint, @@ -33,6 +39,8 @@ export interface ReportRowSelect { title: string | null description: string | null addr: string | null + addr_source: ReportAddressSource | null + addr_precision: AddressPrecision | null status: ReportStatus visibility: "public" | "hidden" lng: number @@ -55,6 +63,8 @@ export function toRecord(r: ReportRowSelect): ReportRecord { title: r.title, description: r.description, addr: r.addr, + addrSource: r.addr_source, + addrPrecision: r.addr_precision, status: r.status, visibility: r.visibility, lat: r.lat, @@ -70,7 +80,8 @@ export function toRecord(r: ReportRowSelect): ReportRecord { export function reportColumns(sql: Queryable) { return sql` - id, reporter_user_id, anon_session_id, category, type, title, description, addr, status, visibility, + id, reporter_user_id, anon_session_id, category, type, title, description, addr, addr_source, + addr_precision, status, visibility, ST_X(geom) AS lng, ST_Y(geom) AS lat, geom_source, jurisdiction_geoid, reference_code, created_at, published_at, deleted_at ` diff --git a/services/api/src/services/social-repository.drizzle.ts b/services/api/src/services/social-repository.drizzle.ts index eec5c366..c80a9870 100644 --- a/services/api/src/services/social-repository.drizzle.ts +++ b/services/api/src/services/social-repository.drizzle.ts @@ -13,6 +13,7 @@ import type { CleanupRecord, CleanupPersonView } from "./cleanup-service.js" import type { CleanupStatus, CleanupType, + EventAddressSource, EventKind, EventVisibility, SocialLinks, @@ -113,6 +114,7 @@ interface CleanupRowSelect { status: CleanupStatus bring: string[] | null address: string | null + address_source: EventAddressSource | null jurisdiction_geoid: string | null reference_code: string | null created_at: Date @@ -149,6 +151,7 @@ function toCleanupRecord(r: CleanupRowSelect): CleanupRecord { status: r.status, bring: r.bring, address: r.address, + addressSource: r.address_source, jurisdictionGeoid: r.jurisdiction_geoid, referenceCode: r.reference_code, createdAt: r.created_at, @@ -208,6 +211,7 @@ function profileEventRows( ${cleanupStatusExpr(sql)} AS status, c.bring, c.address, + c.address_source, c.jurisdiction_geoid, c.reference_code, c.created_at, diff --git a/services/api/test/helpers/cleanups.ts b/services/api/test/helpers/cleanups.ts index 6aba4d84..6ec3b8b4 100644 --- a/services/api/test/helpers/cleanups.ts +++ b/services/api/test/helpers/cleanups.ts @@ -143,6 +143,7 @@ interface StoredCleanup { status: CleanupRecord["status"] bring: string[] | null address: string | null + addressSource: CleanupRecord["addressSource"] capacity: number | null jurisdictionGeoid: string | null referenceCode: string | null @@ -409,6 +410,7 @@ export class InMemoryCleanupRepository implements CleanupRepository { status: seededStatus === "cancelled" ? "cancelled" : "upcoming", bring: over.bring ?? null, address: over.address ?? null, + addressSource: over.addressSource ?? null, capacity: over.capacity ?? null, jurisdictionGeoid: over.jurisdictionGeoid ?? null, referenceCode: over.referenceCode ?? null, @@ -483,6 +485,7 @@ export class InMemoryCleanupRepository implements CleanupRepository { status: deriveCleanupStatus(eventWindowOf(c), this.now().getTime()), bring: c.bring, address: c.address, + addressSource: c.addressSource, capacity: c.capacity, jurisdictionGeoid: c.jurisdictionGeoid, referenceCode: c.referenceCode, @@ -550,6 +553,7 @@ export class InMemoryCleanupRepository implements CleanupRepository { status: args.status, bring: args.bring, address: args.address, + addressSource: args.addressSource, capacity: null, jurisdictionGeoid: args.jurisdictionGeoid, referenceCode, @@ -834,6 +838,7 @@ export class InMemoryCleanupRepository implements CleanupRepository { c.lng = patch.lng } if (patch.address !== undefined) c.address = patch.address + if (patch.addressSource !== undefined) c.addressSource = patch.addressSource if (patch.bring !== undefined) c.bring = patch.bring if (patch.jurisdictionGeoid !== undefined) c.jurisdictionGeoid = patch.jurisdictionGeoid if (patch.endsAt !== undefined) c.endsAt = patch.endsAt diff --git a/services/api/test/helpers/reports.ts b/services/api/test/helpers/reports.ts index 0707d817..f996e133 100644 --- a/services/api/test/helpers/reports.ts +++ b/services/api/test/helpers/reports.ts @@ -114,6 +114,8 @@ export class InMemoryReportRepository implements ReportRepository { title: over.title ?? null, description: over.description ?? null, addr: over.addr ?? null, + addrSource: over.addrSource ?? null, + addrPrecision: over.addrPrecision ?? null, status: over.status ?? "published", visibility: over.visibility ?? "public", lat: over.lat ?? 34.1, @@ -159,6 +161,8 @@ export class InMemoryReportRepository implements ReportRepository { title: args.title, description: args.description, addr: args.addr, + addrSource: args.addrSource, + addrPrecision: args.addrPrecision, status: args.status, visibility: args.visibility, lat: args.lat, diff --git a/services/api/test/helpers/social.ts b/services/api/test/helpers/social.ts index f936f89a..92d6e7e7 100644 --- a/services/api/test/helpers/social.ts +++ b/services/api/test/helpers/social.ts @@ -438,6 +438,7 @@ export function makeCleanupRecord(over: Partial & { organizerUser status: over.status ?? "done", bring: over.bring ?? null, address: over.address ?? null, + addressSource: over.addressSource ?? null, jurisdictionGeoid: over.jurisdictionGeoid ?? null, referenceCode: over.referenceCode ?? null, createdAt: over.createdAt ?? new Date("2024-12-01T10:00:00.000Z"), diff --git a/services/api/test/integration/cleanup-slots-pg.test.ts b/services/api/test/integration/cleanup-slots-pg.test.ts index 7498e0bb..86d358c0 100644 --- a/services/api/test/integration/cleanup-slots-pg.test.ts +++ b/services/api/test/integration/cleanup-slots-pg.test.ts @@ -344,6 +344,7 @@ describe.skipIf(!pg)("signup slots (integration)", () => { status: "upcoming", bring: null, address: null, + addressSource: null, jurisdictionGeoid: null, jurCode: 0, linkedReportIds: [], diff --git a/services/api/test/integration/host-orgs-team-pg.test.ts b/services/api/test/integration/host-orgs-team-pg.test.ts index 1f6f91fb..5ee2bf49 100644 --- a/services/api/test/integration/host-orgs-team-pg.test.ts +++ b/services/api/test/integration/host-orgs-team-pg.test.ts @@ -78,6 +78,7 @@ describe.skipIf(!pg)("host organizations + team (integration)", () => { status: "upcoming", bring: null, address: null, + addressSource: null, jurisdictionGeoid: null, jurCode: 0, linkedReportIds: [], @@ -505,6 +506,7 @@ describe.skipIf(!pg)("host organizations + team (integration)", () => { status: "upcoming" as const, bring: null, address: null, + addressSource: null, jurisdictionGeoid: null, jurCode: 0, linkedReportIds: [], diff --git a/services/api/test/unit/guest-rsvp-service.test.ts b/services/api/test/unit/guest-rsvp-service.test.ts index 9c9b4c6b..f0ccb1ee 100644 --- a/services/api/test/unit/guest-rsvp-service.test.ts +++ b/services/api/test/unit/guest-rsvp-service.test.ts @@ -790,6 +790,7 @@ describe("going: members plus verified, non-cancelled guests", () => { status: "upcoming", bring: null, address: null, + addressSource: null, jurisdictionGeoid: null, jurCode: 1, linkedReportIds: [], diff --git a/services/api/test/unit/report-routes.test.ts b/services/api/test/unit/report-routes.test.ts index ead31c51..44803b32 100644 --- a/services/api/test/unit/report-routes.test.ts +++ b/services/api/test/unit/report-routes.test.ts @@ -35,7 +35,7 @@ async function makeHarness( reportOpts: { geoid?: string | null seed?: (repo: InMemoryReportRepository) => void - reverseGeocode?: (lat: number, lng: number) => Promise + resolveAddress?: ReportServiceOverrides["resolveAddress"] joinReportChatAsOwner?: ReportServiceOverrides["joinReportChatAsOwner"] } = {}, ): Promise { @@ -62,7 +62,7 @@ async function makeHarness( repo, resolveJurisdictionGeoid: () => Promise.resolve("geoid" in reportOpts ? (reportOpts.geoid ?? null) : "0644000"), - ...(reportOpts.reverseGeocode ? { reverseGeocode: reportOpts.reverseGeocode } : {}), + ...(reportOpts.resolveAddress ? { resolveAddress: reportOpts.resolveAddress } : {}), ...(reportOpts.joinReportChatAsOwner ? { joinReportChatAsOwner: reportOpts.joinReportChatAsOwner } : {}), @@ -137,9 +137,13 @@ describe("POST /reports", () => { expect(repo.reports.size).toBe(1) }) - it("reverse-geocodes the pin into addr when the client supplies none", async () => { + it("reverse-geocodes the pin into addr when the client supplies none, and records the rung", async () => { const { app, token } = await makeHarness({ - reverseGeocode: async () => "123 Imperial Hwy, Inglewood, CA", + resolveAddress: async () => ({ + address: "123 Imperial Hwy, Inglewood, CA", + precision: "street", + cityStateLabel: "Inglewood, CA", + }), }) const res = await app.inject({ method: "POST", @@ -156,12 +160,20 @@ describe("POST /reports", () => { }, }) expect(res.statusCode).toBe(201) - expect(res.json().addr).toBe("123 Imperial Hwy, Inglewood, CA") + const dto = res.json() + expect(dto.addr).toBe("123 Imperial Hwy, Inglewood, CA") + // Provenance rides along: the server filled it, at street precision. + expect(dto.addrSource).toBe("resolved") + expect(dto.addrPrecision).toBe("street") }) it("keeps a client-supplied addr instead of reverse-geocoding the pin", async () => { const { app, token } = await makeHarness({ - reverseGeocode: async () => "SHOULD NOT BE USED", + resolveAddress: async () => ({ + address: "SHOULD NOT BE USED", + precision: "street", + cityStateLabel: "Inglewood, CA", + }), }) const res = await app.inject({ method: "POST", @@ -179,7 +191,11 @@ describe("POST /reports", () => { }, }) expect(res.statusCode).toBe(201) - expect(res.json().addr).toBe("NW corner by the bus stop") + const dto = res.json() + expect(dto.addr).toBe("NW corner by the bus stop") + // The reporter's own text is 'user' and carries NO provider precision. + expect(dto.addrSource).toBe("user") + expect(dto.addrPrecision).toBeUndefined() }) // Regression for the live mobile createReport 500 (api.civfix.org, stale deploy): submitReport.ts diff --git a/services/api/test/unit/route-coverage.test.ts b/services/api/test/unit/route-coverage.test.ts index eabea9ec..e04234bc 100644 --- a/services/api/test/unit/route-coverage.test.ts +++ b/services/api/test/unit/route-coverage.test.ts @@ -229,8 +229,8 @@ describe("route-coverage: every shared endpoint is registered (offline boot smok }) } - it("covers ALL 323 endpoints in the registry (no endpoint skipped)", () => { - expect(Object.keys(endpoints).length).toBe(323) + it("covers ALL 324 endpoints in the registry (no endpoint skipped)", () => { + expect(Object.keys(endpoints).length).toBe(324) }) it("the discriminator is not vacuous: a bogus path IS detected as route-missing", async () => { From c4878ac33c46ad8d5a8acbd1fd4f0e7471883d3e Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 19:03:49 -0700 Subject: [PATCH 31/54] cover the address ladder, the cache, the endpoint and both provenance paths Offline throughout - fake providers, a fake geocoder, an in-memory cache and an injected chain, so none of this touches Photon or a database. What the new tests pin, beyond the obvious: - the ladder's ORDER OF CLAIMS, including the rungs the adapters must REFUSE: Photon never answers a city (TIGER owns that rung), Mapbox returns null rather than short-circuiting the chain with a place name, and a residential building's name is never surfaced. - that a throwing provider, a throwing geocoder and an unreachable cache each cost one rung and never an error. - that a negative entry expires in minutes while a resolved one lasts months - the difference between "stop re-firing the chain per pin drag" and "let an outage poison a point for half a year". - that a sub-metre pin nudge is a cache HIT, which is the whole reason the key is the shared 5-decimal one. Found and fixed while writing them: a resolver that REJECTS would have rejected the whole report submit, on both the signed-in and anonymous paths. makeAddressResolver already fails open, but the never-block property belongs at the call site too, so resolveAddressOrNull restates it there rather than inheriting it from whatever dep is wired in. The anon in-memory store now models addr/addr_source/addr_precision, as the real insert writes them. Co-Authored-By: Claude Opus 5 (1M context) --- services/api/src/services/address-resolver.ts | 37 +- services/api/src/services/anon-service.ts | 10 +- services/api/src/services/report-service.ts | 7 +- services/api/test/helpers/anon.ts | 11 + .../api/test/unit/address-resolver.test.ts | 405 ++++++++++++++++++ services/api/test/unit/anon-service.test.ts | 66 +++ .../unit/event-address-verification.test.ts | 289 +++++++++++++ services/api/test/unit/map-routes.test.ts | 90 ++++ services/api/test/unit/report-routes.test.ts | 26 ++ 9 files changed, 928 insertions(+), 13 deletions(-) create mode 100644 services/api/test/unit/address-resolver.test.ts create mode 100644 services/api/test/unit/event-address-verification.test.ts diff --git a/services/api/src/services/address-resolver.ts b/services/api/src/services/address-resolver.ts index 50da4b06..7e8bb39f 100644 --- a/services/api/src/services/address-resolver.ts +++ b/services/api/src/services/address-resolver.ts @@ -70,7 +70,9 @@ export function makeAddressResolver(deps: AddressResolverDeps): AddressResolver if (!Number.isFinite(lat) || !Number.isFinite(lng)) return NOT_RESOLVED const pointKey = geocodePointKey({ lat, lng }) - const cached = await cache.read(pointKey) + // Guarded HERE as well as inside makeGeocodeCache: "never throws" is a property of this function, + // not of whichever cache implementation happens to be wired in. + const cached = await orNull(cache.read(pointKey)) if (cached !== null) { return { address: cached.address, @@ -95,12 +97,14 @@ export function makeAddressResolver(deps: AddressResolverDeps): AddressResolver ? { address: cityStateLabel, precision: "locality", cityStateLabel } : { address: null, precision: null, cityStateLabel } - await cache.write(pointKey, { - address: resolved.address, - precision: resolved.precision, - cityStateLabel, - provider: hit?.provider ?? (resolved.precision === "locality" ? "tiger" : null), - }) + await orNull( + cache.write(pointKey, { + address: resolved.address, + precision: resolved.precision, + cityStateLabel, + provider: hit?.provider ?? (resolved.precision === "locality" ? "tiger" : null), + }), + ) return resolved } @@ -140,3 +144,22 @@ export function addressProvenance( addrPrecision: resolved.precision, } } + +/** + * Call a resolver on the report-create path. `makeAddressResolver` already fails open, but this is the + * ONE place where an address is optional and a filing is not, so the guarantee is re-stated at the call + * site rather than inherited: an injected resolver that rejects (a test double, a future wrapper, a dep + * someone swaps in) must cost the address, never the report. + */ +export async function resolveAddressOrNull( + resolve: AddressResolver | undefined, + lat: number, + lng: number, +): Promise { + if (resolve === undefined) return null + try { + return await resolve(lat, lng) + } catch { + return null + } +} diff --git a/services/api/src/services/anon-service.ts b/services/api/src/services/anon-service.ts index 6250b3fa..eca220f4 100644 --- a/services/api/src/services/anon-service.ts +++ b/services/api/src/services/anon-service.ts @@ -29,7 +29,11 @@ import { } from "../abuse/anon-token.js" import { generateToken, constantTimeStringEqual, sha256Hex } from "../auth/crypto.js" import { UNKNOWN_JURCODE } from "../db/reference-code.js" -import { addressProvenance, type AddressResolver } from "./address-resolver.js" +import { + addressProvenance, + resolveAddressOrNull, + type AddressResolver, +} from "./address-resolver.js" import type { AddressPrecision, ReportAddressSource } from "@civfix/shared" export const ANON_REPORT_CREATE_SCOPE = "anon_report_create" @@ -196,9 +200,9 @@ export function makeAnonService(deps: AnonServiceDeps): AnonService { const suppliedAddr = input.addr?.trim() ?? "" const [jurisdictionGeoid, resolvedAddr] = await Promise.all([ deps.resolveJurisdictionGeoid(input.lat, input.lng), - suppliedAddr.length > 0 || !deps.resolveAddress + suppliedAddr.length > 0 ? Promise.resolve(null) - : deps.resolveAddress(input.lat, input.lng), + : resolveAddressOrNull(deps.resolveAddress, input.lat, input.lng), ]) const jurCode = deps.resolveJurisdictionCode !== undefined diff --git a/services/api/src/services/report-service.ts b/services/api/src/services/report-service.ts index cea0cb7b..b242d913 100644 --- a/services/api/src/services/report-service.ts +++ b/services/api/src/services/report-service.ts @@ -31,7 +31,7 @@ import { type UnsignedReportPin, } from "./report-clustering.js" import { isPubliclyVisibleStatus } from "./report-visibility.js" -import { addressProvenance } from "./address-resolver.js" +import { addressProvenance, resolveAddressOrNull } from "./address-resolver.js" import { REPORT_AUTOFORWARD_JOB, REPORT_CREATE_SCOPE, @@ -226,10 +226,11 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { const category = REPORT_TYPE_TO_CATEGORY[input.type] const suppliedAddr = input.addr?.trim() ?? "" - const wantsReverse = suppliedAddr.length === 0 && deps.resolveAddress !== undefined const [jurisdictionGeoid, reversed] = await Promise.all([ deps.resolveJurisdictionGeoid(input.lat, input.lng), - wantsReverse ? deps.resolveAddress!(input.lat, input.lng) : Promise.resolve(null), + suppliedAddr.length > 0 + ? Promise.resolve(null) + : resolveAddressOrNull(deps.resolveAddress, input.lat, input.lng), ]) const jurCode = deps.resolveJurisdictionCode !== undefined diff --git a/services/api/test/helpers/anon.ts b/services/api/test/helpers/anon.ts index c937ad74..f61d8f3b 100644 --- a/services/api/test/helpers/anon.ts +++ b/services/api/test/helpers/anon.ts @@ -18,6 +18,7 @@ */ import { AppError } from "@civfix/shared" +import type { AddressPrecision, ReportAddressSource } from "@civfix/shared" import type { AnonReportResponse, ReportStatus } from "@civfix/shared" import type { LatLng } from "@civfix/shared" import type { AnonTokenRecord } from "../../src/abuse/anon-token.js" @@ -47,6 +48,10 @@ export interface StoredAnonReport { anonSessionId: string | null category: string description: string | null + /** The creation-time address snapshot and its provenance (0175), as the real insert writes them. */ + addr: string | null + addrSource: ReportAddressSource | null + addrPrecision: AddressPrecision | null status: ReportStatus visibility: "public" | "hidden" lat: number @@ -145,6 +150,9 @@ export class InMemoryAnonStore { anonSessionId: over.anonSessionId ?? null, category: over.category ?? "trash", description: over.description ?? null, + addr: over.addr ?? null, + addrSource: over.addrSource ?? null, + addrPrecision: over.addrPrecision ?? null, status: over.status ?? "held", visibility: over.visibility ?? "public", lat: over.lat ?? 34.1, @@ -245,6 +253,9 @@ export class InMemoryAnonStore { anonSessionId: args.anonSessionId, category: args.category, description: args.description, + addr: args.addr, + addrSource: args.addrSource, + addrPrecision: args.addrPrecision, status: "held", visibility: "public", lat: args.lat, diff --git a/services/api/test/unit/address-resolver.test.ts b/services/api/test/unit/address-resolver.test.ts new file mode 100644 index 00000000..610deca9 --- /dev/null +++ b/services/api/test/unit/address-resolver.test.ts @@ -0,0 +1,405 @@ +/** + * The address ladder (src/services/address-resolver.ts) and its cache (src/services/geocode-cache.ts). + * + * This is the one place a coordinate becomes an address, so what is pinned here is the ORDER OF CLAIMS + * and the failure behavior, not any vendor's wire format (the adapters' own tests cover that): + * + * - the provider chain's rung wins when it has one, verbatim, including a RAW landmark line; + * - a chain miss degrades to the TIGER label at `locality`, which callers read as NOT LOCATED; + * - `cityStateLabel` comes back populated on every path, because the contract promises it; + * - a throwing provider, a throwing geocoder and an unreachable cache each cost one rung and never + * an error - this sits on the report-create path, where a geocoder outage must not cost a filing; + * - a cached point makes NO provider call, and a negative entry expires in minutes while a resolved + * one lasts months. + * + * All offline: fake providers, a fake geocoder and an in-memory cache. + */ + +import { describe, it, expect } from "vitest" +import { FakeGeocoder } from "@civfix/shared/fakes" +import { geocodePointKey } from "@civfix/shared" +import type { ReverseGeocode, ReverseResult } from "../../src/adapters/reverse-geocode.chain.js" +import { + addressProvenance, + makeAddressResolver, + type ResolvedAddress, +} from "../../src/services/address-resolver.js" +import { + GEOCODE_CACHE_NEGATIVE_TTL_MS, + GEOCODE_CACHE_TTL_MS, + isFreshEntry, + makeGeocodeCache, + type GeocodeCache, + type GeocodeCacheEntry, +} from "../../src/services/geocode-cache.js" +import type { Sql } from "../../src/db/client.js" + +const LA = { lat: 34.05223, lng: -118.24368 } + +/** A chain stand-in that records every call, so "did this cost a provider call" is assertable. */ +function chain(result: ReverseResult | null): ReverseGeocode & { calls: number } { + const fn = Object.assign( + async (): Promise => { + fn.calls += 1 + return result + }, + { calls: 0 }, + ) + return fn +} + +function memoryCache(): GeocodeCache & { rows: Map; writes: number } { + const rows = new Map() + const cache = { + rows, + writes: 0, + read: async (key: string) => rows.get(key) ?? null, + write: async (key: string, entry: GeocodeCacheEntry) => { + cache.writes += 1 + rows.set(key, entry) + }, + } + return cache +} + +describe("makeAddressResolver ladder", () => { + it("takes the chain's rung verbatim and still reports the locality hint", async () => { + const resolve = makeAddressResolver({ + streetReverseGeocode: chain({ + line: "123 Main St, Inglewood, CA", + precision: "street", + provider: "photon", + }), + geocoder: new FakeGeocoder(), + }) + + await expect(resolve(LA.lat, LA.lng)).resolves.toEqual({ + address: "123 Main St, Inglewood, CA", + precision: "street", + cityStateLabel: "Los Angeles, CA", + }) + }) + + it("passes a landmark line through RAW - the 'Near ' prefix is a display concern", async () => { + const resolve = makeAddressResolver({ + streetReverseGeocode: chain({ + line: "Vista Hermosa Park, Los Angeles, CA", + precision: "landmark", + provider: "photon", + }), + geocoder: new FakeGeocoder(), + }) + + const out = await resolve(LA.lat, LA.lng) + expect(out.address).toBe("Vista Hermosa Park, Los Angeles, CA") + expect(out.address?.startsWith("Near ")).toBe(false) + expect(out.precision).toBe("landmark") + }) + + it("degrades a chain miss to the TIGER label at locality", async () => { + const resolve = makeAddressResolver({ + streetReverseGeocode: chain(null), + geocoder: new FakeGeocoder(), + }) + + await expect(resolve(LA.lat, LA.lng)).resolves.toEqual({ + address: "Los Angeles, CA", + precision: "locality", + cityStateLabel: "Los Angeles, CA", + }) + }) + + it("answers nothing-at-all when both halves come up empty, with cityStateLabel still a string", async () => { + const resolve = makeAddressResolver({ + streetReverseGeocode: chain(null), + geocoder: { cityStateLabel: async () => null }, + }) + + await expect(resolve(LA.lat, LA.lng)).resolves.toEqual({ + address: null, + precision: null, + cityStateLabel: "", + }) + }) + + it("a THROWING provider costs one rung, never an error", async () => { + const resolve = makeAddressResolver({ + streetReverseGeocode: async () => { + throw new Error("vendor down") + }, + geocoder: new FakeGeocoder(), + }) + + await expect(resolve(LA.lat, LA.lng)).resolves.toEqual({ + address: "Los Angeles, CA", + precision: "locality", + cityStateLabel: "Los Angeles, CA", + }) + }) + + it("a THROWING geocoder still lets the chain's answer through", async () => { + const resolve = makeAddressResolver({ + streetReverseGeocode: chain({ + line: "123 Main St, Inglewood, CA", + precision: "street", + provider: "mapbox", + }), + geocoder: { + cityStateLabel: async () => { + throw new Error("postgis down") + }, + }, + }) + + await expect(resolve(LA.lat, LA.lng)).resolves.toEqual({ + address: "123 Main St, Inglewood, CA", + precision: "street", + cityStateLabel: "", + }) + }) + + it("does not call a provider for a non-finite coordinate", async () => { + const provider = chain({ line: "x", precision: "street", provider: "photon" }) + const resolve = makeAddressResolver({ + streetReverseGeocode: provider, + geocoder: new FakeGeocoder(), + }) + + await expect(resolve(Number.NaN, LA.lng)).resolves.toEqual({ + address: null, + precision: null, + cityStateLabel: "", + }) + expect(provider.calls).toBe(0) + }) +}) + +describe("makeAddressResolver caching", () => { + it("reads through: the second resolve of a point makes NO provider call", async () => { + const provider = chain({ + line: "123 Main St, Inglewood, CA", + precision: "street", + provider: "photon", + }) + const cache = memoryCache() + const resolve = makeAddressResolver({ + streetReverseGeocode: provider, + geocoder: new FakeGeocoder(), + cache, + }) + + await resolve(LA.lat, LA.lng) + await resolve(LA.lat, LA.lng) + + expect(provider.calls).toBe(1) + expect(cache.writes).toBe(1) + }) + + it("keys on the SHARED 5-decimal point key, so a sub-metre pin nudge is a hit", async () => { + const provider = chain({ line: "Main St, Inglewood, CA", precision: "intersection", provider: "photon" }) + const cache = memoryCache() + const resolve = makeAddressResolver({ + streetReverseGeocode: provider, + geocoder: new FakeGeocoder(), + cache, + }) + + await resolve(LA.lat, LA.lng) + // Well inside the 5th decimal (~1.1 m), which is exactly what a dragged pin produces. + await resolve(LA.lat + 0.000001, LA.lng - 0.000002) + + expect(provider.calls).toBe(1) + expect([...cache.rows.keys()]).toEqual([geocodePointKey(LA)]) + }) + + it("caches the NEGATIVE result too, so a dead point does not re-fire the chain per drag", async () => { + const provider = chain(null) + const cache = memoryCache() + const resolve = makeAddressResolver({ + streetReverseGeocode: provider, + geocoder: { cityStateLabel: async () => null }, + cache, + }) + + await resolve(LA.lat, LA.lng) + await resolve(LA.lat, LA.lng) + + expect(provider.calls).toBe(1) + expect(cache.rows.get(geocodePointKey(LA))).toEqual({ + address: null, + precision: null, + cityStateLabel: "", + provider: null, + }) + }) + + it("records which adapter answered, and attributes the locality rung to tiger", async () => { + const cache = memoryCache() + await makeAddressResolver({ + streetReverseGeocode: chain(null), + geocoder: new FakeGeocoder(), + cache, + })(LA.lat, LA.lng) + + expect(cache.rows.get(geocodePointKey(LA))?.provider).toBe("tiger") + }) + + it("an UNREACHABLE cache degrades to no caching, never to an error", async () => { + const provider = chain({ line: "123 Main St, Inglewood, CA", precision: "street", provider: "photon" }) + const resolve = makeAddressResolver({ + streetReverseGeocode: provider, + geocoder: new FakeGeocoder(), + cache: { + read: async () => { + throw new Error("no db") + }, + write: async () => { + throw new Error("no db") + }, + }, + }) + + await expect(resolve(LA.lat, LA.lng)).resolves.toEqual({ + address: "123 Main St, Inglewood, CA", + precision: "street", + cityStateLabel: "Los Angeles, CA", + }) + expect(provider.calls).toBe(1) + }) +}) + +/** + * The cache adapter itself. `makeGeocodeCache` is the layer that must swallow a DB that is missing, + * locked or absent entirely (the fully-faked offline server has no connection at all), so the resolver + * above can treat it as infallible. + */ +function fakeSql(rows: unknown[], calls: string[] = []): Sql { + const fn = (strings: TemplateStringsArray, ..._values: unknown[]): Promise => { + calls.push(strings.join("")) + return Promise.resolve(rows) + } + return fn as unknown as Sql +} + +describe("makeGeocodeCache", () => { + it("returns a fresh row", async () => { + const cache = makeGeocodeCache({ + getSql: () => + fakeSql([ + { + address: "123 Main St, Inglewood, CA", + address_precision: "street", + city_state_label: "Inglewood, CA", + provider: "photon", + resolved_at: new Date(), + }, + ]), + }) + + await expect(cache.read("34.05223,-118.24368")).resolves.toEqual({ + address: "123 Main St, Inglewood, CA", + precision: "street", + cityStateLabel: "Inglewood, CA", + provider: "photon", + }) + }) + + it("reports a row past the positive TTL as a MISS, so it is re-resolved and overwritten", async () => { + const stale = new Date(Date.now() - GEOCODE_CACHE_TTL_MS - 1000) + const cache = makeGeocodeCache({ + getSql: () => + fakeSql([ + { + address: "123 Main St, Inglewood, CA", + address_precision: "street", + city_state_label: "Inglewood, CA", + provider: "photon", + resolved_at: stale, + }, + ]), + }) + + await expect(cache.read("34.05223,-118.24368")).resolves.toBeNull() + }) + + it("expires a NEGATIVE row in minutes: an outage cannot poison a point for months", () => { + const now = new Date() + const aged = (ms: number): Date => new Date(now.getTime() - ms) + + // A negative entry an hour old is already gone... + expect(isFreshEntry({ address: null, resolvedAt: aged(60 * 60 * 1000) }, now)).toBe(false) + // ...while it is still trusted inside its short window (that is what stops the re-fire storm)... + expect(isFreshEntry({ address: null, resolvedAt: aged(GEOCODE_CACHE_NEGATIVE_TTL_MS / 2) }, now)).toBe( + true, + ) + // ...and a resolved line an hour old is nowhere near expiry. + expect(isFreshEntry({ address: "123 Main St", resolvedAt: aged(60 * 60 * 1000) }, now)).toBe(true) + }) + + it("swallows a DB that is not there at all (the offline faked server)", async () => { + const cache = makeGeocodeCache({ + getSql: () => { + throw new Error("no database configured") + }, + }) + + await expect(cache.read("34.05223,-118.24368")).resolves.toBeNull() + await expect( + cache.write("34.05223,-118.24368", { + address: null, + precision: null, + cityStateLabel: "", + provider: null, + }), + ).resolves.toBeUndefined() + }) +}) + +describe("addressProvenance", () => { + const resolved: ResolvedAddress = { + address: "123 Main St, Inglewood, CA", + precision: "street", + cityStateLabel: "Inglewood, CA", + } + + it("marks the reporter's own text 'user' with NO precision", () => { + expect(addressProvenance("NW corner by the bus stop", resolved)).toEqual({ + addr: "NW corner by the bus stop", + addrSource: "user", + addrPrecision: null, + }) + }) + + it("marks a server resolve 'resolved' and keeps the rung it reached", () => { + expect(addressProvenance("", resolved)).toEqual({ + addr: "123 Main St, Inglewood, CA", + addrSource: "resolved", + addrPrecision: "street", + }) + }) + + it("keeps a locality-grade snapshot, labelled honestly rather than dropped", () => { + expect( + addressProvenance("", { address: "Los Angeles, CA", precision: "locality", cityStateLabel: "Los Angeles, CA" }), + ).toEqual({ addr: "Los Angeles, CA", addrSource: "resolved", addrPrecision: "locality" }) + }) + + it("leaves all three NULL when nothing resolved - filing is never blocked on this", () => { + expect(addressProvenance("", null)).toEqual({ + addr: null, + addrSource: null, + addrPrecision: null, + }) + expect(addressProvenance("", { address: null, precision: null, cityStateLabel: "" })).toEqual({ + addr: null, + addrSource: null, + addrPrecision: null, + }) + }) + + it("clamps a server-derived line to the wire cap, which never applied to it", () => { + const long = "A".repeat(500) + const out = addressProvenance("", { address: long, precision: "street", cityStateLabel: "" }) + expect(out.addr).toHaveLength(300) + }) +}) diff --git a/services/api/test/unit/anon-service.test.ts b/services/api/test/unit/anon-service.test.ts index 2eba6bed..54325103 100644 --- a/services/api/test/unit/anon-service.test.ts +++ b/services/api/test/unit/anon-service.test.ts @@ -66,6 +66,7 @@ function req(over: Partial = {}): AnonReportRequest { ...(over.description !== undefined ? { description: over.description } : {}), ...(over.honeypot !== undefined ? { honeypot: over.honeypot } : {}), ...(over.anonToken !== undefined ? { anonToken: over.anonToken } : {}), + ...(over.addr !== undefined ? { addr: over.addr } : {}), } } @@ -544,3 +545,68 @@ function uuid(n: number): string { const h = n.toString(16).padStart(12, "0") return `00000000-0000-4000-8000-${h}` } + +/** + * Address provenance on the anonymous path. It shares one helper with the signed-in path by + * construction, so what is worth pinning here is the WIRING: the resolver is consulted only when the + * reporter typed nothing, and the resulting source/precision land on the row alongside the address. + */ +describe("submitAnonReport: address provenance", () => { + it("records the reporter's own text as 'user', with no provider precision", async () => { + const { store, service } = makeHarness({ + resolveAddress: async () => ({ + address: "SHOULD NOT BE USED", + precision: "street", + cityStateLabel: "Inglewood, CA", + }), + }) + await service.submitAnonReport(req({ addr: "NW corner by the bus stop" }), ctx) + + const stored = store.reports.get("report-1")! + expect(stored.addr).toBe("NW corner by the bus stop") + expect(stored.addrSource).toBe("user") + expect(stored.addrPrecision).toBeNull() + }) + + it("snapshots a server resolve as 'resolved' with the rung it reached", async () => { + const { store, service } = makeHarness({ + resolveAddress: async () => ({ + address: "Vista Hermosa Park, Los Angeles, CA", + precision: "landmark", + cityStateLabel: "Los Angeles, CA", + }), + }) + await service.submitAnonReport(req(), ctx) + + const stored = store.reports.get("report-1")! + // Stored RAW: the "Near " prefix belongs to the viewer's locale, and addrPrecision is what earns it. + expect(stored.addr).toBe("Vista Hermosa Park, Los Angeles, CA") + expect(stored.addrSource).toBe("resolved") + expect(stored.addrPrecision).toBe("landmark") + }) + + it("files the report anyway when the resolver throws - an outage never costs a filing", async () => { + const { store, service } = makeHarness({ + resolveAddress: async () => { + throw new Error("geocoder down") + }, + }) + + await expect(service.submitAnonReport(req(), ctx)).resolves.toMatchObject({ + response: { status: "held" }, + }) + const stored = store.reports.get("report-1")! + expect(stored.addr).toBeNull() + expect(stored.addrSource).toBeNull() + }) + + it("leaves all three NULL with no resolver wired at all", async () => { + const { store, service } = makeHarness() + await service.submitAnonReport(req(), ctx) + + const stored = store.reports.get("report-1")! + expect(stored.addr).toBeNull() + expect(stored.addrSource).toBeNull() + expect(stored.addrPrecision).toBeNull() + }) +}) diff --git a/services/api/test/unit/event-address-verification.test.ts b/services/api/test/unit/event-address-verification.test.ts new file mode 100644 index 00000000..7639bb3b --- /dev/null +++ b/services/api/test/unit/event-address-verification.test.ts @@ -0,0 +1,289 @@ +/** + * Host-verified event addresses (cleanup-service). + * + * Events were never reverse-geocoded: `cleanups.address` held whatever the host typed into "name the + * spot", or nothing. New clients now resolve the pin, show the line to the host and publish with it on + * screen, and the server records WHICH of those happened. The whole design rests on one discriminator, + * so it is what these tests pin: + * + * `addressSource` PRESENT = a new client. The host saw the line; the server only refuses a blank one. + * `addressSource` ABSENT = an old client (the wire keeps `address` optional precisely so the build + * in someone's pocket keeps working). Its text is 'manual'; its silence + * triggers the compat shim. + * + * And the shim's own limit: it stores a resolved line only at a LOCATED rung. "Los Angeles, CA" is not + * a meeting address, and writing it would dress a non-answer up as host-provided. + */ + +import { describe, it, expect, beforeEach } from "vitest" +import { TEST_TICKET_SIGNER } from "../helpers/ticket-signer.js" +import { makeCleanupService, type CleanupService } from "../../src/services/cleanup-service.js" +import { InMemoryCleanupRepository } from "../helpers/cleanups.js" +import { InMemoryCounterStore } from "../../src/abuse/counter-store.js" +import type { AddressResolver } from "../../src/services/address-resolver.js" +import type { CreateCleanupRequest } from "@civfix/shared" + +const HOST = "22222222-2222-2222-2222-222222222222" + +let repo: InMemoryCleanupRepository + +function baseInput(over: Partial = {}): CreateCleanupRequest { + return { + title: "Beach cleanup", + type: "site", + eventKind: "cleanup", + lat: 34.0, + lng: -118.49, + scheduledAt: new Date(Date.now() + 86_400_000).toISOString(), + slots: [{ title: "Volunteers" }], + ...over, + } +} + +/** A resolver that records its calls, so "did the shim even run" is assertable. */ +function resolver( + result: Awaited>, +): AddressResolver & { calls: number } { + const fn = Object.assign( + async (): Promise>> => { + fn.calls += 1 + return result + }, + { calls: 0 }, + ) + return fn +} + +const STREET = { + address: "123 Imperial Hwy, Inglewood, CA", + precision: "street", + cityStateLabel: "Inglewood, CA", +} as const + +const LOCALITY_ONLY = { + address: "Los Angeles, CA", + precision: "locality", + cityStateLabel: "Los Angeles, CA", +} as const + +function serviceWith(resolveAddress?: AddressResolver): CleanupService { + return makeCleanupService({ + tickets: TEST_TICKET_SIGNER, + repo, + counters: new InMemoryCounterStore(), + ...(resolveAddress !== undefined ? { resolveAddress } : {}), + }) +} + +beforeEach(() => { + repo = new InMemoryCleanupRepository() + repo.seedUser({ id: HOST, displayName: "Holly Host", handle: "holly" }) +}) + +describe("createCleanup: a NEW client's confirmed address", () => { + it("stores the host-confirmed line with its source and serves both on the DTO", async () => { + const resolve = resolver(STREET) + const dto = await serviceWith(resolve).createCleanup( + baseInput({ address: "123 Imperial Hwy, Inglewood, CA", addressSource: "resolved" }), + HOST, + ) + + expect(dto.address).toBe("123 Imperial Hwy, Inglewood, CA") + expect(dto.addressSource).toBe("resolved") + // The host already confirmed it; the server has nothing to resolve. + expect(resolve.calls).toBe(0) + }) + + it("keeps 'edited' and 'manual' verbatim - the server never re-judges a host's own wording", async () => { + const edited = await serviceWith().createCleanup( + baseInput({ address: "Boathouse dock, 123 Imperial Hwy", addressSource: "edited" }), + HOST, + ) + expect(edited.addressSource).toBe("edited") + + const manual = await serviceWith().createCleanup( + baseInput({ address: "North gate, by the flagpole", addressSource: "manual" }), + HOST, + ) + expect(manual.addressSource).toBe("manual") + }) + + it("trims before storing", async () => { + const dto = await serviceWith().createCleanup( + baseInput({ address: " North gate ", addressSource: "manual" }), + HOST, + ) + expect(dto.address).toBe("North gate") + }) + + it("REFUSES a source with a blank or too-short address - that combination is a client bug", async () => { + const service = serviceWith() + await expect( + service.createCleanup(baseInput({ address: " ", addressSource: "manual" }), HOST), + ).rejects.toMatchObject({ code: "VALIDATION" }) + + await expect( + service.createCleanup(baseInput({ address: "NW", addressSource: "manual" }), HOST), + ).rejects.toMatchObject({ code: "VALIDATION" }) + + await expect( + service.createCleanup(baseInput({ addressSource: "resolved" }), HOST), + ).rejects.toMatchObject({ code: "VALIDATION" }) + + expect(repo.cleanups.size).toBe(0) + }) +}) + +describe("createCleanup: the OLD-client compat shim", () => { + it("treats an old client's text as the host's own, so it is 'manual'", async () => { + const resolve = resolver(STREET) + const dto = await serviceWith(resolve).createCleanup(baseInput({ address: "Boathouse dock" }), HOST) + + expect(dto.address).toBe("Boathouse dock") + expect(dto.addressSource).toBe("manual") + expect(resolve.calls).toBe(0) + }) + + it("resolves the pin when an old client sent nothing at all, and marks it 'resolved'", async () => { + const resolve = resolver(STREET) + const dto = await serviceWith(resolve).createCleanup(baseInput(), HOST) + + expect(resolve.calls).toBe(1) + expect(dto.address).toBe("123 Imperial Hwy, Inglewood, CA") + expect(dto.addressSource).toBe("resolved") + }) + + it("stores NOTHING when the ladder only reached locality - a city is not a meeting address", async () => { + const dto = await serviceWith(resolver(LOCALITY_ONLY)).createCleanup(baseInput(), HOST) + + expect(dto.address).toBeNull() + expect(dto.addressSource).toBeNull() + }) + + it("stores nothing when the resolve came up empty, and still creates the event", async () => { + const dto = await serviceWith( + resolver({ address: null, precision: null, cityStateLabel: "" }), + ).createCleanup(baseInput(), HOST) + + expect(dto.address).toBeNull() + expect(dto.addressSource).toBeNull() + expect(repo.cleanups.has(dto.id)).toBe(true) + }) + + it("is a no-op with no resolver wired (the pre-0.51.0 behavior, unchanged)", async () => { + const dto = await serviceWith().createCleanup(baseInput(), HOST) + expect(dto.address).toBeNull() + expect(dto.addressSource).toBeNull() + }) +}) + +describe("updateCleanup: the same rules on the edit path", () => { + it("persists a new client's amended line and flips the source to 'edited'", async () => { + const service = serviceWith() + const created = await service.createCleanup( + baseInput({ address: "123 Imperial Hwy, Inglewood, CA", addressSource: "resolved" }), + HOST, + ) + + const updated = await service.updateCleanup( + created.id, + { address: "Boathouse dock, 123 Imperial Hwy", addressSource: "edited" }, + HOST, + ) + + expect(updated.address).toBe("Boathouse dock, 123 Imperial Hwy") + expect(updated.addressSource).toBe("edited") + }) + + it("REFUSES to blank out an address that arrives with a source", async () => { + const service = serviceWith() + const created = await service.createCleanup( + baseInput({ address: "North gate", addressSource: "manual" }), + HOST, + ) + + await expect( + service.updateCleanup(created.id, { address: "", addressSource: "manual" }, HOST), + ).rejects.toMatchObject({ code: "VALIDATION" }) + }) + + it("demotes an old client's edit to 'manual' - it typed the text itself", async () => { + const service = serviceWith() + const created = await service.createCleanup( + baseInput({ address: "123 Imperial Hwy, Inglewood, CA", addressSource: "resolved" }), + HOST, + ) + + const updated = await service.updateCleanup(created.id, { address: "Boathouse dock" }, HOST) + + expect(updated.address).toBe("Boathouse dock") + expect(updated.addressSource).toBe("manual") + }) + + it("an old client clearing the field clears the source with it", async () => { + const service = serviceWith() + const created = await service.createCleanup( + baseInput({ address: "North gate", addressSource: "manual" }), + HOST, + ) + + const updated = await service.updateCleanup(created.id, { address: "" }, HOST) + + expect(updated.address).toBeNull() + expect(updated.addressSource).toBeNull() + }) + + it("leaves both alone when the patch does not mention the address", async () => { + const service = serviceWith() + const created = await service.createCleanup( + baseInput({ address: "North gate", addressSource: "manual" }), + HOST, + ) + + const updated = await service.updateCleanup(created.id, { title: "Renamed sweep" }, HOST) + + expect(updated.address).toBe("North gate") + expect(updated.addressSource).toBe("manual") + }) +}) + +describe("duplicateCleanup", () => { + it("carries the verified address and its provenance over as a pair", async () => { + const service = serviceWith() + const created = await service.createCleanup( + baseInput({ address: "123 Imperial Hwy, Inglewood, CA", addressSource: "edited" }), + HOST, + ) + + const copy = await service.duplicateCleanup(HOST, { + id: created.id, + scheduledAt: new Date(Date.now() + 7 * 86_400_000).toISOString(), + endsAt: new Date(Date.now() + 7 * 86_400_000 + 4 * 3_600_000).toISOString(), + includeTicketTypes: false, + includeQuestions: false, + includePage: false, + }) + + expect(copy.id).not.toBe(created.id) + expect(copy.address).toBe("123 Imperial Hwy, Inglewood, CA") + // NOT demoted to 'manual': a duplicate is the same place at a new time. + expect(copy.addressSource).toBe("edited") + }) + + it("duplicating an addressless legacy event stays addressless", async () => { + const service = serviceWith() + const created = await service.createCleanup(baseInput(), HOST) + + const copy = await service.duplicateCleanup(HOST, { + id: created.id, + scheduledAt: new Date(Date.now() + 7 * 86_400_000).toISOString(), + endsAt: new Date(Date.now() + 7 * 86_400_000 + 4 * 3_600_000).toISOString(), + includeTicketTypes: false, + includeQuestions: false, + includePage: false, + }) + + expect(copy.address).toBeNull() + expect(copy.addressSource).toBeNull() + }) +}) diff --git a/services/api/test/unit/map-routes.test.ts b/services/api/test/unit/map-routes.test.ts index d74b41ca..4343fc1f 100644 --- a/services/api/test/unit/map-routes.test.ts +++ b/services/api/test/unit/map-routes.test.ts @@ -3,6 +3,11 @@ import type { FastifyInstance } from "fastify" import { buildServer } from "../../src/server.js" import { buildContainer } from "../../src/di.js" import { loadEnv } from "../../src/env.js" +import type { Container } from "../../src/di.js" +import type { + ReverseGeocode, + ReverseResult, +} from "../../src/adapters/reverse-geocode.chain.js" /** * Route-level tests for the map plugin that need NO database: tileinfo (pure env read) and @@ -92,6 +97,91 @@ describe("POST /map/reverse-label", () => { }) }) +/** + * The street-level preview the creation flows call. Runs on the fake geocoder for the locality half and + * an injected chain for the street half, so it is offline: the container's real chain is a live Photon + * request, which has no business in a unit test. There is no database here either, which is itself part + * of the contract - the cache swallows its own unavailability and the endpoint still answers. + */ +function withStreetChain(container: Container, result: ReverseResult | null): Container { + ;(container as { streetReverseGeocode: ReverseGeocode }).streetReverseGeocode = async () => result + return container +} + +describe("POST /map/resolve-address", () => { + it("returns the chain's rung plus the locality hint", async () => { + const env = loadEnv({ NODE_ENV: "test" }) + const container = withStreetChain(buildContainer(env), { + line: "123 Imperial Hwy, Inglewood, CA", + precision: "street", + provider: "photon", + }) + app = await buildServer({ env, container }) + + const res = await app.inject({ + method: "POST", + url: "/v1/map/resolve-address", + payload: { lat: 34.05, lng: -118.25 }, + }) + expect(res.statusCode).toBe(200) + expect(res.json()).toEqual({ + address: "123 Imperial Hwy, Inglewood, CA", + precision: "street", + // FakeGeocoder defaults to "Los Angeles, CA". + cityStateLabel: "Los Angeles, CA", + }) + }) + + it("falls back to locality on a chain miss, and says so", async () => { + const env = loadEnv({ NODE_ENV: "test" }) + const container = withStreetChain(buildContainer(env), null) + app = await buildServer({ env, container }) + + const res = await app.inject({ + method: "POST", + url: "/v1/map/resolve-address", + payload: { lat: 34.05, lng: -118.25 }, + }) + expect(res.statusCode).toBe(200) + // precision 'locality' is what tells the client to ask the host to type an address instead. + expect(res.json()).toEqual({ + address: "Los Angeles, CA", + precision: "locality", + cityStateLabel: "Los Angeles, CA", + }) + }) + + it("returns a landmark line RAW, leaving the 'Near ' prefix to the client's locale", async () => { + const env = loadEnv({ NODE_ENV: "test" }) + const container = withStreetChain(buildContainer(env), { + line: "Vista Hermosa Park, Los Angeles, CA", + precision: "landmark", + provider: "photon", + }) + app = await buildServer({ env, container }) + + const res = await app.inject({ + method: "POST", + url: "/v1/map/resolve-address", + payload: { lat: 34.05, lng: -118.25 }, + }) + expect(res.statusCode).toBe(200) + expect(res.json().address).toBe("Vista Hermosa Park, Los Angeles, CA") + expect(res.json().precision).toBe("landmark") + }) + + it("rejects a malformed body with the 422 validation envelope", async () => { + app = await buildServer({ env: loadEnv() }) + const res = await app.inject({ + method: "POST", + url: "/v1/map/resolve-address", + payload: { lat: 999, lng: -118.25 }, + }) + expect(res.statusCode).toBe(422) + expect(res.json().code).toBe("VALIDATION") + }) +}) + describe("POST /map/jurisdictions/:geoid/suggest-contact (validation, no DB)", () => { // Body validation runs BEFORE any DB access, so the "needs a contact" + bad-email cases are clean 422s // with no Postgres wired. The 404 (unknown geoid) and 201 (audit written) paths need the DB and are diff --git a/services/api/test/unit/report-routes.test.ts b/services/api/test/unit/report-routes.test.ts index 44803b32..90d21246 100644 --- a/services/api/test/unit/report-routes.test.ts +++ b/services/api/test/unit/report-routes.test.ts @@ -167,6 +167,32 @@ describe("POST /reports", () => { expect(dto.addrPrecision).toBe("street") }) + it("files the report anyway when the resolver throws - an outage never costs a filing", async () => { + const { app, token } = await makeHarness({ + resolveAddress: async () => { + throw new Error("geocoder down") + }, + }) + const res = await app.inject({ + method: "POST", + url: "/v1/reports", + headers: auth(token), + payload: { + idempotencyKey: KEY_A, + category: "trash", + type: "dump", + lat: 33.95, + lng: -118.35, + geomSource: "device", + mediaUploadIds: [], + }, + }) + expect(res.statusCode).toBe(201) + const dto = res.json() + expect(dto.addr).toBeUndefined() + expect(dto.addrSource).toBeUndefined() + }) + it("keeps a client-supplied addr instead of reverse-geocoding the pin", async () => { const { app, token } = await makeHarness({ resolveAddress: async () => ({ From 64e1162b8d123c2a9b2e988734de5f976d5df409 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 19:36:40 -0700 Subject: [PATCH 32/54] cache only a chain answer, and sweep the geocode cache The locality rung was written to geocode_cache with the 180-day positive TTL, so one Photon/Mapbox blip durably pinned every in-coverage point it touched at city grade. It is not cached at all now: TIGER is a free local query the resolver re-runs per request, and a chain miss leaves the 15-minute negative row instead, which still absorbs a dragged pin and still lets the chain re-run (and upgrade the point) after the blip. A fresh negative row also suppresses the chain without rewriting itself, so a continuously-resolved point cannot push its own expiry out forever. isFreshEntry now decides from the same "did the chain prove a located rung" test, so a row written by the deployed build gets the short TTL too. Nothing ever deleted from geocode_cache - TTLs are applied on read and expired rows are overwritten in place - while POST /map/resolve-address lets an unauthenticated caller mint rows at 30/min/IP. retention.sweep gains a geocode_cache lane at the positive TTL, bounded like every other lane and riding the index 0175 already created. --- docs/retention-cleanup.md | 32 +++- services/api/package.json | 1 + services/api/src/services/address-resolver.ts | 46 +++-- services/api/src/services/geocode-cache.ts | 52 ++++-- .../api/test/unit/address-resolver.test.ts | 159 ++++++++++++++++-- .../media-worker/src/jobs/retention-sweep.ts | 23 +++ .../test/unit/retention-sweep.test.ts | 33 +++- 7 files changed, 299 insertions(+), 47 deletions(-) diff --git a/docs/retention-cleanup.md b/docs/retention-cleanup.md index e9b98042..5e9c4579 100644 --- a/docs/retention-cleanup.md +++ b/docs/retention-cleanup.md @@ -12,7 +12,8 @@ previously accumulated forever (no existing path deleted them). A `retention.sweep` cron in the **media-worker** (the same process that runs `orphan.sweep` and `chat.partition.maintenance`) deletes expired rows from the tables below. Every lane but `inbound_emails` deletes already-expired rows only -and needs no schema of its own. +and needs no schema of its own (`geocode_cache` brings its own index, created +with the table). | Table | Rows deleted | Source schema | |---|---|---| @@ -21,6 +22,7 @@ and needs no schema of its own. | `sessions` | `expires_at < cutoff` | `services/api/src/db/schema/sessions.ts` | | `idempotency_keys` | `created_at < now - 48h` (`RETENTION_IDEMPOTENCY_MS`) | `services/api/src/db/schema/idempotency.ts` | | `notifications` | `created_at < now - 90d` (see F088 below) | `services/api/src/db/schema/notifications.ts` | +| `geocode_cache` | `resolved_at < now - 180d` (`GEOCODE_CACHE_TTL_MS`, see below) | `services/api/src/db/schema/geocode_cache.ts` | | `inbound_emails` | `archived_at < now - 180d` (see H10 below) | `services/api/src/db/schema/inbound_emails.ts` | `cutoff = now - grace`, where `grace` defaults to **1 hour** past expiry (so a @@ -141,6 +143,34 @@ lane (`src/routes/chat-gateway-wiring.ts`) is the remaining step. --- +## `geocode_cache` — bounding the reverse-geocode cache (0175) + +`geocode_cache` (migration `0175_address_resolution.sql`) is written by the +address ladder on every resolve, and the write path is reachable by an +UNAUTHENTICATED caller (`POST /map/resolve-address`, 30/min/IP). Its TTLs are +applied on READ — an expired row is served as a miss and OVERWRITTEN in place — +so nothing in `services/api/src/services/geocode-cache.ts` ever deletes a row, +and a point that is resolved once and never visited again stays forever. Rows are +tiny and derived (a public coordinate → a public address line, no user, report or +event reference, outside the erasure lane by construction), but "tiny × forever" +is still unbounded, so the sweep owns the deletion side. + +| What | Value | +|---|---| +| Table | `geocode_cache` | +| Predicate | `resolved_at < now() - 180 days` (`GEOCODE_CACHE_TTL_MS`, the POSITIVE TTL — imported from `@civfix/api/geocode-cache` so the two can never drift) | +| Index | `geocode_cache_resolved_at_idx` (created by 0175) | +| Cron | the existing `retention.sweep` (`37 4 * * *`, daily) | +| Batching | `DELETE … WHERE point_key IN (SELECT point_key … LIMIT n)`, drained page-wise like the other lanes | +| Tuning | `runRetentionSweep({ geocodeCacheRetentionMs })` | + +The cutoff is the POSITIVE TTL deliberately, even though a non-chain row (a chain +miss) stops being SERVED after 15 minutes: once past 180 days a row cannot be +served on any path, so deleting it loses nothing, and one cutoff keeps the lane a +single index scan. + +--- + ## F106 — `inbound/failed/` poison-message store (adminmail-a) The inbound-mail processor parks parse-poisoned or over-cap `.eml` objects under diff --git a/services/api/package.json b/services/api/package.json index d117730d..bedaa47d 100644 --- a/services/api/package.json +++ b/services/api/package.json @@ -12,6 +12,7 @@ "./db": "./src/db/worker-db.ts", "./media-repo": "./src/services/media-worker-repo.ts", "./inbound-retention-repo": "./src/services/admin/inbound-retention-repository.drizzle.ts", + "./geocode-cache": "./src/services/geocode-cache.ts", "./anon-hold-release": "./src/services/anon-hold-release.ts", "./anon-hold-repo": "./src/services/anon-hold-release-repo.drizzle.ts", "./adapters/storage": "./src/adapters/storage.r2.ts", diff --git a/services/api/src/services/address-resolver.ts b/services/api/src/services/address-resolver.ts index 7e8bb39f..039459a8 100644 --- a/services/api/src/services/address-resolver.ts +++ b/services/api/src/services/address-resolver.ts @@ -20,6 +20,12 @@ * A `landmark` line is stored and returned RAW ("Vista Hermosa Park, Los Angeles, CA"). The "Near " * prefix is added by the display layer, which is the only one that knows the viewer's language. * + * ONLY A CHAIN ANSWER IS CACHED. The `locality` rung is recomputed on every request - it is a free local + * PostGIS query, and storing it would give a transient provider outage the 180-day positive TTL and lock + * every point it touched at city grade for half a year. A chain miss writes the 15-minute NEGATIVE entry + * instead: it still stops a dragged pin from re-firing the chain per micro-drag, and the chain re-runs + * (and can upgrade the point) minutes after the outage ends. + * * NEVER BLOCKS, NEVER THROWS. Both halves are best-effort by seam contract, and this is called on the * report-create path where a geocoder outage must not cost a filing. A provider that throws, a cache * that is unreachable, a TIGER query that fails - each degrades one rung, never into an error. @@ -33,7 +39,7 @@ import { } from "@civfix/shared" import type { Geocoder } from "@civfix/shared" import type { ReverseGeocode } from "../adapters/reverse-geocode.chain.js" -import { NO_GEOCODE_CACHE, type GeocodeCache } from "./geocode-cache.js" +import { isChainAnswer, NO_GEOCODE_CACHE, type GeocodeCache } from "./geocode-cache.js" export interface ResolvedAddress { /** The display line, or null when nothing resolved. Landmarks are raw - no "Near " prefix. */ @@ -70,22 +76,18 @@ export function makeAddressResolver(deps: AddressResolverDeps): AddressResolver if (!Number.isFinite(lat) || !Number.isFinite(lng)) return NOT_RESOLVED const pointKey = geocodePointKey({ lat, lng }) - // Guarded HERE as well as inside makeGeocodeCache: "never throws" is a property of this function, - // not of whichever cache implementation happens to be wired in. const cached = await orNull(cache.read(pointKey)) - if (cached !== null) { + if (cached !== null && isChainAnswer(cached)) { return { address: cached.address, precision: cached.precision, cityStateLabel: cached.cityStateLabel, } } + const chainAlreadyMissed = cached !== null - // Raced, not sequenced: the TIGER label is a local PostGIS query and the chain is a network call, so - // the pair costs what the chain alone costs. The label is needed either way - as the response's - // unconditional cityStateLabel, and as the `locality` rung when the chain misses. const [hit, label] = await Promise.all([ - orNull(deps.streetReverseGeocode(lat, lng)), + chainAlreadyMissed ? Promise.resolve(null) : orNull(deps.streetReverseGeocode(lat, lng)), orNull(Promise.resolve(deps.geocoder.cityStateLabel(lat, lng))), ]) @@ -97,14 +99,25 @@ export function makeAddressResolver(deps: AddressResolverDeps): AddressResolver ? { address: cityStateLabel, precision: "locality", cityStateLabel } : { address: null, precision: null, cityStateLabel } - await orNull( - cache.write(pointKey, { - address: resolved.address, - precision: resolved.precision, - cityStateLabel, - provider: hit?.provider ?? (resolved.precision === "locality" ? "tiger" : null), - }), - ) + if (hit !== null) { + await orNull( + cache.write(pointKey, { + address: hit.line, + precision: hit.precision, + cityStateLabel, + provider: hit.provider, + }), + ) + } else if (!chainAlreadyMissed) { + await orNull( + cache.write(pointKey, { + address: null, + precision: null, + cityStateLabel, + provider: null, + }), + ) + } return resolved } @@ -137,7 +150,6 @@ export function addressProvenance( if (resolved === null || resolved.address === null) { return { addr: null, addrSource: null, addrPrecision: null } } - // The wire cap never applied to a server-derived line; clamp rather than overflow the column. return { addr: resolved.address.slice(0, MAX_REPORT_ADDR_LENGTH), addrSource: "resolved", diff --git a/services/api/src/services/geocode-cache.ts b/services/api/src/services/geocode-cache.ts index 652bf3ed..dca7c368 100644 --- a/services/api/src/services/geocode-cache.ts +++ b/services/api/src/services/geocode-cache.ts @@ -8,26 +8,34 @@ * fine-tune is a cache hit rather than a fresh provider call, and the create that follows a preview * costs zero provider calls. * + * ONLY A CHAIN ANSWER IS A POSITIVE ENTRY. A row earns the long TTL when the provider chain proved a + * located rung (street | intersection | landmark). The `locality` rung never reaches this table at all: + * it comes from a free local TIGER query, so the resolver recomputes it per request rather than letting + * a provider blip freeze a point at city grade for half a year. + * * TTL IS APPLIED ON READ, never by a cron: a row past its TTL is reported as a miss and overwritten in * place by the fresh resolve. Two TTLs, and the difference matters: * - * - POSITIVE (180 days). Addresses do not move. - * - NEGATIVE (15 minutes). A null IS cached, briefly: that is what stops a dragged pin over a - * genuinely unaddressable point - or a provider that is down right now - from re-firing the whole - * chain per micro-drag. Expiring it in minutes is what stops a provider OUTAGE from poisoning those - * points for half a year, which is the failure mode the design spec called out. + * - POSITIVE (180 days), for a chain answer. Addresses do not move. + * - NEGATIVE (15 minutes), for everything else. A chain miss IS cached, briefly: that is what stops a + * dragged pin over a genuinely unaddressable point - or a provider that is down right now - from + * re-firing the whole chain per micro-drag. Expiring it in minutes is what lets the chain re-run, + * and upgrade the point, minutes after an outage ends. + * + * Expired rows are OVERWRITTEN, not deleted, so the table is bounded by the `geocode_cache` lane of the + * media-worker's `retention.sweep` (docs/retention-cleanup.md) rather than by this module. * * EVERY DB ERROR IS SWALLOWED. The cache is an optimisation on a seam whose entire contract is "best * effort, never blocks a submit". A cache table that is missing, locked or unreachable must degrade to * "no cache", never to a failed report. */ -import type { AddressPrecision } from "@civfix/shared" +import { isLocatedPrecision, type AddressPrecision } from "@civfix/shared" import type { Queryable } from "../db/client.js" -/** A resolved address line past this age is re-resolved (and the row overwritten) on next read. */ +/** A chain answer past this age is re-resolved (and the row overwritten) on next read. */ export const GEOCODE_CACHE_TTL_MS = 180 * 24 * 60 * 60 * 1000 -/** A negative entry ("the providers had nothing") is trusted for minutes, not months. */ +/** A negative entry ("the chain had nothing") is trusted for minutes, not months. */ export const GEOCODE_CACHE_NEGATIVE_TTL_MS = 15 * 60 * 1000 export interface GeocodeCacheEntry { @@ -50,9 +58,23 @@ interface GeocodeCacheRowSelect { resolved_at: Date } -/** True while the row may still be served. Negative rows (no address) expire far sooner. */ -export function isFreshEntry(row: { address: string | null; resolvedAt: Date }, now: Date): boolean { - const ttl = row.address === null ? GEOCODE_CACHE_NEGATIVE_TTL_MS : GEOCODE_CACHE_TTL_MS +/** + * Did the provider chain prove a located rung for this row? The one definition of "worth the long TTL", + * read by the freshness rule here and by the resolver that decides what to store. + */ +export function isChainAnswer(entry: { + address: string | null + precision: AddressPrecision | null +}): boolean { + return entry.address !== null && isLocatedPrecision(entry.precision) +} + +/** True while the row may still be served. Anything but a chain answer expires far sooner. */ +export function isFreshEntry( + row: { address: string | null; precision: AddressPrecision | null; resolvedAt: Date }, + now: Date, +): boolean { + const ttl = isChainAnswer(row) ? GEOCODE_CACHE_TTL_MS : GEOCODE_CACHE_NEGATIVE_TTL_MS return now.getTime() - row.resolvedAt.getTime() < ttl } @@ -79,7 +101,11 @@ export function makeGeocodeCache(opts: GeocodeCacheOptions): GeocodeCache { ` const row = rows[0] if (row === undefined) return null - if (!isFreshEntry({ address: row.address, resolvedAt: row.resolved_at }, now())) return null + const fresh = isFreshEntry( + { address: row.address, precision: row.address_precision, resolvedAt: row.resolved_at }, + now(), + ) + if (!fresh) return null return { address: row.address, precision: row.address_precision, @@ -113,7 +139,7 @@ export function makeGeocodeCache(opts: GeocodeCacheOptions): GeocodeCache { resolved_at = EXCLUDED.resolved_at ` } catch { - // See the header: a cache write can never be the reason a submit fails. + return } }, } diff --git a/services/api/test/unit/address-resolver.test.ts b/services/api/test/unit/address-resolver.test.ts index 610deca9..408e60e1 100644 --- a/services/api/test/unit/address-resolver.test.ts +++ b/services/api/test/unit/address-resolver.test.ts @@ -9,8 +9,9 @@ * - `cityStateLabel` comes back populated on every path, because the contract promises it; * - a throwing provider, a throwing geocoder and an unreachable cache each cost one rung and never * an error - this sits on the report-create path, where a geocoder outage must not cost a filing; - * - a cached point makes NO provider call, and a negative entry expires in minutes while a resolved - * one lasts months. + * - a cached point makes NO provider call, and ONLY a chain answer is cached: a chain miss leaves a + * short-TTL null row, so the locality rung is recomputed per request and an outage cannot lock a + * point at city grade for half a year. * * All offline: fake providers, a fake geocoder and an in-memory cache. */ @@ -27,6 +28,7 @@ import { import { GEOCODE_CACHE_NEGATIVE_TTL_MS, GEOCODE_CACHE_TTL_MS, + isChainAnswer, isFreshEntry, makeGeocodeCache, type GeocodeCache, @@ -62,6 +64,24 @@ function memoryCache(): GeocodeCache & { rows: Map; w return cache } +/** The same store, but applying the REAL freshness rule against a clock the test moves. */ +function expiringCache(clock: { now: Date }): GeocodeCache & { + rows: Map +} { + const rows = new Map() + return { + rows, + read: async (key: string) => { + const row = rows.get(key) + if (row === undefined) return null + return isFreshEntry(row, clock.now) ? row : null + }, + write: async (key: string, entry: GeocodeCacheEntry) => { + rows.set(key, { ...entry, resolvedAt: clock.now }) + }, + } +} + describe("makeAddressResolver ladder", () => { it("takes the chain's rung verbatim and still reports the locality hint", async () => { const resolve = makeAddressResolver({ @@ -231,17 +251,98 @@ describe("makeAddressResolver caching", () => { cityStateLabel: "", provider: null, }) + // The second resolve must NOT rewrite the row: a refreshed resolved_at would push the negative + // entry's expiry out forever on a point that is being resolved continuously. + expect(cache.writes).toBe(1) }) - it("records which adapter answered, and attributes the locality rung to tiger", async () => { + it("records which adapter answered", async () => { const cache = memoryCache() await makeAddressResolver({ + streetReverseGeocode: chain({ + line: "123 Main St, Inglewood, CA", + precision: "street", + provider: "mapbox", + }), + geocoder: new FakeGeocoder(), + cache, + })(LA.lat, LA.lng) + + expect(cache.rows.get(geocodePointKey(LA))?.provider).toBe("mapbox") + }) + + it("NEVER caches the locality rung: a chain miss stores a null row, not the TIGER label", async () => { + const cache = memoryCache() + const out = await makeAddressResolver({ streetReverseGeocode: chain(null), geocoder: new FakeGeocoder(), cache, })(LA.lat, LA.lng) - expect(cache.rows.get(geocodePointKey(LA))?.provider).toBe("tiger") + expect(out).toEqual({ + address: "Los Angeles, CA", + precision: "locality", + cityStateLabel: "Los Angeles, CA", + }) + expect(cache.rows.get(geocodePointKey(LA))).toEqual({ + address: null, + precision: null, + cityStateLabel: "Los Angeles, CA", + provider: null, + }) + }) + + it("re-runs the chain once the negative entry expires, and UPGRADES the point", async () => { + const clock = { now: new Date("2026-09-16T00:00:00Z") } + const cache = expiringCache(clock) + let down = true + const provider: ReverseGeocode = async () => + down ? null : { line: "123 Main St, Inglewood, CA", precision: "street", provider: "photon" } + const resolve = makeAddressResolver({ + streetReverseGeocode: provider, + geocoder: new FakeGeocoder(), + cache, + }) + + // The outage: the chain has nothing, so every caller still gets the live locality rung... + await expect(resolve(LA.lat, LA.lng)).resolves.toEqual({ + address: "Los Angeles, CA", + precision: "locality", + cityStateLabel: "Los Angeles, CA", + }) + // ...off a SHORT-TTL null row, which is what makes the re-run possible at all. + expect(cache.rows.get(geocodePointKey(LA))?.address).toBeNull() + + down = false + clock.now = new Date(clock.now.getTime() + GEOCODE_CACHE_NEGATIVE_TTL_MS + 1000) + + await expect(resolve(LA.lat, LA.lng)).resolves.toEqual({ + address: "123 Main St, Inglewood, CA", + precision: "street", + cityStateLabel: "Los Angeles, CA", + }) + expect(cache.rows.get(geocodePointKey(LA))?.precision).toBe("street") + }) + + it("keeps serving a chain answer for months - only the misses are short-lived", async () => { + const clock = { now: new Date("2026-09-16T00:00:00Z") } + const cache = expiringCache(clock) + const provider = chain({ + line: "123 Main St, Inglewood, CA", + precision: "street", + provider: "photon", + }) + const resolve = makeAddressResolver({ + streetReverseGeocode: provider, + geocoder: new FakeGeocoder(), + cache, + }) + + await resolve(LA.lat, LA.lng) + clock.now = new Date(clock.now.getTime() + GEOCODE_CACHE_TTL_MS / 2) + await expect(resolve(LA.lat, LA.lng)).resolves.toMatchObject({ precision: "street" }) + + expect(provider.calls).toBe(1) }) it("an UNREACHABLE cache degrades to no caching, never to an error", async () => { @@ -325,15 +426,49 @@ describe("makeGeocodeCache", () => { it("expires a NEGATIVE row in minutes: an outage cannot poison a point for months", () => { const now = new Date() const aged = (ms: number): Date => new Date(now.getTime() - ms) + const negative = (ms: number): { address: null; precision: null; resolvedAt: Date } => ({ + address: null, + precision: null, + resolvedAt: aged(ms), + }) - // A negative entry an hour old is already gone... - expect(isFreshEntry({ address: null, resolvedAt: aged(60 * 60 * 1000) }, now)).toBe(false) - // ...while it is still trusted inside its short window (that is what stops the re-fire storm)... - expect(isFreshEntry({ address: null, resolvedAt: aged(GEOCODE_CACHE_NEGATIVE_TTL_MS / 2) }, now)).toBe( - true, - ) - // ...and a resolved line an hour old is nowhere near expiry. - expect(isFreshEntry({ address: "123 Main St", resolvedAt: aged(60 * 60 * 1000) }, now)).toBe(true) + expect(isFreshEntry(negative(60 * 60 * 1000), now)).toBe(false) + expect(isFreshEntry(negative(GEOCODE_CACHE_NEGATIVE_TTL_MS / 2), now)).toBe(true) + expect( + isFreshEntry( + { address: "123 Main St", precision: "street", resolvedAt: aged(60 * 60 * 1000) }, + now, + ), + ).toBe(true) + }) + + it("gives a LOCALITY row the negative TTL, so a legacy one cannot outlive the blip that wrote it", () => { + const now = new Date() + const row = { + address: "Los Angeles, CA", + precision: "locality" as const, + resolvedAt: new Date(now.getTime() - 60 * 60 * 1000), + } + + expect(isChainAnswer(row)).toBe(false) + expect(isFreshEntry(row, now)).toBe(false) + }) + + it("a stale LOCALITY row is a read MISS, so the chain gets its next chance", async () => { + const cache = makeGeocodeCache({ + getSql: () => + fakeSql([ + { + address: "Los Angeles, CA", + address_precision: "locality", + city_state_label: "Los Angeles, CA", + provider: "tiger", + resolved_at: new Date(Date.now() - GEOCODE_CACHE_NEGATIVE_TTL_MS - 1000), + }, + ]), + }) + + await expect(cache.read("34.05223,-118.24368")).resolves.toBeNull() }) it("swallows a DB that is not there at all (the offline faked server)", async () => { diff --git a/services/media-worker/src/jobs/retention-sweep.ts b/services/media-worker/src/jobs/retention-sweep.ts index 490b2f3a..1a6fcf86 100644 --- a/services/media-worker/src/jobs/retention-sweep.ts +++ b/services/media-worker/src/jobs/retention-sweep.ts @@ -7,6 +7,7 @@ import { makeDrizzleInboundRetentionRepository, type InboundRetentionRepository, } from "@civfix/api/inbound-retention-repo" +import { GEOCODE_CACHE_TTL_MS } from "@civfix/api/geocode-cache" import { drainPages } from "./drain.js" import { resolveJobObs, type JobObsDeps } from "./obs.js" @@ -15,6 +16,7 @@ export interface RetentionSweepDeps extends JobObsDeps { graceMs?: number idempotencyRetentionMs?: number notificationsRetentionMs?: number + geocodeCacheRetentionMs?: number inboundEmailsRetentionMs?: number batchSize?: number maxPages?: number @@ -27,6 +29,7 @@ export interface RetentionSweepResult { sessions: number idempotencyKeys: number notifications: number + geocodeCache: number inboundEmails: number inboundEmailObjectsLeaked: number errors: number @@ -36,6 +39,7 @@ export const RETENTION_GRACE_MS = 60 * 60 * 1000 export const RETENTION_BATCH = 5000 export const RETENTION_IDEMPOTENCY_MS = 48 * 60 * 60 * 1000 export const RETENTION_NOTIFICATIONS_MS = 90 * 24 * 60 * 60 * 1000 +export const RETENTION_GEOCODE_CACHE_MS = GEOCODE_CACHE_TTL_MS export const RETENTION_INBOUND_EMAILS_MS = INBOUND_EMAIL_RETENTION_MS export const RETENTION_INBOUND_EMAILS_BATCH = INBOUND_EMAIL_RETENTION_BATCH export const RETENTION_MAX_PAGES = 20 @@ -53,6 +57,9 @@ export async function runRetentionSweep(deps: RetentionSweepDeps): Promise (result.notifications += n), ) + await drainTable( + "geocode_cache", + (limit) => deps.sql<{ point_key: string }[]>` + DELETE FROM geocode_cache + WHERE point_key IN ( + SELECT point_key FROM geocode_cache + WHERE resolved_at < ${geocodeCacheCutoff} + LIMIT ${limit} + ) + RETURNING point_key + `, + (n) => (result.geocodeCache += n), + ) + await runInboundEmailRetentionLane(deps, result, { before: inboundEmailsCutoff, maxPages, @@ -165,6 +187,7 @@ export async function runRetentionSweep(deps: RetentionSweepDeps): Promise Pro } describe("retention.sweep", () => { - it("deletes from email_otps, anon_tokens, sessions, idempotency_keys and notifications, returning per-table counts", async () => { - const { sql, calls } = makeSqlSpy([rows(3), rows(2), rows(5), rows(4), rows(6)]) + it("deletes from email_otps, anon_tokens, sessions, idempotency_keys, notifications and geocode_cache, returning per-table counts", async () => { + const { sql, calls } = makeSqlSpy([rows(3), rows(2), rows(5), rows(4), rows(6), rows(7)]) const res = await runRetentionSweep({ sql, now: () => new Date("2026-06-20T00:00:00Z"), log: () => {}, }) - expect(res).toEqual({ ...EMPTY, otps: 3, anonTokens: 2, sessions: 5, idempotencyKeys: 4, notifications: 6 }) - expect(calls.length).toBe(5) + expect(res).toEqual({ + ...EMPTY, + otps: 3, + anonTokens: 2, + sessions: 5, + idempotencyKeys: 4, + notifications: 6, + geocodeCache: 7, + }) + expect(calls.length).toBe(6) expect(calls[0]).toContain("DELETE FROM email_otps") expect(calls[0]).toContain("consumed_at IS NOT NULL OR expires_at <") expect(calls[1]).toContain("DELETE FROM anon_tokens") @@ -73,6 +84,19 @@ describe("retention.sweep", () => { expect(calls[3]).not.toContain("SELECT key FROM idempotency_keys") expect(calls[4]).toContain("DELETE FROM notifications") expect(calls[4]).toContain("created_at <") + expect(calls[5]).toContain("DELETE FROM geocode_cache") + expect(calls[5]).toContain("SELECT point_key FROM geocode_cache") + expect(calls[5]).toContain("resolved_at <") + }) + + it("bounds geocode_cache at the POSITIVE TTL - the table is written by an unauthenticated path", async () => { + const { sql, values } = makeSqlSpy([rows(0), rows(0), rows(0), rows(0), rows(0), rows(1)]) + const now = new Date("2026-06-20T00:00:00Z") + const res = await runRetentionSweep({ sql, now: () => now, log: () => {} }) + + expect(res.geocodeCache).toBe(1) + expect(RETENTION_GEOCODE_CACHE_MS).toBe(GEOCODE_CACHE_TTL_MS) + expect(values[5]?.[0]).toEqual(new Date(now.getTime() - GEOCODE_CACHE_TTL_MS)) }) it("never throws: a per-table failure is counted + reported; the other tables still run", async () => { @@ -331,6 +355,7 @@ describe("inbound_emails retention lane (H10)", () => { rows(0), rows(0), rows(0), + rows(0), [{ id: "e1", attachments: [{ key: "inbound-emails/a/1-x.pdf" }] }], [{ id: "e1" }], ]) From 0e41298dab5b7ec88878a1b3a47bde58ba9a6543 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 19:36:46 -0700 Subject: [PATCH 33/54] copy a stored event address verbatim when duplicating Duplicating an event replays its stored address/source pair through the create path, where the new-client length floor rejected anything under three characters - including the addresses 0175 backfilled to 'manual'. That pair is this server's own row, not a client payload, so the floor no longer applies to it. Slur checks are unchanged. Also drops the inline comments this batch added inside function bodies and the unused redactMapboxToken export with its test. --- .../src/adapters/reverse-geocode.mapbox.ts | 4 --- .../src/adapters/reverse-geocode.photon.ts | 2 -- services/api/src/services/anon-service.ts | 1 - services/api/src/services/cleanup-service.ts | 31 +++++++++++-------- services/api/src/services/report-service.ts | 6 ---- .../unit/event-address-verification.test.ts | 22 +++++++++++++ .../test/unit/reverse-geocode-mapbox.test.ts | 23 -------------- 7 files changed, 40 insertions(+), 49 deletions(-) diff --git a/services/api/src/adapters/reverse-geocode.mapbox.ts b/services/api/src/adapters/reverse-geocode.mapbox.ts index fdccfaba..b93f7f5b 100644 --- a/services/api/src/adapters/reverse-geocode.mapbox.ts +++ b/services/api/src/adapters/reverse-geocode.mapbox.ts @@ -81,10 +81,6 @@ export function mapboxPrecision(p: MapboxReverseProps): AddressPrecision | null return null } -export function redactMapboxToken(url: string): string { - return url.replace(/([?&]access_token=)[^&#\s]*/gi, "$1[redacted]") -} - export function makeMapboxReverseGeocode(opts: MapboxReverseOptions): ReverseGeocode { const baseUrl = opts.url ?? MAPBOX_REVERSE_URL const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS diff --git a/services/api/src/adapters/reverse-geocode.photon.ts b/services/api/src/adapters/reverse-geocode.photon.ts index cfc28da4..4a99f109 100644 --- a/services/api/src/adapters/reverse-geocode.photon.ts +++ b/services/api/src/adapters/reverse-geocode.photon.ts @@ -176,8 +176,6 @@ export function composePhotonReverse( roads.push(road) roadProps ??= c.props } - // A house number is what separates `street` from `intersection`; the pin sits ON a road either way, so - // even a single named road is a materially better answer than the city - it just never claims a number. if (roadProps !== null && roads[0] !== undefined) { const primary = roads.length >= 2 ? `${roads[0]} & ${roads[1]}` : roads[0] return { line: [primary, ...localityTail(roadProps, primary)].join(", "), precision: "intersection" } diff --git a/services/api/src/services/anon-service.ts b/services/api/src/services/anon-service.ts index eca220f4..c834c992 100644 --- a/services/api/src/services/anon-service.ts +++ b/services/api/src/services/anon-service.ts @@ -212,7 +212,6 @@ export function makeAnonService(deps: AnonServiceDeps): AnonService { const reportId = newId() const claimCode = newClaimCode() const claimCodeHash = await sha256Hex(claimCode) - // Identical provenance rules to the signed-in path, by construction (one shared helper). const addressWrite = addressProvenance(suppliedAddr, resolvedAddr) const responseSnapshot: AnonReportResponse = { diff --git a/services/api/src/services/cleanup-service.ts b/services/api/src/services/cleanup-service.ts index a808e9a5..e2ac7b54 100644 --- a/services/api/src/services/cleanup-service.ts +++ b/services/api/src/services/cleanup-service.ts @@ -553,14 +553,25 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { * * The shim stores NOTHING when the ladder only reached `locality`: "Los Angeles, CA" is not a meeting * address, and writing it would dress up a non-answer as a host-provided one. + * + * `fromStoredEvent` marks the DUPLICATE path, whose pair did not come off the wire at all: it is this + * server's own stored row, copied verbatim. The new-client length floor is a check on a client payload + * and would reject a backfilled one-or-two-character address that the host has been running for + * months. Slur checks still apply - they run over the whole input before this. */ - async function resolveEventAddress(input: { - address?: string | undefined - addressSource?: EventAddressSource | undefined - lat: number - lng: number - }): Promise<{ address: string | null; addressSource: EventAddressSource | null }> { + async function resolveEventAddress( + input: { + address?: string | undefined + addressSource?: EventAddressSource | undefined + lat: number + lng: number + }, + opts?: { fromStoredEvent?: boolean }, + ): Promise<{ address: string | null; addressSource: EventAddressSource | null }> { if (input.addressSource !== undefined) { + if (opts?.fromStoredEvent === true && input.address !== undefined) { + return { address: input.address, addressSource: input.addressSource } + } return { address: assertConfirmedAddress(input.address), addressSource: input.addressSource } } const typed = input.address?.trim() ?? "" @@ -570,7 +581,6 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { if (resolved.address === null || !isLocatedPrecision(resolved.precision)) { return { address: null, addressSource: null } } - // The wire cap never applied to a server-derived line; clamp rather than overflow the column. return { address: resolved.address.slice(0, MAX_EVENT_ADDRESS_LENGTH), addressSource: "resolved", @@ -961,9 +971,7 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { throw AppError.validation({ linkedReportIds: "only cleanup events can link reports" }) } await assertReportsLinkable(linkedReportIds) - // Before any budget or host work: a malformed address is a client bug, and rejecting it early keeps - // the compat shim's geocode off the path of a request that was never going to be written. - const addressWrite = await resolveEventAddress(input) + const addressWrite = await resolveEventAddress(input, { fromStoredEvent: copyFrom !== undefined }) if (input.endsAt === null) throw AppError.validation({ endsAt: "required" }) if (input.slots !== undefined && input.slots.length === 0) { throw AppError.validation({ slots: EVENT_NEEDS_A_SLOT_MESSAGE }) @@ -1085,9 +1093,6 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { lng: source.lng, scheduledAt: scheduledAt.toISOString(), ...(source.bring !== null ? { bring: source.bring } : {}), - // A duplicate is the same place at a new time, so the host's verified address and its provenance - // both carry over verbatim. Copied as a PAIR: an address with no source would be re-read as an - // old client's payload and demoted to 'manual', and a source with no address cannot exist. ...(source.address !== null && source.addressSource !== null ? { address: source.address, addressSource: source.addressSource } : source.address !== null diff --git a/services/api/src/services/report-service.ts b/services/api/src/services/report-service.ts index b242d913..8ce13b93 100644 --- a/services/api/src/services/report-service.ts +++ b/services/api/src/services/report-service.ts @@ -212,8 +212,6 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { assertNoSlur(input.title ?? null, "title") assertNoSlur(input.description ?? null, "description") - // Events have slur-checked `address` since forever; reports never checked `addr`, and it is the - // same class of public free text rendered on the same surfaces. assertNoSlur(input.addr ?? null, "addr") const existing = await deps.repo.findIdempotentSnapshot( @@ -239,10 +237,6 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { const h3Cell = reportH3Cell(input.lat, input.lng) const publishedAt = now() const reportId = newId() - // Provenance, snapshotted with the address itself. 'user' text carries no provider precision by - // definition; a server resolve carries exactly the rung the ladder reached, so a `landmark` line - // can be rendered as "Near X" rather than posing as a postal address. A resolve that came back - // empty leaves all three NULL - filing is never blocked on any of this. const addressWrite = addressProvenance(suppliedAddr, reversed) const result = await deps.repo.createReportTx({ diff --git a/services/api/test/unit/event-address-verification.test.ts b/services/api/test/unit/event-address-verification.test.ts index 7639bb3b..de422ba8 100644 --- a/services/api/test/unit/event-address-verification.test.ts +++ b/services/api/test/unit/event-address-verification.test.ts @@ -270,6 +270,28 @@ describe("duplicateCleanup", () => { expect(copy.addressSource).toBe("edited") }) + it("copies a stored address that is SHORTER than the new-client floor, instead of refusing it", async () => { + const service = serviceWith() + const created = await service.createCleanup( + baseInput({ address: "North gate", addressSource: "manual" }), + HOST, + ) + // What 0175's backfill leaves on a legacy event whose host typed two characters. + repo.cleanups.get(created.id)!.address = "NW" + + const copy = await service.duplicateCleanup(HOST, { + id: created.id, + scheduledAt: new Date(Date.now() + 7 * 86_400_000).toISOString(), + endsAt: new Date(Date.now() + 7 * 86_400_000 + 4 * 3_600_000).toISOString(), + includeTicketTypes: false, + includeQuestions: false, + includePage: false, + }) + + expect(copy.address).toBe("NW") + expect(copy.addressSource).toBe("manual") + }) + it("duplicating an addressless legacy event stays addressless", async () => { const service = serviceWith() const created = await service.createCleanup(baseInput(), HOST) diff --git a/services/api/test/unit/reverse-geocode-mapbox.test.ts b/services/api/test/unit/reverse-geocode-mapbox.test.ts index d977fc0b..0f933558 100644 --- a/services/api/test/unit/reverse-geocode-mapbox.test.ts +++ b/services/api/test/unit/reverse-geocode-mapbox.test.ts @@ -3,7 +3,6 @@ import { makeMapboxReverseGeocode, formatMapboxReverse, mapboxPrecision, - redactMapboxToken, } from "../../src/adapters/reverse-geocode.mapbox.js" function okFetch(props: unknown): typeof fetch { @@ -41,28 +40,6 @@ describe("formatMapboxReverse", () => { }) }) -describe("redactMapboxToken", () => { - it("redacts the access_token value while keeping every other query param", () => { - const url = - "https://api.mapbox.com/search/geocode/v6/reverse?longitude=-89.6&latitude=39.8&access_token=pk.secretVALUE123&limit=1" - const out = redactMapboxToken(url) - expect(out).not.toContain("pk.secretVALUE123") - expect(out).toContain("access_token=[redacted]") - expect(out).toContain("longitude=-89.6") - expect(out).toContain("limit=1") - }) - - it("redacts the token when it is the first query param", () => { - expect(redactMapboxToken("https://x/y?access_token=abc&z=1")).toBe( - "https://x/y?access_token=[redacted]&z=1", - ) - }) - - it("is a no-op when there is no token", () => { - expect(redactMapboxToken("https://x/y?z=1")).toBe("https://x/y?z=1") - }) -}) - describe("makeMapboxReverseGeocode", () => { it("returns a formatted address for a coordinate", async () => { const geocode = makeMapboxReverseGeocode({ From 6efdd269bfb629f13b296abad1a7b7034716a601 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 20:51:07 -0700 Subject: [PATCH 34/54] declare the 0175 tables and CHECKs in the schema drift guards --- services/api/test/integration/schema.test.ts | 9 +++++ .../retention-and-phash-pg.test.ts | 36 +++++++++++++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/services/api/test/integration/schema.test.ts b/services/api/test/integration/schema.test.ts index 65af733b..23eef23e 100644 --- a/services/api/test/integration/schema.test.ts +++ b/services/api/test/integration/schema.test.ts @@ -3,6 +3,11 @@ import { afterAll, describe, expect, it } from "vitest" import { randomUUID } from "node:crypto" import { getTableName, is } from "drizzle-orm" import { PgTable, getTableConfig } from "drizzle-orm/pg-core" +import { + AddressPrecisionSchema, + EventAddressSourceSchema, + ReportAddressSourceSchema, +} from "@civfix/shared" import { withPg, type PgHarness } from "../helpers/pg.js" import { seedCleanup } from "../helpers/cleanups.js" import * as schema from "../../src/db/schema/index.js" @@ -111,6 +116,7 @@ const EXPECTED_TABLES = [ "stripe_events", "legal_documents", "consent_records", + "geocode_cache", ] as const const FOREIGN_TABLES = new Set([ @@ -730,6 +736,9 @@ const MIRRORED_CHECKS: readonly MirroredCheck[] = [ { table: "volunteer_hours", column: "source", mirror: ["report", "event", "manual"] }, { table: "reports", column: "verification_verdict", mirror: ["approved", "rejected"] }, { table: "broadcast_unsubscribes", column: "subject_kind", mirror: ["user", "guest"] }, + { table: "cleanups", column: "address_source", mirror: EventAddressSourceSchema.options }, + { table: "reports", column: "addr_source", mirror: ReportAddressSourceSchema.options }, + { table: "reports", column: "addr_precision", mirror: AddressPrecisionSchema.options }, ] const CONSTRAINT_QUALIFIERS = /(?:\s+NO INHERIT)?(?:\s+NOT VALID)?$/ diff --git a/services/media-worker/test/integration/retention-and-phash-pg.test.ts b/services/media-worker/test/integration/retention-and-phash-pg.test.ts index 90ff978f..ad752c33 100644 --- a/services/media-worker/test/integration/retention-and-phash-pg.test.ts +++ b/services/media-worker/test/integration/retention-and-phash-pg.test.ts @@ -2,7 +2,7 @@ * The worker's RAW SQL against the canonical migrated schema (Docker-gated; SKIPS without Docker). * * Two pieces of the worker talk to Postgres in hand-written SQL and were only ever exercised against - * fakes: runRetentionSweep (five paged DELETE ... RETURNING statements plus the inbound-email lane, spy- + * fakes: runRetentionSweep (six paged DELETE ... RETURNING statements plus the inbound-email lane, spy- * tested as STRINGS, so a wrong column or table name was invisible) and makePhashDuplicateLookup (replaced * by an injected stub in every unit test, and its interpolated `AND id <> $1` / `AND report_id IS DISTINCT * FROM $2` fragments plus ORDER BY created_at are exactly the kind of thing a string spy cannot check). @@ -18,6 +18,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest" import { randomUUID } from "node:crypto" import { FakeStorage } from "@civfix/shared/fakes" +import { GEOCODE_CACHE_TTL_MS } from "@civfix/api/geocode-cache" import { runRetentionSweep } from "../../src/jobs/retention-sweep.js" import { buildSeams, type WorkerSeams } from "../../src/seams.js" import { withWorkerPg, type WorkerPgHarness } from "../helpers/pg.js" @@ -59,13 +60,14 @@ afterAll(async () => { describe.skipIf(!pg)("retention.sweep against the real schema", () => { const NOW = new Date("2026-06-20T12:00:00Z") - /** A deterministic baseline: these six tables hold ONLY what each test puts in them. */ + /** A deterministic baseline: these seven tables hold ONLY what each test puts in them. */ beforeEach(async () => { await h.sql`DELETE FROM email_otps` await h.sql`DELETE FROM anon_tokens` await h.sql`DELETE FROM sessions` await h.sql`DELETE FROM idempotency_keys` await h.sql`DELETE FROM notifications` + await h.sql`DELETE FROM geocode_cache` await h.sql`DELETE FROM inbound_emails` }) @@ -122,6 +124,25 @@ describe.skipIf(!pg)("retention.sweep against the real schema", () => { return row!.id } + /** + * geocode_cache is keyed by the shared 5-decimal point key, so each seeded row needs its own point. + * The counter is never reset: uniqueness across the whole file is all the PK asks for. + */ + let geocodePoint = 0 + + async function insertGeocodeEntry(resolvedAt: Date): Promise { + const pointKey = `34.${String(10000 + geocodePoint++).padStart(5, "0")},-118.25000` + const [row] = await h.sql<{ point_key: string }[]>` + INSERT INTO geocode_cache ( + point_key, address, address_precision, city_state_label, provider, resolved_at + ) VALUES ( + ${pointKey}, '1 Main St', 'street', 'Los Angeles, CA', 'photon', ${resolvedAt} + ) + RETURNING point_key + ` + return row!.point_key + } + async function insertInboundEmail(opts: { archivedAt: Date | null attachmentKey?: string @@ -144,7 +165,7 @@ describe.skipIf(!pg)("retention.sweep against the real schema", () => { return row!.id } - it("deletes exactly the expired/consumed rows in all six tables and keeps the live ones", async () => { + it("deletes exactly the expired/consumed rows in all seven tables and keeps the live ones", async () => { // Doomed: past the 1h grace cutoff, or consumed regardless of expiry. const expiredOtp = await insertOtp({ expiresAt: at(-3 * HOUR) }) const consumedButValidOtp = await insertOtp({ expiresAt: at(3 * HOUR), consumedAt: at(-HOUR) }) @@ -152,6 +173,8 @@ describe.skipIf(!pg)("retention.sweep against the real schema", () => { const expiredSession = await insertSession(at(-3 * HOUR)) const oldKey = await insertIdempotencyKey(at(-72 * HOUR)) const oldNotification = await insertNotification(at(-100 * 24 * HOUR)) + // geocode_cache is reaped on resolved_at against the SAME TTL the read path applies (180 days). + const staleGeocode = await insertGeocodeEntry(at(-GEOCODE_CACHE_TTL_MS - HOUR)) // Survivors: still inside the grace window / retention window. const liveOtp = await insertOtp({ expiresAt: at(3 * HOUR) }) @@ -160,6 +183,7 @@ describe.skipIf(!pg)("retention.sweep against the real schema", () => { const liveSession = await insertSession(at(3 * HOUR)) const recentKey = await insertIdempotencyKey(at(-2 * HOUR)) const recentNotification = await insertNotification(at(-24 * HOUR)) + const liveGeocode = await insertGeocodeEntry(at(-GEOCODE_CACHE_TTL_MS + HOUR)) // inbound_emails is reaped on archived_at, 180 days, and only after its attachment objects are gone. const attachmentKey = `inbound/${randomUUID()}.pdf` @@ -180,6 +204,7 @@ describe.skipIf(!pg)("retention.sweep against the real schema", () => { sessions: 1, idempotencyKeys: 1, notifications: 1, + geocodeCache: 1, inboundEmails: 1, inboundEmailObjectsLeaked: 0, errors: 0, @@ -206,6 +231,10 @@ describe.skipIf(!pg)("retention.sweep against the real schema", () => { expect(notifications.map((r) => r.id)).toEqual([recentNotification]) expect(notifications.map((r) => r.id)).not.toContain(oldNotification) + const points = await h.sql<{ point_key: string }[]>`SELECT point_key FROM geocode_cache` + expect(points.map((r) => r.point_key)).toEqual([liveGeocode]) + expect(points.map((r) => r.point_key)).not.toContain(staleGeocode) + const emails = await h.sql<{ id: string }[]>`SELECT id FROM inbound_emails` expect(emails.map((r) => r.id).sort()).toEqual( [recentlyArchivedEmail, unarchivedEmail].sort(), @@ -227,6 +256,7 @@ describe.skipIf(!pg)("retention.sweep against the real schema", () => { sessions: 0, idempotencyKeys: 0, notifications: 0, + geocodeCache: 0, inboundEmails: 0, inboundEmailObjectsLeaked: 0, errors: 0, From c08fad47d4017355c129f7086e0b98a3e99cced8 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 21:50:06 -0700 Subject: [PATCH 35/54] tell a guest how they get in, and give a promoted guest something to read The guest confirmation linked only to the cancel page, so the one email a guest ever receives said nothing about arriving: getGuestEventTicket has no client caller and /guest?token= is cancel-only, so there is no QR code to promise. Say plainly that check-in is by name. email.guest_promoted.* is the copy for the waitlist notice a guest has never had a channel for. It links the public event page rather than a manage link: only sha256(token) is stored, so the raw manage token cannot be reconstructed outside the verify that minted it. Co-Authored-By: Claude Opus 5 (1M context) --- services/api/src/i18n/messages/de.ts | 5 ++++- services/api/src/i18n/messages/en.ts | 5 ++++- services/api/src/i18n/messages/es.ts | 5 ++++- services/api/src/i18n/messages/ko.ts | 6 +++++- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/services/api/src/i18n/messages/de.ts b/services/api/src/i18n/messages/de.ts index 5433ed8d..55679fcc 100644 --- a/services/api/src/i18n/messages/de.ts +++ b/services/api/src/i18n/messages/de.ts @@ -124,7 +124,10 @@ export const de: Partial> = { "Dein Code für die Anmeldung zu {{title}} lautet {{code}}. Er läuft in {{minutes}} Minuten ab. Wenn du ihn nicht angefordert hast, kannst du diese E-Mail ignorieren.", "email.guest_confirmed.subject": "Du stehst auf der Liste für {{title}}", "email.guest_confirmed.body": - "Du bist für {{title}} angemeldet. Anders entschieden? Hier kannst du deine Anmeldung stornieren: {{link}}", + "Du bist für {{title}} angemeldet. Melde dich vor Ort mit deinem Namen an; es gibt kein Ticket zum Ausdrucken. Anders entschieden? Hier kannst du deine Anmeldung stornieren: {{link}}", + "email.guest_promoted.subject": "Ein Platz für {{title}} ist frei geworden", + "email.guest_promoted.body": + "Ein Platz für {{title}} am {{when}} ist frei geworden. Du standest auf der Warteliste, und die Veranstalterin oder der Veranstalter hält einen Platz für dich frei. Sieh dir die Veranstaltung an und melde dich dort, wenn du noch kommen möchtest: {{link}}", "email.guest_updated.subject": "{{title}} hat neue Angaben", "email.guest_updated.body": "Die Angaben zu {{title}} haben sich geändert. Es beginnt jetzt um {{when}} an folgendem Ort: {{place}}. Nutze den Stornierungslink aus deiner Bestätigung, falls du nicht mehr kommen kannst.", diff --git a/services/api/src/i18n/messages/en.ts b/services/api/src/i18n/messages/en.ts index d01e574b..dc87a8db 100644 --- a/services/api/src/i18n/messages/en.ts +++ b/services/api/src/i18n/messages/en.ts @@ -186,7 +186,10 @@ export const en = { "Your code to RSVP for {{title}} is {{code}}. It expires in {{minutes}} minutes. If you did not request it, you can ignore this email.", "email.guest_confirmed.subject": "You are on the list for {{title}}", "email.guest_confirmed.body": - "You are signed up for {{title}}. Change your mind? Cancel your RSVP here: {{link}}", + "You are signed up for {{title}}. Check in by name when you arrive; there is no ticket to print. Change your mind? Cancel your RSVP here: {{link}}", + "email.guest_promoted.subject": "A place opened up for {{title}}", + "email.guest_promoted.body": + "A place opened up for {{title}} on {{when}}. You were on the waitlist and the host is holding a place for you. See the event and contact the host if you still want it: {{link}}", "email.guest_updated.subject": "{{title}} has new details", "email.guest_updated.body": "The details for {{title}} changed. It now starts at {{when}} at {{place}}. Use the cancel link in your confirmation message if you can no longer make it.", diff --git a/services/api/src/i18n/messages/es.ts b/services/api/src/i18n/messages/es.ts index 4ecd3720..9dfed236 100644 --- a/services/api/src/i18n/messages/es.ts +++ b/services/api/src/i18n/messages/es.ts @@ -123,7 +123,10 @@ export const es: Partial> = { "Tu código para confirmar tu asistencia a {{title}} es {{code}}. Caduca en {{minutes}} minutos. Si no lo solicitaste, puedes ignorar este correo.", "email.guest_confirmed.subject": "Estás en la lista de {{title}}", "email.guest_confirmed.body": - "Te has apuntado a {{title}}. ¿Cambiaste de opinión? Cancela tu asistencia aquí: {{link}}", + "Te has apuntado a {{title}}. Al llegar, regístrate con tu nombre; no hay ninguna entrada que imprimir. ¿Cambiaste de opinión? Cancela tu asistencia aquí: {{link}}", + "email.guest_promoted.subject": "Se ha liberado una plaza para {{title}}", + "email.guest_promoted.body": + "Se ha liberado una plaza para {{title}} el {{when}}. Estabas en la lista de espera y la organización te está reservando una plaza. Consulta el evento y ponte en contacto con la organización si todavía quieres asistir: {{link}}", "email.guest_updated.subject": "{{title}} tiene nuevos detalles", "email.guest_updated.body": "Los detalles de {{title}} han cambiado. Ahora empieza a las {{when}} en {{place}}. Usa el enlace de cancelación de tu mensaje de confirmación si ya no puedes asistir.", diff --git a/services/api/src/i18n/messages/ko.ts b/services/api/src/i18n/messages/ko.ts index 09feb2ac..c76b6a58 100644 --- a/services/api/src/i18n/messages/ko.ts +++ b/services/api/src/i18n/messages/ko.ts @@ -122,7 +122,11 @@ export const ko: Partial> = { "email.guest_otp.body": "{{title}} 참가 신청 코드는 {{code}}입니다. {{minutes}}분 후에 만료됩니다. 요청하지 않으셨다면 이 이메일을 무시하셔도 됩니다.", "email.guest_confirmed.subject": "{{title}} 참가자 명단에 등록되었습니다", - "email.guest_confirmed.body": "{{title}}에 참가 신청이 완료되었습니다. 마음이 바뀌셨나요? 여기에서 취소하세요: {{link}}", + "email.guest_confirmed.body": + "{{title}}에 참가 신청이 완료되었습니다. 현장에서는 이름으로 체크인하며, 출력할 티켓은 없습니다. 마음이 바뀌셨나요? 여기에서 취소하세요: {{link}}", + "email.guest_promoted.subject": "{{title}}에 자리가 생겼습니다", + "email.guest_promoted.body": + "{{when}}에 열리는 {{title}}에 자리가 생겼습니다. 대기자 명단에 계셨고, 주최자가 자리를 잡아두고 있습니다. 아직 참가를 원하시면 행사 페이지를 확인하고 주최자에게 문의하세요: {{link}}", "email.guest_updated.subject": "{{title}} 세부 정보가 변경되었습니다", "email.guest_updated.body": "{{title}}의 세부 정보가 변경되었습니다. 이제 {{when}}에 {{place}}에서 시작합니다. 참석이 어려우시면 확인 메시지의 취소 링크를 이용하세요.", From 695100f38ed113fb482e7ecc9ee42a63b51fee17 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 21:50:15 -0700 Subject: [PATCH 36/54] notify a promoted guest, who could hear about it no other way waitlist-service returned early on `userId === null`, so a promoted guest was told nothing at all and the 24 h claim hold expired in silence. A guest has no account, so the bell and push are structurally unreachable: email is the only channel they have. guest-notify.ts is the seam both sides can reach without a wiring cycle - it owns guestManageLinkBase (moved off guest-rsvp-wiring, its only caller) and refuses to mail a cancelled, scrubbed or SMS-channel guest. The send is swallowed like the RSVP confirmation: the promotion stands either way. Co-Authored-By: Claude Opus 5 (1M context) --- services/api/src/services/guest-notify.ts | 47 +++++++ .../services/guest-rsvp-repository.drizzle.ts | 29 ++++ .../api/src/services/guest-rsvp-service.ts | 10 ++ .../api/src/services/guest-rsvp-wiring.ts | 8 +- .../src/services/host/registration-wiring.ts | 11 ++ .../api/src/services/host/waitlist-service.ts | 20 ++- services/api/test/helpers/guest-rsvp.ts | 14 ++ services/api/test/unit/guest-notify.test.ts | 124 ++++++++++++++++++ .../test/unit/host/waitlist-service.test.ts | 95 +++++++++++++- 9 files changed, 343 insertions(+), 15 deletions(-) create mode 100644 services/api/src/services/guest-notify.ts create mode 100644 services/api/test/unit/guest-notify.test.ts diff --git a/services/api/src/services/guest-notify.ts b/services/api/src/services/guest-notify.ts new file mode 100644 index 00000000..69b02ec1 --- /dev/null +++ b/services/api/src/services/guest-notify.ts @@ -0,0 +1,47 @@ +import type { Mailer } from "@civfix/shared/interfaces" +import { renderMessage } from "../i18n/renderMessage.js" +import { formatEventWhen } from "./host/broadcast-render.js" +import { DEFAULT_EVENT_TIME_ZONE } from "./host/event-fields.js" +import type { GuestRsvpRepository } from "./guest-rsvp-service.js" + +export function guestManageLinkBase(webOrigins: readonly string[]): string { + const origin = webOrigins[0] + return origin !== undefined && origin.length > 0 + ? origin.replace(/\/+$/, "") + : "https://civfix.org" +} + +export function guestEventLink(linkBase: string, cleanupId: string): string { + return `${linkBase}/cleanups/${cleanupId}` +} + +export interface GuestPromotionNotifier { + notifyGuestPromoted(guestId: string): Promise +} + +export interface GuestNotifyDeps { + repo: Pick + mailer: Mailer + linkBase: string +} + +export function makeGuestPromotionNotifier(deps: GuestNotifyDeps): GuestPromotionNotifier { + return { + async notifyGuestPromoted(guestId: string): Promise { + const guest = await deps.repo.findGuestForNotice(guestId) + if (guest === null) return + if (guest.cancelledAt !== null || guest.contactScrubbedAt !== null) return + if (guest.email === null || guest.email.length === 0) return + const event = await deps.repo.findEvent(guest.cleanupId) + if (event === null) return + await deps.mailer.sendTransactional(guest.email, "generic", { + subject: renderMessage("en", "email.guest_promoted.subject", { title: event.title }), + message: renderMessage("en", "email.guest_promoted.body", { + title: event.title, + when: formatEventWhen(event.scheduledAt, event.timezone ?? DEFAULT_EVENT_TIME_ZONE), + link: guestEventLink(deps.linkBase, guest.cleanupId), + }), + }) + }, + } +} diff --git a/services/api/src/services/guest-rsvp-repository.drizzle.ts b/services/api/src/services/guest-rsvp-repository.drizzle.ts index 78b84626..3a757dab 100644 --- a/services/api/src/services/guest-rsvp-repository.drizzle.ts +++ b/services/api/src/services/guest-rsvp-repository.drizzle.ts @@ -3,6 +3,7 @@ import type { Sql } from "../db/client.js" import { encodeTimeCursor, pageWith, type TimeCursor } from "../db/cursor-helpers.js" import type { GuestEventView, + GuestNoticeTarget, GuestOtpRecord, GuestRecipient, GuestRosterRow, @@ -221,6 +222,34 @@ export function makeDrizzleGuestRsvpRepository(sql: Sql): GuestRsvpRepository { return { id: row.id, cleanupId: row.cleanup_id, cancelledAt: row.cancelled_at } }, + async findGuestForNotice(guestId: string): Promise { + const rows = await sql< + { + id: string + cleanup_id: string + name: string + email: string | null + cancelled_at: Date | null + contact_scrubbed_at: Date | null + }[] + >` + SELECT id, cleanup_id, name, email, cancelled_at, contact_scrubbed_at + FROM cleanup_guests + WHERE id = ${guestId} + LIMIT 1 + ` + const row = rows[0] + if (row === undefined) return null + return { + id: row.id, + cleanupId: row.cleanup_id, + name: row.name, + email: row.email, + cancelledAt: row.cancelled_at, + contactScrubbedAt: row.contact_scrubbed_at, + } + }, + async cancelGuest(guestId: string, now: Date): Promise { const released = await sql<{ ticket_type_id: string }[]>` WITH cancelled_guest AS ( diff --git a/services/api/src/services/guest-rsvp-service.ts b/services/api/src/services/guest-rsvp-service.ts index 5731d353..11e13b0c 100644 --- a/services/api/src/services/guest-rsvp-service.ts +++ b/services/api/src/services/guest-rsvp-service.ts @@ -132,6 +132,15 @@ export interface GuestRecipient { phone: string | null } +export interface GuestNoticeTarget { + id: string + cleanupId: string + name: string + email: string | null + cancelledAt: Date | null + contactScrubbedAt: Date | null +} + export interface GuestEventView { id: string title: string @@ -188,6 +197,7 @@ export interface GuestRsvpRepository { findGuestByManageTokenHash( hash: string, ): Promise<{ id: string; cleanupId: string; cancelledAt: Date | null } | null> + findGuestForNotice(guestId: string): Promise cancelGuest(guestId: string, now: Date): Promise listGuests(args: { cleanupId: string diff --git a/services/api/src/services/guest-rsvp-wiring.ts b/services/api/src/services/guest-rsvp-wiring.ts index 187bc0c4..efc7e295 100644 --- a/services/api/src/services/guest-rsvp-wiring.ts +++ b/services/api/src/services/guest-rsvp-wiring.ts @@ -3,6 +3,7 @@ import { REVIEWER_OTP_EMAIL } from "../auth/otp.js" import { writeAudit } from "./admin/audit.js" import { requireCapability } from "./host/authz.js" import { makeContainerRegistrationServices } from "./host/registration-wiring.js" +import { guestManageLinkBase } from "./guest-notify.js" import { makeDrizzleGuestRsvpRepository } from "./guest-rsvp-repository.drizzle.js" import { makeGuestRsvpService, @@ -24,13 +25,6 @@ export interface GuestRsvpOverrides { now?: GuestRsvpServiceDeps["now"] } -export function guestManageLinkBase(webOrigins: readonly string[]): string { - const origin = webOrigins[0] - return origin !== undefined && origin.length > 0 - ? origin.replace(/\/+$/, "") - : "https://civfix.org" -} - export function guestReviewerConfig(container: Container): GuestRsvpServiceDeps["reviewer"] { const code = container.env.REVIEWER_OTP_CODE if (!container.env.REVIEWER_OTP_BYPASS || code === undefined || code.length === 0) { diff --git a/services/api/src/services/host/registration-wiring.ts b/services/api/src/services/host/registration-wiring.ts index c0d711b4..e56088eb 100644 --- a/services/api/src/services/host/registration-wiring.ts +++ b/services/api/src/services/host/registration-wiring.ts @@ -2,6 +2,8 @@ import type { CounterStore } from "../../abuse/counter-store.js" import type { Container } from "../../di.js" import type { Sql } from "../../db/client.js" import { makeMediaPresigner } from "../media-presign.js" +import { guestManageLinkBase, makeGuestPromotionNotifier } from "../guest-notify.js" +import { makeDrizzleGuestRsvpRepository } from "../guest-rsvp-repository.drizzle.js" import { writeAudit } from "../admin/audit.js" import type { HostCapability } from "@civfix/shared" import { can, type HostStanding } from "@civfix/shared/host" @@ -136,6 +138,14 @@ export function makeContainerRegistrationServices( }) const notifier = lazyNotifier(container, logger) + const guests = + sql === undefined + ? undefined + : makeGuestPromotionNotifier({ + repo: makeDrizzleGuestRsvpRepository(sql), + mailer: container.mailer, + linkBase: guestManageLinkBase(container.env.WEB_ORIGINS), + }) const counters = overrides?.counters ?? container.getCounterStore() const insightsInvalidator: InsightsInvalidator = overrides?.insightsInvalidator ?? @@ -182,6 +192,7 @@ export function makeContainerRegistrationServices( registrations, jobs: container.jobs, notifier, + ...(guests !== undefined ? { guests } : {}), ...(audit !== undefined ? { audit } : {}), ...(overrides?.now !== undefined ? { now: overrides.now } : {}), ...(logger !== undefined ? { logger } : {}), diff --git a/services/api/src/services/host/waitlist-service.ts b/services/api/src/services/host/waitlist-service.ts index 848247b2..74f288af 100644 --- a/services/api/src/services/host/waitlist-service.ts +++ b/services/api/src/services/host/waitlist-service.ts @@ -13,6 +13,7 @@ import type { } from "@civfix/shared" import type { Jobs } from "@civfix/shared/interfaces" import { sha256Hex } from "../../auth/crypto.js" +import type { GuestPromotionNotifier } from "../guest-notify.js" import { toWaitlistEntryDTO } from "./registration-dto.js" import { enqueueWaitlistPromotion } from "./waitlist-promotion.js" import type { @@ -39,6 +40,7 @@ export interface WaitlistServiceDeps { registrations: Pick jobs?: Jobs notifier?: RegistrationNotifier + guests?: GuestPromotionNotifier audit?: RegistrationAudit now?: () => Date claimWindowMs?: number @@ -73,14 +75,18 @@ export function makeWaitlistService(deps: WaitlistServiceDeps): WaitlistService const claimWindowMs = deps.claimWindowMs ?? WAITLIST_CLAIM_WINDOW_MS async function notifyOffered(offer: WaitlistOffer): Promise { - if (deps.notifier === undefined || offer.userId === null) return try { - await deps.notifier.createNotification(offer.userId, { - type: "cleanup_slot", - title: "A place opened up", - body: "You moved off the waitlist. Claim your place before the hold expires.", - link: `/cleanups/${offer.cleanupId}`, - }) + if (offer.userId !== null) { + await deps.notifier?.createNotification(offer.userId, { + type: "cleanup_slot", + title: "A place opened up", + body: "You moved off the waitlist. Claim your place before the hold expires.", + link: `/cleanups/${offer.cleanupId}`, + }) + return + } + if (offer.guestId === null) return + await deps.guests?.notifyGuestPromoted(offer.guestId) } catch (err) { deps.logger?.warn( { err, cleanupId: offer.cleanupId }, diff --git a/services/api/test/helpers/guest-rsvp.ts b/services/api/test/helpers/guest-rsvp.ts index 076faf4f..247e070a 100644 --- a/services/api/test/helpers/guest-rsvp.ts +++ b/services/api/test/helpers/guest-rsvp.ts @@ -3,6 +3,7 @@ import type { GuestContactChannel } from "@civfix/shared" import { encodeTimeCursor, pageWith, type TimeCursor } from "../../src/db/cursor-helpers.js" import type { GuestEventView, + GuestNoticeTarget, GuestOtpRecord, GuestRecipient, GuestRosterRow, @@ -213,6 +214,19 @@ export class InMemoryGuestRsvpRepository implements GuestRsvpRepository, GuestCo return Promise.resolve({ id: row.id, cleanupId: row.cleanupId, cancelledAt: row.cancelledAt }) } + findGuestForNotice(guestId: string): Promise { + const row = this.guests.find((g) => g.id === guestId) + if (row === undefined) return Promise.resolve(null) + return Promise.resolve({ + id: row.id, + cleanupId: row.cleanupId, + name: row.name, + email: row.email, + cancelledAt: row.cancelledAt, + contactScrubbedAt: row.contactScrubbedAt, + }) + } + releaseOnCancel: string[] = [] async cancelGuest(guestId: string, now: Date): Promise { diff --git a/services/api/test/unit/guest-notify.test.ts b/services/api/test/unit/guest-notify.test.ts new file mode 100644 index 00000000..db0c9fa6 --- /dev/null +++ b/services/api/test/unit/guest-notify.test.ts @@ -0,0 +1,124 @@ +/** + * The guest half of waitlist promotion. + * + * A guest has no account, so `notification_prefs`, the bell and push are all unreachable for them: the + * ONLY way a promoted guest can learn a place opened up is this email. Before this seam existed the + * waitlist service returned early on `userId === null` and a promoted guest was told nothing at all, + * so the 24 h claim hold expired in silence. + * + * What is pinned here is mostly the REFUSALS: a guest whose contact was scrubbed, cancelled or never + * collected (the SMS channel) must not be mailed, and a missing row must not throw into the promoter. + */ + +import { describe, expect, it } from "vitest" +import { randomUUID } from "node:crypto" +import { FakeMailer } from "@civfix/shared/fakes" +import { InMemoryGuestRsvpRepository, type StoredGuest } from "../helpers/guest-rsvp.js" +import { makeGuestPromotionNotifier, guestEventLink } from "../../src/services/guest-notify.js" +import { formatEventWhen } from "../../src/services/host/broadcast-render.js" + +const EVENT_ID = "11111111-1111-1111-1111-111111111111" +const SCHEDULED_AT = new Date(Date.parse("2026-09-05T17:00:00.000Z")) + +function harness(over: Partial = {}): { + repo: InMemoryGuestRsvpRepository + mailer: FakeMailer + guestId: string + notify(guestId: string): Promise +} { + const repo = new InMemoryGuestRsvpRepository() + repo.seedEvent({ + id: EVENT_ID, + title: "Beach cleanup", + scheduledAt: SCHEDULED_AT, + timezone: "America/Los_Angeles", + }) + const guestId = randomUUID() + repo.guests.push({ + id: guestId, + cleanupId: EVENT_ID, + name: "Ada Lovelace", + channel: "email", + email: "ada@example.org", + phone: null, + contactKey: "ada@example.org", + manageTokenHash: "hash", + verifiedAt: new Date(), + cancelledAt: null, + contactScrubbedAt: null, + createdAt: new Date(), + ...over, + }) + const mailer = new FakeMailer() + const notifier = makeGuestPromotionNotifier({ + repo, + mailer, + linkBase: "https://civfix.org", + }) + return { repo, mailer, guestId, notify: (id) => notifier.notifyGuestPromoted(id) } +} + +describe("guest waitlist promotion notice", () => { + it("emails the promoted guest with the title, the local time and the event link", async () => { + const h = harness() + await h.notify(h.guestId) + + expect(h.mailer.sent).toHaveLength(1) + const mail = h.mailer.sent[0] + expect(mail?.to).toBe("ada@example.org") + expect(mail?.template).toBe("generic") + expect(String(mail?.vars?.subject)).toContain("Beach cleanup") + const body = String(mail?.vars?.message) + expect(body).toContain("Beach cleanup") + expect(body).toContain(formatEventWhen(SCHEDULED_AT, "America/Los_Angeles")) + expect(body).toContain("10:00 AM PDT") + expect(body).toContain(guestEventLink("https://civfix.org", EVENT_ID)) + }) + + it("prints Pacific Time when the event carries no timezone, never a raw ISO stamp", async () => { + const h = harness() + h.repo.seedEvent({ id: EVENT_ID, scheduledAt: SCHEDULED_AT, timezone: null }) + await h.notify(h.guestId) + + const body = String(h.mailer.sent[0]?.vars?.message) + expect(body).toContain("10:00 AM PDT") + expect(body).not.toContain("2026-09-05T17:00:00.000Z") + }) + + it("never leaks the manage token: the link is the public event page", async () => { + const h = harness() + await h.notify(h.guestId) + expect(String(h.mailer.sent[0]?.vars?.message)).not.toContain("token=") + }) + + it("sends nothing to an SMS-channel guest, who has no address to mail", async () => { + const h = harness({ channel: "sms", email: null, phone: "+15552223333" }) + await h.notify(h.guestId) + expect(h.mailer.sent).toHaveLength(0) + }) + + it("sends nothing to a cancelled guest", async () => { + const h = harness({ cancelledAt: new Date() }) + await h.notify(h.guestId) + expect(h.mailer.sent).toHaveLength(0) + }) + + it("sends nothing once the retention sweep has scrubbed the contact", async () => { + const h = harness({ contactScrubbedAt: new Date() }) + await h.notify(h.guestId) + expect(h.mailer.sent).toHaveLength(0) + }) + + it("is a no-op for a guest row that is gone, rather than throwing into the promoter", async () => { + const h = harness() + await expect(h.notify(randomUUID())).resolves.toBeUndefined() + expect(h.mailer.sent).toHaveLength(0) + }) + + it("is a no-op when the event itself has vanished", async () => { + const h = harness() + h.repo.events.clear() + await expect(h.notify(h.guestId)).resolves.toBeUndefined() + expect(h.mailer.sent).toHaveLength(0) + }) +}) diff --git a/services/api/test/unit/host/waitlist-service.test.ts b/services/api/test/unit/host/waitlist-service.test.ts index 71e6a853..10b65fe2 100644 --- a/services/api/test/unit/host/waitlist-service.test.ts +++ b/services/api/test/unit/host/waitlist-service.test.ts @@ -15,6 +15,7 @@ const EVENT = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" const ALICE = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" const BOB = "cccccccc-cccc-4ccc-8ccc-cccccccccccc" const HOST = "dddddddd-dddd-4ddd-8ddd-dddddddddddd" +const GUEST = "ffffffff-ffff-4fff-8fff-ffffffffffff" const tokens = makeTicketTokenSigner("waitlist-service-test-secret-long-enough") @@ -25,16 +26,20 @@ interface Harness { advance(ms: number): void enqueued: string[] notified: string[] + guestsNotified: string[] + warnings: unknown[] bumped: string[] } -function build(): Harness { +function build(opts: { guestNotifyFails?: boolean; withGuestNotifier?: boolean } = {}): Harness { const repo = new InMemoryHostRegistrationRepository() repo.seedEvent({ cleanupId: EVENT }) const state = { base: new Date("2026-01-01T12:00:00.000Z").getTime(), tick: 0 } const clock = (): Date => new Date(state.base + state.tick++) const enqueued: string[] = [] const notified: string[] = [] + const guestsNotified: string[] = [] + const warnings: unknown[] = [] const bumped: string[] = [] const seats = (partySize: number): SeatDraft[] => Array.from({ length: partySize }, () => { @@ -67,6 +72,24 @@ function build(): Harness { return Promise.resolve(null) }, }, + ...(opts.withGuestNotifier === false + ? {} + : { + guests: { + notifyGuestPromoted: (guestId: string) => { + guestsNotified.push(guestId) + return opts.guestNotifyFails === true + ? Promise.reject(new Error("mailer down")) + : Promise.resolve() + }, + }, + }), + logger: { + warn: (obj: unknown) => { + warnings.push(obj) + }, + info: () => {}, + }, now: clock, }) return { @@ -78,6 +101,8 @@ function build(): Harness { }, enqueued, notified, + guestsNotified, + warnings, bumped, } } @@ -356,6 +381,74 @@ describe("waitlist service", () => { expect(h.bumped).toEqual([EVENT]) }) + /** + * A guest has no user id, so the bell/push notifier cannot reach them. Until the guest seam existed + * `notifyOffered` returned early on `userId === null` and a promoted guest heard NOTHING, then lost + * the hold when the 24 h claim window expired. + */ + it("mails a promoted GUEST rather than returning early on the null user id", async () => { + const type = h.repo.seedTicketType({ cleanupId: EVENT, capacity: 1, waitlistEnabled: true }) + await h.service.join( + { id: EVENT, ticketTypeId: type.id, partySize: 1 }, + { kind: "guest", guestId: GUEST }, + ) + + expect(await h.service.runPromote({ ticketTypeId: type.id })).toBe(1) + expect(h.guestsNotified).toEqual([GUEST]) + expect(h.notified).toEqual([]) + expect(h.repo.ticketTypes.get(type.id)?.reservedSeats).toBe(1) + }) + + it("mails a guest the host promoted by hand, and audits it the same way", async () => { + const type = h.repo.seedTicketType({ cleanupId: EVENT, capacity: 1, waitlistEnabled: true }) + const joined = await h.service.join( + { id: EVENT, ticketTypeId: type.id, partySize: 1 }, + { kind: "guest", guestId: GUEST }, + ) + + const promoted = await h.service.promote({ id: EVENT, waitlistId: joined.entry.id }, HOST) + expect(promoted.entry.status).toBe("offered") + expect(h.guestsNotified).toEqual([GUEST]) + }) + + it("keeps the promotion when the guest email fails, and says so in the log", async () => { + const broken = build({ guestNotifyFails: true }) + const type = broken.repo.seedTicketType({ cleanupId: EVENT, capacity: 1, waitlistEnabled: true }) + const joined = await broken.service.join( + { id: EVENT, ticketTypeId: type.id, partySize: 1 }, + { kind: "guest", guestId: GUEST }, + ) + + expect(await broken.service.runPromote({ ticketTypeId: type.id })).toBe(1) + expect(broken.repo.waitlist.get(joined.entry.id)?.status).toBe("offered") + expect(broken.repo.ticketTypes.get(type.id)?.reservedSeats).toBe(1) + expect(broken.warnings).toHaveLength(1) + }) + + it("promotes a guest without incident when no guest notifier is wired at all", async () => { + const bare = build({ withGuestNotifier: false }) + const type = bare.repo.seedTicketType({ cleanupId: EVENT, capacity: 1, waitlistEnabled: true }) + await bare.service.join( + { id: EVENT, ticketTypeId: type.id, partySize: 1 }, + { kind: "guest", guestId: GUEST }, + ) + + expect(await bare.service.runPromote({ ticketTypeId: type.id })).toBe(1) + expect(bare.warnings).toEqual([]) + }) + + it("keeps the member notice on the bell, not on the guest lane", async () => { + const type = h.repo.seedTicketType({ cleanupId: EVENT, capacity: 1, waitlistEnabled: true }) + await h.service.join( + { id: EVENT, ticketTypeId: type.id, partySize: 1 }, + { kind: "user", userId: ALICE }, + ) + + await h.service.runPromote({ ticketTypeId: type.id }) + expect(h.notified).toEqual([ALICE]) + expect(h.guestsNotified).toEqual([]) + }) + it("leaves the insights generation alone when a waitlist mutation changes nothing", async () => { const type = h.repo.seedTicketType({ cleanupId: EVENT, capacity: 1, waitlistEnabled: true }) await h.service.leave({ id: EVENT, ticketTypeId: type.id }, { kind: "user", userId: ALICE }) From 76e6510be306c992d3b41126f6363c48c9bdc56e Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 21:50:23 -0700 Subject: [PATCH 37/54] sweep the guest mechanism with unit tests end to end Pins the parts of the guest path that had no test: the IP throttle, the name and slur gates, the enumeration-proof request response, the guest: idempotency key into the registration bridge, the swallowed seat write that leaves a guest with no registration, token rotation killing the old manage link, re-RSVP after a cancel, the cancelled-event retention branch, the guest half of every broadcast segment, guests being planned on the email channel alone, and getGuestEventTicket refusing a cancelled, rotated or unknown token with one message. Records two designed behaviours the tests now make explicit: all_registered selects cleanup_guests directly, with no join to a registration, so a guest whose seat write was swallowed still hears from the host; and a VALIDATION out of the registration bridge escapes after the guest row is already written. Co-Authored-By: Claude Opus 5 (1M context) --- .../api/test/unit/guest-rsvp-service.test.ts | 345 ++++++++++++++++++ .../test/unit/host-broadcast-audience.test.ts | 69 ++++ .../test/unit/host-broadcast-pipeline.test.ts | 116 ++++++ .../api/test/unit/host-export-csv.test.ts | 38 ++ .../test/unit/host/checkin-service.test.ts | 163 ++++++++- 5 files changed, 726 insertions(+), 5 deletions(-) diff --git a/services/api/test/unit/guest-rsvp-service.test.ts b/services/api/test/unit/guest-rsvp-service.test.ts index f0ccb1ee..baabdc70 100644 --- a/services/api/test/unit/guest-rsvp-service.test.ts +++ b/services/api/test/unit/guest-rsvp-service.test.ts @@ -5,6 +5,7 @@ import { FakeAbuseChecks, FakeMailer, FakeSmsSender } from "@civfix/shared/fakes import { GUEST_OTP_ERROR_FIELD, GuestOtpErrorReason, + MAX_GUEST_NAME, type GuestRsvpRequestRequest, type GuestRsvpVerifyRequest, } from "@civfix/shared" @@ -13,11 +14,14 @@ import { InMemoryCacheClient } from "../../src/auth/cache.js" import { InMemoryCounterStore, type CounterStore } from "../../src/abuse/counter-store.js" import { InMemoryGuestRsvpRepository } from "../helpers/guest-rsvp.js" import { InMemoryCleanupRepository } from "../helpers/cleanups.js" +import { InMemoryHostRegistrationRepository } from "../../src/services/host/registration-repository.memory.js" import { smsFailure } from "../../src/errors/sms-failure.js" import { formatEventWhen } from "../../src/services/host/broadcast-render.js" import { generateToken } from "../../src/auth/crypto.js" +import { OTP_VERIFY_IP_FAIL_MAX } from "../../src/auth/otp.js" import { GUEST_CONTACT_MAX_PER_DAY, + GUEST_IP_MAX_PER_HOUR, MAX_GUESTS_PER_EVENT, SMS_BUDGET_KEY_PREFIX, makeGuestRsvpService, @@ -279,6 +283,60 @@ describe("guest rsvp: requesting a code", () => { }) }) + it("caps one NETWORK at ten code requests an hour, across different contacts", async () => { + for (let i = 0; i < GUEST_IP_MAX_PER_HOUR; i++) { + await expect( + h.service.requestCode(emailRequest({ email: `ip${i}@example.org` }), ctx), + ).resolves.toMatchObject({ sent: true }) + } + await expect( + h.service.requestCode(emailRequest({ email: "one-too-many@example.org" }), ctx), + ).rejects.toMatchObject({ code: "RATE_LIMITED" }) + expect(h.mailer.sent).toHaveLength(GUEST_IP_MAX_PER_HOUR) + + // A request with no resolvable IP is not silently exempted from the other throttles. + await expect( + h.service.requestCode(emailRequest({ email: "no-ip@example.org" }), { ip: null }), + ).resolves.toMatchObject({ sent: true }) + }) + + it("refuses an empty or over-long name before anything is written", async () => { + await expect(h.service.requestCode(emailRequest({ name: " " }), ctx)).rejects.toMatchObject({ + code: "VALIDATION", + }) + await expect( + h.service.requestCode(emailRequest({ name: "a".repeat(MAX_GUEST_NAME + 1) }), ctx), + ).rejects.toMatchObject({ code: "VALIDATION" }) + expect(h.repo.otps).toHaveLength(0) + expect(h.mailer.sent).toHaveLength(0) + }) + + it("refuses a slur in the guest name, which would otherwise land on the host's roster", async () => { + await expect( + h.service.requestCode(emailRequest({ name: "retarded mcfly" }), ctx), + ).rejects.toMatchObject({ code: "VALIDATION" }) + expect(h.repo.otps).toHaveLength(0) + }) + + it("answers identically whether or not that contact already RSVP'd (no enumeration)", async () => { + const first = await h.service.requestCode(emailRequest(), ctx) + await h.service.verifyCode(emailVerify(), ctx) + h.advance(61_000) + + const known = await h.service.requestCode(emailRequest(), ctx) + const unknown = await h.service.requestCode( + emailRequest({ email: "nobody@example.org" }), + ctx, + ) + const honeypot = await h.service.requestCode( + emailRequest({ email: "bot@example.org", website: "spam" }), + ctx, + ) + const reviewer = await h.service.requestCode(emailRequest({ email: REVIEWER_EMAIL }), ctx) + + expect([known, unknown, honeypot, reviewer]).toEqual([first, first, first, first]) + }) + it("skips every send and every counter for the reviewer contact", async () => { const result = await h.service.requestCode(emailRequest({ email: REVIEWER_EMAIL }), ctx) @@ -389,6 +447,121 @@ describe("guest rsvp: verifying a code", () => { ) }) + /** + * `/guest?token=` is a CANCEL page and nothing else, and `getGuestEventTicket` has no client caller, + * so a guest never sees a QR code. The confirmation has to say how they actually get in, or it is + * quietly promising a ticket that never arrives. + */ + it("tells the guest how they actually get in, since no ticket is ever linked", async () => { + await h.service.verifyCode(emailVerify(), ctx) + const body = String(h.mailer.sent.at(-1)?.vars?.message) + + expect(body).toContain("Check in by name") + expect(body).toContain("no ticket to print") + }) + + it("rejects a code that was already consumed by an earlier successful verify", async () => { + await h.service.verifyCode(emailVerify(), ctx) + await expect(h.service.verifyCode(emailVerify(), ctx)).rejects.toMatchObject({ + code: "UNAUTHORIZED", + fields: { [GUEST_OTP_ERROR_FIELD]: GuestOtpErrorReason.invalidCode }, + }) + expect(h.repo.guests).toHaveLength(1) + }) + + it("locks the NETWORK out after enough wrong codes, before the event is even loaded", async () => { + for (let i = 0; i < OTP_VERIFY_IP_FAIL_MAX; i++) { + await h.service + .verifyCode(emailVerify({ email: `wrong${i}@example.org`, code: "000000" }), ctx) + .catch(() => undefined) + } + await expect(h.service.verifyCode(emailVerify(), ctx)).rejects.toMatchObject({ + code: "UNAUTHORIZED", + fields: { [GUEST_OTP_ERROR_FIELD]: GuestOtpErrorReason.lockedOut }, + }) + expect(h.repo.guests).toHaveLength(0) + }) + + it("keys the registration bridge on the guest row, so a replayed verify cannot double-book", async () => { + const calls: { idempotencyKey: string; subject: unknown }[] = [] + const harness = build({ + registrations: { + register: (input, subject) => { + calls.push({ idempotencyKey: input.idempotencyKey as string, subject }) + return Promise.resolve({ + outcome: "registered" as const, + registration: null, + ticketTokens: ["tok-1"], + }) + }, + }, + }) + await harness.service.requestCode(emailRequest(), ctx) + const first = await harness.service.verifyCode(emailVerify(), ctx) + harness.advance(61_000) + await harness.service.requestCode(emailRequest(), ctx) + await harness.service.verifyCode(emailVerify(), ctx) + + const guestId = harness.repo.guests[0]?.id as string + expect(calls.map((c) => c.idempotencyKey)).toEqual([`guest:${guestId}`, `guest:${guestId}`]) + expect(calls[0]?.subject).toEqual({ kind: "guest", guestId }) + expect(first.ticketTokens).toEqual(["tok-1"]) + }) + + it("never puts a guest on the waitlist: the bridge is asked not to", async () => { + const seen: unknown[] = [] + const harness = build({ + registrations: { + register: (input) => { + seen.push(input.joinWaitlistIfFull) + return Promise.resolve({ + outcome: "registered" as const, + registration: null, + ticketTokens: [], + }) + }, + }, + }) + await harness.service.requestCode(emailRequest(), ctx) + await harness.service.verifyCode(emailVerify(), ctx) + expect(seen).toEqual([false]) + }) + + /** + * The RSVP is deliberately allowed to stand when the seat write fails: the guest is on the list and + * the host sees them, but there is NO registration and NO seat, so no ticket exists for them. This + * pins the shape of that half-state rather than pretending it cannot happen. + */ + it("keeps the RSVP when the seat write fails, leaving a guest with no registration", async () => { + const harness = build({ + registrations: { register: () => Promise.reject(new Error("registrations are down")) }, + }) + await harness.service.requestCode(emailRequest(), ctx) + const result = await harness.service.verifyCode(emailVerify(), ctx) + + expect(result.joined).toBe(true) + expect(result.registration).toBeNull() + expect(result.registrationOutcome).toBeNull() + expect(result.ticketTokens).toEqual([]) + expect(harness.repo.guests).toHaveLength(1) + expect(result.manageToken.length).toBeGreaterThanOrEqual(20) + }) + + it("lets a VALIDATION from the bridge escape, though the guest row is already written", async () => { + const harness = build({ + registrations: { + register: () => Promise.reject(AppError.validation({ partySize: "too large" })), + }, + }) + await harness.service.requestCode(emailRequest(), ctx) + + await expect(harness.service.verifyCode(emailVerify(), ctx)).rejects.toMatchObject({ + code: "VALIDATION", + }) + expect(harness.repo.guests).toHaveLength(1) + expect(harness.repo.otps[0]?.consumedAt).not.toBeNull() + }) + it("rejects a wrong code without joining, naming the reason in fields", async () => { await expect(h.service.verifyCode(emailVerify({ code: "000000" }), ctx)).rejects.toMatchObject({ code: "UNAUTHORIZED", @@ -501,6 +674,129 @@ describe("guest rsvp: cancelling", () => { }) }) + it("kills the OLD manage link the moment a re-verify rotates the token", async () => { + const h = build() + await h.service.requestCode(emailRequest(), ctx) + const first = await h.service.verifyCode(emailVerify(), ctx) + h.advance(61_000) + await h.service.requestCode(emailRequest(), ctx) + const second = await h.service.verifyCode(emailVerify(), ctx) + + await expect(h.service.cancelRsvp(first.manageToken)).rejects.toMatchObject({ + code: "NOT_FOUND", + }) + await expect(h.service.cancelRsvp(second.manageToken)).resolves.toEqual({ ok: true }) + }) + + /** + * Cancel NULLs contact_key, which is what frees `cleanup_guests_active_contact_uidx`. Without that + * the same person could never RSVP again to the same event after changing their mind twice. + */ + it("lets the same address RSVP again after cancelling, on a fresh row and a fresh token", async () => { + const h = build() + await h.service.requestCode(emailRequest(), ctx) + const first = await h.service.verifyCode(emailVerify(), ctx) + await h.service.cancelRsvp(first.manageToken) + h.advance(61_000) + + await h.service.requestCode(emailRequest(), ctx) + const second = await h.service.verifyCode(emailVerify(), ctx) + + expect(second.joined).toBe(true) + expect(second.going).toBe(4) + expect(second.manageToken).not.toBe(first.manageToken) + expect(h.repo.guests).toHaveLength(2) + expect(h.repo.activeGuestCount(EVENT_ID)).toBe(1) + expect(h.repo.guests.filter((g) => g.cancelledAt !== null)).toHaveLength(1) + }) + + it("cancels the seats, blanks the attendee name and gives the ticket type its seats back", async () => { + const registrations = new InMemoryHostRegistrationRepository() + registrations.seedEvent({ cleanupId: EVENT_ID }) + const type = registrations.seedTicketType({ cleanupId: EVENT_ID, capacity: 5 }) + const h = build({ + registrations: { + register: async (input, subject) => { + const seatId = randomUUID() + const outcome = await registrations.registerTx({ + cleanupId: input.id, + subject, + ticketTypeId: type.id, + seats: [{ id: seatId, attendeeName: "Ada Lovelace", tokenHash: `hash-${seatId}` }], + accessCodeHash: null, + answers: [], + consent: null, + slotId: null, + source: "self", + idempotencyKey: input.idempotencyKey as string, + waitlistId: null, + now: new Date(), + }) + if (outcome.kind !== "registered") throw new Error(`unexpected ${outcome.kind}`) + return { outcome: "registered" as const, registration: null, ticketTokens: [seatId] } + }, + }, + }) + h.repo.registrations = registrations + + await h.service.requestCode(emailRequest(), ctx) + const { manageToken } = await h.service.verifyCode(emailVerify(), ctx) + expect(registrations.ticketTypes.get(type.id)?.reservedSeats).toBe(1) + + await h.service.cancelRsvp(manageToken) + + const registration = [...registrations.registrations.values()][0] + expect(registration?.status).toBe("cancelled") + expect(registration?.seats.every((seat) => seat.status === "cancelled")).toBe(true) + expect(registration?.seats.every((seat) => seat.attendeeName === null)).toBe(true) + expect(registrations.ticketTypes.get(type.id)?.reservedSeats).toBe(0) + }) + + it("cancels a guest's waitlist rows too, releasing any offered hold", async () => { + const registrations = new InMemoryHostRegistrationRepository() + registrations.seedEvent({ cleanupId: EVENT_ID }) + const type = registrations.seedTicketType({ + cleanupId: EVENT_ID, + capacity: 1, + waitlistEnabled: true, + }) + const h = build() + h.repo.registrations = registrations + + await h.service.requestCode(emailRequest(), ctx) + const { manageToken } = await h.service.verifyCode(emailVerify(), ctx) + const guestId = h.repo.guests[0]?.id as string + + await registrations.joinWaitlist({ + cleanupId: EVENT_ID, + ticketTypeId: type.id, + subject: { kind: "guest", guestId }, + partySize: 1, + accessCodeHash: null, + now: new Date(), + }) + const entry = [...registrations.waitlist.values()][0] + entry!.status = "offered" + entry!.offeredAt = new Date() + const seeded = registrations.ticketTypes.get(type.id) + seeded!.reservedSeats = 1 + + await h.service.cancelRsvp(manageToken) + + expect([...registrations.waitlist.values()][0]?.status).toBe("cancelled") + expect(registrations.ticketTypes.get(type.id)?.reservedSeats).toBe(0) + }) + + it("counts only ACTIVE guests against the per-event cap", async () => { + const h = build() + await h.service.requestCode(emailRequest(), ctx) + const { manageToken } = await h.service.verifyCode(emailVerify(), ctx) + await h.service.cancelRsvp(manageToken) + + await expect(h.repo.countActiveGuests(EVENT_ID)).resolves.toBe(0) + expect(h.repo.guests).toHaveLength(1) + }) + it("reports the registration outcome when the event has no seat left", async () => { const h = build({ registrations: { @@ -718,6 +1014,55 @@ describe("guest rsvp: retention", () => { expect(result.scrubbedGuests).toBe(0) expect(h.repo.guests[0]?.email).toBe("ada@example.org") }) + + /** + * A cancelled event never "happens", so its scheduled_at can sit in the future forever. The scrub + * has a second branch that ages those guests off their own created_at instead, or their contact + * would be held indefinitely. + */ + it("ages a CANCELLED event's guests off their own signup date, not the event date", async () => { + const h = build() + await h.service.requestCode(emailRequest(), ctx) + await h.service.verifyCode(emailVerify(), ctx) + + h.repo.seedEvent({ + id: EVENT_ID, + status: "cancelled", + scheduledAt: new Date(Date.parse("2030-01-01T00:00:00.000Z")), + }) + expect((await h.service.runRetentionSweep()).scrubbedGuests).toBe(0) + + h.advance(31 * 24 * 60 * 60 * 1000) + expect((await h.service.runRetentionSweep()).scrubbedGuests).toBe(1) + expect(h.repo.guests[0]?.email).toBeNull() + expect(h.repo.guests[0]?.contactKey).toBeNull() + }) + + it("reaps guest OTPs on their own 24h clock, independent of the contact lane", async () => { + const h = build() + await h.service.requestCode(emailRequest(), ctx) + + expect(await h.service.runRetentionSweep()).toEqual({ scrubbedGuests: 0, deletedOtps: 0 }) + h.advance(23 * 60 * 60 * 1000) + expect((await h.service.runRetentionSweep()).deletedOtps).toBe(0) + + h.advance(2 * 60 * 60 * 1000) + expect((await h.service.runRetentionSweep()).deletedOtps).toBe(1) + expect(h.repo.otps).toHaveLength(0) + }) + + it("scrubs a live guest's contact without cancelling their RSVP", async () => { + const h = build() + await h.service.requestCode(emailRequest(), ctx) + await h.service.verifyCode(emailVerify(), ctx) + h.repo.seedEvent({ id: EVENT_ID, scheduledAt: new Date(Date.parse("2026-01-01T00:00:00.000Z")) }) + + await h.service.runRetentionSweep() + + expect(h.repo.guests[0]?.cancelledAt).toBeNull() + expect(h.repo.activeGuestCount(EVENT_ID)).toBe(1) + await expect(h.repo.listContactableGuests(EVENT_ID, 10)).resolves.toEqual([]) + }) }) describe("guest rsvp: SMS title truncation", () => { diff --git a/services/api/test/unit/host-broadcast-audience.test.ts b/services/api/test/unit/host-broadcast-audience.test.ts index 0312ad27..3da859d7 100644 --- a/services/api/test/unit/host-broadcast-audience.test.ts +++ b/services/api/test/unit/host-broadcast-audience.test.ts @@ -156,6 +156,75 @@ describe("broadcast audience", () => { expect(result.guests).toEqual([u(11)]) }) + it("resolves the GUEST half of every segment, and only slots comes back empty", async () => { + const repo = new InMemoryBroadcastRepository() + repo.seedMembers(EVENT, []) + repo.seedGuests(EVENT, [ + { guestId: u(21), ticketTypeId: TYPE_A, checkedIn: true }, + { guestId: u(22), ticketTypeId: TYPE_B }, + { guestId: u(23), waitlisted: true, registered: false }, + ]) + + // all_registered / guests_only select cleanup_guests DIRECTLY, with no join to a registration: a + // verified RSVP is the membership, so a guest whose seat write was swallowed still hears from the + // host. u(23) is on the waitlist only and is in this segment for the same reason. + expect((await resolve(repo, { kind: "all_registered" })).guests).toEqual([u(21), u(22), u(23)]) + expect((await resolve(repo, { kind: "guests_only" })).guests).toEqual([u(21), u(22), u(23)]) + expect((await resolve(repo, { kind: "ticket_types", ids: [TYPE_B] })).guests).toEqual([u(22)]) + expect((await resolve(repo, { kind: "waitlist" })).guests).toEqual([u(23)]) + expect((await resolve(repo, { kind: "checked_in" })).guests).toEqual([u(21)]) + expect((await resolve(repo, { kind: "not_checked_in" })).guests).toEqual([u(22)]) + // A guest can never hold a slot claim (cleanup_slot_claims is user_id-keyed). + expect((await resolve(repo, { kind: "slots", ids: [SLOT] })).guests).toEqual([]) + }) + + it("excludes a guest with no email address at all: there is no other channel for them", async () => { + const repo = new InMemoryBroadcastRepository() + repo.seedMembers(EVENT, []) + repo.seedGuests(EVENT, [ + { guestId: u(31), email: null }, + { guestId: u(32), email: "grace@example.org" }, + ]) + for (const kind of ["host_broadcast", "event_cancelled"] as const) { + expect((await resolve(repo, { kind: "all_registered" }, kind)).guests, kind).toEqual([u(32)]) + } + }) + + it("keeps a guest unsubscribe OFF the critical lanes, exactly like a member's", async () => { + const repo = seed() + await repo.recordUnsubscribe({ + scope: "event", + cleanupId: EVENT, + subjectKind: "guest", + subjectId: u(11), + }) + for (const kind of ["event_updated", "event_cancelled"] as const) { + expect((await resolve(repo, { kind: "all_registered" }, kind)).guests, kind).toEqual([u(11)]) + } + for (const kind of ["host_broadcast", "announcement", "reminder", "thank_you"] as const) { + expect((await resolve(repo, { kind: "all_registered" }, kind)).guests, kind).toEqual([]) + } + }) + + it("honours a GLOBAL guest unsubscribe for bulk", async () => { + const repo = seed() + await repo.recordUnsubscribe({ + scope: "global", + cleanupId: null, + subjectKind: "guest", + subjectId: u(11), + }) + expect((await resolve(repo, { kind: "all_registered" })).guests).toEqual([]) + expect((await resolve(repo, { kind: "all_registered" }, "event_cancelled")).guests).toEqual([ + u(11), + ]) + }) + + it("keeps cancelled and scrubbed guests out of the guests_only segment too", async () => { + const result = await resolve(seed(), { kind: "guests_only" }, "event_cancelled") + expect(result.guests).toEqual([u(11)]) + }) + it("pages by keyset independently per recipient kind", async () => { const repo = seed() const first = await repo.audiencePage({ diff --git a/services/api/test/unit/host-broadcast-pipeline.test.ts b/services/api/test/unit/host-broadcast-pipeline.test.ts index b32885e6..c0992b09 100644 --- a/services/api/test/unit/host-broadcast-pipeline.test.ts +++ b/services/api/test/unit/host-broadcast-pipeline.test.ts @@ -7,6 +7,7 @@ import { insightsGenerationKey } from "../../src/services/host/host-analytics-ca import { InMemoryCounterStore } from "../../src/abuse/counter-store.js" import { InMemoryBroadcastRepository } from "../../src/services/host/broadcast-repository.memory.js" import { + emailHashOf, makeBroadcastService, type BroadcastConfig, } from "../../src/services/host/broadcast-service.js" @@ -524,6 +525,121 @@ describe("broadcast chunk", () => { }) }) +/** + * Guests have no account, so the bell and push are structurally unreachable for them: email is the + * whole channel set. Every producing lane asks for ["inapp","push","email"], so if the guest half of + * the planner ever stopped filtering, a guest would get a delivery row on a channel that can never be + * processed and the broadcast would sit unfinished. + */ +describe("broadcast to guests", () => { + const GUEST_A = u(101) + const GUEST_B = u(102) + + function withGuests(guests: Parameters[1]): Harness { + const h = harness({ members: 0 }) + h.repo.seedMembers(EVENT, [{ userId: u(1) }]) + h.repo.seedGuests(EVENT, guests) + return h + } + + it("plans EMAIL ONLY for a guest, whatever channels the host picked", async () => { + const h = withGuests([{ guestId: GUEST_A, email: "ada@example.test", name: "Ada" }]) + const id = await draftSending(h, ["inapp", "push", "email"]) + await h.pipeline.plan(id) + + const rows = h.repo.allDeliveries() + const guestRows = rows.filter((d) => d.guestId === GUEST_A) + expect(guestRows.map((d) => d.channel)).toEqual(["email"]) + expect(rows.filter((d) => d.userId === u(1))).toHaveLength(3) + }) + + it("sends the guest their email and leaves them out of the in-app fan-out", async () => { + const h = withGuests([{ guestId: GUEST_A, email: "ada@example.test", name: "Ada" }]) + const id = await draftSending(h, ["inapp", "email"]) + await h.pipeline.plan(id) + await h.pipeline.runChunk(id, 0) + await h.pipeline.runChunk(id, 1) + + const guestRow = h.repo.allDeliveries().find((d) => d.guestId === GUEST_A) + expect(guestRow?.status).toBe("sent") + const envelope = h.mailer.sent.find((m) => m.to === "ada@example.test")?.outbound + expect(envelope?.headers?.["List-Unsubscribe"]).toContain("/v1/broadcasts/unsubscribe?t=") + expect(envelope?.text).toContain("Beach Cleanup") + const fanout = h.notifications.calls as { userIds: string[] }[] + expect(fanout.flatMap((c) => c.userIds)).toEqual([u(1)]) + }) + + it("greets the guest by the name they gave at RSVP", async () => { + const h = withGuests([{ guestId: GUEST_A, email: "ada@example.test", name: "Ada" }]) + const record = await h.repo.create({ + cleanupId: EVENT, + createdBy: HOST, + kind: "host_broadcast", + subject: "Hello {first_name}", + bodyMd: "See you soon, {first_name}.", + segment: { kind: "guests_only" }, + channels: ["email"], + status: "sending", + }) + await h.pipeline.plan(record.id) + await h.pipeline.runChunk(record.id, 0) + + const envelope = h.mailer.sent.find((m) => m.to === "ada@example.test")?.outbound + expect(envelope?.subject).toBe("Hello Ada") + expect(envelope?.text).toContain("See you soon, Ada.") + }) + + it("labels a guest whose contact was scrubbed between plan and send contact_scrubbed", async () => { + const h = withGuests([{ guestId: GUEST_A, email: "ada@example.test", name: "Ada" }]) + const id = await draftSending(h) + await h.pipeline.plan(id) + h.repo.seedGuests(EVENT, [ + { guestId: GUEST_A, email: "ada@example.test", name: "Ada", scrubbed: true }, + ]) + await h.pipeline.runChunk(id, 0) + + const guestRow = h.repo.allDeliveries().find((d) => d.guestId === GUEST_A) + expect(guestRow?.status).toBe("suppressed") + expect(guestRow?.suppressionReason).toBe("contact_scrubbed") + }) + + it("blocks a guest on the address-level bounce suppression list, like any member", async () => { + const h = withGuests([ + { guestId: GUEST_A, email: "bounced@example.test", name: "Ada" }, + { guestId: GUEST_B, email: "fine@example.test", name: "Grace" }, + ]) + await h.repo.suppressEmail(emailHashOf("bounced@example.test"), "hard_bounce") + const id = await draftSending(h) + await h.pipeline.plan(id) + for (const chunk of [...h.chunks]) await h.pipeline.runChunk(id, chunk.chunkNo) + + const rows = h.repo.allDeliveries() + expect(rows.find((d) => d.guestId === GUEST_A)?.suppressionReason).toBe("bounce_suppressed") + expect(rows.find((d) => d.guestId === GUEST_B)?.status).toBe("sent") + }) + + it("reaches guests on the ANNOUNCEMENT kind, which asks for inapp + push + email", async () => { + const h = withGuests([{ guestId: GUEST_A, email: "ada@example.test", name: "Ada" }]) + const record = await h.repo.create({ + cleanupId: EVENT, + createdBy: HOST, + kind: "announcement", + subject: "Parking has moved", + bodyMd: "Use the north lot.", + segment: { kind: "all_registered" }, + channels: ["inapp", "push", "email"], + status: "sending", + }) + await h.pipeline.plan(record.id) + for (const chunk of [...h.chunks]) await h.pipeline.runChunk(record.id, chunk.chunkNo) + + const guestRows = h.repo.allDeliveries().filter((d) => d.guestId === GUEST_A) + expect(guestRows.map((d) => d.channel)).toEqual(["email"]) + expect(guestRows[0]?.status).toBe("sent") + expect(h.mailer.sent.some((m) => m.to === "ada@example.test")).toBe(true) + }) +}) + describe("broadcast template variables", () => { it("renders every allow-listed variable non-empty on the real send path", async () => { const h = harness({ members: 0 }) diff --git a/services/api/test/unit/host-export-csv.test.ts b/services/api/test/unit/host-export-csv.test.ts index 1c6154e3..e6089c2c 100644 --- a/services/api/test/unit/host-export-csv.test.ts +++ b/services/api/test/unit/host-export-csv.test.ts @@ -7,9 +7,11 @@ import { toHostExportDTO, } from "../../src/services/host/export-service.js" import { + hostExportBuilder, registerHostExportBuilder, resetHostExportBuildersForTests, } from "../../src/services/host/export-builders.js" +import { registerEventExportBuilders } from "../../src/services/host/host-export-builders.js" import type { HostExportRecord, HostExportRepository, @@ -268,6 +270,42 @@ describe("roster export query source", () => { expect(rosterQuery).not.toMatch(/users\.phone/) expect(rosterQuery).toMatch(/g\.email AS guest_email/) }) + + /** + * The CSV is the LAST place a host can hold guest contact after `EventGuestsBlock`, and it is the + * only artefact that leaves the platform. The header and the provenance block are the promise the + * file makes about that, so they are pinned here rather than left to a DB-backed test. + */ + it("carries guest_email and guest_phone on the roster, with the scrub caveat written on the file", async () => { + resetHostExportBuildersForTests() + registerEventExportBuilders(() => { + throw new Error("no SQL: header and provenance must not touch the database") + }) + const ctx = { + exportId: "11111111-1111-4111-8111-111111111111", + cleanupId: "22222222-2222-4222-8222-222222222222", + organizationId: null, + requestedBy: "33333333-3333-4333-8333-333333333333", + filters: {}, + now: new Date("2026-02-05T12:00:00Z"), + } + + const roster = hostExportBuilder("roster") + const header = await roster.header(ctx) + expect(header).toContain("guest_email") + expect(header).toContain("guest_phone") + expect(header).toContain("attendee_kind") + expect((await roster.provenance(ctx)).join(" ")).toContain( + "guest contact is blank once the 30-day retention scrub has run", + ) + + // The check-in export is a different capability's artefact and must NOT carry contact. + const checkins = await hostExportBuilder("checkins").header(ctx) + expect(checkins).not.toContain("guest_email") + expect(checkins).not.toContain("guest_phone") + + resetHostExportBuildersForTests() + }) }) describe("host export run guards", () => { diff --git a/services/api/test/unit/host/checkin-service.test.ts b/services/api/test/unit/host/checkin-service.test.ts index 82725835..f3986fc9 100644 --- a/services/api/test/unit/host/checkin-service.test.ts +++ b/services/api/test/unit/host/checkin-service.test.ts @@ -14,15 +14,25 @@ const EVENT = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" const OTHER_EVENT = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee" const ATTENDEE = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" const STAFF = "cccccccc-cccc-4ccc-8ccc-cccccccccccc" +const GUEST = "dddddddd-dddd-4ddd-8ddd-dddddddddddd" +const GUEST_NAME = "Grace Hopper" const NOW = new Date("2026-01-01T12:00:00.000Z") const tokens = makeTicketTokenSigner("checkin-service-test-secret-long-enough") +interface GuestTokenRecord { + id: string + cleanupId: string + cancelledAt: Date | null +} + interface Harness { repo: InMemoryHostRegistrationRepository service: CheckinService audits: string[] bumped: string[] + /** Raw manage token -> the guest row it resolves to, mirroring cleanup_guests.manage_token_hash. */ + guestTokens: Map } function seats(partySize: number, name: string | null = null): SeatDraft[] { @@ -39,6 +49,9 @@ function build(): Harness { repo.seedEvent({ cleanupId: OTHER_EVENT }) const audits: string[] = [] const bumped: string[] = [] + const guestTokens = new Map([ + ["guest-manage-token", { id: GUEST, cleanupId: EVENT, cancelledAt: null }], + ]) const service = makeCheckinService({ repo, tokens, @@ -49,10 +62,10 @@ function build(): Harness { }, }, guestByManageToken: async (hash) => { - const known = await sha256Hex("guest-manage-token") - return hash === known - ? { id: "guest-1", cleanupId: EVENT, cancelledAt: null } - : null + for (const [raw, record] of guestTokens) { + if (hash === (await sha256Hex(raw))) return record + } + return null }, audit: (input) => { audits.push(input.action) @@ -60,7 +73,35 @@ function build(): Harness { }, now: () => NOW, }) - return { repo, service, audits, bumped } + return { repo, service, audits, bumped, guestTokens } +} + +/** Seat a guest the way the guest RSVP bridge does: a cleanup_guests row plus a guest-subject registration. */ +async function registerGuest( + repo: InMemoryHostRegistrationRepository, + partySize = 1, + attendeeName: string | null = null, +): Promise<{ registrationId: string; seatIds: string[] }> { + repo.guests.set(GUEST, { id: GUEST, cleanupId: EVENT, name: GUEST_NAME }) + const outcome = await repo.registerTx({ + cleanupId: EVENT, + subject: { kind: "guest", guestId: GUEST }, + ticketTypeId: null, + seats: seats(partySize, attendeeName), + accessCodeHash: null, + answers: [], + consent: null, + slotId: null, + source: "self", + idempotencyKey: `guest:${GUEST}`, + waitlistId: null, + now: NOW, + }) + if (outcome.kind !== "registered") throw new Error(`expected registered, got ${outcome.kind}`) + return { + registrationId: outcome.registration.id, + seatIds: outcome.registration.seats.map((seat) => seat.id), + } } async function register( @@ -238,6 +279,118 @@ describe("check-in service", () => { ).rejects.toBeInstanceOf(AppError) }) + it("names a guest on a scanned seat that carries no attendee name of its own", async () => { + const { seatIds } = await registerGuest(h.repo) + const result = await h.service.scan( + { id: EVENT, token: tokens.tokenFor(seatIds[0] as string) }, + STAFF, + ) + expect(result.outcome).toBe("checked_in") + expect(result.attendeeName).toBe(GUEST_NAME) + }) + + it("names a guest on a MANUAL seat check-in too", async () => { + const { seatIds } = await registerGuest(h.repo) + const result = await h.service.checkIn( + { id: EVENT, seatId: seatIds[0] as string, method: "manual" }, + STAFF, + ) + expect(result.attendeeName).toBe(GUEST_NAME) + }) + + it("prefers a per-seat attendee name over the guest's own name", async () => { + const { seatIds } = await registerGuest(h.repo, 1, "Plus One") + const result = await h.service.scan( + { id: EVENT, token: tokens.tokenFor(seatIds[0] as string) }, + STAFF, + ) + expect(result.attendeeName).toBe("Plus One") + }) +}) + +/** + * `getGuestEventTicket` is the guest's ONLY read of their own seat tokens: it is `auth: "public"` and + * secured solely by the bearer manage token in the POST body. These tests pin that the token is the + * whole authorisation - a cancelled RSVP, a rotated token and an unknown token are all refused with + * the same 404, so the endpoint cannot be used to probe whether an address ever RSVP'd. + */ +describe("guest ticket", () => { + let h: Harness + + beforeEach(() => { + h = build() + }) + + it("hands a guest their live seat tokens behind a valid manage token", async () => { + const { registrationId, seatIds } = await registerGuest(h.repo, 2) + + const ticket = await h.service.guestTicket({ token: "guest-manage-token" }) + + expect(ticket.cleanupId).toBe(EVENT) + expect(ticket.registrationId).toBe(registrationId) + expect(ticket.status).toBe("registered") + expect(ticket.seats.map((seat) => seat.ticketToken)).toEqual( + seatIds.map((seatId) => tokens.tokenFor(seatId)), + ) + expect(ticket.canCancel).toBe(true) + }) + + it("hands back a token a host scan then accepts, closing the loop", async () => { + await registerGuest(h.repo) + const ticket = await h.service.guestTicket({ token: "guest-manage-token" }) + const token = ticket.seats[0]?.ticketToken as string + + expect((await h.service.scan({ id: EVENT, token }, STAFF)).outcome).toBe("checked_in") + }) + + it("refuses a guest whose RSVP was cancelled, even though the token still resolves", async () => { + await registerGuest(h.repo) + h.guestTokens.set("guest-manage-token", { + id: GUEST, + cleanupId: EVENT, + cancelledAt: NOW, + }) + + await expect(h.service.guestTicket({ token: "guest-manage-token" })).rejects.toMatchObject({ + code: "NOT_FOUND", + }) + }) + + it("kills the OLD token once re-verifying rotated the manage token", async () => { + await registerGuest(h.repo) + await expect(h.service.guestTicket({ token: "guest-manage-token" })).resolves.toBeDefined() + + h.guestTokens.delete("guest-manage-token") + h.guestTokens.set("guest-manage-token-rotated", { + id: GUEST, + cleanupId: EVENT, + cancelledAt: null, + }) + + await expect(h.service.guestTicket({ token: "guest-manage-token" })).rejects.toMatchObject({ + code: "NOT_FOUND", + }) + await expect( + h.service.guestTicket({ token: "guest-manage-token-rotated" }), + ).resolves.toBeDefined() + }) + + it("refuses a guest row that never got a registration (the swallowed-bridge case)", async () => { + await expect(h.service.guestTicket({ token: "guest-manage-token" })).rejects.toMatchObject({ + code: "NOT_FOUND", + }) + }) + + it("refuses every failure mode with the SAME message, so it cannot enumerate RSVPs", async () => { + const messages: string[] = [] + for (const token of ["guest-manage-token", "x".repeat(32)]) { + await h.service.guestTicket({ token }).catch((err: unknown) => { + messages.push((err as AppError).message) + }) + } + expect(new Set(messages).size).toBe(1) + }) + it("bumps the insights generation on every attendance change", async () => { const { seatIds } = await register(h.repo, EVENT, 2) const seatId = seatIds[0] as string From d404385230ce79bcb58398ccae5fd23a312c04f9 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 22:06:51 -0700 Subject: [PATCH 38/54] teach the mailer real email shapes, not one bare paragraph Every sendTransactional call rendered a single paragraph in the shell. renderTemplate (now exported, with renderOtp) gains guest_otp, guest_confirmed, guest_promoted and a generic structured "action" template: heading, when/where rows, styled code block, CTA button, quote and muted note, all built from the existing block set. Footers now break into lines with clickable links in the HTML part, and the no-reply footer stops inviting replies nobody reads. Guest email copy moves to per-piece catalog keys in all four locales. Co-Authored-By: Claude Fable 5 --- services/api/src/adapters/email-layout.ts | 18 ++- services/api/src/adapters/mailer.oci.ts | 91 ++++++++++- services/api/src/i18n/messages/de.ts | 21 ++- services/api/src/i18n/messages/en.ts | 21 ++- services/api/src/i18n/messages/es.ts | 20 ++- services/api/src/i18n/messages/ko.ts | 20 ++- services/api/test/unit/email-format.test.ts | 148 +++++++++++++++++- .../test/unit/host-broadcast-render.test.ts | 2 +- 8 files changed, 307 insertions(+), 34 deletions(-) diff --git a/services/api/src/adapters/email-layout.ts b/services/api/src/adapters/email-layout.ts index 78a22c51..6dc66ebc 100644 --- a/services/api/src/adapters/email-layout.ts +++ b/services/api/src/adapters/email-layout.ts @@ -28,9 +28,9 @@ const WORDMARK_HTML = WORDMARK.map( ([letter, color]) => `${letter}`, ).join("") -const DEFAULT_FOOTER = - "civfix is a civic reporting platform that connects residents with their local government. " + - "Reply to this email to respond. civfix.org" +export const DEFAULT_FOOTER = + "civfix is a civic reporting platform that connects residents with their local government.\n" + + "This mailbox is not monitored. civfix.org" export const CITY_FOOTER = "You're receiving this because a resident routed civic activity to your office through civfix, " + @@ -69,7 +69,15 @@ export function eventFooter(opts: EventFooterOptions): string { lines.push(`Manage your signup: ${opts.manageUrl}`) } lines.push("civfix.org") - return lines.join(" ") + return lines.join("\n") +} + +const FOOTER_URL_RE = /https?:\/\/[^\s<>"]+/g + +function footerHtml(footer: string): string { + return escapeHtml(footer) + .replace(FOOTER_URL_RE, (url) => `${url}`) + .replace(/\n/g, "
") } export interface RenderEmailOptions { @@ -110,7 +118,7 @@ export function renderEmailBody(opts: RenderEmailOptions): { text: string; html: `` + `` + `` + - `` + + `` + `` const text = [...opts.blocks.map((b) => b.text), "--", footerText].join("\n\n") diff --git a/services/api/src/adapters/mailer.oci.ts b/services/api/src/adapters/mailer.oci.ts index cd7b83c4..e6ab14c4 100644 --- a/services/api/src/adapters/mailer.oci.ts +++ b/services/api/src/adapters/mailer.oci.ts @@ -5,7 +5,7 @@ import type { Mailer, OutboundEmail, SentMail } from "@civfix/shared/interfaces" import type { Transporter } from "nodemailer" import { domainOf, escapeHtml, sanitizeHeaderValue } from "./mail-text.js" import { mailFailure } from "./mail-failure.js" -import { code, paragraph } from "./email-blocks.js" +import { button, code, heading, kvTable, paragraph, quote, type EmailBlock } from "./email-blocks.js" import { renderEmailBody } from "./email-layout.js" import { renderMessage } from "../i18n/renderMessage.js" import { resolveLocale, type Locale } from "../i18n/locales.js" @@ -185,7 +185,7 @@ export class OciMailer implements Mailer { } } -function renderOtp(passcode: string, locale: Locale): Rendered { +export function renderOtp(passcode: string, locale: Locale): Rendered { const subject = renderMessage(locale, "email.otp.subject") const { text, html } = renderEmailBody({ preheader: subject, @@ -198,7 +198,7 @@ function renderOtp(passcode: string, locale: Locale): Rendered { return { subject, text, html } } -function renderTemplate(template: string, vars: Record): Rendered { +export function renderTemplate(template: string, vars: Record): Rendered { const locale = resolveLocale(typeof vars.locale === "string" ? vars.locale : undefined) switch (template) { case "report_update": { @@ -208,6 +208,80 @@ function renderTemplate(template: string, vars: Record): Render const { text, html } = renderEmailBody({ preheader: subject, blocks: [paragraph(message)] }) return { subject, text, html } } + case "guest_otp": { + const title = stringVar(vars, "title", "the event") + const passcode = stringVar(vars, "code", "") + const minutes = stringVar(vars, "minutes", "5") + const subject = renderMessage(locale, "email.guest_otp.subject", { title }) + const { text, html } = renderEmailBody({ + preheader: subject, + blocks: [ + paragraph(renderMessage(locale, "email.guest_otp.html_intro", { title })), + code(passcode), + paragraph(renderMessage(locale, "email.guest_otp.body_expiry", { minutes }), { + muted: true, + }), + ], + }) + return { subject, text, html } + } + case "guest_confirmed": { + const title = stringVar(vars, "title", "the event") + const when = optionalVar(vars, "when") + const place = optionalVar(vars, "place") + const cancelUrl = optionalVar(vars, "cancelUrl") + const subject = renderMessage(locale, "email.guest_confirmed.subject", { title }) + const details: Array<[string, string]> = [] + if (when !== undefined) details.push([renderMessage(locale, "email.event.when"), when]) + if (place !== undefined) details.push([renderMessage(locale, "email.event.where"), place]) + const blocks: EmailBlock[] = [heading(title)] + if (details.length > 0) blocks.push(kvTable(details)) + blocks.push(paragraph(renderMessage(locale, "email.guest_confirmed.checkin"))) + if (cancelUrl !== undefined) { + blocks.push( + paragraph(renderMessage(locale, "email.guest_confirmed.cancel_hint"), { muted: true }), + button(cancelUrl, renderMessage(locale, "email.guest_confirmed.cancel_cta")), + ) + } + const { text, html } = renderEmailBody({ preheader: subject, blocks }) + return { subject, text, html } + } + case "guest_promoted": { + const title = stringVar(vars, "title", "the event") + const when = stringVar(vars, "when", "") + const eventUrl = optionalVar(vars, "eventUrl") + const subject = renderMessage(locale, "email.guest_promoted.subject", { title }) + const blocks: EmailBlock[] = [ + paragraph(renderMessage(locale, "email.guest_promoted.intro", { title, when })), + ] + if (eventUrl !== undefined) { + blocks.push(button(eventUrl, renderMessage(locale, "email.guest_promoted.cta"))) + } + blocks.push(paragraph(renderMessage(locale, "email.guest_promoted.ignore"), { muted: true })) + const { text, html } = renderEmailBody({ preheader: subject, blocks }) + return { subject, text, html } + } + case "action": { + const subject = stringVar(vars, "subject", renderMessage(locale, "email.generic.subject")) + const blocks: EmailBlock[] = paragraphsVar(vars).map((p) => paragraph(p)) + const quoteHeading = optionalVar(vars, "quoteHeading") + const quoted = optionalVar(vars, "quote") + if (quoted !== undefined) { + if (quoteHeading !== undefined) blocks.push(heading(quoteHeading)) + blocks.push(quote(quoted)) + } + const ctaUrl = optionalVar(vars, "ctaUrl") + if (ctaUrl !== undefined) { + blocks.push(button(ctaUrl, stringVar(vars, "ctaLabel", "Open"))) + } + const note = optionalVar(vars, "note") + if (note !== undefined) blocks.push(paragraph(note, { muted: true })) + if (blocks.length === 0) { + blocks.push(paragraph(renderMessage(locale, "email.generic.body"))) + } + const { text, html } = renderEmailBody({ preheader: subject, blocks }) + return { subject, text, html } + } default: { const subject = stringVar(vars, "subject", renderMessage(locale, "email.generic.subject")) const message = stringVar(vars, "message", renderMessage(locale, "email.generic.body")) @@ -222,6 +296,17 @@ function stringVar(vars: Record, key: string, fallback: string) return typeof v === "string" && v.length > 0 ? v : fallback } +function optionalVar(vars: Record, key: string): string | undefined { + const v = vars[key] + return typeof v === "string" && v.length > 0 ? v : undefined +} + +function paragraphsVar(vars: Record): string[] { + const v = vars.paragraphs + if (!Array.isArray(v)) return [] + return v.filter((p): p is string => typeof p === "string" && p.length > 0) +} + function textToHtml(text: string): string { return `

${escapeHtml(text).replace(/\n/g, "
")}

` } diff --git a/services/api/src/i18n/messages/de.ts b/services/api/src/i18n/messages/de.ts index 55679fcc..bf7d415c 100644 --- a/services/api/src/i18n/messages/de.ts +++ b/services/api/src/i18n/messages/de.ts @@ -120,14 +120,23 @@ export const de: Partial> = { // ---- Guest event RSVP ------------------------------------------------------------------------- "email.guest_otp.subject": "Dein Code für die Anmeldung zu {{title}}", - "email.guest_otp.body": - "Dein Code für die Anmeldung zu {{title}} lautet {{code}}. Er läuft in {{minutes}} Minuten ab. Wenn du ihn nicht angefordert hast, kannst du diese E-Mail ignorieren.", + "email.guest_otp.html_intro": "Dein Code für die Anmeldung zu {{title}} lautet:", + "email.guest_otp.body_expiry": + "Er läuft in {{minutes}} Minuten ab. Wenn du ihn nicht angefordert hast, kannst du diese E-Mail ignorieren.", "email.guest_confirmed.subject": "Du stehst auf der Liste für {{title}}", - "email.guest_confirmed.body": - "Du bist für {{title}} angemeldet. Melde dich vor Ort mit deinem Namen an; es gibt kein Ticket zum Ausdrucken. Anders entschieden? Hier kannst du deine Anmeldung stornieren: {{link}}", + "email.guest_confirmed.checkin": + "Du stehst auf der Liste. Melde dich vor Ort mit deinem Namen an; es gibt kein Ticket zum Ausdrucken.", + "email.guest_confirmed.cancel_hint": + "Anders entschieden? Storniere deine Anmeldung, damit jemand anderes den Platz bekommen kann.", + "email.guest_confirmed.cancel_cta": "Anmeldung stornieren", + "email.event.when": "Wann", + "email.event.where": "Wo", "email.guest_promoted.subject": "Ein Platz für {{title}} ist frei geworden", - "email.guest_promoted.body": - "Ein Platz für {{title}} am {{when}} ist frei geworden. Du standest auf der Warteliste, und die Veranstalterin oder der Veranstalter hält einen Platz für dich frei. Sieh dir die Veranstaltung an und melde dich dort, wenn du noch kommen möchtest: {{link}}", + "email.guest_promoted.intro": + "Ein Platz für {{title}} am {{when}} ist frei geworden. Du standest auf der Warteliste, und die Veranstalterin oder der Veranstalter hält einen Platz für dich frei.", + "email.guest_promoted.cta": "Veranstaltung ansehen", + "email.guest_promoted.ignore": + "Wenn du den Platz nicht mehr möchtest, musst du nichts weiter tun.", "email.guest_updated.subject": "{{title}} hat neue Angaben", "email.guest_updated.body": "Die Angaben zu {{title}} haben sich geändert. Es beginnt jetzt um {{when}} an folgendem Ort: {{place}}. Nutze den Stornierungslink aus deiner Bestätigung, falls du nicht mehr kommen kannst.", diff --git a/services/api/src/i18n/messages/en.ts b/services/api/src/i18n/messages/en.ts index dc87a8db..1b42a0fa 100644 --- a/services/api/src/i18n/messages/en.ts +++ b/services/api/src/i18n/messages/en.ts @@ -182,14 +182,23 @@ export const en = { // Sent to people who are NOT users, so there is no users.locale to key off: the guest service renders // these in "en". They are catalogued anyway so a future guest-locale field needs no code change. "email.guest_otp.subject": "Your code to RSVP for {{title}}", - "email.guest_otp.body": - "Your code to RSVP for {{title}} is {{code}}. It expires in {{minutes}} minutes. If you did not request it, you can ignore this email.", + "email.guest_otp.html_intro": "Your code to RSVP for {{title}} is:", + "email.guest_otp.body_expiry": + "It expires in {{minutes}} minutes. If you did not request it, you can ignore this email.", "email.guest_confirmed.subject": "You are on the list for {{title}}", - "email.guest_confirmed.body": - "You are signed up for {{title}}. Check in by name when you arrive; there is no ticket to print. Change your mind? Cancel your RSVP here: {{link}}", + "email.guest_confirmed.checkin": + "You are on the list. Check in by name when you arrive; there is no ticket to print.", + "email.guest_confirmed.cancel_hint": + "Plans changed? Cancel your RSVP so someone else can take the place.", + "email.guest_confirmed.cancel_cta": "Cancel RSVP", + "email.event.when": "When", + "email.event.where": "Where", "email.guest_promoted.subject": "A place opened up for {{title}}", - "email.guest_promoted.body": - "A place opened up for {{title}} on {{when}}. You were on the waitlist and the host is holding a place for you. See the event and contact the host if you still want it: {{link}}", + "email.guest_promoted.intro": + "A place opened up for {{title}} on {{when}}. You were on the waitlist and the host is holding a place for you.", + "email.guest_promoted.cta": "See the event", + "email.guest_promoted.ignore": + "If you no longer want the place, there is nothing you need to do.", "email.guest_updated.subject": "{{title}} has new details", "email.guest_updated.body": "The details for {{title}} changed. It now starts at {{when}} at {{place}}. Use the cancel link in your confirmation message if you can no longer make it.", diff --git a/services/api/src/i18n/messages/es.ts b/services/api/src/i18n/messages/es.ts index 9dfed236..ae977290 100644 --- a/services/api/src/i18n/messages/es.ts +++ b/services/api/src/i18n/messages/es.ts @@ -119,14 +119,22 @@ export const es: Partial> = { // ---- Guest event RSVP ------------------------------------------------------------------------- "email.guest_otp.subject": "Tu código para confirmar tu asistencia a {{title}}", - "email.guest_otp.body": - "Tu código para confirmar tu asistencia a {{title}} es {{code}}. Caduca en {{minutes}} minutos. Si no lo solicitaste, puedes ignorar este correo.", + "email.guest_otp.html_intro": "Tu código para confirmar tu asistencia a {{title}} es:", + "email.guest_otp.body_expiry": + "Caduca en {{minutes}} minutos. Si no lo solicitaste, puedes ignorar este correo.", "email.guest_confirmed.subject": "Estás en la lista de {{title}}", - "email.guest_confirmed.body": - "Te has apuntado a {{title}}. Al llegar, regístrate con tu nombre; no hay ninguna entrada que imprimir. ¿Cambiaste de opinión? Cancela tu asistencia aquí: {{link}}", + "email.guest_confirmed.checkin": + "Estás en la lista. Al llegar, regístrate con tu nombre; no hay ninguna entrada que imprimir.", + "email.guest_confirmed.cancel_hint": + "¿Cambiaron tus planes? Cancela tu asistencia para que otra persona pueda ocupar la plaza.", + "email.guest_confirmed.cancel_cta": "Cancelar asistencia", + "email.event.when": "Cuándo", + "email.event.where": "Dónde", "email.guest_promoted.subject": "Se ha liberado una plaza para {{title}}", - "email.guest_promoted.body": - "Se ha liberado una plaza para {{title}} el {{when}}. Estabas en la lista de espera y la organización te está reservando una plaza. Consulta el evento y ponte en contacto con la organización si todavía quieres asistir: {{link}}", + "email.guest_promoted.intro": + "Se ha liberado una plaza para {{title}} el {{when}}. Estabas en la lista de espera y la organización te está reservando una plaza.", + "email.guest_promoted.cta": "Ver el evento", + "email.guest_promoted.ignore": "Si ya no quieres la plaza, no tienes que hacer nada.", "email.guest_updated.subject": "{{title}} tiene nuevos detalles", "email.guest_updated.body": "Los detalles de {{title}} han cambiado. Ahora empieza a las {{when}} en {{place}}. Usa el enlace de cancelación de tu mensaje de confirmación si ya no puedes asistir.", diff --git a/services/api/src/i18n/messages/ko.ts b/services/api/src/i18n/messages/ko.ts index c76b6a58..74726104 100644 --- a/services/api/src/i18n/messages/ko.ts +++ b/services/api/src/i18n/messages/ko.ts @@ -119,14 +119,22 @@ export const ko: Partial> = { // ---- Guest event RSVP ------------------------------------------------------------------------- "email.guest_otp.subject": "{{title}} 참가 신청 코드", - "email.guest_otp.body": - "{{title}} 참가 신청 코드는 {{code}}입니다. {{minutes}}분 후에 만료됩니다. 요청하지 않으셨다면 이 이메일을 무시하셔도 됩니다.", + "email.guest_otp.html_intro": "{{title}} 참가 신청 코드:", + "email.guest_otp.body_expiry": + "{{minutes}}분 후에 만료됩니다. 요청하지 않으셨다면 이 이메일을 무시하셔도 됩니다.", "email.guest_confirmed.subject": "{{title}} 참가자 명단에 등록되었습니다", - "email.guest_confirmed.body": - "{{title}}에 참가 신청이 완료되었습니다. 현장에서는 이름으로 체크인하며, 출력할 티켓은 없습니다. 마음이 바뀌셨나요? 여기에서 취소하세요: {{link}}", + "email.guest_confirmed.checkin": + "참가자 명단에 등록되었습니다. 현장에서는 이름으로 체크인하며, 출력할 티켓은 없습니다.", + "email.guest_confirmed.cancel_hint": + "일정이 바뀌셨나요? 다른 분이 자리를 이용할 수 있도록 참가를 취소해 주세요.", + "email.guest_confirmed.cancel_cta": "참가 취소", + "email.event.when": "일시", + "email.event.where": "장소", "email.guest_promoted.subject": "{{title}}에 자리가 생겼습니다", - "email.guest_promoted.body": - "{{when}}에 열리는 {{title}}에 자리가 생겼습니다. 대기자 명단에 계셨고, 주최자가 자리를 잡아두고 있습니다. 아직 참가를 원하시면 행사 페이지를 확인하고 주최자에게 문의하세요: {{link}}", + "email.guest_promoted.intro": + "{{when}}에 열리는 {{title}}에 자리가 생겼습니다. 대기자 명단에 계셨고, 주최자가 자리를 잡아두고 있습니다.", + "email.guest_promoted.cta": "행사 보기", + "email.guest_promoted.ignore": "자리가 더 이상 필요 없으시면 따로 하실 일은 없습니다.", "email.guest_updated.subject": "{{title}} 세부 정보가 변경되었습니다", "email.guest_updated.body": "{{title}}의 세부 정보가 변경되었습니다. 이제 {{when}}에 {{place}}에서 시작합니다. 참석이 어려우시면 확인 메시지의 취소 링크를 이용하세요.", diff --git a/services/api/test/unit/email-format.test.ts b/services/api/test/unit/email-format.test.ts index 06ec535a..ab42103b 100644 --- a/services/api/test/unit/email-format.test.ts +++ b/services/api/test/unit/email-format.test.ts @@ -4,8 +4,10 @@ import { buildDiscussionForwardPacket, buildEventPacket, } from "../../src/services/admin/mail-format.js" -import { renderEmailBody } from "../../src/adapters/email-layout.js" +import { eventFooter, renderEmailBody } from "../../src/adapters/email-layout.js" import { paragraph, kvTable, linkList } from "../../src/adapters/email-blocks.js" +import { renderOtp, renderTemplate } from "../../src/adapters/mailer.oci.js" +import { buildDataExportEmail } from "../../src/services/data-export-service.js" import type { AdminReportRecord } from "../../src/services/admin/admin-report-types.js" function reportRecord(overrides: Partial = {}): AdminReportRecord { @@ -210,3 +212,147 @@ describe("buildEventPacket", () => { expect(packet.text).toContain("Reference: EVT-9") }) }) + +describe("footer rendering", () => { + it("splits the event footer into lines and links its URLs in the HTML part", () => { + const footer = eventFooter({ + eventTitle: "Beach Cleanup", + unsubscribeUrl: "https://civfix.org/unsubscribe?t=abc", + manageUrl: "https://civfix.org/e/beach", + }) + expect(footer).toContain("\n") + const { html, text } = renderEmailBody({ blocks: [paragraph("x")], footer }) + expect(html).toContain('href="https://civfix.org/unsubscribe?t=abc"') + expect(html).toContain('href="https://civfix.org/e/beach"') + expect(html).toContain("never gave them your email address.
") + expect(text).toContain("Stop receiving messages about this event: https://civfix.org/unsubscribe?t=abc") + }) + + it("escapes HTML in the footer before linkifying", () => { + const { html } = renderEmailBody({ blocks: [paragraph("x")], footer: "bold https://civfix.org" }) + expect(html).not.toContain("bold") + expect(html).toContain("<b>bold</b>") + expect(html).toContain('href="https://civfix.org"') + }) +}) + +describe("renderOtp", () => { + it("renders the passcode in the styled code block with a muted expiry line", () => { + const out = renderOtp("482913", "en") + expect(out.subject).toBe("Your civfix sign-in code") + expect(out.html).toContain("482913") + expect(out.html).toContain("letter-spacing:6px") + expect(out.text).toContain("482913") + expect(out.text).toContain("expires in 5 minutes") + }) +}) + +describe("renderTemplate", () => { + it("guest_otp: shows the code in the big code block, not buried in a sentence", () => { + const out = renderTemplate("guest_otp", { title: "Beach Cleanup", code: "738201", minutes: "5" }) + expect(out.subject).toBe("Your code to RSVP for Beach Cleanup") + expect(out.html).toContain("letter-spacing:6px") + expect(out.html).toContain(">738201<") + expect(out.text).toContain("738201") + expect(out.text).toContain("expires in 5 minutes") + }) + + it("guest_confirmed: event heading, when/where rows, honest check-in line and a cancel button", () => { + const out = renderTemplate("guest_confirmed", { + title: "Beach Cleanup", + when: "Saturday, October 3 at 9:00 AM PDT", + place: "Ballona Creek Trailhead", + cancelUrl: "https://civfix.org/guest?token=abc", + }) + expect(out.subject).toBe("You are on the list for Beach Cleanup") + expect(out.html).toContain("Cancel RSVP") + expect(out.html).not.toContain(">https://civfix.org/guest?token=abc<") + expect(out.text).toContain("Cancel RSVP: https://civfix.org/guest?token=abc") + }) + + it("guest_confirmed: omits the details table when when/where are absent", () => { + const out = renderTemplate("guest_confirmed", { + title: "Beach Cleanup", + cancelUrl: "https://civfix.org/guest?token=abc", + }) + expect(out.html).not.toContain(">When<") + expect(out.html).toContain("Check in by name") + }) + + it("guest_promoted: intro paragraph, event button, muted opt-out line", () => { + const out = renderTemplate("guest_promoted", { + title: "Beach Cleanup", + when: "Saturday at 9:00 AM", + eventUrl: "https://civfix.org/cleanups/e1", + }) + expect(out.subject).toBe("A place opened up for Beach Cleanup") + expect(out.html).toContain("holding a place for you") + expect(out.html).toContain('href="https://civfix.org/cleanups/e1"') + expect(out.html).toContain(">See the event") + expect(out.html).toContain("nothing you need to do") + }) + + it("action: paragraphs, quote with heading, CTA button and muted note", () => { + const out = renderTemplate("action", { + subject: "You have been invited", + paragraphs: ["First paragraph.", "Second paragraph."], + quoteHeading: "Reason", + quote: "The uploaded letter names a different entity.", + ctaUrl: "https://civfix.org/accept#token=x", + ctaLabel: "Accept the invitation", + note: "The invitation expires in 14 days.", + }) + expect(out.subject).toBe("You have been invited") + expect(out.html).toContain("First paragraph.") + expect(out.html).toContain("Second paragraph.") + expect(out.html).toContain(">Reason") + expect(out.html).toContain("different entity") + expect(out.html).toContain('href="https://civfix.org/accept#token=x"') + expect(out.html).toContain(">Accept the invitation") + expect(out.html).not.toContain(">https://civfix.org/accept#token=x<") + expect(out.text).toContain("Accept the invitation: https://civfix.org/accept#token=x") + expect(out.text).toContain("The invitation expires in 14 days.") + }) + + it("action: escapes HTML-significant user content", () => { + const out = renderTemplate("action", { + subject: "s", + paragraphs: [""], + }) + expect(out.html).not.toContain("") + expect(out.html).toContain("<script>") + }) + + it("action: falls back to the generic body when nothing usable is passed", () => { + const out = renderTemplate("action", { subject: "s", paragraphs: [42, ""] }) + expect(out.html).toContain("You have a new civfix notification.") + }) + + it("unknown templates still fall back to the generic paragraph shell", () => { + const out = renderTemplate("whatever", { subject: "Hi", message: "Body text." }) + expect(out.subject).toBe("Hi") + expect(out.html).toContain("Body text.") + }) +}) + +describe("buildDataExportEmail", () => { + it("renders the branded shell with the truncation note only when sections were clipped", () => { + const full = buildDataExportEmail("support@civfix.org", []) + expect(full.subject).toBe("Your civfix data export") + expect(full.html).toContain("") + expect(full.html).toContain("civfix-export.json") + expect(full.html).not.toContain("only part of them") + + const clipped = buildDataExportEmail("support@civfix.org", ["messages", "comments"]) + expect(clipped.html).toContain("messages, comments") + expect(clipped.html).toContain("support@civfix.org") + expect(clipped.text).toContain("If you did not request this, you can ignore this email.") + }) +}) diff --git a/services/api/test/unit/host-broadcast-render.test.ts b/services/api/test/unit/host-broadcast-render.test.ts index 5cb9d919..0ef42d37 100644 --- a/services/api/test/unit/host-broadcast-render.test.ts +++ b/services/api/test/unit/host-broadcast-render.test.ts @@ -64,7 +64,7 @@ describe("broadcast rendering", () => { ctx, ) expect(out.html).not.toContain("javascript:") - expect(out.html).not.toContain("click") }) it("renders a CTA button when a url is present", () => { From 08d207d99ff559bdfe0eb0b492a76cfb7a6a7266 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 22:07:03 -0700 Subject: [PATCH 39/54] send guests the structured emails: big code, event card, cancel button The OTP now lands in the styled code block instead of mid-sentence, the confirmation shows the event title, when and where (the loaded event is passed through joinAsGuest instead of just its title) with a Cancel RSVP button in place of a raw token URL, and the waitlist promotion gets a See-the-event button. SMS paths are untouched. Co-Authored-By: Claude Fable 5 --- services/api/src/services/guest-notify.ts | 12 ++-- .../api/src/services/guest-rsvp-service.ts | 55 ++++++++----------- services/api/test/unit/guest-notify.test.ts | 21 ++++--- .../api/test/unit/guest-rsvp-service.test.ts | 22 ++++++-- 4 files changed, 52 insertions(+), 58 deletions(-) diff --git a/services/api/src/services/guest-notify.ts b/services/api/src/services/guest-notify.ts index 69b02ec1..86dd0a16 100644 --- a/services/api/src/services/guest-notify.ts +++ b/services/api/src/services/guest-notify.ts @@ -1,5 +1,4 @@ import type { Mailer } from "@civfix/shared/interfaces" -import { renderMessage } from "../i18n/renderMessage.js" import { formatEventWhen } from "./host/broadcast-render.js" import { DEFAULT_EVENT_TIME_ZONE } from "./host/event-fields.js" import type { GuestRsvpRepository } from "./guest-rsvp-service.js" @@ -34,13 +33,10 @@ export function makeGuestPromotionNotifier(deps: GuestNotifyDeps): GuestPromotio if (guest.email === null || guest.email.length === 0) return const event = await deps.repo.findEvent(guest.cleanupId) if (event === null) return - await deps.mailer.sendTransactional(guest.email, "generic", { - subject: renderMessage("en", "email.guest_promoted.subject", { title: event.title }), - message: renderMessage("en", "email.guest_promoted.body", { - title: event.title, - when: formatEventWhen(event.scheduledAt, event.timezone ?? DEFAULT_EVENT_TIME_ZONE), - link: guestEventLink(deps.linkBase, guest.cleanupId), - }), + await deps.mailer.sendTransactional(guest.email, "guest_promoted", { + title: event.title, + when: formatEventWhen(event.scheduledAt, event.timezone ?? DEFAULT_EVENT_TIME_ZONE), + eventUrl: guestEventLink(deps.linkBase, guest.cleanupId), }) }, } diff --git a/services/api/src/services/guest-rsvp-service.ts b/services/api/src/services/guest-rsvp-service.ts index 11e13b0c..22d243ef 100644 --- a/services/api/src/services/guest-rsvp-service.ts +++ b/services/api/src/services/guest-rsvp-service.ts @@ -445,10 +445,6 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi return `${deps.manageLinkBase}/guest?token=${encodeURIComponent(rawToken)}` } - async function sendGuestEmail(to: string, subject: string, message: string): Promise { - await deps.mailer.sendTransactional(to, "generic", { subject, message }) - } - async function deliverCode(args: { channel: GuestContactChannel contact: string @@ -456,15 +452,11 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi eventTitle: string }): Promise { if (args.channel === "email") { - await sendGuestEmail( - args.contact, - renderMessage("en", "email.guest_otp.subject", { title: args.eventTitle }), - renderMessage("en", "email.guest_otp.body", { - title: args.eventTitle, - code: args.code, - minutes: Math.floor(GUEST_OTP_TTL_SECONDS / 60), - }), - ) + await deps.mailer.sendTransactional(args.contact, "guest_otp", { + title: args.eventTitle, + code: args.code, + minutes: String(Math.floor(GUEST_OTP_TTL_SECONDS / 60)), + }) return } await deps.smsSender.send( @@ -616,8 +608,7 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi } async function joinAsGuest(args: { - cleanupId: string - eventTitle: string + event: GuestEventView name: string channel: GuestContactChannel contact: string @@ -627,7 +618,7 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi const rawToken = newToken() const manageTokenHash = await sha256Hex(rawToken) const guest = await deps.repo.upsertVerifiedGuest({ - cleanupId: args.cleanupId, + cleanupId: args.event.id, name: args.name, channel: args.channel, contactKey: args.contact, @@ -637,15 +628,15 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi now: new Date(now()), }) const seat = await registerVerifiedGuest( - args.cleanupId, + args.event.id, guest.id, args.registration ?? {}, ) - const going = await deps.repo.goingCount(args.cleanupId) + const going = await deps.repo.goingCount(args.event.id) if (args.confirm) { await sendConfirmation({ ...args, rawToken }).catch((err: unknown) => { deps.logger?.warn( - { err, cleanupId: args.cleanupId }, + { err, cleanupId: args.event.id }, "guest rsvp: confirmation message failed (suppressed; the RSVP stands)", ) }) @@ -661,20 +652,20 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi } async function sendConfirmation(args: { - eventTitle: string + event: GuestEventView channel: GuestContactChannel contact: string rawToken: string }): Promise { if (args.channel === "email") { - await sendGuestEmail( - args.contact, - renderMessage("en", "email.guest_confirmed.subject", { title: args.eventTitle }), - renderMessage("en", "email.guest_confirmed.body", { - title: args.eventTitle, - link: manageLink(args.rawToken), - }), - ) + await deps.mailer.sendTransactional(args.contact, "guest_confirmed", { + title: args.event.title, + when: formatEventWhen(args.event.scheduledAt, args.event.timezone ?? DEFAULT_EVENT_TIME_ZONE), + ...(args.event.address !== null && args.event.address.length > 0 + ? { place: args.event.address } + : {}), + cancelUrl: manageLink(args.rawToken), + }) return } if (!deps.smsGuestEnabled) return @@ -688,7 +679,7 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi await deps.smsSender.send( args.contact, renderMessage("en", "sms.guest_confirmed.body", { - title: smsTitle(args.eventTitle), + title: smsTitle(args.event.title), link: manageLink(args.rawToken), }), ) @@ -848,8 +839,7 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi if (isReviewerContact(input.channel, contact)) { if (reviewer !== null && constantTimeStringEqual(input.code, reviewer.code)) { return joinAsGuest({ - cleanupId: event.id, - eventTitle: event.title, + event, name: REVIEWER_DISPLAY_NAME, channel: input.channel, contact, @@ -899,8 +889,7 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi }) return joinAsGuest({ - cleanupId: event.id, - eventTitle: event.title, + event, name: record.name, channel: input.channel, contact, diff --git a/services/api/test/unit/guest-notify.test.ts b/services/api/test/unit/guest-notify.test.ts index db0c9fa6..cb6a17c4 100644 --- a/services/api/test/unit/guest-notify.test.ts +++ b/services/api/test/unit/guest-notify.test.ts @@ -66,13 +66,12 @@ describe("guest waitlist promotion notice", () => { expect(h.mailer.sent).toHaveLength(1) const mail = h.mailer.sent[0] expect(mail?.to).toBe("ada@example.org") - expect(mail?.template).toBe("generic") - expect(String(mail?.vars?.subject)).toContain("Beach cleanup") - const body = String(mail?.vars?.message) - expect(body).toContain("Beach cleanup") - expect(body).toContain(formatEventWhen(SCHEDULED_AT, "America/Los_Angeles")) - expect(body).toContain("10:00 AM PDT") - expect(body).toContain(guestEventLink("https://civfix.org", EVENT_ID)) + expect(mail?.template).toBe("guest_promoted") + expect(String(mail?.vars?.title)).toBe("Beach cleanup") + const when = String(mail?.vars?.when) + expect(when).toBe(formatEventWhen(SCHEDULED_AT, "America/Los_Angeles")) + expect(when).toContain("10:00 AM PDT") + expect(mail?.vars?.eventUrl).toBe(guestEventLink("https://civfix.org", EVENT_ID)) }) it("prints Pacific Time when the event carries no timezone, never a raw ISO stamp", async () => { @@ -80,15 +79,15 @@ describe("guest waitlist promotion notice", () => { h.repo.seedEvent({ id: EVENT_ID, scheduledAt: SCHEDULED_AT, timezone: null }) await h.notify(h.guestId) - const body = String(h.mailer.sent[0]?.vars?.message) - expect(body).toContain("10:00 AM PDT") - expect(body).not.toContain("2026-09-05T17:00:00.000Z") + const when = String(h.mailer.sent[0]?.vars?.when) + expect(when).toContain("10:00 AM PDT") + expect(when).not.toContain("2026-09-05T17:00:00.000Z") }) it("never leaks the manage token: the link is the public event page", async () => { const h = harness() await h.notify(h.guestId) - expect(String(h.mailer.sent[0]?.vars?.message)).not.toContain("token=") + expect(JSON.stringify(h.mailer.sent[0]?.vars)).not.toContain("token=") }) it("sends nothing to an SMS-channel guest, who has no address to mail", async () => { diff --git a/services/api/test/unit/guest-rsvp-service.test.ts b/services/api/test/unit/guest-rsvp-service.test.ts index baabdc70..e93ebd8f 100644 --- a/services/api/test/unit/guest-rsvp-service.test.ts +++ b/services/api/test/unit/guest-rsvp-service.test.ts @@ -19,6 +19,7 @@ import { smsFailure } from "../../src/errors/sms-failure.js" import { formatEventWhen } from "../../src/services/host/broadcast-render.js" import { generateToken } from "../../src/auth/crypto.js" import { OTP_VERIFY_IP_FAIL_MAX } from "../../src/auth/otp.js" +import { renderTemplate } from "../../src/adapters/mailer.oci.js" import { GUEST_CONTACT_MAX_PER_DAY, GUEST_IP_MAX_PER_HOUR, @@ -162,8 +163,12 @@ describe("guest rsvp: requesting a code", () => { expect(result).toEqual({ sent: true, resendAfterSec: 60 }) expect(h.mailer.sent).toHaveLength(1) expect(h.mailer.sent[0]?.to).toBe("ada@example.org") - expect(String(h.mailer.sent[0]?.vars?.message)).toContain(CODE) - expect(String(h.mailer.sent[0]?.vars?.subject)).toContain("Beach cleanup") + expect(h.mailer.sent[0]?.template).toBe("guest_otp") + expect(String(h.mailer.sent[0]?.vars?.code)).toBe(CODE) + expect(String(h.mailer.sent[0]?.vars?.title)).toContain("Beach cleanup") + const rendered = renderTemplate("guest_otp", h.mailer.sent[0]?.vars ?? {}) + expect(rendered.subject).toContain("Beach cleanup") + expect(rendered.text).toContain(CODE) expect(h.repo.otps).toHaveLength(1) expect(h.repo.otps[0]?.name).toBe("Ada Lovelace") }) @@ -442,7 +447,12 @@ describe("guest rsvp: verifying a code", () => { const result = await h.service.verifyCode(emailVerify(), ctx) const confirmation = h.mailer.sent.at(-1) - expect(String(confirmation?.vars?.message)).toContain( + expect(confirmation?.template).toBe("guest_confirmed") + expect(confirmation?.vars?.cancelUrl).toBe( + `https://civfix.org/guest?token=${encodeURIComponent(result.manageToken)}`, + ) + const rendered = renderTemplate("guest_confirmed", confirmation?.vars ?? {}) + expect(rendered.text).toContain( `https://civfix.org/guest?token=${encodeURIComponent(result.manageToken)}`, ) }) @@ -454,10 +464,10 @@ describe("guest rsvp: verifying a code", () => { */ it("tells the guest how they actually get in, since no ticket is ever linked", async () => { await h.service.verifyCode(emailVerify(), ctx) - const body = String(h.mailer.sent.at(-1)?.vars?.message) + const rendered = renderTemplate("guest_confirmed", h.mailer.sent.at(-1)?.vars ?? {}) - expect(body).toContain("Check in by name") - expect(body).toContain("no ticket to print") + expect(rendered.text).toContain("Check in by name") + expect(rendered.text).toContain("no ticket to print") }) it("rejects a code that was already consumed by an earlier successful verify", async () => { From df7b54852a0845643cda72501573a5149dfcce22 Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 22:07:03 -0700 Subject: [PATCH 40/54] structure the invite, verification and data-export emails Org invites, org verification decisions and event team invites move off hand-glued sentences onto the action template via exported vars builders, so the accept link becomes a button and the expiry a muted note; a rejection quotes its reason. The DSAR export mail trades bare plain text for the standard branded shell. Co-Authored-By: Claude Fable 5 --- .../api/src/services/data-export-service.ts | 51 ++++++++---- .../src/services/host/host-team-service.ts | 22 +++++- .../src/services/host/organization-service.ts | 78 ++++++++++++++++--- .../unit/admin-host-platform-routes.test.ts | 2 +- .../test/unit/host/host-team-service.test.ts | 9 +-- .../unit/host/organization-service.test.ts | 15 ++-- 6 files changed, 134 insertions(+), 43 deletions(-) diff --git a/services/api/src/services/data-export-service.ts b/services/api/src/services/data-export-service.ts index f22aa9a1..2e66d593 100644 --- a/services/api/src/services/data-export-service.ts +++ b/services/api/src/services/data-export-service.ts @@ -1,6 +1,8 @@ import type { Sql } from "../db/client.js" import type { Mailer } from "@civfix/shared/interfaces" import type { UserStore } from "../auth/stores.js" +import { heading, paragraph } from "../adapters/email-blocks.js" +import { renderEmailBody } from "../adapters/email-layout.js" export interface DataExportServiceDeps { sql: Sql @@ -20,6 +22,38 @@ export const DATA_EXPORT_FREE_TEXT_MAX_ROWS = 5_000 export const DATA_EXPORT_BYTE_BUDGET = 8_000_000 +export function buildDataExportEmail( + supportEmail: string, + truncatedSections: readonly string[], +): { subject: string; text: string; html: string } { + const subject = "Your civfix data export" + const blocks = [ + heading("Your civfix data export"), + paragraph("Attached is a copy of your civfix data as a JSON file (civfix-export.json)."), + paragraph( + "It includes your profile, reports, posts, comments, messages, events, registrations, " + + "organizations, volunteer hours, connections, and your issued service-hours transcripts.", + ), + paragraph( + "Secrets (login codes, session tokens, raw device tokens, ticket tokens) and other " + + "people's material (host-private notes, another organization's verification evidence) " + + "are intentionally excluded.", + { muted: true }, + ), + ] + if (truncatedSections.length > 0) { + blocks.push( + paragraph( + `Note: some sections (${truncatedSections.join(", ")}) were very large and this export ` + + `contains only part of them. Email ${supportEmail} to request a complete copy of those sections.`, + ), + ) + } + blocks.push(paragraph("If you did not request this, you can ignore this email.", { muted: true })) + const { text, html } = renderEmailBody({ preheader: subject, blocks }) + return { subject, text, html } +} + export function makeDataExportService(deps: DataExportServiceDeps): DataExportService { const { sql, mailer, users, fromNoReply, supportEmail } = deps @@ -435,23 +469,14 @@ export function makeDataExportService(deps: DataExportServiceDeps): DataExportSe const bytes = new TextEncoder().encode(JSON.stringify(exportObject)) - const text = - "Attached is a copy of your civfix data (JSON). It includes your profile, reports, posts, " + - "comments, messages, events, registrations, organizations, volunteer hours, " + - "connections, and your issued service-hours transcripts. Secrets (login codes, session " + - "tokens, raw device tokens, ticket tokens) and other people's material (host-private notes, " + - "another organization's verification evidence) are intentionally excluded." + - (truncatedSections.length > 0 - ? `\n\nNote: some sections (${truncatedSections.join(", ")}) were very large and this export ` + - `contains only part of them. Email ${supportEmail} to request a complete copy of those sections.` - : "") + - "\n\nIf you did not request this, you can ignore this email." + const rendered = buildDataExportEmail(supportEmail, truncatedSections) await mailer.sendOutbound({ from: fromNoReply, to: email, - subject: "Your civfix data export", - text, + subject: rendered.subject, + text: rendered.text, + html: rendered.html, attachments: [ { filename: "civfix-export.json", diff --git a/services/api/src/services/host/host-team-service.ts b/services/api/src/services/host/host-team-service.ts index 03e9e609..e102d8ac 100644 --- a/services/api/src/services/host/host-team-service.ts +++ b/services/api/src/services/host/host-team-service.ts @@ -200,6 +200,23 @@ function toMemberDTO( } } +export function teamInviteEmailVars(args: { + title: string + role: string + link: string +}): Record { + return { + subject: `You've been invited to help run ${args.title}`, + paragraphs: [ + `You've been invited to join ${args.title} as ${args.role}.`, + "Sign in with this email address to accept.", + ], + ctaUrl: args.link, + ctaLabel: "View the invitation", + note: "The invitation expires in 14 days. If you weren't expecting it, you can ignore this email.", + } +} + export function makeHostTeamService(deps: HostTeamServiceDeps): HostTeamService { const counters = deps.counters ?? fallbackCounters const now = deps.now ?? (() => new Date()) @@ -217,10 +234,7 @@ export function makeHostTeamService(deps: HostTeamServiceDeps): HostTeamService const base = deps.webOrigin ?? "https://civfix.org" const link = `${base}/cleanups/${cleanupId}#teamInvite=${encodeURIComponent(token)}` try { - await deps.mailer.sendTransactional(email, "generic", { - subject: `You've been invited to help run ${title}`, - message: `You've been invited to join ${title} as ${role}. Open ${link} to accept. The invitation expires in 14 days.`, - }) + await deps.mailer.sendTransactional(email, "action", teamInviteEmailVars({ title, role, link })) } catch (err) { deps.logger?.warn({ err, cleanupId }, "event team invite email failed (suppressed)") } diff --git a/services/api/src/services/host/organization-service.ts b/services/api/src/services/host/organization-service.ts index 1597fded..a70dab53 100644 --- a/services/api/src/services/host/organization-service.ts +++ b/services/api/src/services/host/organization-service.ts @@ -146,6 +146,56 @@ function verificationKindLabel(kind: OrgVerificationKind | null): string { } } +export function orgInviteEmailVars(args: { + inviterName: string + orgName: string + role: "admin" | "member" + link: string +}): Record { + const roleLabel = args.role === "admin" ? "an admin" : "a member" + return { + subject: `${args.inviterName} invited you to join ${args.orgName} on civfix`, + paragraphs: [ + `${args.inviterName} invited you to join ${args.orgName} on civfix as ${roleLabel}.`, + "Sign in with this email address to accept.", + ], + ctaUrl: args.link, + ctaLabel: "Accept the invitation", + note: "The invitation expires in 14 days. If you weren't expecting it, you can ignore this email.", + } +} + +export function orgVerificationDecisionEmailVars(args: { + orgName: string + kindLabel: string + approved: boolean + reason: string + orgUrl: string + verifyUrl: string +}): Record { + if (args.approved) { + return { + subject: `${args.orgName} is now verified as a ${args.kindLabel} on civfix`, + paragraphs: [ + `Good news: ${args.orgName} is now verified as a ${args.kindLabel} on civfix. Its profile now carries the verified badge.`, + ], + ctaUrl: args.orgUrl, + ctaLabel: "View the profile", + } + } + return { + subject: `Your verification application for ${args.orgName} was not approved`, + paragraphs: [ + `An operator reviewed the verification application for ${args.orgName} and did not approve it.`, + ], + quoteHeading: "Reason", + quote: args.reason, + ctaUrl: args.verifyUrl, + ctaLabel: "Re-apply for verification", + note: "You can address the reason and re-apply at any time from the organization's verification page.", + } +} + export interface OrganizationService { createOrganization(input: CreateOrganizationRequest, actorId: string): Promise listMyOrganizations(actorId: string): Promise @@ -543,10 +593,11 @@ export function makeOrganizationService(deps: OrganizationServiceDeps): Organiza ) } try { - await deps.mailer.sendTransactional(email, "generic", { - subject: `${inviterName} invited you to join ${org.name} on civfix`, - message: `${inviterName} invited you to join ${org.name} on civfix as ${role === "admin" ? "an admin" : "a member"}. Open ${link} to accept - sign in with this email address. The invitation expires in 14 days.`, - }) + await deps.mailer.sendTransactional( + email, + "action", + orgInviteEmailVars({ inviterName, orgName: org.name, role, link }), + ) } catch (err) { deps.logger?.warn?.({ err, organization: org.name }, "org invite email failed (suppressed)") } @@ -598,15 +649,20 @@ export function makeOrganizationService(deps: OrganizationServiceDeps): Organiza const verifyPath = `/manage/orgs/${org.id}/verification` const kindLabel = verificationKindLabel(org.verifiedKind ?? null) const approved = decision === "verified" - const subject = approved - ? `${org.name} is now verified as a ${kindLabel} on civfix` - : `Your verification application for ${org.name} was not approved` - const message = approved - ? `Good news: ${org.name} is now verified as a ${kindLabel} on civfix. Its profile shows the verified badge at ${base}${orgPath}.` - : `An operator reviewed the verification application for ${org.name} and did not approve it.\n\nReason: ${reason}\n\nYou can address the reason and re-apply at any time from the organization's verification page: ${base}${verifyPath}.` if (deps.mailer !== undefined && owner.email !== null) { try { - await deps.mailer.sendTransactional(owner.email, "generic", { subject, message }) + await deps.mailer.sendTransactional( + owner.email, + "action", + orgVerificationDecisionEmailVars({ + orgName: org.name, + kindLabel, + approved, + reason, + orgUrl: `${base}${orgPath}`, + verifyUrl: `${base}${verifyPath}`, + }), + ) } catch (err) { deps.logger?.warn?.( { err, organizationId: org.id, decision }, diff --git a/services/api/test/unit/admin-host-platform-routes.test.ts b/services/api/test/unit/admin-host-platform-routes.test.ts index f04accd4..c494fa33 100644 --- a/services/api/test/unit/admin-host-platform-routes.test.ts +++ b/services/api/test/unit/admin-host-platform-routes.test.ts @@ -375,7 +375,7 @@ describe("admin org verification decision", () => { target: `organization:${id}`, meta: { reason: "no determination letter" }, }) - expect(String(h.mails[0]?.vars.message)).toContain("no determination letter") + expect(String(h.mails[0]?.vars.quote)).toContain("no determination letter") expect(h.notes).toEqual([{ userId: HOST, type: "system" }]) }) diff --git a/services/api/test/unit/host/host-team-service.test.ts b/services/api/test/unit/host/host-team-service.test.ts index 2e707e90..800c0d76 100644 --- a/services/api/test/unit/host/host-team-service.test.ts +++ b/services/api/test/unit/host/host-team-service.test.ts @@ -201,12 +201,9 @@ describe("inviteEventTeamMember", () => { }) expect(sentMail).toHaveLength(1) const token = "token-1-aaaaaaaaaaaaaaaaaaaaaaaa" - const message = String(sentMail[0]?.vars.message) - expect(message).toContain(`https://civfix.test/cleanups/${EVENT}#teamInvite=${token}`) - expect(message).not.toContain("?teamInvite=") - const url = new URL( - message.split(/\s+/).find((word) => word.startsWith("https://")) ?? "", - ) + const link = String(sentMail[0]?.vars.ctaUrl) + expect(link).toBe(`https://civfix.test/cleanups/${EVENT}#teamInvite=${token}`) + const url = new URL(link) expect(url.hash).toBe(`#teamInvite=${token}`) expect(url.search).toBe("") expect(`${url.origin}${url.pathname}${url.search}`).not.toContain(token) diff --git a/services/api/test/unit/host/organization-service.test.ts b/services/api/test/unit/host/organization-service.test.ts index 2fb39e33..3bc8bf09 100644 --- a/services/api/test/unit/host/organization-service.test.ts +++ b/services/api/test/unit/host/organization-service.test.ts @@ -689,11 +689,11 @@ describe("verification", () => { await svc.adminDecideVerification(id, OPERATOR, { decision: "verified" }) expect(mails).toHaveLength(1) - expect(mails[0]).toMatchObject({ to: "olive@x.org", template: "generic" }) + expect(mails[0]).toMatchObject({ to: "olive@x.org", template: "action" }) expect(mails[0]?.vars.subject).toBe( "Ballona Creek Trust is now verified as a nonprofit on civfix", ) - expect(String(mails[0]?.vars.message)).toContain("https://web.test/orgs/ballona-creek-trust") + expect(mails[0]?.vars.ctaUrl).toBe("https://web.test/orgs/ballona-creek-trust") expect(notes).toHaveLength(1) expect(notes[0]).toMatchObject({ @@ -727,10 +727,9 @@ describe("verification", () => { expect(mails[0]?.vars.subject).toBe( "Your verification application for Ballona Creek Trust was not approved", ) - const message = String(mails[0]?.vars.message) - expect(message).toContain("Reason: no determination letter") - expect(message).toContain("re-apply") - expect(message).toContain(`https://web.test/manage/orgs/${id}/verification`) + expect(mails[0]?.vars.quote).toBe("no determination letter") + expect(String(mails[0]?.vars.note)).toContain("re-apply") + expect(mails[0]?.vars.ctaUrl).toBe(`https://web.test/manage/orgs/${id}/verification`) expect(notes[0]).toMatchObject({ userId: OWNER, @@ -849,10 +848,10 @@ describe("org invites (0.41.0)", () => { expect(mails[0]?.to).toBe("newcomer@example.com") expect(String(mails[0]?.vars.subject)).toContain("Olive Owner invited you to join Ballona Creek Trust") // The token rides the URL FRAGMENT, never the query string (DECISIONS §32). - expect(String(mails[0]?.vars.message)).toContain( + expect(mails[0]?.vars.ctaUrl).toBe( `https://civfix.test/manage/org-invites/accept#token=${OPERATOR_TOKEN}`, ) - expect(String(mails[0]?.vars.message)).not.toContain("?token=") + expect(JSON.stringify(mails[0]?.vars)).not.toContain("?token=") expect(repo.audits.filter((a) => a.action === "org.invite_created")).toHaveLength(1) const listed = await service.listInvites(orgId, OWNER) expect(listed.items).toHaveLength(1) From 6f6714d639c4dda19ec6e659a3505519ef29d69d Mon Sep 17 00:00:00 2001 From: Theo Date: Wed, 16 Sep 2026 22:07:03 -0700 Subject: [PATCH 41/54] add a render-everything email gallery script pnpm email:gallery renders all seventeen outbound email types with realistic sample data plus an index page, straight from the exported renderers, no DB or mailer. The Home Turf builders are exported so the gallery shows the real thing. Co-Authored-By: Claude Fable 5 --- services/api/package.json | 1 + services/api/scripts/render-email-gallery.ts | 333 +++++++++++++++++++ services/api/src/routes/forms.routes.ts | 6 +- 3 files changed, 337 insertions(+), 3 deletions(-) create mode 100644 services/api/scripts/render-email-gallery.ts diff --git a/services/api/package.json b/services/api/package.json index bedaa47d..7a868eab 100644 --- a/services/api/package.json +++ b/services/api/package.json @@ -51,6 +51,7 @@ "db:backfill:user-activity": "tsx src/db/backfill-user-activity.ts", "db:backfill:signup-seats": "tsx src/db/backfill-signup-seats.ts", "db:backfill:post-geom": "tsx src/db/backfill-post-geom.ts", + "email:gallery": "tsx scripts/render-email-gallery.ts", "db:certificate:revoke": "tsx scripts/revoke-certificate.ts", "db:prepare-boundaries": "tsx scripts/prepare-boundaries.ts", "db:boundaries:refresh": "tsx scripts/refresh-boundaries.ts", diff --git a/services/api/scripts/render-email-gallery.ts b/services/api/scripts/render-email-gallery.ts new file mode 100644 index 00000000..66dcbd66 --- /dev/null +++ b/services/api/scripts/render-email-gallery.ts @@ -0,0 +1,333 @@ +import { mkdirSync, writeFileSync } from "node:fs" +import { join, resolve } from "node:path" +import { renderOtp, renderTemplate } from "../src/adapters/mailer.oci.js" +import { renderBroadcast } from "../src/services/host/broadcast-render.js" +import { + buildDiscussionForwardPacket, + buildEventPacket, + buildReportPacket, +} from "../src/services/admin/mail-format.js" +import { buildConfirmationEmail, buildNotificationEmail } from "../src/routes/forms.routes.js" +import { buildDataExportEmail } from "../src/services/data-export-service.js" +import { + orgInviteEmailVars, + orgVerificationDecisionEmailVars, +} from "../src/services/host/organization-service.js" +import { teamInviteEmailVars } from "../src/services/host/host-team-service.js" +import type { AdminReportRecord } from "../src/services/admin/admin-report-types.js" + +interface GalleryEntry { + name: string + label: string + subject: string + html: string + text: string +} + +const WEB = "https://civfix.org" +const EVENT_TITLE = "Ballona Creek Cleanup" +const EVENT_WHEN = "Saturday, October 3 at 9:00 AM PDT" +const EVENT_WHERE = "Ballona Creek Trailhead, 13500 Jefferson Blvd, Los Angeles" +const CANCEL_URL = `${WEB}/guest?token=k3v9pT2wXane4Yr8LqZs0dHc7fGbJmQu` +const UNSUB_URL = `${WEB}/v1/broadcasts/unsubscribe?t=9fJ2mQx7Lw4TzAb1` + +function sampleReport(): AdminReportRecord { + return { + id: "11111111-2222-3333-4444-555555555555", + category: "graffiti", + status: "submitted", + flagged: false, + title: "Tagging on the Jefferson Blvd underpass", + place: "Del Rey", + reporter: { + id: "u1", + name: "Dana Reporter", + handle: "dana", + emailVerified: true, + hasOauth: false, + joinedAt: new Date("2025-01-01T00:00:00Z"), + }, + confirmations: 3, + address: "13500 Jefferson Blvd", + desc: "Fresh tags along the north wall of the underpass.\nAbout 20 feet of coverage, appeared this week.", + lat: 33.9812, + lng: -118.4185, + hasPhoto: true, + previewMedia: null, + createdAt: new Date("2026-09-01T17:20:00Z"), + referenceCode: "LA-4F7K2", + verificationVerdict: null, + verifiedAt: null, + reporterReportVerified: null, + } +} + +function collect(): GalleryEntry[] { + const entries: GalleryEntry[] = [] + const push = (name: string, label: string, r: { subject: string; html?: string; text: string }): void => { + entries.push({ name, label, subject: r.subject, html: r.html ?? `
${r.text}
`, text: r.text }) + } + + push("signin-otp", "Sign-in passcode (Mailer.sendOtp)", renderOtp("482913", "en")) + + push( + "guest-otp", + "Guest RSVP code (template guest_otp)", + renderTemplate("guest_otp", { title: EVENT_TITLE, code: "738201", minutes: "5" }), + ) + + push( + "guest-confirmed", + "Guest RSVP confirmation (template guest_confirmed)", + renderTemplate("guest_confirmed", { + title: EVENT_TITLE, + when: EVENT_WHEN, + place: EVENT_WHERE, + cancelUrl: CANCEL_URL, + }), + ) + + push( + "guest-promoted", + "Guest waitlist promotion (template guest_promoted)", + renderTemplate("guest_promoted", { + title: EVENT_TITLE, + when: EVENT_WHEN, + eventUrl: `${WEB}/cleanups/e7a1c9d2-5b34-4f68-9a21-8c3d5e7f0a1b`, + }), + ) + + push( + "org-invite", + "Organization member invite (template action)", + renderTemplate( + "action", + orgInviteEmailVars({ + inviterName: "Maria Chen", + orgName: "Ballona Creek Trust", + role: "admin", + link: `${WEB}/manage/org-invites/accept#token=q8Zw2Xv7Kp4Nr9Ty`, + }), + ), + ) + + push( + "org-verification-approved", + "Org verification approved (template action)", + renderTemplate( + "action", + orgVerificationDecisionEmailVars({ + orgName: "Ballona Creek Trust", + kindLabel: "nonprofit", + approved: true, + reason: "", + orgUrl: `${WEB}/orgs/ballona-creek-trust`, + verifyUrl: `${WEB}/manage/orgs/1/verification`, + }), + ), + ) + + push( + "org-verification-rejected", + "Org verification rejected (template action)", + renderTemplate( + "action", + orgVerificationDecisionEmailVars({ + orgName: "Ballona Creek Trust", + kindLabel: "nonprofit", + approved: false, + reason: + "The determination letter you uploaded names a different legal entity than the organization profile.", + orgUrl: `${WEB}/orgs/ballona-creek-trust`, + verifyUrl: `${WEB}/manage/orgs/1/verification`, + }), + ), + ) + + push( + "team-invite", + "Event team invite (template action)", + renderTemplate( + "action", + teamInviteEmailVars({ + title: EVENT_TITLE, + role: "co-host", + link: `${WEB}/cleanups/e7a1c9d2#teamInvite=m2Pq8Rv5Tx7Zw1Ka`, + }), + ), + ) + + push( + "generic-fallback", + "Generic fallback (template generic)", + renderTemplate("generic", { + subject: "A civfix notification", + message: "You have a new civfix notification.", + }), + ) + + const announcement = renderBroadcast( + { + subject: "This Saturday: {event_title} - what to bring", + bodyMd: + "Hi {first_name},\n\n" + + "We are all set for {event_when} at the trailhead. A few notes before the day:\n\n" + + "- We provide grabbers, gloves and bags\n" + + "- Wear closed-toe shoes and bring a refillable water bottle\n" + + "- Parking is free in the lot off Jefferson after 8:30 AM\n\n" + + "**Meet at the blue canopy** near the bike path entrance. See the full plan at [the event page](https://civfix.org/e/ballona-creek-cleanup).", + ctaLabel: "View event details", + ctaUrl: `${WEB}/e/ballona-creek-cleanup`, + }, + { + eventTitle: EVENT_TITLE, + vars: { + first_name: "Jordan", + event_title: EVENT_TITLE, + event_when: EVENT_WHEN, + event_where: EVENT_WHERE, + }, + unsubscribeUrl: UNSUB_URL, + manageUrl: `${WEB}/e/ballona-creek-cleanup`, + replyTo: "hosts@ballonacreektrust.org", + }, + ) + push("broadcast-announcement", "Host announcement (broadcast pipeline)", announcement) + + const critical = renderBroadcast( + { + subject: "{event_title} moved to 10:00 AM", + bodyMd: + "Hi {first_name},\n\nCity crews are resurfacing the parking lot early Saturday, so we are pushing the start to **10:00 AM**. Same meeting point at the blue canopy. If you can no longer make it, you can cancel from the event page.", + }, + { + eventTitle: EVENT_TITLE, + vars: { + first_name: "Jordan", + event_title: EVENT_TITLE, + event_when: EVENT_WHEN, + }, + manageUrl: `${WEB}/e/ballona-creek-cleanup`, + critical: true, + }, + ) + push("broadcast-event-updated", "Event updated notice (critical broadcast)", critical) + + const form = { + coachName: "Sam Alvarez", + role: "Head coach", + school: "Venice High School", + city: "Los Angeles", + teamSize: "18", + email: "coach.alvarez@venicehigh.edu", + phone: "(310) 555-0142", + notes: "Season runs through November; weekday afternoons work best.", + turnstileToken: "x", + honeypot: "", + } + push( + "home-turf-notification", + "Home Turf sign-up (to staff)", + buildNotificationEmail(form, "no-reply@civfix.org", "events@civfix.org"), + ) + push( + "home-turf-confirmation", + "Home Turf confirmation (to submitter)", + buildConfirmationEmail(form, "no-reply@civfix.org", "events@civfix.org"), + ) + + push( + "data-export", + "Data export ready (DSAR)", + buildDataExportEmail("support@civfix.org", ["messages", "comments"]), + ) + + push( + "admin-report-packet", + "Admin report forward (to city office)", + buildReportPacket( + sampleReport(), + { geoid: "0644000", dept: "Public Works", place: "Los Angeles", contact: null, routed: true }, + [`${WEB}/m/a?t=1`, `${WEB}/m/b?t=2`], + "Second report at this location this month; residents flagged it as recurring.", + ), + ) + + push( + "admin-discussion-forward", + "Admin discussion forward (to city office)", + buildDiscussionForwardPacket( + { reportId: "11111111-2222-3333-4444-555555555555", category: "graffiti", place: "Del Rey", org: null }, + "The tags are back again this week - is there a schedule for abatement on this wall?", + ), + ) + + push( + "admin-event-packet", + "Admin event resource request (to city office)", + buildEventPacket( + { + title: EVENT_TITLE, + host: "Ballona Creek Trust", + place: "Del Rey", + address: "13500 Jefferson Blvd, Los Angeles", + lat: 33.9812, + lng: -118.4185, + referenceCode: "LA-4F7K2", + }, + "We expect around 60 volunteers and would appreciate a dumpster drop-off on Friday plus pickup Monday.", + ), + ) + + return entries +} + +function escapeHtml(value: string): string { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) +} + +function indexHtml(entries: GalleryEntry[]): string { + const rows = entries + .map( + (e) => + `
  • ${escapeHtml(e.label)}${escapeHtml(e.subject)}
    • `, + ) + .join("\n") + return `civfix email gallery + +

    civfix email gallery (${entries.length})

    +
      +${rows} +
    +` +} + +function main(): void { + const outdirArg = process.argv[2] + if (outdirArg === undefined || outdirArg.length === 0) { + console.error("usage: pnpm exec tsx scripts/render-email-gallery.ts ") + process.exit(1) + } + const outdir = resolve(outdirArg) + mkdirSync(outdir, { recursive: true }) + const entries = collect() + for (const entry of entries) { + writeFileSync(join(outdir, `${entry.name}.html`), entry.html) + } + writeFileSync(join(outdir, "index.html"), indexHtml(entries)) + console.log(`wrote ${entries.length} emails + index.html to ${outdir}`) +} + +main() diff --git a/services/api/src/routes/forms.routes.ts b/services/api/src/routes/forms.routes.ts index 179f7218..b21a32ac 100644 --- a/services/api/src/routes/forms.routes.ts +++ b/services/api/src/routes/forms.routes.ts @@ -166,7 +166,7 @@ export async function registerHomeTurfRoutes( ) } -interface FormOutboundEmail { +export interface FormOutboundEmail { from: string to: string replyTo?: string @@ -191,7 +191,7 @@ function formRows(form: HomeTurfForm): Array<[string, string]> { return rows } -function buildNotificationEmail(form: HomeTurfForm, from: string, to: string): FormOutboundEmail { +export function buildNotificationEmail(form: HomeTurfForm, from: string, to: string): FormOutboundEmail { const subject = sanitizeHeaderValue(`Home Turf sign-up: ${form.school}`) const { text, html } = renderEmailBody({ preheader: subject, @@ -200,7 +200,7 @@ function buildNotificationEmail(form: HomeTurfForm, from: string, to: string): F return { from, to, replyTo: form.email, subject, text, html } } -function buildConfirmationEmail(form: HomeTurfForm, from: string, notifyTo: string): FormOutboundEmail { +export function buildConfirmationEmail(form: HomeTurfForm, from: string, notifyTo: string): FormOutboundEmail { const subject = "We got your Home Turf sign-up" const { text, html } = renderEmailBody({ preheader: subject, From f25986f96ad188992f39578715041d11b7f67758 Mon Sep 17 00:00:00 2001 From: Theo Date: Mon, 21 Sep 2026 23:00:04 -0700 Subject: [PATCH 42/54] push token reclaim of revoked rows + trim recency fix --- services/api/.env.example | 5 ++ .../notification-repository.drizzle.ts | 5 +- services/api/test/helpers/notifications.ts | 7 ++- .../social-notifications-pg.test.ts | 49 +++++++++++-------- .../test/unit/notification-service.test.ts | 29 +++++++++++ 5 files changed, 71 insertions(+), 24 deletions(-) diff --git a/services/api/.env.example b/services/api/.env.example index fbc8ea83..922043c2 100644 --- a/services/api/.env.example +++ b/services/api/.env.example @@ -225,6 +225,11 @@ VAPID_PUBLIC_KEY= # [OPT] VAPID public key VAPID_PRIVATE_KEY= # [OPT] VAPID private key VAPID_SUBJECT= # [OPT] VAPID subject (mailto: or https URL) +# ===== push: Expo (bypassed by USE_FAKE_PUSH) [OPT] ===== +# Only needed when the Expo project has Enhanced Push Security enabled; without it Expo rejects +# every send from this server with 401/"Unauthorized" and iOS/Android pushes silently stop. +EXPO_ACCESS_TOKEN= # [OPT] Expo access token; required only with Enhanced Push Security + # ===== error reporting: GlitchTip [OPT] ===== GLITCHTIP_DSN= # [OPT] Sentry-compatible DSN; reporting is no-op if absent GLITCHTIP_DATABASE_URL= # [OPT] separate DB URL for a self-hosted GlitchTip diff --git a/services/api/src/services/notification-repository.drizzle.ts b/services/api/src/services/notification-repository.drizzle.ts index 7b28351c..de3e925a 100644 --- a/services/api/src/services/notification-repository.drizzle.ts +++ b/services/api/src/services/notification-repository.drizzle.ts @@ -291,8 +291,9 @@ export function makeDrizzleNotificationRepository(sql: Sql): NotificationReposit ON CONFLICT (platform, token) DO UPDATE SET user_id = EXCLUDED.user_id, device_id = EXCLUDED.device_id, - revoked_at = NULL - WHERE push_tokens.user_id = EXCLUDED.user_id + revoked_at = NULL, + created_at = now() + WHERE push_tokens.user_id = EXCLUDED.user_id OR push_tokens.revoked_at IS NOT NULL RETURNING id ` if (rows.length === 0) return "conflict" diff --git a/services/api/test/helpers/notifications.ts b/services/api/test/helpers/notifications.ts index 136f862b..38685c3b 100644 --- a/services/api/test/helpers/notifications.ts +++ b/services/api/test/helpers/notifications.ts @@ -182,9 +182,14 @@ export class InMemoryNotificationRepository implements NotificationRepository { (t) => t.platform === args.platform && t.token === args.token, ) if (existing) { - if (existing.userId !== args.userId) return Promise.resolve("conflict") + if (existing.userId !== args.userId && existing.revokedAt === null) { + return Promise.resolve("conflict") + } + existing.userId = args.userId existing.deviceId = args.deviceId existing.revokedAt = null + this.pushTokens.splice(this.pushTokens.indexOf(existing), 1) + this.pushTokens.push(existing) this.capActiveTokens(args.userId) return Promise.resolve("stored") } diff --git a/services/api/test/integration/social-notifications-pg.test.ts b/services/api/test/integration/social-notifications-pg.test.ts index e77c24e2..d2730721 100644 --- a/services/api/test/integration/social-notifications-pg.test.ts +++ b/services/api/test/integration/social-notifications-pg.test.ts @@ -333,43 +333,50 @@ describe.skipIf(!pg)("social + notifications (integration)", () => { expect(cleared.quietEnd).toBeNull() }) - it("push tokens: owner re-register re-activates; foreign user CANNOT hijack; device_id does NOT authorize a cross-account rebind (F153)", async () => { + it("push tokens: owner re-register re-activates; device_id does NOT authorize a cross-account rebind of an ACTIVE row (F153); a REVOKED row can be re-claimed", async () => { const repo = makeDrizzleNotificationRepository(h.sql) const userA = await newUser("Token Owner A") const userB = await newUser("Token Owner B") + const readRow = async (): Promise<{ + user_id: string + device_id: string | null + revoked_at: Date | null + }> => { + const rows = await h.sql<{ user_id: string; device_id: string | null; revoked_at: Date | null }[]>` + SELECT user_id, device_id, revoked_at FROM push_tokens WHERE platform = 'ios' AND token = ${"tok-int"} + ` + expect(rows).toHaveLength(1) + return rows[0]! + } expect(await repo.upsertPushToken({ userId: userA, platform: "ios", token: "tok-int", deviceId: "d1" })).toBe( "stored", ) await h.sql`UPDATE push_tokens SET revoked_at = now() WHERE token = ${"tok-int"}` - expect(await repo.upsertPushToken({ userId: userB, platform: "ios", token: "tok-int", deviceId: "d2" })).toBe( - "conflict", - ) - let rows = await h.sql<{ user_id: string; device_id: string | null; revoked_at: Date | null }[]>` - SELECT user_id, device_id, revoked_at FROM push_tokens WHERE platform = 'ios' AND token = ${"tok-int"} - ` - expect(rows).toHaveLength(1) - expect(rows[0]!.user_id).toBe(userA) - expect(rows[0]!.device_id).toBe("d1") - expect(rows[0]!.revoked_at).not.toBeNull() - expect(await repo.upsertPushToken({ userId: userA, platform: "ios", token: "tok-int", deviceId: "d1" })).toBe( "stored", ) - rows = await h.sql<{ user_id: string; device_id: string | null; revoked_at: Date | null }[]>` - SELECT user_id, device_id, revoked_at FROM push_tokens WHERE platform = 'ios' AND token = ${"tok-int"} - ` - expect(rows[0]!.user_id).toBe(userA) - expect(rows[0]!.revoked_at).toBeNull() + let row = await readRow() + expect(row.user_id).toBe(userA) + expect(row.device_id).toBe("d1") + expect(row.revoked_at).toBeNull() expect(await repo.upsertPushToken({ userId: userB, platform: "ios", token: "tok-int", deviceId: "d1" })).toBe( "conflict", ) - rows = await h.sql<{ user_id: string; device_id: string | null; revoked_at: Date | null }[]>` - SELECT user_id, device_id, revoked_at FROM push_tokens WHERE platform = 'ios' AND token = ${"tok-int"} - ` - expect(rows[0]!.user_id).toBe(userA) + row = await readRow() + expect(row.user_id).toBe(userA) + expect(row.revoked_at).toBeNull() + + await h.sql`UPDATE push_tokens SET revoked_at = now() WHERE token = ${"tok-int"}` + expect(await repo.upsertPushToken({ userId: userB, platform: "ios", token: "tok-int", deviceId: "d2" })).toBe( + "stored", + ) + row = await readRow() + expect(row.user_id).toBe(userB) + expect(row.device_id).toBe("d2") + expect(row.revoked_at).toBeNull() }) it("new_follower hook end to end: a NEW follow records a notification for the followee", async () => { diff --git a/services/api/test/unit/notification-service.test.ts b/services/api/test/unit/notification-service.test.ts index 73036b91..83670d69 100644 --- a/services/api/test/unit/notification-service.test.ts +++ b/services/api/test/unit/notification-service.test.ts @@ -353,6 +353,20 @@ describe("registerPushToken", () => { expect(repo.pushTokens[0]).toMatchObject({ userId: U, deviceId: "shared-device", revokedAt: null }) }) + it("a DIFFERENT user CAN claim a (platform,token) row that is already revoked (device handoff)", async () => { + const { repo, push, service } = makeHarness() + await service.registerPushToken(U, { platform: "android", token: TOK_X, deviceId: "d1" }) + await service.unregisterPushToken(U, { platform: "android", token: TOK_X }) + expect(repo.pushTokens[0]?.revokedAt).not.toBeNull() + + await expect( + service.registerPushToken(V, { platform: "android", token: TOK_X, deviceId: "d2" }), + ).resolves.toEqual({ ok: true }) + expect(repo.pushTokens).toHaveLength(1) + expect(repo.pushTokens[0]).toMatchObject({ userId: V, deviceId: "d2", revokedAt: null }) + expect(push.tokens.some((t) => t.userId === V && t.token === TOK_X)).toBe(true) + }) + it("H12: a self-declared device_id does NOT revoke another account's token (mass-revoke closed)", async () => { const { repo, service } = makeHarness() await service.registerPushToken(U, { platform: "ios", token: TOK_U, deviceId: "shared" }) @@ -631,6 +645,21 @@ describe("push token cap per user (F087)", () => { expect(repo.pushTokens.find((t) => t.token === capToken(0))?.revokedAt).not.toBeNull() expect(repo.pushTokens.find((t) => t.token === capToken(total - 1))?.revokedAt).toBeNull() }) + + it("re-registering an existing token with a full cap of newer tokens leaves it ACTIVE", async () => { + const { repo, service } = makeHarness() + await service.registerPushToken(U, { platform: "ios", token: TOK_1 }) + for (let i = 0; i < MAX_ACTIVE_PUSH_TOKENS_PER_USER; i++) { + await service.registerPushToken(U, { platform: "ios", token: capToken(i) }) + } + expect(repo.pushTokens.find((t) => t.token === TOK_1)?.revokedAt).not.toBeNull() + + await service.registerPushToken(U, { platform: "ios", token: TOK_1 }) + expect(repo.pushTokens.find((t) => t.token === TOK_1)?.revokedAt).toBeNull() + expect(repo.pushTokens.filter((t) => t.userId === U && t.revokedAt === null)).toHaveLength( + MAX_ACTIVE_PUSH_TOKENS_PER_USER, + ) + }) }) describe("toggle-bell de-duplication (F015)", () => { From 82e91ceb7d3a4a8f2042f83c125cf59440ada72d Mon Sep 17 00:00:00 2001 From: Theo Date: Mon, 21 Sep 2026 23:06:05 -0700 Subject: [PATCH 43/54] adopt @civfix/shared 0.54.0 --- pnpm-lock.yaml | 14 +++++++------- services/api/package.json | 2 +- .../src/services/admin/forward-template-service.ts | 1 + services/api/test/unit/feed-ranking.test.ts | 6 +++--- services/api/test/unit/route-coverage.test.ts | 4 ++-- services/media-worker/package.json | 2 +- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 60ea3f27..2eaed5fc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -65,8 +65,8 @@ importers: specifier: ^3.700.0 version: 3.1057.0 '@civfix/shared': - specifier: ^0.53.0 - version: 0.53.0 + specifier: ^0.54.0 + version: 0.54.0 '@fastify/cookie': specifier: ^11.0.1 version: 11.0.2 @@ -210,8 +210,8 @@ importers: specifier: workspace:* version: link:../api '@civfix/shared': - specifier: ^0.53.0 - version: 0.53.0 + specifier: ^0.54.0 + version: 0.54.0 '@sentry/node': specifier: ^10.73.0 version: 10.73.0(@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1)) @@ -414,8 +414,8 @@ packages: '@balena/dockerignore@1.0.2': resolution: {integrity: sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==} - '@civfix/shared@0.53.0': - resolution: {integrity: sha512-TvY+VHFkg3G7GNNNjhk9kwSrvXVsEDq9NAUbNohmN2X29JAC2uLaUS3FfIJIeew6Qc710+Zr7Ik2+C7Cbs0YkQ==} + '@civfix/shared@0.54.0': + resolution: {integrity: sha512-H9vFZ+t9w/4UidgUYBNApeuqE2q9vuzXD0BmhZmeWOHJ2SeKjL7iPZOBL3Zpj8NGn+tq9ClcueYDUq4IXQcavw==} engines: {node: '>=20'} '@derhuerst/http-basic@8.2.4': @@ -4440,7 +4440,7 @@ snapshots: '@balena/dockerignore@1.0.2': {} - '@civfix/shared@0.53.0': + '@civfix/shared@0.54.0': dependencies: zod: 3.25.76 diff --git a/services/api/package.json b/services/api/package.json index 22e2c533..be885540 100644 --- a/services/api/package.json +++ b/services/api/package.json @@ -60,7 +60,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.700.0", "@aws-sdk/s3-request-presigner": "^3.700.0", - "@civfix/shared": "^0.53.0", + "@civfix/shared": "^0.54.0", "@fastify/cookie": "^11.0.1", "@fastify/cors": "^10.0.1", "@fastify/helmet": "^12.0.1", diff --git a/services/api/src/services/admin/forward-template-service.ts b/services/api/src/services/admin/forward-template-service.ts index 59192718..ee5d5068 100644 --- a/services/api/src/services/admin/forward-template-service.ts +++ b/services/api/src/services/admin/forward-template-service.ts @@ -82,6 +82,7 @@ function sampleReport(): AdminReportRecord { lat: Number(sampleValue("lat")), lng: Number(sampleValue("lng")), hasPhoto: true, + previewMedia: null, createdAt: SAMPLE_CREATED_AT, referenceCode: sampleValue("referenceCode"), verificationVerdict: null, diff --git a/services/api/test/unit/feed-ranking.test.ts b/services/api/test/unit/feed-ranking.test.ts index 76de8162..08160ba2 100644 --- a/services/api/test/unit/feed-ranking.test.ts +++ b/services/api/test/unit/feed-ranking.test.ts @@ -229,7 +229,7 @@ describe("feed ranking: the worked examples from the design", () => { expect(score).toBeCloseTo(115.9, 1) }) - it("stranger 5 km away, verified org, report, image, 12 likes, 3 d old scores ~45.8", () => { + it("stranger 5 km away, verified org, report, image, 12 likes, 3 d old scores ~125.1", () => { const score = scoreCandidate( candidate({ createdAtMs: at(72), @@ -242,7 +242,7 @@ describe("feed ranking: the worked examples from the design", () => { CFG, NOW, ) - expect(score).toBeCloseTo(45.8, 1) + expect(score).toBeCloseTo(125.1, 1) }) it("stranger, no location, 2 likes, 1 h old scores ~13.1 and clears the cutoff", () => { @@ -354,7 +354,7 @@ describe("FeedRankingConfigSchema", () => { it("fills every knob from an empty object", () => { const parsed = FeedRankingConfigSchema.parse({}) expect(parsed).toEqual(DEFAULT_FEED_RANKING) - expect(Object.keys(parsed)).toHaveLength(27) + expect(Object.keys(parsed)).toHaveLength(28) }) it("merges a partial override onto the defaults", () => { diff --git a/services/api/test/unit/route-coverage.test.ts b/services/api/test/unit/route-coverage.test.ts index d812b26f..a232c54a 100644 --- a/services/api/test/unit/route-coverage.test.ts +++ b/services/api/test/unit/route-coverage.test.ts @@ -233,8 +233,8 @@ describe("route-coverage: every shared endpoint is registered (offline boot smok }) } - it("covers ALL 324 endpoints in the registry (no endpoint skipped)", () => { - expect(Object.keys(endpoints).length).toBe(324) + it("covers ALL 330 endpoints in the registry (no endpoint skipped)", () => { + expect(Object.keys(endpoints).length).toBe(330) }) it("the discriminator is not vacuous: a bogus path IS detected as route-missing", async () => { diff --git a/services/media-worker/package.json b/services/media-worker/package.json index 6aa72f37..095c71fe 100644 --- a/services/media-worker/package.json +++ b/services/media-worker/package.json @@ -23,7 +23,7 @@ "@aws-sdk/client-s3": "^3.700.0", "@aws-sdk/s3-request-presigner": "^3.700.0", "@civfix/api": "workspace:*", - "@civfix/shared": "^0.53.0", + "@civfix/shared": "^0.54.0", "@sentry/node": "^10.73.0", "@smithy/node-http-handler": "4.7.5", "drizzle-orm": "^0.45.2", From 6f2eabfbd05b485b7afc9682370989dc43bbe897 Mon Sep 17 00:00:00 2001 From: Theo Date: Mon, 21 Sep 2026 23:13:42 -0700 Subject: [PATCH 44/54] feed scoring: global+viewer split, location-first, seeded jitter --- services/api/src/services/feed-ranking.ts | 76 +++++- services/api/src/services/post-service.ts | 25 +- .../api/test/unit/feed-ranked-service.test.ts | 191 ++++++++++++++- .../api/test/unit/feed-ranking-env.test.ts | 17 ++ services/api/test/unit/feed-ranking.test.ts | 231 ++++++++++++++++-- 5 files changed, 504 insertions(+), 36 deletions(-) diff --git a/services/api/src/services/feed-ranking.ts b/services/api/src/services/feed-ranking.ts index 7e9f5f56..57bfbc68 100644 --- a/services/api/src/services/feed-ranking.ts +++ b/services/api/src/services/feed-ranking.ts @@ -52,31 +52,92 @@ function logScale(count: number): number { return Math.log1p(Math.max(0, count)) } -export function rawScore(candidate: FeedCandidate, cfg: FeedRankingConfig): number { +export type GlobalCandidate = Omit< + FeedCandidate, + "authorFollowed" | "authorIsViewer" | "viewerMentioned" | "distanceKm" +> + +export type ViewerCandidate = Pick< + FeedCandidate, + "authorFollowed" | "authorIsViewer" | "viewerMentioned" | "distanceKm" +> + +export function globalScore(candidate: GlobalCandidate, cfg: FeedRankingConfig): number { return ( cfg.baseWeight + - (candidate.authorFollowed ? cfg.followWeight : 0) + - (candidate.authorIsViewer ? cfg.selfWeight : 0) + - (candidate.viewerMentioned ? cfg.mentionWeight : 0) + (candidate.authorOrgVerified ? cfg.orgVerifiedWeight : 0) + (candidate.hasLiveEvent ? cfg.attachEventWeight : 0) + (candidate.hasReport ? cfg.attachReportWeight : 0) + (candidate.hasMedia ? cfg.imageWeight : 0) + - cfg.nearbyWeight * proximity(candidate.distanceKm, cfg.nearbyRadiusKm) + cfg.likeWeight * logScale(candidate.likeCount) + cfg.replyWeight * logScale(candidate.replyCount) + cfg.repostWeight * logScale(candidate.repostCount) ) } +export function viewerScore(candidate: ViewerCandidate, cfg: FeedRankingConfig): number { + return ( + (candidate.authorFollowed ? cfg.followWeight : 0) + + (candidate.authorIsViewer ? cfg.selfWeight : 0) + + (candidate.viewerMentioned ? cfg.mentionWeight : 0) + + cfg.nearbyWeight * proximity(candidate.distanceKm, cfg.nearbyRadiusKm) + ) +} + +export function rawScore(candidate: FeedCandidate, cfg: FeedRankingConfig): number { + return globalScore(candidate, cfg) + viewerScore(candidate, cfg) +} + +const FNV_OFFSET_BASIS = 2_166_136_261 +const FNV_PRIME = 16_777_619 +const UINT32_SPAN = 4_294_967_296 + +function fnv1a32(text: string): number { + let hash = FNV_OFFSET_BASIS + for (let i = 0; i < text.length; i += 1) { + hash ^= text.charCodeAt(i) + hash = Math.imul(hash, FNV_PRIME) + } + return hash >>> 0 +} + +function avalanche(value: number): number { + let hash = value >>> 0 + hash ^= hash >>> 16 + hash = Math.imul(hash, 2_246_822_507) + hash ^= hash >>> 13 + hash = Math.imul(hash, 3_266_489_909) + hash ^= hash >>> 16 + return hash >>> 0 +} + +export function jitterUnit(seed: number, postId: string): number { + return avalanche(fnv1a32(`${seed}:${postId}`)) / UINT32_SPAN +} + +export function jitterMultiplier(cfg: FeedRankingConfig, seed: number, postId: string): number { + if (cfg.jitterAmount === 0) return 1 + return 1 + cfg.jitterAmount * (2 * jitterUnit(seed, postId) - 1) +} + +export function bucketSeed(viewerKey: string, filter: string, nowBucketMs: number): number { + return avalanche(fnv1a32(`${viewerKey}|${filter}|${nowBucketMs}`)) +} + export function scoreCandidate( candidate: FeedCandidate, cfg: FeedRankingConfig, nowBucketMs: number, + seed: number, ): number { const ageHours = Math.max(0, (nowBucketMs - candidate.createdAtMs) / MS_PER_HOUR) const seen = candidate.alreadySeen ? cfg.seenDiscount : 1 - return Math.max(0, rawScore(candidate, cfg)) * recency(ageHours, cfg) * seen + return ( + Math.max(0, rawScore(candidate, cfg)) * + recency(ageHours, cfg) * + seen * + jitterMultiplier(cfg, seed, candidate.id) + ) } function byScoreThenId(a: RankedCandidate, b: RankedCandidate): number { @@ -88,12 +149,13 @@ export function rankCandidates( candidates: readonly FeedCandidate[], cfg: FeedRankingConfig, nowMs: number, + seed: number, ): RankedCandidate[] { const nowBucketMs = quantizeClock(nowMs, cfg.clockBucketSeconds) const base = candidates.map((candidate) => ({ id: candidate.id, authorId: candidate.authorId, - score: quantizeFeedScore(scoreCandidate(candidate, cfg, nowBucketMs)), + score: quantizeFeedScore(scoreCandidate(candidate, cfg, nowBucketMs, seed)), })) base.sort(byScoreThenId) diff --git a/services/api/src/services/post-service.ts b/services/api/src/services/post-service.ts index 99626039..29a8f944 100644 --- a/services/api/src/services/post-service.ts +++ b/services/api/src/services/post-service.ts @@ -15,6 +15,7 @@ import type { PostDTO, } from "@civfix/shared" import type { UserChannel } from "@civfix/shared/interfaces" +import { randomInt } from "node:crypto" import type { Sql } from "../db/client.js" import { assertNoSlur } from "../abuse/slur-filter.js" import { resolveMentionTargets } from "./mention-resolver.drizzle.js" @@ -30,6 +31,8 @@ import { } from "./post-repository.drizzle.js" import { applyCutoff, + bucketSeed, + quantizeClock, rankCandidates, type FeedCandidate, type RankedCandidate, @@ -52,6 +55,10 @@ export const FEED_FANOUT_CONCURRENCY = 16 export const FEED_DISTANCE_RESOLUTION_KM = 1 +export const FEED_SEED_SPAN = 2 ** 31 + +export const PUBLIC_FEED_SEED_KEY = "public" + function coarseDistanceKm(distanceKm: number | null): number | null { if (distanceKm === null || !Number.isFinite(distanceKm)) return null return Math.round(distanceKm / FEED_DISTANCE_RESOLUTION_KM) * FEED_DISTANCE_RESOLUTION_KM @@ -72,6 +79,7 @@ export interface PostServiceDeps { feedPresence?: FeedPresence userChannel?: UserChannel now?: () => number + feedSeed?: () => number } export interface PostService { @@ -100,6 +108,7 @@ export function makePostService(deps: PostServiceDeps): PostService { const isBlocked = deps.isBlockedEitherWay ?? (() => Promise.resolve(false)) const feedConfig = deps.feedRanking ?? DEFAULT_FEED_RANKING const nowMs = deps.now ?? (() => Date.now()) + const mintSeed = deps.feedSeed ?? (() => randomInt(0, FEED_SEED_SPAN)) function fanout(recipients: readonly string[], topic: "feed" | "feed_counts", id: string): void { const channel = deps.userChannel @@ -205,6 +214,18 @@ export function makePostService(deps: PostServiceDeps): PostService { return { items, nextCursor } } + function seedFor( + viewerId: string, + filter: string, + isFirstPage: boolean, + nowBucketMs: number, + ): number { + const presence = presenceFor(viewerId) + if (isFirstPage && presence !== undefined && presence.snapshotsAvailable) return mintSeed() + const key = viewerId === NIL_VIEWER_ID ? PUBLIC_FEED_SEED_KEY : viewerId + return bucketSeed(key, filter, nowBucketMs) + } + async function rankCandidateSet( viewerId: string, query: HomeFeedQuery, @@ -230,10 +251,12 @@ export function makePostService(deps: PostServiceDeps): PostService { rows.map((r) => r.id), ) + const nowBucketMs = quantizeClock(nowMs(), feedConfig.clockBucketSeconds) const ranked = rankCandidates( rows.map((row) => toCandidate(row, seen)), feedConfig, - nowMs(), + nowBucketMs, + seedFor(viewerId, query.filter, isFirstPage, nowBucketMs), ) return { page: applyCutoff(ranked, feedConfig, true), diff --git a/services/api/test/unit/feed-ranked-service.test.ts b/services/api/test/unit/feed-ranked-service.test.ts index 8fc1f99a..a1eefa52 100644 --- a/services/api/test/unit/feed-ranked-service.test.ts +++ b/services/api/test/unit/feed-ranked-service.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import { DEFAULT_FEED_RANKING, parseFeedScoreCursor } from "@civfix/shared" -import type { PostDTO, UserSignal } from "@civfix/shared" +import type { FeedRankingConfig, PostDTO, UserSignal } from "@civfix/shared" import type { UserChannel } from "@civfix/shared/interfaces" import type { Sql } from "../../src/db/client.js" import { makePostService, type PostService } from "../../src/services/post-service.js" @@ -18,6 +18,8 @@ const AUTHOR = "22222222-2222-2222-2222-222222222222" const POST = "33333333-3333-3333-3333-333333333333" const NOW = Date.UTC(2026, 8, 15, 12, 0, 0) +const NO_JITTER: FeedRankingConfig = { ...DEFAULT_FEED_RANKING, jitterAmount: 0 } + const throwingSql = (() => { throw new Error("sql must not be called in these unit paths") }) as unknown as Sql @@ -210,9 +212,9 @@ describe("ranked feed: cursor continuation covers every item exactly once", () = describe("ranked feed: warm snapshot continues a cursor without re-ranking", () => { const rows = Array.from({ length: 60 }, (_, i) => candidateRow(i + 1)) - function harness() { + function harness(config: FeedRankingConfig = DEFAULT_FEED_RANKING) { const cache = new InMemoryCacheClient(() => Date.now()) - const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + const presence = makeFeedPresence({ cache, config }) let candidateQueries = 0 const svc = makePostService({ repo: repoOver({ @@ -223,6 +225,7 @@ describe("ranked feed: warm snapshot continues a cursor without re-ranking", () }), sql: throwingSql, feedPresence: presence, + feedRanking: config, now: () => NOW, }) return { svc, presence, candidateQueries: () => candidateQueries } @@ -246,7 +249,7 @@ describe("ranked feed: warm snapshot continues a cursor without re-ranking", () }) it("re-ranks rather than ending the feed when the snapshot has expired under a dwelling reader", async () => { - const { svc } = harness() + const { svc } = harness(NO_JITTER) const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) const expired = makePostService({ @@ -254,8 +257,9 @@ describe("ranked feed: warm snapshot continues a cursor without re-ranking", () sql: throwingSql, feedPresence: makeFeedPresence({ cache: new InMemoryCacheClient(() => Date.now()), - config: DEFAULT_FEED_RANKING, + config: NO_JITTER, }), + feedRanking: NO_JITTER, now: () => NOW, }) const page = await expired.homeFeed(VIEWER, { @@ -271,11 +275,11 @@ describe("ranked feed: warm snapshot continues a cursor without re-ranking", () }) it("rebuilds the snapshot on the re-ranked continuation so the next page is served from cache", async () => { - const { svc } = harness() + const { svc } = harness(NO_JITTER) const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) const cache = new InMemoryCacheClient(() => Date.now()) - const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + const presence = makeFeedPresence({ cache, config: NO_JITTER }) let candidateQueries = 0 const expired = makePostService({ repo: repoOver({ @@ -286,6 +290,7 @@ describe("ranked feed: warm snapshot continues a cursor without re-ranking", () }), sql: throwingSql, feedPresence: presence, + feedRanking: NO_JITTER, now: () => NOW, }) @@ -306,7 +311,7 @@ describe("ranked feed: warm snapshot continues a cursor without re-ranking", () }) it("re-ranking a moved clock never re-serves a page-1 item", async () => { - const { svc } = harness() + const { svc } = harness(NO_JITTER) const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) const later = makePostService({ @@ -314,8 +319,9 @@ describe("ranked feed: warm snapshot continues a cursor without re-ranking", () sql: throwingSql, feedPresence: makeFeedPresence({ cache: new InMemoryCacheClient(() => Date.now()), - config: DEFAULT_FEED_RANKING, + config: NO_JITTER, }), + feedRanking: NO_JITTER, now: () => NOW + 10 * 60_000, }) const page = await later.homeFeed(VIEWER, { @@ -414,11 +420,15 @@ describe("ranked feed: no Redis is a slower feed, not a one-page feed", () => { describe("ranked feed: a signed-out reader keeps scrolling", () => { const rows = Array.from({ length: 60 }, (_, i) => candidateRow(i + 1, { author_followed: false })) - function guestService(presence?: ReturnType): PostService { + function guestService( + presence?: ReturnType, + config: FeedRankingConfig = DEFAULT_FEED_RANKING, + ): PostService { return makePostService({ repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), sql: throwingSql, ...(presence === undefined ? {} : { feedPresence: presence }), + feedRanking: config, now: () => NOW, }) } @@ -448,10 +458,11 @@ describe("ranked feed: a signed-out reader keeps scrolling", () => { }) it("keeps a guest continuation stable across a clock bucket boundary", async () => { - const first = await guestService().publicFeed({ filter: "all", limit: 20 }) + const first = await guestService(undefined, NO_JITTER).publicFeed({ filter: "all", limit: 20 }) const later = makePostService({ repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), sql: throwingSql, + feedRanking: NO_JITTER, now: () => NOW + 5 * 60_000, }) const second = await later.publicFeed({ filter: "all", limit: 20, cursor: first.nextCursor! }) @@ -461,6 +472,26 @@ describe("ranked feed: a signed-out reader keeps scrolling", () => { expect(second.items.some((item) => firstIds.has(item.id))).toBe(false) }) + it("keys the guest shuffle to the shared public bucket, not to a viewer id", async () => { + const first = await guestService().publicFeed({ filter: "all", limit: 20 }) + const again = await guestService().publicFeed({ filter: "all", limit: 20 }) + expect(again.items.map((item) => item.id)).toEqual(first.items.map((item) => item.id)) + + const seen: string[] = [] + let cursor: string | undefined = first.nextCursor ?? undefined + seen.push(...first.items.map((item) => item.id)) + while (cursor !== undefined) { + const page: { items: PostDTO[]; nextCursor: string | null } = await guestService().publicFeed({ + filter: "all", + limit: 20, + cursor, + }) + seen.push(...page.items.map((item) => item.id)) + cursor = page.nextCursor ?? undefined + } + expect(new Set(seen).size).toBe(seen.length) + }) + it("never writes a guest into the shared presence keys", async () => { const cache = new InMemoryCacheClient(() => Date.now()) const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) @@ -555,6 +586,144 @@ describe("ranked feed: served set feeds the seen discount", () => { }) }) +describe("ranked feed: a refresh reshuffles the near-ties, a cursor page never does", () => { + const rows = Array.from({ length: 60 }, (_, i) => + candidateRow(i + 1, { like_count: 0, created_at: new Date(NOW), author_id: `author-${i}` }), + ) + + function seededService(seed: number) { + const cache = new InMemoryCacheClient(() => Date.now()) + const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) + const svc = makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: presence, + now: () => NOW, + feedSeed: () => seed, + }) + return { svc, presence } + } + + it("hands two refreshes with different seeds two different orders", async () => { + const a = await seededService(11).svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + const b = await seededService(22).svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + + expect(a.items.map((item) => item.id)).not.toEqual(b.items.map((item) => item.id)) + expect(a.items).toHaveLength(20) + expect(b.items).toHaveLength(20) + }) + + it("reproduces the same first page for the same minted seed", async () => { + const a = await seededService(11).svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + const b = await seededService(11).svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + expect(a.items.map((item) => item.id)).toEqual(b.items.map((item) => item.id)) + }) + + it("mints the seed once per first-page request, never per candidate", async () => { + let mints = 0 + const cache = new InMemoryCacheClient(() => Date.now()) + const svc = makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }), + now: () => NOW, + feedSeed: () => { + mints += 1 + return 7 + }, + }) + + const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + expect(mints).toBe(1) + + await svc.homeFeed(VIEWER, { filter: "all", limit: 20, cursor: first.nextCursor! }) + expect(mints).toBe(1) + }) + + it("pages a jittered snapshot end to end with no duplicates and no holes", async () => { + const { svc } = seededService(99) + const seen: string[] = [] + let cursor: string | undefined + + for (let page = 0; page < 6; page += 1) { + const result = await svc.homeFeed(VIEWER, { + filter: "all", + limit: 20, + ...(cursor === undefined ? {} : { cursor }), + }) + seen.push(...result.items.map((item) => item.id)) + if (result.nextCursor === null) break + cursor = result.nextCursor + } + + expect(seen).toHaveLength(60) + expect(new Set(seen).size).toBe(60) + }) + + it("falls back to a deterministic seed when no snapshot can carry the minted one", async () => { + let mints = 0 + function noSnapshotService() { + return makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: makeFeedPresence({ config: DEFAULT_FEED_RANKING }), + now: () => NOW, + feedSeed: () => { + mints += 1 + return mints + }, + }) + } + + const first = await noSnapshotService().homeFeed(VIEWER, { filter: "all", limit: 20 }) + const again = await noSnapshotService().homeFeed(VIEWER, { filter: "all", limit: 20 }) + + expect(mints).toBe(0) + expect(again.items.map((item) => item.id)).toEqual(first.items.map((item) => item.id)) + }) + + it("agrees between two fallback recomputations inside the same clock bucket", async () => { + function fallbackService(atMs: number) { + return makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: makeFeedPresence({ config: DEFAULT_FEED_RANKING }), + now: () => atMs, + }) + } + + const first = await fallbackService(NOW + 1_000).homeFeed(VIEWER, { filter: "all", limit: 20 }) + const second = await fallbackService(NOW + 55_000).homeFeed(VIEWER, { + filter: "all", + limit: 20, + cursor: first.nextCursor!, + }) + const third = await fallbackService(NOW + 30_000).homeFeed(VIEWER, { + filter: "all", + limit: 20, + cursor: first.nextCursor!, + }) + + expect(second.items.map((item) => item.id)).toEqual(third.items.map((item) => item.id)) + const firstIds = new Set(first.items.map((item) => item.id)) + expect(second.items.some((item) => firstIds.has(item.id))).toBe(false) + }) + + it("gives two viewers different shuffles of the same candidate set", async () => { + const other = "99999999-9999-9999-9999-999999999999" + const svc = makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: makeFeedPresence({ config: DEFAULT_FEED_RANKING }), + now: () => NOW, + }) + + const mine = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + const theirs = await svc.homeFeed(other, { filter: "all", limit: 20 }) + expect(mine.items.map((item) => item.id)).not.toEqual(theirs.items.map((item) => item.id)) + }) +}) + describe("ranked feed: realtime fanout never blocks the request path", () => { it("publishes topic feed to the author's followers on a new top-level post", async () => { const { channel, sent } = recordingChannel() diff --git a/services/api/test/unit/feed-ranking-env.test.ts b/services/api/test/unit/feed-ranking-env.test.ts index 76b5cb92..f1da515f 100644 --- a/services/api/test/unit/feed-ranking-env.test.ts +++ b/services/api/test/unit/feed-ranking-env.test.ts @@ -43,6 +43,23 @@ describe("FEED_RANKING env var", () => { expect(() => load('{"decayFloor":1.5}')).toThrow(/FEED_RANKING\.decayFloor/) }) + it("accepts a jitterAmount override, including switching jitter off entirely", () => { + expect(load('{"jitterAmount":0.4}').FEED_RANKING.jitterAmount).toBe(0.4) + expect(load('{"jitterAmount":0}').FEED_RANKING.jitterAmount).toBe(0) + expect(load('{"jitterAmount":1}').FEED_RANKING.jitterAmount).toBe(1) + }) + + it("keeps the rest of the profile intact when only jitterAmount is overridden", () => { + const env = load('{"jitterAmount":0.05}') + expect(env.FEED_RANKING).toEqual({ ...DEFAULT_FEED_RANKING, jitterAmount: 0.05 }) + }) + + it("refuses an out-of-range jitterAmount rather than clamping it", () => { + expect(() => load('{"jitterAmount":1.5}')).toThrow(/FEED_RANKING\.jitterAmount/) + expect(() => load('{"jitterAmount":-0.1}')).toThrow(/FEED_RANKING\.jitterAmount/) + expect(() => load('{"jitterAmount":"0.2"}')).toThrow(/FEED_RANKING\.jitterAmount/) + }) + it("refuses a JSON scalar where an object is required", () => { expect(() => load("42")).toThrow(/FEED_RANKING/) expect(() => load('"forty-two"')).toThrow(/FEED_RANKING/) diff --git a/services/api/test/unit/feed-ranking.test.ts b/services/api/test/unit/feed-ranking.test.ts index 08160ba2..f9cb835a 100644 --- a/services/api/test/unit/feed-ranking.test.ts +++ b/services/api/test/unit/feed-ranking.test.ts @@ -3,20 +3,28 @@ import { DEFAULT_FEED_RANKING, FeedRankingConfigSchema } from "@civfix/shared" import type { FeedRankingConfig } from "@civfix/shared" import { applyCutoff, + bucketSeed, diversityMultiplier, + globalScore, + jitterUnit, proximity, quantizeClock, rankCandidates, rawScore, recency, scoreCandidate, + viewerScore, type FeedCandidate, } from "../../src/services/feed-ranking.js" const NOW = Date.UTC(2026, 8, 15, 12, 0, 0) const HOUR = 3_600_000 -const CFG = DEFAULT_FEED_RANKING +const SEED = 0 + +const CFG: FeedRankingConfig = { ...DEFAULT_FEED_RANKING, jitterAmount: 0 } + +const JITTERED = DEFAULT_FEED_RANKING function candidate(over: Partial = {}): FeedCandidate { return { @@ -164,11 +172,11 @@ describe("feed ranking: author diversity", () => { const other = candidate({ id: "b0", authorId: "someone-else", authorFollowed: true }) const undiscounted = [...flood, other] - .map((c) => ({ id: c.id, score: scoreCandidate(c, CFG, NOW) })) + .map((c) => ({ id: c.id, score: scoreCandidate(c, CFG, NOW, SEED) })) .sort((a, b) => b.score - a.score) expect(undiscounted.findIndex((entry) => entry.id === "b0")).toBe(flood.length) - const ranked = rankCandidates([...flood, other], CFG, NOW) + const ranked = rankCandidates([...flood, other], CFG, NOW, SEED) expect(ranked.findIndex((entry) => entry.id === "b0")).toBeLessThan(flood.length) }) @@ -181,6 +189,7 @@ describe("feed ranking: author diversity", () => { ], CFG, NOW, + SEED, ) const scores = new Set(ranked.map((entry) => entry.score)) expect(scores.size).toBe(1) @@ -189,8 +198,8 @@ describe("feed ranking: author diversity", () => { describe("feed ranking: seen discount", () => { it("demotes a post the viewer was already served", () => { - const fresh = scoreCandidate(candidate(), CFG, NOW) - const seen = scoreCandidate(candidate({ alreadySeen: true }), CFG, NOW) + const fresh = scoreCandidate(candidate(), CFG, NOW, SEED) + const seen = scoreCandidate(candidate({ alreadySeen: true }), CFG, NOW, SEED) expect(seen).toBeCloseTo(fresh * CFG.seenDiscount, 10) }) }) @@ -204,15 +213,15 @@ describe("feed ranking: clock quantisation keeps a cursor stable", () => { it("produces byte-identical scores 59 seconds apart", () => { const rows = [candidate({ id: "a", createdAtMs: NOW - 5 * HOUR, likeCount: 7 })] - const first = rankCandidates(rows, CFG, NOW + 1_000) - const second = rankCandidates(rows, CFG, NOW + 59_000) + const first = rankCandidates(rows, CFG, NOW + 1_000, SEED) + const second = rankCandidates(rows, CFG, NOW + 59_000, SEED) expect(second).toEqual(first) }) it("does move the score once the bucket rolls over", () => { const rows = [candidate({ id: "a", createdAtMs: NOW - 5 * HOUR })] - const inBucket = rankCandidates(rows, CFG, NOW) - const nextBucket = rankCandidates(rows, CFG, NOW + 61_000) + const inBucket = rankCandidates(rows, CFG, NOW, SEED) + const nextBucket = rankCandidates(rows, CFG, NOW + 61_000, SEED) expect(nextBucket[0]!.score).toBeLessThan(inBucket[0]!.score) }) }) @@ -225,6 +234,7 @@ describe("feed ranking: the worked examples from the design", () => { candidate({ authorFollowed: true, createdAtMs: at(2), likeCount: 3, replyCount: 1 }), CFG, NOW, + SEED, ) expect(score).toBeCloseTo(115.9, 1) }) @@ -241,12 +251,13 @@ describe("feed ranking: the worked examples from the design", () => { }), CFG, NOW, + SEED, ) expect(score).toBeCloseTo(125.1, 1) }) it("stranger, no location, 2 likes, 1 h old scores ~13.1 and clears the cutoff", () => { - const score = scoreCandidate(candidate({ createdAtMs: at(1), likeCount: 2 }), CFG, NOW) + const score = scoreCandidate(candidate({ createdAtMs: at(1), likeCount: 2 }), CFG, NOW, SEED) expect(score).toBeCloseTo(13.1, 1) expect(score).toBeGreaterThanOrEqual(CFG.minScore) }) @@ -256,13 +267,14 @@ describe("feed ranking: the worked examples from the design", () => { candidate({ createdAtMs: at(24 * 30), likeCount: 2 }), CFG, NOW, + SEED, ) expect(score).toBeCloseTo(2.0, 1) expect(score).toBeLessThan(CFG.minScore) }) it("stranger, no location, no engagement, 1 h old scores ~9.9, below the cutoff", () => { - const score = scoreCandidate(candidate({ createdAtMs: at(1) }), CFG, NOW) + const score = scoreCandidate(candidate({ createdAtMs: at(1) }), CFG, NOW, SEED) expect(score).toBeCloseTo(9.84, 2) expect(score).toBeLessThan(CFG.minScore) }) @@ -278,6 +290,7 @@ describe("feed ranking: ordering", () => { ], CFG, NOW, + SEED, ) expect(ranked.map((entry) => entry.id)).toEqual([ "cccccccc-0000-0000-0000-000000000000", @@ -295,12 +308,12 @@ describe("feed ranking: ordering", () => { likeCount: i, }), ) - expect(rankCandidates(rows, CFG, NOW)).toEqual(rankCandidates(rows, CFG, NOW)) + expect(rankCandidates(rows, CFG, NOW, SEED)).toEqual(rankCandidates(rows, CFG, NOW, SEED)) }) it("never emits a negative score", () => { const hostile: FeedRankingConfig = { ...CFG, baseWeight: 0 } - const score = scoreCandidate(candidate({ createdAtMs: NOW - 24 * 365 * HOUR }), hostile, NOW) + const score = scoreCandidate(candidate({ createdAtMs: NOW - 24 * 365 * HOUR }), hostile, NOW, SEED) expect(score).toBeGreaterThanOrEqual(0) }) }) @@ -316,13 +329,13 @@ describe("feed ranking: cutoff and cold start", () => { ) it("suspends the cutoff on page 1 when too little clears it", () => { - const ranked = rankCandidates(weak(), CFG, NOW) + const ranked = rankCandidates(weak(), CFG, NOW, SEED) expect(ranked.every((entry) => entry.score < CFG.minScore)).toBe(true) expect(applyCutoff(ranked, CFG, true)).toHaveLength(3) }) it("never suspends the cutoff on a later page: running out of feed is normal", () => { - const ranked = rankCandidates(weak(), CFG, NOW) + const ranked = rankCandidates(weak(), CFG, NOW, SEED) expect(applyCutoff(ranked, CFG, false)).toHaveLength(0) }) @@ -335,7 +348,7 @@ describe("feed ranking: cutoff and cold start", () => { createdAtMs: NOW, }), ) - const ranked = rankCandidates([...strong, ...weak()], CFG, NOW) + const ranked = rankCandidates([...strong, ...weak()], CFG, NOW, SEED) const kept = applyCutoff(ranked, CFG, true) expect(kept).toHaveLength(6) expect(kept.every((entry) => entry.score >= CFG.minScore)).toBe(true) @@ -345,11 +358,195 @@ describe("feed ranking: cutoff and cold start", () => { const strangers = Array.from({ length: 3 }, (_, i) => candidate({ id: `n${i}`, authorId: `author-${i}`, createdAtMs: NOW - HOUR }), ) - const ranked = rankCandidates(strangers, CFG, NOW) + const ranked = rankCandidates(strangers, CFG, NOW, SEED) expect(applyCutoff(ranked, CFG, true).length).toBeGreaterThan(0) }) }) +describe("feed ranking: the global half is viewer-independent by construction", () => { + const viewerFields: Array> = [ + { authorFollowed: true }, + { authorIsViewer: true }, + { viewerMentioned: true }, + { distanceKm: 0 }, + { distanceKm: 2 }, + { authorFollowed: true, authorIsViewer: true, viewerMentioned: true, distanceKm: 1 }, + ] + + it("scores the same post identically for every viewer", () => { + const anonymous = globalScore(candidate(), CFG) + for (const over of viewerFields) { + expect(globalScore(candidate(over), CFG)).toBe(anonymous) + } + }) + + it("still reacts to the post's own properties", () => { + const plain = globalScore(candidate(), CFG) + expect(globalScore(candidate({ authorOrgVerified: true }), CFG) - plain).toBeCloseTo( + CFG.orgVerifiedWeight, + 10, + ) + expect(globalScore(candidate({ hasReport: true }), CFG) - plain).toBeCloseTo( + CFG.attachReportWeight, + 10, + ) + expect(globalScore(candidate({ likeCount: 12 }), CFG)).toBeGreaterThan(plain) + }) + + it("carries every viewer-specific term in the viewer half", () => { + expect(viewerScore(candidate(), CFG)).toBe(0) + expect(viewerScore(candidate({ authorFollowed: true }), CFG)).toBeCloseTo(CFG.followWeight, 10) + expect(viewerScore(candidate({ authorIsViewer: true }), CFG)).toBeCloseTo(CFG.selfWeight, 10) + expect(viewerScore(candidate({ viewerMentioned: true }), CFG)).toBeCloseTo(CFG.mentionWeight, 10) + expect(viewerScore(candidate({ distanceKm: 0 }), CFG)).toBeCloseTo(CFG.nearbyWeight, 10) + }) + + it("sums back to the raw score, so the split is a regrouping not a rewrite", () => { + for (const over of viewerFields) { + const c = candidate({ ...over, likeCount: 9, hasReport: true, authorOrgVerified: true }) + expect(globalScore(c, CFG) + viewerScore(c, CFG)).toBe(rawScore(c, CFG)) + } + }) +}) + +describe("feed ranking: location outranks affinity under the 0.54.0 weights", () => { + it("puts a stranger two kilometres away above a followed author across town", () => { + const nearbyStranger = candidate({ + id: "aaaaaaaa-0000-0000-0000-000000000000", + authorId: "stranger", + distanceKm: 2, + }) + const distantFriend = candidate({ + id: "ffffffff-0000-0000-0000-000000000000", + authorId: "friend", + authorFollowed: true, + distanceKm: 35, + }) + + const ranked = rankCandidates([distantFriend, nearbyStranger], CFG, NOW, SEED) + expect(ranked[0]!.id).toBe(nearbyStranger.id) + expect(rawScore(nearbyStranger, CFG)).toBeGreaterThan(rawScore(distantFriend, CFG)) + }) + + it("keeps that ordering even at the widest jitter swing", () => { + const nearbyStranger = candidate({ id: "near", authorId: "stranger", distanceKm: 2 }) + const distantFriend = candidate({ + id: "far", + authorId: "friend", + authorFollowed: true, + distanceKm: 35, + }) + const worst = rawScore(nearbyStranger, CFG) * (1 - JITTERED.jitterAmount) + const best = rawScore(distantFriend, CFG) * (1 + JITTERED.jitterAmount) + expect(worst).toBeGreaterThan(best) + }) + + it("does not let the follow boost alone beat being in the neighbourhood", () => { + const followedNowhere = candidate({ authorFollowed: true, distanceKm: null }) + const strangerNextDoor = candidate({ distanceKm: 0 }) + expect(viewerScore(strangerNextDoor, CFG)).toBeGreaterThan(viewerScore(followedNowhere, CFG)) + }) +}) + +describe("feed ranking: seeded jitter", () => { + const spread = () => + Array.from({ length: 40 }, (_, i) => + candidate({ + id: `${i}`.padStart(8, "0") + "-0000-0000-0000-000000000000", + authorId: `author-${i}`, + authorFollowed: true, + }), + ) + + it("draws a uniform value in [0, 1) for any seed and post id", () => { + for (let seed = 0; seed < 50; seed += 1) { + for (const id of ["a", "post-1", "11111111-1111-1111-1111-111111111111"]) { + const u = jitterUnit(seed, id) + expect(u).toBeGreaterThanOrEqual(0) + expect(u).toBeLessThan(1) + } + } + }) + + it("is a pure function of the seed and the post id", () => { + expect(jitterUnit(7, "post-a")).toBe(jitterUnit(7, "post-a")) + expect(jitterUnit(7, "post-a")).not.toBe(jitterUnit(8, "post-a")) + expect(jitterUnit(7, "post-a")).not.toBe(jitterUnit(7, "post-b")) + }) + + it("never moves a score outside plus or minus jitterAmount", () => { + for (const row of spread()) { + const plain = scoreCandidate(row, CFG, NOW, SEED) + for (let seed = 0; seed < 25; seed += 1) { + const jittered = scoreCandidate(row, JITTERED, NOW, seed) + expect(jittered).toBeGreaterThanOrEqual(plain * (1 - JITTERED.jitterAmount)) + expect(jittered).toBeLessThanOrEqual(plain * (1 + JITTERED.jitterAmount)) + } + } + }) + + it("reproduces the unjittered score byte for byte when jitterAmount is 0", () => { + const rows = spread() + const first = rankCandidates(rows, CFG, NOW, 1) + const second = rankCandidates(rows, CFG, NOW, 987_654_321) + expect(second).toEqual(first) + for (const row of rows) { + expect(scoreCandidate(row, CFG, NOW, 4_242)).toBe(scoreCandidate(row, CFG, NOW, SEED)) + } + }) + + it("produces an identical ranking for the same seed, every time", () => { + const rows = spread() + expect(rankCandidates(rows, JITTERED, NOW, 31)).toEqual(rankCandidates(rows, JITTERED, NOW, 31)) + }) + + it("reshuffles a near-tied set when the seed changes", () => { + const rows = spread() + const unjittered = rankCandidates(rows, CFG, NOW, SEED).map((entry) => entry.id) + expect(new Set(rankCandidates(rows, CFG, NOW, SEED).map((e) => e.score)).size).toBe(1) + + const a = rankCandidates(rows, JITTERED, NOW, 1).map((entry) => entry.id) + const b = rankCandidates(rows, JITTERED, NOW, 2).map((entry) => entry.id) + expect(a).not.toEqual(b) + expect(a).not.toEqual(unjittered) + expect([...a].sort()).toEqual([...b].sort()) + }) + + it("keeps the score non-negative at the widest jitter", () => { + const hostile: FeedRankingConfig = { ...JITTERED, jitterAmount: 1, baseWeight: 0 } + for (let seed = 0; seed < 25; seed += 1) { + const score = scoreCandidate(candidate({ createdAtMs: NOW - HOUR }), hostile, NOW, seed) + expect(score).toBeGreaterThanOrEqual(0) + } + }) +}) + +describe("feed ranking: the deterministic bucket seed", () => { + it("agrees for two recomputations inside the same clock bucket", () => { + const bucket = quantizeClock(NOW + 5_000, CFG.clockBucketSeconds) + expect(bucketSeed("viewer-1", "all", bucket)).toBe( + bucketSeed("viewer-1", "all", quantizeClock(NOW + 55_000, CFG.clockBucketSeconds)), + ) + }) + + it("moves with the viewer, the filter and the bucket", () => { + const bucket = quantizeClock(NOW, CFG.clockBucketSeconds) + const base = bucketSeed("viewer-1", "all", bucket) + expect(bucketSeed("viewer-2", "all", bucket)).not.toBe(base) + expect(bucketSeed("viewer-1", "events", bucket)).not.toBe(base) + expect(bucketSeed("viewer-1", "all", bucket + 60_000)).not.toBe(base) + }) + + it("stays a non-negative 32-bit integer", () => { + for (let i = 0; i < 50; i += 1) { + const seed = bucketSeed(`viewer-${i}`, "all", NOW + i * 60_000) + expect(Number.isInteger(seed)).toBe(true) + expect(seed).toBeGreaterThanOrEqual(0) + expect(seed).toBeLessThan(2 ** 32) + } + }) +}) + describe("FeedRankingConfigSchema", () => { it("fills every knob from an empty object", () => { const parsed = FeedRankingConfigSchema.parse({}) From 61c6e68b48adf994e33873f0dd1ae65dd29dd574 Mon Sep 17 00:00:00 2001 From: Theo Date: Mon, 21 Sep 2026 23:31:50 -0700 Subject: [PATCH 45/54] feed batch cleanup --- .../src/adapters/reverse-geocode.photon.ts | 13 +++++----- .../src/routes/host/announcements.routes.ts | 12 --------- services/api/src/services/address-resolver.ts | 21 +++++---------- .../services/admin/admin-report-service.ts | 14 +++++----- services/api/src/services/cleanup-service.ts | 11 ++++---- services/api/src/services/feed-ranking.ts | 16 +++++------- .../src/services/host/announcement-service.ts | 20 ++++++++------ .../src/services/host/broadcast-service.ts | 20 +++----------- .../api/src/services/host/comms-wiring.ts | 1 - .../services/host/event-analytics-service.ts | 2 +- .../notification-repository.drizzle.ts | 18 ------------- .../api/src/services/notification-service.ts | 20 -------------- .../src/services/post-repository.drizzle.ts | 26 ++++++++----------- services/api/src/services/post-service.ts | 5 ++-- services/api/test/helpers/notifications.ts | 21 --------------- 15 files changed, 62 insertions(+), 158 deletions(-) diff --git a/services/api/src/adapters/reverse-geocode.photon.ts b/services/api/src/adapters/reverse-geocode.photon.ts index 4a99f109..f292db58 100644 --- a/services/api/src/adapters/reverse-geocode.photon.ts +++ b/services/api/src/adapters/reverse-geocode.photon.ts @@ -87,15 +87,16 @@ export function distanceMeters( return 2 * EARTH_RADIUS_M * Math.asin(Math.min(1, Math.sqrt(s))) } -/** The ", City, ST" tail, skipping whatever is already the primary and dropping a US country. */ -function localityTail(p: PhotonReverseProps, primary: string): string[] { - return [ +/** ", City, ST" - skips whatever is already the primary and drops a US country. */ +function addressLine(p: PhotonReverseProps, primary: string): string { + const tail = [ p.city && p.city !== primary ? p.city : null, p.state, p.country && p.country !== "United States" && p.country !== "United States of America" ? p.country : null, ].filter((v): v is string => !!v) + return [primary, ...tail].join(", ") } /** @@ -110,7 +111,7 @@ export function formatPhotonReverse(p: PhotonReverseProps): string | null { const street = [p.housenumber, p.street].filter(Boolean).join(" ").trim() const primary = street || p.name || p.district || p.city || "" if (!primary) return null - return [primary, ...localityTail(p, primary)].join(", ") + return addressLine(p, primary) } /** True when a feature's `name` could be a private residence label rather than a public landmark. */ @@ -178,14 +179,14 @@ export function composePhotonReverse( } if (roadProps !== null && roads[0] !== undefined) { const primary = roads.length >= 2 ? `${roads[0]} & ${roads[1]}` : roads[0] - return { line: [primary, ...localityTail(roadProps, primary)].join(", "), precision: "intersection" } + return { line: addressLine(roadProps, primary), precision: "intersection" } } for (const c of candidates) { if (c.meters > LANDMARK_RADIUS_M) continue const name = landmarkName(c.props) if (name === null) continue - return { line: [name, ...localityTail(c.props, name)].join(", "), precision: "landmark" } + return { line: addressLine(c.props, name), precision: "landmark" } } return null diff --git a/services/api/src/routes/host/announcements.routes.ts b/services/api/src/routes/host/announcements.routes.ts index 1c24a1f7..f3e85c9f 100644 --- a/services/api/src/routes/host/announcements.routes.ts +++ b/services/api/src/routes/host/announcements.routes.ts @@ -28,16 +28,6 @@ import type { export const ANNOUNCEMENT_CREATE_RATE_LIMIT = perIdentity({ max: 10, timeWindow: "1 minute" }) export const ANNOUNCEMENT_READ_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute" }) -export interface AnnouncementOverrides { - runtime: CommsRuntime -} - -declare module "fastify" { - interface FastifyInstance { - announcementOverrides?: AnnouncementOverrides - } -} - function mergeParams(request: FastifyRequest): Record { const params = (request.params ?? {}) as Record const body = (request.body ?? {}) as Record @@ -58,8 +48,6 @@ export async function registerHostAnnouncementRoutes( let cached: CommsRuntime | undefined function runtime(): CommsRuntime { - const override = app.announcementOverrides - if (override) return override.runtime return (cached ??= makeCommsRuntime(container, app.log)) } diff --git a/services/api/src/services/address-resolver.ts b/services/api/src/services/address-resolver.ts index 039459a8..a9743479 100644 --- a/services/api/src/services/address-resolver.ts +++ b/services/api/src/services/address-resolver.ts @@ -87,8 +87,8 @@ export function makeAddressResolver(deps: AddressResolverDeps): AddressResolver const chainAlreadyMissed = cached !== null const [hit, label] = await Promise.all([ - chainAlreadyMissed ? Promise.resolve(null) : orNull(deps.streetReverseGeocode(lat, lng)), - orNull(Promise.resolve(deps.geocoder.cityStateLabel(lat, lng))), + chainAlreadyMissed ? null : orNull(deps.streetReverseGeocode(lat, lng)), + orNull(deps.geocoder.cityStateLabel(lat, lng)), ]) const cityStateLabel = label ?? "" @@ -99,22 +99,13 @@ export function makeAddressResolver(deps: AddressResolverDeps): AddressResolver ? { address: cityStateLabel, precision: "locality", cityStateLabel } : { address: null, precision: null, cityStateLabel } - if (hit !== null) { + if (hit !== null || !chainAlreadyMissed) { await orNull( cache.write(pointKey, { - address: hit.line, - precision: hit.precision, + address: hit?.line ?? null, + precision: hit?.precision ?? null, cityStateLabel, - provider: hit.provider, - }), - ) - } else if (!chainAlreadyMissed) { - await orNull( - cache.write(pointKey, { - address: null, - precision: null, - cityStateLabel, - provider: null, + provider: hit?.provider ?? null, }), ) } diff --git a/services/api/src/services/admin/admin-report-service.ts b/services/api/src/services/admin/admin-report-service.ts index 53d09769..31389f0a 100644 --- a/services/api/src/services/admin/admin-report-service.ts +++ b/services/api/src/services/admin/admin-report-service.ts @@ -38,6 +38,7 @@ import { } from "./mail-format.js" import { pickPreviewMedia, previewThumbnailUrl } from "./admin-report-types.js" import type { + AdminReportMediaRecord, AdminReportRecord, AdminReportService, AdminReportServiceDeps, @@ -95,13 +96,15 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor await deps.reportChatEmitter.emit(event) } - async function previewMediaDTO(record: AdminReportRecord): Promise { - const m = record.previewMedia - if (m === null) return null + async function toMediaDTO(m: AdminReportMediaRecord): Promise { const { url, thumbUrl } = await presignMedia(m.r2Key, m.thumbKey) return { id: m.id, kind: m.kind, url, thumbUrl: thumbUrl ?? null } } + async function previewMediaDTO(record: AdminReportRecord): Promise { + return record.previewMedia === null ? null : toMediaDTO(record.previewMedia) + } + function toListItem( record: AdminReportRecord, ref: Date, @@ -193,10 +196,7 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor contact: routing?.contact ?? null, routed: routing?.routed ?? false, } - const mediaDtos: ReportMedia[] = await mapWithLimit(media, PRESIGN_CONCURRENCY, async (m) => { - const { url, thumbUrl } = await presignMedia(m.r2Key, m.thumbKey) - return { id: m.id, kind: m.kind, url, thumbUrl: thumbUrl ?? null } - }) + const mediaDtos: ReportMedia[] = await mapWithLimit(media, PRESIGN_CONCURRENCY, toMediaDTO) const base = toListItem(record, ref, pickPreviewMedia(mediaDtos)) const linkedEvents: LinkedEventRef[] = (linkedEventsMap.get(id) ?? []).map(toLinkedEventRef) const outreachDTO: ReportOutreach = { diff --git a/services/api/src/services/cleanup-service.ts b/services/api/src/services/cleanup-service.ts index 753921c1..7e727504 100644 --- a/services/api/src/services/cleanup-service.ts +++ b/services/api/src/services/cleanup-service.ts @@ -1093,11 +1093,12 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { lng: source.lng, scheduledAt: scheduledAt.toISOString(), ...(source.bring !== null ? { bring: source.bring } : {}), - ...(source.address !== null && source.addressSource !== null - ? { address: source.address, addressSource: source.addressSource } - : source.address !== null - ? { address: source.address } - : {}), + ...(source.address !== null + ? { + address: source.address, + ...(source.addressSource !== null ? { addressSource: source.addressSource } : {}), + } + : {}), slots: slots.length > 0 ? slots.map((slot) => ({ diff --git a/services/api/src/services/feed-ranking.ts b/services/api/src/services/feed-ranking.ts index 57bfbc68..7c4f8efd 100644 --- a/services/api/src/services/feed-ranking.ts +++ b/services/api/src/services/feed-ranking.ts @@ -115,7 +115,7 @@ export function jitterUnit(seed: number, postId: string): number { return avalanche(fnv1a32(`${seed}:${postId}`)) / UINT32_SPAN } -export function jitterMultiplier(cfg: FeedRankingConfig, seed: number, postId: string): number { +function jitterMultiplier(cfg: FeedRankingConfig, seed: number, postId: string): number { if (cfg.jitterAmount === 0) return 1 return 1 + cfg.jitterAmount * (2 * jitterUnit(seed, postId) - 1) } @@ -160,17 +160,13 @@ export function rankCandidates( base.sort(byScoreThenId) const seenPerAuthor = new Map() - const discounted = base.map((entry) => { + for (const entry of base) { const rank = seenPerAuthor.get(entry.authorId) ?? 0 seenPerAuthor.set(entry.authorId, rank + 1) - return { - id: entry.id, - authorId: entry.authorId, - score: quantizeFeedScore(entry.score * diversityMultiplier(rank, cfg)), - } - }) - discounted.sort(byScoreThenId) - return discounted + entry.score = quantizeFeedScore(entry.score * diversityMultiplier(rank, cfg)) + } + base.sort(byScoreThenId) + return base } export function applyCutoff( diff --git a/services/api/src/services/host/announcement-service.ts b/services/api/src/services/host/announcement-service.ts index ecf9a05e..adcc0fac 100644 --- a/services/api/src/services/host/announcement-service.ts +++ b/services/api/src/services/host/announcement-service.ts @@ -14,7 +14,6 @@ import type { ListEventAnnouncementsResponse, PersonDTO, } from "@civfix/shared" -import type { FastifyBaseLogger } from "fastify" import { encodeTimeCursor, parseTimeCursor } from "../../db/cursor-helpers.js" import type { BroadcastRepository } from "./broadcast-repository.js" import type { BroadcastRecord } from "./broadcast-types.js" @@ -33,7 +32,6 @@ export interface AnnouncementServiceDeps { identities: AnnouncementIdentityRepository broadcasts: BroadcastService config: Pick - logger?: Pick now?: () => Date } @@ -151,6 +149,16 @@ export function makeAnnouncementService(deps: AnnouncementServiceDeps): Announce return broadcastLinkWarnings(url, config.linkAllowedHosts).length === 0 ? url : null } + async function hydrateOne( + cleanupId: string, + record: BroadcastRecord, + projection: AnnouncementProjection, + ): Promise { + const [dto] = await hydrate(cleanupId, [record], projection) + if (dto === undefined) throw notFound() + return dto + } + return { async create(cleanupId, actorId, body) { await deps.broadcasts.assertComposeAllowed(cleanupId, actorId) @@ -187,9 +195,7 @@ export function makeAnnouncementService(deps: AnnouncementServiceDeps): Announce const sent = await repo.findForEvent(cleanupId, draft.id) if (sent === null) throw notFound() - const [dto] = await hydrate(cleanupId, [sent], { host: true }) - if (dto === undefined) throw notFound() - return dto + return hydrateOne(cleanupId, sent, { host: true }) }, async list(cleanupId, query, projection) { @@ -215,9 +221,7 @@ export function makeAnnouncementService(deps: AnnouncementServiceDeps): Announce if (!projection.host && record.status !== "sending" && record.status !== "sent") { throw notFound() } - const [dto] = await hydrate(cleanupId, [record], projection) - if (dto === undefined) throw notFound() - return dto + return hydrateOne(cleanupId, record, projection) }, } } diff --git a/services/api/src/services/host/broadcast-service.ts b/services/api/src/services/host/broadcast-service.ts index 493b2b05..466a101c 100644 --- a/services/api/src/services/host/broadcast-service.ts +++ b/services/api/src/services/host/broadcast-service.ts @@ -141,10 +141,6 @@ export function toBroadcastDTO(record: BroadcastRecord): BroadcastDTO { } } -export interface BroadcastSendOptions { - skipEventSendCounters?: boolean -} - export interface BroadcastService { list(cleanupId: string, query: ListEventBroadcastsRequest): Promise<{ items: BroadcastDTO[] @@ -168,12 +164,7 @@ export interface BroadcastService { body: PreviewEventBroadcastRequest, ): Promise testSend(cleanupId: string, actorId: string, broadcastId: string): Promise<{ ok: true }> - send( - cleanupId: string, - actorId: string, - broadcastId: string, - options?: BroadcastSendOptions, - ): Promise + send(cleanupId: string, actorId: string, broadcastId: string): Promise sendAnnouncement(cleanupId: string, actorId: string, broadcastId: string): Promise schedule( cleanupId: string, @@ -481,13 +472,8 @@ export function makeBroadcastService(deps: BroadcastServiceDeps): BroadcastServi return { ok: true } }, - async send(cleanupId, actorId, broadcastId, options) { - return sendBroadcast( - cleanupId, - actorId, - broadcastId, - options?.skipEventSendCounters === true, - ) + async send(cleanupId, actorId, broadcastId) { + return sendBroadcast(cleanupId, actorId, broadcastId, false) }, async sendAnnouncement(cleanupId, actorId, broadcastId) { diff --git a/services/api/src/services/host/comms-wiring.ts b/services/api/src/services/host/comms-wiring.ts index 6da06254..0f806762 100644 --- a/services/api/src/services/host/comms-wiring.ts +++ b/services/api/src/services/host/comms-wiring.ts @@ -139,7 +139,6 @@ export function makeCommsRuntime(container: Container, logger?: CommsLogger): Co ), broadcasts, config, - ...(logger !== undefined ? { logger } : {}), }) const pipeline = makeBroadcastPipeline({ diff --git a/services/api/src/services/host/event-analytics-service.ts b/services/api/src/services/host/event-analytics-service.ts index d58b1501..5dea79b0 100644 --- a/services/api/src/services/host/event-analytics-service.ts +++ b/services/api/src/services/host/event-analytics-service.ts @@ -373,7 +373,7 @@ export function makeEventAnalyticsService( registeredPublishable: boolean, full: boolean, ): Panel { - const panel = toPanel(breakdown([...rows], { totalPublishable: registeredPublishable })) + const panel = toPanel(breakdown(rows, { totalPublishable: registeredPublishable })) if (full) return panel return { ...panel, rows: panel.rows.slice(0, EVENT_ANALYTICS_CARD_SLOT_ROWS) } } diff --git a/services/api/src/services/notification-repository.drizzle.ts b/services/api/src/services/notification-repository.drizzle.ts index de3e925a..63290f24 100644 --- a/services/api/src/services/notification-repository.drizzle.ts +++ b/services/api/src/services/notification-repository.drizzle.ts @@ -312,24 +312,6 @@ export function makeDrizzleNotificationRepository(sql: Sql): NotificationReposit return "stored" }, - async revokeDeviceTokensForOtherUsers(args: { - userId: string - token: string - platform: PushPlatform - deviceId: string | null - }): Promise { - const rows = await sql<{ id: string }[]>` - UPDATE push_tokens - SET revoked_at = now() - WHERE user_id <> ${args.userId} - AND revoked_at IS NULL - AND platform = ${args.platform} - AND token = ${args.token} - RETURNING id - ` - return rows.length - }, - async revokeToken(userId: string, platform: PushPlatform, token: string): Promise { await sql` UPDATE push_tokens diff --git a/services/api/src/services/notification-service.ts b/services/api/src/services/notification-service.ts index c92bf616..e7e33e35 100644 --- a/services/api/src/services/notification-service.ts +++ b/services/api/src/services/notification-service.ts @@ -148,13 +148,6 @@ export interface NotificationRepository { deviceId: string | null }): Promise - revokeDeviceTokensForOtherUsers(args: { - userId: string - token: string - platform: PushPlatform - deviceId: string | null - }): Promise - revokeToken(userId: string, platform: PushPlatform, token: string): Promise deletePushTokensForUser(userId: string): Promise @@ -562,19 +555,6 @@ export function makeNotificationService(deps: NotificationServiceDeps): Notifica "This push token is registered to another account. Sign out on the other account or reinstall the app.", ) } - try { - const revoked = await deps.repo.revokeDeviceTokensForOtherUsers({ - userId, - token: req.token, - platform: req.platform, - deviceId: req.deviceId ?? null, - }) - if (revoked > 0) { - deps.logger?.warn({ userId, platform: req.platform, revoked }, "device-claim revoked other accounts' push tokens") - } - } catch (err) { - deps.logger?.warn({ err, userId }, "device-claim revoke failed (suppressed)") - } try { await deps.pushSender.registerToken(userId, req.token, req.platform, req.deviceId) } catch (err) { diff --git a/services/api/src/services/post-repository.drizzle.ts b/services/api/src/services/post-repository.drizzle.ts index 00f0c71c..206772ae 100644 --- a/services/api/src/services/post-repository.drizzle.ts +++ b/services/api/src/services/post-repository.drizzle.ts @@ -339,6 +339,14 @@ function feedFilterClause(sql: Queryable, filter: FeedFilter): postgres.Fragment return sql`` } +function viewerBlockClause(sql: Queryable, viewerId: string): postgres.Fragment { + return sql`AND NOT EXISTS ( + SELECT 1 FROM user_blocks b + WHERE (b.blocker_id = ${viewerId} AND b.blocked_id = p.author_id) + OR (b.blocker_id = p.author_id AND b.blocked_id = ${viewerId}) + )` +} + export function feedCandidatesStatement( sql: Queryable, args: FeedCandidateArgs, @@ -350,11 +358,7 @@ export function feedCandidatesStatement( AND p.reply_to_id IS NULL AND p.visibility = 'public' AND p.created_at >= (SELECT since FROM eligible_window) - AND NOT EXISTS ( - SELECT 1 FROM user_blocks b - WHERE (b.blocker_id = ${viewerId} AND b.blocked_id = p.author_id) - OR (b.blocker_id = p.author_id AND b.blocked_id = ${viewerId}) - ) + ${viewerBlockClause(sql, viewerId)} ${filterClause} ` return sql` @@ -1212,11 +1216,7 @@ export function makeDrizzlePostRepository(sql: Sql, deps: PostRepoDeps): PostRep WHERE p.id = ANY(${[...ids]}::uuid[]) AND p.deleted_at IS NULL AND p.visibility = 'public' - AND NOT EXISTS ( - SELECT 1 FROM user_blocks b - WHERE (b.blocker_id = ${viewerId} AND b.blocked_id = p.author_id) - OR (b.blocker_id = p.author_id AND b.blocked_id = ${viewerId}) - ) + ${viewerBlockClause(sql, viewerId)} ` const order = new Map(ids.map((id, index) => [id, index])) rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0)) @@ -1246,11 +1246,7 @@ export function makeDrizzlePostRepository(sql: Sql, deps: PostRepoDeps): PostRep WHERE p.id = ANY(${[...postIds]}::uuid[]) AND p.deleted_at IS NULL AND p.visibility = 'public' - AND NOT EXISTS ( - SELECT 1 FROM user_blocks b - WHERE (b.blocker_id = ${viewerId} AND b.blocked_id = p.author_id) - OR (b.blocker_id = p.author_id AND b.blocked_id = ${viewerId}) - ) + ${viewerBlockClause(sql, viewerId)} ` }, diff --git a/services/api/src/services/post-service.ts b/services/api/src/services/post-service.ts index 29a8f944..a097a0f0 100644 --- a/services/api/src/services/post-service.ts +++ b/services/api/src/services/post-service.ts @@ -258,9 +258,10 @@ export function makePostService(deps: PostServiceDeps): PostService { nowBucketMs, seedFor(viewerId, query.filter, isFirstPage, nowBucketMs), ) + const durable = applyCutoff(ranked, feedConfig, false) return { - page: applyCutoff(ranked, feedConfig, true), - durable: applyCutoff(ranked, feedConfig, false), + page: isFirstPage ? applyCutoff(ranked, feedConfig, true) : durable, + durable, } } diff --git a/services/api/test/helpers/notifications.ts b/services/api/test/helpers/notifications.ts index 38685c3b..69a68c55 100644 --- a/services/api/test/helpers/notifications.ts +++ b/services/api/test/helpers/notifications.ts @@ -213,27 +213,6 @@ export class InMemoryNotificationRepository implements NotificationRepository { } } - revokeDeviceTokensForOtherUsers(args: { - userId: string - token: string - platform: PushPlatform - deviceId: string | null - }): Promise { - let revoked = 0 - for (const t of this.pushTokens) { - if ( - t.userId !== args.userId && - t.revokedAt === null && - t.platform === args.platform && - t.token === args.token - ) { - t.revokedAt = this.now() - revoked++ - } - } - return Promise.resolve(revoked) - } - revokeToken(userId: string, platform: PushPlatform, token: string): Promise { for (const t of this.pushTokens) { if ( From c2917b5f23c9bed1132998c132293ee9cde3eb6b Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 22 Sep 2026 00:09:51 -0700 Subject: [PATCH 46/54] feed ranking pagination and snapshot fixes --- services/api/src/services/feed-presence.ts | 9 +- services/api/src/services/feed-ranking.ts | 3 +- .../src/services/post-repository.drizzle.ts | 2 +- services/api/src/services/post-service.ts | 79 ++++++-- .../api/test/unit/feed-candidates-sql.test.ts | 6 +- services/api/test/unit/feed-presence.test.ts | 2 +- .../api/test/unit/feed-ranked-service.test.ts | 177 +++++++++++++++++- services/api/test/unit/feed-ranking.test.ts | 15 +- 8 files changed, 250 insertions(+), 43 deletions(-) diff --git a/services/api/src/services/feed-presence.ts b/services/api/src/services/feed-presence.ts index 16bb3d1d..59e73646 100644 --- a/services/api/src/services/feed-presence.ts +++ b/services/api/src/services/feed-presence.ts @@ -26,7 +26,7 @@ export interface FeedSnapshotEntry { export interface FeedPresence { readonly snapshotsAvailable: boolean readSnapshot(userId: string, filter: string): Promise - writeSnapshot(userId: string, filter: string, ranked: readonly RankedCandidate[]): Promise + writeSnapshot(userId: string, filter: string, ranked: readonly RankedCandidate[]): Promise touchSnapshot(userId: string, filter: string): Promise seenBy(userId: string, postIds: readonly string[]): Promise> recordServed(userId: string, postIds: readonly string[]): Promise @@ -90,9 +90,9 @@ export function makeFeedPresence(deps: FeedPresenceDeps): FeedPresence { userId: string, filter: string, ranked: readonly RankedCandidate[], - ): Promise { - if (cfg.snapshotTtlSeconds <= 0) return Promise.resolve() - return guarded(undefined, "snapshot write", async () => { + ): Promise { + if (cfg.snapshotTtlSeconds <= 0) return Promise.resolve(false) + return guarded(false, "snapshot write", async () => { const payload = ranked .slice(0, cfg.candidateCap) .map((entry) => [entry.id, entry.score] as const) @@ -101,6 +101,7 @@ export function makeFeedPresence(deps: FeedPresenceDeps): FeedPresence { JSON.stringify(payload), cfg.snapshotTtlSeconds, ) + return true }) }, diff --git a/services/api/src/services/feed-ranking.ts b/services/api/src/services/feed-ranking.ts index 7c4f8efd..dd3875db 100644 --- a/services/api/src/services/feed-ranking.ts +++ b/services/api/src/services/feed-ranking.ts @@ -172,9 +172,8 @@ export function rankCandidates( export function applyCutoff( ranked: readonly RankedCandidate[], cfg: FeedRankingConfig, - isFirstPage: boolean, ): RankedCandidate[] { const kept = ranked.filter((entry) => entry.score >= cfg.minScore) - if (isFirstPage && kept.length < cfg.minPageItems) return [...ranked] + if (kept.length < cfg.minPageItems) return [...ranked] return kept } diff --git a/services/api/src/services/post-repository.drizzle.ts b/services/api/src/services/post-repository.drizzle.ts index 206772ae..621dedf5 100644 --- a/services/api/src/services/post-repository.drizzle.ts +++ b/services/api/src/services/post-repository.drizzle.ts @@ -436,7 +436,7 @@ export function feedCandidatesStatement( p.reply_count, p.repost_count, (p.report_id IS NOT NULL) AS has_report, - (p.event_id IS NOT NULL AND ev.ends_at >= now()) AS has_live_event, + COALESCE(ev.status <> 'cancelled' AND ev.ends_at > now(), false) AS has_live_event, EXISTS ( SELECT 1 FROM media_assets ma WHERE ma.post_id = p.id AND ma.status = 'ready' AND ma.served_key IS NOT NULL diff --git a/services/api/src/services/post-service.ts b/services/api/src/services/post-service.ts index a097a0f0..c00f9df1 100644 --- a/services/api/src/services/post-service.ts +++ b/services/api/src/services/post-service.ts @@ -45,6 +45,21 @@ function isVisible(brief: PostBrief): boolean { return brief.deletedAt === null && brief.visibility === "public" } +interface FeedScoreCursor { + score: number + postId: string +} + +interface RankedSet { + ranked: RankedCandidate[] + recomputable: () => RankedCandidate[] +} + +function normalizeScoreCursor(cursor: FeedScoreCursor | null): FeedScoreCursor | null { + if (cursor === null) return null + return { score: cursor.score, postId: cursor.postId.toLowerCase() } +} + function isLegacyTimeCursor(cursor: string | null | undefined): boolean { if (cursor === null || cursor === undefined || cursor === "") return false if (parseFeedScoreCursor(cursor) !== null) return false @@ -231,7 +246,7 @@ export function makePostService(deps: PostServiceDeps): PostService { query: HomeFeedQuery, location: FeedViewerLocation | undefined, isFirstPage: boolean, - ): Promise<{ page: RankedCandidate[]; durable: RankedCandidate[] }> { + ): Promise { const rows = await deps.repo.feedCandidates({ viewerId, filter: query.filter, @@ -252,31 +267,51 @@ export function makePostService(deps: PostServiceDeps): PostService { ) const nowBucketMs = quantizeClock(nowMs(), feedConfig.clockBucketSeconds) - const ranked = rankCandidates( - rows.map((row) => toCandidate(row, seen)), - feedConfig, - nowBucketMs, - seedFor(viewerId, query.filter, isFirstPage, nowBucketMs), - ) - const durable = applyCutoff(ranked, feedConfig, false) + function rankWith(seenSet: ReadonlySet, seed: number): RankedCandidate[] { + return applyCutoff( + rankCandidates( + rows.map((row) => toCandidate(row, seenSet)), + feedConfig, + nowBucketMs, + seed, + ), + feedConfig, + ) + } + return { - page: isFirstPage ? applyCutoff(ranked, feedConfig, true) : durable, - durable, + ranked: rankWith(seen, seedFor(viewerId, query.filter, isFirstPage, nowBucketMs)), + recomputable: () => + rankWith(new Set(), seedFor(viewerId, query.filter, false, nowBucketMs)), } } function persistSnapshot( viewerId: string, filter: string, - durable: readonly RankedCandidate[], + ranked: readonly RankedCandidate[], ): void { const presence = presenceFor(viewerId) if (presence === undefined || !presence.snapshotsAvailable) return - void presence.writeSnapshot(viewerId, filter, durable).catch((err: unknown) => { + void presence.writeSnapshot(viewerId, filter, ranked).catch((err: unknown) => { deps.logger?.warn({ err }, "post: feed snapshot write failed (suppressed)") }) } + async function storeFirstSnapshot( + viewerId: string, + filter: string, + ranked: readonly RankedCandidate[], + ): Promise { + const presence = presenceFor(viewerId) + if (presence === undefined || !presence.snapshotsAvailable) return true + try { + return await presence.writeSnapshot(viewerId, filter, ranked) + } catch { + return false + } + } + async function continueRankedPage( viewerId: string, query: HomeFeedQuery, @@ -295,9 +330,9 @@ export function makePostService(deps: PostServiceDeps): PostService { } } - const { durable } = await rankCandidateSet(viewerId, query, location, false) - persistSnapshot(viewerId, query.filter, durable) - return pageFrom(viewerId, sliceAfterCursor(durable, cursor), limit) + const { ranked } = await rankCandidateSet(viewerId, query, location, false) + persistSnapshot(viewerId, query.filter, ranked) + return pageFrom(viewerId, sliceAfterCursor(ranked, cursor), limit) } async function rankedFeed( @@ -306,15 +341,21 @@ export function makePostService(deps: PostServiceDeps): PostService { location: FeedViewerLocation | undefined, ): Promise { const limit = query.limit ?? POSTS_DEFAULT_LIMIT - const cursor = parseFeedScoreCursor(query.cursor) + const cursor = normalizeScoreCursor(parseFeedScoreCursor(query.cursor)) if (cursor !== null) { return continueRankedPage(viewerId, query, location, cursor, limit) } - const { page, durable } = await rankCandidateSet(viewerId, query, location, true) - persistSnapshot(viewerId, query.filter, durable) - return pageFrom(viewerId, page, limit) + const { ranked, recomputable } = await rankCandidateSet(viewerId, query, location, true) + if (await storeFirstSnapshot(viewerId, query.filter, ranked)) { + return pageFrom(viewerId, ranked, limit) + } + deps.logger?.warn( + { viewerId, filter: query.filter }, + "post: feed snapshot unavailable, serving the reproducible bucket order", + ) + return pageFrom(viewerId, recomputable(), limit) } async function safeNotify(fn: () => Promise): Promise { diff --git a/services/api/test/unit/feed-candidates-sql.test.ts b/services/api/test/unit/feed-candidates-sql.test.ts index cc0705a6..4a283496 100644 --- a/services/api/test/unit/feed-candidates-sql.test.ts +++ b/services/api/test/unit/feed-candidates-sql.test.ts @@ -113,9 +113,11 @@ describe("feed candidate SQL: security invariants", () => { expect(stmt.sql).toMatch(/ma\.status = 'ready' AND ma\.served_key IS NOT NULL/) }) - it("treats an ended event as not live", async () => { + it("reuses the canonical ongoing-event predicate and never yields SQL NULL", async () => { const stmt = await emitted() - expect(stmt.sql).toMatch(/p\.event_id IS NOT NULL AND ev\.ends_at >= now\(\)/) + expect(stmt.sql).toMatch( + /COALESCE\(ev\.status <> 'cancelled' AND ev\.ends_at > now\(\), false\) AS has_live_event/, + ) }) }) diff --git a/services/api/test/unit/feed-presence.test.ts b/services/api/test/unit/feed-presence.test.ts index 586d670b..6ca2a520 100644 --- a/services/api/test/unit/feed-presence.test.ts +++ b/services/api/test/unit/feed-presence.test.ts @@ -240,7 +240,7 @@ describe("feed presence: degrades instead of failing the request", () => { expect(await presence.readSnapshot(VIEWER, "all")).toBeNull() await expect( presence.writeSnapshot(VIEWER, "all", [{ id: POST, authorId: "a", score: 1 }]), - ).resolves.toBeUndefined() + ).resolves.toBe(false) expect([...(await presence.seenBy(VIEWER, [POST]))]).toEqual([]) await expect(presence.recordServed(VIEWER, [POST])).resolves.toBeUndefined() expect(await presence.viewersOf(POST)).toEqual([]) diff --git a/services/api/test/unit/feed-ranked-service.test.ts b/services/api/test/unit/feed-ranked-service.test.ts index a1eefa52..2201597f 100644 --- a/services/api/test/unit/feed-ranked-service.test.ts +++ b/services/api/test/unit/feed-ranked-service.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { DEFAULT_FEED_RANKING, parseFeedScoreCursor } from "@civfix/shared" +import { DEFAULT_FEED_RANKING, formatFeedScoreCursor, parseFeedScoreCursor } from "@civfix/shared" import type { FeedRankingConfig, PostDTO, UserSignal } from "@civfix/shared" import type { UserChannel } from "@civfix/shared/interfaces" import type { Sql } from "../../src/db/client.js" @@ -209,6 +209,49 @@ describe("ranked feed: cursor continuation covers every item exactly once", () = }) }) +describe("ranked feed: a cursor is hex, and hex has no case", () => { + const rows = Array.from({ length: 60 }, (_, i) => + candidateRow(i + 1, { + id: `ab0000${String(i + 1).padStart(2, "0")}-0000-0000-0000-000000000000`, + author_id: `author-${i}`, + like_count: 0, + created_at: new Date(NOW), + }), + ) + + function service(): PostService { + const cache = new InMemoryCacheClient(() => Date.now()) + return makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: makeFeedPresence({ cache, config: NO_JITTER }), + feedRanking: NO_JITTER, + now: () => NOW, + }) + } + + it("resolves an upper-cased cursor against the lower-cased snapshot ids", async () => { + const svc = service() + const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + const parsed = parseFeedScoreCursor(first.nextCursor)! + expect(parsed.postId.toUpperCase()).not.toBe(parsed.postId) + + const shouted = formatFeedScoreCursor({ + score: parsed.score, + postId: parsed.postId.toUpperCase(), + }) + const second = await svc.homeFeed(VIEWER, { filter: "all", limit: 20, cursor: shouted }) + const exact = await svc.homeFeed(VIEWER, { + filter: "all", + limit: 20, + cursor: first.nextCursor!, + }) + + expect(second.items).toHaveLength(20) + expect(second.items.map((item) => item.id)).toEqual(exact.items.map((item) => item.id)) + }) +}) + describe("ranked feed: warm snapshot continues a cursor without re-ranking", () => { const rows = Array.from({ length: 60 }, (_, i) => candidateRow(i + 1)) @@ -504,12 +547,12 @@ describe("ranked feed: a signed-out reader keeps scrolling", () => { }) }) -describe("ranked feed: the cold-start fallback is one request, not one session", () => { +describe("ranked feed: the cold-start leniency pages to exhaustion, it does not dead-end", () => { const rows = Array.from({ length: 40 }, (_, i) => candidateRow(i + 1, { author_followed: false, like_count: 0, created_at: new Date(NOW) }), ) - it("serves an uncut first page but never lets the cutoff stay suspended on page 2", async () => { + function service(): { svc: PostService; presence: ReturnType } { const cache = new InMemoryCacheClient(() => Date.now()) const presence = makeFeedPresence({ cache, config: DEFAULT_FEED_RANKING }) const svc = makePostService({ @@ -518,20 +561,138 @@ describe("ranked feed: the cold-start fallback is one request, not one session", feedPresence: presence, now: () => NOW, }) + return { svc, presence } + } + it("snapshots the same lenient list it served, so the page-1 cursor resolves", async () => { + const { svc, presence } = service() const first = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) - await new Promise((resolve) => setTimeout(resolve, 0)) expect(first.items).toHaveLength(20) - expect(await presence.readSnapshot(VIEWER, "all")).toEqual([]) + const snapshot = await presence.readSnapshot(VIEWER, "all") + expect(snapshot).toHaveLength(40) - const second = await svc.homeFeed(VIEWER, { + const cursor = parseFeedScoreCursor(first.nextCursor) + expect(cursor).not.toBeNull() + expect(snapshot!.some((entry) => entry.id === cursor!.postId)).toBe(true) + }) + + it("pages every lenient item exactly once instead of ending at page 1", async () => { + const { svc } = service() + const seen: string[] = [] + let cursor: string | undefined + let pages = 0 + + for (;;) { + const page = await svc.homeFeed(VIEWER, { + filter: "all", + limit: 20, + ...(cursor === undefined ? {} : { cursor }), + }) + seen.push(...page.items.map((item) => item.id)) + pages += 1 + if (page.nextCursor === null) break + cursor = page.nextCursor + expect(pages).toBeLessThan(10) + } + + expect(pages).toBe(2) + expect(seen).toHaveLength(40) + expect(new Set(seen).size).toBe(40) + }) + + it("keeps the re-ranked continuation lenient too, so an expired snapshot still pages", async () => { + const first = await service().svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + + const expired = makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: makeFeedPresence({ + cache: new InMemoryCacheClient(() => Date.now()), + config: DEFAULT_FEED_RANKING, + }), + now: () => NOW, + }) + const second = await expired.homeFeed(VIEWER, { filter: "all", limit: 20, cursor: first.nextCursor!, }) - expect(second.items).toEqual([]) - expect(second.nextCursor).toBeNull() + + expect(second.items.length).toBeGreaterThan(0) + }) +}) + +describe("ranked feed: a page-1 snapshot that cannot be stored is served reproducibly", () => { + const rows = Array.from({ length: 60 }, (_, i) => + candidateRow(i + 1, { like_count: 0, created_at: new Date(NOW), author_id: `author-${i}` }), + ) + + function bucketOrderService(): PostService { + return makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: makeFeedPresence({ config: DEFAULT_FEED_RANKING }), + now: () => NOW, + }) + } + + function failingSnapshotService(warnings: unknown[]): PostService { + const base = makeFeedPresence({ + cache: new InMemoryCacheClient(() => Date.now()), + config: DEFAULT_FEED_RANKING, + }) + return makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: { ...base, writeSnapshot: () => Promise.reject(new Error("redis down")) }, + feedRanking: DEFAULT_FEED_RANKING, + now: () => NOW, + feedSeed: () => 4242, + logger: { warn: (obj) => warnings.push(obj) }, + }) + } + + it("serves the order a later fallback recompute reproduces in the same bucket", async () => { + const warnings: unknown[] = [] + const served = await failingSnapshotService(warnings).homeFeed(VIEWER, { + filter: "all", + limit: 20, + }) + const fallback = await bucketOrderService().homeFeed(VIEWER, { filter: "all", limit: 20 }) + + expect(served.items.map((item) => item.id)).toEqual(fallback.items.map((item) => item.id)) + expect(warnings).toHaveLength(1) + }) + + it("still serves the minted shuffle when the snapshot write succeeds", async () => { + const svc = makePostService({ + repo: repoOver({ feedCandidates: () => Promise.resolve(rows) }), + sql: throwingSql, + feedPresence: makeFeedPresence({ + cache: new InMemoryCacheClient(() => Date.now()), + config: DEFAULT_FEED_RANKING, + }), + now: () => NOW, + feedSeed: () => 4242, + }) + const minted = await svc.homeFeed(VIEWER, { filter: "all", limit: 20 }) + const fallback = await bucketOrderService().homeFeed(VIEWER, { filter: "all", limit: 20 }) + + expect(minted.items.map((item) => item.id)).not.toEqual(fallback.items.map((item) => item.id)) + }) + + it("continues the fallback page from its own cursor without repeating an item", async () => { + const first = await failingSnapshotService([]).homeFeed(VIEWER, { filter: "all", limit: 20 }) + const second = await bucketOrderService().homeFeed(VIEWER, { + filter: "all", + limit: 20, + cursor: first.nextCursor!, + }) + + const firstIds = new Set(first.items.map((item) => item.id)) + expect(second.items).toHaveLength(20) + expect(second.items.some((item) => firstIds.has(item.id))).toBe(false) }) }) diff --git a/services/api/test/unit/feed-ranking.test.ts b/services/api/test/unit/feed-ranking.test.ts index f9cb835a..7527929e 100644 --- a/services/api/test/unit/feed-ranking.test.ts +++ b/services/api/test/unit/feed-ranking.test.ts @@ -328,15 +328,18 @@ describe("feed ranking: cutoff and cold start", () => { }), ) - it("suspends the cutoff on page 1 when too little clears it", () => { + it("suspends the cutoff when too little clears it", () => { const ranked = rankCandidates(weak(), CFG, NOW, SEED) expect(ranked.every((entry) => entry.score < CFG.minScore)).toBe(true) - expect(applyCutoff(ranked, CFG, true)).toHaveLength(3) + expect(applyCutoff(ranked, CFG)).toHaveLength(3) }) - it("never suspends the cutoff on a later page: running out of feed is normal", () => { + it("decides leniency from the ranked set alone, so every page of one set agrees", () => { const ranked = rankCandidates(weak(), CFG, NOW, SEED) - expect(applyCutoff(ranked, CFG, false)).toHaveLength(0) + expect(applyCutoff(ranked, CFG)).toEqual(applyCutoff(ranked, CFG)) + expect(applyCutoff(ranked, CFG).map((entry) => entry.id)).toEqual( + ranked.map((entry) => entry.id), + ) }) it("applies the cutoff normally once page 1 has enough strong items", () => { @@ -349,7 +352,7 @@ describe("feed ranking: cutoff and cold start", () => { }), ) const ranked = rankCandidates([...strong, ...weak()], CFG, NOW, SEED) - const kept = applyCutoff(ranked, CFG, true) + const kept = applyCutoff(ranked, CFG) expect(kept).toHaveLength(6) expect(kept.every((entry) => entry.score >= CFG.minScore)).toBe(true) }) @@ -359,7 +362,7 @@ describe("feed ranking: cutoff and cold start", () => { candidate({ id: `n${i}`, authorId: `author-${i}`, createdAtMs: NOW - HOUR }), ) const ranked = rankCandidates(strangers, CFG, NOW, SEED) - expect(applyCutoff(ranked, CFG, true).length).toBeGreaterThan(0) + expect(applyCutoff(ranked, CFG).length).toBeGreaterThan(0) }) }) From 72329224851b49daee0d90d1af3f9619db200174 Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 22 Sep 2026 00:09:51 -0700 Subject: [PATCH 47/54] review fixes: post rate limit, announcement cap race, mapbox precision --- .../src/adapters/reverse-geocode.mapbox.ts | 13 +++++-- services/api/src/routes/posts.routes.ts | 2 +- .../src/services/host/announcement-service.ts | 38 ++++++++++--------- .../host/broadcast-repository.drizzle.ts | 34 ++++++++++++++--- .../host/broadcast-repository.memory.ts | 15 ++++++++ .../src/services/host/broadcast-repository.ts | 9 +++++ .../unit/host/announcement-service.test.ts | 31 +++++++++++++++ services/api/test/unit/posts-routes.test.ts | 8 +--- .../test/unit/reverse-geocode-mapbox.test.ts | 14 ++++++- 9 files changed, 130 insertions(+), 34 deletions(-) diff --git a/services/api/src/adapters/reverse-geocode.mapbox.ts b/services/api/src/adapters/reverse-geocode.mapbox.ts index b93f7f5b..33e62b6f 100644 --- a/services/api/src/adapters/reverse-geocode.mapbox.ts +++ b/services/api/src/adapters/reverse-geocode.mapbox.ts @@ -11,8 +11,9 @@ * worse, short-circuit the rest of the chain with the least specific line available. So this adapter * returns only what it can prove - * - * street a house number is present (context.address.address_number, or a name that starts - * with one) "123 Main St, Inglewood, CA" + * street a house number is present (context.address.address_number, or an address label - + * context.address.name, or the feature's own name when feature_type is `address` - + * that starts with one) "123 Main St, Inglewood, CA" * intersection a named street but no number "Main St, Inglewood, CA" * * - and hands anything coarser to the next provider, ending at the local TIGER `locality` label. @@ -34,6 +35,7 @@ interface MapboxReverseContext { } export interface MapboxReverseProps { name?: string + feature_type?: string full_address?: string place_formatted?: string context?: MapboxReverseContext @@ -60,7 +62,10 @@ export function formatMapboxReverse(p: MapboxReverseProps): string | null { return [primary, ...tail].join(", ") } -/** A leading house number is the only thing that earns the `street` rung. */ +/** + * A leading house number earns the `street` rung, but ONLY on a label the response itself calls an + * address. A digit can lead any POI name ("24 Hour Fitness"), and a POI is not a rooftop. + */ function startsWithHouseNumber(name: string | undefined): boolean { return name !== undefined && /^\d/.test(name.trim()) } @@ -74,7 +79,7 @@ export function mapboxPrecision(p: MapboxReverseProps): AddressPrecision | null const hasNumber = (ctx.address?.address_number?.trim().length ?? 0) > 0 || startsWithHouseNumber(ctx.address?.name) || - startsWithHouseNumber(p.name) + (p.feature_type === "address" && startsWithHouseNumber(p.name)) if (hasNumber) return "street" const named = ctx.street?.name ?? ctx.address?.street_name ?? null if (named !== null && named.trim().length > 0) return "intersection" diff --git a/services/api/src/routes/posts.routes.ts b/services/api/src/routes/posts.routes.ts index 0cd2b3cf..8fad3608 100644 --- a/services/api/src/routes/posts.routes.ts +++ b/services/api/src/routes/posts.routes.ts @@ -24,7 +24,7 @@ export const FEED_COUNTS_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minu export const HOME_FEED_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute" }) -export const CREATE_POST_RATE_LIMIT = perIdentity({ max: 120, timeWindow: "1 minute" }) +export const CREATE_POST_RATE_LIMIT = perIdentity({ max: 12, timeWindow: "1 minute" }) export const POST_INTERACTION_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute" }) diff --git a/services/api/src/services/host/announcement-service.ts b/services/api/src/services/host/announcement-service.ts index adcc0fac..911fe8e9 100644 --- a/services/api/src/services/host/announcement-service.ts +++ b/services/api/src/services/host/announcement-service.ts @@ -15,7 +15,7 @@ import type { PersonDTO, } from "@civfix/shared" import { encodeTimeCursor, parseTimeCursor } from "../../db/cursor-helpers.js" -import type { BroadcastRepository } from "./broadcast-repository.js" +import type { AnnouncementCap, BroadcastRepository } from "./broadcast-repository.js" import type { BroadcastRecord } from "./broadcast-types.js" import { broadcastLinkWarnings } from "./broadcast-render.js" import { announcementPath } from "./broadcast-pipeline.js" @@ -138,10 +138,11 @@ export function makeAnnouncementService(deps: AnnouncementServiceDeps): Announce ) } - async function assertUnderDailyCap(cleanupId: string): Promise { - const since = new Date(now().getTime() - ANNOUNCEMENT_WINDOW_MS) - const used = await repo.countAnnouncementsSince(cleanupId, since) - if (used >= MAX_EVENT_ANNOUNCEMENTS_PER_DAY) throw announcementRateLimited() + function dailyCap(): AnnouncementCap { + return { + since: new Date(now().getTime() - ANNOUNCEMENT_WINDOW_MS), + max: MAX_EVENT_ANNOUNCEMENTS_PER_DAY, + } } function ctaFor(cleanupId: string, announcementId: string): string | null { @@ -162,22 +163,25 @@ export function makeAnnouncementService(deps: AnnouncementServiceDeps): Announce return { async create(cleanupId, actorId, body) { await deps.broadcasts.assertComposeAllowed(cleanupId, actorId) - await assertUnderDailyCap(cleanupId) const event = await repo.eventContext(cleanupId) if (event === null) throw AppError.notFound("Cleanup not found") - const draft = await repo.create({ - cleanupId, - createdBy: actorId, - kind: ANNOUNCEMENT_BROADCAST_KIND, - subject: announcementTitleOf(body.title, event.title), - bodyMd: body.bodyMd, - segment: announcementAudienceToSegment(body.audience), - channels: [...ANNOUNCEMENT_CHANNELS] as BroadcastChannel[], - status: "draft", - chunkSize: config.chunkSize, - }) + const draft = await repo.createAnnouncementUnderCap( + { + cleanupId, + createdBy: actorId, + kind: ANNOUNCEMENT_BROADCAST_KIND, + subject: announcementTitleOf(body.title, event.title), + bodyMd: body.bodyMd, + segment: announcementAudienceToSegment(body.audience), + channels: [...ANNOUNCEMENT_CHANNELS] as BroadcastChannel[], + status: "draft", + chunkSize: config.chunkSize, + }, + dailyCap(), + ) + if (draft === null) throw announcementRateLimited() try { const cta = ctaFor(cleanupId, draft.id) diff --git a/services/api/src/services/host/broadcast-repository.drizzle.ts b/services/api/src/services/host/broadcast-repository.drizzle.ts index c8122296..9975c796 100644 --- a/services/api/src/services/host/broadcast-repository.drizzle.ts +++ b/services/api/src/services/host/broadcast-repository.drizzle.ts @@ -6,10 +6,11 @@ import type { BroadcastStatus, DeliveryStatus, } from "@civfix/shared" -import type { Sql } from "../../db/client.js" +import type { Queryable, Sql } from "../../db/client.js" import { listGuestAudiencePage, listMemberAudiencePage } from "./broadcast-audience-sql.js" import type { AdminBroadcastListQuery, + AnnouncementCap, AnnouncementListQuery, AudiencePageQuery, BroadcastListQuery, @@ -125,9 +126,11 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { return rows[0] ? toRecord(rows[0]) : null } - return { - async create(input: BroadcastCreateInput): Promise { - const rows = await sql` + async function insertBroadcast( + db: Queryable, + input: BroadcastCreateInput, + ): Promise { + const rows = await db` INSERT INTO broadcasts ( cleanup_id, created_by, kind, reminder_offset_min, status, subject, body_md, cta_label, cta_url, segment, channels, reply_to, scheduled_at, chunk_size @@ -142,7 +145,28 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { cta_label, cta_url, segment, channels, reply_to, scheduled_at, planned_at, started_at, finished_at, chunk_size, chunk_count, recipient_count, sent_count, failed_count, suppressed_count, content_scrubbed_at, created_at, updated_at` - return toRecord(rows[0]!) + return toRecord(rows[0]!) + } + + return { + create(input: BroadcastCreateInput): Promise { + return insertBroadcast(sql, input) + }, + + createAnnouncementUnderCap( + input: BroadcastCreateInput, + cap: AnnouncementCap, + ): Promise { + return sql.begin(async (tx) => { + await tx`SELECT id FROM cleanups WHERE id = ${input.cleanupId} LIMIT 1 FOR UPDATE` + const rows = await tx<{ n: number }[]>` + SELECT count(*)::int AS n FROM broadcasts + WHERE cleanup_id = ${input.cleanupId} + AND kind = ${ANNOUNCEMENT_BROADCAST_KIND} + AND created_at >= ${cap.since}` + if ((rows[0]?.n ?? 0) >= cap.max) return null + return insertBroadcast(tx, input) + }) as Promise }, async createIfAbsent(input: BroadcastCreateInput): Promise { diff --git a/services/api/src/services/host/broadcast-repository.memory.ts b/services/api/src/services/host/broadcast-repository.memory.ts index 843fe4dd..1e321d78 100644 --- a/services/api/src/services/host/broadcast-repository.memory.ts +++ b/services/api/src/services/host/broadcast-repository.memory.ts @@ -3,6 +3,7 @@ import { ANNOUNCEMENT_BROADCAST_KIND } from "@civfix/shared" import type { BroadcastKind, BroadcastStatus } from "@civfix/shared" import type { AdminBroadcastListQuery, + AnnouncementCap, AnnouncementListQuery, AudiencePageQuery, BroadcastListQuery, @@ -177,6 +178,20 @@ export class InMemoryBroadcastRepository implements BroadcastRepository { return Promise.resolve(record) } + createAnnouncementUnderCap( + input: BroadcastCreateInput, + cap: AnnouncementCap, + ): Promise { + const used = [...this.broadcasts.values()].filter( + (b) => + b.cleanupId === input.cleanupId && + b.kind === ANNOUNCEMENT_BROADCAST_KIND && + b.createdAt.getTime() >= cap.since.getTime(), + ).length + if (used >= cap.max) return Promise.resolve(null) + return this.create(input) + } + async createIfAbsent(input: BroadcastCreateInput): Promise { const exists = [...this.broadcasts.values()].some( (b) => diff --git a/services/api/src/services/host/broadcast-repository.ts b/services/api/src/services/host/broadcast-repository.ts index 71a6977d..ff6a5d25 100644 --- a/services/api/src/services/host/broadcast-repository.ts +++ b/services/api/src/services/host/broadcast-repository.ts @@ -71,9 +71,18 @@ export interface AudiencePageQuery { limit: number } +export interface AnnouncementCap { + since: Date + max: number +} + export interface BroadcastRepository { create(input: BroadcastCreateInput): Promise createIfAbsent(input: BroadcastCreateInput): Promise + createAnnouncementUnderCap( + input: BroadcastCreateInput, + cap: AnnouncementCap, + ): Promise findById(broadcastId: string): Promise findForEvent(cleanupId: string, broadcastId: string): Promise diff --git a/services/api/test/unit/host/announcement-service.test.ts b/services/api/test/unit/host/announcement-service.test.ts index fb7716da..229cc47a 100644 --- a/services/api/test/unit/host/announcement-service.test.ts +++ b/services/api/test/unit/host/announcement-service.test.ts @@ -240,6 +240,37 @@ describe("createEventAnnouncement", () => { ).rejects.toMatchObject({ code: "RATE_LIMITED" }) }) + it("admits exactly one of two concurrent creates that race for the last daily slot", async () => { + for (let i = 0; i < MAX_EVENT_ANNOUNCEMENTS_PER_DAY - 1; i += 1) { + await h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: `Body ${i}`, + audience: { kind: "all_registered" }, + }) + } + + const outcomes = await Promise.allSettled([ + h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: "Racer A", + audience: { kind: "all_registered" }, + }), + h.service.create(EVENT, HOST, { + id: EVENT, + bodyMd: "Racer B", + audience: { kind: "all_registered" }, + }), + ]) + + expect(outcomes.filter((o) => o.status === "fulfilled")).toHaveLength(1) + const rejected = outcomes.find((o) => o.status === "rejected") + expect(rejected).toBeDefined() + expect((rejected as PromiseRejectedResult).reason).toMatchObject({ code: "RATE_LIMITED" }) + expect(await h.repo.countAnnouncementsSince(EVENT, new Date(0))).toBe( + MAX_EVENT_ANNOUNCEMENTS_PER_DAY, + ) + }) + it("lets the host announce again once the 24h window has rolled past", async () => { for (let i = 0; i < MAX_EVENT_ANNOUNCEMENTS_PER_DAY; i += 1) { const dto = await h.service.create(EVENT, HOST, { diff --git a/services/api/test/unit/posts-routes.test.ts b/services/api/test/unit/posts-routes.test.ts index 6b5373e6..2b01bc77 100644 --- a/services/api/test/unit/posts-routes.test.ts +++ b/services/api/test/unit/posts-routes.test.ts @@ -840,7 +840,7 @@ describe("POST /posts (createPost)", () => { expect(h.repo.posts.get(res.json().id)?.reportId).toBe(reportId) }) - it("rate limits createPost at 120/min PER IP (the route carries its own config.rateLimit bucket)", async () => { + it("rate limits createPost at 12/min PER IP (the route carries its own config.rateLimit bucket)", async () => { const h = await makeHarness() const me = await h.signIn("burst@example.com", "Burst") const post = () => @@ -851,13 +851,9 @@ describe("POST /posts (createPost)", () => { payload: { kind: "post", body: "burst" }, }) - for (let i = 0; i < 120; i += 1) { + for (let i = 0; i < 12; i += 1) { expect((await post()).statusCode, `request ${i + 1}`).toBe(201) } - // Without the route bucket this would sit at the global 300/min and return a 121st 201. The bucket is - // keyed by IP (the inherited global keyGenerator), so it is shared by every user behind one exit - - // which is why it is 120 and not the 20-30 the other creates use: this endpoint carries thread - // replies, and a whole crew replying from one venue Wi-Fi must not 429 each other. expect((await post()).statusCode).toBe(429) }) diff --git a/services/api/test/unit/reverse-geocode-mapbox.test.ts b/services/api/test/unit/reverse-geocode-mapbox.test.ts index 0f933558..3922bcc5 100644 --- a/services/api/test/unit/reverse-geocode-mapbox.test.ts +++ b/services/api/test/unit/reverse-geocode-mapbox.test.ts @@ -45,6 +45,7 @@ describe("makeMapboxReverseGeocode", () => { const geocode = makeMapboxReverseGeocode({ token: "pk.test", fetchImpl: okFetch({ + feature_type: "address", name: "1 Main St", context: { place: { name: "Springfield" }, region: { region_code: "IL" }, country: { country_code: "us" } }, }), @@ -114,7 +115,18 @@ describe("mapboxPrecision", () => { expect(mapboxPrecision({ context: { address: { address_number: "123", name: "123 Main St" } } })).toBe( "street", ) - expect(mapboxPrecision({ name: "123 Main St" })).toBe("street") + expect(mapboxPrecision({ feature_type: "address", name: "123 Main St" })).toBe("street") + }) + + it("refuses street for a digit-leading name the response never called an address", () => { + expect(mapboxPrecision({ name: "123 Main St" })).toBeNull() + expect( + mapboxPrecision({ + feature_type: "poi", + name: "24 Hour Fitness", + context: { street: { name: "Sepulveda Blvd" } }, + }), + ).toBe("intersection") }) it("claims intersection for a named street with no number", () => { From dc9af49574f2fc550075c024cd9015fdab0b9a02 Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 22 Sep 2026 00:40:37 -0700 Subject: [PATCH 48/54] broadcast repo: email_verified column fix --- .../host/broadcast-repository.drizzle.ts | 12 +- .../host-broadcast-identity-pg.test.ts | 121 ++++++++++++++++++ .../test/unit/host-broadcast-columns.test.ts | 70 ++++++++++ 3 files changed, 197 insertions(+), 6 deletions(-) create mode 100644 services/api/test/integration/host-broadcast-identity-pg.test.ts create mode 100644 services/api/test/unit/host-broadcast-columns.test.ts diff --git a/services/api/src/services/host/broadcast-repository.drizzle.ts b/services/api/src/services/host/broadcast-repository.drizzle.ts index 9975c796..79030a1c 100644 --- a/services/api/src/services/host/broadcast-repository.drizzle.ts +++ b/services/api/src/services/host/broadcast-repository.drizzle.ts @@ -717,12 +717,12 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { { id: string email: string | null - email_verified_at: Date | null + email_verified: boolean | null display_name: string | null locale: string | null }[] >` - SELECT id, email, email_verified_at, display_name, locale + SELECT id, email, email_verified, display_name, locale FROM users WHERE id = ANY(${[...userIds]}::uuid[]) AND deleted_at IS NULL` for (const row of rows) { @@ -730,7 +730,7 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { out.set(row.id, { userId: row.id, email: row.email, - emailVerified: row.email_verified_at !== null, + emailVerified: row.email_verified === true, displayName, firstName: firstName(displayName), locale: row.locale, @@ -859,9 +859,9 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { async hostMessagingState(userId: string): Promise { const rows = await sql< - { suspended: boolean | null; email_verified_at: Date | null; created_at: Date }[] + { suspended: boolean | null; email_verified: boolean | null; created_at: Date }[] >` - SELECT um.host_messaging_suspended AS suspended, u.email_verified_at, u.created_at + SELECT um.host_messaging_suspended AS suspended, u.email_verified, u.created_at FROM users u LEFT JOIN user_moderation um ON um.user_id = u.id WHERE u.id = ${userId} AND u.deleted_at IS NULL @@ -870,7 +870,7 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { if (row === undefined) return null return { suspended: row.suspended === true, - emailVerified: row.email_verified_at !== null, + emailVerified: row.email_verified === true, accountCreatedAt: row.created_at, } }, diff --git a/services/api/test/integration/host-broadcast-identity-pg.test.ts b/services/api/test/integration/host-broadcast-identity-pg.test.ts new file mode 100644 index 00000000..3d8e136e --- /dev/null +++ b/services/api/test/integration/host-broadcast-identity-pg.test.ts @@ -0,0 +1,121 @@ +import { randomUUID } from "node:crypto" +import { afterAll, beforeAll, describe, expect, it } from "vitest" +import { withPg, type PgHarness } from "../helpers/pg.js" +import { makeDrizzleBroadcastRepository } from "../../src/services/host/broadcast-repository.drizzle.js" +import type { BroadcastRepository } from "../../src/services/host/broadcast-repository.js" + +const pg = await withPg() + +describe.skipIf(!pg)("broadcast host/member identity reads (integration)", () => { + let h: PgHarness + let repo: BroadcastRepository + + beforeAll(() => { + h = pg as PgHarness + repo = makeDrizzleBroadcastRepository(h.sql) + }) + + afterAll(async () => { + await h.teardown() + }) + + async function newUser(over: { + displayName: string + email?: string | null + emailVerified?: boolean + locale?: string + }): Promise { + const handle = `u${randomUUID().replace(/-/g, "").slice(0, 15)}` + const [row] = await h.sql<{ id: string }[]>` + INSERT INTO users (display_name, handle, email, email_verified, locale) + VALUES ( + ${over.displayName}, + ${handle}, + ${over.email ?? null}, + ${over.emailVerified ?? false}, + ${over.locale ?? "en"} + ) + RETURNING id` + return row!.id + } + + async function suspend(userId: string): Promise { + await h.sql` + INSERT INTO user_moderation (user_id, host_messaging_suspended) + VALUES (${userId}, true) + ON CONFLICT (user_id) DO UPDATE SET host_messaging_suspended = true` + } + + it("hostMessagingState runs against the real schema and reports a verified host", async () => { + const userId = await newUser({ + displayName: "Ada Host", + email: "ada@example.org", + emailVerified: true, + }) + const state = await repo.hostMessagingState(userId) + expect(state).not.toBeNull() + expect(state!.emailVerified).toBe(true) + expect(state!.suspended).toBe(false) + expect(state!.accountCreatedAt).toBeInstanceOf(Date) + }) + + it("hostMessagingState reports an UNVERIFIED host as unverified", async () => { + const userId = await newUser({ + displayName: "Unverified Host", + email: "nope@example.org", + emailVerified: false, + }) + const state = await repo.hostMessagingState(userId) + expect(state!.emailVerified).toBe(false) + }) + + it("hostMessagingState reflects a host_messaging_suspended moderation row", async () => { + const userId = await newUser({ + displayName: "Suspended Host", + email: "susp@example.org", + emailVerified: true, + }) + await suspend(userId) + const state = await repo.hostMessagingState(userId) + expect(state!.suspended).toBe(true) + expect(state!.emailVerified).toBe(true) + }) + + it("hostMessagingState is null for a soft-deleted user", async () => { + const userId = await newUser({ displayName: "Gone", email: "gone@example.org" }) + await h.sql`UPDATE users SET deleted_at = now() WHERE id = ${userId}` + expect(await repo.hostMessagingState(userId)).toBeNull() + }) + + it("memberContacts runs against the real schema and maps verification per user", async () => { + const verified = await newUser({ + displayName: "Grace Hopper", + email: "grace@example.org", + emailVerified: true, + locale: "es", + }) + const unverified = await newUser({ + displayName: "Alan Turing", + email: "alan@example.org", + emailVerified: false, + }) + const contacts = await repo.memberContacts([verified, unverified]) + expect(contacts.size).toBe(2) + const g = contacts.get(verified)! + expect(g.email).toBe("grace@example.org") + expect(g.emailVerified).toBe(true) + expect(g.displayName).toBe("Grace Hopper") + expect(g.firstName).toBe("Grace") + expect(g.locale).toBe("es") + const a = contacts.get(unverified)! + expect(a.emailVerified).toBe(false) + expect(a.email).toBe("alan@example.org") + }) + + it("memberContacts omits a soft-deleted user", async () => { + const userId = await newUser({ displayName: "Deleted Member", email: "del@example.org" }) + await h.sql`UPDATE users SET deleted_at = now() WHERE id = ${userId}` + const contacts = await repo.memberContacts([userId]) + expect(contacts.size).toBe(0) + }) +}) diff --git a/services/api/test/unit/host-broadcast-columns.test.ts b/services/api/test/unit/host-broadcast-columns.test.ts new file mode 100644 index 00000000..750aa3cf --- /dev/null +++ b/services/api/test/unit/host-broadcast-columns.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest" +import { makeFakeSql } from "../helpers/fake-sql.js" +import type { Sql } from "../../src/db/client.js" +import { makeDrizzleBroadcastRepository } from "../../src/services/host/broadcast-repository.drizzle.js" + +const USER_ID = "11111111-1111-1111-1111-111111111111" + +async function hostMessagingStateStatement(emailVerified: boolean) { + const fake = makeFakeSql([ + { + match: /host_messaging_suspended/, + rows: [{ suspended: false, email_verified: emailVerified, created_at: new Date(0) }], + }, + ]) + const repo = makeDrizzleBroadcastRepository(fake.sql as unknown as Sql) + const state = await repo.hostMessagingState(USER_ID) + const stmt = fake.statements[0] + expect(stmt, "hostMessagingState should emit a statement").toBeDefined() + return { sql: stmt!.sql, state } +} + +async function memberContactsStatement(emailVerified: boolean) { + const fake = makeFakeSql([ + { + match: /FROM users/, + rows: [ + { + id: USER_ID, + email: "host@example.org", + email_verified: emailVerified, + display_name: "Ada Host", + locale: "en", + }, + ], + }, + ]) + const repo = makeDrizzleBroadcastRepository(fake.sql as unknown as Sql) + const contacts = await repo.memberContacts([USER_ID]) + const stmt = fake.statements[0] + expect(stmt, "memberContacts should emit a statement").toBeDefined() + return { sql: stmt!.sql, contact: contacts.get(USER_ID) } +} + +describe("broadcast repository selects the columns users actually has", () => { + it("hostMessagingState reads email_verified, never the phantom email_verified_at", async () => { + const { sql } = await hostMessagingStateStatement(true) + expect(sql).not.toMatch(/email_verified_at/) + expect(sql).toMatch(/\bemail_verified\b/) + }) + + it("recipient identities read email_verified, never the phantom email_verified_at", async () => { + const { sql } = await memberContactsStatement(true) + expect(sql).not.toMatch(/email_verified_at/) + expect(sql).toMatch(/\bemail_verified\b/) + }) + + it("hostMessagingState maps the boolean, so an unverified host stays unverified", async () => { + const verified = await hostMessagingStateStatement(true) + const unverified = await hostMessagingStateStatement(false) + expect(verified.state?.emailVerified).toBe(true) + expect(unverified.state?.emailVerified).toBe(false) + }) + + it("recipient identities map the boolean, so an unverified member stays unverified", async () => { + const verified = await memberContactsStatement(true) + const unverified = await memberContactsStatement(false) + expect(verified.contact?.emailVerified).toBe(true) + expect(unverified.contact?.emailVerified).toBe(false) + }) +}) From ee0629e6a49474ba5fdcb9e7e1717e83aba2e27d Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 22 Sep 2026 01:05:27 -0700 Subject: [PATCH 49/54] adopt @civfix/shared 0.55.0 --- pnpm-lock.yaml | 14 +++++++------- services/api/package.json | 2 +- services/media-worker/package.json | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2eaed5fc..389e1231 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -65,8 +65,8 @@ importers: specifier: ^3.700.0 version: 3.1057.0 '@civfix/shared': - specifier: ^0.54.0 - version: 0.54.0 + specifier: ^0.55.0 + version: 0.55.0 '@fastify/cookie': specifier: ^11.0.1 version: 11.0.2 @@ -210,8 +210,8 @@ importers: specifier: workspace:* version: link:../api '@civfix/shared': - specifier: ^0.54.0 - version: 0.54.0 + specifier: ^0.55.0 + version: 0.55.0 '@sentry/node': specifier: ^10.73.0 version: 10.73.0(@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1)) @@ -414,8 +414,8 @@ packages: '@balena/dockerignore@1.0.2': resolution: {integrity: sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==} - '@civfix/shared@0.54.0': - resolution: {integrity: sha512-H9vFZ+t9w/4UidgUYBNApeuqE2q9vuzXD0BmhZmeWOHJ2SeKjL7iPZOBL3Zpj8NGn+tq9ClcueYDUq4IXQcavw==} + '@civfix/shared@0.55.0': + resolution: {integrity: sha512-f24XyngBWMV4L7ipARvX8oATSObFKkrEvCBQxJDobbpenHduZ3LkU6sC5uZfN/ZrsySiH0xcInvxgit966S3gA==} engines: {node: '>=20'} '@derhuerst/http-basic@8.2.4': @@ -4440,7 +4440,7 @@ snapshots: '@balena/dockerignore@1.0.2': {} - '@civfix/shared@0.54.0': + '@civfix/shared@0.55.0': dependencies: zod: 3.25.76 diff --git a/services/api/package.json b/services/api/package.json index be885540..f095cd11 100644 --- a/services/api/package.json +++ b/services/api/package.json @@ -60,7 +60,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.700.0", "@aws-sdk/s3-request-presigner": "^3.700.0", - "@civfix/shared": "^0.54.0", + "@civfix/shared": "^0.55.0", "@fastify/cookie": "^11.0.1", "@fastify/cors": "^10.0.1", "@fastify/helmet": "^12.0.1", diff --git a/services/media-worker/package.json b/services/media-worker/package.json index 095c71fe..71b07c8b 100644 --- a/services/media-worker/package.json +++ b/services/media-worker/package.json @@ -23,7 +23,7 @@ "@aws-sdk/client-s3": "^3.700.0", "@aws-sdk/s3-request-presigner": "^3.700.0", "@civfix/api": "workspace:*", - "@civfix/shared": "^0.54.0", + "@civfix/shared": "^0.55.0", "@sentry/node": "^10.73.0", "@smithy/node-http-handler": "4.7.5", "drizzle-orm": "^0.45.2", From cac6b915bb6264bad5b7f2e0530a26b3234a593a Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 22 Sep 2026 01:11:56 -0700 Subject: [PATCH 50/54] host analytics summary endpoint --- .../api/src/routes/host/analytics.routes.ts | 20 ++ .../host/analytics-repository.drizzle.ts | 185 ++++++++++++++ .../src/services/host/analytics-service.ts | 93 +++++++ .../services/host/event-analytics-service.ts | 1 - services/api/test/unit/host-analytics.test.ts | 226 ++++++++++++++++++ .../unit/host/event-analytics-service.test.ts | 62 ++++- services/api/test/unit/route-coverage.test.ts | 4 +- 7 files changed, 587 insertions(+), 4 deletions(-) diff --git a/services/api/src/routes/host/analytics.routes.ts b/services/api/src/routes/host/analytics.routes.ts index 4cd27377..95da3d3c 100644 --- a/services/api/src/routes/host/analytics.routes.ts +++ b/services/api/src/routes/host/analytics.routes.ts @@ -3,6 +3,7 @@ import { EventAnalyticsRequestSchema, GetEventAnalyticsRequestSchema, GetEventInsightsRequestSchema, + HostAnalyticsSummaryRequestSchema, HostedEventsAnalyticsRequestSchema, type AnalyticsRange, type PortfolioAnalyticsRange, @@ -169,4 +170,23 @@ export async function registerHostAnalyticsRoutes( .status(200) .send(await analytics().portfolio(userId, query.orgId ?? null, range, viewerScope)) }) + + route(app, "hostedEventsAnalyticsSummary", { config: { rateLimit: ANALYTICS_RATE_LIMIT } }, async (request, reply) => { + const userId = requireAuth(request) + const query = parse(HostAnalyticsSummaryRequestSchema, request.query) + let viewerScope = "self" + if (query.orgId !== undefined) { + const standing = await requireOrgCapability( + container.getDb().sql, + query.orgId, + userId, + "view_analytics", + ) + viewerScope = `org:${standing.orgRole ?? "none"}` + } + const range: AnalyticsRange = query.range ?? "30d" + reply + .status(200) + .send(await analytics().summary(userId, query.orgId ?? null, range, viewerScope)) + }) } diff --git a/services/api/src/services/host/analytics-repository.drizzle.ts b/services/api/src/services/host/analytics-repository.drizzle.ts index 7531e15f..1f633c84 100644 --- a/services/api/src/services/host/analytics-repository.drizzle.ts +++ b/services/api/src/services/host/analytics-repository.drizzle.ts @@ -1,3 +1,4 @@ +import { MAX_HOST_SUMMARY_EVENT_ROWS } from "@civfix/shared" import type { BroadcastKind, CleanupStatus, RegistrationSource } from "@civfix/shared" import type { DayCount, DayTimeCount, KeyCount } from "@civfix/shared/host" import type { Sql } from "../../db/client.js" @@ -8,6 +9,8 @@ export const INSIGHTS_TREND_LIMIT = 400 export const INSIGHTS_SOURCE_LIMIT = 10 +export const SUMMARY_DAY_ZONE = "UTC" + export interface EventKpiRow { registered: number checkedIn: number @@ -113,6 +116,46 @@ export interface PortfolioTotals { repeatAttendees: number } +export interface HostActivityTotals { + registrations: number + cancellations: number + hoursTotal: number + hoursVolunteers: number + reportsLinked: number + reportsResolved: number + postsCreated: number +} + +export const ZERO_HOST_ACTIVITY_TOTALS: HostActivityTotals = Object.freeze({ + registrations: 0, + cancellations: 0, + hoursTotal: 0, + hoursVolunteers: 0, + reportsLinked: 0, + reportsResolved: 0, + postsCreated: 0, +}) + +export interface HeldEventTotals { + events: number + registered: number + checkedIn: number + noShow: number +} + +export const ZERO_HELD_EVENT_TOTALS: HeldEventTotals = Object.freeze({ + events: 0, + registered: 0, + checkedIn: 0, + noShow: 0, +}) + +export interface HostSummarySignups { + daily: DayCount[] + byEvent: KeyCount[] + hoursByEvent: KeyCount[] +} + export interface AnalyticsRepository { eventKpis(cleanupId: string): Promise registrationsByDay(cleanupId: string, timezone: string, from: string, to: string): Promise @@ -139,6 +182,13 @@ export interface AnalyticsRepository { cleanupId: string, hostedEventIds: readonly string[], ): Promise + activityTotals(cleanupIds: readonly string[], from: Date, to: Date): Promise + heldEventTotals(cleanupIds: readonly string[], from: Date, to: Date): Promise + signupsByDayAcross( + cleanupIds: readonly string[], + fromDay: string, + toDay: string, + ): Promise } export function makeDrizzleAnalyticsRepository(sql: Sql): AnalyticsRepository { @@ -550,5 +600,140 @@ export function makeDrizzleAnalyticsRepository(sql: Sql): AnalyticsRepository { count: Number(row.n), })) }, + + async activityTotals(cleanupIds, from, to) { + if (cleanupIds.length === 0) return { ...ZERO_HOST_ACTIVITY_TOTALS } + const ids = [...cleanupIds] + const rows = await sql< + { + registrations: string + cancellations: string + hours_total: number + hours_volunteers: string + reports_linked: string + reports_resolved: string + posts_created: string + }[] + >` + SELECT + (SELECT count(*) FROM cleanup_registrations r + WHERE r.cleanup_id = ANY(${ids}::uuid[]) + AND r.status = 'registered' + AND r.registered_at >= ${from} AND r.registered_at < ${to})::text AS registrations, + (SELECT count(*) FROM cleanup_registrations r + WHERE r.cleanup_id = ANY(${ids}::uuid[]) + AND r.cancelled_at IS NOT NULL + AND r.cancelled_at >= ${from} AND r.cancelled_at < ${to})::text AS cancellations, + (SELECT COALESCE(sum(vh.hours), 0)::float8 FROM volunteer_hours vh + WHERE vh.cleanup_id = ANY(${ids}::uuid[]) + AND vh.source = 'event' AND vh.voided_at IS NULL + AND vh.created_at >= ${from} AND vh.created_at < ${to}) AS hours_total, + (SELECT count(DISTINCT vh.user_id) FROM volunteer_hours vh + WHERE vh.cleanup_id = ANY(${ids}::uuid[]) + AND vh.source = 'event' AND vh.voided_at IS NULL + AND vh.created_at >= ${from} AND vh.created_at < ${to})::text AS hours_volunteers, + (SELECT count(*) FROM cleanup_reports cr + WHERE cr.cleanup_id = ANY(${ids}::uuid[]) + AND cr.linked_at >= ${from} AND cr.linked_at < ${to})::text AS reports_linked, + (SELECT count(*) FROM cleanup_reports cr + JOIN reports rep ON rep.id = cr.report_id + WHERE cr.cleanup_id = ANY(${ids}::uuid[]) + AND cr.linked_at >= ${from} AND cr.linked_at < ${to} + AND rep.status = 'resolved')::text AS reports_resolved, + (SELECT count(*) FROM posts p + WHERE p.event_id = ANY(${ids}::uuid[]) + AND p.created_at >= ${from} AND p.created_at < ${to} + AND p.deleted_at IS NULL + AND p.reply_to_id IS NULL)::text AS posts_created` + const row = rows[0] + if (row === undefined) return { ...ZERO_HOST_ACTIVITY_TOTALS } + return { + registrations: Number(row.registrations), + cancellations: Number(row.cancellations), + hoursTotal: Number(row.hours_total), + hoursVolunteers: Number(row.hours_volunteers), + reportsLinked: Number(row.reports_linked), + reportsResolved: Number(row.reports_resolved), + postsCreated: Number(row.posts_created), + } + }, + + async heldEventTotals(cleanupIds, from, to) { + if (cleanupIds.length === 0) return { ...ZERO_HELD_EVENT_TOTALS } + const ids = [...cleanupIds] + const rows = await sql< + { events: string; registered: string; checked_in: string; no_show: string }[] + >` + WITH held AS ( + SELECT c.id + FROM cleanups c + WHERE c.id = ANY(${ids}::uuid[]) + AND COALESCE(c.completed_at, c.ends_at, c.scheduled_at) >= ${from} + AND COALESCE(c.completed_at, c.ends_at, c.scheduled_at) < ${to} + ) + SELECT (SELECT count(*) FROM held)::text AS events, + (SELECT count(*) FROM cleanup_registrations r + WHERE r.cleanup_id IN (SELECT id FROM held) + AND r.status = 'registered')::text AS registered, + (SELECT count(*) FROM cleanup_registration_seats s + WHERE s.cleanup_id IN (SELECT id FROM held) + AND s.checked_in_at IS NOT NULL)::text AS checked_in, + (SELECT count(*) FROM cleanup_registration_seats s + WHERE s.cleanup_id IN (SELECT id FROM held) + AND s.no_show_at IS NOT NULL)::text AS no_show` + const row = rows[0] + if (row === undefined) return { ...ZERO_HELD_EVENT_TOTALS } + return { + events: Number(row.events), + registered: Number(row.registered), + checkedIn: Number(row.checked_in), + noShow: Number(row.no_show), + } + }, + + async signupsByDayAcross(cleanupIds, fromDay, toDay) { + if (cleanupIds.length === 0) return { daily: [], byEvent: [], hoursByEvent: [] } + const ids = [...cleanupIds] + const [dailyRows, byEventRows, hoursRows] = await Promise.all([ + sql<{ day: string; n: string }[]>` + SELECT to_char((registered_at AT TIME ZONE ${SUMMARY_DAY_ZONE})::date, 'YYYY-MM-DD') AS day, + count(*)::text AS n + FROM cleanup_registrations + WHERE cleanup_id = ANY(${ids}::uuid[]) + AND status = 'registered' + AND (registered_at AT TIME ZONE ${SUMMARY_DAY_ZONE})::date BETWEEN ${fromDay}::date AND ${toDay}::date + GROUP BY 1 ORDER BY 1 LIMIT 400`, + sql<{ key: string; n: string }[]>` + SELECT c.title AS key, count(r.id)::text AS n + FROM cleanups c + JOIN cleanup_registrations r + ON r.cleanup_id = c.id + AND r.status = 'registered' + AND (r.registered_at AT TIME ZONE ${SUMMARY_DAY_ZONE})::date + BETWEEN ${fromDay}::date AND ${toDay}::date + WHERE c.id = ANY(${ids}::uuid[]) + GROUP BY c.id, c.title + ORDER BY 2 DESC, c.title ASC + LIMIT ${MAX_HOST_SUMMARY_EVENT_ROWS}`, + sql<{ key: string; n: string }[]>` + SELECT c.title AS key, round(sum(vh.hours))::text AS n + FROM cleanups c + JOIN volunteer_hours vh + ON vh.cleanup_id = c.id + AND vh.source = 'event' + AND vh.voided_at IS NULL + AND (vh.created_at AT TIME ZONE ${SUMMARY_DAY_ZONE})::date + BETWEEN ${fromDay}::date AND ${toDay}::date + WHERE c.id = ANY(${ids}::uuid[]) + GROUP BY c.id, c.title + ORDER BY sum(vh.hours) DESC, c.title ASC + LIMIT ${MAX_HOST_SUMMARY_EVENT_ROWS}`, + ]) + return { + daily: await dayCounts(dailyRows), + byEvent: byEventRows.map((row) => ({ key: row.key, count: Number(row.n) })), + hoursByEvent: hoursRows.map((row) => ({ key: row.key, count: Number(row.n) })), + } + }, } } diff --git a/services/api/src/services/host/analytics-service.ts b/services/api/src/services/host/analytics-service.ts index 416010ce..ac5ef014 100644 --- a/services/api/src/services/host/analytics-service.ts +++ b/services/api/src/services/host/analytics-service.ts @@ -9,6 +9,7 @@ import { type EventAnalyticsRegistrationsResponse, type EventAnalyticsSourcesResponse, type FunnelStep, + type HostAnalyticsSummaryResponse, type HostedEventsAnalyticsResponse, type Panel, type PortfolioAnalyticsRange, @@ -95,6 +96,12 @@ export interface AnalyticsService { range: PortfolioAnalyticsRange, viewerScope: string, ): Promise + summary( + userId: string, + organizationId: string | null, + range: AnalyticsRange, + viewerScope: string, + ): Promise } export function makeAnalyticsService(deps: AnalyticsServiceDeps): AnalyticsService { @@ -356,6 +363,92 @@ export function makeAnalyticsService(deps: AnalyticsServiceDeps): AnalyticsServi } }) }, + + summary(userId, organizationId, range, viewerScope) { + const scope = organizationId === null ? `host:${userId}` : `org:${organizationId}` + return cached("summary", scope, range, viewerScope, async () => { + const window = utcRangeWindow(RANGE_DAYS[range]) + const bounds = dayBounds(window) + const cleanupIds = await deps.analytics.hostedEventIds( + userId, + organizationId, + PORTFOLIO_EVENT_LIMIT, + ) + const envelope = { + generatedAt: now().toISOString(), + range, + k: ANALYTICS_SUPPRESSION_K, + window, + } + if (cleanupIds.length === 0) return emptySummary(envelope, window) + const [activity, held, signups, metricRows] = await Promise.all([ + deps.analytics.activityTotals(cleanupIds, bounds.from, bounds.to), + deps.analytics.heldEventTotals(cleanupIds, bounds.from, bounds.to), + deps.analytics.signupsByDayAcross(cleanupIds, window.from, window.to), + deps.metrics.readMany(cleanupIds, ["donation_clicks"], window.from, window.to), + ]) + return { + ...envelope, + activity: { + signups: activity.registrations, + cancellations: activity.cancellations, + hoursTotal: round2(activity.hoursTotal), + hoursVolunteers: activity.hoursVolunteers, + reportsLinked: activity.reportsLinked, + reportsResolved: activity.reportsResolved, + postsCreated: activity.postsCreated, + donationClicks: sumMetric(metricRows, "donation_clicks"), + }, + eventsHeld: { + count: held.events, + registered: held.registered, + checkIns: held.checkedIn, + noShows: held.noShow, + checkInRate: exactRate(held.checkedIn, held.registered), + }, + totals: { events: cleanupIds.length }, + signupsDaily: exactSeries(signups.daily, window), + byEvent: exactPanel(signups.byEvent), + hoursByEvent: exactPanel(signups.hoursByEvent), + } + }) + }, + } +} + +function dayBounds(window: DayRange): { from: Date; to: Date } { + const from = new Date(`${window.from}T00:00:00.000Z`) + const to = new Date(Date.parse(`${window.to}T00:00:00.000Z`) + 86_400_000) + return { from, to } +} + +function emptySummary( + envelope: Pick, + window: DayRange, +): HostAnalyticsSummaryResponse { + return { + ...envelope, + activity: { + signups: 0, + cancellations: 0, + hoursTotal: 0, + hoursVolunteers: 0, + reportsLinked: 0, + reportsResolved: 0, + postsCreated: 0, + donationClicks: 0, + }, + eventsHeld: { + count: 0, + registered: 0, + checkIns: 0, + noShows: 0, + checkInRate: exactRate(0, 0), + }, + totals: { events: 0 }, + signupsDaily: exactSeries([], window), + byEvent: exactPanel([]), + hoursByEvent: exactPanel([]), } } diff --git a/services/api/src/services/host/event-analytics-service.ts b/services/api/src/services/host/event-analytics-service.ts index 5dea79b0..617893e8 100644 --- a/services/api/src/services/host/event-analytics-service.ts +++ b/services/api/src/services/host/event-analytics-service.ts @@ -276,7 +276,6 @@ export function makeEventAnalyticsService( const donationClicks = sumMetric(metricRows, "donation_clicks") const steps: KeyCount[] = [ - { key: "page_views", count: pageViews }, { key: "signups", count: kpis.registered }, { key: "checked_in", count: kpis.checkedIn }, { key: "logged_hours", count: hours.attendeesCredited }, diff --git a/services/api/test/unit/host-analytics.test.ts b/services/api/test/unit/host-analytics.test.ts index 96ec591b..487c4538 100644 --- a/services/api/test/unit/host-analytics.test.ts +++ b/services/api/test/unit/host-analytics.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest" import { avatarGradient, + HostAnalyticsSummaryResponseSchema, HostedEventsAnalyticsResponseSchema, MAX_PORTFOLIO_TOP_VOLUNTEERS, } from "@civfix/shared" @@ -69,6 +70,31 @@ function analyticsRepo(overrides: Partial = {}): AnalyticsR returningAttendees: () => Promise.resolve({ seats: 0, ofRegistered: 0 }), topVolunteers: () => Promise.resolve([]), hoursTotals: () => Promise.resolve({ credited: 0, volunteersCredited: 0 }), + activityTotals: () => + Promise.resolve({ + registrations: 20, + cancellations: 2, + hoursTotal: 48.755, + hoursVolunteers: 11, + reportsLinked: 3, + reportsResolved: 1, + postsCreated: 7, + }), + heldEventTotals: () => + Promise.resolve({ events: 2, registered: 20, checkedIn: 12, noShow: 1 }), + signupsByDayAcross: () => + Promise.resolve({ + daily: [ + { day: "2026-02-08", count: 6 }, + { day: "2026-02-09", count: 2 }, + { day: "2026-02-10", count: 7 }, + ], + byEvent: [ + { key: "Beach Cleanup", count: 12 }, + { key: "Park Cleanup", count: 3 }, + ], + hoursByEvent: [{ key: "Beach Cleanup", count: 31 }], + }), ...overrides, } } @@ -654,3 +680,203 @@ describe("#110: portfolio hours and top volunteers", () => { expect(legacy.volunteersCredited).toBeUndefined() }) }) + +describe("hosted-events analytics summary", () => { + const OWNER = "00000000-0000-0000-0000-0000000000aa" + const ORG = "33333333-3333-4333-8333-333333333333" + + it("returns a payload the shared summary schema accepts", async () => { + const { service } = build() + const payload = await service.summary(OWNER, null, "30d", "self") + expect(() => HostAnalyticsSummaryResponseSchema.parse(payload)).not.toThrow() + expect(payload.k).toBe(5) + expect(payload.range).toBe("30d") + expect(payload.generatedAt).toBe(NOW.toISOString()) + }) + + it("spans the 30-day window inclusive of both end days", async () => { + const { service } = build() + const payload = await service.summary(OWNER, null, "30d", "self") + expect(payload.window).toEqual({ from: "2026-01-12", to: "2026-02-10" }) + expect(payload.signupsDaily).toHaveLength(30) + expect(payload.signupsDaily[0]?.day).toBe("2026-01-12") + expect(payload.signupsDaily.at(-1)?.day).toBe("2026-02-10") + }) + + it("hands the repositories the window's half-open instant bounds", async () => { + const asked: { from: Date; to: Date }[] = [] + const { service } = build({ + activityTotals: (_ids, from, to) => { + asked.push({ from, to }) + return Promise.resolve({ + registrations: 0, + cancellations: 0, + hoursTotal: 0, + hoursVolunteers: 0, + reportsLinked: 0, + reportsResolved: 0, + postsCreated: 0, + }) + }, + heldEventTotals: (_ids, from, to) => { + asked.push({ from, to }) + return Promise.resolve({ events: 0, registered: 0, checkedIn: 0, noShow: 0 }) + }, + }) + await service.summary(OWNER, null, "30d", "self") + expect(asked).toHaveLength(2) + for (const call of asked) { + expect(call.from.toISOString()).toBe("2026-01-12T00:00:00.000Z") + expect(call.to.toISOString()).toBe("2026-02-11T00:00:00.000Z") + } + }) + + it("carries the live activity counts straight through, sub-k included", async () => { + const { service } = build({}, [ + { cleanupId: EVENT, day: "2026-02-09", metric: "donation_clicks", bucket: "", value: 4 }, + ]) + const payload = await service.summary(OWNER, null, "30d", "self") + expect(payload.activity).toEqual({ + signups: 20, + cancellations: 2, + hoursTotal: 48.76, + hoursVolunteers: 11, + reportsLinked: 3, + reportsResolved: 1, + postsCreated: 7, + donationClicks: 4, + }) + }) + + it("publishes every signup day exactly, sub-k days included", async () => { + const { service } = build() + const payload = await service.summary(OWNER, null, "30d", "self") + const byDay = new Map(payload.signupsDaily.map((point) => [point.day, point])) + expect(byDay.get("2026-02-08")?.value).toBe(6) + expect(byDay.get("2026-02-09")?.value).toBe(2) + expect(byDay.get("2026-02-10")?.value).toBe(7) + expect(payload.signupsDaily.every((point) => point.suppressed === false)).toBe(true) + }) + + it("publishes the per-event panels exactly, the way the portfolio does", async () => { + const { service } = build() + const payload = await service.summary(OWNER, null, "30d", "self") + expect(payload.byEvent.panelSuppressed).toBe(false) + expect(payload.byEvent.rows.map((row) => [row.key, row.value])).toEqual([ + ["Beach Cleanup", 12], + ["Park Cleanup", 3], + ]) + expect(payload.hoursByEvent.rows.map((row) => row.value)).toEqual([31]) + expect(payload.byEvent.rows.every((row) => row.suppressed === false)).toBe(true) + }) + + it("derives the held-event check-in rate exactly, never suppressed above zero", async () => { + const { service } = build() + const payload = await service.summary(OWNER, null, "30d", "self") + expect(payload.eventsHeld).toMatchObject({ + count: 2, + registered: 20, + checkIns: 12, + noShows: 1, + }) + expect(payload.eventsHeld.checkInRate.value).toBeCloseTo(12 / 20, 4) + expect(payload.eventsHeld.checkInRate.numerator).toBe(12) + expect(payload.eventsHeld.checkInRate.denominator).toBe(20) + expect(payload.eventsHeld.checkInRate.suppressed).toBe(false) + }) + + it("keeps a sub-k held-event rate exact rather than hiding it", async () => { + const { service } = build({ + heldEventTotals: () => + Promise.resolve({ events: 1, registered: 3, checkedIn: 2, noShow: 1 }), + }) + const payload = await service.summary(OWNER, null, "30d", "self") + expect(payload.eventsHeld.checkInRate.value).toBeCloseTo(2 / 3, 4) + expect(payload.eventsHeld.checkInRate.suppressed).toBe(false) + }) + + it("leaves the rate null when nobody registered", async () => { + const { service } = build({ + heldEventTotals: () => + Promise.resolve({ events: 1, registered: 0, checkedIn: 0, noShow: 0 }), + }) + const payload = await service.summary(OWNER, null, "30d", "self") + expect(payload.eventsHeld.checkInRate.value).toBeNull() + expect(payload.eventsHeld.checkInRate.suppressed).toBe(true) + }) + + it("short-circuits a host with no events without touching the window reads", async () => { + const touched: string[] = [] + const { service } = build({ + hostedEventIds: () => Promise.resolve([]), + activityTotals: () => { + touched.push("activityTotals") + return Promise.resolve({ + registrations: 1, + cancellations: 1, + hoursTotal: 1, + hoursVolunteers: 1, + reportsLinked: 1, + reportsResolved: 1, + postsCreated: 1, + }) + }, + heldEventTotals: () => { + touched.push("heldEventTotals") + return Promise.resolve({ events: 1, registered: 1, checkedIn: 1, noShow: 1 }) + }, + signupsByDayAcross: () => { + touched.push("signupsByDayAcross") + return Promise.resolve({ daily: [], byEvent: [], hoursByEvent: [] }) + }, + }) + const payload = await service.summary(OWNER, null, "30d", "self") + + expect(touched).toEqual([]) + expect(() => HostAnalyticsSummaryResponseSchema.parse(payload)).not.toThrow() + expect(payload.totals.events).toBe(0) + expect(payload.eventsHeld).toMatchObject({ count: 0, registered: 0, checkIns: 0, noShows: 0 }) + expect(Object.values(payload.activity).every((value) => value === 0)).toBe(true) + expect(payload.byEvent.rows).toEqual([]) + expect(payload.hoursByEvent.rows).toEqual([]) + expect(payload.signupsDaily.every((point) => point.value === 0)).toBe(true) + }) + + it("scopes the hosted-event lookup to the organization when one is asked for", async () => { + const asked: Array = [] + const { service } = build({ + hostedEventIds: (_userId, organizationId) => { + asked.push(organizationId) + return Promise.resolve([EVENT]) + }, + }) + await service.summary(OWNER, ORG, "30d", "org:admin") + await service.summary(OWNER, null, "30d", "self") + expect(asked).toEqual([ORG, null]) + }) + + it("does not serve an org summary from the personal one's cache entry", async () => { + let calls = 0 + const { service } = build({ + hostedEventIds: () => { + calls += 1 + return Promise.resolve([EVENT]) + }, + }) + await service.summary(OWNER, null, "30d", "self") + await service.summary(OWNER, null, "30d", "self") + expect(calls).toBe(1) + await service.summary(OWNER, ORG, "30d", "org:admin") + expect(calls).toBe(2) + }) + + it("widens the window with the range", async () => { + const { service } = build() + expect((await service.summary(OWNER, null, "7d", "self")).window).toEqual({ + from: "2026-02-04", + to: "2026-02-10", + }) + expect((await service.summary(OWNER, null, "90d", "self")).window.from).toBe("2025-11-13") + expect((await service.summary(OWNER, null, "all", "self")).signupsDaily).toHaveLength(365) + }) +}) diff --git a/services/api/test/unit/host/event-analytics-service.test.ts b/services/api/test/unit/host/event-analytics-service.test.ts index f9f93bbd..b7e31ef3 100644 --- a/services/api/test/unit/host/event-analytics-service.test.ts +++ b/services/api/test/unit/host/event-analytics-service.test.ts @@ -89,6 +89,18 @@ function analyticsRepo(over: Partial = {}): AnalyticsReposi topVolunteers: () => Promise.resolve([]), hoursTotals: () => Promise.resolve({ credited: 0, volunteersCredited: 0 }), returningAttendees: () => Promise.resolve({ seats: 0, ofRegistered: 0 }), + activityTotals: () => + Promise.resolve({ + registrations: 0, + cancellations: 0, + hoursTotal: 0, + hoursVolunteers: 0, + reportsLinked: 0, + reportsResolved: 0, + postsCreated: 0, + }), + heldEventTotals: () => Promise.resolve({ events: 0, registered: 0, checkedIn: 0, noShow: 0 }), + signupsByDayAcross: () => Promise.resolve({ daily: [], byEvent: [], hoursByEvent: [] }), ...over, } } @@ -308,7 +320,6 @@ describe("getEventAnalytics", () => { it("keeps the full scope's funnel and breakdown panels", async () => { const payload = await service().analytics(EVENT, "full", VIEWER) expect(payload.reach.funnel.map((step) => step.step)).toEqual([ - "page_views", "signups", "checked_in", "logged_hours", @@ -318,6 +329,55 @@ describe("getEventAnalytics", () => { expect(payload.eventDay.bySlot?.rows).toHaveLength(1) }) + it("draws the funnel from the same live counts the KPI tiles show", async () => { + const payload = await service().analytics(EVENT, "full", VIEWER) + expect(payload.reach.funnel.map((step) => step.value)).toEqual([20, 12, 10]) + expect(payload.reach.funnel.every((step) => step.suppressed === false)).toBe(true) + expect(payload.kpis.signups).toBe(20) + expect(payload.kpis.checkedIn).toBe(12) + expect(payload.kpis.hoursVolunteers).toBe(10) + }) + + it("does not let an empty page-view rollup zero a funnel with live signups", async () => { + const payload = await service({ metrics: [] }).analytics(EVENT, "full", VIEWER) + expect(payload.kpis.pageViews).toBeNull() + expect(payload.reach.funnel.map((step) => step.value)).toEqual([20, 12, 10]) + }) + + it(`suppresses the whole funnel when its head is under k=${ANALYTICS_SUPPRESSION_K}`, async () => { + const thin = service({ + analytics: { + eventKpis: () => + Promise.resolve({ + registered: 4, + checkedIn: 2, + waitlisted: 0, + cancelled: 0, + noShow: 0, + capacity: 10, + }), + }, + }) + const payload = await thin.analytics(EVENT, "full", VIEWER) + expect(payload.reach.funnel.map((step) => step.step)).toEqual([ + "signups", + "checked_in", + "logged_hours", + ]) + expect(payload.reach.funnel.every((step) => step.value === null && step.suppressed)).toBe(true) + }) + + it("clamps a funnel step that exceeds the one above it", async () => { + const clamped = service({ + analytics: { + eventHoursTotals: () => + Promise.resolve({ credited: 90, attendeesCredited: 30, attendeesCheckedIn: 12 }), + }, + }) + const payload = await clamped.analytics(EVENT, "full", VIEWER) + expect(payload.reach.funnel.map((step) => step.value)).toEqual([20, 12, 12]) + }) + it("returns the host's own median comparison once there is enough history", async () => { const withHistory = service({ events: { diff --git a/services/api/test/unit/route-coverage.test.ts b/services/api/test/unit/route-coverage.test.ts index a232c54a..d95c2bca 100644 --- a/services/api/test/unit/route-coverage.test.ts +++ b/services/api/test/unit/route-coverage.test.ts @@ -233,8 +233,8 @@ describe("route-coverage: every shared endpoint is registered (offline boot smok }) } - it("covers ALL 330 endpoints in the registry (no endpoint skipped)", () => { - expect(Object.keys(endpoints).length).toBe(330) + it("covers ALL 331 endpoints in the registry (no endpoint skipped)", () => { + expect(Object.keys(endpoints).length).toBe(331) }) it("the discriminator is not vacuous: a bogus path IS detected as route-missing", async () => { From 430378d909b06a8b6206177eb210ea4244d288cb Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 22 Sep 2026 01:47:51 -0700 Subject: [PATCH 51/54] analytics summary: key event panels by id --- .../host/analytics-repository.drizzle.ts | 30 +++++++++---- .../src/services/host/analytics-service.ts | 22 +++++++--- services/api/test/unit/host-analytics.test.ts | 42 +++++++++++++++---- 3 files changed, 73 insertions(+), 21 deletions(-) diff --git a/services/api/src/services/host/analytics-repository.drizzle.ts b/services/api/src/services/host/analytics-repository.drizzle.ts index 1f633c84..665c56b2 100644 --- a/services/api/src/services/host/analytics-repository.drizzle.ts +++ b/services/api/src/services/host/analytics-repository.drizzle.ts @@ -150,10 +150,14 @@ export const ZERO_HELD_EVENT_TOTALS: HeldEventTotals = Object.freeze({ noShow: 0, }) +export interface LabeledKeyCount extends KeyCount { + label: string +} + export interface HostSummarySignups { daily: DayCount[] - byEvent: KeyCount[] - hoursByEvent: KeyCount[] + byEvent: LabeledKeyCount[] + hoursByEvent: LabeledKeyCount[] } export interface AnalyticsRepository { @@ -703,8 +707,8 @@ export function makeDrizzleAnalyticsRepository(sql: Sql): AnalyticsRepository { AND status = 'registered' AND (registered_at AT TIME ZONE ${SUMMARY_DAY_ZONE})::date BETWEEN ${fromDay}::date AND ${toDay}::date GROUP BY 1 ORDER BY 1 LIMIT 400`, - sql<{ key: string; n: string }[]>` - SELECT c.title AS key, count(r.id)::text AS n + sql<{ key: string; label: string; n: string }[]>` + SELECT c.id::text AS key, c.title AS label, count(r.id)::text AS n FROM cleanups c JOIN cleanup_registrations r ON r.cleanup_id = c.id @@ -713,10 +717,10 @@ export function makeDrizzleAnalyticsRepository(sql: Sql): AnalyticsRepository { BETWEEN ${fromDay}::date AND ${toDay}::date WHERE c.id = ANY(${ids}::uuid[]) GROUP BY c.id, c.title - ORDER BY 2 DESC, c.title ASC + ORDER BY 3 DESC, c.title ASC LIMIT ${MAX_HOST_SUMMARY_EVENT_ROWS}`, - sql<{ key: string; n: string }[]>` - SELECT c.title AS key, round(sum(vh.hours))::text AS n + sql<{ key: string; label: string; n: string }[]>` + SELECT c.id::text AS key, c.title AS label, round(sum(vh.hours))::text AS n FROM cleanups c JOIN volunteer_hours vh ON vh.cleanup_id = c.id @@ -731,8 +735,16 @@ export function makeDrizzleAnalyticsRepository(sql: Sql): AnalyticsRepository { ]) return { daily: await dayCounts(dailyRows), - byEvent: byEventRows.map((row) => ({ key: row.key, count: Number(row.n) })), - hoursByEvent: hoursRows.map((row) => ({ key: row.key, count: Number(row.n) })), + byEvent: byEventRows.map((row) => ({ + key: row.key, + label: row.label, + count: Number(row.n), + })), + hoursByEvent: hoursRows.map((row) => ({ + key: row.key, + label: row.label, + count: Number(row.n), + })), } }, } diff --git a/services/api/src/services/host/analytics-service.ts b/services/api/src/services/host/analytics-service.ts index ac5ef014..b2980591 100644 --- a/services/api/src/services/host/analytics-service.ts +++ b/services/api/src/services/host/analytics-service.ts @@ -33,7 +33,7 @@ import { type KeyCount, type SeriesClosure, } from "@civfix/shared/host" -import type { AnalyticsRepository } from "./analytics-repository.drizzle.js" +import type { AnalyticsRepository, LabeledKeyCount } from "./analytics-repository.drizzle.js" import { leaderboardEntryOf } from "../volunteer-hours-service.js" import type { MetricRow, MetricsRepository } from "./metrics-repository.drizzle.js" import { hostAnalyticsCacheKey, type HostAnalyticsCache } from "./host-analytics-cache.js" @@ -408,8 +408,8 @@ export function makeAnalyticsService(deps: AnalyticsServiceDeps): AnalyticsServi }, totals: { events: cleanupIds.length }, signupsDaily: exactSeries(signups.daily, window), - byEvent: exactPanel(signups.byEvent), - hoursByEvent: exactPanel(signups.hoursByEvent), + byEvent: labeledPanel(signups.byEvent), + hoursByEvent: labeledPanel(signups.hoursByEvent), } }) }, @@ -447,8 +447,8 @@ function emptySummary( }, totals: { events: 0 }, signupsDaily: exactSeries([], window), - byEvent: exactPanel([]), - hoursByEvent: exactPanel([]), + byEvent: labeledPanel([]), + hoursByEvent: labeledPanel([]), } } @@ -478,6 +478,18 @@ function exactPanel(rows: readonly KeyCount[]): Panel { } } +function labeledPanel(rows: readonly LabeledKeyCount[]): Panel { + return { + panelSuppressed: false, + rows: rows.map((row) => ({ + key: row.key, + label: row.label, + value: row.count, + suppressed: false, + })), + } +} + function exactRate(numerator: number, denominator: number): SuppressedRate { return toSuppressedRate(suppressRate(numerator, denominator, EXACT_K), numerator, denominator) } diff --git a/services/api/test/unit/host-analytics.test.ts b/services/api/test/unit/host-analytics.test.ts index 487c4538..b6761384 100644 --- a/services/api/test/unit/host-analytics.test.ts +++ b/services/api/test/unit/host-analytics.test.ts @@ -15,6 +15,8 @@ import type { AnalyticsRepository } from "../../src/services/host/analytics-repo import type { MetricsRepository } from "../../src/services/host/metrics-repository.drizzle.js" const EVENT = "00000000-0000-0000-0000-0000000000ee" +const EVENT_A = "11111111-1111-4111-8111-111111111111" +const EVENT_B = "22222222-2222-4222-8222-222222222222" const NOW = new Date("2026-02-10T00:00:00Z") function analyticsRepo(overrides: Partial = {}): AnalyticsRepository { @@ -90,10 +92,10 @@ function analyticsRepo(overrides: Partial = {}): AnalyticsR { day: "2026-02-10", count: 7 }, ], byEvent: [ - { key: "Beach Cleanup", count: 12 }, - { key: "Park Cleanup", count: 3 }, + { key: EVENT_A, label: "Beach Cleanup", count: 12 }, + { key: EVENT_B, label: "Park Cleanup", count: 3 }, ], - hoursByEvent: [{ key: "Beach Cleanup", count: 31 }], + hoursByEvent: [{ key: EVENT_A, label: "Beach Cleanup", count: 31 }], }), ...overrides, } @@ -762,14 +764,40 @@ describe("hosted-events analytics summary", () => { const { service } = build() const payload = await service.summary(OWNER, null, "30d", "self") expect(payload.byEvent.panelSuppressed).toBe(false) - expect(payload.byEvent.rows.map((row) => [row.key, row.value])).toEqual([ - ["Beach Cleanup", 12], - ["Park Cleanup", 3], + expect(payload.byEvent.rows.map((row) => [row.key, row.label, row.value])).toEqual([ + [EVENT_A, "Beach Cleanup", 12], + [EVENT_B, "Park Cleanup", 3], + ]) + expect(payload.hoursByEvent.rows.map((row) => [row.key, row.label, row.value])).toEqual([ + [EVENT_A, "Beach Cleanup", 31], ]) - expect(payload.hoursByEvent.rows.map((row) => row.value)).toEqual([31]) expect(payload.byEvent.rows.every((row) => row.suppressed === false)).toBe(true) }) + it("keeps per-event rows distinct when two events share a title", async () => { + const { service } = build({ + signupsByDayAcross: () => + Promise.resolve({ + daily: [], + byEvent: [ + { key: EVENT_A, label: "Test1", count: 5 }, + { key: EVENT_B, label: "Test1", count: 2 }, + ], + hoursByEvent: [ + { key: EVENT_A, label: "Test1", count: 9 }, + { key: EVENT_B, label: "Test1", count: 4 }, + ], + }), + }) + const payload = await service.summary(OWNER, null, "30d", "self") + expect(payload.byEvent.rows.map((row) => row.key)).toEqual([EVENT_A, EVENT_B]) + expect(new Set(payload.byEvent.rows.map((row) => row.key)).size).toBe(2) + expect(payload.byEvent.rows.map((row) => row.label)).toEqual(["Test1", "Test1"]) + expect(payload.hoursByEvent.rows.map((row) => row.key)).toEqual([EVENT_A, EVENT_B]) + expect(new Set(payload.hoursByEvent.rows.map((row) => row.key)).size).toBe(2) + expect(payload.hoursByEvent.rows.map((row) => row.value)).toEqual([9, 4]) + }) + it("derives the held-event check-in rate exactly, never suppressed above zero", async () => { const { service } = build() const payload = await service.summary(OWNER, null, "30d", "self") From 80484386c85868ed4f6297ff9c07fcf0c9d09deb Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 22 Sep 2026 02:09:16 -0700 Subject: [PATCH 52/54] =?UTF-8?q?per-event=20count=20kpis=20exact=20per=20?= =?UTF-8?q?DECISIONS=20=C2=A754?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/host/event-analytics-service.ts | 29 ++++++----- .../unit/host/event-analytics-service.test.ts | 52 +++++++++++++++++-- 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/services/api/src/services/host/event-analytics-service.ts b/services/api/src/services/host/event-analytics-service.ts index 617893e8..1ee5f663 100644 --- a/services/api/src/services/host/event-analytics-service.ts +++ b/services/api/src/services/host/event-analytics-service.ts @@ -23,7 +23,6 @@ import { eventPhase, funnel, seriesClosure, - suppressCount, suppressRate, type DayCount, type DayRange, @@ -128,8 +127,10 @@ function toFunnelSteps( })) } -function sumMetric(rows: readonly MetricRow[], metric: string): number { - return rows.filter((row) => row.metric === metric).reduce((acc, row) => acc + row.value, 0) +function metricTotal(rows: readonly MetricRow[], metric: string): number | null { + const matching = rows.filter((row) => row.metric === metric) + if (matching.length === 0) return null + return matching.reduce((acc, row) => acc + row.value, 0) } function seriesOf(rows: readonly MetricRow[], metric: string): DayCount[] { @@ -272,8 +273,8 @@ export function makeEventAnalyticsService( const registeredPublishable = closureAllowsTotal(closure) const cancellations = dailySeries(cancellationDays, window, { suppressPoints: true }) const viewsSeries = dailySeries(seriesOf(metricRows, "page_views"), window) - const pageViews = sumMetric(metricRows, "page_views") - const donationClicks = sumMetric(metricRows, "donation_clicks") + const pageViews = metricTotal(metricRows, "page_views") + const donationClicks = metricTotal(metricRows, "donation_clicks") const steps: KeyCount[] = [ { key: "signups", count: kpis.registered }, @@ -298,17 +299,17 @@ export function makeEventAnalyticsService( completedAt: clock.completedAt?.toISOString() ?? null, }, kpis: { - signups: registeredPublishable ? suppressCount(kpis.registered).value : null, + signups: kpis.registered, capacity: kpis.capacity, - waitlisted: suppressCount(kpis.waitlisted).value, - cancelled: suppressCount(kpis.cancelled).value, - checkedIn: suppressCount(kpis.checkedIn).value, - noShow: suppressCount(kpis.noShow).value, - walkUps: suppressCount(facts.walkUps).value, - pageViews: suppressCount(pageViews).value, + waitlisted: kpis.waitlisted, + cancelled: kpis.cancelled, + checkedIn: kpis.checkedIn, + noShow: kpis.noShow, + walkUps: facts.walkUps, + pageViews, uniqueViewers: null, shares: null, - donationClicks: suppressCount(donationClicks).value, + donationClicks, hoursTotal: hours.credited, hoursVolunteers: hours.attendeesCredited, reportsLinked: facts.reportsLinked, @@ -323,7 +324,7 @@ export function makeEventAnalyticsService( ? emptyRate() : toRate(kpis.registered, kpis.capacity), viewToSignup: - pageViews === 0 || !registeredPublishable + pageViews === null || pageViews === 0 || !registeredPublishable ? emptyRate() : toRate(kpis.registered, pageViews), waitlistConversion: toRate(waitlist.promoted, waitlist.joined), diff --git a/services/api/test/unit/host/event-analytics-service.test.ts b/services/api/test/unit/host/event-analytics-service.test.ts index b7e31ef3..a406d630 100644 --- a/services/api/test/unit/host/event-analytics-service.test.ts +++ b/services/api/test/unit/host/event-analytics-service.test.ts @@ -225,10 +225,12 @@ describe("getEventAnalytics", () => { expect(payload.kpis.shares).toBeNull() }) - it(`suppresses a head count under k=${ANALYTICS_SUPPRESSION_K} but publishes one at or above it`, async () => { + it(`publishes the host's own whole-event counts exactly, including under k=${ANALYTICS_SUPPRESSION_K}`, async () => { const payload = await service().analytics(EVENT, "full", VIEWER) - expect(payload.kpis.walkUps).toBeNull() - expect(payload.kpis.waitlisted).toBeNull() + expect(payload.kpis.walkUps).toBe(4) + expect(payload.kpis.waitlisted).toBe(3) + expect(payload.kpis.cancelled).toBe(2) + expect(payload.kpis.noShow).toBe(1) expect(payload.kpis.checkedIn).toBe(12) }) @@ -259,6 +261,7 @@ describe("getEventAnalytics", () => { expect(payload.rates.checkIn.value).toBeNull() expect(payload.rates.fill.suppressed).toBe(true) expect(payload.kpis.capacity).toBe(4) + expect(payload.kpis.signups).toBe(3) }) it("survives an event with no data at all and still parses", async () => { @@ -338,6 +341,49 @@ describe("getEventAnalytics", () => { expect(payload.kpis.hoursVolunteers).toBe(10) }) + it("publishes the whole-event signup count when the daily buckets are mostly under k", async () => { + const thinDays = service({ + analytics: { + eventClock: () => Promise.resolve(clock({ createdAt: new Date("2026-02-07T00:00:00Z") })), + eventKpis: () => + Promise.resolve({ + registered: 25, + checkedIn: 12, + waitlisted: 3, + cancelled: 2, + noShow: 1, + capacity: 40, + }), + registrationsByDay: () => + Promise.resolve([ + { day: "2026-02-07", count: 8 }, + { day: "2026-02-08", count: 7 }, + { day: "2026-02-09", count: 7 }, + { day: "2026-02-19", count: 1 }, + { day: "2026-02-20", count: 2 }, + ]), + }, + }) + + const payload = await thinDays.analytics(EVENT, "full", VIEWER) + expect(payload.kpis.signups).toBe(25) + expect(payload.reach.funnel[0]?.value).toBe(25) + expect(payload.signups.daily).toHaveLength(14) + expect(payload.signups.daily.filter((point) => point.suppressed)).toHaveLength(11) + expect(payload.signups.cumulative.every((point) => point.suppressed)).toBe(true) + expect(payload.signups.bySlot?.panelSuppressed).toBe(true) + expect(payload.rates.checkIn.suppressed).toBe(true) + }) + + it("keeps rollup counts exact when the rollup has rows and null when it has none", async () => { + const sparse = service({ + metrics: [{ day: "2026-02-18", metric: "page_views", bucket: "", value: 3 }], + }) + const payload = await sparse.analytics(EVENT, "full", VIEWER) + expect(payload.kpis.pageViews).toBe(3) + expect(payload.kpis.donationClicks).toBeNull() + }) + it("does not let an empty page-view rollup zero a funnel with live signups", async () => { const payload = await service({ metrics: [] }).analytics(EVENT, "full", VIEWER) expect(payload.kpis.pageViews).toBeNull() From 1983573758bd16a2c273488b04a43cc235218886 Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 22 Sep 2026 02:14:24 -0700 Subject: [PATCH 53/54] ws lifecycle test: deterministic revalidation timing --- .../api/test/unit/ws-socket-lifecycle.test.ts | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/services/api/test/unit/ws-socket-lifecycle.test.ts b/services/api/test/unit/ws-socket-lifecycle.test.ts index 0bf29993..7ed1fcee 100644 --- a/services/api/test/unit/ws-socket-lifecycle.test.ts +++ b/services/api/test/unit/ws-socket-lifecycle.test.ts @@ -186,10 +186,21 @@ function baseOpts(extra: Partial = {}): GatewayOptions { } } +const realSetTimeout = globalThis.setTimeout +const realNowMs = performance.now.bind(performance) + +const SETTLE_BUDGET_MS = 10_000 + function flush(): Promise { return new Promise((r) => setTimeout(r, 0)) } +function realTick(): Promise { + return new Promise((resolve) => { + realSetTimeout(resolve, 1) + }) +} + async function microflush(): Promise { for (let i = 0; i < 20; i += 1) await Promise.resolve() } @@ -201,13 +212,18 @@ async function settle(): Promise { } } -async function settleUntil(done: () => boolean, maxTicks = 200): Promise { - for (let i = 0; i < maxTicks; i += 1) { +async function settleUntil(done: () => boolean, budgetMs = SETTLE_BUDGET_MS): Promise { + const deadline = realNowMs() + budgetMs + for (;;) { if (done()) return if (vi.isFakeTimers()) await vi.advanceTimersByTimeAsync(1) else await flush() + if (done()) return + if (realNowMs() >= deadline) { + throw new Error("settleUntil: condition never became true") + } + await realTick() } - if (!done()) throw new Error("settleUntil: condition never became true") } describe("frames sent during the async handshake are buffered, not dropped", () => { @@ -857,7 +873,7 @@ describe("H2: a ?ticket socket is re-validated against session revocation, exact ) const socket = new MockSocket() handler(socket as unknown as WebSocket, ticketRequest(ticket)) - await settle() + await settleUntil(() => socket.closes.length > 0) expect(socket.closes).toHaveLength(1) expect(socket.closes[0]?.reason).toBe("unauthenticated") From 6edbf7f1f766bfccf99e3a8b7e1b34e727bf3c79 Mon Sep 17 00:00:00 2001 From: Theo Date: Tue, 22 Sep 2026 02:16:26 -0700 Subject: [PATCH 54/54] =?UTF-8?q?overview=20counts=20exact=20per=20DECISIO?= =?UTF-8?q?NS=20=C2=A754?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/services/host/analytics-service.ts | 30 +++++++----- services/api/test/unit/host-analytics.test.ts | 49 +++++++++++++++++-- 2 files changed, 61 insertions(+), 18 deletions(-) diff --git a/services/api/src/services/host/analytics-service.ts b/services/api/src/services/host/analytics-service.ts index b2980591..c826b437 100644 --- a/services/api/src/services/host/analytics-service.ts +++ b/services/api/src/services/host/analytics-service.ts @@ -146,7 +146,7 @@ export function makeAnalyticsService(deps: AnalyticsServiceDeps): AnalyticsServi return cached("overview", cleanupId, range, viewerScope, async () => { const timezone = await timezoneOf(cleanupId) const window = eventRangeWindow(RANGE_DAYS[range], timezone) - const [kpis, metricRows, registrationDays, cancellationDays] = await Promise.all([ + const [kpis, metricRows, registrationDays] = await Promise.all([ deps.analytics.eventKpis(cleanupId), deps.metrics.read( cleanupId, @@ -155,14 +155,12 @@ export function makeAnalyticsService(deps: AnalyticsServiceDeps): AnalyticsServi window.to, ), deps.analytics.registrationsByDay(cleanupId, timezone, window.from, window.to), - deps.analytics.cancellationsByDay(cleanupId, timezone, window.from, window.to), ]) const registeredPublishable = closureAllowsTotal(closureOf(registrationDays, window)) - const cancelledPublishable = closureAllowsTotal(closureOf(cancellationDays, window)) - const pageViews = sumMetric(metricRows, "page_views") - const donationClicks = sumMetric(metricRows, "donation_clicks") + const pageViews = metricTotal(metricRows, "page_views") + const donationClicks = metricTotal(metricRows, "donation_clicks") const steps: KeyCount[] = [ - { key: "page_views", count: pageViews }, + ...(pageViews === null ? [] : [{ key: "page_views", count: pageViews }]), { key: "registered", count: kpis.registered }, { key: "checked_in", count: kpis.checkedIn }, ] @@ -170,14 +168,14 @@ export function makeAnalyticsService(deps: AnalyticsServiceDeps): AnalyticsServi return { ...envelope(range), kpis: { - registered: registeredPublishable ? suppressCount(kpis.registered).value : null, - checkedIn: suppressCount(kpis.checkedIn).value, - waitlisted: suppressCount(kpis.waitlisted).value, - cancelled: cancelledPublishable ? suppressCount(kpis.cancelled).value : null, - noShow: suppressCount(kpis.noShow).value, + registered: kpis.registered, + checkedIn: kpis.checkedIn, + waitlisted: kpis.waitlisted, + cancelled: kpis.cancelled, + noShow: kpis.noShow, capacity: kpis.capacity, - pageViews: suppressCount(pageViews).value, - donationClicks: suppressCount(donationClicks).value, + pageViews, + donationClicks, }, checkInRate: registeredPublishable ? toRate(kpis.checkedIn, kpis.registered) @@ -527,6 +525,12 @@ function sumMetric(rows: readonly MetricRow[], metric: string, bucket?: string): .reduce((acc, row) => acc + row.value, 0) } +function metricTotal(rows: readonly MetricRow[], metric: string): number | null { + const matching = rows.filter((row) => row.metric === metric) + if (matching.length === 0) return null + return matching.reduce((acc, row) => acc + row.value, 0) +} + function seriesOf(rows: readonly MetricRow[], metric: string): DayCount[] { const byDay = new Map() for (const row of rows) { diff --git a/services/api/test/unit/host-analytics.test.ts b/services/api/test/unit/host-analytics.test.ts index b6761384..0ddf7003 100644 --- a/services/api/test/unit/host-analytics.test.ts +++ b/services/api/test/unit/host-analytics.test.ts @@ -154,7 +154,27 @@ describe("host analytics envelopes", () => { expect(payload).not.toMatch(/"clicks"/) }) - it("suppresses a count below k rather than showing it", async () => { + it("publishes the host's own whole-event counts exactly, including under k", async () => { + const { service } = build({ + eventKpis: () => + Promise.resolve({ + registered: 4, + checkedIn: 2, + waitlisted: 3, + cancelled: 1, + noShow: 2, + capacity: 10, + }), + }) + const payload = await service.overview(EVENT, "30d", "organizer:none") + expect(payload.kpis.registered).toBe(4) + expect(payload.kpis.checkedIn).toBe(2) + expect(payload.kpis.waitlisted).toBe(3) + expect(payload.kpis.cancelled).toBe(1) + expect(payload.kpis.noShow).toBe(2) + }) + + it("keeps the rates suppressed even though the counts behind them publish", async () => { const { service } = build({ eventKpis: () => Promise.resolve({ @@ -167,8 +187,27 @@ describe("host analytics envelopes", () => { }), }) const payload = await service.overview(EVENT, "30d", "organizer:none") - expect(payload.kpis.registered).toBeNull() - expect(payload.kpis.checkedIn).toBeNull() + expect(payload.checkInRate.suppressed).toBe(true) + expect(payload.checkInRate.value).toBeNull() + expect(payload.checkInRate.denominator).toBeNull() + expect(payload.noShowRate.suppressed).toBe(true) + }) + + it("reports a rollup count exactly when it has rows and null when it has none", async () => { + const { service } = build({}, [ + { cleanupId: EVENT, day: "2026-02-09", metric: "page_views", bucket: "all", value: 3 }, + ]) + const payload = await service.overview(EVENT, "30d", "organizer:none") + expect(payload.kpis.pageViews).toBe(3) + expect(payload.kpis.donationClicks).toBeNull() + }) + + it("does not let an empty page-view rollup zero a funnel with live signups", async () => { + const { service } = build() + const payload = await service.overview(EVENT, "30d", "organizer:none") + expect(payload.kpis.pageViews).toBeNull() + expect(payload.funnel.map((step) => step.step)).toEqual(["registered", "checked_in"]) + expect(payload.funnel.map((step) => step.value)).toEqual([20, 12]) }) it("shows a count at exactly k", async () => { @@ -457,7 +496,7 @@ describe("host analytics envelopes", () => { expect(derivable.has(2)).toBe(false) }) - it("withholds the registered KPI that would close the same chain the series left open", async () => { + it("publishes the registered KPI while the series the chain left open stays closed", async () => { const registrationsByDay = () => Promise.resolve([ { day: "2026-02-08", count: 6 }, @@ -480,7 +519,7 @@ describe("host analytics envelopes", () => { PAGE_VIEW_ROWS, ) const overview = await service.overview(EVENT, "30d", "organizer:none") - expect(overview.kpis.registered).toBeNull() + expect(overview.kpis.registered).toBe(15) expect(overview.checkInRate.suppressed).toBe(true) expect(overview.checkInRate.denominator).toBeNull() expect(overview.noShowRate.denominator).toBeNull()