diff --git a/services/api/src/routes/admin/_route-utils.ts b/services/api/src/routes/admin/_route-utils.ts index 7d75b4a9..cea553d3 100644 --- a/services/api/src/routes/admin/_route-utils.ts +++ b/services/api/src/routes/admin/_route-utils.ts @@ -1,4 +1,4 @@ -import { AppError, IdSchema, type AdminOkResponse } from "@civfix/shared" +import { AppError, IdSchema, type AdminOkResponse, type RoomKind } from "@civfix/shared" import type { FastifyBaseLogger, FastifyInstance, FastifyReply, FastifyRequest } from "fastify" import type { ZodTypeAny, z } from "zod" import { parse } from "../_validate.js" @@ -15,6 +15,9 @@ import { export { parse } +// A one-message page around the target is the smallest read that still returns a tombstoned message. +const TARGET_ONLY = 1 + // The columns these feed are `uuid`: a malformed id must be a 422 here rather than reach SQL, where // `invalid input syntax for type uuid` becomes a hidden 500 and a GlitchTip capture. function idSegment(value: unknown, field: string): string { @@ -90,18 +93,22 @@ export function makeContainerMessageUpdateAnnouncer( const presign = makePrivateMediaPresigner(container.storage) const chatRepo = makeDrizzleChatRepository(sql, presign) const dmRepo = makeDrizzleDmRepository(sql, presign) - const TARGET_ONLY = 1 + const historyAround = (kind: RoomKind, roomId: string, messageId: string) => { + switch (kind) { + case "dm": + return dmRepo.history(roomId, undefined, TARGET_ONLY, null, messageId) + case "report": + return chatRepo.reportHistory(roomId, undefined, TARGET_ONLY, null, messageId) + case "group": + return chatRepo.groupHistory(roomId, undefined, TARGET_ONLY, null, messageId) + default: + return chatRepo.history(roomId, undefined, TARGET_ONLY, null, messageId) + } + } return makeMessageUpdateAnnouncer({ findRoom: (messageId) => findMessageRoom(sql, messageId), loadMessage: async (kind, roomId, messageId) => { - const page = - kind === "dm" - ? await dmRepo.history(roomId, undefined, TARGET_ONLY, null, messageId) - : kind === "report" - ? await chatRepo.reportHistory(roomId, undefined, TARGET_ONLY, null, messageId) - : kind === "group" - ? await chatRepo.groupHistory(roomId, undefined, TARGET_ONLY, null, messageId) - : await chatRepo.history(roomId, undefined, TARGET_ONLY, null, messageId) + const page = await historyAround(kind, roomId, messageId) return page.items.find((m) => m.id === messageId) ?? null }, broadcast: (kind, roomId, message) => diff --git a/services/api/src/routes/admin/auth.routes.ts b/services/api/src/routes/admin/auth.routes.ts index 10de9613..282e9ca4 100644 --- a/services/api/src/routes/admin/auth.routes.ts +++ b/services/api/src/routes/admin/auth.routes.ts @@ -33,6 +33,8 @@ import { route } from "../../versioning/route.js" const ADMIN_AUTH_RATE_LIMIT = { max: 10, timeWindow: "1 minute" } as const const UNNAMED_OPERATOR_DISPLAY_NAME = "Operator" const UNAUTHORIZED_OPERATOR_MESSAGE = "This account is not authorized for the operator dashboard." +const CF_ACCESS_JWT_HEADER = "cf-access-jwt-assertion" +const LOGIN_VIA_CF_ACCESS = "cf-access" export interface AdminAuthOverrides { auditSink(input: WriteAuditInput): Promise @@ -68,7 +70,7 @@ export async function registerAdminAuthRoutes( actorId: user.id, action: "operator.login_denied", target: `user:${user.id}`, - meta: { email: user.email, ...detail, via: "cf-access" }, + meta: { email: user.email, ...detail, via: LOGIN_VIA_CF_ACCESS }, }).catch((err: unknown) => { request.log.warn({ err }, "operator.login_denied audit write failed") }) @@ -107,7 +109,7 @@ export async function registerAdminAuthRoutes( actorId: operator.id, action: "operator.login", target: `user:${operator.id}`, - meta: { email: operator.email, via: "cf-access" }, + meta: { email: operator.email, via: LOGIN_VIA_CF_ACCESS }, }) return operator } @@ -174,7 +176,7 @@ async function verifyHeader( verify: VerifyAccessJwt, request: FastifyRequest, ): Promise { - const token = request.headers["cf-access-jwt-assertion"] + const token = request.headers[CF_ACCESS_JWT_HEADER] if (typeof token !== "string" || token.length === 0) throw AppError.unauthorized() try { return await verify(token) diff --git a/services/api/src/routes/admin/broadcasts.routes.ts b/services/api/src/routes/admin/broadcasts.routes.ts index eed5be8c..866b4700 100644 --- a/services/api/src/routes/admin/broadcasts.routes.ts +++ b/services/api/src/routes/admin/broadcasts.routes.ts @@ -27,11 +27,20 @@ declare module "fastify" { } } -export const ADMIN_BROADCAST_DEFAULT_LIMIT = 50 +const ADMIN_BROADCAST_DEFAULT_LIMIT = 50 -export const ADMIN_HOST_DEFAULT_LIMIT = 50 +const ADMIN_HOST_DEFAULT_LIMIT = 50 -export const ADMIN_HOST_DEFAULT_WINDOW_DAYS = 30 +const ADMIN_HOST_DEFAULT_WINDOW_DAYS = 30 + +const DAY_MS = 24 * 60 * 60 * 1000 + +// Enough of the digest to tell subjects apart in the console without echoing the subject text itself. +const SUBJECT_HASH_HEX_LENGTH = 16 + +function hashSubject(subject: string): string { + return createHash("sha256").update(subject).digest("hex").slice(0, SUBJECT_HASH_HEX_LENGTH) +} function mergeParams(request: FastifyRequest): Record { const params = (request.params ?? {}) as Record @@ -69,10 +78,7 @@ export async function registerAdminBroadcastRoutes( eventTitle: row.eventTitle, kind: row.kind, status: row.status, - subjectHash: - row.subject === null - ? null - : createHash("sha256").update(row.subject).digest("hex").slice(0, 16), + subjectHash: row.subject === null ? null : hashSubject(row.subject), createdBy: row.createdBy === null ? null @@ -103,7 +109,7 @@ export async function registerAdminBroadcastRoutes( const rows = await broadcastRepo().listAdminHosts({ ...(query.q !== undefined ? { q: query.q } : {}), ...(query.suspended !== undefined ? { suspended: query.suspended } : {}), - windowStart: new Date(Date.now() - windowDays * 24 * 60 * 60 * 1000), + windowStart: new Date(Date.now() - windowDays * DAY_MS), cursor: parseKeysetCursor(query.cursor, { direction: "desc" }), limit: limit + 1, }) diff --git a/services/api/src/routes/admin/discovery.routes.ts b/services/api/src/routes/admin/discovery.routes.ts index a4f9bcf2..c50c626f 100644 --- a/services/api/src/routes/admin/discovery.routes.ts +++ b/services/api/src/routes/admin/discovery.routes.ts @@ -29,6 +29,9 @@ import { } from "../../services/admin/discovery-service.js" import { makeDrizzleDiscoveryRepository } from "../../services/admin/discovery-repository.drizzle.js" +const FALLBACK_OPERATOR_LABEL = "operator" +const ACTOR_ID_LABEL_CHARS = 8 + /** Test-only: an in-memory repo so the whole HTTP flow runs offline. */ export interface DiscoveryRouteOverrides { repo: DiscoveryRepository @@ -103,9 +106,9 @@ export async function registerAdminDiscoveryRoutes( async function operatorLabel(app: FastifyInstance, actorId: string): Promise { try { const user = await app.authServices.users.findById(actorId) - if (!user) return "operator" - return user.handle ?? user.displayName ?? user.email ?? actorId.slice(0, 8) + if (!user) return FALLBACK_OPERATOR_LABEL + return user.handle ?? user.displayName ?? user.email ?? actorId.slice(0, ACTOR_ID_LABEL_CHARS) } catch { - return "operator" + return FALLBACK_OPERATOR_LABEL } } diff --git a/services/api/src/routes/admin/gov.routes.ts b/services/api/src/routes/admin/gov.routes.ts index 15deb4b2..81ac2b94 100644 --- a/services/api/src/routes/admin/gov.routes.ts +++ b/services/api/src/routes/admin/gov.routes.ts @@ -27,7 +27,7 @@ import { } from "../../services/admin/gov-claims-service.js" import { makeDrizzleGovClaimsRepository } from "../../services/admin/gov-claims-repository.drizzle.js" import type { RevokeAllSessions } from "../../services/admin/role-change.js" -import type { UserStore } from "../../auth/stores.js" +import type { UserRecord, UserStore } from "../../auth/stores.js" export interface GovClaimsRouteOverrides { repo: GovClaimsRepository @@ -42,22 +42,22 @@ declare module "fastify" { } } +function toProvisionedUser(user: UserRecord): ProvisionedUser { + return { id: user.id, email: user.email, role: user.role, emailVerified: user.emailVerified } +} + function provisionerFromUserStore(store: UserStore): UserProvisioner { const norm = (email: string): string => email.trim().toLowerCase() return { async findByEmail(email: string): Promise { const user = await store.findByEmail(norm(email)) - return user - ? { id: user.id, email: user.email, role: user.role, emailVerified: user.emailVerified } - : null + return user ? toProvisionedUser(user) : null }, async create(email: string, displayName: string): Promise { - const user = await store.create(norm(email), { displayName, role: "citizen" }) - return { id: user.id, email: user.email, role: user.role, emailVerified: user.emailVerified } + return toProvisionedUser(await store.create(norm(email), { displayName, role: "citizen" })) }, async setRole(id: string, role: Role): Promise { - const user = await store.setRole(id, role) - return { id: user.id, email: user.email, role: user.role, emailVerified: user.emailVerified } + return toProvisionedUser(await store.setRole(id, role)) }, } } diff --git a/services/api/src/routes/admin/inbox.routes.ts b/services/api/src/routes/admin/inbox.routes.ts index b4571d77..e7091175 100644 --- a/services/api/src/routes/admin/inbox.routes.ts +++ b/services/api/src/routes/admin/inbox.routes.ts @@ -26,6 +26,8 @@ import { /** Defends against a crafted mail with thousands of parts. */ const MAX_INBOX_ATTACHMENTS = 50 +const INBOUND_EMAIL_NOT_FOUND = "Inbound email not found." + /** Test-only: an in-memory repo and a fake Storage so the HTTP flow runs offline with no DB and no R2. */ export interface AdminInboxRouteOverrides { repo: InboundRepository @@ -62,7 +64,7 @@ export async function registerAdminInboxRoutes( route(app, "getInboxMessage", async (request, reply) => { const { id } = idParam(request) const dto = await repo().get(id) - if (dto === null) throw AppError.notFound("Inbound email not found.") + if (dto === null) throw AppError.notFound(INBOUND_EMAIL_NOT_FOUND) await auditRead(request, container, requireOperator(request), { action: "inbox.message_viewed", target: `inbound_email:${id}`, @@ -92,7 +94,7 @@ export async function registerAdminInboxRoutes( const operatorId = requireOperator(request) const { id, body } = parseBodyWithId(SetInboxStatusRequestSchema, request) const ok = await repo().setStatus(id, body.status, operatorId) - if (!ok) throw AppError.notFound("Inbound email not found.") + if (!ok) throw AppError.notFound(INBOUND_EMAIL_NOT_FOUND) sendOk(reply) }) } diff --git a/services/api/src/routes/admin/legal.routes.ts b/services/api/src/routes/admin/legal.routes.ts index 3fe5740d..3ee77b3a 100644 --- a/services/api/src/routes/admin/legal.routes.ts +++ b/services/api/src/routes/admin/legal.routes.ts @@ -6,10 +6,8 @@ import { legalDocumentVersions } from "../../services/legal-service.js" export async function registerAdminLegalRoutes( app: FastifyInstance, - container: Container, + _container: Container, ): Promise { - void container - route(app, "adminGetLegalVersions", async (_request, reply) => { const payload: GetLegalVersionsResponse = { documents: legalDocumentVersions(), diff --git a/services/api/src/routes/admin/pages.routes.ts b/services/api/src/routes/admin/pages.routes.ts index cc9c0d50..3853e9ab 100644 --- a/services/api/src/routes/admin/pages.routes.ts +++ b/services/api/src/routes/admin/pages.routes.ts @@ -24,7 +24,7 @@ import { makeDrizzleHostRegistrationRepository } from "../../services/host/regis import { toEventPageDTO } from "../../services/host/registration-dto.js" import { makeEventMediaPresigner } from "../../services/host/event-media.js" -export const ADMIN_EVENT_PAGE_DEFAULT_LIMIT = 50 +const ADMIN_EVENT_PAGE_DEFAULT_LIMIT = 50 type RecordAudit = (entry: WriteAuditInput) => Promise diff --git a/services/api/src/routes/admin/report-chat.routes.ts b/services/api/src/routes/admin/report-chat.routes.ts index e7e95926..892f1b10 100644 --- a/services/api/src/routes/admin/report-chat.routes.ts +++ b/services/api/src/routes/admin/report-chat.routes.ts @@ -6,7 +6,7 @@ import { type ChatHistoryResponse, } from "@civfix/shared" import type { FastifyInstance } from "fastify" -import { perIdentity } from "../../plugins/rate-limit.js" +import { perIdentity, type RouteRateLimitSpec } from "../../plugins/rate-limit.js" import type { Container } from "../../di.js" import { route } from "../../versioning/route.js" import { @@ -48,17 +48,15 @@ declare module "fastify" { } } -export const ADMIN_REPORT_CHAT_SEND_RATE_LIMIT = perIdentity({ +const OPERATOR_CHAT_WRITE_LIMIT: RouteRateLimitSpec = { max: 60, timeWindow: "1 minute", skipOnError: false, -}) +} -export const ADMIN_REPORT_CHAT_REMOVE_RATE_LIMIT = perIdentity({ - max: 60, - timeWindow: "1 minute", - skipOnError: false, -}) +export const ADMIN_REPORT_CHAT_SEND_RATE_LIMIT = perIdentity(OPERATOR_CHAT_WRITE_LIMIT) + +export const ADMIN_REPORT_CHAT_REMOVE_RATE_LIMIT = perIdentity(OPERATOR_CHAT_WRITE_LIMIT) export async function registerAdminReportChatRoutes( app: FastifyInstance, diff --git a/services/api/src/routes/admin/reports.routes.ts b/services/api/src/routes/admin/reports.routes.ts index 6cfff247..857a1d30 100644 --- a/services/api/src/routes/admin/reports.routes.ts +++ b/services/api/src/routes/admin/reports.routes.ts @@ -37,6 +37,7 @@ import { type OutboundMailService, } from "../../services/admin/outbound-mail-service.js" import { makeDrizzleCleanupRepository } from "../../services/cleanup-repository.drizzle.js" +import type { LinkedEventView } from "../../services/cleanup-service.js" import { makePacketMediaPresigner, makePrivateMediaPresigner, @@ -46,6 +47,18 @@ import { makeContainerReportChatEmitter } from "../../services/report-chat-emitt import { makeRouteNotificationService } from "../../services/route-notifier.js" import type { ReportChatSystemEmitter } from "../../services/report-timeline-event.js" +export const ROUTE_REPORT_RATE_LIMIT = perIdentity({ + max: 10, + timeWindow: "1 minute", + skipOnError: false, +}) + +export const ADMIN_REPORT_MUTATION_RATE_LIMIT = perIdentity({ + max: 60, + timeWindow: "1 minute", + skipOnError: false, +}) + export interface AdminReportRouteOverrides { repo: AdminReportRepository outboundMail: OutboundMailService @@ -53,9 +66,7 @@ export interface AdminReportRouteOverrides { r2Key: string, thumbKey: string | null, ) => Promise<{ url: string; thumbUrl?: string }> - loadLinkedEventsForReports?: ( - reportIds: string[], - ) => Promise> + loadLinkedEventsForReports?: (reportIds: string[]) => Promise> now?: () => Date reportChatEmitter?: ReportChatSystemEmitter forwardTemplates?: ForwardTemplateRepository @@ -208,15 +219,3 @@ export async function registerAdminReportsRoutes( }, ) } - -export const ROUTE_REPORT_RATE_LIMIT = perIdentity({ - max: 10, - timeWindow: "1 minute", - skipOnError: false, -}) - -export const ADMIN_REPORT_MUTATION_RATE_LIMIT = perIdentity({ - max: 60, - timeWindow: "1 minute", - skipOnError: false, -}) diff --git a/services/api/src/routes/host/analytics.routes.ts b/services/api/src/routes/host/analytics.routes.ts index 055e96a0..562f379c 100644 --- a/services/api/src/routes/host/analytics.routes.ts +++ b/services/api/src/routes/host/analytics.routes.ts @@ -8,8 +8,9 @@ import { type AnalyticsRange, type PortfolioAnalyticsRange, } from "@civfix/shared" -import { can } from "@civfix/shared/host" +import { can, type HostStanding } from "@civfix/shared/host" import type { FastifyInstance, FastifyRequest } from "fastify" +import type { ZodType } from "zod" import type { Container } from "../../di.js" import { requireAuth } from "../../auth/context.js" import { perIdentity } from "../../plugins/rate-limit.js" @@ -28,6 +29,12 @@ import type { InsightsService } from "../../services/host/insights-service.js" export const ANALYTICS_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute" }) +const DEFAULT_EVENT_RANGE: AnalyticsRange = "30d" +const DEFAULT_PORTFOLIO_RANGE: PortfolioAnalyticsRange = "90d" +const DEFAULT_EVENT_ANALYTICS_SCOPE = "full" +const SELF_VIEWER_SCOPE = "self" +const NO_ROLE = "none" + export interface HostAnalyticsOverrides { analytics: AnalyticsService eventAnalytics: EventAnalyticsService @@ -46,43 +53,60 @@ function mergeQuery(request: FastifyRequest): Record { return { ...query, ...params } } +function viewerScopeOf(standing: HostStanding): string { + return `${standing.eventRole ?? NO_ROLE}:${standing.orgRole ?? NO_ROLE}` +} + export async function registerHostAnalyticsRoutes( app: FastifyInstance, container: Container, ): Promise { let cached: CommsRuntime | undefined - function analytics(): AnalyticsService { - const override = app.hostAnalyticsOverrides - if (override) return override.analytics - return (cached ??= makeCommsRuntime(container, app.log)).analytics + function services(): HostAnalyticsOverrides { + return app.hostAnalyticsOverrides ?? (cached ??= makeCommsRuntime(container, app.log)) } - function eventAnalytics(): EventAnalyticsService { - const override = app.hostAnalyticsOverrides - if (override) return override.eventAnalytics - return (cached ??= makeCommsRuntime(container, app.log)).eventAnalytics - } + const analytics = (): AnalyticsService => services().analytics + const eventAnalytics = (): EventAnalyticsService => services().eventAnalytics + const insights = (): InsightsService => services().insights - function insights(): InsightsService { - const override = app.hostAnalyticsOverrides - if (override) return override.insights - return (cached ??= makeCommsRuntime(container, app.log)).insights + async function authorizeEventView( + request: FastifyRequest, + schema: ZodType, + input: unknown, + ) { + const userId = requireAuth(request) + const query = parse(schema, input) + const resolution = await requireCapability( + container.getDb().sql, + query.id, + userId, + "view_analytics", + ) + return { userId, query, resolution, viewerScope: viewerScopeOf(resolution.standing) } } async function eventScope( request: FastifyRequest, ): Promise<{ cleanupId: string; range: AnalyticsRange; viewerScope: string }> { - const userId = requireAuth(request) - const query = parse(EventAnalyticsRequestSchema, mergeQuery(request)) - const resolution = await requireCapability( + const { query, viewerScope } = await authorizeEventView( + request, + EventAnalyticsRequestSchema, + mergeQuery(request), + ) + return { cleanupId: query.id, range: query.range ?? DEFAULT_EVENT_RANGE, viewerScope } + } + + async function portfolioViewerScope(userId: string, orgId: string | undefined): Promise { + if (orgId === undefined) return SELF_VIEWER_SCOPE + const standing = await requireOrgCapability( container.getDb().sql, - query.id, + orgId, userId, "view_analytics", ) - const viewerScope = `${resolution.standing.eventRole ?? "none"}:${resolution.standing.orgRole ?? "none"}` - return { cleanupId: query.id, range: query.range ?? "30d", viewerScope } + return `org:${standing.orgRole ?? NO_ROLE}` } route( @@ -90,17 +114,13 @@ export async function registerHostAnalyticsRoutes( "getEventAnalytics", { config: { rateLimit: ANALYTICS_RATE_LIMIT } }, async (request, reply) => { - const userId = requireAuth(request) - const query = parse(GetEventAnalyticsRequestSchema, mergeQuery(request)) - const resolution = await requireCapability( - container.getDb().sql, - query.id, - userId, - "view_analytics", + const { userId, query, resolution, viewerScope } = await authorizeEventView( + request, + GetEventAnalyticsRequestSchema, + mergeQuery(request), ) - const viewerScope = `${resolution.standing.eventRole ?? "none"}:${resolution.standing.orgRole ?? "none"}` reply.status(200).send( - await eventAnalytics().analytics(query.id, query.scope ?? "full", { + await eventAnalytics().analytics(query.id, query.scope ?? DEFAULT_EVENT_ANALYTICS_SCOPE, { userId, organizationId: resolution.organizationId, viewerScope, @@ -174,24 +194,15 @@ export async function registerHostAnalyticsRoutes( "getEventInsights", { config: { rateLimit: ANALYTICS_RATE_LIMIT } }, async (request, reply) => { - const userId = requireAuth(request) - const params = parse(GetEventInsightsRequestSchema, request.params) - const resolution = await requireCapability( - container.getDb().sql, - params.id, - userId, - "view_analytics", + const { userId, query, resolution, viewerScope } = await authorizeEventView( + request, + GetEventInsightsRequestSchema, + request.params, ) if (!can(resolution.standing, "view_roster")) { throw AppError.forbidden(hostForbiddenCopy("view_roster")) } - const viewerScope = `${resolution.standing.eventRole ?? "none"}:${resolution.standing.orgRole ?? "none"}` - reply.status(200).send( - await insights().insights(params.id, { - userId, - viewerScope, - }), - ) + reply.status(200).send(await insights().insights(query.id, { userId, viewerScope })) }, ) @@ -202,17 +213,8 @@ export async function registerHostAnalyticsRoutes( async (request, reply) => { const userId = requireAuth(request) const query = parse(HostedEventsAnalyticsRequestSchema, request.query) - let viewerScope = "self" - if (query.orgId !== undefined) { - const standing = await requireOrgCapability( - container.getDb().sql, - query.orgId, - userId, - "view_analytics", - ) - viewerScope = `org:${standing.orgRole ?? "none"}` - } - const range: PortfolioAnalyticsRange = query.range ?? "90d" + const viewerScope = await portfolioViewerScope(userId, query.orgId) + const range: PortfolioAnalyticsRange = query.range ?? DEFAULT_PORTFOLIO_RANGE reply .status(200) .send(await analytics().portfolio(userId, query.orgId ?? null, range, viewerScope)) @@ -226,17 +228,8 @@ export async function registerHostAnalyticsRoutes( async (request, reply) => { const userId = requireAuth(request) const query = parse(HostAnalyticsSummaryRequestSchema, request.query) - let viewerScope = "self" - if (query.orgId !== undefined) { - const standing = await requireOrgCapability( - container.getDb().sql, - query.orgId, - userId, - "view_analytics", - ) - viewerScope = `org:${standing.orgRole ?? "none"}` - } - const range: AnalyticsRange = query.range ?? "30d" + const viewerScope = await portfolioViewerScope(userId, query.orgId) + const range: AnalyticsRange = query.range ?? DEFAULT_EVENT_RANGE reply .status(200) .send(await analytics().summary(userId, query.orgId ?? null, range, viewerScope)) diff --git a/services/api/src/routes/host/announcements.routes.ts b/services/api/src/routes/host/announcements.routes.ts index 9e330074..d0211eb0 100644 --- a/services/api/src/routes/host/announcements.routes.ts +++ b/services/api/src/routes/host/announcements.routes.ts @@ -13,7 +13,7 @@ import { perIdentity } from "../../plugins/rate-limit.js" import { route } from "../../versioning/route.js" import { parse, trimTextFields } from "../_validate.js" import { requireCapability, resolveVisibleStanding } from "../../services/host/authz.js" -import { BroadcastCapError, capError } from "../../services/host/broadcast-service.js" +import { withCaps } from "../../services/host/broadcast-service.js" import { auditBestEffort, makeCommsRuntime } from "../../services/host/comms-wiring.js" import type { CommsRuntime } from "../../services/host/comms-wiring.js" import type { @@ -70,13 +70,7 @@ export async function registerHostAnnouncementRoutes( mergeParams(request), ) await requireCapability(container.getDb().sql, body.id, userId, "broadcast") - let payload: AnnouncementDTO - try { - payload = await service().create(body.id, userId, body) - } catch (err) { - if (err instanceof BroadcastCapError) throw capError(err.kind) - throw err - } + const payload = await withCaps(() => service().create(body.id, userId, body)) await auditBestEffort( container.getDb().sql, { diff --git a/services/api/src/routes/host/broadcasts.routes.ts b/services/api/src/routes/host/broadcasts.routes.ts index 268307f0..aa8d4535 100644 --- a/services/api/src/routes/host/broadcasts.routes.ts +++ b/services/api/src/routes/host/broadcasts.routes.ts @@ -25,9 +25,8 @@ import { route } from "../../versioning/route.js" import { parse } from "../_validate.js" import { requireCapability, resolveVisibleStanding } from "../../services/host/authz.js" import { - BroadcastCapError, - capError, emailHashOf, + withCaps, type BroadcastService, } from "../../services/host/broadcast-service.js" import { auditBestEffort, makeCommsRuntime } from "../../services/host/comms-wiring.js" @@ -76,15 +75,6 @@ export async function registerHostBroadcastRoutes( return runtime().broadcasts } - async function withCaps(fn: () => Promise): Promise { - try { - return await fn() - } catch (err) { - if (err instanceof BroadcastCapError) throw capError(err.kind) - throw err - } - } - route( app, "listEventBroadcasts", diff --git a/services/api/src/routes/host/exports.routes.ts b/services/api/src/routes/host/exports.routes.ts index 70f446c7..056b5fb5 100644 --- a/services/api/src/routes/host/exports.routes.ts +++ b/services/api/src/routes/host/exports.routes.ts @@ -18,12 +18,19 @@ import { requireCapability } from "../../services/host/authz.js" import { HOST_EXPORT_JOB } from "../../services/host/broadcast-queues.js" import { makeCommsRuntime } from "../../services/host/comms-wiring.js" import type { CommsRuntime } from "../../services/host/comms-wiring.js" -import { toHostExportDTO, type HostExportService } from "../../services/host/export-service.js" +import { + EXPORT_NOT_FOUND, + toHostExportDTO, + type HostExportService, +} from "../../services/host/export-service.js" export const EXPORT_REQUEST_RATE_LIMIT = perIdentity({ max: 10, timeWindow: "1 hour" }) export const EXPORT_READ_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute" }) export const EXPORT_DOWNLOAD_RATE_LIMIT = perIdentity({ max: 30, timeWindow: "1 minute" }) +const EXPORT_JOB_RETRY_LIMIT = 2 +const EXPORT_AUDIT_ACTION = "event.roster_exported" + export interface HostExportOverrides { exports: HostExportService } @@ -74,7 +81,7 @@ export async function registerHostExportRoutes( kind: body.kind, filters: body.filters, audit: (exportId) => ({ - action: "event.roster_exported", + action: EXPORT_AUDIT_ACTION, actorId: userId, target: `cleanup:${body.id}`, meta: { exportId, kind: body.kind }, @@ -83,7 +90,7 @@ export async function registerHostExportRoutes( await container.jobs.enqueue( HOST_EXPORT_JOB, { exportId: payload.id }, - { singletonKey: `export:${payload.id}`, retryLimit: 2 }, + { singletonKey: `export:${payload.id}`, retryLimit: EXPORT_JOB_RETRY_LIMIT }, ) reply.status(200).send(payload) }, @@ -112,7 +119,7 @@ export async function registerHostExportRoutes( const params = parse(GetEventExportRequestSchema, mergeParams(request)) await requireCapability(container.getDb().sql, params.id, userId, "export") const record = await exports().get(params.exportId) - if (record.cleanupId !== params.id) throw AppError.notFound("Export not found") + if (record.cleanupId !== params.id) throw AppError.notFound(EXPORT_NOT_FOUND) reply.status(200).send(toHostExportDTO(record)) }, ) @@ -125,8 +132,8 @@ export async function registerHostExportRoutes( const userId = requireAuth(request) const params = parse(DownloadHostExportRequestSchema, request.params) const record = await exports().get(params.id) - if (record.requestedBy !== userId) throw AppError.notFound("Export not found") - if (record.cleanupId === null) throw AppError.notFound("Export not found") + if (record.requestedBy !== userId) throw AppError.notFound(EXPORT_NOT_FOUND) + if (record.cleanupId === null) throw AppError.notFound(EXPORT_NOT_FOUND) await requireCapability(container.getDb().sql, record.cleanupId, userId, "export") const payload: DownloadHostExportResponse = await exports().downloadUrl(record) reply.status(200).send(payload) diff --git a/services/api/src/routes/webhooks/inbound-mail.routes.ts b/services/api/src/routes/webhooks/inbound-mail.routes.ts index 6e41eef0..2bc89e15 100644 --- a/services/api/src/routes/webhooks/inbound-mail.routes.ts +++ b/services/api/src/routes/webhooks/inbound-mail.routes.ts @@ -16,7 +16,7 @@ export const CF_WEBHOOK_TIMESTAMP_HEADER = "x-cf-timestamp" export const WEBHOOK_MAX_CLOCK_SKEW_SEC = 5 * 60 -export const ACCEPT_LEGACY_UNTIMESTAMPED_SIGNATURE = true +const ACCEPT_LEGACY_UNTIMESTAMPED_SIGNATURE = true const LEGACY_WARN_INTERVAL_SEC = 300 let lastLegacyWarnAtSec = 0 @@ -29,6 +29,9 @@ export const INBOUND_WEBHOOK_RATE_LIMIT = perHost({ max: 60, timeWindow: "1 minu const INBOUND_WEBHOOK_BODY_LIMIT = 4096 +const SIGNATURE_HMAC_ALGORITHM = "sha256" +const INVALID_SIGNATURE_MESSAGE = "Invalid inbound mail webhook signature." + export type InboundMailWebhookOverrides = InboundProcessorDeps declare module "fastify" { @@ -110,7 +113,7 @@ export function assertSignature( const header = request.headers[CF_WEBHOOK_SIGNATURE_HEADER] const presented = Array.isArray(header) ? header[0] : header if (!presented) { - throw rejected(request, nowSec, "no-signature", "Invalid inbound mail webhook signature.") + throw rejected(request, nowSec, "no-signature", INVALID_SIGNATURE_MESSAGE) } const tsHeader = request.headers[CF_WEBHOOK_TIMESTAMP_HEADER] @@ -121,12 +124,12 @@ export function assertSignature( if (!Number.isFinite(ts) || Math.abs(nowSec - ts) > WEBHOOK_MAX_CLOCK_SKEW_SEC) { throw rejected(request, nowSec, "expired", "Inbound mail webhook signature expired.") } - const expected = createHmac("sha256", expectedSecret) + const expected = createHmac(SIGNATURE_HMAC_ALGORITHM, expectedSecret) .update(`${rawTs}.`) .update(rawBuf) .digest("hex") if (!constantTimeStringEqual(presented, expected)) { - throw rejected(request, nowSec, "mismatch", "Invalid inbound mail webhook signature.") + throw rejected(request, nowSec, "mismatch", INVALID_SIGNATURE_MESSAGE) } return } @@ -139,14 +142,15 @@ export function assertSignature( "Inbound mail webhook signature is missing its timestamp.", ) } - const nowSecForWarn = nowSec - if (nowSecForWarn - lastLegacyWarnAtSec >= LEGACY_WARN_INTERVAL_SEC) { - lastLegacyWarnAtSec = nowSecForWarn + if (nowSec - lastLegacyWarnAtSec >= LEGACY_WARN_INTERVAL_SEC) { + lastLegacyWarnAtSec = nowSec request.log.warn("inbound-mail webhook: legacy untimestamped signature accepted (replayable)") } - const legacyExpected = createHmac("sha256", expectedSecret).update(rawBuf).digest("hex") + const legacyExpected = createHmac(SIGNATURE_HMAC_ALGORITHM, expectedSecret) + .update(rawBuf) + .digest("hex") if (!constantTimeStringEqual(presented, legacyExpected)) { - throw rejected(request, nowSec, "mismatch", "Invalid inbound mail webhook signature.") + throw rejected(request, nowSec, "mismatch", INVALID_SIGNATURE_MESSAGE) } } diff --git a/services/api/src/services/admin/activity-repository.drizzle.ts b/services/api/src/services/admin/activity-repository.drizzle.ts index fed8339a..0d0a1227 100644 --- a/services/api/src/services/admin/activity-repository.drizzle.ts +++ b/services/api/src/services/admin/activity-repository.drizzle.ts @@ -27,10 +27,11 @@ import { keysetInstant, keysetPredicate, paginateKeyset, + type KeysetAnchor, } from "./pagination.js" import { AUDIT_READ_ACTIONS } from "./audit.js" import { likePrefix } from "./like.js" -import { ilikeAnyOf, type SqlFragment } from "./sql-fragments.js" +import { anyOf, ilikeAnyOf, type SqlFragment } from "./sql-fragments.js" import { AUDIT_ACTION_RULES, AUDIT_FALLBACK_KIND, @@ -69,72 +70,71 @@ function toRecord(r: ActivityRowSelect): ActivitySourceRecord { } } -export function makeDrizzleActivityRepository(sql: Sql): ActivityRepository { - // The catch-all kind matches none of the rules; it must stay derived from the one rule table or the - // chip drifts from the label. - function auditKindFilter(args: ListActivityArgs): SqlFragment { - if (args.filter === "all") return sql`` - const own = auditRulesForKind(args.filter) - if (own.length > 0) return sql`AND ${anyRule(own)}` - if (args.filter === AUDIT_FALLBACK_KIND) return sql`AND NOT ${anyRule(AUDIT_ACTION_RULES)}` - // The kind cannot come from the audit branch at all; sourcesForKind already pruned it, so this is - // unreachable defense rather than a filter. - return sql`AND false` - } +interface PageSpec { + anchor: KeysetAnchor | null + desc: boolean + limit: number +} + +type BranchBuilder = (sql: Sql, args: ListActivityArgs, page: PageSpec) => SqlFragment + +// The catch-all kind matches none of the rules; it must stay derived from the one rule table or the chip +// drifts from the label. +function auditKindFilter(sql: Sql, args: ListActivityArgs): SqlFragment { + if (args.filter === "all") return sql`` + const own = auditRulesForKind(args.filter) + if (own.length > 0) return sql`AND ${anyRule(sql, own)}` + if (args.filter === AUDIT_FALLBACK_KIND) return sql`AND NOT ${anyRule(sql, AUDIT_ACTION_RULES)}` + // The kind cannot come from the audit branch at all; sourcesForKind already pruned it, so this is + // unreachable defense rather than a filter. + return sql`AND false` +} - function anyRule(rules: readonly AuditActionRule[]): SqlFragment { - // The prefixes contain `_` (gov_claim.), a LIKE wildcard, so they are escaped to match literally, as - // the service classifier's startsWith does. - const branches = rules.map((rule) => +function anyRule(sql: Sql, rules: readonly AuditActionRule[]): SqlFragment { + // The prefixes contain `_` (gov_claim.), a LIKE wildcard, so they are escaped to match literally, as + // the service classifier's startsWith does. + return anyOf( + sql, + rules.map((rule) => rule.exact !== undefined ? sql`a.action = ${rule.exact}` : sql`a.action LIKE ${likePrefix(rule.prefix)} ESCAPE '\\'`, - ) - const first = branches[0] - if (first === undefined) return sql`(false)` - return sql`(${branches.slice(1).reduce((acc, b) => sql`${acc} OR ${b}`, first)})` - } - - function mailKindFilter(args: ListActivityArgs): SqlFragment { - const types = [...MAIL_BOUNCE_EVENT_TYPES] - if (args.filter === "outreach_bounce") return sql`AND e.type = ANY(${types}::text[])` - if (args.filter === "outreach_open") return sql`AND e.type <> ALL(${types}::text[])` - return sql`` - } + ), + ) +} - return { - async list( - args: ListActivityArgs, - ): Promise<{ records: ActivitySourceRecord[]; nextCursor: string | null }> { - const limit = clampLimit(args.limit) - // requireUuid is false: the keyset compares `id::text`, never `${id}::uuid`, so a non-uuid id in a - // hand-made cursor cannot raise a 22P02; it simply anchors past every real row. - const anchor = decodeCursor(args.cursor) - const desc = args.sort === "newest" - const sources = args.filter === "all" ? null : new Set(sourcesForKind(args.filter)) - const wants = (source: ActivitySource): boolean => sources === null || sources.has(source) - - const pageWindow = (tsCol: SqlFragment, idText: SqlFragment): SqlFragment => { - const keyset = - anchor === null - ? sql`` - : sql`AND ${keysetPredicate(sql, tsCol, idText, anchor, { - direction: desc ? "desc" : "asc", - idType: "text", - })}` - const order = desc - ? sql`ORDER BY ${tsCol} DESC, ${idText} DESC` - : sql`ORDER BY ${tsCol} ASC, ${idText} ASC` - return sql`${keyset} ${order} LIMIT ${limit + 1}` - } +function mailKindFilter(sql: Sql, args: ListActivityArgs): SqlFragment { + const types = [...MAIL_BOUNCE_EVENT_TYPES] + if (args.filter === "outreach_bounce") return sql`AND e.type = ANY(${types}::text[])` + if (args.filter === "outreach_open") return sql`AND e.type <> ALL(${types}::text[])` + return sql`` +} - const search = (columns: readonly SqlFragment[]): SqlFragment => - args.q === null ? sql`` : sql`AND ${ilikeAnyOf(sql, columns, args.q)}` +function pageWindow( + sql: Sql, + page: PageSpec, + tsCol: SqlFragment, + idText: SqlFragment, +): SqlFragment { + const keyset = + page.anchor === null + ? sql`` + : sql`AND ${keysetPredicate(sql, tsCol, idText, page.anchor, { + direction: page.desc ? "desc" : "asc", + idType: "text", + })}` + const order = page.desc + ? sql`ORDER BY ${tsCol} DESC, ${idText} DESC` + : sql`ORDER BY ${tsCol} ASC, ${idText} ASC` + return sql`${keyset} ${order} LIMIT ${page.limit + 1}` +} - const branches: SqlFragment[] = [] +function searchFilter(sql: Sql, q: string | null, columns: readonly SqlFragment[]): SqlFragment { + return q === null ? sql`` : sql`AND ${ilikeAnyOf(sql, columns, q)}` +} - if (wants("audit")) { - branches.push(sql`( +function auditBranch(sql: Sql, args: ListActivityArgs, page: PageSpec): SqlFragment { + return sql`( SELECT 'audit'::text AS source, a.id::text AS id, a.created_at AS ts, ${keysetInstant(sql, sql`a.created_at`)} AS cursor_at, u.display_name AS who, a.target AS where_label, @@ -145,14 +145,14 @@ export function makeDrizzleActivityRepository(sql: Sql): ActivityRepository { -- fill this branch's whole per-source window and evict every real action from the feed. They -- remain in the audit-log view. WHERE a.action <> ALL(${[...AUDIT_READ_ACTIONS]}) - ${auditKindFilter(args)} - ${search([sql`a.action`, sql`u.display_name`, sql`a.target`])} - ${pageWindow(sql`a.created_at`, sql`a.id::text`)} - )`) - } + ${auditKindFilter(sql, args)} + ${searchFilter(sql, args.q, [sql`a.action`, sql`u.display_name`, sql`a.target`])} + ${pageWindow(sql, page, sql`a.created_at`, sql`a.id::text`)} + )` +} - if (wants("report")) { - branches.push(sql`( +function reportBranch(sql: Sql, args: ListActivityArgs, page: PageSpec): SqlFragment { + return sql`( SELECT 'report'::text AS source, r.id::text AS id, r.created_at AS ts, ${keysetInstant(sql, sql`r.created_at`)} AS cursor_at, ru.display_name AS who, j.name AS where_label, @@ -161,15 +161,15 @@ export function makeDrizzleActivityRepository(sql: Sql): ActivityRepository { LEFT JOIN users ru ON ru.id = r.reporter_user_id LEFT JOIN jurisdictions j ON j.geoid = r.jurisdiction_geoid WHERE r.deleted_at IS NULL AND r.visibility = 'public' - ${search([sql`r.category`, sql`ru.display_name`, sql`j.name`])} + ${searchFilter(sql, args.q, [sql`r.category`, sql`ru.display_name`, sql`j.name`])} -- reports.created_at is DEFAULT now() (effectively non-null on every row), so ordering on the -- bare column lets the (created_at) index serve the sort instead of a seq-scan+top-N heap sort. - ${pageWindow(sql`r.created_at`, sql`r.id::text`)} - )`) - } + ${pageWindow(sql, page, sql`r.created_at`, sql`r.id::text`)} + )` +} - if (wants("cleanup")) { - branches.push(sql`( +function cleanupBranch(sql: Sql, args: ListActivityArgs, page: PageSpec): SqlFragment { + return sql`( SELECT 'cleanup'::text AS source, c.id::text AS id, c.created_at AS ts, ${keysetInstant(sql, sql`c.created_at`)} AS cursor_at, cu.display_name AS who, c.address AS where_label, @@ -177,15 +177,15 @@ export function makeDrizzleActivityRepository(sql: Sql): ActivityRepository { FROM cleanups c LEFT JOIN users cu ON cu.id = c.organizer_user_id WHERE true - ${search([sql`c.title`, sql`cu.display_name`, sql`c.address`])} + ${searchFilter(sql, args.q, [sql`c.title`, sql`cu.display_name`, sql`c.address`])} -- cleanups.created_at is DEFAULT now() (set on insert, effectively non-null), so ordering on the -- bare column lets the (created_at) index serve the sort. - ${pageWindow(sql`c.created_at`, sql`c.id::text`)} - )`) - } + ${pageWindow(sql, page, sql`c.created_at`, sql`c.id::text`)} + )` +} - if (wants("mail_event")) { - branches.push(sql`( +function mailEventBranch(sql: Sql, args: ListActivityArgs, page: PageSpec): SqlFragment { + return sql`( SELECT 'mail_event'::text AS source, e.id::text AS id, e.created_at AS ts, ${keysetInstant(sql, sql`e.created_at`)} AS cursor_at, t.org AS who, COALESCE(t.org, t.jurisdiction_geoid) AS where_label, @@ -193,11 +193,37 @@ export function makeDrizzleActivityRepository(sql: Sql): ActivityRepository { FROM mail_events e LEFT JOIN mail_threads t ON t.id = e.thread_id WHERE true - ${mailKindFilter(args)} - ${search([sql`e.type`, sql`t.org`, sql`t.jurisdiction_geoid`, sql`t.subject`])} - ${pageWindow(sql`e.created_at`, sql`e.id::text`)} - )`) + ${mailKindFilter(sql, args)} + ${searchFilter(sql, args.q, [sql`e.type`, sql`t.org`, sql`t.jurisdiction_geoid`, sql`t.subject`])} + ${pageWindow(sql, page, sql`e.created_at`, sql`e.id::text`)} + )` +} + +// Union order is fixed so the statement text is the same for every request that selects the same sources. +const SOURCE_BRANCHES: readonly (readonly [ActivitySource, BranchBuilder])[] = [ + ["audit", auditBranch], + ["report", reportBranch], + ["cleanup", cleanupBranch], + ["mail_event", mailEventBranch], +] + +export function makeDrizzleActivityRepository(sql: Sql): ActivityRepository { + return { + async list( + args: ListActivityArgs, + ): Promise<{ records: ActivitySourceRecord[]; nextCursor: string | null }> { + const limit = clampLimit(args.limit) + // requireUuid is false: the keyset compares `id::text`, never `${id}::uuid`, so a non-uuid id in a + // hand-made cursor cannot raise a 22P02; it simply anchors past every real row. + const page: PageSpec = { + anchor: decodeCursor(args.cursor), + desc: args.sort === "newest", + limit, } + const sources = args.filter === "all" ? null : new Set(sourcesForKind(args.filter)) + const branches = SOURCE_BRANCHES.filter( + ([source]) => sources === null || sources.has(source), + ).map(([, build]) => build(sql, args, page)) const first = branches[0] // No branch can produce the requested kind (today: filter=claim). An empty union is not valid SQL, so @@ -207,7 +233,7 @@ export function makeDrizzleActivityRepository(sql: Sql): ActivityRepository { .slice(1) .reduce((acc, branch) => sql`${acc} UNION ALL ${branch}`, first) - const outerOrder = desc + const outerOrder = page.desc ? sql`ORDER BY feed.ts DESC, feed.id DESC` : sql`ORDER BY feed.ts ASC, feed.id ASC` // The union is wrapped in a FROM subquery (rather than trailing the branches with a bare ORDER BY) so diff --git a/services/api/src/services/admin/activity-service.ts b/services/api/src/services/admin/activity-service.ts index f02e4a0e..b99a27d7 100644 --- a/services/api/src/services/admin/activity-service.ts +++ b/services/api/src/services/admin/activity-service.ts @@ -12,7 +12,7 @@ import type { ActivityListQuery, ActivityListResponse, } from "@civfix/shared" -import { clampLimit } from "./pagination.js" +import { ADMIN_DEFAULT_LIMIT, clampLimit } from "./pagination.js" export type ActivitySource = "audit" | "report" | "cleanup" | "mail_event" @@ -27,8 +27,6 @@ export interface ActivitySourceRecord { subject?: string | null } -export const ACTIVITY_DEFAULT_LIMIT = 25 - export type ActivityFilter = "all" | ActivityKind export type ActivitySort = "newest" | "oldest" @@ -51,13 +49,13 @@ export interface ActivityRepository { * `filter` / `sort` are free-form strings on the wire (ActivityListQuerySchema), so an unrecognized value * degrades to the default rather than 422 or reach SQL: the server owns this vocabulary. */ -export function parseActivityFilter(filter: string | undefined): ActivityFilter { +function parseActivityFilter(filter: string | undefined): ActivityFilter { if (filter === undefined || filter === "" || filter === "all") return "all" const parsed = ActivityKindSchema.safeParse(filter) return parsed.success ? parsed.data : "all" } -export function parseActivitySort(sort: string | undefined): ActivitySort { +function parseActivitySort(sort: string | undefined): ActivitySort { return sort === "oldest" ? "oldest" : "newest" } @@ -94,7 +92,7 @@ export const AUDIT_FALLBACK_KIND: ActivityKind = "mod_action" export const MAIL_BOUNCE_EVENT_TYPES = ["bounced", "failed"] as const -export function isMailBounceEventType(type: string): boolean { +function isMailBounceEventType(type: string): boolean { return (MAIL_BOUNCE_EVENT_TYPES as readonly string[]).includes(type) } @@ -143,100 +141,110 @@ export function classifyAuditAction(action: string): ActivityKind { * Keep this covering every AdminAuditAction (audit.ts): an unlabelled action renders as its raw dotted * string in the operator's feed. An unknown action still falls back to that rather than failing. */ +const AUDIT_ACTION_LABELS: Record = { + "operator.login": "Operator signed in", + "operator.logout": "Operator signed out", + "operator.login_denied": "An operator sign-in was denied", + "account.deleted": "An account was deleted", + "data_export.undeliverable": "A data export could not be delivered", + "discovery.contacts_saved": "Saved routing contacts", + "discovery.draft_saved": "Saved a routing draft", + "discovery.note_added": "Added a discovery note", + "discovery.flagged": "Flagged a jurisdiction", + // Written by the public suggest-contact endpoint, not an operator (the feed's `who` is the row's actor). + "discovery.contact_suggested": "Suggested a routing contact", + "jurisdiction.patched": "Updated a jurisdiction", + "report.status_changed": "Changed a report status", + "report.flagged": "Flagged a report", + "report.unflagged": "Unflagged a report", + "report.removed": "Removed a report", + "report.followup_sent": "Sent a report follow-up", + "report.message_posted": "Posted in a report chat", + "report.verdict_set": "Set a report verdict", + "report.routed": "Forwarded a report to the city", + "report.takedown_requested": "Requested a takedown of their report", + "report_message.removed": "Removed a report chat message", + "event.status_changed": "Changed an event status", + "event.flagged": "Flagged an event", + "event.unflagged": "Unflagged an event", + "event.cancelled": "Cancelled an event", + "event.message_posted": "Messaged event attendees", + "event.outcome_logged": "Logged an event outcome", + "event.reports_linked": "Linked reports to an event", + "event.report_unlinked": "Unlinked a report from an event", + "event.announcement_sent": "Sent an event announcement", + "user.flagged": "Flagged an account", + "user.unflagged": "Unflagged an account", + "user.status_changed": "Changed an account status", + "user.banned": "Banned an account", + "user.role_changed": "Changed an account role", + "user.verified": "Verified an account", + "user.unverified": "Removed an account's verification", + "user.report_verified": "Granted report-verified status", + "user.report_unverified": "Revoked report-verified status", + "message.removed": "Removed a message", + "gov_claim.verified": "Verified a gov claim check", + "gov_claim.approved": "Approved a gov claim", + "gov_claim.rejected": "Rejected a gov claim", + "moderation.approved": "Approved a held item", + "moderation.removed": "Removed a flagged item", + "moderation.held": "Extended a hold", + "moderation.appeal_decided": "Decided an appeal", + "mail.sent": "Sent outreach mail", + "mail.replied": "Replied to a thread", + "mail.resent": "Resent a message", + "mail.send_failed": "A send to a jurisdiction failed", + "mail.status_changed": "Updated a mail thread", + "mail.forward_template_set": "Updated the default forwarding template", + "outreach.digest_sent": "Sent an outreach digest", + "inbox.status_changed": "Updated an inbox message", + // Read audits are filtered out of the feed at the repo (AUDIT_READ_ACTIONS), so these labels only + // matter if a row reaches the classifier another way (a fake, or a future feed that includes them). + "user.detail_viewed": "Viewed an account", + "user.messages_viewed": "Viewed an account's messages", + "inbox.message_viewed": "Viewed an inbox message", + "mail.thread_viewed": "Viewed a mail thread", +} + export function describeAuditAction(action: string): string { - const map: Record = { - "operator.login": "Operator signed in", - "operator.logout": "Operator signed out", - "operator.login_denied": "An operator sign-in was denied", - "account.deleted": "An account was deleted", - "data_export.undeliverable": "A data export could not be delivered", - "discovery.contacts_saved": "Saved routing contacts", - "discovery.draft_saved": "Saved a routing draft", - "discovery.note_added": "Added a discovery note", - "discovery.flagged": "Flagged a jurisdiction", - // Written by the public suggest-contact endpoint, not an operator (the feed's `who` is the row's actor). - "discovery.contact_suggested": "Suggested a routing contact", - "jurisdiction.patched": "Updated a jurisdiction", - "report.status_changed": "Changed a report status", - "report.flagged": "Flagged a report", - "report.unflagged": "Unflagged a report", - "report.removed": "Removed a report", - "report.followup_sent": "Sent a report follow-up", - "report.message_posted": "Posted in a report chat", - "report.verdict_set": "Set a report verdict", - "report.routed": "Forwarded a report to the city", - "report.takedown_requested": "Requested a takedown of their report", - "report_message.removed": "Removed a report chat message", - "event.status_changed": "Changed an event status", - "event.flagged": "Flagged an event", - "event.unflagged": "Unflagged an event", - "event.cancelled": "Cancelled an event", - "event.message_posted": "Messaged event attendees", - "event.outcome_logged": "Logged an event outcome", - "event.reports_linked": "Linked reports to an event", - "event.report_unlinked": "Unlinked a report from an event", - "event.announcement_sent": "Sent an event announcement", - "user.flagged": "Flagged an account", - "user.unflagged": "Unflagged an account", - "user.status_changed": "Changed an account status", - "user.banned": "Banned an account", - "user.role_changed": "Changed an account role", - "user.verified": "Verified an account", - "user.unverified": "Removed an account's verification", - "user.report_verified": "Granted report-verified status", - "user.report_unverified": "Revoked report-verified status", - "message.removed": "Removed a message", - "gov_claim.verified": "Verified a gov claim check", - "gov_claim.approved": "Approved a gov claim", - "gov_claim.rejected": "Rejected a gov claim", - "moderation.approved": "Approved a held item", - "moderation.removed": "Removed a flagged item", - "moderation.held": "Extended a hold", - "moderation.appeal_decided": "Decided an appeal", - "mail.sent": "Sent outreach mail", - "mail.replied": "Replied to a thread", - "mail.resent": "Resent a message", - "mail.send_failed": "A send to a jurisdiction failed", - "mail.status_changed": "Updated a mail thread", - "mail.forward_template_set": "Updated the default forwarding template", - "outreach.digest_sent": "Sent an outreach digest", - "inbox.status_changed": "Updated an inbox message", - // Read audits are filtered out of the feed at the repo (AUDIT_READ_ACTIONS), so these labels only - // matter if a row reaches the classifier another way (a fake, or a future feed that includes them). - "user.detail_viewed": "Viewed an account", - "user.messages_viewed": "Viewed an account's messages", - "inbox.message_viewed": "Viewed an inbox message", - "mail.thread_viewed": "Viewed a mail thread", - } - return map[action] ?? action + return AUDIT_ACTION_LABELS[action] ?? action +} + +const NEIGHBOR_ACTOR = "A neighbor" +const MAIL_ACTOR = "Mail" +const OPERATOR_ACTOR = "Operator" + +function nonBlankOr(value: string | null | undefined, fallback: string): string { + return value && value.trim() !== "" ? value : fallback } export function classifyActivity(record: ActivitySourceRecord, ref: Date): ActivityItemDTO { const ts = relativeAgo(record.ts, ref) if (record.source === "report") { - const cat = record.subject && record.subject.trim() !== "" ? record.subject : "issue" - return item("pin", record.who || "A neighbor", `New ${cat} report`, record.where, ts) + const cat = nonBlankOr(record.subject, "issue") + return item("pin", record.who || NEIGHBOR_ACTOR, `New ${cat} report`, record.where, ts) } if (record.source === "cleanup") { - const title = record.subject && record.subject.trim() !== "" ? record.subject : "cleanup" - return item("cleanup_plan", record.who || "A neighbor", `Planned: ${title}`, record.where, ts) - } - if (record.source === "mail_event") { - const type = record.eventType ?? "" - // The kind comes from the same list the repo's outreach_bounce filter uses; only the label - // distinguishes the two members. - if (isMailBounceEventType(type)) { - const what = type === "failed" ? "Outreach failed" : "Outreach bounced" - return item("outreach_bounce", record.who || "Mail", what, record.where, ts) - } - if (type === "delivered") { - return item("outreach_open", record.who || "Mail", "City replied", record.where, ts) - } - return item("outreach_open", record.who || "Mail", "Outreach sent", record.where, ts) + const title = nonBlankOr(record.subject, "cleanup") + return item("cleanup_plan", record.who || NEIGHBOR_ACTOR, `Planned: ${title}`, record.where, ts) } + if (record.source === "mail_event") return classifyMailEvent(record, ts) const action = record.action ?? "" const kind = classifyAuditAction(action) - return item(kind, record.who || "Operator", describeAuditAction(action), record.where, ts) + return item(kind, record.who || OPERATOR_ACTOR, describeAuditAction(action), record.where, ts) +} + +function classifyMailEvent(record: ActivitySourceRecord, ts: string): ActivityItemDTO { + const type = record.eventType ?? "" + const who = record.who || MAIL_ACTOR + // The kind comes from the same list the repo's outreach_bounce filter uses; only the label + // distinguishes the two members. + if (isMailBounceEventType(type)) { + const what = type === "failed" ? "Outreach failed" : "Outreach bounced" + return item("outreach_bounce", who, what, record.where, ts) + } + if (type === "delivered") return item("outreach_open", who, "City replied", record.where, ts) + return item("outreach_open", who, "Outreach sent", record.where, ts) } function item( @@ -268,7 +276,7 @@ export function makeActivityService(deps: ActivityServiceDeps): ActivityService filter: parseActivityFilter(query.filter), sort: parseActivitySort(query.sort), cursor: query.cursor ?? null, - limit: clampLimit(query.limit ?? ACTIVITY_DEFAULT_LIMIT), + limit: clampLimit(query.limit ?? ADMIN_DEFAULT_LIMIT), } const { records, nextCursor } = await deps.repo.list(args) return { items: records.map((r) => classifyActivity(r, ref)), nextCursor } diff --git a/services/api/src/services/admin/admin-event-helpers.ts b/services/api/src/services/admin/admin-event-helpers.ts index 0bb05371..d35e3236 100644 --- a/services/api/src/services/admin/admin-event-helpers.ts +++ b/services/api/src/services/admin/admin-event-helpers.ts @@ -2,6 +2,13 @@ import type { EventStatus, EventTimelineItem } from "@civfix/shared" +export const EVENT_NOTE_FLAGGED = "Flagged for review" + +export const EVENT_NOTE_UNFLAGGED = "Flag cleared" + +/** The note an operator broadcast writes; the Drizzle repo spells it inline in its INSERT. */ +export const EVENT_NOTE_MESSAGE_POSTED = "Posted an update to attendees" + export function resolveEventFilter(filter: string | undefined): { status: EventStatus | null flaggedOnly: boolean @@ -60,9 +67,9 @@ export function timelineDefaultNote(kind: string): string { case "cancel": return "Event cancelled" case "flag": - return "Flagged for review" + return EVENT_NOTE_FLAGGED case "unflag": - return "Flag cleared" + return EVENT_NOTE_UNFLAGGED case "report_linked": return "Linked a report" case "report_unlinked": diff --git a/services/api/src/services/admin/admin-event-repository.drizzle.ts b/services/api/src/services/admin/admin-event-repository.drizzle.ts index 0e57f8b8..fe449ddf 100644 --- a/services/api/src/services/admin/admin-event-repository.drizzle.ts +++ b/services/api/src/services/admin/admin-event-repository.drizzle.ts @@ -9,7 +9,12 @@ import { toRecord, type EventRowSelect, } from "./admin-event-sql.js" -import { ADMIN_EVENT_MESSAGE_CAP, eventOutcomeNote } from "./admin-event-helpers.js" +import { + ADMIN_EVENT_MESSAGE_CAP, + EVENT_NOTE_FLAGGED, + EVENT_NOTE_UNFLAGGED, + eventOutcomeNote, +} from "./admin-event-helpers.js" import { andAll, type SqlFragment } from "./sql-fragments.js" import { publicReportFilter } from "../report-sql.js" import { CIVFIX_OFFICIAL_USER_ID } from "../../auth/official-account.js" @@ -25,6 +30,8 @@ import type { AdminEventCounts, ReportCategory, ReportStatus } from "@civfix/sha const LINK_REPORTS_MAX = 100 +const SYSTEM_ACTOR_NAME = "system" + export function makeDrizzleAdminEventRepository(sql: Sql): AdminEventRepository { return { async listEvents( @@ -115,7 +122,7 @@ export function makeDrizzleAdminEventRepository(sql: Sql): AdminEventRepository return rows.map((r) => ({ kind: r.kind, note: r.note, - who: r.who ?? "system", + who: r.who ?? SYSTEM_ACTOR_NAME, createdAt: r.created_at, })) }, @@ -130,7 +137,7 @@ export function makeDrizzleAdminEventRepository(sql: Sql): AdminEventRepository LIMIT ${ADMIN_EVENT_MESSAGE_CAP} ` return rows.reverse().map((r) => ({ - who: r.who ?? "system", + who: r.who ?? SYSTEM_ACTOR_NAME, text: r.body ?? "", createdAt: r.created_at, })) @@ -178,7 +185,7 @@ export function makeDrizzleAdminEventRepository(sql: Sql): AdminEventRepository INSERT INTO cleanup_timeline (cleanup_id, kind, note, actor_id) VALUES ( ${id}, ${nowFlagged ? "flag" : "unflag"}, - ${nowFlagged ? "Flagged for review" : "Flag cleared"}, + ${nowFlagged ? EVENT_NOTE_FLAGGED : EVENT_NOTE_UNFLAGGED}, ${input.actorId} ) ` diff --git a/services/api/src/services/admin/admin-event-repository.memory.ts b/services/api/src/services/admin/admin-event-repository.memory.ts index 820c51d7..615fed03 100644 --- a/services/api/src/services/admin/admin-event-repository.memory.ts +++ b/services/api/src/services/admin/admin-event-repository.memory.ts @@ -10,6 +10,9 @@ import { isUuid } from "../../db/cursor-helpers.js" import { pageInMemoryById } from "./pagination.js" import { ADMIN_EVENT_MESSAGE_CAP, + EVENT_NOTE_FLAGGED, + EVENT_NOTE_MESSAGE_POSTED, + EVENT_NOTE_UNFLAGGED, eventOutcomeNote, flaggedFromTimeline, } from "./admin-event-helpers.js" @@ -29,6 +32,13 @@ import type { import type { LinkedReportView } from "../cleanup-service.js" import type { AdminEventCounts, EventKind, EventStatus, ReportCategory } from "@civfix/shared" +const HOUR_MS = 3_600_000 + +const IN_PROGRESS_SEED_OFFSET_MS = -HOUR_MS +const UPCOMING_SEED_OFFSET_MS = 7 * 24 * HOUR_MS + +const SEEDED_ACTOR_NAME = "operator" + function seededWindow( status: string, scheduledAt: Date | undefined, @@ -42,9 +52,11 @@ function seededWindow( } function defaultStartOffsetMs(status: string): number { - if (status === "in_progress" || status === "active") return -3_600_000 - if (status === "completed" || status === "done") return -DEFAULT_EVENT_DURATION_MS - 3_600_000 - return 7 * 86_400_000 + if (status === "in_progress" || status === "active") return IN_PROGRESS_SEED_OFFSET_MS + if (status === "completed" || status === "done") { + return IN_PROGRESS_SEED_OFFSET_MS - DEFAULT_EVENT_DURATION_MS + } + return UPCOMING_SEED_OFFSET_MS } export interface RecordedMemberNotification { @@ -299,7 +311,7 @@ export class InMemoryAdminEventRepository implements AdminEventRepository { this.appendTimeline(id, { kind: "outcome", note: eventOutcomeNote(input.bags), - who: "operator", + who: SEEDED_ACTOR_NAME, createdAt: this.nextDate(), }) this.audits.push({ @@ -320,8 +332,8 @@ export class InMemoryAdminEventRepository implements AdminEventRepository { seeded.record.flagged = next this.appendTimeline(id, { kind: next ? "flag" : "unflag", - note: next ? "Flagged for review" : "Flag cleared", - who: "operator", + note: next ? EVENT_NOTE_FLAGGED : EVENT_NOTE_UNFLAGGED, + who: SEEDED_ACTOR_NAME, createdAt: this.nextDate(), }) this.audits.push({ @@ -343,7 +355,7 @@ export class InMemoryAdminEventRepository implements AdminEventRepository { this.appendTimeline(id, { kind: "cancel", note: input.note, - who: "operator", + who: SEEDED_ACTOR_NAME, createdAt: this.nextDate(), }) this.audits.push({ @@ -365,8 +377,8 @@ export class InMemoryAdminEventRepository implements AdminEventRepository { this.messages.set(id, list) this.appendTimeline(id, { kind: "message", - note: "Posted an update to attendees", - who: "operator", + note: EVENT_NOTE_MESSAGE_POSTED, + who: SEEDED_ACTOR_NAME, createdAt: this.nextDate(), }) // Mirrors the Drizzle repo's set-based in-transaction fan-out. @@ -414,7 +426,7 @@ export class InMemoryAdminEventRepository implements AdminEventRepository { async linkReports( id: string, reportIds: string[], - actorId: string | null, + _actorId: string | null, ): Promise<{ linked: string[] } | null> { if (!this.events.has(id)) return null const linked: string[] = [] @@ -426,7 +438,7 @@ export class InMemoryAdminEventRepository implements AdminEventRepository { this.appendTimeline(id, { kind: "report_linked", note: `Linked report ${reportId}`, - who: "operator", + who: SEEDED_ACTOR_NAME, createdAt: this.nextDate(), }) linked.push(reportId) @@ -436,14 +448,13 @@ export class InMemoryAdminEventRepository implements AdminEventRepository { target: `cleanup:${id}`, meta: { reportIds: linked }, }) - void actorId return { linked } } async unlinkReport( id: string, reportId: string, - actorId: string | null, + _actorId: string | null, ): Promise { if (!this.events.has(id)) return null const idx = this.links.findIndex((l) => l.cleanupId === id && l.reportId === reportId) @@ -452,7 +463,7 @@ export class InMemoryAdminEventRepository implements AdminEventRepository { this.appendTimeline(id, { kind: "report_unlinked", note: `Unlinked report ${reportId}`, - who: "operator", + who: SEEDED_ACTOR_NAME, createdAt: this.nextDate(), }) this.audits.push({ @@ -460,7 +471,6 @@ export class InMemoryAdminEventRepository implements AdminEventRepository { target: `cleanup:${id}`, meta: { reportId }, }) - void actorId return true } diff --git a/services/api/src/services/admin/admin-event-service.ts b/services/api/src/services/admin/admin-event-service.ts index 433baca7..8e6b6da0 100644 --- a/services/api/src/services/admin/admin-event-service.ts +++ b/services/api/src/services/admin/admin-event-service.ts @@ -29,7 +29,7 @@ import { timelineDefaultNote, } from "./admin-event-helpers.js" -// Re-exported so importers of the helpers' old home (tests, the memory repo) keep their import path. +// Re-exported so tests that import the helpers from their old home keep their import path. export { resolveEventFilter, eventStatusNote, @@ -37,6 +37,23 @@ export { eventTimelineKind, } from "./admin-event-helpers.js" +const ADMIN_EVENTS_DEFAULT_LIMIT = 25 + +const EVENT_NOT_FOUND = "Event not found" + +// A cleanup always HAS an organizer, so this stands in for corrupt data, not for a supported "no organizer" +// state (the reports surface's anonymous case). Hence `id: ""`, not null: the contract's organizer.id is a +// plain string. +const MISSING_ORGANIZER = { id: "", name: "Unknown", handle: "unknown" } + +const ZERO_EVENT_COUNTS: AdminEventCounts = { + all: 0, + upcoming: 0, + in_progress: 0, + completed: 0, + flagged: 0, +} + /** Field-identical to a report's reporter; the alias keeps the events domain's name for this person. */ export type AdminOrganizerRecord = AdminPersonRecord @@ -170,10 +187,7 @@ export function makeAdminEventService(deps: AdminEventServiceDeps): AdminEventSe attendees: record.attendees, capacity: record.capacity, bags: record.bags, - // A cleanup always HAS an organizer, so these fallbacks stand in for corrupt data, not for a - // supported "no organizer" state (the reports surface's anonymous case). Hence `id: ""`, not null: - // the contract's organizer.id is a plain string. - organizer: toPersonDTO(record.organizer, ref, { id: "", name: "Unknown", handle: "unknown" }), + organizer: toPersonDTO(record.organizer, ref, MISSING_ORGANIZER), date: toRelAbs(record.scheduledAt, ref), coords: [record.lat, record.lng], } @@ -201,7 +215,7 @@ export function makeAdminEventService(deps: AdminEventServiceDeps): AdminEventSe status, flaggedOnly, cursor: query.cursor ?? null, - limit: query.limit ?? 25, + limit: query.limit ?? ADMIN_EVENTS_DEFAULT_LIMIT, } // The counts describe the whole searched set rather than the page, so they are computed on page 1 // only (the shared admin-list policy; the console reads them off the first page). @@ -209,13 +223,7 @@ export function makeAdminEventService(deps: AdminEventServiceDeps): AdminEventSe deps.repo.listEvents(args), args.cursor === null ? deps.repo.countByBucket({ q: args.q }) - : Promise.resolve({ - all: 0, - upcoming: 0, - in_progress: 0, - completed: 0, - flagged: 0, - }), + : Promise.resolve({ ...ZERO_EVENT_COUNTS }), ]) return { items: records.map((r) => toListItem(r, ref)), nextCursor, counts } }, @@ -240,7 +248,7 @@ export function makeAdminEventService(deps: AdminEventServiceDeps): AdminEventSe async get(id: string): Promise { const ref = now() const record = await deps.repo.getEvent(id) - if (!record) throw AppError.notFound("Event not found") + if (!record) throw AppError.notFound(EVENT_NOT_FOUND) const [timeline, messages, linkedViews] = await Promise.all([ deps.repo.listTimeline(id), deps.repo.listMessages(id), @@ -278,12 +286,12 @@ export function makeAdminEventService(deps: AdminEventServiceDeps): AdminEventSe note: eventStatusNote("cancelled"), actorId: input.actorId, }) - if (!ok) throw AppError.notFound("Event not found") + if (!ok) throw AppError.notFound(EVENT_NOT_FOUND) }, async setOutcome(id: string, input: { bags: number; actorId: string | null }): Promise { const ok = await deps.repo.setBags(id, input) - if (!ok) throw AppError.notFound("Event not found") + if (!ok) throw AppError.notFound(EVENT_NOT_FOUND) }, async flag( @@ -291,7 +299,7 @@ export function makeAdminEventService(deps: AdminEventServiceDeps): AdminEventSe input: { reason: string | null; actorId: string | null }, ): Promise { const flagged = await deps.repo.toggleFlag(id, input) - if (flagged === null) throw AppError.notFound("Event not found") + if (flagged === null) throw AppError.notFound(EVENT_NOT_FOUND) return flagged }, @@ -304,12 +312,12 @@ export function makeAdminEventService(deps: AdminEventServiceDeps): AdminEventSe ? `Event cancelled: ${input.reason.trim()}` : eventStatusNote("cancelled") const ok = await deps.repo.cancel(id, { note, actorId: input.actorId }) - if (!ok) throw AppError.notFound("Event not found") + if (!ok) throw AppError.notFound(EVENT_NOT_FOUND) }, async postMessage(id: string, input: { body: string; actorId: string }): Promise { const result = await deps.repo.postMessage(id, input) - if (result === null) throw AppError.notFound("Event not found") + if (result === null) throw AppError.notFound(EVENT_NOT_FOUND) return result.notified }, @@ -320,18 +328,18 @@ export function makeAdminEventService(deps: AdminEventServiceDeps): AdminEventSe ): Promise<{ linked: string[] }> { // Rejected before any write, so a non-cleanup event never gains a link. const record = await deps.repo.getEvent(id) - if (!record) throw AppError.notFound("Event not found") + if (!record) throw AppError.notFound(EVENT_NOT_FOUND) if (record.eventKind !== "cleanup") { throw AppError.validation({ reportIds: "only cleanup events can link reports" }) } const result = await deps.repo.linkReports(id, reportIds, actorId) - if (result === null) throw AppError.notFound("Event not found") + if (result === null) throw AppError.notFound(EVENT_NOT_FOUND) return result }, async unlinkReport(id: string, reportId: string, actorId: string | null): Promise { const result = await deps.repo.unlinkReport(id, reportId, actorId) - if (result === null) throw AppError.notFound("Event not found") + if (result === null) throw AppError.notFound(EVENT_NOT_FOUND) // result === false means there was no such link; the unlink is idempotent so that is still a success. }, } diff --git a/services/api/src/services/admin/admin-report-chat-service.ts b/services/api/src/services/admin/admin-report-chat-service.ts index 26ac5109..3a45e749 100644 --- a/services/api/src/services/admin/admin-report-chat-service.ts +++ b/services/api/src/services/admin/admin-report-chat-service.ts @@ -12,8 +12,8 @@ import { sendReportChatMessage, type ReportChatSendDeps } from "../report-chat-s import type { AdminReportChatRepository } from "./admin-report-chat-repository.drizzle.js" import { CIVFIX_OFFICIAL_USER_ID } from "../../auth/official-account.js" -export const ADMIN_REPORT_CHAT_HISTORY_DEFAULT = 30 -export const ADMIN_REPORT_CHAT_HISTORY_MAX = 50 +const ADMIN_REPORT_CHAT_HISTORY_DEFAULT = 30 +const ADMIN_REPORT_CHAT_HISTORY_MAX = 50 /** Called only after the operator's change committed; never throws. */ export type MessageUpdateAnnouncer = (messageId: string) => Promise diff --git a/services/api/src/services/admin/admin-report-repository.drizzle.ts b/services/api/src/services/admin/admin-report-repository.drizzle.ts index 205c75e4..322dacf8 100644 --- a/services/api/src/services/admin/admin-report-repository.drizzle.ts +++ b/services/api/src/services/admin/admin-report-repository.drizzle.ts @@ -28,7 +28,7 @@ import { type AdminReportTimelineRecord, type ListReportsArgs, type ReportOutreachState, -} from "./admin-report-service.js" +} from "./admin-report-types.js" import type { AdminReportCounts, AdminReportStatus, @@ -68,6 +68,9 @@ function searchReportsFragment(sql: Queryable, q: string | null): SqlFragment { const MEDIA_CAP = 20 +const UNTITLED_REPORT_TITLE = "Untitled report" +const UNNAMED_REPORTER_NAME = "Neighbor" + interface ReportRowSelect { id: string category: ReportCategory @@ -110,7 +113,7 @@ function toRecord(r: ReportRowSelect): AdminReportRecord { hasOauth: r.reporter_has_oauth, joinedAt: r.reporter_joined, }, - "Neighbor", + UNNAMED_REPORTER_NAME, ) return { id: r.id, @@ -118,7 +121,7 @@ function toRecord(r: ReportRowSelect): AdminReportRecord { status: r.status, visibility: r.visibility, flagged: r.flagged, - title: r.title ?? "Untitled report", + title: r.title ?? UNTITLED_REPORT_TITLE, place: r.place ?? "", reporter, confirmations: Number(r.confirmations ?? "0"), diff --git a/services/api/src/services/admin/admin-report-repository.memory.ts b/services/api/src/services/admin/admin-report-repository.memory.ts index 7afd5047..c7db6551 100644 --- a/services/api/src/services/admin/admin-report-repository.memory.ts +++ b/services/api/src/services/admin/admin-report-repository.memory.ts @@ -10,7 +10,7 @@ import type { AdminReportTimelineRecord, ListReportsArgs, ReportOutreachState, -} from "./admin-report-service.js" +} from "./admin-report-types.js" import type { AdminReportCounts, AdminReportStatus, @@ -20,7 +20,7 @@ import type { } from "@civfix/shared" import { mapOutreachStatus } from "./admin-report-repository.drizzle.js" import { isPacketKind, type MailMessageKind } from "./mail-repository.js" -import { pickPreviewMedia, REPORT_VERIFIED_THRESHOLD } from "./admin-report-service.js" +import { pickPreviewMedia, REPORT_VERIFIED_THRESHOLD } from "./admin-report-types.js" import { STATUS_BUCKETS } from "./admin-report-status.js" export interface RecordedAudit { @@ -278,7 +278,6 @@ export class InMemoryAdminReportRepository implements AdminReportRepository { ): Promise { const seeded = this.reports.get(id) if (!seeded) return - void input.body this.appendTimeline(id, { status: seeded.record.status, note: input.note, @@ -304,7 +303,6 @@ export class InMemoryAdminReportRepository implements AdminReportRepository { ): Promise { const seeded = this.reports.get(id) if (!seeded || seeded.deletedAt !== null) return false - void input.body seeded.record.status = input.status if (input.status === "rejected") seeded.deletedAt = seeded.deletedAt ?? this.nextDate() this.appendTimeline(id, { diff --git a/services/api/src/services/admin/admin-report-service.ts b/services/api/src/services/admin/admin-report-service.ts index 8d417183..fc72bee9 100644 --- a/services/api/src/services/admin/admin-report-service.ts +++ b/services/api/src/services/admin/admin-report-service.ts @@ -21,8 +21,9 @@ import type { PacketAttachment } from "./outbound-mail-service.js" import { toLinkedEventRef, type LinkedEventView } from "../cleanup-service.js" import { toRelAbs } from "./admin-format.js" import { toPersonDTO } from "./admin-person.js" -import { mapWithLimit, PRESIGN_CONCURRENCY } from "../media-presign.js" +import { mapWithLimit, PRESIGN_CONCURRENCY, type PresignPacketMedia } from "../media-presign.js" import { isPubliclyVisibleStatus } from "../report-visibility.js" +import { ADMIN_DEFAULT_LIMIT } from "./pagination.js" import { JURISDICTION_REPLY_NOTE, resolveListFilter, @@ -52,17 +53,10 @@ import type { export * from "./admin-report-types.js" export * from "./admin-report-status.js" -export { - buildReportPacket, - attachmentFilename, - MAX_PACKET_ATTACHMENTS, - MAX_PACKET_ATTACHMENT_BYTES, - type ReportPacket, -} from "./mail-format.js" const ROUTABLE_FROM_STATUSES: readonly AdminReportStatus[] = ["submitted", "held", "published"] -export const ALREADY_ROUTED_CONFLICT = "This report has already been sent to its jurisdiction" +const ALREADY_ROUTED_CONFLICT = "This report has already been sent to its jurisdiction" export const SEND_IN_FLIGHT_CONFLICT = "A send to this jurisdiction is still in progress. Check back shortly to see the outcome on the outreach trail." @@ -75,6 +69,33 @@ export function isAlreadyRoutedConflict(err: unknown): boolean { ) } +const REPORT_NOT_FOUND = "Report not found" + +const REPORTER_FOLLOWUP_NOTE = "Follow-up sent to the reporter" + +const ANONYMOUS_REPORTER = { id: null, name: "Anonymous", handle: "anonymous" } as const + +const EMPTY_REPORT_COUNTS: AdminReportCounts = { + all: 0, + submitted: 0, + in_progress: 0, + completed: 0, + flagged: 0, + needsVerification: 0, +} + +const FALLBACK_ATTACHMENT_MIME = "image/jpeg" + +const JPEG_SIGNATURE = [0xff, 0xd8, 0xff] +const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a] +// A WebP file is a RIFF container: "RIFF", a 4-byte size, then "WEBP". +const RIFF_SIGNATURE = [0x52, 0x49, 0x46, 0x46] +const WEBP_FORM_TYPE = [0x57, 0x45, 0x42, 0x50] +const WEBP_FORM_TYPE_OFFSET = 8 + +const PG_SERIALIZATION_FAILURE = "40001" +const PG_DEADLOCK_DETECTED = "40P01" + function firstTemplate(...candidates: (string | null | undefined)[]): string | null { for (const candidate of candidates) { if (typeof candidate === "string" && candidate.trim() !== "") return candidate @@ -91,17 +112,6 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor const presignPacketMedia = deps.presignPacketMedia ?? (async (r2Key: string) => (await presignMedia(r2Key, null)).url) - async function emitTimeline(event: { - reportId: string - status: AdminReportStatus - kind?: ReportTimelineItem["kind"] - note?: string | null - body?: string | null - }): Promise { - if (deps.reportChatEmitter === undefined) return - await deps.reportChatEmitter.emit(event) - } - async function toMediaDTO(m: AdminReportMediaRecord): Promise { const { url, thumbUrl } = await presignMedia(m.r2Key, m.thumbKey) return { id: m.id, kind: m.kind, url, thumbUrl: thumbUrl ?? null } @@ -123,11 +133,7 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor flagged: record.flagged, title: record.title, place: record.place, - reporter: toPersonDTO(record.reporter, ref, { - id: null, - name: "Anonymous", - handle: "anonymous", - }), + reporter: toPersonDTO(record.reporter, ref, ANONYMOUS_REPORTER), confirmations: record.confirmations, submitted: toRelAbs(record.createdAt, ref), coords: [record.lat, record.lng], @@ -138,17 +144,11 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor } function toTimelineDTO(record: AdminReportTimelineRecord, ref: Date): ReportTimelineItem { - const kind: ReportTimelineItem["kind"] = - (record.kind ?? null) !== null - ? (record.kind as ReportTimelineItem["kind"]) - : record.note?.startsWith(JURISDICTION_REPLY_NOTE) === true - ? "reply" - : timelineKindForStatus(record.status) return { who: record.who, what: record.note ?? statusChangeNote(record.status), when: toRelAbs(record.createdAt, ref).rel, - kind, + kind: timelineRecordKind(record), } } @@ -162,20 +162,13 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor flaggedOnly, needsVerificationOnly, cursor: query.cursor ?? null, - limit: query.limit ?? 25, + limit: query.limit ?? ADMIN_DEFAULT_LIMIT, } const [{ records, nextCursor }, counts] = await Promise.all([ deps.repo.listReports(args), args.cursor === null ? deps.repo.countByBucket({ q: args.q }) - : Promise.resolve({ - all: 0, - submitted: 0, - in_progress: 0, - completed: 0, - flagged: 0, - needsVerification: 0, - }), + : Promise.resolve({ ...EMPTY_REPORT_COUNTS }), ]) const items = await mapWithLimit(records, PRESIGN_CONCURRENCY, async (r) => toListItem(r, ref, await previewMediaDTO(r)), @@ -186,7 +179,7 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor async get(id: string): Promise { const ref = now() const record = await deps.repo.getReport(id) - if (!record) throw AppError.notFound("Report not found") + if (!record) throw AppError.notFound(REPORT_NOT_FOUND) const [timeline, routing, media, linkedEventsMap, outreach] = await Promise.all([ deps.repo.listTimeline(id), deps.repo.getRouting(id), @@ -237,7 +230,7 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor input: { status: AdminReportStatus; actorId: string | null }, ): Promise { const record = await deps.repo.getReport(id) - if (!record) throw AppError.notFound("Report not found") + if (!record) throw AppError.notFound(REPORT_NOT_FOUND) if (input.status === "rejected") { throw AppError.validation({ status: "use_remove" }, "Use Remove to reject a report") } @@ -265,7 +258,7 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor ) } await notifyReporterOfStatus(deps, id, record.reporter?.id ?? null, input.status) - await emitTimeline({ + await emitTimeline(deps, { reportId: id, status: input.status, kind: timelineKindForStatus(input.status), @@ -278,12 +271,12 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor input: { reason: string | null; actorId: string | null }, ): Promise { const flagged = await deps.repo.toggleFlag(id, input) - if (flagged === null) throw AppError.notFound("Report not found") + if (flagged === null) throw AppError.notFound(REPORT_NOT_FOUND) // The toggle is committed; failing the request would invite a retry, and a retried toggle unflags. try { const record = await deps.repo.getReport(id) if (record) { - await emitTimeline({ + await emitTimeline(deps, { reportId: id, status: record.status, kind: "status", @@ -305,8 +298,8 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor ? `Report removed: ${input.reason.trim()}` : statusChangeNote("rejected") const ok = await deps.repo.remove(id, { note, actorId: input.actorId }) - if (!ok) throw AppError.notFound("Report not found") - await emitTimeline({ + if (!ok) throw AppError.notFound(REPORT_NOT_FOUND) + await emitTimeline(deps, { reportId: id, status: "rejected", kind: timelineKindForStatus("rejected"), @@ -319,74 +312,11 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor input: { to: "reporter" | "city"; body: string; actorId: string | null }, ): Promise { const record = await deps.repo.getReport(id) - if (!record) throw AppError.notFound("Report not found") + if (!record) throw AppError.notFound(REPORT_NOT_FOUND) - if (input.to === "reporter") { - const reporterId = record.reporter?.id - if (!reporterId || reporterId === "") { - throw AppError.validation({ to: "report has no reporter account to notify" }) - } - const notifications = deps.notifications - if (notifications === undefined) { - throw AppError.internal("Reporter notifications are not wired on this instance") - } - // Audit before the citizen is messaged, so no operator message ever reaches a reporter unrecorded. - await recordFollowup(deps, id, { - note: "Follow-up sent to the reporter", - actorId: input.actorId, - to: "reporter", - destination: reporterId, - }) - await notifications.createNotification(reporterId, { - type: "report_update", - title: "Update on your report", - body: input.body, - link: `/reports/${id}`, - }) - await emitTimeline({ - reportId: id, - status: record.status, - kind: "status", - note: "Follow-up sent to the reporter", - }) - return { to: "reporter", destination: reporterId } - } - - const routing = await deps.repo.getRouting(id) - const outreach = await deps.repo.getOutreach(id) - assertNoSendInFlight(outreach) - if (outreach.threadId === null) { - throw AppError.validation( - { to: "not_routed" }, - "Send the report to the jurisdiction first; follow-ups go on that conversation", - ) - } - const destination = outreach.routedTo ?? routing?.contact ?? null - if (destination === null || destination === "") { - throw AppError.validation({ to: "no city contact on file for this report" }) - } - await deps.outboundMail.appendOutbound(outreach.threadId, { - body: input.body, - toAddr: destination, - audit: { - actorId: input.actorId, - action: "mail.replied", - meta: { reportId: id, to: destination }, - }, - }) - await recordFollowup(deps, id, { - note: `Follow-up sent to ${destination}`, - actorId: input.actorId, - to: "city", - destination, - }) - await emitTimeline({ - reportId: id, - status: record.status, - kind: "status", - note: `Follow-up sent to ${destination}`, - }) - return { to: "city", destination } + return input.to === "reporter" + ? followupToReporter(deps, id, record, input) + : followupToCity(deps, id, record, input) }, async routeToJurisdiction( @@ -394,7 +324,7 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor input: { note: string | null; actorId: string | null }, ): Promise { const record = await deps.repo.getReport(id) - if (!record) throw AppError.notFound("Report not found") + if (!record) throw AppError.notFound(REPORT_NOT_FOUND) const routing = await deps.repo.getRouting(id) const toAddr = routing?.contact ?? null @@ -406,27 +336,12 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor assertRoutable(await deps.repo.getOutreach(id), toAddr) - const media = await deps.repo.listMedia(id) - const publiclyVisible = - isPubliclyVisibleStatus(record.status) && record.visibility === "public" - const packetMedia: PacketMediaLink[] = [] - const attachments: PacketAttachment[] = [] - let attachedBytesTotal = 0 - for (const m of media) { - packetMedia.push({ kind: m.kind, url: await presignPacketMedia(m.r2Key, publiclyVisible) }) - if (m.kind !== "image" || attachments.length >= MAX_PACKET_ATTACHMENTS) continue - const bytes = deps.loadMediaBytes ? await deps.loadMediaBytes(m.r2Key) : null - if (bytes === null || bytes.byteLength > MAX_PACKET_ATTACHMENT_BYTES) continue - if (attachedBytesTotal + bytes.byteLength > MAX_PACKET_TOTAL_BYTES) continue - attachedBytesTotal += bytes.byteLength - const contentType = attachmentContentType(m.contentType, bytes) - attachments.push({ - key: m.r2Key, - filename: attachmentFilename(m.r2Key, attachments.length, contentType), - contentType, - content: bytes, - }) - } + const { packetMedia, attachments } = await collectPacketMedia( + await deps.repo.listMedia(id), + isPubliclyVisibleStatus(record.status) && record.visibility === "public", + presignPacketMedia, + deps.loadMediaBytes, + ) const defaults = deps.forwardTemplates !== undefined ? await deps.forwardTemplates.get() : null @@ -455,31 +370,17 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor }) }) - const recordRouteOutcome = async (): Promise => { - let advanced = false - await retryOnce(async () => { - advanced = await deps.repo.advanceStatusIfIn(id, { - from: ROUTABLE_FROM_STATUSES, - to: "acknowledged", - note: routeNote, - actorId: input.actorId, - kind: "route", - }) - if (!advanced) { - await deps.repo.appendSystemTimeline(id, { note: routeNote, kind: "route" }) - } - }) - const current = advanced - ? "acknowledged" - : ((await deps.repo.getReport(id))?.status ?? record.status) - await emitTimeline({ reportId: id, status: current, kind: "route", note: routeNote }) + const outcome: RouteOutcome = { + reportId: id, + note: routeNote, + actorId: input.actorId, + statusBefore: record.status, } - - await prepared.deliver({ onLateSuccess: recordRouteOutcome }) + await prepared.deliver({ onLateSuccess: () => recordRouteOutcome(deps, outcome) }) const threadId = prepared.thread.id - await recordRouteOutcome() + await recordRouteOutcome(deps, outcome) return { threadId, routedTo: toAddr } }, @@ -496,9 +397,9 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor actorId: input.actorId, note, }) - if (!ok) throw AppError.notFound("Report not found") + if (!ok) throw AppError.notFound(REPORT_NOT_FOUND) const record = await deps.repo.getReport(input.id) - await emitTimeline({ + await emitTimeline(deps, { reportId: input.id, status: record?.status ?? "submitted", kind: "status", @@ -508,30 +409,108 @@ export function makeAdminReportService(deps: AdminReportServiceDeps): AdminRepor } } +function timelineRecordKind(record: AdminReportTimelineRecord): ReportTimelineItem["kind"] { + if ((record.kind ?? null) !== null) return record.kind as ReportTimelineItem["kind"] + if (record.note?.startsWith(JURISDICTION_REPLY_NOTE) === true) return "reply" + return timelineKindForStatus(record.status) +} + +async function emitTimeline( + deps: AdminReportServiceDeps, + event: { + reportId: string + status: AdminReportStatus + kind?: ReportTimelineItem["kind"] + note?: string | null + body?: string | null + }, +): Promise { + if (deps.reportChatEmitter === undefined) return + await deps.reportChatEmitter.emit(event) +} + +/** + * Every media item gets a link; only images within the per-file, count and total byte caps are also + * attached. Presigning and loading stay sequential, in media order, so attachment filenames are stable. + */ +async function collectPacketMedia( + media: readonly AdminReportMediaRecord[], + publiclyVisible: boolean, + presignPacketMedia: PresignPacketMedia, + loadMediaBytes: AdminReportServiceDeps["loadMediaBytes"], +): Promise<{ packetMedia: PacketMediaLink[]; attachments: PacketAttachment[] }> { + const packetMedia: PacketMediaLink[] = [] + const attachments: PacketAttachment[] = [] + let attachedBytesTotal = 0 + for (const m of media) { + packetMedia.push({ kind: m.kind, url: await presignPacketMedia(m.r2Key, publiclyVisible) }) + if (m.kind !== "image" || attachments.length >= MAX_PACKET_ATTACHMENTS) continue + const bytes = loadMediaBytes ? await loadMediaBytes(m.r2Key) : null + if (bytes === null || bytes.byteLength > MAX_PACKET_ATTACHMENT_BYTES) continue + if (attachedBytesTotal + bytes.byteLength > MAX_PACKET_TOTAL_BYTES) continue + attachedBytesTotal += bytes.byteLength + const contentType = attachmentContentType(m.contentType, bytes) + attachments.push({ + key: m.r2Key, + filename: attachmentFilename(m.r2Key, attachments.length, contentType), + contentType, + content: bytes, + }) + } + return { packetMedia, attachments } +} + +interface RouteOutcome { + reportId: string + note: string + actorId: string | null + statusBefore: AdminReportStatus +} + +async function recordRouteOutcome( + deps: AdminReportServiceDeps, + outcome: RouteOutcome, +): Promise { + const { reportId, note, actorId } = outcome + let advanced = false + await retryOnce(async () => { + advanced = await deps.repo.advanceStatusIfIn(reportId, { + from: ROUTABLE_FROM_STATUSES, + to: "acknowledged", + note, + actorId, + kind: "route", + }) + if (!advanced) { + await deps.repo.appendSystemTimeline(reportId, { note, kind: "route" }) + } + }) + const current = advanced + ? "acknowledged" + : ((await deps.repo.getReport(reportId))?.status ?? outcome.statusBefore) + await emitTimeline(deps, { reportId, status: current, kind: "route", note }) +} + export function attachmentContentType( stored: string | null | undefined, bytes: Uint8Array, ): string { if (stored !== null && stored !== undefined && stored.trim() !== "") return stored - return sniffImageMime(bytes) ?? "image/jpeg" + return sniffImageMime(bytes) ?? FALLBACK_ATTACHMENT_MIME +} + +function hasBytesAt(bytes: Uint8Array, offset: number, expected: readonly number[]): boolean { + return ( + bytes.length >= offset + expected.length && expected.every((b, i) => bytes[offset + i] === b) + ) } function sniffImageMime(bytes: Uint8Array): string | null { - if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { - return "image/jpeg" - } - const PNG = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a] - if (bytes.length >= PNG.length && PNG.every((b, i) => bytes[i] === b)) return "image/png" + if (hasBytesAt(bytes, 0, JPEG_SIGNATURE)) return "image/jpeg" + if (hasBytesAt(bytes, 0, PNG_SIGNATURE)) return "image/png" if ( - bytes.length >= 12 && - bytes[0] === 0x52 && - bytes[1] === 0x49 && - bytes[2] === 0x46 && - bytes[3] === 0x46 && - bytes[8] === 0x57 && - bytes[9] === 0x45 && - bytes[10] === 0x42 && - bytes[11] === 0x50 + hasBytesAt(bytes, 0, RIFF_SIGNATURE) && + hasBytesAt(bytes, WEBP_FORM_TYPE_OFFSET, WEBP_FORM_TYPE) ) { return "image/webp" } @@ -594,6 +573,90 @@ async function notifyReporterOfStatus( } } +interface FollowupInput { + body: string + actorId: string | null +} + +async function followupToReporter( + deps: AdminReportServiceDeps, + id: string, + record: AdminReportRecord, + input: FollowupInput, +): Promise { + const reporterId = record.reporter?.id + if (!reporterId || reporterId === "") { + throw AppError.validation({ to: "report has no reporter account to notify" }) + } + const notifications = deps.notifications + if (notifications === undefined) { + throw AppError.internal("Reporter notifications are not wired on this instance") + } + // Audit before the citizen is messaged, so no operator message ever reaches a reporter unrecorded. + await recordFollowup(deps, id, { + note: REPORTER_FOLLOWUP_NOTE, + actorId: input.actorId, + to: "reporter", + destination: reporterId, + }) + await notifications.createNotification(reporterId, { + type: "report_update", + title: "Update on your report", + body: input.body, + link: `/reports/${id}`, + }) + await emitTimeline(deps, { + reportId: id, + status: record.status, + kind: "status", + note: REPORTER_FOLLOWUP_NOTE, + }) + return { to: "reporter", destination: reporterId } +} + +async function followupToCity( + deps: AdminReportServiceDeps, + id: string, + record: AdminReportRecord, + input: FollowupInput, +): Promise { + const routing = await deps.repo.getRouting(id) + const outreach = await deps.repo.getOutreach(id) + assertNoSendInFlight(outreach) + if (outreach.threadId === null) { + throw AppError.validation( + { to: "not_routed" }, + "Send the report to the jurisdiction first; follow-ups go on that conversation", + ) + } + const destination = outreach.routedTo ?? routing?.contact ?? null + if (destination === null || destination === "") { + throw AppError.validation({ to: "no city contact on file for this report" }) + } + await deps.outboundMail.appendOutbound(outreach.threadId, { + body: input.body, + toAddr: destination, + audit: { + actorId: input.actorId, + action: "mail.replied", + meta: { reportId: id, to: destination }, + }, + }) + await recordFollowup(deps, id, { + note: `Follow-up sent to ${destination}`, + actorId: input.actorId, + to: "city", + destination, + }) + await emitTimeline(deps, { + reportId: id, + status: record.status, + kind: "status", + note: `Follow-up sent to ${destination}`, + }) + return { to: "city", destination } +} + async function recordFollowup( deps: AdminReportServiceDeps, id: string, @@ -613,5 +676,5 @@ async function retryOnce(fn: () => Promise): Promise { function isRetryableSerializationFailure(err: unknown): boolean { const code = (err as { code?: unknown } | null | undefined)?.code - return code === "40001" || code === "40P01" + return code === PG_SERIALIZATION_FAILURE || code === PG_DEADLOCK_DETECTED } diff --git a/services/api/src/services/admin/admin-report-status.ts b/services/api/src/services/admin/admin-report-status.ts index 8d8605c4..55ec7f10 100644 --- a/services/api/src/services/admin/admin-report-status.ts +++ b/services/api/src/services/admin/admin-report-status.ts @@ -21,40 +21,37 @@ export const STATUS_BUCKETS: Record< completed: ["resolved"], } -export function resolveListFilter(filter: string | undefined): { +export interface ReportListFilter { statuses: AdminReportStatus[] | null flaggedOnly: boolean needsVerificationOnly: boolean -} { +} + +function listFilter( + statuses: AdminReportStatus[] | null, + flags: { flaggedOnly?: boolean; needsVerificationOnly?: boolean } = {}, +): ReportListFilter { + return { + statuses, + flaggedOnly: flags.flaggedOnly ?? false, + needsVerificationOnly: flags.needsVerificationOnly ?? false, + } +} + +export function resolveListFilter(filter: string | undefined): ReportListFilter { switch (filter) { case "submitted": - return { - statuses: STATUS_BUCKETS.submitted, - flaggedOnly: false, - needsVerificationOnly: false, - } + return listFilter(STATUS_BUCKETS.submitted) case "in_progress": - return { - statuses: STATUS_BUCKETS.in_progress, - flaggedOnly: false, - needsVerificationOnly: false, - } + return listFilter(STATUS_BUCKETS.in_progress) case "completed": - return { - statuses: STATUS_BUCKETS.completed, - flaggedOnly: false, - needsVerificationOnly: false, - } + return listFilter(STATUS_BUCKETS.completed) case "flagged": - return { statuses: null, flaggedOnly: true, needsVerificationOnly: false } + return listFilter(null, { flaggedOnly: true }) case "needs_verification": - return { - statuses: STATUS_BUCKETS.submitted, - flaggedOnly: false, - needsVerificationOnly: true, - } + return listFilter(STATUS_BUCKETS.submitted, { needsVerificationOnly: true }) default: - return { statuses: null, flaggedOnly: false, needsVerificationOnly: false } + return listFilter(null) } } diff --git a/services/api/src/services/admin/admin-report-types.ts b/services/api/src/services/admin/admin-report-types.ts index 5c59896d..d8f5eead 100644 --- a/services/api/src/services/admin/admin-report-types.ts +++ b/services/api/src/services/admin/admin-report-types.ts @@ -3,7 +3,6 @@ import type { AdminReportDTO, AdminReportListQuery, AdminReportListResponse, - AdminReportListItemDTO, AdminReportStatus, ReportCategory, ReportMedia, @@ -207,5 +206,3 @@ export interface AdminReportService { actorId: string | null }): Promise } - -export type { AdminReportListItemDTO } diff --git a/services/api/src/services/admin/admin-user-repository.drizzle.ts b/services/api/src/services/admin/admin-user-repository.drizzle.ts index 03df25b8..de662320 100644 --- a/services/api/src/services/admin/admin-user-repository.drizzle.ts +++ b/services/api/src/services/admin/admin-user-repository.drizzle.ts @@ -31,12 +31,36 @@ import type { } from "@civfix/shared" import { likeContains } from "./like.js" -export const ADMIN_USER_ORGANIZATIONS_LIMIT = 25 +const ADMIN_USER_ORGANIZATIONS_LIMIT = 25 const EPOCH = new Date(0) const CITY_MATCH_USER_CAP = 5000 +const UNNAMED_USER_NAME = "Neighbor" +const UNTITLED_REPORT_TITLE = "Untitled report" +const UNTITLED_CLEANUP_TITLE = "Cleanup" + +interface FacetCountRow { + all: string + active: string + suspended: string + flagged: string + deleted: string + banned: string +} + +function toFacetCounts(row: FacetCountRow | undefined): AdminUserCounts { + return { + all: Number(row?.all ?? "0"), + active: Number(row?.active ?? "0"), + suspended: Number(row?.suspended ?? "0"), + flagged: Number(row?.flagged ?? "0"), + deleted: Number(row?.deleted ?? "0"), + banned: Number(row?.banned ?? "0"), + } +} + function searchUsersFragment( sql: Queryable, q: string | null, @@ -86,7 +110,7 @@ interface UserRowSelect { function toRecord(r: UserRowSelect): AdminUserRecord { return { id: r.id, - name: r.name ?? "Neighbor", + name: r.name ?? UNNAMED_USER_NAME, handle: r.handle, emailVerified: r.email_verified, city: r.city ?? "", @@ -195,16 +219,7 @@ export function makeDrizzleAdminUserRepository(sql: Sql): AdminUserRepository { async countByFacet(args: { q: string | null }): Promise { if (args.q === null) { - const rows = await sql< - { - all: string - active: string - suspended: string - flagged: string - deleted: string - banned: string - }[] - >` + const rows = await sql` SELECT COUNT(*)::text AS all, COUNT(*) FILTER (WHERE COALESCE(um.account_status, 'active') = 'active')::text AS active, @@ -215,29 +230,12 @@ export function makeDrizzleAdminUserRepository(sql: Sql): AdminUserRepository { FROM users u LEFT JOIN user_moderation um ON um.user_id = u.id ` - const exact = rows[0] - return { - all: Number(exact?.all ?? "0"), - active: Number(exact?.active ?? "0"), - suspended: Number(exact?.suspended ?? "0"), - flagged: Number(exact?.flagged ?? "0"), - deleted: Number(exact?.deleted ?? "0"), - banned: Number(exact?.banned ?? "0"), - } + return toFacetCounts(rows[0]) } const cityUserIds = await resolveCityMatchUserIds(sql, args.q) const search = searchUsersFragment(sql, args.q, cityUserIds) - const rows = await sql< - { - all: string - active: string - suspended: string - flagged: string - deleted: string - banned: string - }[] - >` + const rows = await sql` SELECT COUNT(*)::text AS all, COUNT(*) FILTER (WHERE COALESCE(um.account_status, 'active') = 'active')::text AS active, @@ -250,15 +248,7 @@ export function makeDrizzleAdminUserRepository(sql: Sql): AdminUserRepository { WHERE TRUE ${search} ` - const r = rows[0] - return { - all: Number(r?.all ?? "0"), - active: Number(r?.active ?? "0"), - suspended: Number(r?.suspended ?? "0"), - flagged: Number(r?.flagged ?? "0"), - deleted: Number(r?.deleted ?? "0"), - banned: Number(r?.banned ?? "0"), - } + return toFacetCounts(rows[0]) }, async userExists(id: string): Promise { @@ -315,7 +305,7 @@ export function makeDrizzleAdminUserRepository(sql: Sql): AdminUserRepository { records: items.map((r) => ({ id: r.id, category: r.category, - title: r.title ?? "Untitled report", + title: r.title ?? UNTITLED_REPORT_TITLE, place: r.place ?? "", status: r.status, createdAt: r.created_at, @@ -368,7 +358,7 @@ export function makeDrizzleAdminUserRepository(sql: Sql): AdminUserRepository { return { records: items.map((r) => ({ id: r.id, - title: r.title ?? "Cleanup", + title: r.title ?? UNTITLED_CLEANUP_TITLE, place: r.place ?? "", role: r.role, attendees: Number(r.attendees ?? "0"), diff --git a/services/api/src/services/admin/admin-user-repository.memory.ts b/services/api/src/services/admin/admin-user-repository.memory.ts index 82c6e8c7..de5ecd01 100644 --- a/services/api/src/services/admin/admin-user-repository.memory.ts +++ b/services/api/src/services/admin/admin-user-repository.memory.ts @@ -17,6 +17,20 @@ export interface RecordedUserAudit { meta: Record } +function appendTo(map: Map, key: string, row: T): void { + const list = map.get(key) ?? [] + list.push(row) + map.set(key, list) +} + +function userSearchMatcher(q: string): (record: AdminUserRecord) => boolean { + const needle = q.toLowerCase() + return (r) => + r.name.toLowerCase().includes(needle) || + (r.handle?.toLowerCase().includes(needle) ?? false) || + r.city.toLowerCase().includes(needle) +} + export class InMemoryAdminUserRepository implements AdminUserRepository { readonly users = new Map() readonly reports = new Map() @@ -76,15 +90,11 @@ export class InMemoryAdminUserRepository implements AdminUserRepository { } seedReport(userId: string, row: UserReportRecord): void { - const list = this.reports.get(userId) ?? [] - list.push(row) - this.reports.set(userId, list) + appendTo(this.reports, userId, row) } seedEvent(userId: string, row: UserEventRecord): void { - const list = this.events.get(userId) ?? [] - list.push(row) - this.events.set(userId, list) + appendTo(this.events, userId, row) } seedMessage( @@ -95,14 +105,12 @@ export class InMemoryAdminUserRepository implements AdminUserRepository { sourceId?: string | null }, ): void { - const list = this.messages.get(userId) ?? [] - list.push({ + appendTo(this.messages, userId, { ...row, deletedAt: row.deletedAt ?? null, source: row.source ?? "chat", sourceId: row.sourceId ?? null, }) - this.messages.set(userId, list) } async listUsers( @@ -110,15 +118,7 @@ export class InMemoryAdminUserRepository implements AdminUserRepository { ): Promise<{ records: AdminUserRecord[]; nextCursor: string | null }> { let rows = [...this.users.values()] - if (args.q !== null) { - const needle = args.q.toLowerCase() - rows = rows.filter( - (r) => - r.name.toLowerCase().includes(needle) || - (r.handle?.toLowerCase().includes(needle) ?? false) || - r.city.toLowerCase().includes(needle), - ) - } + if (args.q !== null) rows = rows.filter(userSearchMatcher(args.q)) if (args.status !== null) rows = rows.filter((r) => r.accountStatus === args.status) if (args.flaggedOnly) rows = rows.filter((r) => r.flagged) if (args.deletedOnly) rows = rows.filter((r) => r.deletedAt !== null) @@ -151,15 +151,7 @@ export class InMemoryAdminUserRepository implements AdminUserRepository { async countByFacet(args: { q: string | null }): Promise { let rows = [...this.users.values()] - if (args.q !== null) { - const needle = args.q.toLowerCase() - rows = rows.filter( - (r) => - r.name.toLowerCase().includes(needle) || - (r.handle?.toLowerCase().includes(needle) ?? false) || - r.city.toLowerCase().includes(needle), - ) - } + if (args.q !== null) rows = rows.filter(userSearchMatcher(args.q)) let active = 0 let suspended = 0 let flagged = 0 diff --git a/services/api/src/services/admin/admin-user-service.ts b/services/api/src/services/admin/admin-user-service.ts index dccda855..c5308574 100644 --- a/services/api/src/services/admin/admin-user-service.ts +++ b/services/api/src/services/admin/admin-user-service.ts @@ -21,6 +21,7 @@ import type { CleanupMemberRole, } from "@civfix/shared" import { toRelAbs } from "./admin-format.js" +import { ADMIN_DEFAULT_LIMIT } from "./pagination.js" import { applyRoleChange } from "./role-change.js" import type { MessageUpdateAnnouncer } from "./admin-report-chat-service.js" import { @@ -141,6 +142,19 @@ export interface SessionControl { const GRANTABLE_ROLES: ReadonlySet = new Set(["citizen", "gov_user", "gov_admin"]) +const USER_SUBLIST_DEFAULT_LIMIT = 20 + +const USER_NOT_FOUND = "User not found" + +const EMPTY_USER_COUNTS: AdminUserCounts = { + all: 0, + active: 0, + suspended: 0, + flagged: 0, + deleted: 0, + banned: 0, +} + export function resolveUserFilter(filter: string | undefined): { status: UserStatus | null flaggedOnly: boolean @@ -148,11 +162,9 @@ export function resolveUserFilter(filter: string | undefined): { } { switch (filter) { case "active": - return { status: "active", flaggedOnly: false, deletedOnly: false } case "suspended": - return { status: "suspended", flaggedOnly: false, deletedOnly: false } case "banned": - return { status: "banned", flaggedOnly: false, deletedOnly: false } + return { status: filter, flaggedOnly: false, deletedOnly: false } case "flagged": return { status: null, flaggedOnly: true, deletedOnly: false } case "deleted": @@ -169,8 +181,6 @@ export interface AdminUserServiceDeps { announceMessageUpdate?: MessageUpdateAnnouncer } -export const USER_SUBLIST_DEFAULT_LIMIT = 20 - export interface AdminUserService { list(query: AdminUserListQuery): Promise get(id: string): Promise @@ -226,20 +236,13 @@ export function makeAdminUserService(deps: AdminUserServiceDeps): AdminUserServi flaggedOnly, deletedOnly, cursor: query.cursor ?? null, - limit: query.limit ?? 25, + limit: query.limit ?? ADMIN_DEFAULT_LIMIT, } const [{ records, nextCursor }, counts] = await Promise.all([ deps.repo.listUsers(args), args.cursor === null ? deps.repo.countByFacet({ q: args.q }) - : Promise.resolve({ - all: 0, - active: 0, - suspended: 0, - flagged: 0, - deleted: 0, - banned: 0, - }), + : Promise.resolve({ ...EMPTY_USER_COUNTS }), ]) return { items: records.map((r) => toListItem(r, ref)), nextCursor, counts } }, @@ -247,7 +250,7 @@ export function makeAdminUserService(deps: AdminUserServiceDeps): AdminUserServi async get(id: string): Promise { const ref = now() const record = await deps.repo.getUser(id) - if (!record) throw AppError.notFound("User not found") + if (!record) throw AppError.notFound(USER_NOT_FOUND) const organizations = await deps.repo.listUserOrganizations(id) return { ...toListItem(record, ref), @@ -322,7 +325,7 @@ export function makeAdminUserService(deps: AdminUserServiceDeps): AdminUserServi ): Promise { assertTargetIsNotOfficialAccount(id, "flag") const flagged = await deps.repo.toggleFlag(id, input) - if (flagged === null) throw AppError.notFound("User not found") + if (flagged === null) throw AppError.notFound(USER_NOT_FOUND) return flagged }, @@ -332,7 +335,7 @@ export function makeAdminUserService(deps: AdminUserServiceDeps): AdminUserServi ): Promise<{ revokedSessions: number }> { await assertTargetIsNotOperator(deps.repo, id, "ban or change the status of") const ok = await deps.repo.setStatus(id, input) - if (!ok) throw AppError.notFound("User not found") + if (!ok) throw AppError.notFound(USER_NOT_FOUND) const revokedSessions = await deps.sessions.applyStatus(id, input.status) return { revokedSessions } }, @@ -348,7 +351,7 @@ export function makeAdminUserService(deps: AdminUserServiceDeps): AdminUserServi throw AppError.forbidden("You cannot change your own role.") } const target = await deps.repo.getUser(id) - if (!target) throw AppError.notFound("User not found") + if (!target) throw AppError.notFound(USER_NOT_FOUND) if (target.role === "operator") { throw AppError.forbidden( "Operator accounts are managed through ADMIN_EMAILS; they cannot be changed from the console.", @@ -359,7 +362,7 @@ export function makeAdminUserService(deps: AdminUserServiceDeps): AdminUserServi { write: async (userId, role) => { const ok = await deps.repo.applyRole(userId, { role, actorId: input.actorId }) - if (!ok) throw AppError.notFound("User not found") + if (!ok) throw AppError.notFound(USER_NOT_FOUND) }, revokeAll: deps.sessions.revokeAll.bind(deps.sessions), }, @@ -374,7 +377,7 @@ export function makeAdminUserService(deps: AdminUserServiceDeps): AdminUserServi ): Promise { assertTargetIsNotOfficialAccount(id, "verify") const ok = await deps.repo.setReportVerified(id, input) - if (!ok) throw AppError.notFound("User not found") + if (!ok) throw AppError.notFound(USER_NOT_FOUND) }, async removeMessage( @@ -390,7 +393,7 @@ export function makeAdminUserService(deps: AdminUserServiceDeps): AdminUserServi } async function assertUserExists(repo: AdminUserRepository, id: string): Promise { - if (!(await repo.userExists(id))) throw AppError.notFound("User not found") + if (!(await repo.userExists(id))) throw AppError.notFound(USER_NOT_FOUND) } async function assertTargetIsNotOperator( @@ -400,6 +403,6 @@ async function assertTargetIsNotOperator( ): Promise { assertTargetIsNotOfficialAccount(id, verb) const target = await repo.getUser(id) - if (!target) throw AppError.notFound("User not found") + if (!target) throw AppError.notFound(USER_NOT_FOUND) assertTargetIsNotOperatorRole(target.role, verb) } diff --git a/services/api/src/services/admin/analytics-repository.drizzle.ts b/services/api/src/services/admin/analytics-repository.drizzle.ts index aa08ed4b..f83433a3 100644 --- a/services/api/src/services/admin/analytics-repository.drizzle.ts +++ b/services/api/src/services/admin/analytics-repository.drizzle.ts @@ -49,7 +49,6 @@ function withCacheIn( key: string, run: () => Promise, ): Promise { - if (ttlMs <= 0) return run() const hit = analyticsCache.get(key) if (hit !== undefined && Date.now() - hit.at <= ttlMs) return hit.value as Promise const value = run() @@ -63,6 +62,40 @@ function withCacheIn( return value } +interface RetentionCell { + y: string + m: string + size: string + period: string + active: string +} + +function toRetentionRows(cells: readonly RetentionCell[], cohorts: number): RetentionRow[] { + const byCohort = new Map() + for (const cell of cells) { + const year = num(cell.y) + const month = num(cell.m) + const key = `${year}-${month}` + let cohort = byCohort.get(key) + if (!cohort) { + cohort = { year, month, size: num(cell.size), activeByPeriod: [] } + byCohort.set(key, cohort) + } + const period = num(cell.period) + if (period >= 0 && period < cohorts) { + cohort.activeByPeriod[period] = num(cell.active) + } + } + const result: RetentionRow[] = [] + for (const cohort of byCohort.values()) { + const dense: number[] = [] + for (let i = 0; i < cohorts; i++) dense.push(cohort.activeByPeriod[i] ?? 0) + result.push({ ...cohort, activeByPeriod: dense }) + } + result.sort((a, b) => b.year * 12 + b.month - (a.year * 12 + a.month)) + return result +} + export function makeDrizzleAnalyticsRepository( sql: Sql, opts?: { cacheTtlMs?: number }, @@ -413,9 +446,7 @@ export function makeDrizzleAnalyticsRepository( }, async retention(cohorts: number): Promise { - const rows = await sql< - { y: string; m: string; size: string; period: string; active: string }[] - >` + const rows = await sql` WITH cohort_users AS ( SELECT u.id AS user_id, @@ -463,29 +494,7 @@ export function makeDrizzleAnalyticsRepository( LEFT JOIN active_counts ac ON ac.cohort_month = cs.cohort_month ORDER BY cs.cohort_month, period ` - const byCohort = new Map() - for (const row of rows) { - const year = num(row.y) - const month = num(row.m) - const key = `${year}-${month}` - let cohort = byCohort.get(key) - if (!cohort) { - cohort = { year, month, size: num(row.size), activeByPeriod: [] } - byCohort.set(key, cohort) - } - const period = num(row.period) - if (period >= 0 && period < cohorts) { - cohort.activeByPeriod[period] = num(row.active) - } - } - const result: RetentionRow[] = [] - for (const cohort of byCohort.values()) { - const dense: number[] = [] - for (let i = 0; i < cohorts; i++) dense.push(cohort.activeByPeriod[i] ?? 0) - result.push({ ...cohort, activeByPeriod: dense }) - } - result.sort((a, b) => b.year * 12 + b.month - (a.year * 12 + a.month)) - return result + return toRetentionRows(rows, cohorts) }, } @@ -493,22 +502,22 @@ export function makeDrizzleAnalyticsRepository( if (ttl <= 0) return base const cache = cacheFor(sql) - const withCache = (ttlMs: number, key: string, run: () => Promise): Promise => - withCacheIn(cache, ttlMs, key, run) + const withCache = (key: string, run: () => Promise): Promise => + withCacheIn(cache, ttl, key, run) return { - kpis: () => withCache(ttl, "kpis", () => base.kpis()), - pinsByWeek: (weeks) => withCache(ttl, `pinsByWeek:${weeks}`, () => base.pinsByWeek(weeks)), - byCategory: () => withCache(ttl, "byCategory", () => base.byCategory()), - funnel: () => withCache(ttl, "funnel", () => base.funnel()), - coverage: () => withCache(ttl, "coverage", () => base.coverage()), + kpis: () => withCache("kpis", () => base.kpis()), + pinsByWeek: (weeks) => withCache(`pinsByWeek:${weeks}`, () => base.pinsByWeek(weeks)), + byCategory: () => withCache("byCategory", () => base.byCategory()), + funnel: () => withCache("funnel", () => base.funnel()), + coverage: () => withCache("coverage", () => base.coverage()), resolutionByCategory: () => - withCache(ttl, "resolutionByCategory", () => base.resolutionByCategory()), - events: (months) => withCache(ttl, `events:${months}`, () => base.events(months)), + withCache("resolutionByCategory", () => base.resolutionByCategory()), + events: (months) => withCache(`events:${months}`, () => base.events(months)), topJurisdictions: (limit) => - withCache(ttl, `topJurisdictions:${limit}`, () => base.topJurisdictions(limit)), + withCache(`topJurisdictions:${limit}`, () => base.topJurisdictions(limit)), topContributors: (limit) => - withCache(ttl, `topContributors:${limit}`, () => base.topContributors(limit)), - heatmap: (limit) => withCache(ttl, `heatmap:${limit}`, () => base.heatmap(limit)), - retention: (cohorts) => withCache(ttl, `retention:${cohorts}`, () => base.retention(cohorts)), + withCache(`topContributors:${limit}`, () => base.topContributors(limit)), + heatmap: (limit) => withCache(`heatmap:${limit}`, () => base.heatmap(limit)), + retention: (cohorts) => withCache(`retention:${cohorts}`, () => base.retention(cohorts)), } } diff --git a/services/api/src/services/admin/analytics-shaping.ts b/services/api/src/services/admin/analytics-shaping.ts index 4091b58c..4f33f9b7 100644 --- a/services/api/src/services/admin/analytics-shaping.ts +++ b/services/api/src/services/admin/analytics-shaping.ts @@ -37,6 +37,11 @@ const MONTH_ABBR = [ "Dec", ] as const +const DAY_MS = 24 * 60 * 60 * 1000 +const WEEK_MS = 7 * DAY_MS + +const ROUTE_TIME_PENDING_DELTA = "Phase 3" + export function round1(n: number): number { return Math.round(n * 10) / 10 } @@ -92,7 +97,7 @@ export function buildKpis(agg: KpiAggregates): AnalyticsKpi[] { // its KPI strip, but the value is part of the contract. label: "Avg. route time", num: 0, - delta: "Phase 3", + delta: ROUTE_TIME_PENDING_DELTA, dir: "flat", }, { @@ -234,9 +239,6 @@ export function buildRetention(rows: RetentionRow[], periods: number): Analytics return { cohorts, periodLabels } } -const DAY_MS = 24 * 60 * 60 * 1000 -const WEEK_MS = 7 * DAY_MS - /** UTC start-of-week anchored to Monday (matches date_trunc('week', ...) in Postgres, which is Monday). */ export function startOfWeekUtc(d: Date): Date { const copy = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate())) diff --git a/services/api/src/services/admin/audit-repository.drizzle.ts b/services/api/src/services/admin/audit-repository.drizzle.ts index cf74d030..b8a0980e 100644 --- a/services/api/src/services/admin/audit-repository.drizzle.ts +++ b/services/api/src/services/admin/audit-repository.drizzle.ts @@ -1,4 +1,4 @@ -import type { Sql } from "../../db/client.js" +import type { Queryable, Sql } from "../../db/client.js" import { clampLimit, decodeCursor, @@ -9,6 +9,7 @@ import { import { isUuid } from "../../db/cursor-helpers.js" import type { AuditRecord, AuditRepository, ListAuditArgs } from "./audit-service.js" import { likeContains } from "./like.js" +import type { SqlFragment } from "./sql-fragments.js" interface AuditRowSelect { id: string @@ -33,6 +34,17 @@ function toRecord(r: AuditRowSelect): AuditRecord { } } +function auditActorFilter(sql: Queryable, actor: string | null): SqlFragment { + if (actor === null) return sql`` + const like = likeContains(actor) + const asUuid = isUuid(actor) ? actor : null + return sql`AND (${asUuid}::uuid IS NOT NULL AND a.actor_id = ${asUuid}::uuid OR u.display_name ILIKE ${like} ESCAPE '\\')` +} + +function containsFilter(sql: Queryable, column: SqlFragment, term: string | null): SqlFragment { + return term === null ? sql`` : sql`AND ${column} ILIKE ${likeContains(term)} ESCAPE '\\'` +} + export function makeDrizzleAuditRepository(sql: Sql): AuditRepository { return { async list( @@ -44,22 +56,9 @@ export function makeDrizzleAuditRepository(sql: Sql): AuditRepository { anchor !== null ? sql`AND ${keysetPredicate(sql, sql`a.created_at`, sql`a.id`, anchor)}` : sql`` - const actorFilter = - args.actor !== null - ? (() => { - const like = likeContains(args.actor) - const asUuid = isUuid(args.actor) ? args.actor : null - return sql`AND (${asUuid}::uuid IS NOT NULL AND a.actor_id = ${asUuid}::uuid OR u.display_name ILIKE ${like} ESCAPE '\\')` - })() - : sql`` - const actionFilter = - args.action !== null - ? sql`AND a.action ILIKE ${likeContains(args.action)} ESCAPE '\\'` - : sql`` - const targetFilter = - args.target !== null - ? sql`AND a.target ILIKE ${likeContains(args.target)} ESCAPE '\\'` - : sql`` + const actorFilter = auditActorFilter(sql, args.actor) + const actionFilter = containsFilter(sql, sql`a.action`, args.action) + const targetFilter = containsFilter(sql, sql`a.target`, args.target) const rows = await sql` SELECT a.id, a.actor_id, u.display_name AS actor_name, a.action, a.target, a.meta, a.created_at, diff --git a/services/api/src/services/admin/autoforward-jobs.ts b/services/api/src/services/admin/autoforward-jobs.ts index a88ebb34..20fbb3ed 100644 --- a/services/api/src/services/admin/autoforward-jobs.ts +++ b/services/api/src/services/admin/autoforward-jobs.ts @@ -15,8 +15,6 @@ import { import { makePacketMediaPresigner, makePrivateMediaPresigner } from "../media-presign.js" import { makeContainerReportChatEmitter } from "../report-chat-emitter.js" -export { REPORT_AUTOFORWARD_JOB } - export interface AutoForwardLogger { info: (obj: unknown, msg?: string) => void warn: (obj: unknown, msg?: string) => void @@ -37,7 +35,7 @@ export async function registerAutoForwardJobs( }) } -export async function runAutoForward( +async function runAutoForward( container: Container, reportId: string, logger?: AutoForwardJobLogger, diff --git a/services/api/src/services/admin/category-counts.ts b/services/api/src/services/admin/category-counts.ts index 5b680741..12b75717 100644 --- a/services/api/src/services/admin/category-counts.ts +++ b/services/api/src/services/admin/category-counts.ts @@ -55,11 +55,7 @@ export function parseCategoryCounts( } function joinColumns(sql: Queryable, parts: SqlFragment[]): SqlFragment { - let out = sql`` - let first = true - for (const part of parts) { - out = first ? part : sql`${out}, ${part}` - first = false - } - return out + const [first, ...rest] = parts + if (first === undefined) return sql`` + return rest.reduce((acc, part) => sql`${acc}, ${part}`, first) } diff --git a/services/api/src/services/admin/discovery-repository.drizzle.ts b/services/api/src/services/admin/discovery-repository.drizzle.ts index 5f9ebfc1..f5495ac0 100644 --- a/services/api/src/services/admin/discovery-repository.drizzle.ts +++ b/services/api/src/services/admin/discovery-repository.drizzle.ts @@ -26,6 +26,8 @@ const SAMPLE_PIN_CAP = 50 const DISCOVERY_NOTE_CAP = 200 +const DISCOVERY_DETAIL_ZOOM = 11 + interface TaskAggRow extends CategoryCountRow { id: string geoid: string | null @@ -141,6 +143,15 @@ async function taskAggregateSql( return rows as unknown as TaskAggRow[] } +function taskAggregateByIdSql(sql: Queryable, id: string): Promise { + return taskAggregateSql( + sql, + sql`AND t.id = ${id}`, + sql``, + sql`SELECT dt.geoid FROM jurisdiction_discovery_tasks dt WHERE dt.id = ${id} AND dt.geoid IS NOT NULL`, + ) +} + export function makeDrizzleDiscoveryRepository(sql: Sql): DiscoveryRepository { return { async listTasks( @@ -178,12 +189,7 @@ export function makeDrizzleDiscoveryRepository(sql: Sql): DiscoveryRepository { }, async getDetail(id: string): Promise { - const rows = await taskAggregateSql( - sql, - sql`AND t.id = ${id}`, - sql``, - sql`SELECT dt.geoid FROM jurisdiction_discovery_tasks dt WHERE dt.id = ${id} AND dt.geoid IS NOT NULL`, - ) + const rows = await taskAggregateByIdSql(sql, id) const row = rows[0] if (!row) return null const task = toTaskRecord(row) @@ -255,12 +261,7 @@ export function makeDrizzleDiscoveryRepository(sql: Sql): DiscoveryRepository { }, async getTask(id: string): Promise { - const rows = await taskAggregateSql( - sql, - sql`AND t.id = ${id}`, - sql``, - sql`SELECT dt.geoid FROM jurisdiction_discovery_tasks dt WHERE dt.id = ${id} AND dt.geoid IS NOT NULL`, - ) + const rows = await taskAggregateByIdSql(sql, id) const row = rows[0] return row ? toTaskRecord(row) : null }, @@ -429,7 +430,7 @@ async function loadGeometry( const c = centerRows[0] const center: [number, number] | null = c && c.lat !== null && c.lng !== null ? [c.lat, c.lng] : null - const zoom = center !== null ? 11 : null + const zoom = center !== null ? DISCOVERY_DETAIL_ZOOM : null return { placeGeojson, center, zoom } } @@ -461,18 +462,12 @@ export function invalidateDirectoryFacetCache(): void { directoryFacetCache = null } -export interface UpsertDefaultContactOpts { - setEmail?: boolean - setFormUrl?: boolean -} - export async function upsertJurisdictionContacts( tx: Queryable, geoid: string, contacts: Partial>, defaultEmails: string[], formUrl: string | null, - opts?: UpsertDefaultContactOpts, ): Promise { for (const [category, rawEmail] of Object.entries(contacts) as [ ReportCategory, @@ -495,14 +490,12 @@ export async function upsertJurisdictionContacts( const defaultEmail = defaultEmails.find((e) => e.trim() !== "")?.trim() ?? null const form = formUrl && formUrl.trim() !== "" ? formUrl.trim() : null - const setEmail = opts?.setEmail ?? defaultEmail !== null - const setForm = opts?.setFormUrl ?? form !== null - const writeEmail = setEmail && defaultEmail !== null - const writeForm = setForm && form !== null - if (writeEmail || writeForm) { + const setEmail = defaultEmail !== null + const setForm = form !== null + if (setEmail || setForm) { await tx` INSERT INTO jurisdiction_contacts (geoid, category, email, form_url, updated_at, bounced_at) - VALUES (${geoid}, NULL, ${writeEmail ? defaultEmail : null}, ${writeForm ? form : null}, now(), NULL) + VALUES (${geoid}, NULL, ${defaultEmail}, ${form}, now(), NULL) ON CONFLICT (geoid) WHERE category IS NULL DO UPDATE SET email = CASE WHEN ${setEmail} THEN EXCLUDED.email ELSE jurisdiction_contacts.email END, @@ -510,16 +503,6 @@ export async function upsertJurisdictionContacts( updated_at = now(), bounced_at = CASE WHEN ${setEmail} THEN NULL ELSE jurisdiction_contacts.bounced_at END ` - } else if (setEmail || setForm) { - await tx` - UPDATE jurisdiction_contacts - SET - email = CASE WHEN ${setEmail} THEN NULL ELSE email END, - form_url = CASE WHEN ${setForm} THEN NULL ELSE form_url END, - updated_at = now(), - bounced_at = CASE WHEN ${setEmail} THEN NULL ELSE bounced_at END - WHERE geoid = ${geoid} AND category IS NULL - ` } if (defaultEmails.length > 0 || form !== null) { diff --git a/services/api/src/services/admin/discovery-repository.memory.ts b/services/api/src/services/admin/discovery-repository.memory.ts index d1baab77..ebdf8e2b 100644 --- a/services/api/src/services/admin/discovery-repository.memory.ts +++ b/services/api/src/services/admin/discovery-repository.memory.ts @@ -218,34 +218,12 @@ export class InMemoryDiscoveryRepository implements DiscoveryRepository { for (const seeded of this.tasks.values()) { if (seeded.task.geoid === input.geoid && seeded.task.status !== "done") return false } - const id = randomUUID() - this.tasks.set(id, { - task: { - id, - geoid: input.geoid, - place: input.geoid, - layer: "place", - population: input.population ?? null, - status: "open", - perCategory: {}, - total: 0, - oldestWaitingAt: null, - newestWaitingAt: null, - contactCategories: [], - hasDefaultContact: false, - }, - contacts: [], - placeGeojson: null, - samplePins: [], - center: null, - zoom: null, - }) + this.seedTask({ geoid: input.geoid, place: input.geoid, population: input.population ?? null }) return true } } -/** Shared with the in-memory contacts repository so both mutate the seeded state identically. */ -export function upsertContacts( +function upsertContacts( seeded: SeededDiscoveryTask, contacts: Partial>, defaultEmails: string[], diff --git a/services/api/src/services/admin/discovery-service.ts b/services/api/src/services/admin/discovery-service.ts index 44892358..402d8747 100644 --- a/services/api/src/services/admin/discovery-service.ts +++ b/services/api/src/services/admin/discovery-service.ts @@ -11,6 +11,7 @@ import { AppError, REPORT_CATEGORY_LABELS, relativeAgo } from "@civfix/shared" import { ADMIN_CATEGORIES } from "./category-counts.js" +import { ADMIN_DEFAULT_LIMIT } from "./pagination.js" import type { DiscoveryContact, DiscoveryListQuery, @@ -34,6 +35,14 @@ const DISCOVERY_CATEGORIES = ADMIN_CATEGORIES /** A task breaches when its oldest waiting report is older than this. */ export const DISCOVERY_SLA_HOURS = 24 +const HOUR_MS = 60 * 60 * 1000 + +const DISCOVERY_TASK_NOT_FOUND = "Discovery task not found" + +const SUGGESTION_NOTE_AUTHOR = "Reporter" + +const NO_WAITING_REPORT_LABEL = "-" + export interface DiscoveryTaskRecord { id: string geoid: string @@ -186,7 +195,7 @@ export function computeContactState(record: DiscoveryTaskRecord): { export function isOverSla(oldestWaitingAt: Date | null, now: Date): boolean { if (oldestWaitingAt === null) return false const ageMs = now.getTime() - oldestWaitingAt.getTime() - return ageMs > DISCOVERY_SLA_HOURS * 60 * 60 * 1000 + return ageMs > DISCOVERY_SLA_HOURS * HOUR_MS } /** @@ -199,13 +208,13 @@ export function derivePriority(record: DiscoveryTaskRecord, now: Date): Priority return "low" } -export function suggestionToNote(s: DiscoveryContactSuggestionRecord): DiscoveryNoteRecord { +function suggestionToNote(s: DiscoveryContactSuggestionRecord): DiscoveryNoteRecord { const contact = [s.email, s.formUrl] .filter((v): v is string => !!v && v.trim() !== "") .join(" / ") const head = `Suggested contact: ${contact || "(none provided)"}` return { - who: "Reporter", + who: SUGGESTION_NOTE_AUTHOR, text: s.note && s.note.trim() !== "" ? `${head} (note: ${s.note.trim()})` : head, createdAt: s.createdAt, } @@ -255,8 +264,14 @@ export function makeDiscoveryService(deps: DiscoveryServiceDeps): DiscoveryServi pop: record.population ?? 0, reports: record.total, perCategoryCounts: fullPerCategoryCounts(record.perCategory), - lastReport: record.newestWaitingAt !== null ? relativeAgo(record.newestWaitingAt, ref) : "-", - age: record.oldestWaitingAt !== null ? relativeAgo(record.oldestWaitingAt, ref) : "-", + lastReport: + record.newestWaitingAt !== null + ? relativeAgo(record.newestWaitingAt, ref) + : NO_WAITING_REPORT_LABEL, + age: + record.oldestWaitingAt !== null + ? relativeAgo(record.oldestWaitingAt, ref) + : NO_WAITING_REPORT_LABEL, overSla: isOverSla(record.oldestWaitingAt, ref), priority: derivePriority(record, ref), contactState: computeContactState(record), @@ -272,7 +287,7 @@ export function makeDiscoveryService(deps: DiscoveryServiceDeps): DiscoveryServi filter: query.filter ?? "all", sort: query.sort ?? "pop", cursor: query.cursor ?? null, - limit: query.limit ?? 25, + limit: query.limit ?? ADMIN_DEFAULT_LIMIT, } const { records, nextCursor } = await deps.repo.listTasks(args) // Only the detail renders notes, so list rows carry an empty notes[] rather than a note read @@ -284,7 +299,7 @@ export function makeDiscoveryService(deps: DiscoveryServiceDeps): DiscoveryServi async getTask(id: string): Promise { const ref = now() const [detail, notes] = await Promise.all([deps.repo.getDetail(id), deps.repo.listNotes(id)]) - if (!detail) throw AppError.notFound("Discovery task not found") + if (!detail) throw AppError.notFound(DISCOVERY_TASK_NOT_FOUND) const suggestions = await deps.repo.listContactSuggestions(detail.task.geoid) const merged = [...notes, ...suggestions.map(suggestionToNote)].sort( @@ -319,14 +334,14 @@ export function makeDiscoveryService(deps: DiscoveryServiceDeps): DiscoveryServi ): Promise { const ref = now() const task = await deps.repo.getTask(id) - if (!task) throw AppError.notFound("Discovery task not found") + if (!task) throw AppError.notFound(DISCOVERY_TASK_NOT_FOUND) const note = await deps.repo.addNote(id, input) return toNoteDTO(note, ref) }, async flag(id: string, input: { reason: string | null; actorId: string }): Promise { const ok = await deps.repo.flagTask(id, input) - if (!ok) throw AppError.notFound("Discovery task not found") + if (!ok) throw AppError.notFound(DISCOVERY_TASK_NOT_FOUND) }, async saveDraft( @@ -339,7 +354,7 @@ export function makeDiscoveryService(deps: DiscoveryServiceDeps): DiscoveryServi }, ): Promise { const ok = await deps.repo.saveDraft(id, input) - if (!ok) throw AppError.notFound("Discovery task not found") + if (!ok) throw AppError.notFound(DISCOVERY_TASK_NOT_FOUND) }, } } diff --git a/services/api/src/services/admin/event-status.ts b/services/api/src/services/admin/event-status.ts index 1147306c..ea210394 100644 --- a/services/api/src/services/admin/event-status.ts +++ b/services/api/src/services/admin/event-status.ts @@ -7,8 +7,6 @@ import type { EventStatus } from "@civfix/shared" -export type StoredCleanupStatus = "upcoming" | "active" | "done" | "cancelled" - export function toEventStatus(stored: string): EventStatus { switch (stored) { case "active": diff --git a/services/api/src/services/admin/forward-template-repository.drizzle.ts b/services/api/src/services/admin/forward-template-repository.drizzle.ts index 69fb5d11..7ae14d75 100644 --- a/services/api/src/services/admin/forward-template-repository.drizzle.ts +++ b/services/api/src/services/admin/forward-template-repository.drizzle.ts @@ -6,7 +6,7 @@ import type { SetForwardTemplateInput, } from "./forward-template-types.js" -export const FORWARD_TEMPLATE_AUDIT_TARGET = "mail:forward-template" +const FORWARD_TEMPLATE_AUDIT_TARGET = "mail:forward-template" interface ForwardTemplateRowSelect { subject_template: string | null diff --git a/services/api/src/services/admin/gov-claims-repository.drizzle.ts b/services/api/src/services/admin/gov-claims-repository.drizzle.ts index 1055272c..317170d2 100644 --- a/services/api/src/services/admin/gov-claims-repository.drizzle.ts +++ b/services/api/src/services/admin/gov-claims-repository.drizzle.ts @@ -14,6 +14,7 @@ import { } from "./pagination.js" import { ilikeAnyOf } from "./sql-fragments.js" import { + GOV_CHECKS, type GovCheckRecord, type GovClaimRecord, type GovClaimsRepository, @@ -36,7 +37,7 @@ interface GovClaimRow { created_at: Date } -const CHECK_KEYS: readonly string[] = ["linkedin", "directory", "callback"] +const CHECK_KEYS: readonly string[] = GOV_CHECKS const CHECK_STATUSES: readonly GovCheckStatus[] = ["verified", "pending"] @@ -60,6 +61,10 @@ function parseChecks(raw: unknown): Partial { + const claim = await deps.repo.getClaim(id) + if (!claim) throw AppError.notFound(GOV_CLAIM_NOT_FOUND) + throw AppError.conflict(GOV_CLAIM_NOT_PENDING) + } + + // Find-or-create carries no privilege, so it runs before the claim transition commits. + async function findOrCreateClaimUser(email: string, claimName: string): Promise { + const user = await deps.users.findByEmail(email) + if (!user) { + // The placeholder's email is unverified: it has no sessions and only becomes usable when the real + // owner signs in via Email-OTP, which proves control of the address. + return deps.users.create(email, newAccountDisplayName(claimName, "citizen")) + } + // Security: contact_email is unverified free text. Elevating a pre-existing account whose email + // is not verified would let a duped or compromised operator elevate an arbitrary victim account + // by entering its address. + if (!user.emailVerified) { + throw AppError.validation( + { contactEmail: "unverified" }, + "Cannot elevate an existing account whose email is not verified", + ) + } + // Same rule as the users console's setRole: an operator account is not changeable from the + // console. Otherwise a claim carrying an operator's address would demote them and revoke their + // sessions, one operator stripping another through the gov queue. Operator authority is governed + // by ADMIN_EMAILS + Cloudflare Access. + if (user.role === OPERATOR_ROLE) { + throw AppError.forbidden( + "Operator accounts are managed through ADMIN_EMAILS; they cannot be changed from the console.", + ) + } + return user + } + return { async list(query: GovClaimListQuery): Promise { const ref = now() @@ -203,7 +243,7 @@ export function makeGovClaimsService(deps: GovClaimsServiceDeps): GovClaimsServi async getClaim(id: string): Promise { const ref = now() const record = await deps.repo.getClaim(id) - if (!record) throw AppError.notFound("Gov claim not found") + if (!record) throw AppError.notFound(GOV_CLAIM_NOT_FOUND) return toDTO(record, ref) }, @@ -218,20 +258,14 @@ export function makeGovClaimsService(deps: GovClaimsServiceDeps): GovClaimsServi }, ): Promise { const updated = await deps.repo.setCheck(id, input) - if (!updated) { - // setCheck returns null for both a missing claim and a decided one; re-read so an operator racing - // a colleague's decision gets a conflict, not "not found". - const claim = await deps.repo.getClaim(id) - if (!claim) throw AppError.notFound("Gov claim not found") - throw AppError.conflict("Gov claim is not pending") - } + if (!updated) await throwMissingOrDecided(id) }, async approve(id: string, input: { actorId: string; note: string | null }): Promise { const claim = await deps.repo.getClaim(id) - if (!claim) throw AppError.notFound("Gov claim not found") + if (!claim) throw AppError.notFound(GOV_CLAIM_NOT_FOUND) if (claim.status !== "pending") { - throw AppError.conflict("Gov claim is not pending") + throw AppError.conflict(GOV_CLAIM_NOT_PENDING) } const email = claim.contactEmail?.trim() if (!email) { @@ -243,32 +277,7 @@ export function makeGovClaimsService(deps: GovClaimsServiceDeps): GovClaimsServi // The privilege grant happens only after the claim transition commits, so a concurrent decision or // a failed transition never leaves a gov_admin elevation with no approved claim to justify it. - // Find-or-create first: it carries no privilege. - let user = await deps.users.findByEmail(email) - if (user) { - // Security: contact_email is unverified free text. Elevating a pre-existing account whose email - // is not verified would let a duped or compromised operator elevate an arbitrary victim account - // by entering its address. - if (!user.emailVerified) { - throw AppError.validation( - { contactEmail: "unverified" }, - "Cannot elevate an existing account whose email is not verified", - ) - } - // Same rule as the users console's setRole: an operator account is not changeable from the - // console. Otherwise a claim carrying an operator's address would demote them and revoke their - // sessions, one operator stripping another through the gov queue. Operator authority is governed - // by ADMIN_EMAILS + Cloudflare Access. - if (user.role === OPERATOR_ROLE) { - throw AppError.forbidden( - "Operator accounts are managed through ADMIN_EMAILS; they cannot be changed from the console.", - ) - } - } else { - // The placeholder's email is unverified: it has no sessions and only becomes usable when the real - // owner signs in via Email-OTP, which proves control of the address. - user = await deps.users.create(email, newAccountDisplayName(claim.name, "citizen")) - } + const user = await findOrCreateClaimUser(email, claim.name) const updated = await deps.repo.approve(id, { userId: user.id, @@ -277,7 +286,7 @@ export function makeGovClaimsService(deps: GovClaimsServiceDeps): GovClaimsServi }) if (!updated) { // A concurrent decision won. The user has not been elevated, so there is nothing to compensate. - throw AppError.conflict("Gov claim is not pending") + throw AppError.conflict(GOV_CLAIM_NOT_PENDING) } // A failure from here on leaves the claim approved without the role, the recoverable direction: @@ -285,7 +294,6 @@ export function makeGovClaimsService(deps: GovClaimsServiceDeps): GovClaimsServi // needs too: live sessions carry the old role in the Redis projection and sliding expiry keeps them // alive indefinitely. if (user.role !== GOV_ADMIN_ROLE) { - const target = user await applyRoleChange( { write: async (userId, role) => { @@ -293,7 +301,7 @@ export function makeGovClaimsService(deps: GovClaimsServiceDeps): GovClaimsServi }, revokeAll: deps.revokeSessions, }, - target.id, + user.id, GOV_ADMIN_ROLE, ) } @@ -301,11 +309,7 @@ export function makeGovClaimsService(deps: GovClaimsServiceDeps): GovClaimsServi async reject(id: string, input: { reason: string; actorId: string }): Promise { const updated = await deps.repo.reject(id, input) - if (!updated) { - const claim = await deps.repo.getClaim(id) - if (!claim) throw AppError.notFound("Gov claim not found") - throw AppError.conflict("Gov claim is not pending") - } + if (!updated) await throwMissingOrDecided(id) }, } } diff --git a/services/api/src/services/admin/home-service.ts b/services/api/src/services/admin/home-service.ts index 6f4be94d..fb0b9613 100644 --- a/services/api/src/services/admin/home-service.ts +++ b/services/api/src/services/admin/home-service.ts @@ -7,7 +7,7 @@ import type { ReportStatus, } from "@civfix/shared" import type { AnalyticsRepository } from "./analytics-types.js" -import { buildCoverage, buildPinsByWeek, pct } from "./analytics-shaping.js" +import { buildCoverage, buildPinsByWeek, round1 } from "./analytics-shaping.js" import { PINS_BY_WEEK_WEEKS } from "./analytics-types.js" import type { DiscoverySectionCounts, @@ -78,7 +78,7 @@ export function toMapPin(record: HomeMapPinRecord): HomeMapPin { return record.attendees !== null ? { ...base, attendees: record.attendees } : base } -export const HOME_MAP_PIN_LIMIT = 200 +const HOME_MAP_PIN_LIMIT = 200 export const HOME_SUMMARY_CONCURRENCY = 3 @@ -88,7 +88,7 @@ type SectionResults = { -readonly [K in keyof T]: Awaited> } -export async function runBounded( +async function runBounded( limit: number, tasks: T, ): Promise> { @@ -127,7 +127,7 @@ export function makeHomeService(deps: HomeServiceDeps): HomeService { ): AnalyticsMini { return { pinsThisMonth: kpis ? kpis.pins.current : 0, - resolvedPct: kpis ? round1Pct(kpis.resolvedRatio.current) : 0, + resolvedPct: kpis ? round1(kpis.resolvedRatio.current * 100) : 0, coveragePct: coverage ? coverage.pct : 0, cleanups: kpis ? kpis.cleanupsPlanned.current : 0, eventsThisMonth: kpis ? kpis.events.current : 0, @@ -198,7 +198,3 @@ export function makeHomeService(deps: HomeServiceDeps): HomeService { }, } } - -function round1Pct(ratio: number): number { - return pct(ratio * 100, 100) -} diff --git a/services/api/src/services/admin/inbound-bounce.ts b/services/api/src/services/admin/inbound-bounce.ts index 7fa2ac0b..b56dffd1 100644 --- a/services/api/src/services/admin/inbound-bounce.ts +++ b/services/api/src/services/admin/inbound-bounce.ts @@ -21,6 +21,16 @@ function ourMailDomains(container: Container): string[] { const DSN_SCAN_PREFIX_BYTES = 64 * 1024 +const DAEMON_LOCAL_PARTS: readonly string[] = ["mailer-daemon", "postmaster"] +const DAEMON_SENDER_RE = new RegExp(`(${DAEMON_LOCAL_PARTS.join("|")})@`, "i") +const DSN_CONTENT_TYPE_RE = /report-type["']?\s*[=:]\s*["']?delivery-status/i +const FINAL_RECIPIENT_LINE_RE = /^final-recipient:\s*(?:rfc822;)?\s*(.+)$/im +const TO_LINE_RE = /^to:\s*(.+)$/im +const ORIGINAL_MESSAGE_ID_LINE_RE = /^original-message-id:\s*(.+)$/im +const MESSAGE_ID_LINE_RE = /^message-id:\s*(.+)$/im +const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/ +const BRACKET_ID_RE = /<[^>]+>/ + export interface BounceDetection { isBounce: boolean failedRecipient: string | null @@ -32,8 +42,8 @@ export function detectBounce(mail: ParsedMail): BounceDetection { const contentType = mail.headers["content-type"] ?? "" const failedHeader = mail.headers["x-failed-recipients"] ?? "" const isBounce = - /(mailer-daemon|postmaster)@/i.test(fromAddr) || - /report-type["']?\s*[=:]\s*["']?delivery-status/i.test(contentType) || + DAEMON_SENDER_RE.test(fromAddr) || + DSN_CONTENT_TYPE_RE.test(contentType) || failedHeader.length > 0 if (!isBounce) { return { isBounce: false, failedRecipient: null, originalMessageId: null } @@ -41,17 +51,15 @@ export function detectBounce(mail: ParsedMail): BounceDetection { const body = (mail.text ?? mail.html ?? "").slice(0, DSN_SCAN_PREFIX_BYTES) const failedRecipient = extractEmail(failedHeader) ?? - extractEmail(matchLine(body, /^final-recipient:\s*(?:rfc822;)?\s*(.+)$/im)) ?? - extractEmail(matchLine(body, /^to:\s*(.+)$/im)) + extractEmail(matchLine(body, FINAL_RECIPIENT_LINE_RE)) ?? + extractEmail(matchLine(body, TO_LINE_RE)) const originalMessageId = - matchBracketId(matchLine(body, /^original-message-id:\s*(.+)$/im)) ?? - matchBracketId(matchLine(body, /^message-id:\s*(.+)$/im)) + matchBracketId(matchLine(body, ORIGINAL_MESSAGE_ID_LINE_RE)) ?? + matchBracketId(matchLine(body, MESSAGE_ID_LINE_RE)) return { isBounce: true, failedRecipient, originalMessageId } } -export const DAEMON_LOCAL_PARTS: readonly string[] = ["mailer-daemon", "postmaster"] - -export const PROVIDER_DAEMON_DOMAINS: readonly string[] = [ +const PROVIDER_DAEMON_DOMAINS: readonly string[] = [ "googlemail.com", "google.com", "outlook.com", @@ -127,7 +135,7 @@ export const CONSUMER_MAIL_DOMAINS: ReadonlySet = new Set([ "t-online.de", ]) -export function isPlausibleBounceSender(input: { +function isPlausibleBounceSender(input: { fromAddr: string | null failedRecipient: string ourMailDomains: readonly string[] @@ -246,19 +254,21 @@ export async function geoidForContact(sql: Sql, email: string): Promise]+>/) - return m ? m[0] : value.trim().length > 0 ? value.trim() : null + const m = value.match(BRACKET_ID_RE) + if (m) return m[0] + const trimmed = value.trim() + return trimmed.length > 0 ? trimmed : null } diff --git a/services/api/src/services/admin/inbound-html-sanitizer.ts b/services/api/src/services/admin/inbound-html-sanitizer.ts index aba61d55..1f0de2d4 100644 --- a/services/api/src/services/admin/inbound-html-sanitizer.ts +++ b/services/api/src/services/admin/inbound-html-sanitizer.ts @@ -187,6 +187,10 @@ function closeAfter( return k < slots.length ? (slots[k] ?? -1) : -1 } +function escapedTail(html: string, from: number): string { + return html.slice(from).replace(/", step.lt + 1) if (gt === -1) { - out += html.slice(step.lt).replace(/ = { newline: "\n", } +const MAX_CODE_POINT = 0x10ffff + const ENTITY_RE = /&(?:#x([0-9a-f]+);?|#(\d+);?|(amp|quot|apos|lt|gt|nbsp|tab|newline);)/gi function normalizeUrlAttr(value: string): string { @@ -341,7 +347,7 @@ function decodeEntities(value: string): string { } function safeFromCodePoint(code: number): string { - if (!Number.isFinite(code) || code < 0 || code > 0x10ffff) return "" + if (!Number.isFinite(code) || code < 0 || code > MAX_CODE_POINT) return "" try { return String.fromCodePoint(code) } catch { diff --git a/services/api/src/services/admin/inbound-jobs.ts b/services/api/src/services/admin/inbound-jobs.ts index a521f2bb..d05649aa 100644 --- a/services/api/src/services/admin/inbound-jobs.ts +++ b/services/api/src/services/admin/inbound-jobs.ts @@ -9,8 +9,8 @@ export async function registerInboundJobs(container: Container): Promise { const result = await runInboundSweep(container, { deps: { logger: { - warn: (obj, msg) => console.warn(msg ?? "inbound.sweep", obj), - error: (obj, msg) => console.error(msg ?? "inbound.sweep", obj), + warn: (obj, msg) => console.warn(msg ?? INBOUND_SWEEP_JOB, obj), + error: (obj, msg) => console.error(msg ?? INBOUND_SWEEP_JOB, obj), }, }, }) diff --git a/services/api/src/services/admin/inbound-processor.ts b/services/api/src/services/admin/inbound-processor.ts index a98c4a70..aeb7dc85 100644 --- a/services/api/src/services/admin/inbound-processor.ts +++ b/services/api/src/services/admin/inbound-processor.ts @@ -13,7 +13,7 @@ import { type InboundRepository, } from "./inbound-repository.drizzle.js" import type { AdminReportRepository, ReporterNotifier } from "./admin-report-service.js" -import { detectBounce, handleBounce, type BounceDetection } from "./inbound-bounce.js" +import { detectBounce, handleBounce } from "./inbound-bounce.js" import { applyInboundEffects, findThreadByReferences, @@ -22,7 +22,6 @@ import { type InboundLogger, isJurisdictionSender, isSelfOriginated, - parseMessageIdList, resolveMessageId, } from "./inbound-thread-correlation.js" import type { CleanupRepository } from "../cleanup-service.js" @@ -31,14 +30,7 @@ import { readMailAuthVerdict, type MailAuthVerdict } from "../../adapters/inboun import { sanitizeInboundHtml } from "./inbound-html-sanitizer.js" import { htmlToText } from "./mail-preview.js" -export { - detectBounce, - resolveMessageId, - parseMessageIdList, - type BounceDetection, - type InboundLogger, -} -export { readMailAuthVerdict, sanitizeInboundHtml, type MailAuthVerdict } +export { detectBounce, resolveMessageId } export const INBOUND_PENDING_PREFIX = "inbound/pending/" const INBOUND_FAILED_PREFIX = "inbound/failed/" @@ -47,12 +39,26 @@ export const INBOUND_PENDING_KEY_RE = /^inbound\/pending\/[^/]+\.eml$/ export const INBOUND_ATTACHMENT_MAX_BYTES = 25 * 1024 * 1024 export const INBOUND_ATTACHMENT_MAX_COUNT = 50 -export const INBOUND_ATTACHMENT_TOTAL_MAX_BYTES = 30 * 1024 * 1024 +const INBOUND_ATTACHMENT_TOTAL_MAX_BYTES = 30 * 1024 * 1024 -export const INBOUND_OBJECT_MAX_BYTES = 30 * 1024 * 1024 +const INBOUND_OBJECT_MAX_BYTES = 30 * 1024 * 1024 export const INBOUND_PARSE_TIMEOUT_MS = 20_000 +export const INBOUND_HTML_SOURCE_MAX_CHARS = 1024 * 1024 +export const INBOUND_BODY_TEXT_MAX_CHARS = 256 * 1024 +const INBOUND_HEADER_VALUE_MAX_CHARS = 4 * 1024 +const BODY_TRUNCATION_MARKER = "\n… [truncated]" + +const THREADED_ATTACHMENT_PREFIX = "inbound-mail/" +const INBOX_ATTACHMENT_PREFIX = "inbound-emails/" +const ATTACHMENT_CONTENT_TYPE = "application/octet-stream" +const RAW_MAIL_CONTENT_TYPE = "message/rfc822" +const ATTACHMENT_FILENAME_MAX_CHARS = 120 + +const CIVFIX_HEADER_PREFIX = "x-civfix-" +const AUTH_VERDICT_HEADER = `${CIVFIX_HEADER_PREFIX}auth-verdict` + // At the 5-minute sweep cadence this parks a DSN after roughly half an hour of failing bookkeeping, long // enough to ride out a transient fault and short enough that poison DSNs cannot fill the sweep batch. export const INBOUND_BOUNCE_MAX_ATTEMPTS = 6 @@ -288,7 +294,7 @@ async function insertThreadedMessage( // attachment. const { attachments, oversize } = await streamAttachments( storage, - `inbound-mail/${thread.id}/${contentDigest(raw)}`, + `${THREADED_ATTACHMENT_PREFIX}${thread.id}/${contentDigest(raw)}`, mail, ) const message = await mailRepo.insertMessage({ @@ -315,8 +321,6 @@ function plainTextBody(mail: ParsedMail): string | null { return clipBodyText(htmlToText(mail.html.slice(0, INBOUND_HTML_SOURCE_MAX_CHARS))) } -export const INBOUND_HTML_SOURCE_MAX_CHARS = 1024 * 1024 - function errorText(err: unknown): string { return err instanceof Error ? err.message : String(err) } @@ -333,7 +337,7 @@ async function routeInbox( // attachment objects, so a folder another row also pointed at would lose that row's evidence. const { attachments } = await streamAttachments( storage, - `inbound-emails/${contentDigest(raw)}`, + `${INBOX_ATTACHMENT_PREFIX}${contentDigest(raw)}`, mail, ) const recipient = mail.to[0]?.address ?? null @@ -371,13 +375,10 @@ async function discardUnreferenced( for (const key of orphans) await storage.delete(key) } -export const INBOUND_BODY_TEXT_MAX_CHARS = 256 * 1024 -const INBOUND_HEADER_VALUE_MAX_CHARS = 4 * 1024 - function clipBodyText(text: string | null | undefined): string | null { if (text === null || text === undefined) return null if (text.length <= INBOUND_BODY_TEXT_MAX_CHARS) return text - return `${text.slice(0, INBOUND_BODY_TEXT_MAX_CHARS)}\n… [truncated]` + return `${text.slice(0, INBOUND_BODY_TEXT_MAX_CHARS)}${BODY_TRUNCATION_MARKER}` } const STORED_HEADER_ALLOWLIST = new Set([ @@ -401,14 +402,11 @@ function buildStoredHeaders( const out: Record = {} for (const [rawKey, value] of Object.entries(headers)) { const key = rawKey.toLowerCase() - if (!STORED_HEADER_ALLOWLIST.has(key) && !key.startsWith("x-civfix-")) continue + if (!STORED_HEADER_ALLOWLIST.has(key) && !key.startsWith(CIVFIX_HEADER_PREFIX)) continue if (typeof value !== "string") continue - out[key] = - value.length > INBOUND_HEADER_VALUE_MAX_CHARS - ? value.slice(0, INBOUND_HEADER_VALUE_MAX_CHARS) - : value + out[key] = value.slice(0, INBOUND_HEADER_VALUE_MAX_CHARS) } - out["x-civfix-auth-verdict"] = authVerdict + out[AUTH_VERDICT_HEADER] = authVerdict return out } @@ -444,7 +442,7 @@ async function streamAttachments( continue } const objKey = `${keyPrefix}/${contentDigest(bytes)}/${sanitizeFilename(filename)}` - await storage.put(objKey, bytes, { contentType: "application/octet-stream" }) + await storage.put(objKey, bytes, { contentType: ATTACHMENT_CONTENT_TYPE }) attachments.push({ key: objKey, filename, size: bytes.byteLength }) totalBytes += bytes.byteLength } @@ -480,11 +478,11 @@ async function moveToFailed(storage: Storage, key: string, bytes: Uint8Array): P const failedKey = key.startsWith(INBOUND_PENDING_PREFIX) ? INBOUND_FAILED_PREFIX + key.slice(INBOUND_PENDING_PREFIX.length) : INBOUND_FAILED_PREFIX + key - await storage.put(failedKey, bytes, { contentType: "message/rfc822" }) + await storage.put(failedKey, bytes, { contentType: RAW_MAIL_CONTENT_TYPE }) await storage.delete(key) } function sanitizeFilename(name: string): string { const cleaned = name.replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^\.+/, "") - return cleaned.length > 0 ? cleaned.slice(0, 120) : "file" + return cleaned.length > 0 ? cleaned.slice(0, ATTACHMENT_FILENAME_MAX_CHARS) : "file" } diff --git a/services/api/src/services/admin/inbound-repository.drizzle.ts b/services/api/src/services/admin/inbound-repository.drizzle.ts index 3d8e29fa..0f48f203 100644 --- a/services/api/src/services/admin/inbound-repository.drizzle.ts +++ b/services/api/src/services/admin/inbound-repository.drizzle.ts @@ -41,8 +41,6 @@ export interface InboundRepository { clearBounceFailures(objectKey: string): Promise } -export { toPreview } - export function localPartOf(recipient: string | null): string { if (!recipient) return "" const at = recipient.indexOf("@") diff --git a/services/api/src/services/admin/inbound-retention-repository.drizzle.ts b/services/api/src/services/admin/inbound-retention-repository.drizzle.ts index ef863f3f..4663c92d 100644 --- a/services/api/src/services/admin/inbound-retention-repository.drizzle.ts +++ b/services/api/src/services/admin/inbound-retention-repository.drizzle.ts @@ -1,6 +1,10 @@ import type { Sql } from "../../db/client.js" -export const INBOUND_EMAIL_RETENTION_MS = 180 * 24 * 60 * 60 * 1000 +const DAY_MS = 24 * 60 * 60 * 1000 + +const INBOUND_EMAIL_RETENTION_DAYS = 180 + +export const INBOUND_EMAIL_RETENTION_MS = INBOUND_EMAIL_RETENTION_DAYS * DAY_MS export const INBOUND_EMAIL_RETENTION_BATCH = 200 diff --git a/services/api/src/services/admin/inbound-sweep.ts b/services/api/src/services/admin/inbound-sweep.ts index 61f8b0ec..7786e2bb 100644 --- a/services/api/src/services/admin/inbound-sweep.ts +++ b/services/api/src/services/admin/inbound-sweep.ts @@ -11,11 +11,11 @@ import { inboundEffectDeps, } from "./inbound-thread-correlation.js" -export const INBOUND_SWEEP_BATCH = 200 +const INBOUND_SWEEP_BATCH = 200 const LIST_PAGE = 100 -export const INBOUND_EFFECTS_REDRIVE_BATCH = 100 -export const INBOUND_EFFECTS_REDRIVE_MIN_AGE_MS = 60_000 +const INBOUND_EFFECTS_REDRIVE_BATCH = 100 +const INBOUND_EFFECTS_REDRIVE_MIN_AGE_MS = 60_000 export interface InboundSweepResult { scanned: number @@ -28,15 +28,20 @@ export interface InboundSweepResult { listError?: string } +export interface EffectsRedriveOptions { + deps?: InboundProcessorDeps + now?: () => Date + effectsBatch?: number + effectsMinAgeMs?: number +} + +export interface InboundSweepOptions extends EffectsRedriveOptions { + batch?: number +} + export async function runInboundSweep( container: Container, - opts: { - batch?: number - deps?: InboundProcessorDeps - now?: () => Date - effectsBatch?: number - effectsMinAgeMs?: number - } = {}, + opts: InboundSweepOptions = {}, ): Promise { const storage = opts.deps?.storage ?? container.inboundStorage const cap = opts.batch ?? INBOUND_SWEEP_BATCH @@ -89,12 +94,7 @@ export async function runInboundSweep( async function redriveEffects( container: Container, - opts: { - deps?: InboundProcessorDeps - now?: () => Date - effectsBatch?: number - effectsMinAgeMs?: number - }, + opts: EffectsRedriveOptions, ): Promise<{ effectsRedriven: number; effectsErrors: number; effectsError?: string }> { const now = (opts.now ?? (() => new Date()))() const minAge = opts.effectsMinAgeMs ?? INBOUND_EFFECTS_REDRIVE_MIN_AGE_MS diff --git a/services/api/src/services/admin/inbound-thread-correlation.ts b/services/api/src/services/admin/inbound-thread-correlation.ts index f6db60ee..dd133b90 100644 --- a/services/api/src/services/admin/inbound-thread-correlation.ts +++ b/services/api/src/services/admin/inbound-thread-correlation.ts @@ -42,13 +42,20 @@ export const EFFECTS_LEASE_MS = 10 * 60 * 1000 const EVENT_REPLY_FALLBACK_NOTE = "Jurisdiction replied" -export const EFFECTS_STAGE_TIMELINE = 1 -export const EFFECTS_STAGE_CHAT = 2 -export const EFFECTS_STAGE_NOTIFIED = 3 +const EFFECTS_STAGE_TIMELINE = 1 +const EFFECTS_STAGE_CHAT = 2 +const EFFECTS_STAGE_NOTIFIED = 3 + +const JURISDICTION_REPLY_NOTIFICATION_TITLE = "Your report got a response" export const JURISDICTION_REPLY_NOTIFICATION_BODY = "The city responded. See their reply in the report chat." +const BRACKETED_MESSAGE_ID_RE = /<[^>]+>/g + +const DERIVED_ID_BODY_PREFIX_CHARS = 4096 +const DERIVED_ID_PREFIX = "derived:" + export function inboundEffectDeps( deps: { adminReportRepo?: AdminReportRepository @@ -132,7 +139,7 @@ export function stripQuotedHistory(raw: string, replyDomain = DEFAULT_REPLY_DOMA return lines.slice(0, end).join("\n").trim() } -export function clipToMessageBody(text: string, max: number = MESSAGE_BODY_MAX): string { +function clipToMessageBody(text: string, max: number = MESSAGE_BODY_MAX): string { if (text.length <= max) return text let kept = "" for (const cluster of segmentGraphemes(text)) { @@ -166,9 +173,7 @@ export const MESSAGE_ID_LIST_CAP = 20 export function parseMessageIdList(value: string | undefined): string[] { if (!value || value.length === 0) return [] const out: string[] = [] - const re = /<[^>]+>/g - let m: RegExpExecArray | null - while ((m = re.exec(value)) !== null) { + for (const m of value.matchAll(BRACKETED_MESSAGE_ID_RE)) { out.push(m[0]) if (out.length >= MESSAGE_ID_LIST_CAP) return out } @@ -266,7 +271,7 @@ export async function applyInboundEffects( } } -export async function onJurisdictionReply( +async function onJurisdictionReply( container: Container, injected: InboundEffectDeps, mailRepo: MailRepository, @@ -282,20 +287,22 @@ export async function onJurisdictionReply( const record = await reportRepo.getReport(reportId) if (!record) return - const note = JURISDICTION_REPLY_NOTE - if (stage < EFFECTS_STAGE_TIMELINE) { const advances = record.status === "published" || record.status === "acknowledged" if (advances) { await reportRepo.setStatus(reportId, { status: "in_progress", - note, + note: JURISDICTION_REPLY_NOTE, actorId: null, kind: "reply", body: null, }) } else { - await reportRepo.appendSystemTimeline(reportId, { note, kind: "reply", body: null }) + await reportRepo.appendSystemTimeline(reportId, { + note: JURISDICTION_REPLY_NOTE, + kind: "reply", + body: null, + }) } await mailRepo.setMessageEffectsStage(messageId, EFFECTS_STAGE_TIMELINE) } @@ -309,7 +316,7 @@ export async function onJurisdictionReply( reportId, status: current?.status ?? record.status, kind: "reply", - note, + note: JURISDICTION_REPLY_NOTE, body: cityReplyChatBody(message.body, container.env.MAIL_REPLY_DOMAIN), }) await mailRepo.setMessageEffectsStage(messageId, EFFECTS_STAGE_CHAT) @@ -321,7 +328,7 @@ export async function onJurisdictionReply( const notifications = injected.notifications ?? container.getNotificationService() await notifications.createNotification(reporterUserId, { type: "report_update", - title: "Your report got a response", + title: JURISDICTION_REPLY_NOTIFICATION_TITLE, body: JURISDICTION_REPLY_NOTIFICATION_BODY, link: `/reports/${reportId}`, }) @@ -357,8 +364,6 @@ export async function onEventReply( await mailRepo.setThreadStatus(thread.id, "replied") } -const DERIVED_ID_BODY_PREFIX_CHARS = 4096 - export function resolveMessageId(mail: ParsedMail): string { if (mail.messageId && mail.messageId.length > 0) return mail.messageId const body = mail.text ?? mail.html ?? "" @@ -369,5 +374,5 @@ export function resolveMessageId(mail: ParsedMail): string { String(body.length), body.slice(0, DERIVED_ID_BODY_PREFIX_CHARS), ].join("|") - return `derived:${createHash("sha256").update(basis).digest("hex")}` + return `${DERIVED_ID_PREFIX}${createHash("sha256").update(basis).digest("hex")}` } diff --git a/services/api/src/services/admin/jurisdiction-contacts-repository.drizzle.ts b/services/api/src/services/admin/jurisdiction-contacts-repository.drizzle.ts index 866c15bc..9c190c44 100644 --- a/services/api/src/services/admin/jurisdiction-contacts-repository.drizzle.ts +++ b/services/api/src/services/admin/jurisdiction-contacts-repository.drizzle.ts @@ -17,6 +17,8 @@ import { type CategoryCountRow, } from "./category-counts.js" import type { + DirectoryFilter, + DirectorySort, JurisdictionContactsRepository, JurisdictionDirectoryRecord, JurisdictionGeometryRecord, @@ -27,11 +29,13 @@ import type { } from "./jurisdiction-contacts-types.js" import { AppError } from "@civfix/shared" import type { JurisdictionLayer, ReportCategory } from "@civfix/shared" -import { ilikeAnyOf } from "./sql-fragments.js" +import { ilikeAnyOf, type SqlFragment } from "./sql-fragments.js" const PG_UNIQUE_VIOLATION = "23505" const JURISDICTION_HANDLE_CONSTRAINT = "jurisdictions_handle_lower_key" +const HANDLE_TAKEN_BY_JURISDICTION = "That @handle is already used by another jurisdiction." + function isJurisdictionHandleConflict(err: unknown): boolean { if (typeof err !== "object" || err === null) return false const e = err as { code?: unknown; constraint_name?: unknown } @@ -105,6 +109,77 @@ async function loadUnmappedAggregate( return { total: parseCount(r?.total), perCategoryCounts: parseCategoryCounts(r) } } +function directoryMethodFilter( + sql: Sql, + filter: DirectoryFilter, + hasEmailExpr: SqlFragment, + hasFormExpr: SqlFragment, +): SqlFragment { + switch (filter) { + case "email": + return sql`AND ${hasEmailExpr}` + case "form": + return sql`AND NOT ${hasEmailExpr} AND ${hasFormExpr}` + case "none": + return sql`AND NOT (${hasEmailExpr} OR ${hasFormExpr})` + case "routed": + return sql`AND (${hasEmailExpr} OR ${hasFormExpr})` + case "needs_mapping": + return sql`AND NOT (${hasEmailExpr} OR ${hasFormExpr}) AND COALESCE(w.total, 0) > 0` + default: + return sql`` + } +} + +function directoryOrderBy(sql: Sql, sort: DirectorySort): SqlFragment { + switch (sort) { + case "reports": + return sql`ORDER BY COALESCE(w.total, 0) DESC, j.geoid ASC` + case "name": + return sql`ORDER BY j.name ASC, j.geoid ASC` + case "oldest": + return sql`ORDER BY w.oldest_waiting_at ASC NULLS LAST, j.geoid ASC` + default: + return sql`ORDER BY COALESCE(j.population, 0) DESC, j.geoid ASC` + } +} + +interface FacetAggregateRow { + total: string + routed: string + unrouted: string +} + +async function directoryTotals( + args: ListDirectoryArgs, + offset: number, + filteredTotal: string | undefined, + queryFacets: () => Promise, +): Promise> { + if (offset !== 0) return { total: null, facets: null } + const isDefaultView = args.filter === "all" && args.q === null && args.layer === null + const cached = isDefaultView ? readDirectoryFacetCache() : null + if (cached !== null) return { total: cached.total, facets: cached.facets } + const a = await queryFacets() + const facets = { routed: Number(a?.routed ?? "0"), unrouted: Number(a?.unrouted ?? "0") } + if (isDefaultView) writeDirectoryFacetCache({ total: Number(a?.total ?? "0"), facets }) + return { total: Number(filteredTotal ?? "0"), facets } +} + +async function withUnmappedRecord( + sql: Sql, + args: ListDirectoryArgs, + result: ListDirectoryResult, +): Promise { + if (!shouldIncludeUnmapped(args)) return result + const unmapped = await loadUnmappedAggregate(sql) + if (unmapped.total <= 0) return result + return { + ...result, + records: [buildUnmappedRecord(unmapped.total, unmapped.perCategoryCounts), ...result.records], + } +} + export function makeDrizzleJurisdictionContactsRepository( sql: Sql, ): JurisdictionContactsRepository { @@ -217,7 +292,7 @@ export function makeDrizzleJurisdictionContactsRepository( LIMIT 1 ` if (dupeJurisdiction.length > 0) { - throw AppError.conflict("That @handle is already used by another jurisdiction.") + throw AppError.conflict(HANDLE_TAKEN_BY_JURISDICTION) } const dupeUser = await tx<{ id: string }[]>` SELECT id FROM users WHERE lower(handle::text) = lower(${handle}) LIMIT 1 @@ -229,7 +304,7 @@ export function makeDrizzleJurisdictionContactsRepository( await tx`UPDATE jurisdictions SET handle = ${handle} WHERE geoid = ${geoid}` } catch (err) { if (isJurisdictionHandleConflict(err)) { - throw AppError.conflict("That @handle is already used by another jurisdiction.") + throw AppError.conflict(HANDLE_TAKEN_BY_JURISDICTION) } throw err } @@ -289,29 +364,11 @@ export function makeDrizzleJurisdictionContactsRepository( ) )` const hasFormExpr = sql`(j.report_form_url IS NOT NULL AND btrim(j.report_form_url) <> '')` - const methodFilter = - args.filter === "email" - ? sql`AND ${hasEmailExpr}` - : args.filter === "form" - ? sql`AND NOT ${hasEmailExpr} AND ${hasFormExpr}` - : args.filter === "none" - ? sql`AND NOT (${hasEmailExpr} OR ${hasFormExpr})` - : args.filter === "routed" - ? sql`AND (${hasEmailExpr} OR ${hasFormExpr})` - : args.filter === "needs_mapping" - ? sql`AND NOT (${hasEmailExpr} OR ${hasFormExpr}) AND COALESCE(w.total, 0) > 0` - : sql`` + const methodFilter = directoryMethodFilter(sql, args.filter, hasEmailExpr, hasFormExpr) const layerFilter = args.layer !== null ? sql`AND j.layer = ${args.layer}` : sql`` - const orderBy = - args.sort === "reports" - ? sql`ORDER BY COALESCE(w.total, 0) DESC, j.geoid ASC` - : args.sort === "name" - ? sql`ORDER BY j.name ASC, j.geoid ASC` - : args.sort === "oldest" - ? sql`ORDER BY w.oldest_waiting_at ASC NULLS LAST, j.geoid ASC` - : sql`ORDER BY COALESCE(j.population, 0) DESC, j.geoid ASC` + const orderBy = directoryOrderBy(sql, args.sort) const rows = await sql` WITH waiting AS ( @@ -400,16 +457,12 @@ export function makeDrizzleJurisdictionContactsRepository( const page = (hasMore ? rows.slice(0, limit) : rows).map(toRecord) const nextCursor = hasMore ? encodeOffsetCursor(offset + limit) : null - let total: number | null = null - let facets: { routed: number; unrouted: number } | null = null - if (offset === 0) { - const isDefaultView = args.q === null && args.layer === null - const cached = args.filter === "all" && isDefaultView ? readDirectoryFacetCache() : null - if (cached !== null) { - total = cached.total - facets = cached.facets - } else { - const agg = await sql<{ total: string; routed: string; unrouted: string }[]>` + const { total, facets } = await directoryTotals( + args, + offset, + rows[0]?.filtered_total, + async () => { + const agg = await sql` SELECT COUNT(*)::text AS total, COUNT(*) FILTER (WHERE ${hasEmailExpr} OR ${hasFormExpr})::text AS routed, @@ -417,27 +470,11 @@ export function makeDrizzleJurisdictionContactsRepository( FROM jurisdictions j WHERE true ${search} ${layerFilter} ` - const a = agg[0] - total = Number(rows[0]?.filtered_total ?? "0") - facets = { routed: Number(a?.routed ?? "0"), unrouted: Number(a?.unrouted ?? "0") } - if (args.filter === "all" && isDefaultView) { - writeDirectoryFacetCache({ total: Number(a?.total ?? "0"), facets }) - } - } - } + return agg[0] + }, + ) - if (shouldIncludeUnmapped(args)) { - const unmapped = await loadUnmappedAggregate(sql) - if (unmapped.total > 0) { - return { - records: [buildUnmappedRecord(unmapped.total, unmapped.perCategoryCounts), ...page], - nextCursor, - total, - facets, - } - } - } - return { records: page, nextCursor, total, facets } + return withUnmappedRecord(sql, args, { records: page, nextCursor, total, facets }) }, async getGeometry(geoid: string): Promise { diff --git a/services/api/src/services/admin/jurisdiction-contacts-repository.memory.ts b/services/api/src/services/admin/jurisdiction-contacts-repository.memory.ts index c0214afc..5364cb48 100644 --- a/services/api/src/services/admin/jurisdiction-contacts-repository.memory.ts +++ b/services/api/src/services/admin/jurisdiction-contacts-repository.memory.ts @@ -55,6 +55,8 @@ export interface SeededContactsTask { const NON_WAITING = new Set(["rejected", "resolved", "acknowledged", "in_progress"]) +const HANDLE_TAKEN_BY_JURISDICTION = "That @handle is already used by another jurisdiction." + export interface RecordedContactsAudit { actorId: string | null action: string @@ -175,14 +177,7 @@ export class InMemoryJurisdictionContactsRepository implements JurisdictionConta if (!j) return { taskResolved: false } applyContacts(j, input) - if (input.forwardSubjectTemplate !== undefined) { - const template = input.forwardSubjectTemplate - j.forwardSubjectTemplate = template === null || template === "" ? null : template - } - if (input.forwardBodyTemplate !== undefined) { - const template = input.forwardBodyTemplate - j.forwardBodyTemplate = template === null || template === "" ? null : template - } + applyForwardTemplates(j, input) j.contactUpdatedAt = this.now let taskResolved = false @@ -240,20 +235,13 @@ export class InMemoryJurisdictionContactsRepository implements JurisdictionConta other.handle !== null && other.handle.toLowerCase() === handle.toLowerCase() ) { - throw AppError.conflict("That @handle is already used by another jurisdiction.") + throw AppError.conflict(HANDLE_TAKEN_BY_JURISDICTION) } } j.handle = handle } } - if (input.forwardSubjectTemplate !== undefined) { - const t = input.forwardSubjectTemplate - j.forwardSubjectTemplate = t === null || t === "" ? null : t - } - if (input.forwardBodyTemplate !== undefined) { - const t = input.forwardBodyTemplate - j.forwardBodyTemplate = t === null || t === "" ? null : t - } + applyForwardTemplates(j, input) if (touchedContact) j.contactUpdatedAt = this.now this.audits.push({ actorId: audit.actorId, @@ -278,43 +266,17 @@ export class InMemoryJurisdictionContactsRepository implements JurisdictionConta async listDirectory(args: ListDirectoryArgs): Promise { const all = [...this.jurisdictions.values()].map((j) => toRecord(j, this.reports)) + const needle = args.q?.toLowerCase() ?? null const matched = - args.q !== null - ? (() => { - const needle = args.q.toLowerCase() - return all.filter( - (r) => - r.name.toLowerCase().includes(needle) || r.geoid.toLowerCase().includes(needle), - ) - })() + needle !== null + ? all.filter( + (r) => r.name.toLowerCase().includes(needle) || r.geoid.toLowerCase().includes(needle), + ) : all const searched = args.layer !== null ? matched.filter((r) => r.layer === args.layer) : matched - const records = - args.filter === "all" - ? searched.slice() - : args.filter === "routed" - ? searched.filter((r) => directoryMethod(r) !== "none") - : args.filter === "needs_mapping" - ? searched.filter((r) => directoryMethod(r) === "none" && r.reportsWaiting > 0) - : searched.filter((r) => directoryMethod(r) === args.filter) - - const byGeoid = (a: JurisdictionDirectoryRecord, b: JurisdictionDirectoryRecord) => - a.geoid < b.geoid ? -1 : a.geoid > b.geoid ? 1 : 0 - records.sort((a, b) => { - if (args.sort === "name") return a.name < b.name ? -1 : a.name > b.name ? 1 : byGeoid(a, b) - if (args.sort === "oldest") { - const at = a.oldestReportAt?.getTime() ?? null - const bt = b.oldestReportAt?.getTime() ?? null - if (at === null && bt === null) return byGeoid(a, b) - if (at === null) return 1 - if (bt === null) return -1 - return at - bt || byGeoid(a, b) - } - const av = args.sort === "reports" ? a.reportsWaiting : (a.population ?? 0) - const bv = args.sort === "reports" ? b.reportsWaiting : (b.population ?? 0) - return bv - av || byGeoid(a, b) - }) + const records = filterByMethod(searched, args.filter) + records.sort(directoryComparator(args.sort)) const limit = clampLimit(args.limit) const offset = decodeOffsetCursor(args.cursor) @@ -365,6 +327,57 @@ export class InMemoryJurisdictionContactsRepository implements JurisdictionConta } } +function blankToNull(template: string | null): string | null { + return template === null || template === "" ? null : template +} + +function applyForwardTemplates( + j: SeededJurisdiction, + input: { forwardSubjectTemplate?: string | null; forwardBodyTemplate?: string | null }, +): void { + if (input.forwardSubjectTemplate !== undefined) { + j.forwardSubjectTemplate = blankToNull(input.forwardSubjectTemplate) + } + if (input.forwardBodyTemplate !== undefined) { + j.forwardBodyTemplate = blankToNull(input.forwardBodyTemplate) + } +} + +function filterByMethod( + searched: JurisdictionDirectoryRecord[], + filter: ListDirectoryArgs["filter"], +): JurisdictionDirectoryRecord[] { + if (filter === "all") return searched.slice() + if (filter === "routed") return searched.filter((r) => directoryMethod(r) !== "none") + if (filter === "needs_mapping") { + return searched.filter((r) => directoryMethod(r) === "none" && r.reportsWaiting > 0) + } + return searched.filter((r) => directoryMethod(r) === filter) +} + +function byGeoid(a: JurisdictionDirectoryRecord, b: JurisdictionDirectoryRecord): number { + return a.geoid < b.geoid ? -1 : a.geoid > b.geoid ? 1 : 0 +} + +function directoryComparator( + sort: ListDirectoryArgs["sort"], +): (a: JurisdictionDirectoryRecord, b: JurisdictionDirectoryRecord) => number { + return (a, b) => { + if (sort === "name") return a.name < b.name ? -1 : a.name > b.name ? 1 : byGeoid(a, b) + if (sort === "oldest") { + const at = a.oldestReportAt?.getTime() ?? null + const bt = b.oldestReportAt?.getTime() ?? null + if (at === null && bt === null) return byGeoid(a, b) + if (at === null) return 1 + if (bt === null) return -1 + return at - bt || byGeoid(a, b) + } + const av = sort === "reports" ? a.reportsWaiting : (a.population ?? 0) + const bv = sort === "reports" ? b.reportsWaiting : (b.population ?? 0) + return bv - av || byGeoid(a, b) + } +} + function applyContacts(j: SeededJurisdiction, input: SaveContactsInput): void { let wroteEmail = false for (const [category, email] of Object.entries(input.contacts) as [ diff --git a/services/api/src/services/admin/jurisdiction-contacts-service.ts b/services/api/src/services/admin/jurisdiction-contacts-service.ts index 0b4cd100..ecfebd5b 100644 --- a/services/api/src/services/admin/jurisdiction-contacts-service.ts +++ b/services/api/src/services/admin/jurisdiction-contacts-service.ts @@ -6,6 +6,8 @@ import type { JurisdictionListQuery } from "@civfix/shared" import { toDirectoryDTO, hasAnyContact } from "./jurisdiction-directory-projection.js" import type { DirectoryFilter, DirectorySort } from "./jurisdiction-contacts-types.js" import { isReservedHandle } from "../../auth/reserved-handles.js" +import { ADMIN_DEFAULT_LIMIT } from "./pagination.js" +import { isThrottled } from "./outreach-service.js" import { OUTREACH_DIGEST_JOB, type JurisdictionContactsService, @@ -18,6 +20,8 @@ import { export * from "./jurisdiction-contacts-types.js" export * from "./jurisdiction-directory-projection.js" +const JURISDICTION_NOT_FOUND = "Jurisdiction not found" + export function makeJurisdictionContactsService( deps: JurisdictionContactsServiceDeps, ): JurisdictionContactsService { @@ -30,7 +34,7 @@ export function makeJurisdictionContactsService( actorId: string, ): Promise { const exists = await deps.repo.jurisdictionExists(geoid) - if (!exists) throw AppError.notFound("Jurisdiction not found") + if (!exists) throw AppError.notFound(JURISDICTION_NOT_FOUND) if (!hasAnyContact(input)) { throw AppError.validation({ contacts: "At least one contact is required to route." }) } @@ -53,7 +57,7 @@ export function makeJurisdictionContactsService( throw AppError.validation({ handle: "That @handle is reserved." }) } const ok = await deps.repo.patch(geoid, input, { actorId }) - if (!ok) throw AppError.notFound("Jurisdiction not found") + if (!ok) throw AppError.notFound(JURISDICTION_NOT_FOUND) }, async listDirectory(query: JurisdictionListQuery) { @@ -64,7 +68,7 @@ export function makeJurisdictionContactsService( layer: query.layer ?? null, sort: (query.sort ?? "population") as DirectorySort, cursor: query.cursor ?? null, - limit: query.limit ?? 25, + limit: query.limit ?? ADMIN_DEFAULT_LIMIT, }) return { items: records.map(toDirectoryDTO), @@ -87,10 +91,7 @@ export function makeJurisdictionContactsService( if (!deps.outreachDigestEnabled) return false const state = await deps.repo.getOutreachState(geoid) if (state?.suppressed) return false - if (state?.lastOutreachAt) { - const windowMs = deps.throttleDays * 24 * 60 * 60 * 1000 - if (now().getTime() - state.lastOutreachAt.getTime() < windowMs) return false - } + if (isThrottled(state?.lastOutreachAt ?? null, now(), deps.throttleDays)) return false await deps.jobs.enqueue(OUTREACH_DIGEST_JOB, { geoid }, { singletonKey: geoid }) return true } diff --git a/services/api/src/services/admin/jurisdiction-directory-projection.ts b/services/api/src/services/admin/jurisdiction-directory-projection.ts index 2f22284d..bd937aec 100644 --- a/services/api/src/services/admin/jurisdiction-directory-projection.ts +++ b/services/api/src/services/admin/jurisdiction-directory-projection.ts @@ -41,7 +41,7 @@ export function directoryStatus( return "pending" } -export function primaryEmail(record: JurisdictionDirectoryRecord): string | null { +function primaryEmail(record: JurisdictionDirectoryRecord): string | null { const def = record.defaultEmails.find((e) => e.trim() !== "") if (def) return def const cat = record.categoryContacts.find((c) => c.email !== null && c.email.trim() !== "") diff --git a/services/api/src/services/admin/mail-format.ts b/services/api/src/services/admin/mail-format.ts index bb851f31..9f23cacd 100644 --- a/services/api/src/services/admin/mail-format.ts +++ b/services/api/src/services/admin/mail-format.ts @@ -39,18 +39,26 @@ const ATTACHMENT_EXTENSIONS: Record = { } const EXTENSION_FALLBACK = ".jpg" +const ATTACHMENT_FILENAME_MAX_CHARS = 120 +const FILE_EXTENSION_RE = /\.[A-Za-z0-9]{2,5}$/ +const UNSAFE_FILENAME_CHARS_RE = /[^A-Za-z0-9._-]+/g +const LEADING_DOTS_RE = /^\.+/ + +const SHORT_ID_CHARS = 8 +const UNKNOWN_PLACE_LABEL = "the area" +const MAP_LINK_ZOOM = 18 function hasExtension(name: string): boolean { - return /\.[A-Za-z0-9]{2,5}$/.test(name) + return FILE_EXTENSION_RE.test(name) } export function attachmentFilename(r2Key: string, index: number, contentType?: string): string { const ext = ATTACHMENT_EXTENSIONS[(contentType ?? "").trim().toLowerCase()] ?? EXTENSION_FALLBACK const tail = r2Key.split("/").pop() ?? "" - const cleaned = tail.replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^\.+/, "") + const cleaned = tail.replace(UNSAFE_FILENAME_CHARS_RE, "_").replace(LEADING_DOTS_RE, "") if (cleaned.length === 0) return `photo-${index + 1}${ext}` - if (hasExtension(cleaned)) return cleaned.slice(0, 120) - return `${cleaned.slice(0, 120 - ext.length)}${ext}` + if (hasExtension(cleaned)) return cleaned.slice(0, ATTACHMENT_FILENAME_MAX_CHARS) + return `${cleaned.slice(0, ATTACHMENT_FILENAME_MAX_CHARS - ext.length)}${ext}` } export interface ReportPacket { @@ -93,7 +101,7 @@ function mediaListHeading(media: readonly PacketMediaLink[]): string { } function mapLinkFor(lat: number, lng: number): string { - return `https://www.openstreetmap.org/?mlat=${lat}&mlon=${lng}#map=18/${lat}/${lng}` + return `https://www.openstreetmap.org/?mlat=${lat}&mlon=${lng}#map=${MAP_LINK_ZOOM}/${lat}/${lng}` } // The API container runs in UTC, which turns a US evening report into the next day. No jurisdiction @@ -115,7 +123,8 @@ function buildTemplateValues( labelledMedia: readonly LabelledMediaLink[], noteText: string | null, ): Record { - const ref = record.referenceCode ?? record.id.slice(0, 8) + const shortId = record.id.slice(0, SHORT_ID_CHARS) + const ref = record.referenceCode ?? shortId const categoryLabel = REPORT_CATEGORY_LABELS[record.category] const place = routing?.place ?? record.place const address = record.address && record.address.trim() !== "" ? record.address : place @@ -123,7 +132,7 @@ function buildTemplateValues( return { referenceCode: ref, reportId: record.id, - shortId: record.id.slice(0, 8), + shortId, title: record.title, category: categoryLabel, status: ADMIN_REPORT_STATUS_LABELS[record.status], @@ -234,9 +243,9 @@ export interface DiscussionForwardInput { displayName?: string | null } -export const DISCUSSION_FORWARD_ANONYMOUS_AUTHOR = "A neighbor" +const DISCUSSION_FORWARD_ANONYMOUS_AUTHOR = "A neighbor" -export function discussionForwardAuthor(displayName: string | null | undefined): string { +function discussionForwardAuthor(displayName: string | null | undefined): string { const trimmed = (displayName ?? "").trim() return trimmed === "" ? DISCUSSION_FORWARD_ANONYMOUS_AUTHOR : trimmed } @@ -245,8 +254,8 @@ export function buildDiscussionForwardPacket( input: DiscussionForwardInput, comment: string, ): ReportPacket { - const id8 = input.reportId.slice(0, 8) - const place = input.place ?? input.org ?? "the area" + const id8 = input.reportId.slice(0, SHORT_ID_CHARS) + const place = input.place ?? input.org ?? UNKNOWN_PLACE_LABEL const subject = `civfix report: ${sanitizeHeaderValue(`${input.category} in ${place}`)} [${id8}]` const body = comment.trim() !== "" ? comment.trim() : "(no comment provided)" const author = discussionForwardAuthor(input.displayName) @@ -280,7 +289,7 @@ export function buildEventPacket(event: EventPacketInput, message: string): Repo const ref = event.referenceCode ?? null const subject = ref !== null ? `civfix event: ${safeTitle} [${ref}]` : `civfix event: ${safeTitle}` - const place = event.place ?? "the area" + const place = event.place ?? UNKNOWN_PLACE_LABEL const address = event.address && event.address.trim() !== "" ? event.address : place const msgText = message.trim() !== "" ? message.trim() : "(no message provided)" diff --git a/services/api/src/services/admin/mail-mappers.ts b/services/api/src/services/admin/mail-mappers.ts index 9539868b..15992ff2 100644 --- a/services/api/src/services/admin/mail-mappers.ts +++ b/services/api/src/services/admin/mail-mappers.ts @@ -1,4 +1,3 @@ -import type { CursorAnchor } from "./pagination.js" import { toPreview } from "./mail-preview.js" import type { MailMessageKind, @@ -107,18 +106,23 @@ export function toOutreachRecord(r: OutreachRowSelect): OutreachStateRecord { } const TOKEN_ALPHABET = "abcdefghijklmnopqrstuvwxyz234567" +const THREAD_TOKEN_CHARS = 12 +const TOKEN_ALPHABET_MASK = TOKEN_ALPHABET.length - 1 + +const INBOUND_WHO_FALLBACK = "Inbound" +const OUTBOUND_WHO_FALLBACK = "civfix" export function mintThreadToken(): string { - const bytes = new Uint8Array(12) + const bytes = new Uint8Array(THREAD_TOKEN_CHARS) globalThis.crypto.getRandomValues(bytes) let out = "" - for (const b of bytes) out += TOKEN_ALPHABET[b & 31] + for (const b of bytes) out += TOKEN_ALPHABET[b & TOKEN_ALPHABET_MASK] return out } export function deriveWho(direction: MailDirection, fromAddr: string | null): string { if (fromAddr && fromAddr.length > 0) return fromAddr - return direction === "in" ? "Inbound" : "civfix" + return direction === "in" ? INBOUND_WHO_FALLBACK : OUTBOUND_WHO_FALLBACK } export function toThreadListItem( @@ -172,7 +176,3 @@ export function toThreadDTO( messages: messages.map(toMessageDTO), } } - -export function anchorOf(thread: MailThreadRecord): CursorAnchor { - return { createdAt: thread.lastMessageAt ?? thread.createdAt, id: thread.id } -} diff --git a/services/api/src/services/admin/mail-preview.ts b/services/api/src/services/admin/mail-preview.ts index 7199b3de..a1612fb0 100644 --- a/services/api/src/services/admin/mail-preview.ts +++ b/services/api/src/services/admin/mail-preview.ts @@ -21,6 +21,17 @@ const LINK_HREF_RE = /\shref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))/i const HTML_TAG_NAME_RE = /[a-zA-Z][^\s/>]{0,16}/y +const ABSOLUTE_HTTP_URL_RE = /^https?:\/\//i + +const EXTRA_BLANK_LINES_RE = /\n{3,}/g + +const WHITESPACE_RUN_RE = /\s+/g + +const MAX_CODE_POINT = 0x10ffff +const SURROGATE_MIN = 0xd800 +const SURROGATE_MAX = 0xdfff +const REPLACEMENT_CHARACTER = 0xfffd + function tagNameAt(html: string, at: number): { name: string; end: number } | null { HTML_TAG_NAME_RE.lastIndex = at const m = HTML_TAG_NAME_RE.exec(html) @@ -46,7 +57,7 @@ export function htmlToText(html: string): string { let href: string | null = null let anchorText = "" const breakLine = (): void => { - const text = line.replace(/\s+/g, " ").trim() + const text = collapse(line) lines.push(quoteDepth > 0 && text !== "" ? `> ${text}` : text) line = "" } @@ -89,7 +100,7 @@ export function htmlToText(html: string): string { href = m === null ? null : decodeTextEntities(m[1] ?? m[2] ?? m[3] ?? "") anchorText = "" } else if (href !== null) { - if (/^https?:\/\//i.test(href) && anchorText.trim() !== href) line += ` (${href})` + if (ABSOLUTE_HTTP_URL_RE.test(href) && anchorText.trim() !== href) line += ` (${href})` href = null } continue @@ -102,10 +113,7 @@ export function htmlToText(html: string): string { i = closeGt + 1 } breakLine() - return lines - .join("\n") - .replace(/\n{3,}/g, "\n\n") - .trim() + return lines.join("\n").replace(EXTRA_BLANK_LINES_RE, "\n\n").trim() } const TEXT_NAMED_ENTITIES: ReadonlyMap = new Map( @@ -153,14 +161,15 @@ function decodeTextEntities(text: string): string { } function codePointText(code: number): string { - const invalid = code === 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff) - return String.fromCodePoint(invalid ? 0xfffd : code) + const invalid = + code === 0 || code > MAX_CODE_POINT || (code >= SURROGATE_MIN && code <= SURROGATE_MAX) + return String.fromCodePoint(invalid ? REPLACEMENT_CHARACTER : code) } function collapse(value: string): string { - return value.replace(/\s+/g, " ").trim() + return value.replace(WHITESPACE_RUN_RE, " ").trim() } function truncate(value: string): string { - return value.length > PREVIEW_LEN ? value.slice(0, PREVIEW_LEN) : value + return value.slice(0, PREVIEW_LEN) } diff --git a/services/api/src/services/admin/mail-repository.drizzle.ts b/services/api/src/services/admin/mail-repository.drizzle.ts index 0f706c44..bce2bcce 100644 --- a/services/api/src/services/admin/mail-repository.drizzle.ts +++ b/services/api/src/services/admin/mail-repository.drizzle.ts @@ -21,6 +21,7 @@ import { type ThreadRowSelect, } from "./mail-mappers.js" import { buildMailStats } from "./mail-stats.js" +import { parseCount } from "./category-counts.js" import { sendInFlightExpr } from "./outbound-send-sql.js" import { MAIL_STATS_WINDOW_DAYS, @@ -55,17 +56,7 @@ import type { } from "@civfix/shared" export * from "./mail-repository.js" -export { - anchorOf, - deriveWho, - mintThreadToken, - toMessageDTO, - toMessageRecord, - toOutreachRecord, - toThreadDTO, - toThreadListItem, - toThreadRecord, -} from "./mail-mappers.js" +export { deriveWho, mintThreadToken, toThreadDTO } from "./mail-mappers.js" export { buildMailStats } from "./mail-stats.js" const MAIL_THREAD_MESSAGE_CAP = 500 @@ -154,6 +145,49 @@ async function insertOrSelectThread( return toThreadRecord(row) } +interface ThreadListRowSelect extends ThreadRowSelect { + lm_direction: MailDirection | null + lm_from_addr: string | null + lm_to_addr: string | null + lm_body: string | null +} + +function latestPreviewMessage( + r: ThreadListRowSelect, + thread: MailThreadRecord, +): MailMessageRecord | null { + if (r.lm_direction === null) return null + return { + id: "", + threadId: r.id, + direction: r.lm_direction, + fromAddr: r.lm_from_addr, + toAddr: r.lm_to_addr, + subject: null, + body: r.lm_body, + html: null, + kind: null, + attachments: [], + messageId: null, + inReplyTo: null, + unaffiliated: false, + effectsClaimedAt: null, + effectsAppliedAt: null, + effectsStage: 0, + createdAt: thread.lastMessageAt ?? thread.createdAt, + } +} + +async function writeMailAudit(tx: Queryable, audit: MailAuditInput | undefined): Promise { + if (!audit) return + await writeAudit(tx, { + actorId: audit.actorId, + action: audit.action, + target: audit.target, + meta: audit.meta ?? null, + }) +} + function bounceEventMatch(sql: Queryable, input: BounceEventKey): SqlFragment { return sql` thread_id = ${input.threadId}::uuid @@ -332,14 +366,7 @@ export function makeDrizzleMailRepository(sql: Sql): MailRepository { unread = ${setUnread} OR unread WHERE id = ${input.threadId} ` - if (input.audit) { - await writeAudit(tx, { - actorId: input.audit.actorId, - action: input.audit.action, - target: input.audit.target, - meta: input.audit.meta ?? null, - }) - } + await writeMailAudit(tx, input.audit) return toMessageRecord(row) }) }, @@ -367,15 +394,7 @@ export function makeDrizzleMailRepository(sql: Sql): MailRepository { input.q !== undefined && input.q.trim().length > 0 ? sql`AND ${ilikeAnyOf(sql, [sql`t.org`, sql`t.subject`, sql`lm.from_addr`], input.q.trim())}` : sql`` - const rows = await sql< - (ThreadRowSelect & { - lm_direction: MailDirection | null - lm_from_addr: string | null - lm_to_addr: string | null - lm_body: string | null - cursor_at: string - })[] - >` + const rows = await sql<(ThreadListRowSelect & { cursor_at: string })[]>` SELECT ${threadColumns(sql, "t")}, lm.direction AS lm_direction, lm.from_addr AS lm_from_addr, lm.to_addr AS lm_to_addr, lm.body AS lm_body, ${keysetInstant(sql, activityAt)} AS cursor_at @@ -404,29 +423,7 @@ export function makeDrizzleMailRepository(sql: Sql): MailRepository { })) const items = page.map((r) => { const thread = toThreadRecord(r) - const latest: MailMessageRecord | null = - r.lm_direction !== null - ? { - id: "", - threadId: r.id, - direction: r.lm_direction, - fromAddr: r.lm_from_addr, - toAddr: r.lm_to_addr, - subject: null, - body: r.lm_body, - html: null, - kind: null, - attachments: [], - messageId: null, - inReplyTo: null, - unaffiliated: false, - effectsClaimedAt: null, - effectsAppliedAt: null, - effectsStage: 0, - createdAt: thread.lastMessageAt ?? thread.createdAt, - } - : null - return toThreadListItem(thread, latest) + return toThreadListItem(thread, latestPreviewMessage(r, thread)) }) return { items, nextCursor } }, @@ -638,14 +635,7 @@ export function makeDrizzleMailRepository(sql: Sql): MailRepository { UPDATE mail_threads SET status = ${status} WHERE id = ${id} RETURNING id ` if (rows.length === 0) return false - if (audit) { - await writeAudit(tx, { - actorId: audit.actorId, - action: audit.action, - target: audit.target, - meta: audit.meta ?? null, - }) - } + await writeMailAudit(tx, audit) return true }) }, @@ -670,14 +660,7 @@ export function makeDrizzleMailRepository(sql: Sql): MailRepository { VALUES (${input.threadId}, ${input.messageId}, 'failed', ${meta}) ` await tx`UPDATE mail_threads SET status = 'needs_action' WHERE id = ${input.threadId}` - if (input.audit) { - await writeAudit(tx, { - actorId: input.audit.actorId, - action: input.audit.action, - target: input.audit.target, - meta: input.audit.meta ?? null, - }) - } + await writeMailAudit(tx, input.audit) }) }, @@ -733,8 +716,7 @@ export function makeDrizzleMailRepository(sql: Sql): MailRepository { ` const counts = { sent: 0, bounced: 0, failed: 0 } for (const row of eventRows) { - const n = Number.parseInt(row.n, 10) - if (row.type in counts) counts[row.type as keyof typeof counts] = Number.isNaN(n) ? 0 : n + if (row.type in counts) counts[row.type as keyof typeof counts] = parseCount(row.n) } const mailbox = await sql<{ unread: string; threads: string }[]>` SELECT @@ -742,11 +724,9 @@ export function makeDrizzleMailRepository(sql: Sql): MailRepository { COUNT(*)::text AS threads FROM mail_threads ` - const unread = Number.parseInt(mailbox[0]?.unread ?? "0", 10) - const threads = Number.parseInt(mailbox[0]?.threads ?? "0", 10) return buildMailStats({ - unread: Number.isNaN(unread) ? 0 : unread, - threads: Number.isNaN(threads) ? 0 : threads, + unread: parseCount(mailbox[0]?.unread), + threads: parseCount(mailbox[0]?.threads), counts, }) }, diff --git a/services/api/src/services/admin/mail-repository.memory.ts b/services/api/src/services/admin/mail-repository.memory.ts index ef6e23d1..a7e045dc 100644 --- a/services/api/src/services/admin/mail-repository.memory.ts +++ b/services/api/src/services/admin/mail-repository.memory.ts @@ -23,7 +23,7 @@ import { type RecordSendFailureInput, type ThreadInit, } from "./mail-repository.js" -import { deriveWho, mintThreadToken, toMessageDTO, toThreadListItem } from "./mail-mappers.js" +import { mintThreadToken, toMessageDTO, toThreadListItem } from "./mail-mappers.js" import { buildMailStats } from "./mail-stats.js" import { ROUTE_CLAIM_STALE_SECONDS, @@ -32,6 +32,8 @@ import { import { clampLimit, decodeCursor, encodeCursor } from "./pagination.js" import type { MailDelivery, MailStatsResponse, MailStatus, MailThreadDTO } from "@civfix/shared" +const DAY_MS = 24 * 60 * 60 * 1000 + export interface StoredMailEvent { id: string threadId: string | null @@ -158,31 +160,37 @@ export class InMemoryMailRepository implements MailRepository { return Promise.resolve({ ...record }) } - findOrCreateReportThread(reportId: string, init: ThreadInit = {}): Promise { + private newestThread(matches: (t: MailThreadRecord) => boolean): MailThreadRecord | null { let best: MailThreadRecord | null = null for (const t of this.threads.values()) { - if (t.reportId !== reportId) continue - if (best === null || cmpThreadNewest(t, best) > 0) best = t + if (!matches(t)) continue + if (best === null || cmpCreated(t, best) > 0) best = t } + return best + } + + private latestMessage(matches: (m: MailMessageRecord) => boolean): MailMessageRecord | null { + let best: MailMessageRecord | null = null + for (const m of this.messages) { + if (!matches(m)) continue + if (best === null || cmpCreated(m, best) > 0) best = m + } + return best + } + + findOrCreateReportThread(reportId: string, init: ThreadInit = {}): Promise { + const best = this.newestThread((t) => t.reportId === reportId) if (best) return Promise.resolve({ ...best }) return this.createThread({ ...init, reportId, threadToken: mintThreadToken() }) } findReportThread(reportId: string): Promise { - let best: MailThreadRecord | null = null - for (const t of this.threads.values()) { - if (t.reportId !== reportId) continue - if (best === null || cmpThreadNewest(t, best) > 0) best = t - } + const best = this.newestThread((t) => t.reportId === reportId) return Promise.resolve(best === null ? null : { ...best }) } findOrCreateEventThread(cleanupId: string, init: ThreadInit = {}): Promise { - let best: MailThreadRecord | null = null - for (const t of this.threads.values()) { - if (t.cleanupId !== cleanupId) continue - if (best === null || cmpThreadNewest(t, best) > 0) best = t - } + const best = this.newestThread((t) => t.cleanupId === cleanupId) if (best) return Promise.resolve({ ...best }) return this.createThread({ ...init, cleanupId, threadToken: mintThreadToken() }) } @@ -196,11 +204,9 @@ export class InMemoryMailRepository implements MailRepository { } upsertThreadByGeoid(geoid: string, init: ThreadInit = {}): Promise { - let best: MailThreadRecord | null = null - for (const t of this.threads.values()) { - if (t.jurisdictionGeoid !== geoid || t.reportId !== null || t.cleanupId !== null) continue - if (best === null || cmpThreadNewest(t, best) > 0) best = t - } + const best = this.newestThread( + (t) => t.jurisdictionGeoid === geoid && t.reportId === null && t.cleanupId === null, + ) if (best) return Promise.resolve({ ...best }) return this.createThread({ ...init, @@ -226,7 +232,7 @@ export class InMemoryMailRepository implements MailRepository { if (m.direction !== "out" || m.messageId === null || !ids.has(m.messageId)) continue const t = this.threads.get(m.threadId) if (!t) continue - if (best === null || cmpThreadNewest(t, best) > 0) best = t + if (best === null || cmpCreated(t, best) > 0) best = t } return Promise.resolve(best ? { ...best } : null) } @@ -382,13 +388,10 @@ export class InMemoryMailRepository implements MailRepository { } getLastOutboundRecipient(threadId: string): Promise { - let best: MailMessageRecord | null = null - for (const m of this.messages) { - if (m.threadId !== threadId) continue - if (m.direction !== "out") continue - if (m.toAddr === null || m.toAddr === "") continue - if (best === null || cmpCreated(m, best) > 0) best = m - } + const best = this.latestMessage( + (m) => + m.threadId === threadId && m.direction === "out" && m.toAddr !== null && m.toAddr !== "", + ) return Promise.resolve(best?.toAddr ?? null) } @@ -400,11 +403,7 @@ export class InMemoryMailRepository implements MailRepository { hasSendInFlight(threadId: string): Promise { const inflightBefore = this.now.getTime() - ROUTE_DEADLINE_INFLIGHT_SECONDS * 1000 const staleBefore = this.now.getTime() - ROUTE_CLAIM_STALE_SECONDS * 1000 - let latest: MailMessageRecord | null = null - for (const m of this.messages) { - if (m.threadId !== threadId || m.direction !== "out") continue - if (latest === null || cmpCreated(m, latest) > 0) latest = m - } + const latest = this.latestMessage((m) => m.threadId === threadId && m.direction === "out") if (latest === null) return Promise.resolve(false) const own = this.events.filter((e) => e.messageId === latest.id) if (own.some((e) => e.type === "sent")) return Promise.resolve(false) @@ -542,11 +541,7 @@ export class InMemoryMailRepository implements MailRepository { } latestOutboundMessageId(threadId: string): Promise { - let best: MailMessageRecord | null = null - for (const m of this.messages) { - if (m.threadId !== threadId || m.direction !== "out") continue - if (best === null || cmpCreated(m, best) > 0) best = m - } + const best = this.latestMessage((m) => m.threadId === threadId && m.direction === "out") return Promise.resolve(best?.id ?? null) } @@ -564,7 +559,7 @@ export class InMemoryMailRepository implements MailRepository { } stats7d(): Promise { - const cutoff = this.now.getTime() - MAIL_STATS_WINDOW_DAYS * 24 * 60 * 60 * 1000 + const cutoff = this.now.getTime() - MAIL_STATS_WINDOW_DAYS * DAY_MS const counts = { sent: 0, bounced: 0, failed: 0 } for (const e of this.events) { if (e.createdAt.getTime() < cutoff) continue @@ -602,16 +597,11 @@ export class InMemoryMailRepository implements MailRepository { } } -function cmpCreated(a: MailMessageRecord, b: MailMessageRecord): number { - const d = a.createdAt.getTime() - b.createdAt.getTime() - if (d !== 0) return d - return a.id < b.id ? -1 : a.id > b.id ? 1 : 0 -} - -function cmpThreadNewest(a: MailThreadRecord, b: MailThreadRecord): number { +function cmpCreated( + a: { createdAt: Date; id: string }, + b: { createdAt: Date; id: string }, +): number { const d = a.createdAt.getTime() - b.createdAt.getTime() if (d !== 0) return d return a.id < b.id ? -1 : a.id > b.id ? 1 : 0 } - -export { deriveWho } diff --git a/services/api/src/services/admin/mail-service.ts b/services/api/src/services/admin/mail-service.ts index ef1ab53f..87074438 100644 --- a/services/api/src/services/admin/mail-service.ts +++ b/services/api/src/services/admin/mail-service.ts @@ -20,6 +20,8 @@ import { import { mapWithLimit, PRESIGN_CONCURRENCY } from "../media-presign.js" import type { OutboundMailService } from "./outbound-mail-service.js" +const MAIL_THREAD_NOT_FOUND = "Mail thread not found" + export interface MailServiceDeps { repo: MailRepository outboundMail: OutboundMailService @@ -73,7 +75,7 @@ export function makeMailService(deps: MailServiceDeps): MailService { async function requireThreadDTO(id: string): Promise { const dto = await repo.getThread(id) - if (!dto) throw AppError.notFound("Mail thread not found") + if (!dto) throw AppError.notFound(MAIL_THREAD_NOT_FOUND) return dto } @@ -89,7 +91,7 @@ export function makeMailService(deps: MailServiceDeps): MailService { noRecipientMsg: string, ): Promise { const record = await repo.getThreadRecord(id) - if (!record) throw AppError.notFound("Mail thread not found") + if (!record) throw AppError.notFound(MAIL_THREAD_NOT_FOUND) const toAddr = await repo.getLastOutboundRecipient(id) if (toAddr === null) throw AppError.validation({ [noRecipientField]: noRecipientMsg }) return toAddr @@ -108,11 +110,7 @@ export function makeMailService(deps: MailServiceDeps): MailService { return repo.listThreads(input) }, - async getThread(id: string): Promise { - const dto = await repo.getThread(id) - if (!dto) throw AppError.notFound("Mail thread not found") - return dto - }, + getThread: requireThreadDTO, async compose(input: ComposeRequest, actorId: string): Promise { const thread = await outboundMail.compose({ @@ -143,7 +141,7 @@ export function makeMailService(deps: MailServiceDeps): MailService { async markRead(id: string): Promise { const ok = await repo.markThreadRead(id) - if (!ok) throw AppError.notFound("Mail thread not found") + if (!ok) throw AppError.notFound(MAIL_THREAD_NOT_FOUND) }, async setStatus(id: string, status: MailStatus, actorId: string): Promise { @@ -153,12 +151,12 @@ export function makeMailService(deps: MailServiceDeps): MailService { target: `mail:${id}`, meta: { status }, }) - if (!ok) throw AppError.notFound("Mail thread not found") + if (!ok) throw AppError.notFound(MAIL_THREAD_NOT_FOUND) }, async resend(id: string, actorId: string): Promise { const thread = await repo.getThreadRecord(id) - if (!thread) throw AppError.notFound("Mail thread not found") + if (!thread) throw AppError.notFound(MAIL_THREAD_NOT_FOUND) await assertNoSendInFlight(id) const lastId = await repo.latestOutboundMessageId(id) const last = lastId === null ? null : await repo.getOutboundMessageForResend(lastId) diff --git a/services/api/src/services/admin/moderation-repository.drizzle.ts b/services/api/src/services/admin/moderation-repository.drizzle.ts index dc17e545..f3afa97d 100644 --- a/services/api/src/services/admin/moderation-repository.drizzle.ts +++ b/services/api/src/services/admin/moderation-repository.drizzle.ts @@ -4,9 +4,9 @@ import { writeAudit } from "./audit.js" import { clampLimit, decodeCursor, - encodeKeysetCursor, keysetInstant, keysetPredicate, + paginateKeyset, } from "./pagination.js" import { ADMIN_CATEGORIES } from "./category-counts.js" import { ilikeAnyOf, type SqlFragment } from "./sql-fragments.js" @@ -17,6 +17,10 @@ import { } from "../../auth/operator-target.js" import { moderationMediaKeyExpr, moderationMediaFilter } from "../media-served-key.js" import { + isMessageSubject, + isUserSubject, + MODERATION_APPROVED_NOTE, + MODERATION_REMOVED_NOTE, type CreateModerationItemInput, type ListModerationArgs, type ModerationItemRecord, @@ -27,9 +31,8 @@ import { type SubjectType = ModerationItemRecord["subjectType"] -function isMessageSubject(subjectType: SubjectType): boolean { - return subjectType === "chat" || subjectType === "message" -} +const UNKNOWN_USER_NAME = "Unknown" +const DEFAULT_ITEM_PRIORITY = "med" function itemColumns(sql: Queryable): SqlFragment { return sql` @@ -210,7 +213,7 @@ function parseMeta(raw: unknown): { user = { id: typeof u.id === "string" ? u.id : null, handle: typeof u.handle === "string" ? u.handle : "", - name: typeof u.name === "string" ? u.name : "Unknown", + name: typeof u.name === "string" ? u.name : UNKNOWN_USER_NAME, joined: typeof u.joined === "string" ? u.joined : "", priorReports: typeof u.priorReports === "number" ? u.priorReports : 0, priorRemovals: typeof u.priorRemovals === "number" ? u.priorRemovals : 0, @@ -274,6 +277,21 @@ async function loadMedia( })) } +function facetFilter(sql: Queryable, filter: ListModerationArgs["filter"]): SqlFragment { + if (filter === "high") return sql`AND priority = 'high'` + if (filter !== "all") return sql`AND kind = ${filter}` + return sql`` +} + +function searchFilter(sql: Queryable, q: string | null): SqlFragment { + if (q === null) return sql`` + return sql`AND ${ilikeAnyOf( + sql, + [sql`COALESCE(flag, '')`, sql`COALESCE(meta->>'reporter', '')`, sql`COALESCE(reason, '')`], + q, + )}` +} + export function makeDrizzleModerationRepository(sql: Sql): ModerationRepository { return { async listOpen( @@ -281,25 +299,6 @@ export function makeDrizzleModerationRepository(sql: Sql): ModerationRepository ): Promise<{ records: ModerationItemRecord[]; nextCursor: string | null }> { const limit = clampLimit(args.limit) const anchor = decodeCursor(args.cursor, true) - - const facet = - args.filter === "high" - ? sql`AND priority = 'high'` - : args.filter !== "all" - ? sql`AND kind = ${args.filter}` - : sql`` - const search = - args.q !== null - ? sql`AND ${ilikeAnyOf( - sql, - [ - sql`COALESCE(flag, '')`, - sql`COALESCE(meta->>'reporter', '')`, - sql`COALESCE(reason, '')`, - ], - args.q, - )}` - : sql`` const keyset = anchor ? sql`AND ${keysetPredicate(sql, sql`created_at`, sql`id`, anchor)}` : sql`` @@ -308,20 +307,19 @@ export function makeDrizzleModerationRepository(sql: Sql): ModerationRepository SELECT ${itemColumns(sql)}, ${keysetInstant(sql, sql`created_at`)} AS cursor_at FROM moderation_items WHERE status = 'open' - ${facet} - ${search} + ${facetFilter(sql, args.filter)} + ${searchFilter(sql, args.q)} ${keyset} ORDER BY created_at DESC, id DESC LIMIT ${limit + 1} `) as unknown as (ModerationItemRow & { cursor_at: string })[] - const hasMore = rows.length > limit - const page = hasMore ? rows.slice(0, limit) : rows + const { items: page, nextCursor } = paginateKeyset(rows, limit, (r) => ({ + atText: r.cursor_at, + id: r.id, + })) await attachDestinationRefs(sql, page) - const records = page.map((r) => toRecord(r, [])) - const last = page[page.length - 1] - const nextCursor = hasMore && last ? encodeKeysetCursor(last.cursor_at, last.id) : null - return { records, nextCursor } + return { records: page.map((r) => toRecord(r, [])), nextCursor } }, async getItem(id: string): Promise { @@ -356,7 +354,7 @@ export function makeDrizzleModerationRepository(sql: Sql): ModerationRepository publishedReport = true await tx` INSERT INTO report_timeline (report_id, status, note, actor_id) - VALUES (${resolved.subject_id}, 'published', ${"Approved in moderation"}, ${input.actorId}) + VALUES (${resolved.subject_id}, 'published', ${MODERATION_APPROVED_NOTE}, ${input.actorId}) ` } } @@ -394,7 +392,7 @@ export function makeDrizzleModerationRepository(sql: Sql): ModerationRepository if (removedReport) { await tx` INSERT INTO report_timeline (report_id, status, note, actor_id) - VALUES (${resolved.subject_id}, 'rejected', ${input.reason ?? "Removed in moderation"}, ${input.actorId}) + VALUES (${resolved.subject_id}, 'rejected', ${input.reason ?? MODERATION_REMOVED_NOTE}, ${input.actorId}) ` } if (removed && !isOwnerTakedown(resolved.meta)) { @@ -702,7 +700,7 @@ async function buildUserSnapshot( return { id: row.id, handle: row.handle ?? "", - name: row.name ?? "Unknown", + name: row.name ?? UNKNOWN_USER_NAME, joined: row.joined ?? "", priorReports: Number.parseInt(row.prior_reports ?? "0", 10) || 0, priorRemovals: row.removals ?? 0, @@ -711,10 +709,6 @@ async function buildUserSnapshot( } } -function isUserSubject(subjectType: SubjectType): boolean { - return subjectType === "user" || subjectType === "profile" -} - function abuseSubjectTypeFor(subjectType: SubjectType): string { if (subjectType === "profile") return "user" if (subjectType === "photo") return "media" @@ -938,7 +932,7 @@ export async function insertModerationItem( ${input.reason ?? null}, ${input.category ?? null}, ${input.place ?? null}, - ${input.priority ?? "med"}, + ${input.priority ?? DEFAULT_ITEM_PRIORITY}, ${input.autoAction ?? null}, ${tx.json((input.signals ?? []) as Parameters[0])}, ${tx.json((input.similar ?? []) as Parameters[0])}, diff --git a/services/api/src/services/admin/moderation-repository.memory.ts b/services/api/src/services/admin/moderation-repository.memory.ts index bff01c79..a4b9f87a 100644 --- a/services/api/src/services/admin/moderation-repository.memory.ts +++ b/services/api/src/services/admin/moderation-repository.memory.ts @@ -5,6 +5,8 @@ import { assertTargetIsNotOperatorRole, } from "../../auth/operator-target.js" import { + isMessageSubject, + isUserSubject, type CreateModerationItemInput, type ListModerationArgs, type ModerationItemRecord, @@ -20,8 +22,16 @@ export interface SeededHeldReport { createdAt: Date } -function isUserSubjectType(subjectType: ModerationItemRecord["subjectType"]): boolean { - return subjectType === "user" || subjectType === "profile" +const DEFAULT_ITEM_PRIORITY = "med" + +function directDestination( + subjectType: ModerationItemRecord["subjectType"], + subjectId: string, +): Pick { + if (subjectType === "report") return { destinationKind: "report", destinationId: subjectId } + if (subjectType === "event") return { destinationKind: "event", destinationId: subjectId } + if (isUserSubject(subjectType)) return { destinationKind: "user", destinationId: subjectId } + return { destinationKind: null, destinationId: null } } export class InMemoryModerationRepository implements ModerationRepository { @@ -42,23 +52,11 @@ export class InMemoryModerationRepository implements ModerationRepository { return new Date(this.now.getTime() + this.tick) } - private directDestination( - subjectType: ModerationItemRecord["subjectType"], - subjectId: string, - ): Pick { - if (subjectType === "report") return { destinationKind: "report", destinationId: subjectId } - if (subjectType === "event") return { destinationKind: "event", destinationId: subjectId } - if (subjectType === "user" || subjectType === "profile") { - return { destinationKind: "user", destinationId: subjectId } - } - return { destinationKind: null, destinationId: null } - } - seedItem(input: Partial & { id?: string }): ModerationItemRecord { const id = input.id ?? randomUUID() const subjectType = input.subjectType ?? "report" const subjectId = input.subjectId ?? randomUUID() - const destination = this.directDestination(subjectType, subjectId) + const destination = directDestination(subjectType, subjectId) const record: ModerationItemRecord = { id, kind: input.kind ?? "image", @@ -72,7 +70,7 @@ export class InMemoryModerationRepository implements ModerationRepository { reason: input.reason ?? null, category: input.category ?? null, place: input.place ?? null, - priority: input.priority ?? "med", + priority: input.priority ?? DEFAULT_ITEM_PRIORITY, autoAction: input.autoAction ?? null, reporter: input.reporter ?? null, reporterId: input.reporterId ?? null, @@ -180,7 +178,7 @@ export class InMemoryModerationRepository implements ModerationRepository { _input: { actorId: string | null; reason: string | null }, ): Promise { const pending = this.items.get(id) - if (pending && isUserSubjectType(pending.subjectType)) { + if (pending && isUserSubject(pending.subjectType)) { assertTargetIsNotOfficialAccount(pending.subjectId, "remove") assertTargetIsNotOperatorRole(this.userRoles.get(pending.subjectId), "remove") } @@ -191,13 +189,8 @@ export class InMemoryModerationRepository implements ModerationRepository { this.reportStatus.set(item.subjectId, "rejected") item.reportTimelineStatus = "rejected" } - if (item.subjectType === "chat" || item.subjectType === "message") { - this.tombstoned.add(item.subjectId) - } - if ( - isUserSubjectType(item.subjectType) && - this.accountStatus.get(item.subjectId) !== "banned" - ) { + if (isMessageSubject(item.subjectType)) this.tombstoned.add(item.subjectId) + if (isUserSubject(item.subjectType) && this.accountStatus.get(item.subjectId) !== "banned") { this.accountStatus.set(item.subjectId, "suspended") item.suspendedUserId = item.subjectId } @@ -220,11 +213,9 @@ export class InMemoryModerationRepository implements ModerationRepository { item.status = "approved" this.suspensions.set(item.subjectId, input.decision === "uphold") delete item.restoredUserId - if (input.decision === "overturn" && this.restoreSubject(item.subjectType, item.subjectId)) { - if (item.subjectType === "user" || item.subjectType === "profile") { - item.restoredUserId = item.subjectId - } - } + const restored = + input.decision === "overturn" && this.restoreSubject(item.subjectType, item.subjectId) + if (restored && isUserSubject(item.subjectType)) item.restoredUserId = item.subjectId return item } @@ -232,10 +223,8 @@ export class InMemoryModerationRepository implements ModerationRepository { subjectType: ModerationItemRecord["subjectType"], subjectId: string, ): boolean { - if (subjectType === "chat" || subjectType === "message") { - return this.tombstoned.delete(subjectId) - } - if (subjectType === "user" || subjectType === "profile") { + if (isMessageSubject(subjectType)) return this.tombstoned.delete(subjectId) + if (isUserSubject(subjectType)) { if (this.deletedUserIds.has(subjectId)) return false if (this.accountStatus.get(subjectId) !== "suspended") return false this.accountStatus.set(subjectId, "active") @@ -265,7 +254,7 @@ export class InMemoryModerationRepository implements ModerationRepository { reason: input.reason ?? null, category: input.category ?? null, place: input.place ?? null, - priority: input.priority ?? "med", + priority: input.priority ?? DEFAULT_ITEM_PRIORITY, autoAction: input.autoAction ?? null, reporter: input.reporter ?? null, reporterId: input.reporterUserId ?? null, diff --git a/services/api/src/services/admin/moderation-service.ts b/services/api/src/services/admin/moderation-service.ts index 59cfb53a..d89e4a19 100644 --- a/services/api/src/services/admin/moderation-service.ts +++ b/services/api/src/services/admin/moderation-service.ts @@ -22,6 +22,12 @@ import type { MessageUpdateAnnouncer } from "./admin-report-chat-service.js" export type ModerationFilter = "all" | ModerationKind | "high" +/** Shared by the service's chat mirror and the repository's report_timeline row, which must agree. */ +export const MODERATION_APPROVED_NOTE = "Approved in moderation" +export const MODERATION_REMOVED_NOTE = "Removed in moderation" + +const MODERATION_ITEM_NOT_FOUND = "Moderation item not found" + export interface ListModerationArgs { q: string | null filter: ModerationFilter @@ -118,7 +124,7 @@ export interface ModerationRepository { backfillFromHeldReports(): Promise } -export const NEUTRAL_USER_SNAPSHOT: ModerationUserSnapshot = { +const NEUTRAL_USER_SNAPSHOT: ModerationUserSnapshot = { id: null, handle: "", name: "Unknown", @@ -156,10 +162,14 @@ export interface ModerationServiceDeps { announceMessageUpdate?: MessageUpdateAnnouncer } -function isMessageSubject(subjectType: ModerationItemRecord["subjectType"]): boolean { +export function isMessageSubject(subjectType: ModerationSubjectType): boolean { return subjectType === "chat" || subjectType === "message" } +export function isUserSubject(subjectType: ModerationSubjectType): boolean { + return subjectType === "user" || subjectType === "profile" +} + export interface ModerationService { list(query: ModerationListQuery): Promise getItem(id: string): Promise @@ -235,7 +245,7 @@ export function makeModerationService(deps: ModerationServiceDeps): ModerationSe const ref = now() const args: ListModerationArgs = { q: query.q && query.q.trim() !== "" ? query.q.trim() : null, - filter: (query.filter ?? "all") as ModerationFilter, + filter: query.filter ?? "all", cursor: query.cursor ?? null, limit: clampLimit(query.limit), } @@ -246,7 +256,7 @@ export function makeModerationService(deps: ModerationServiceDeps): ModerationSe async getItem(id: string): Promise { const ref = now() const record = await deps.repo.getItem(id) - if (!record) throw AppError.notFound("Moderation item not found") + if (!record) throw AppError.notFound(MODERATION_ITEM_NOT_FOUND) return toDetailDTO(record, ref) }, @@ -255,13 +265,13 @@ export function makeModerationService(deps: ModerationServiceDeps): ModerationSe input: { actorId: string | null; note: string | null }, ): Promise { const result = await deps.repo.approve(id, input) - if (!result) throw AppError.notFound("Moderation item not found") + if (!result) throw AppError.notFound(MODERATION_ITEM_NOT_FOUND) if (deps.reportChatEmitter && result.reportTimelineStatus === "published") { await deps.reportChatEmitter.emit({ reportId: result.subjectId, status: "published", kind: timelineKindForStatus("published"), - note: "Approved in moderation", + note: MODERATION_APPROVED_NOTE, }) } }, @@ -271,7 +281,7 @@ export function makeModerationService(deps: ModerationServiceDeps): ModerationSe input: { actorId: string | null; reason: string | null }, ): Promise { const result = await deps.repo.remove(id, input) - if (!result) throw AppError.notFound("Moderation item not found") + if (!result) throw AppError.notFound(MODERATION_ITEM_NOT_FOUND) if (isMessageSubject(result.subjectType)) { await deps.announceMessageUpdate?.(result.subjectId) } @@ -283,14 +293,14 @@ export function makeModerationService(deps: ModerationServiceDeps): ModerationSe reportId: result.subjectId, status: "rejected", kind: timelineKindForStatus("rejected"), - note: input.reason ?? "Removed in moderation", + note: input.reason ?? MODERATION_REMOVED_NOTE, }) } }, async hold(id: string, input: { actorId: string | null; note: string | null }): Promise { const result = await deps.repo.hold(id, input) - if (!result) throw AppError.notFound("Moderation item not found") + if (!result) throw AppError.notFound(MODERATION_ITEM_NOT_FOUND) }, async appeal( @@ -298,7 +308,7 @@ export function makeModerationService(deps: ModerationServiceDeps): ModerationSe input: { decision: "uphold" | "overturn"; actorId: string | null; note: string | null }, ): Promise { const result = await deps.repo.decideAppeal(id, input) - if (!result) throw AppError.notFound("Moderation item not found") + if (!result) throw AppError.notFound(MODERATION_ITEM_NOT_FOUND) if (input.decision === "overturn" && isMessageSubject(result.subjectType)) { await deps.announceMessageUpdate?.(result.subjectId) } diff --git a/services/api/src/services/admin/outbound-mail-service.ts b/services/api/src/services/admin/outbound-mail-service.ts index e4a83dd3..813fed5a 100644 --- a/services/api/src/services/admin/outbound-mail-service.ts +++ b/services/api/src/services/admin/outbound-mail-service.ts @@ -124,10 +124,14 @@ export interface OutboundMailServiceDeps { sendMinThroughputBytesPerSec?: number } -export const OUTBOUND_DEADLINE_REASON = "deadline" +const OUTBOUND_DEADLINE_REASON = "deadline" const THREAD_STATUS_CLEARED_BY_DELIVERY: readonly MailStatus[] = ["needs_action", "bounced"] +const REFERENCES_HEADER_MAX = 10 + +const MAIL_THREAD_NOT_FOUND = "Mail thread not found" + export class OutboundSendDeadlineError extends AppError { readonly outboundSendDeadline = true readonly deadlineMs: number @@ -147,7 +151,7 @@ export function isOutboundSendDeadlineError(err: unknown): boolean { return (err as { outboundSendDeadline?: unknown }).outboundSendDeadline === true } -export function attachmentMetadata( +function attachmentMetadata( attachments: readonly PacketAttachment[] | undefined, ): MailAttachment[] { const stored: MailAttachment[] = [] @@ -180,6 +184,155 @@ function raceDeadline(work: Promise, ms: number): Promise { }) as Promise } +interface DeliveryArgs { + threadId: string + messageId: string + fromHeader: string + toAddr: string + subject: string + body: string + html?: string + attachments?: PacketAttachment[] + eventMeta?: Record + onLateSuccess?: (() => Promise) | undefined +} + +interface DeliveryContext { + repo: MailRepository + logger: OutboundMailLogger +} + +// A long thread's References keeps its root plus the most recent replies, so the header stays bounded +// while a client can still place the message in the conversation. +function threadingHeaders(priorIds: string[]): { + inReplyTo: string | undefined + references: string[] +} { + const inReplyTo = priorIds.length > 0 ? priorIds[priorIds.length - 1] : undefined + const references = + priorIds.length > REFERENCES_HEADER_MAX + ? [priorIds[0] as string, ...priorIds.slice(-(REFERENCES_HEADER_MAX - 1))] + : priorIds + return { inReplyTo, references } +} + +async function recordFailed( + { repo, logger }: DeliveryContext, + args: DeliveryArgs, + err: unknown, + extra: Record, +): Promise { + const error = err instanceof Error ? err.message : String(err) + const reportId = args.eventMeta?.reportId + try { + await repo.recordSendFailure({ + threadId: args.threadId, + messageId: args.messageId, + meta: { + from: args.fromHeader, + to: args.toAddr, + error, + ...extra, + ...(args.eventMeta ?? {}), + }, + ...(typeof reportId === "string" && reportId.length > 0 + ? { + audit: { + actorId: null, + action: "mail.send_failed" as const, + target: `report:${reportId}`, + meta: { + threadId: args.threadId, + messageId: args.messageId, + to: args.toAddr, + reportId, + error, + }, + }, + } + : {}), + }) + } catch (recordErr) { + logger.warn( + { err: recordErr, threadId: args.threadId, messageId: args.messageId }, + "outbound mail send failed AND recording the 'failed' event failed", + ) + } +} + +async function recordSent( + ctx: DeliveryContext, + args: DeliveryArgs, + result: SentMail, + extra: Record, +): Promise { + await ctx.repo.setMessageMessageId(args.messageId, result.messageId) + await ctx.repo.recordEvent({ + threadId: args.threadId, + messageId: args.messageId, + type: "sent", + meta: { + from: args.fromHeader, + to: args.toAddr, + ...extra, + ...(args.eventMeta ?? {}), + }, + }) + await clearDeliveryFailureStatus(ctx, args) +} + +async function clearDeliveryFailureStatus( + { repo, logger }: DeliveryContext, + args: DeliveryArgs, +): Promise { + try { + const fresh = await repo.getThreadRecord(args.threadId) + if (fresh === null || !THREAD_STATUS_CLEARED_BY_DELIVERY.includes(fresh.status)) return + await repo.setThreadStatus(args.threadId, "sent") + } catch (err) { + logger.warn( + { err, threadId: args.threadId, messageId: args.messageId }, + "outbound mail delivered but clearing the thread's failure status failed", + ) + } +} + +function attachLateSettlement( + ctx: DeliveryContext, + args: DeliveryArgs, + send: Promise, + failureWrite: Promise, + deadlineMs: number, +): void { + const { logger } = ctx + void send.then( + async (late: SentMail) => { + try { + // The caller already awaits failureWrite and sees its error; here it only orders the + // late 'sent' after the 'failed' row. + await failureWrite.catch(() => {}) + await recordSent(ctx, args, late, { late: true }) + if (args.onLateSuccess !== undefined) await args.onLateSuccess() + logger.warn( + { threadId: args.threadId, messageId: args.messageId, deadlineMs }, + "outbound mail delivered AFTER its deadline; recorded 'sent' (late)", + ) + } catch (recordErr) { + logger.warn( + { err: recordErr, threadId: args.threadId, messageId: args.messageId }, + "outbound mail delivered late but recording the 'sent' event failed", + ) + } + }, + (lateErr: unknown) => { + logger.warn( + { err: lateErr, threadId: args.threadId, messageId: args.messageId, deadlineMs }, + "outbound mail rejected AFTER its deadline; the 'failed' event already recorded stands", + ) + }, + ) +} + export function makeOutboundMailService(deps: OutboundMailServiceDeps): OutboundMailService { const { repo, mailer, env } = deps const logger: OutboundMailLogger = deps.logger ?? console @@ -194,18 +347,8 @@ export function makeOutboundMailService(deps: OutboundMailServiceDeps): Outbound return `"civfix" ` } - async function deliverAndRecord(args: { - threadId: string - messageId: string - fromHeader: string - toAddr: string - subject: string - body: string - html?: string - attachments?: PacketAttachment[] - eventMeta?: Record - onLateSuccess?: (() => Promise) | undefined - }): Promise { + async function deliverAndRecord(args: DeliveryArgs): Promise { + const ctx: DeliveryContext = { repo, logger } const rfcMessageId = `` // Threading headers are a courtesy to the city's mail client; losing them must not block the send. const priorIds = await repo @@ -217,9 +360,7 @@ export function makeOutboundMailService(deps: OutboundMailServiceDeps): Outbound ) return [] }) - const inReplyTo = priorIds.length > 0 ? priorIds[priorIds.length - 1] : undefined - const references = - priorIds.length > 10 ? [priorIds[0] as string, ...priorIds.slice(-9)] : priorIds + const { inReplyTo, references } = threadingHeaders(priorIds) const bytes = outboundPayloadBytes({ body: args.body, html: args.html, @@ -245,118 +386,25 @@ export function makeOutboundMailService(deps: OutboundMailServiceDeps): Outbound ...(args.attachments !== undefined ? { attachments: args.attachments } : {}), }) - async function recordFailed(err: unknown, extra: Record): Promise { - const error = err instanceof Error ? err.message : String(err) - const reportId = args.eventMeta?.reportId - try { - await repo.recordSendFailure({ - threadId: args.threadId, - messageId: args.messageId, - meta: { - from: args.fromHeader, - to: args.toAddr, - error, - ...extra, - ...(args.eventMeta ?? {}), - }, - ...(typeof reportId === "string" && reportId.length > 0 - ? { - audit: { - actorId: null, - action: "mail.send_failed" as const, - target: `report:${reportId}`, - meta: { - threadId: args.threadId, - messageId: args.messageId, - to: args.toAddr, - reportId, - error, - }, - }, - } - : {}), - }) - } catch (recordErr) { - logger.warn( - { err: recordErr, threadId: args.threadId, messageId: args.messageId }, - "outbound mail send failed AND recording the 'failed' event failed", - ) - } - } - - async function recordSent(result: SentMail, extra: Record): Promise { - await repo.setMessageMessageId(args.messageId, result.messageId) - await repo.recordEvent({ - threadId: args.threadId, - messageId: args.messageId, - type: "sent", - meta: { - from: args.fromHeader, - to: args.toAddr, - ...extra, - ...(args.eventMeta ?? {}), - }, - }) - await clearDeliveryFailureStatus() - } - - async function clearDeliveryFailureStatus(): Promise { - try { - const fresh = await repo.getThreadRecord(args.threadId) - if (fresh === null || !THREAD_STATUS_CLEARED_BY_DELIVERY.includes(fresh.status)) return - await repo.setThreadStatus(args.threadId, "sent") - } catch (err) { - logger.warn( - { err, threadId: args.threadId, messageId: args.messageId }, - "outbound mail delivered but clearing the thread's failure status failed", - ) - } - } - let sent: SentMail try { sent = await raceDeadline(send, deadlineMs) } catch (err) { if (!isOutboundSendDeadlineError(err)) { - await recordFailed(err, {}) + await recordFailed(ctx, args, err, {}) throw err } - const failureWrite = recordFailed(err, { + const failureWrite = recordFailed(ctx, args, err, { reason: OUTBOUND_DEADLINE_REASON, deadlineMs, bytes, }) - void send.then( - async (late: SentMail) => { - try { - // The caller already awaits failureWrite and sees its error; here it only orders the - // late 'sent' after the 'failed' row. - await failureWrite.catch(() => {}) - await recordSent(late, { late: true }) - if (args.onLateSuccess !== undefined) await args.onLateSuccess() - logger.warn( - { threadId: args.threadId, messageId: args.messageId, deadlineMs }, - "outbound mail delivered AFTER its deadline; recorded 'sent' (late)", - ) - } catch (recordErr) { - logger.warn( - { err: recordErr, threadId: args.threadId, messageId: args.messageId }, - "outbound mail delivered late but recording the 'sent' event failed", - ) - } - }, - (lateErr: unknown) => { - logger.warn( - { err: lateErr, threadId: args.threadId, messageId: args.messageId, deadlineMs }, - "outbound mail rejected AFTER its deadline; the 'failed' event already recorded stands", - ) - }, - ) + attachLateSettlement(ctx, args, send, failureWrite, deadlineMs) await failureWrite throw err } try { - await recordSent(sent, {}) + await recordSent(ctx, args, sent, {}) } catch (err) { logger.warn( { err, threadId: args.threadId, messageId: args.messageId }, @@ -573,7 +621,7 @@ export function makeOutboundMailService(deps: OutboundMailServiceDeps): Outbound async appendOutbound(threadId: string, input: AppendOutboundInput): Promise { const thread = await repo.getThreadRecord(threadId) if (!thread) { - throw AppError.notFound("Mail thread not found") + throw AppError.notFound(MAIL_THREAD_NOT_FOUND) } const subject = input.subject ?? replySubject(thread.subject) const fromHeader = fromHeaderForThread(thread) diff --git a/services/api/src/services/admin/outbound-send-policy.ts b/services/api/src/services/admin/outbound-send-policy.ts index d9a762d1..83184171 100644 --- a/services/api/src/services/admin/outbound-send-policy.ts +++ b/services/api/src/services/admin/outbound-send-policy.ts @@ -4,17 +4,27 @@ export const OUTBOUND_SEND_PHASE_BUDGET_MS = 45_000 export const OUTBOUND_SEND_MIN_THROUGHPUT_BPS = 256 * 1024 -export const OUTBOUND_SEND_MIN_THROUGHPUT_FLOOR_BPS = 1024 +const OUTBOUND_SEND_MIN_THROUGHPUT_FLOOR_BPS = 1024 -export const OUTBOUND_SMTP_TIMEOUT_CEILING_MS = 60_000 +const OUTBOUND_SMTP_TIMEOUT_CEILING_MS = 60_000 export const OUTBOUND_SEND_MAX_DEADLINE_MS = 2 ** 31 - 1 -export const BASE64_EXPANSION_NUMERATOR = 4 +const BASE64_EXPANSION_NUMERATOR = 4 -export const BASE64_EXPANSION_DENOMINATOR = 3 +const BASE64_EXPANSION_DENOMINATOR = 3 -export const OUTBOUND_INFLIGHT_SLACK_SECONDS = 120 +const OUTBOUND_INFLIGHT_SLACK_SECONDS = 120 + +// nodemailer's connect, greeting and socket timeouts are each the full SMTP timeout and are inactivity +// timeouts, so the phase budget allows all three before the transfer time is added. +const SMTP_TIMEOUT_PHASES = 3 + +const ROUTE_INFLIGHT_FLOOR_SECONDS = 15 * 60 + +// The default OCI_EMAIL_SMTP_TIMEOUT_MS. The floor dominates at this timeout, and assertOutboundSendPolicy +// refuses to boot a configuration whose largest deadline would outlive the resulting window. +const ROUTE_GUARD_SMTP_TIMEOUT_MS = OUTBOUND_SMTP_TIMEOUT_CEILING_MS / 4 export function base64Bytes(rawBytes: number): number { return Math.ceil( @@ -48,7 +58,7 @@ export function maxOutboundSendDeadlineMs(input: { } export function phaseBudgetFor(smtpTimeoutMs: number): number { - return Math.max(0, smtpTimeoutMs) * 3 + return Math.max(0, smtpTimeoutMs) * SMTP_TIMEOUT_PHASES } export function inflightWindowSeconds(input: { @@ -59,9 +69,9 @@ export function inflightWindowSeconds(input: { } export const ROUTE_DEADLINE_INFLIGHT_SECONDS = Math.max( - 15 * 60, + ROUTE_INFLIGHT_FLOOR_SECONDS, inflightWindowSeconds({ - smtpTimeoutMs: OUTBOUND_SMTP_TIMEOUT_CEILING_MS / 4, + smtpTimeoutMs: ROUTE_GUARD_SMTP_TIMEOUT_MS, minThroughputBytesPerSec: OUTBOUND_SEND_MIN_THROUGHPUT_BPS, }), ) diff --git a/services/api/src/services/admin/outreach-jobs.ts b/services/api/src/services/admin/outreach-jobs.ts index 3c4036c2..0ad3871f 100644 --- a/services/api/src/services/admin/outreach-jobs.ts +++ b/services/api/src/services/admin/outreach-jobs.ts @@ -13,8 +13,6 @@ import { type OutreachService, } from "./outreach-service.js" -export { OUTREACH_DIGEST_JOB } - export async function registerOutreachJobs( container: Container, logger?: OutboundMailLogger, diff --git a/services/api/src/services/admin/outreach-service.ts b/services/api/src/services/admin/outreach-service.ts index 667e8101..7307e6aa 100644 --- a/services/api/src/services/admin/outreach-service.ts +++ b/services/api/src/services/admin/outreach-service.ts @@ -23,13 +23,15 @@ export interface OutreachRepository { claimOutreachWindow?(geoid: string, window: { at: Date; windowStart: Date }): Promise } -export const OUTREACH_SWEEP_BATCH_SIZE = 200 +const OUTREACH_SWEEP_BATCH_SIZE = 200 + +const DAY_MS = 24 * 60 * 60 * 1000 export const OUTREACH_CATEGORIES: readonly ReportCategory[] = ADMIN_CATEGORIES export function isThrottled(lastOutreachAt: Date | null, now: Date, throttleDays: number): boolean { if (lastOutreachAt === null) return false - const windowMs = throttleDays * 24 * 60 * 60 * 1000 + const windowMs = throttleDays * DAY_MS return now.getTime() - lastOutreachAt.getTime() < windowMs } @@ -124,7 +126,7 @@ export function makeOutreachService(deps: OutreachServiceDeps): OutreachService const at = now() const claim = deps.outreachRepo.claimOutreachWindow if (claim) { - const windowStart = new Date(at.getTime() - deps.throttleDays * 24 * 60 * 60 * 1000) + const windowStart = new Date(at.getTime() - deps.throttleDays * DAY_MS) const won = await claim(geoid, { at, windowStart }) if (!won) { return { geoid, sent: false, skipped: "throttled", reportCount: 0 } diff --git a/services/api/src/services/admin/pagination.ts b/services/api/src/services/admin/pagination.ts index 33a65390..a827d181 100644 --- a/services/api/src/services/admin/pagination.ts +++ b/services/api/src/services/admin/pagination.ts @@ -10,7 +10,7 @@ export { export const ADMIN_DEFAULT_LIMIT = 25 /** Mirrors the shared AdminListQuery limit ceiling. */ -export const ADMIN_MAX_LIMIT = 100 +const ADMIN_MAX_LIMIT = 100 export interface CursorAnchor { createdAt: Date diff --git a/services/api/src/services/admin/sql-fragments.ts b/services/api/src/services/admin/sql-fragments.ts index 8d4bd613..7660d8ce 100644 --- a/services/api/src/services/admin/sql-fragments.ts +++ b/services/api/src/services/admin/sql-fragments.ts @@ -15,6 +15,11 @@ export function ilikeAnyOf( ...columns.map((col) => sql`${col} ILIKE ${like} ESCAPE '\\'`), ...extraBranches, ] + return anyOf(sql, branches) +} + +// An empty branch list matches nothing rather than rendering the invalid `()`. +export function anyOf(sql: Queryable, branches: readonly SqlFragment[]): SqlFragment { const first = branches[0] if (first === undefined) return sql`(false)` const ored = branches diff --git a/services/api/src/services/admin/system-health-probes.ts b/services/api/src/services/admin/system-health-probes.ts index f13a6c48..4c8bf123 100644 --- a/services/api/src/services/admin/system-health-probes.ts +++ b/services/api/src/services/admin/system-health-probes.ts @@ -1,10 +1,8 @@ import type { Sql } from "../../db/client.js" import type { ProbeResult, SystemHealthProbes } from "./system-health-service.js" -import { MEDIA_WORKER_BACKLOG_WARN } from "./system-health-service.js" +import { MEDIA_WORKER_BACKLOG_WARN, PG_UNDEFINED_TABLE } from "./system-health-service.js" -export const PROBE_TIMEOUT_MS = 2000 - -const PG_UNDEFINED_TABLE = "42P01" +const PROBE_TIMEOUT_MS = 2000 /** * Only the media worker's pg-boss queues, so another queue's backlog (outreach digest, inbound sweep) @@ -12,30 +10,31 @@ const PG_UNDEFINED_TABLE = "42P01" */ const MEDIA_QUEUE_LIKE = "media.%" +// The timer is unref'd so a hung probe never holds the process open on shutdown. +function timeoutAfter(ms: number, onTimeout?: () => void): Promise { + return new Promise((_resolve, reject) => { + const t = setTimeout(() => { + onTimeout?.() + reject(new Error(`probe timed out after ${ms}ms`)) + }, ms) + if (typeof t === "object" && t && "unref" in t) (t as { unref: () => void }).unref() + }) +} + function withTimeout(run: () => Promise, ms: number): Promise { - return Promise.race([ - run(), - new Promise((_resolve, reject) => { - const t = setTimeout(() => reject(new Error(`probe timed out after ${ms}ms`)), ms) - if (typeof t === "object" && t && "unref" in t) (t as { unref: () => void }).unref() - }), - ]) + return Promise.race([run(), timeoutAfter(ms)]) } function withQueryTimeout( query: PromiseLike & { cancel: () => void }, ms: number, ): Promise { - return Promise.race([ - query, - new Promise((_resolve, reject) => { - const t = setTimeout(() => { - query.cancel() - reject(new Error(`probe timed out after ${ms}ms`)) - }, ms) - if (typeof t === "object" && t && "unref" in t) (t as { unref: () => void }).unref() - }), - ]) + return Promise.race([query, timeoutAfter(ms, () => query.cancel())]) +} + +function countOf(rows: readonly { n: string }[]): number { + const n = Number.parseInt(rows[0]?.n ?? "0", 10) + return Number.isNaN(n) ? 0 : n } function isUndefinedTable(err: unknown): boolean { @@ -77,8 +76,7 @@ export function makeSystemHealthProbes(deps: SystemProbeDeps): SystemHealthProbe `, PROBE_TIMEOUT_MS, ) - const n = Number.parseInt(rows[0]?.n ?? "0", 10) - return { val: `${Number.isNaN(n) ? 0 : n} events 7d`, status: "ok" } + return { val: `${countOf(rows)} events 7d`, status: "ok" } } probes.mediaWorker = async (): Promise => { @@ -93,11 +91,10 @@ export function makeSystemHealthProbes(deps: SystemProbeDeps): SystemHealthProbe `, PROBE_TIMEOUT_MS, ) - const depth = Number.parseInt(rows[0]?.n ?? "0", 10) - const value = Number.isNaN(depth) ? 0 : depth + const depth = countOf(rows) return { - val: `depth ${value}`, - status: value > MEDIA_WORKER_BACKLOG_WARN ? "warn" : "ok", + val: `depth ${depth}`, + status: depth > MEDIA_WORKER_BACKLOG_WARN ? "warn" : "ok", } } catch (err) { if (isUndefinedTable(err)) { diff --git a/services/api/src/services/admin/system-health-service.ts b/services/api/src/services/admin/system-health-service.ts index cb36f32c..66a2785b 100644 --- a/services/api/src/services/admin/system-health-service.ts +++ b/services/api/src/services/admin/system-health-service.ts @@ -33,6 +33,18 @@ export interface SystemHealthService { export const MEDIA_WORKER_BACKLOG_WARN = 50 +export const PG_UNDEFINED_TABLE = "42P01" + +const PG_INVALID_PASSWORD = "28P01" + +const PG_INVALID_AUTHORIZATION = "28000" + +const PG_INVALID_CATALOG_NAME = "3D000" + +const NO_PROBE: { status: HealthStatus; val: string } = { status: "down", val: "No probe" } + +const ROUTING_NOT_DEPLOYED_VAL = "Phase 3" + async function probeRow( name: string, probe: (() => Promise) | undefined, @@ -55,8 +67,8 @@ function classifyProbeError(err: unknown): string { const message = err instanceof Error ? err.message : String(err) if (message.includes("timed out") || code === "ETIMEDOUT") return "Timed out" if (code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "ECONNRESET") return "Unreachable" - if (code === "28P01" || code === "28000") return "Auth failed" - if (code === "42P01" || code === "3D000") return "Not provisioned" + if (code === PG_INVALID_PASSWORD || code === PG_INVALID_AUTHORIZATION) return "Auth failed" + if (code === PG_UNDEFINED_TABLE || code === PG_INVALID_CATALOG_NAME) return "Not provisioned" return "Unavailable" } @@ -67,15 +79,15 @@ export function makeSystemHealthService(deps: SystemHealthServiceDeps): SystemHe const api: SystemService = { name: "API", status: "ok", val: "Responding" } const [postgres, redis, mediaWorker, ociEmail] = await Promise.all([ - probeRow("Postgres", probes.postgres, { status: "down", val: "No probe" }, log), - probeRow("Redis", probes.redis, { status: "down", val: "No probe" }, log), + probeRow("Postgres", probes.postgres, NO_PROBE, log), + probeRow("Redis", probes.redis, NO_PROBE, log), probeRow( "Media worker", env.jobsIsFake ? undefined : probes.mediaWorker, { status: "not_deployed", val: "Jobs queue not wired" }, log, ), - probeRow("OCI Email", probes.ociEmail, { status: "down", val: "No probe" }, log), + probeRow("OCI Email", probes.ociEmail, NO_PROBE, log), ]) const ociEmailRow: SystemService = @@ -94,7 +106,7 @@ export function makeSystemHealthService(deps: SystemHealthServiceDeps): SystemHe const routing: SystemService = { name: "Routing (VRP)", status: "not_deployed", - val: "Phase 3", + val: ROUTING_NOT_DEPLOYED_VAL, } return { diff --git a/services/api/src/services/data-export-jobs.ts b/services/api/src/services/data-export-jobs.ts index fffb9df5..cf8535c3 100644 --- a/services/api/src/services/data-export-jobs.ts +++ b/services/api/src/services/data-export-jobs.ts @@ -7,7 +7,9 @@ import { isFinalJobAttempt } from "./job-attempt.js" export const DATA_EXPORT_JOB = "data.export" -export interface DataExportJob { +const SUPPORT_MAILBOX_LOCAL_PART = "support" + +interface DataExportJob { userId: string } @@ -22,7 +24,7 @@ export interface RegisterDataExportOpts { } export function dataExportSupportEmail(env: Env): string { - return `support@${env.MAIL_REPLY_DOMAIN}` + return `${SUPPORT_MAILBOX_LOCAL_PART}@${env.MAIL_REPLY_DOMAIN}` } export async function registerDataExportJobs( diff --git a/services/api/src/services/data-export-service.ts b/services/api/src/services/data-export-service.ts index 4c43a627..3b208da2 100644 --- a/services/api/src/services/data-export-service.ts +++ b/services/api/src/services/data-export-service.ts @@ -21,10 +21,14 @@ export interface DataExportServiceDeps { */ export type DataExportUndeliverable = "oversize" | "permanent" | "rejected" +export type DataExportResult = { + ok: true + email: string | null + undeliverable?: DataExportUndeliverable +} + export interface DataExportService { - exportData( - userId: string, - ): Promise<{ ok: true; email: string | null; undeliverable?: DataExportUndeliverable }> + exportData(userId: string): Promise recordUndeliverable(userId: string, kind: DataExportUndeliverable): Promise } @@ -34,14 +38,26 @@ export const DATA_EXPORT_FREE_TEXT_MAX_ROWS = 5_000 export const DATA_EXPORT_BYTE_BUDGET = 8_000_000 +const DATA_EXPORT_FORMAT = "civfix-data-export@1" + +const DATA_EXPORT_FILENAME = "civfix-export.json" + +const DATA_EXPORT_CONTENT_TYPE = "application/json" + +const REDACTED_TOKEN = "[REDACTED]" + +const DATA_EXPORT_SUBJECT = "Your civfix data export" + +const SMTP_PERMANENT_MIN = 500 + export function buildDataExportEmail( supportEmail: string, truncatedSections: readonly string[], ): { subject: string; text: string; html: string } { - const subject = "Your civfix data export" + const subject = DATA_EXPORT_SUBJECT const blocks = [ - heading("Your civfix data export"), - paragraph("Attached is a copy of your civfix data as a JSON file (civfix-export.json)."), + heading(DATA_EXPORT_SUBJECT), + paragraph(`Attached is a copy of your civfix data as a JSON file (${DATA_EXPORT_FILENAME}).`), paragraph( "It includes your profile, reports, posts, comments, messages, events, registrations, " + "organizations, volunteer hours, connections, and your issued service-hours transcripts.", @@ -66,11 +82,29 @@ export function buildDataExportEmail( return { subject, text, html } } +function buildDataExportUndeliverableEmail(supportEmail: string): { + subject: string + text: string + html: string +} { + const subject = DATA_EXPORT_SUBJECT + const blocks = [ + heading(DATA_EXPORT_SUBJECT), + paragraph( + "Your data export was too large to send by email. Your request is on record, and our team " + + `will send you a complete copy. You can also email ${supportEmail} about it.`, + ), + paragraph("If you did not request this, you can ignore this email.", { muted: true }), + ] + const { text, html } = renderEmailBody({ preheader: subject, blocks }) + return { subject, text, html } +} + /** - * The JSON key order of the export's sections. Truncation is reported in this order even though the byte - * budget is spent on the small structured sections first. + * The export body in JSON key order, which is also the order its queries run in. Truncation is reported + * in this order even though the byte budget is spent in DATA_EXPORT_FIT_ORDER. */ -const DATA_EXPORT_SECTION_ORDER = [ +const DATA_EXPORT_BODY_ORDER = [ "reports", "posts", "comments", @@ -82,6 +116,7 @@ const DATA_EXPORT_SECTION_ORDER = [ "following", "followers", "blocks", + "notificationPrefs", "pushTokens", "certificates", "organizations", @@ -92,22 +127,507 @@ const DATA_EXPORT_SECTION_ORDER = [ "eventCheckins", ] as const -export function buildDataExportUndeliverableEmail(supportEmail: string): { - subject: string - text: string - html: string -} { - const subject = "Your civfix data export" - const blocks = [ - heading("Your civfix data export"), - paragraph( - "Your data export was too large to send by email. Your request is on record, and our team " + - `will send you a complete copy. You can also email ${supportEmail} about it.`, - ), - paragraph("If you did not request this, you can ignore this email.", { muted: true }), - ] - const { text, html } = renderEmailBody({ preheader: subject, blocks }) - return { subject, text, html } +type BodyKey = (typeof DATA_EXPORT_BODY_ORDER)[number] + +type SingleRecordKey = "notificationPrefs" + +type SectionName = Exclude + +/** + * The budget is spent on the small structured sections first, so one heavy free-text history (thousands of + * chat messages) cannot crowd out a user's certificates, consents or registrations. + */ +const DATA_EXPORT_FIT_ORDER = [ + "pushTokens", + "certificates", + "organizations", + "eventConsents", + "eventRegistrations", + "eventCheckins", + "eventTeamMemberships", + "volunteerHours", + "cleanupsOrganized", + "cleanupsJoined", + "following", + "followers", + "blocks", + "reports", + "comments", + "eventAnswers", + "posts", + "chatMessages", + "dmMessages", +] as const satisfies readonly SectionName[] + +type UnfittedSection = Exclude + +// A section missing from the fit order would silently drop out of every export. +const _everySectionIsFitted: UnfittedSection extends never ? true : never = true + +interface SectionPart { + kind: "section" + rowCap: number + /** Fetches one row past `rowCap` so a clipped section is detectable. */ + load(sql: Sql, userId: string, rowCap: number): PromiseLike + /** Runs before the byte budget is charged, so the budget measures the shape that ships. */ + redact?(row: object): object + /** Runs on the kept rows only; the budget was charged on the loaded shape. */ + present?(row: object): unknown +} + +interface SingleRecordPart { + kind: "record" + load(sql: Sql, userId: string): PromiseLike +} + +type BodyParts = { readonly [K in SectionName]: SectionPart } & { + readonly [K in SingleRecordKey]: SingleRecordPart +} + +interface ProfileRow { + id: string + display_name: string + handle: string | null + email: string | null + email_verified: boolean + bio: string | null + avatar_url: string | null + donation_url: string | null + created_at: Date + deleted_at: Date | null +} + +interface PushTokenRow { + id: string + platform: string + created_at: Date + revoked_at: Date | null +} + +async function loadProfile(sql: Sql, userId: string): Promise { + const rows = await sql` + SELECT id, display_name, handle, email, email_verified, bio, avatar_url, donation_url, + created_at, deleted_at + FROM users WHERE id = ${userId} LIMIT 1 + ` + return rows[0] ?? null +} + +const DATA_EXPORT_BODY: BodyParts = { + reports: { + kind: "section", + rowCap: DATA_EXPORT_MAX_ROWS, + load: (sql, userId, rowCap) => sql< + { + id: string + category: string + title: string | null + description: string | null + place: string | null + status: string + created_at: Date + }[] + >` + SELECT r.id, r.category, r.title, r.description, j.name AS place, r.status, r.created_at + FROM reports r + LEFT JOIN jurisdictions j ON j.geoid = r.jurisdiction_geoid + WHERE r.reporter_user_id = ${userId} + ORDER BY r.created_at DESC + LIMIT ${rowCap + 1} + `, + }, + + posts: { + kind: "section", + rowCap: DATA_EXPORT_FREE_TEXT_MAX_ROWS, + load: (sql, userId, rowCap) => sql< + { + id: string + kind: string + body: string | null + visibility: string + reply_to_id: string | null + repost_of_id: string | null + created_at: Date + updated_at: Date + deleted_at: Date | null + }[] + >` + SELECT id, kind, body, visibility, reply_to_id, repost_of_id, created_at, updated_at, deleted_at + FROM posts + WHERE author_id = ${userId} + ORDER BY created_at DESC + LIMIT ${rowCap + 1} + `, + }, + + comments: { + kind: "section", + rowCap: DATA_EXPORT_MAX_ROWS, + load: () => + Promise.resolve< + { id: string; report_id: string; body: string; created_at: Date; deleted_at: Date | null }[] + >([]), + }, + + chatMessages: { + kind: "section", + rowCap: DATA_EXPORT_FREE_TEXT_MAX_ROWS, + load: (sql, userId, rowCap) => sql< + { + id: string + cleanup_id: string | null + report_id: string | null + group_id: string | null + body: string | null + created_at: Date + deleted_at: Date | null + }[] + >` + SELECT id, cleanup_id, report_id, group_id, body, created_at, deleted_at + FROM chat_messages + WHERE sender_id = ${userId} + ORDER BY created_at DESC + LIMIT ${rowCap + 1} + `, + }, + + dmMessages: { + kind: "section", + rowCap: DATA_EXPORT_FREE_TEXT_MAX_ROWS, + load: (sql, userId, rowCap) => sql< + { + id: string + thread_id: string + body: string | null + created_at: Date + deleted_at: Date | null + }[] + >` + SELECT id, thread_id, body, created_at, deleted_at + FROM dm_messages + WHERE sender_id = ${userId} + ORDER BY created_at DESC + LIMIT ${rowCap + 1} + `, + }, + + volunteerHours: { + kind: "section", + rowCap: DATA_EXPORT_MAX_ROWS, + load: (sql, userId, rowCap) => sql< + { + id: string + source: string + report_id: string | null + cleanup_id: string | null + jurisdiction_geoid: string | null + hours: number + logged_by_user_id: string | null + created_at: Date + }[] + >` + SELECT id, source, report_id, cleanup_id, jurisdiction_geoid, + hours::float8 AS hours, logged_by_user_id, created_at + FROM volunteer_hours + WHERE user_id = ${userId} + ORDER BY created_at DESC + LIMIT ${rowCap + 1} + `, + }, + + cleanupsOrganized: { + kind: "section", + rowCap: DATA_EXPORT_MAX_ROWS, + load: (sql, userId, rowCap) => sql<{ id: string; title: string | null; created_at: Date }[]>` + SELECT id, title, created_at FROM cleanups + WHERE organizer_user_id = ${userId} + ORDER BY created_at DESC + LIMIT ${rowCap + 1} + `, + }, + + cleanupsJoined: { + kind: "section", + rowCap: DATA_EXPORT_MAX_ROWS, + load: (sql, userId, rowCap) => sql<{ cleanup_id: string; role: string; joined_at: Date }[]>` + SELECT cleanup_id, role, joined_at FROM cleanup_members + WHERE user_id = ${userId} + ORDER BY joined_at DESC + LIMIT ${rowCap + 1} + `, + }, + + following: { + kind: "section", + rowCap: DATA_EXPORT_MAX_ROWS, + load: (sql, userId, rowCap) => sql<{ followee_id: string }[]>` + SELECT followee_id FROM follows_people WHERE follower_id = ${userId} + LIMIT ${rowCap + 1} + `, + present: (row: { followee_id: string }) => row.followee_id, + }, + + followers: { + kind: "section", + rowCap: DATA_EXPORT_MAX_ROWS, + load: (sql, userId, rowCap) => sql<{ follower_id: string }[]>` + SELECT follower_id FROM follows_people WHERE followee_id = ${userId} + LIMIT ${rowCap + 1} + `, + present: (row: { follower_id: string }) => row.follower_id, + }, + + blocks: { + kind: "section", + rowCap: DATA_EXPORT_MAX_ROWS, + load: (sql, userId, rowCap) => sql<{ blocked_id: string }[]>` + SELECT blocked_id FROM user_blocks WHERE blocker_id = ${userId} + LIMIT ${rowCap + 1} + `, + present: (row: { blocked_id: string }) => row.blocked_id, + }, + + notificationPrefs: { + kind: "record", + load: (sql, userId) => sql< + { + user_id: string + push: boolean + cleanup_chat: boolean + report_updates: boolean + follows: boolean + quiet_start: string | null + quiet_end: string | null + mentions: boolean + host_broadcasts: boolean + }[] + >` + SELECT user_id, push, cleanup_chat, report_updates, follows, quiet_start, quiet_end, mentions, + host_broadcasts + FROM notification_prefs WHERE user_id = ${userId} LIMIT 1 + `, + }, + + pushTokens: { + kind: "section", + rowCap: DATA_EXPORT_MAX_ROWS, + load: (sql, userId, rowCap) => sql` + SELECT id, platform, created_at, revoked_at FROM push_tokens WHERE user_id = ${userId} + LIMIT ${rowCap + 1} + `, + redact: (t: PushTokenRow) => ({ + id: t.id, + platform: t.platform, + token: REDACTED_TOKEN, + createdAt: t.created_at, + revokedAt: t.revoked_at, + }), + }, + + certificates: { + kind: "section", + rowCap: DATA_EXPORT_MAX_ROWS, + load: (sql, userId, rowCap) => sql< + { + code: string + locale: string + holder_name: string + holder_handle: string | null + total_hours: number + entry_count: number + period_start: Date | null + period_end: Date | null + document_sha256: string + byte_size: number + issued_at: Date + revoked_at: Date | null + revoked_reason: string | null + }[] + >` + SELECT + code, locale, holder_name, holder_handle, total_hours::float8 AS total_hours, entry_count, + period_start, period_end, document_sha256, byte_size, issued_at, revoked_at, revoked_reason + FROM service_hours_certificates + WHERE user_id = ${userId} + ORDER BY issued_at DESC + LIMIT ${rowCap + 1} + `, + }, + + organizations: { + kind: "section", + rowCap: DATA_EXPORT_MAX_ROWS, + load: (sql, userId, rowCap) => sql< + { organization_id: string; slug: string; name: string; role: string; joined_at: Date }[] + >` + SELECT om.organization_id, o.slug, o.name, om.role, om.joined_at + FROM organization_members om + JOIN organizations o ON o.id = om.organization_id + WHERE om.user_id = ${userId} + ORDER BY om.joined_at DESC + LIMIT ${rowCap + 1} + `, + }, + + eventTeamMemberships: { + kind: "section", + rowCap: DATA_EXPORT_MAX_ROWS, + load: (sql, userId, rowCap) => sql< + { cleanup_id: string; role: string; joined_at: Date | null }[] + >` + SELECT cleanup_id, role, joined_at FROM cleanup_members + WHERE user_id = ${userId} AND role <> 'member' + ORDER BY joined_at DESC NULLS LAST + LIMIT ${rowCap + 1} + `, + }, + + eventConsents: { + kind: "section", + rowCap: DATA_EXPORT_MAX_ROWS, + load: (sql, userId, rowCap) => sql< + { + cleanup_id: string + terms_version: string + disclosure_version: string + host_contact_opt_in: boolean + sms_opt_in: boolean + accepted_at: Date + }[] + >` + SELECT cleanup_id, terms_version, disclosure_version, host_contact_opt_in, sms_opt_in, + accepted_at + FROM event_consents WHERE user_id = ${userId} + ORDER BY accepted_at DESC + LIMIT ${rowCap + 1} + `, + }, + + eventRegistrations: { + kind: "section", + rowCap: DATA_EXPORT_MAX_ROWS, + load: (sql, userId, rowCap) => sql< + { + id: string + cleanup_id: string + ticket_type_name: string | null + party_size: number + status: string + source: string + registered_at: Date + cancelled_at: Date | null + }[] + >` + SELECT r.id, r.cleanup_id, t.name AS ticket_type_name, r.party_size, r.status, r.source, + r.registered_at, r.cancelled_at + FROM cleanup_registrations r + LEFT JOIN cleanup_ticket_types t ON t.id = r.ticket_type_id + WHERE r.user_id = ${userId} + ORDER BY r.registered_at DESC + LIMIT ${rowCap + 1} + `, + }, + + eventAnswers: { + kind: "section", + rowCap: DATA_EXPORT_FREE_TEXT_MAX_ROWS, + load: (sql, userId, rowCap) => sql< + { cleanup_id: string; prompt: string; value: string | null }[] + >` + SELECT a.cleanup_id, q.prompt, + COALESCE(a.value_text, a.value_json::text) AS value + FROM cleanup_answers a + JOIN cleanup_questions q ON q.id = a.question_id + JOIN cleanup_registrations r ON r.id = a.registration_id + WHERE r.user_id = ${userId} AND a.scrubbed_at IS NULL + ORDER BY a.created_at DESC + LIMIT ${rowCap + 1} + `, + }, + + eventCheckins: { + kind: "section", + rowCap: DATA_EXPORT_MAX_ROWS, + load: (sql, userId, rowCap) => sql< + { + cleanup_id: string + seat_index: number + checked_in_at: Date + checkin_method: string | null + }[] + >` + SELECT s.cleanup_id, s.seat_index, s.checked_in_at, s.checkin_method + FROM cleanup_registration_seats s + JOIN cleanup_registrations r ON r.id = s.registration_id + WHERE r.user_id = ${userId} AND s.checked_in_at IS NOT NULL + ORDER BY s.checked_in_at DESC + LIMIT ${rowCap + 1} + `, + }, +} + +function loadBodyPart(sql: Sql, userId: string, key: BodyKey): PromiseLike { + const part = DATA_EXPORT_BODY[key] + return part.kind === "section" ? part.load(sql, userId, part.rowCap) : part.load(sql, userId) +} + +/** Queries run in body order: the statements go out in the same sequence on every export. */ +async function loadBody(sql: Sql, userId: string): Promise> { + const loaded = await Promise.all( + DATA_EXPORT_BODY_ORDER.map((key) => loadBodyPart(sql, userId, key)), + ) + return new Map(DATA_EXPORT_BODY_ORDER.map((key, i) => [key, loaded[i] ?? []])) +} + +interface FittedBody { + body: Record + truncatedSections: SectionName[] + sectionCaps: Record +} + +function fitBody(loaded: ReadonlyMap): FittedBody { + const truncatedCaps = new Map() + let usedBytes = 0 + + const fit = (name: SectionName, rows: readonly object[], rowCap: number): object[] => { + let truncated = rows.length > rowCap + const kept: object[] = [] + for (const row of truncated ? rows.slice(0, rowCap) : rows) { + const size = Buffer.byteLength(JSON.stringify(row), "utf8") + 1 + if (usedBytes + size > DATA_EXPORT_BYTE_BUDGET) { + truncated = true + break + } + usedBytes += size + kept.push(row) + } + if (truncated) truncatedCaps.set(name, rowCap) + return kept + } + + const sections = new Map() + for (const name of DATA_EXPORT_FIT_ORDER) { + const { rowCap, redact, present } = DATA_EXPORT_BODY[name] + const rows = loaded.get(name) ?? [] + const kept = fit(name, redact ? rows.map(redact) : rows, rowCap) + sections.set(name, present ? kept.map(present) : kept) + } + + const body = Object.fromEntries( + DATA_EXPORT_BODY_ORDER.map((key) => [ + key, + key === "notificationPrefs" ? (loaded.get(key)?.[0] ?? null) : sections.get(key), + ]), + ) as Record + + const truncatedSections = DATA_EXPORT_BODY_ORDER.filter( + (key): key is SectionName => key !== "notificationPrefs" && truncatedCaps.has(key), + ) + const sectionCaps = Object.fromEntries( + truncatedSections.map((name) => [name, truncatedCaps.get(name)]), + ) + return { body, truncatedSections, sectionCaps } } export function makeDataExportService(deps: DataExportServiceDeps): DataExportService { @@ -139,438 +659,55 @@ export function makeDataExportService(deps: DataExportServiceDeps): DataExportSe } } - return { - async exportData( - userId: string, - ): Promise<{ ok: true; email: string | null; undeliverable?: DataExportUndeliverable }> { - const profileRows = await sql< - { - id: string - display_name: string - handle: string | null - email: string | null - email_verified: boolean - bio: string | null - avatar_url: string | null - donation_url: string | null - created_at: Date - deleted_at: Date | null - }[] - >` - SELECT id, display_name, handle, email, email_verified, bio, avatar_url, donation_url, - created_at, deleted_at - FROM users WHERE id = ${userId} LIMIT 1 - ` - - const reports = sql< - { - id: string - category: string - title: string | null - description: string | null - place: string | null - status: string - created_at: Date - }[] - >` - SELECT r.id, r.category, r.title, r.description, j.name AS place, r.status, r.created_at - FROM reports r - LEFT JOIN jurisdictions j ON j.geoid = r.jurisdiction_geoid - WHERE r.reporter_user_id = ${userId} - ORDER BY r.created_at DESC - LIMIT ${DATA_EXPORT_MAX_ROWS + 1} - ` - - const posts = sql< - { - id: string - kind: string - body: string | null - visibility: string - reply_to_id: string | null - repost_of_id: string | null - created_at: Date - updated_at: Date - deleted_at: Date | null - }[] - >` - SELECT id, kind, body, visibility, reply_to_id, repost_of_id, created_at, updated_at, deleted_at - FROM posts - WHERE author_id = ${userId} - ORDER BY created_at DESC - LIMIT ${DATA_EXPORT_FREE_TEXT_MAX_ROWS + 1} - ` - - const comments = Promise.resolve< - { id: string; report_id: string; body: string; created_at: Date; deleted_at: Date | null }[] - >([]) - - const chatMessages = sql< - { - id: string - cleanup_id: string | null - report_id: string | null - group_id: string | null - body: string | null - created_at: Date - deleted_at: Date | null - }[] - >` - SELECT id, cleanup_id, report_id, group_id, body, created_at, deleted_at - FROM chat_messages - WHERE sender_id = ${userId} - ORDER BY created_at DESC - LIMIT ${DATA_EXPORT_FREE_TEXT_MAX_ROWS + 1} - ` - - const dmMessages = sql< - { - id: string - thread_id: string - body: string | null - created_at: Date - deleted_at: Date | null - }[] - >` - SELECT id, thread_id, body, created_at, deleted_at - FROM dm_messages - WHERE sender_id = ${userId} - ORDER BY created_at DESC - LIMIT ${DATA_EXPORT_FREE_TEXT_MAX_ROWS + 1} - ` - - const volunteerHours = sql< - { - id: string - source: string - report_id: string | null - cleanup_id: string | null - jurisdiction_geoid: string | null - hours: number - logged_by_user_id: string | null - created_at: Date - }[] - >` - SELECT id, source, report_id, cleanup_id, jurisdiction_geoid, - hours::float8 AS hours, logged_by_user_id, created_at - FROM volunteer_hours - WHERE user_id = ${userId} - ORDER BY created_at DESC - LIMIT ${DATA_EXPORT_MAX_ROWS + 1} - ` - - const cleanupsOrganized = sql<{ id: string; title: string | null; created_at: Date }[]>` - SELECT id, title, created_at FROM cleanups - WHERE organizer_user_id = ${userId} - ORDER BY created_at DESC - LIMIT ${DATA_EXPORT_MAX_ROWS + 1} - ` - const cleanupsJoined = sql<{ cleanup_id: string; role: string; joined_at: Date }[]>` - SELECT cleanup_id, role, joined_at FROM cleanup_members - WHERE user_id = ${userId} - ORDER BY joined_at DESC - LIMIT ${DATA_EXPORT_MAX_ROWS + 1} - ` - - const following = sql<{ followee_id: string }[]>` - SELECT followee_id FROM follows_people WHERE follower_id = ${userId} - LIMIT ${DATA_EXPORT_MAX_ROWS + 1} - ` - const followers = sql<{ follower_id: string }[]>` - SELECT follower_id FROM follows_people WHERE followee_id = ${userId} - LIMIT ${DATA_EXPORT_MAX_ROWS + 1} - ` - const blocks = sql<{ blocked_id: string }[]>` - SELECT blocked_id FROM user_blocks WHERE blocker_id = ${userId} - LIMIT ${DATA_EXPORT_MAX_ROWS + 1} - ` - - const notificationPrefs = sql< - { - user_id: string - push: boolean - cleanup_chat: boolean - report_updates: boolean - follows: boolean - quiet_start: string | null - quiet_end: string | null - mentions: boolean - host_broadcasts: boolean - }[] - >` - SELECT user_id, push, cleanup_chat, report_updates, follows, quiet_start, quiet_end, mentions, - host_broadcasts - FROM notification_prefs WHERE user_id = ${userId} LIMIT 1 - ` - - const pushTokenRowsQuery = sql< - { id: string; platform: string; created_at: Date; revoked_at: Date | null }[] - >` - SELECT id, platform, created_at, revoked_at FROM push_tokens WHERE user_id = ${userId} - LIMIT ${DATA_EXPORT_MAX_ROWS + 1} - ` - - const certificates = sql< - { - code: string - locale: string - holder_name: string - holder_handle: string | null - total_hours: number - entry_count: number - period_start: Date | null - period_end: Date | null - document_sha256: string - byte_size: number - issued_at: Date - revoked_at: Date | null - revoked_reason: string | null - }[] - >` - SELECT - code, locale, holder_name, holder_handle, total_hours::float8 AS total_hours, entry_count, - period_start, period_end, document_sha256, byte_size, issued_at, revoked_at, revoked_reason - FROM service_hours_certificates - WHERE user_id = ${userId} - ORDER BY issued_at DESC - LIMIT ${DATA_EXPORT_MAX_ROWS + 1} - ` - - const organizations = sql< - { organization_id: string; slug: string; name: string; role: string; joined_at: Date }[] - >` - SELECT om.organization_id, o.slug, o.name, om.role, om.joined_at - FROM organization_members om - JOIN organizations o ON o.id = om.organization_id - WHERE om.user_id = ${userId} - ORDER BY om.joined_at DESC - LIMIT ${DATA_EXPORT_MAX_ROWS + 1} - ` - - const eventTeamMemberships = sql< - { cleanup_id: string; role: string; joined_at: Date | null }[] - >` - SELECT cleanup_id, role, joined_at FROM cleanup_members - WHERE user_id = ${userId} AND role <> 'member' - ORDER BY joined_at DESC NULLS LAST - LIMIT ${DATA_EXPORT_MAX_ROWS + 1} - ` - - const eventConsents = sql< - { - cleanup_id: string - terms_version: string - disclosure_version: string - host_contact_opt_in: boolean - sms_opt_in: boolean - accepted_at: Date - }[] - >` - SELECT cleanup_id, terms_version, disclosure_version, host_contact_opt_in, sms_opt_in, - accepted_at - FROM event_consents WHERE user_id = ${userId} - ORDER BY accepted_at DESC - LIMIT ${DATA_EXPORT_MAX_ROWS + 1} - ` - - const eventRegistrations = sql< - { - id: string - cleanup_id: string - ticket_type_name: string | null - party_size: number - status: string - source: string - registered_at: Date - cancelled_at: Date | null - }[] - >` - SELECT r.id, r.cleanup_id, t.name AS ticket_type_name, r.party_size, r.status, r.source, - r.registered_at, r.cancelled_at - FROM cleanup_registrations r - LEFT JOIN cleanup_ticket_types t ON t.id = r.ticket_type_id - WHERE r.user_id = ${userId} - ORDER BY r.registered_at DESC - LIMIT ${DATA_EXPORT_MAX_ROWS + 1} - ` - - const eventAnswers = sql<{ cleanup_id: string; prompt: string; value: string | null }[]>` - SELECT a.cleanup_id, q.prompt, - COALESCE(a.value_text, a.value_json::text) AS value - FROM cleanup_answers a - JOIN cleanup_questions q ON q.id = a.question_id - JOIN cleanup_registrations r ON r.id = a.registration_id - WHERE r.user_id = ${userId} AND a.scrubbed_at IS NULL - ORDER BY a.created_at DESC - LIMIT ${DATA_EXPORT_FREE_TEXT_MAX_ROWS + 1} - ` - - const eventCheckins = sql< - { - cleanup_id: string - seat_index: number - checked_in_at: Date - checkin_method: string | null - }[] - >` - SELECT s.cleanup_id, s.seat_index, s.checked_in_at, s.checkin_method - FROM cleanup_registration_seats s - JOIN cleanup_registrations r ON r.id = s.registration_id - WHERE r.user_id = ${userId} AND s.checked_in_at IS NOT NULL - ORDER BY s.checked_in_at DESC - LIMIT ${DATA_EXPORT_MAX_ROWS + 1} - ` - - const [ - reportRows, - postRows, - commentRows, - chatRows, - dmRows, - volunteerHourRows, - cleanupsOrganizedRows, - cleanupsJoinedRows, - followingRows, - followerRows, - blockRows, - notificationPrefRows, - pushTokenRows, - certificateRows, - organizationRows, - eventTeamMembershipRows, - eventConsentRows, - eventRegistrationRows, - eventAnswerRows, - eventCheckinRows, - ] = await Promise.all([ - reports, - posts, - comments, - chatMessages, - dmMessages, - volunteerHours, - cleanupsOrganized, - cleanupsJoined, - following, - followers, - blocks, - notificationPrefs, - pushTokenRowsQuery, - certificates, - organizations, - eventTeamMemberships, - eventConsents, - eventRegistrations, - eventAnswers, - eventCheckins, - ]) - - const profile = profileRows[0] ?? null - const email = - profile?.email ?? (users ? ((await users.findById(userId))?.email ?? null) : null) - - const truncatedCaps = new Map() - let usedBytes = 0 - - const fit = ( - name: (typeof DATA_EXPORT_SECTION_ORDER)[number], - rows: T[], - rowCap: number, - ): T[] => { - let truncated = false - let source = rows - if (source.length > rowCap) { - source = source.slice(0, rowCap) - truncated = true - } - const kept: T[] = [] - for (const row of source) { - const size = Buffer.byteLength(JSON.stringify(row), "utf8") + 1 - if (usedBytes + size > DATA_EXPORT_BYTE_BUDGET) { - truncated = true - break - } - usedBytes += size - kept.push(row) - } - if (truncated) truncatedCaps.set(name, rowCap) - return kept - } + async function resolveEmail(userId: string, profile: ProfileRow | null): Promise { + if (profile?.email != null) return profile.email + if (!users) return null + return (await users.findById(userId))?.email ?? null + } - // The budget is spent on the small structured sections first, so one heavy free-text history - // (thousands of chat messages) cannot crowd out a user's certificates, consents or registrations. - const pushTokensFit = fit( - "pushTokens", - pushTokenRows.map((t) => ({ - id: t.id, - platform: t.platform, - token: "[REDACTED]", - createdAt: t.created_at, - revokedAt: t.revoked_at, - })), - DATA_EXPORT_MAX_ROWS, - ) - const certificatesFit = fit("certificates", certificateRows, DATA_EXPORT_MAX_ROWS) - const organizationsFit = fit("organizations", organizationRows, DATA_EXPORT_MAX_ROWS) - const eventConsentsFit = fit("eventConsents", eventConsentRows, DATA_EXPORT_MAX_ROWS) - const eventRegistrationsFit = fit( - "eventRegistrations", - eventRegistrationRows, - DATA_EXPORT_MAX_ROWS, - ) - const eventCheckinsFit = fit("eventCheckins", eventCheckinRows, DATA_EXPORT_MAX_ROWS) - const eventTeamMembershipsFit = fit( - "eventTeamMemberships", - eventTeamMembershipRows, - DATA_EXPORT_MAX_ROWS, - ) - const volunteerHoursFit = fit("volunteerHours", volunteerHourRows, DATA_EXPORT_MAX_ROWS) - const cleanupsOrganizedFit = fit( - "cleanupsOrganized", - cleanupsOrganizedRows, - DATA_EXPORT_MAX_ROWS, - ) - const cleanupsJoinedFit = fit("cleanupsJoined", cleanupsJoinedRows, DATA_EXPORT_MAX_ROWS) - const followingFit = fit("following", followingRows, DATA_EXPORT_MAX_ROWS) - const followersFit = fit("followers", followerRows, DATA_EXPORT_MAX_ROWS) - const blocksFit = fit("blocks", blockRows, DATA_EXPORT_MAX_ROWS) - const reportsFit = fit("reports", reportRows, DATA_EXPORT_MAX_ROWS) - const commentsFit = fit("comments", commentRows, DATA_EXPORT_MAX_ROWS) - const eventAnswersFit = fit("eventAnswers", eventAnswerRows, DATA_EXPORT_FREE_TEXT_MAX_ROWS) - const postsFit = fit("posts", postRows, DATA_EXPORT_FREE_TEXT_MAX_ROWS) - const chatMessagesFit = fit("chatMessages", chatRows, DATA_EXPORT_FREE_TEXT_MAX_ROWS) - const dmMessagesFit = fit("dmMessages", dmRows, DATA_EXPORT_FREE_TEXT_MAX_ROWS) - - const truncatedSections = DATA_EXPORT_SECTION_ORDER.filter((name) => truncatedCaps.has(name)) - const sectionCaps = Object.fromEntries( - truncatedSections.map((name) => [name, truncatedCaps.get(name)]), - ) + async function deliver( + userId: string, + email: string, + exportObject: object, + truncatedSections: readonly string[], + ): Promise { + const bytes = new TextEncoder().encode(JSON.stringify(exportObject)) + const rendered = buildDataExportEmail(supportEmail, truncatedSections) + try { + await mailer.sendOutbound({ + from: fromNoReply, + to: email, + subject: rendered.subject, + text: rendered.text, + html: rendered.html, + attachments: [ + { filename: DATA_EXPORT_FILENAME, contentType: DATA_EXPORT_CONTENT_TYPE, content: bytes }, + ], + }) + } catch (err) { + const kind = undeliverableKind(err) + if (kind === null) throw err + await recordUndeliverable(userId, kind) + // A rejected recipient would bounce a notice too; only a size refusal can still reach the user. + if (kind === "oversize") await sendUndeliverableNotice(email) + return { ok: true, email, undeliverable: kind } + } + return { ok: true, email } + } + + return { + async exportData(userId: string): Promise { + const profile = await loadProfile(sql, userId) + const loaded = await loadBody(sql, userId) + const email = await resolveEmail(userId, profile) + const { body, truncatedSections, sectionCaps } = fitBody(loaded) const exportObject = { exportedAt: new Date().toISOString(), - format: "civfix-data-export@1", + format: DATA_EXPORT_FORMAT, userId, profile, - reports: reportsFit, - posts: postsFit, - comments: commentsFit, - chatMessages: chatMessagesFit, - dmMessages: dmMessagesFit, - volunteerHours: volunteerHoursFit, - cleanupsOrganized: cleanupsOrganizedFit, - cleanupsJoined: cleanupsJoinedFit, - following: followingFit.map((f) => f.followee_id), - followers: followersFit.map((f) => f.follower_id), - blocks: blocksFit.map((b) => b.blocked_id), - notificationPrefs: notificationPrefRows[0] ?? null, - pushTokens: pushTokensFit, - certificates: certificatesFit, - organizations: organizationsFit, - eventTeamMemberships: eventTeamMembershipsFit, - eventConsents: eventConsentsFit, - eventRegistrations: eventRegistrationsFit, - eventAnswers: eventAnswersFit, - eventCheckins: eventCheckinsFit, + ...body, truncated: truncatedSections.length > 0 ? { @@ -584,44 +721,13 @@ export function makeDataExportService(deps: DataExportServiceDeps): DataExportSe } if (email === null) return { ok: true, email: null } - - const bytes = new TextEncoder().encode(JSON.stringify(exportObject)) - - const rendered = buildDataExportEmail(supportEmail, truncatedSections) - - try { - await mailer.sendOutbound({ - from: fromNoReply, - to: email, - subject: rendered.subject, - text: rendered.text, - html: rendered.html, - attachments: [ - { - filename: "civfix-export.json", - contentType: "application/json", - content: bytes, - }, - ], - }) - } catch (err) { - const kind = undeliverableKind(err) - if (kind === null) throw err - await recordUndeliverable(userId, kind) - // A rejected recipient would bounce a notice too; only a size refusal can still reach the user. - if (kind === "oversize") await sendUndeliverableNotice(email) - return { ok: true, email, undeliverable: kind } - } - - return { ok: true, email } + return deliver(userId, email, exportObject, truncatedSections) }, recordUndeliverable, } } -const SMTP_PERMANENT_MIN = 500 - // A 5xx the provider sends in answer to the message body refuses this message, not our credentials or // sender, so rebuilding and resending the same export can only be refused again. function isMessageRejection(failure: ReturnType): boolean { diff --git a/services/api/src/services/host/analytics-repository.drizzle.ts b/services/api/src/services/host/analytics-repository.drizzle.ts index e8666686..79a8ab47 100644 --- a/services/api/src/services/host/analytics-repository.drizzle.ts +++ b/services/api/src/services/host/analytics-repository.drizzle.ts @@ -9,7 +9,7 @@ export const INSIGHTS_TREND_LIMIT = 400 export const INSIGHTS_SOURCE_LIMIT = 10 -export const SUMMARY_DAY_ZONE = "UTC" +const SUMMARY_DAY_ZONE = "UTC" export interface EventKpiRow { registered: number @@ -98,7 +98,7 @@ export interface PortfolioHoursTotals { volunteersCredited: number } -export const ZERO_PORTFOLIO_HOURS_TOTALS: PortfolioHoursTotals = Object.freeze({ +const ZERO_PORTFOLIO_HOURS_TOTALS: PortfolioHoursTotals = Object.freeze({ credited: 0, volunteersCredited: 0, }) @@ -126,7 +126,7 @@ export interface HostActivityTotals { postsCreated: number } -export const ZERO_HOST_ACTIVITY_TOTALS: HostActivityTotals = Object.freeze({ +const ZERO_HOST_ACTIVITY_TOTALS: HostActivityTotals = Object.freeze({ registrations: 0, cancellations: 0, hoursTotal: 0, @@ -143,7 +143,7 @@ export interface HeldEventTotals { noShow: number } -export const ZERO_HELD_EVENT_TOTALS: HeldEventTotals = Object.freeze({ +const ZERO_HELD_EVENT_TOTALS: HeldEventTotals = Object.freeze({ events: 0, registered: 0, checkedIn: 0, @@ -205,11 +205,19 @@ export interface AnalyticsRepository { ): Promise } -export function makeDrizzleAnalyticsRepository(sql: Sql): AnalyticsRepository { - async function dayCounts(rows: { day: string; n: string }[]): Promise { - return rows.map((row) => ({ day: row.day, count: Number(row.n) })) - } +function dayCounts(rows: { day: string; n: string }[]): DayCount[] { + return rows.map((row) => ({ day: row.day, count: Number(row.n) })) +} + +function keyCounts(rows: { key: string; n: string }[]): KeyCount[] { + return rows.map((row) => ({ key: row.key, count: Number(row.n) })) +} + +function labeledKeyCounts(rows: { key: string; label: string; n: string }[]): LabeledKeyCount[] { + return rows.map((row) => ({ key: row.key, label: row.label, count: Number(row.n) })) +} +export function makeDrizzleAnalyticsRepository(sql: Sql): AnalyticsRepository { return { async eventKpis(cleanupId: string): Promise { const rows = await sql< @@ -274,7 +282,7 @@ export function makeDrizzleAnalyticsRepository(sql: Sql): AnalyticsRepository { LEFT JOIN cleanup_ticket_types t ON t.id = r.ticket_type_id WHERE r.cleanup_id = ${cleanupId} AND r.status = 'registered' GROUP BY 1 ORDER BY 2 DESC LIMIT 50` - return rows.map((row) => ({ key: row.key, count: Number(row.n) })) + return keyCounts(rows) }, async registrationsByAudience(cleanupId) { @@ -284,7 +292,7 @@ export function makeDrizzleAnalyticsRepository(sql: Sql): AnalyticsRepository { FROM cleanup_registrations WHERE cleanup_id = ${cleanupId} AND status = 'registered' GROUP BY 1` - return rows.map((row) => ({ key: row.key, count: Number(row.n) })) + return keyCounts(rows) }, async checkinsByTicketType(cleanupId) { @@ -295,7 +303,7 @@ export function makeDrizzleAnalyticsRepository(sql: Sql): AnalyticsRepository { LEFT JOIN cleanup_ticket_types t ON t.id = r.ticket_type_id WHERE s.cleanup_id = ${cleanupId} AND s.checked_in_at IS NOT NULL GROUP BY 1 ORDER BY 2 DESC LIMIT 50` - return rows.map((row) => ({ key: row.key, count: Number(row.n) })) + return keyCounts(rows) }, async checkinsBySlot(cleanupId) { @@ -308,7 +316,7 @@ export function makeDrizzleAnalyticsRepository(sql: Sql): AnalyticsRepository { LEFT JOIN cleanup_slots sl ON sl.id = sc.slot_id WHERE s.cleanup_id = ${cleanupId} AND s.checked_in_at IS NOT NULL GROUP BY 1 ORDER BY 2 DESC LIMIT 50` - return rows.map((row) => ({ key: row.key, count: Number(row.n) })) + return keyCounts(rows) }, async arrivalOffsets(cleanupId, limit) { @@ -401,7 +409,7 @@ export function makeDrizzleAnalyticsRepository(sql: Sql): AnalyticsRepository { GROUP BY c.id, c.title ORDER BY 2 DESC LIMIT ${limit}` - return rows.map((row) => ({ key: row.key, count: Number(row.n) })) + return keyCounts(rows) }, async broadcastsSent(cleanupId, timezone, from, to) { @@ -744,17 +752,9 @@ export function makeDrizzleAnalyticsRepository(sql: Sql): AnalyticsRepository { LIMIT ${MAX_HOST_SUMMARY_EVENT_ROWS}`, ]) return { - daily: await dayCounts(dailyRows), - byEvent: byEventRows.map((row) => ({ - key: row.key, - label: row.label, - count: Number(row.n), - })), - hoursByEvent: hoursRows.map((row) => ({ - key: row.key, - label: row.label, - count: Number(row.n), - })), + daily: dayCounts(dailyRows), + byEvent: labeledKeyCounts(byEventRows), + hoursByEvent: labeledKeyCounts(hoursRows), } }, } diff --git a/services/api/src/services/host/analytics-service.ts b/services/api/src/services/host/analytics-service.ts index 3a1c265c..e8e43f84 100644 --- a/services/api/src/services/host/analytics-service.ts +++ b/services/api/src/services/host/analytics-service.ts @@ -3,12 +3,12 @@ import { MAX_PORTFOLIO_TOP_VOLUNTEERS, type AnalyticsRange, type BreakdownRow, + type BroadcastChannel, type EventAnalyticsBroadcastsResponse, type EventAnalyticsCheckinsResponse, type EventAnalyticsOverviewResponse, type EventAnalyticsRegistrationsResponse, type EventAnalyticsSourcesResponse, - type FunnelStep, type HostAnalyticsSummaryResponse, type HostedEventsAnalyticsResponse, type Panel, @@ -31,7 +31,6 @@ import { type DerivedBreakdownRow, type DerivedPanel, type KeyCount, - type SeriesClosure, } from "@civfix/shared/host" import type { AnalyticsRepository, LabeledKeyCount } from "./analytics-repository.drizzle.js" import { leaderboardEntryOf } from "../volunteer-hours-service.js" @@ -39,17 +38,34 @@ import type { MetricRow, MetricsRepository } from "./metrics-repository.drizzle. import { hostAnalyticsCacheKey, type HostAnalyticsCache } from "./host-analytics-cache.js" import { eventDayKey } from "./event-day.js" import { DEFAULT_EVENT_TIME_ZONE } from "./event-fields.js" - -function shiftDayKey(day: string, deltaDays: number): string { - const [year, month, date] = day.split("-").map(Number) - const shifted = new Date( - Date.UTC(year ?? 1970, (month ?? 1) - 1, date ?? 1) + deltaDays * 86_400_000, - ) - return shifted.toISOString().slice(0, 10) -} - -export const PORTFOLIO_EVENT_LIMIT = 200 +import { + METRIC_BROADCAST_FAILED, + METRIC_BROADCAST_RECIPIENTS, + METRIC_BROADCAST_SENT, + METRIC_BROADCAST_SUPPRESSED, + METRIC_DONATION_CLICKS, + METRIC_PAGE_VIEWS, + METRIC_REGISTRATIONS, + METRIC_SOURCE, + METRIC_UNSUBSCRIBES, +} from "./event-metric-names.js" +import { + DAY_MS, + closureAllowsTotal, + emptyRate, + isoDayOf, + metricTotal, + seriesOf, + shiftDayKey, + toFunnelSteps, + toRate, + toSeries, + toSuppressedRate, +} from "./host-analytics-shaping.js" + +const PORTFOLIO_EVENT_LIMIT = 200 export const ARRIVAL_SAMPLE_LIMIT = 20_000 +const PORTFOLIO_BY_EVENT_LIMIT = 50 const RANGE_DAYS: Record = { "7d": 7, @@ -68,6 +84,8 @@ const ALL_RANGE_DAYS = 365 const EXACT_K = 1 +const BROADCAST_CHANNELS: readonly BroadcastChannel[] = ["inapp", "push", "email", "sms"] + export interface AnalyticsServiceDeps { analytics: AnalyticsRepository metrics: MetricsRepository @@ -121,8 +139,8 @@ export function makeAnalyticsService(deps: AnalyticsServiceDeps): AnalyticsServi function utcRangeWindow(days: number | null): { from: string; to: string } { const to = now() const span = days ?? ALL_RANGE_DAYS - const from = new Date(to.getTime() - (span - 1) * 86_400_000) - return { from: from.toISOString().slice(0, 10), to: to.toISOString().slice(0, 10) } + const from = new Date(to.getTime() - (span - 1) * DAY_MS) + return { from: isoDayOf(from), to: isoDayOf(to) } } function eventRangeWindow(days: number | null, timezone: string): { from: string; to: string } { @@ -159,12 +177,17 @@ export function makeAnalyticsService(deps: AnalyticsServiceDeps): AnalyticsServi const window = eventRangeWindow(RANGE_DAYS[range], timezone) const [kpis, metricRows, registrationDays] = await Promise.all([ deps.analytics.eventKpis(cleanupId), - deps.metrics.read(cleanupId, ["page_views", "donation_clicks"], window.from, window.to), + deps.metrics.read( + cleanupId, + [METRIC_PAGE_VIEWS, METRIC_DONATION_CLICKS], + window.from, + window.to, + ), deps.analytics.registrationsByDay(cleanupId, timezone, window.from, window.to), ]) const registeredPublishable = closureAllowsTotal(closureOf(registrationDays, window)) - const pageViews = metricTotal(metricRows, "page_views") - const donationClicks = metricTotal(metricRows, "donation_clicks") + const pageViews = metricTotal(metricRows, METRIC_PAGE_VIEWS) + const donationClicks = metricTotal(metricRows, METRIC_DONATION_CLICKS) const steps: KeyCount[] = [ ...(pageViews === null ? [] : [{ key: "page_views", count: pageViews }]), { key: "registered", count: kpis.registered }, @@ -266,39 +289,33 @@ export function makeAnalyticsService(deps: AnalyticsServiceDeps): AnalyticsServi deps.metrics.read( cleanupId, [ - "broadcast_recipients", - "broadcast_sent", - "broadcast_failed", - "broadcast_suppressed", - "unsubscribes", + METRIC_BROADCAST_RECIPIENTS, + METRIC_BROADCAST_SENT, + METRIC_BROADCAST_FAILED, + METRIC_BROADCAST_SUPPRESSED, + METRIC_UNSUBSCRIBES, ], window.from, window.to, ), deps.analytics.broadcastsSent(cleanupId, timezone, window.from, window.to), ]) - const recipients = sumMetric(rows, "broadcast_recipients") - const channels: Array<"inapp" | "push" | "email" | "sms"> = [ - "inapp", - "push", - "email", - "sms", - ] - const sentByChannel = channelColumn(rows, "broadcast_sent", channels) - const failedByChannel = channelColumn(rows, "broadcast_failed", channels) - const suppressedByChannel = channelColumn(rows, "broadcast_suppressed", channels) + const recipients = sumMetric(rows, METRIC_BROADCAST_RECIPIENTS) + const sentByChannel = channelColumn(rows, METRIC_BROADCAST_SENT) + const failedByChannel = channelColumn(rows, METRIC_BROADCAST_FAILED) + const suppressedByChannel = channelColumn(rows, METRIC_BROADCAST_SUPPRESSED) return { ...envelope(range), broadcastsSent, recipients: suppressCount(recipients).value, - unsubscribes: suppressCount(sumMetric(rows, "unsubscribes")).value, - byChannel: channels.map((channel) => ({ + unsubscribes: suppressCount(sumMetric(rows, METRIC_UNSUBSCRIBES)).value, + byChannel: BROADCAST_CHANNELS.map((channel) => ({ channel, sent: sentByChannel.get(channel) ?? null, failed: failedByChannel.get(channel) ?? null, suppressed: suppressedByChannel.get(channel) ?? null, })), - series: toSeries(dailySeries(seriesOf(rows, "broadcast_sent"), window).points), + series: toSeries(dailySeries(seriesOf(rows, METRIC_BROADCAST_SENT), window).points), } }) }, @@ -309,20 +326,20 @@ export function makeAnalyticsService(deps: AnalyticsServiceDeps): AnalyticsServi const window = eventRangeWindow(RANGE_DAYS[range], timezone) const rows = await deps.metrics.read( cleanupId, - ["page_views", "source", "donation_clicks"], + [METRIC_PAGE_VIEWS, METRIC_SOURCE, METRIC_DONATION_CLICKS], window.from, window.to, ) const byBucket = new Map() for (const row of rows) { - if (row.metric !== "source") continue + if (row.metric !== METRIC_SOURCE) continue byBucket.set(row.bucket, (byBucket.get(row.bucket) ?? 0) + row.value) } return { ...envelope(range), - pageViews: toSeries(dailySeries(seriesOf(rows, "page_views"), window).points), + pageViews: toSeries(dailySeries(seriesOf(rows, METRIC_PAGE_VIEWS), window).points), bySource: toPanel(breakdown([...byBucket].map(([key, count]) => ({ key, count })))), - donationClicks: suppressCount(sumMetric(rows, "donation_clicks")).value, + donationClicks: suppressCount(sumMetric(rows, METRIC_DONATION_CLICKS)).value, } }) }, @@ -338,9 +355,9 @@ export function makeAnalyticsService(deps: AnalyticsServiceDeps): AnalyticsServi ) const [totals, byEvent, dayTime, metricRows, hours, topVolunteers] = await Promise.all([ deps.analytics.portfolioTotals(cleanupIds), - deps.analytics.portfolioByEvent(cleanupIds, 50), + deps.analytics.portfolioByEvent(cleanupIds, PORTFOLIO_BY_EVENT_LIMIT), deps.analytics.portfolioDayTime(cleanupIds), - deps.metrics.readMany(cleanupIds, ["registrations"], window.from, window.to), + deps.metrics.readMany(cleanupIds, [METRIC_REGISTRATIONS], window.from, window.to), deps.analytics.hoursTotals(cleanupIds), deps.analytics.topVolunteers(cleanupIds, MAX_PORTFOLIO_TOP_VOLUNTEERS), ]) @@ -357,7 +374,7 @@ export function makeAnalyticsService(deps: AnalyticsServiceDeps): AnalyticsServi totalHours: round2(hours.credited), volunteersCredited: hours.volunteersCredited, topVolunteers: topVolunteers.map((row, index) => leaderboardEntryOf(row, index + 1)), - series: exactSeries(seriesOf(metricRows, "registrations"), window), + series: exactSeries(seriesOf(metricRows, METRIC_REGISTRATIONS), window), byEvent: exactPanel(byEvent), repeatAttendance: exactRate(totals.repeatAttendees, totals.uniqueAttendees), averageCheckInRate: exactRate(totals.checkIns, totals.registrations), @@ -387,7 +404,7 @@ export function makeAnalyticsService(deps: AnalyticsServiceDeps): AnalyticsServi deps.analytics.activityTotals(cleanupIds, bounds.from, bounds.to), deps.analytics.heldEventTotals(cleanupIds, bounds.from, bounds.to), deps.analytics.signupsByDayAcross(cleanupIds, window.from, window.to), - deps.metrics.readMany(cleanupIds, ["donation_clicks"], window.from, window.to), + deps.metrics.readMany(cleanupIds, [METRIC_DONATION_CLICKS], window.from, window.to), ]) return { ...envelope, @@ -399,7 +416,7 @@ export function makeAnalyticsService(deps: AnalyticsServiceDeps): AnalyticsServi reportsLinked: activity.reportsLinked, reportsResolved: activity.reportsResolved, postsCreated: activity.postsCreated, - donationClicks: sumMetric(metricRows, "donation_clicks"), + donationClicks: sumMetric(metricRows, METRIC_DONATION_CLICKS), }, eventsHeld: { count: held.events, @@ -420,7 +437,7 @@ export function makeAnalyticsService(deps: AnalyticsServiceDeps): AnalyticsServi function dayBounds(window: DayRange): { from: Date; to: Date } { const from = new Date(`${window.from}T00:00:00.000Z`) - const to = new Date(Date.parse(`${window.to}T00:00:00.000Z`) + 86_400_000) + const to = new Date(Date.parse(`${window.to}T00:00:00.000Z`) + DAY_MS) return { from, to } } @@ -508,17 +525,12 @@ function closureOf(points: readonly DayCount[], window: { from: string; to: stri return seriesClosure(points, window, { suppressPoints: true }) } -function closureAllowsTotal(closure: SeriesClosure): boolean { - return closure.panelSuppressed || closure.totalPublishable -} - -function channelColumn( - rows: readonly MetricRow[], - metric: string, - channels: readonly string[], -): Map { +function channelColumn(rows: readonly MetricRow[], metric: string): Map { const panel = breakdown( - channels.map((channel) => ({ key: channel, count: sumMetric(rows, metric, channel) })), + BROADCAST_CHANNELS.map((channel) => ({ + key: channel, + count: sumMetric(rows, metric, channel), + })), ) return new Map(panel.rows.map((row) => [row.key, row.value])) } @@ -529,31 +541,6 @@ function sumMetric(rows: readonly MetricRow[], metric: string, bucket?: string): .reduce((acc, row) => acc + row.value, 0) } -function metricTotal(rows: readonly MetricRow[], metric: string): number | null { - const matching = rows.filter((row) => row.metric === metric) - if (matching.length === 0) return null - return matching.reduce((acc, row) => acc + row.value, 0) -} - -function seriesOf(rows: readonly MetricRow[], metric: string): DayCount[] { - const byDay = new Map() - for (const row of rows) { - if (row.metric !== metric) continue - byDay.set(row.day, (byDay.get(row.day) ?? 0) + row.value) - } - return [...byDay].map(([day, count]) => ({ day, count })) -} - -function toSeries( - points: readonly { day: string; value: number | null; suppressed: boolean }[], -): SeriesPoint[] { - return points.map((point) => ({ - day: point.day, - value: point.value, - suppressed: point.suppressed, - })) -} - function toPanel(panel: DerivedPanel): Panel { return { panelSuppressed: panel.panelSuppressed, @@ -567,36 +554,3 @@ function toPanel(panel: DerivedPanel): Panel { ), } } - -function toFunnelSteps( - panel: DerivedPanel<{ key: string; value: number | null; suppressed: boolean }>, -): FunnelStep[] { - return panel.rows.map((row) => ({ - step: row.key, - label: row.key, - value: row.value, - suppressed: row.suppressed, - })) -} - -function toRate(numerator: number, denominator: number): SuppressedRate { - const ratio = suppressRate(numerator, denominator) - return toSuppressedRate(ratio, numerator, denominator) -} - -function toSuppressedRate( - ratio: { value: number | null; suppressed: boolean }, - numerator: number, - denominator: number, -): SuppressedRate { - return { - value: ratio.value, - numerator: ratio.suppressed ? null : numerator, - denominator: ratio.suppressed ? null : denominator, - suppressed: ratio.suppressed, - } -} - -function emptyRate(): SuppressedRate { - return { value: null, numerator: null, denominator: null, suppressed: true } -} diff --git a/services/api/src/services/host/announcement-service.ts b/services/api/src/services/host/announcement-service.ts index dc8a26e8..39324d09 100644 --- a/services/api/src/services/host/announcement-service.ts +++ b/services/api/src/services/host/announcement-service.ts @@ -18,15 +18,15 @@ import type { FastifyBaseLogger } from "fastify" import { paginateKeyset, parseKeysetCursor } from "../../db/cursor-helpers.js" import type { AnnouncementCap, BroadcastRepository } from "./broadcast-repository.js" import type { BroadcastRecord } from "./broadcast-types.js" -import { broadcastLinkWarnings } from "./broadcast-render.js" -import { announcementPath } from "./broadcast-pipeline.js" +import { ANNOUNCEMENT_VISIBLE_STATUSES } from "./broadcast-types.js" +import { announcementPath, broadcastLinkWarnings } from "./broadcast-render.js" import type { BroadcastConfig, BroadcastService } from "./broadcast-service.js" import type { AnnouncementIdentityRepository } from "./announcement-repository.drizzle.js" -export const ANNOUNCEMENT_DEFAULT_LIMIT = 20 -export const ANNOUNCEMENT_MAX_LIMIT = 50 -export const ANNOUNCEMENT_WINDOW_MS = 24 * 60 * 60 * 1000 -export const ANNOUNCEMENT_CTA_LABEL = "View announcement" +const ANNOUNCEMENT_DEFAULT_LIMIT = 20 +const ANNOUNCEMENT_MAX_LIMIT = 50 +const ANNOUNCEMENT_WINDOW_MS = 24 * 60 * 60 * 1000 +const ANNOUNCEMENT_CTA_LABEL = "View announcement" export interface AnnouncementServiceDeps { repo: BroadcastRepository @@ -59,7 +59,7 @@ export interface AnnouncementService { ): Promise } -export function announcementRateLimited(): AppError { +function announcementRateLimited(): AppError { return AppError.rateLimited( "Announcement limit reached for today. Use the group chat for ongoing discussion.", ) @@ -226,7 +226,7 @@ export function makeAnnouncementService(deps: AnnouncementServiceDeps): Announce async get(cleanupId, announcementId, projection) { const record = await repo.findForEvent(cleanupId, announcementId) if (record === null || record.kind !== ANNOUNCEMENT_BROADCAST_KIND) throw notFound() - if (!projection.host && record.status !== "sending" && record.status !== "sent") { + if (!projection.host && !ANNOUNCEMENT_VISIBLE_STATUSES.includes(record.status)) { throw notFound() } return hydrateOne(cleanupId, record, projection) diff --git a/services/api/src/services/host/broadcast-audience-sql.ts b/services/api/src/services/host/broadcast-audience-sql.ts index 786f4fde..02351d34 100644 --- a/services/api/src/services/host/broadcast-audience-sql.ts +++ b/services/api/src/services/host/broadcast-audience-sql.ts @@ -2,7 +2,8 @@ import type { BroadcastKind, BroadcastSegment } from "@civfix/shared" import type { Queryable } from "../../db/client.js" import { CRITICAL_BROADCAST_KINDS, HOST_COMPOSED_BROADCAST_KINDS } from "./broadcast-types.js" -export const AUDIENCE_PAGE_SIZE = 1000 +// Sorts before every real id, so a first page starts at the beginning of the keyset. +const FIRST_UUID = "00000000-0000-0000-0000-000000000000" export interface AudienceQuery { cleanupId: string @@ -78,7 +79,7 @@ function guestSuppressionTail(sql: Queryable, cleanupId: string, kind: Broadcast function memberQuery(sql: Queryable, q: AudienceQuery): Promise { const tail = memberSuppressionTail(sql, q.cleanupId, q.kind) - const after = q.after ?? "00000000-0000-0000-0000-000000000000" + const after = q.after ?? FIRST_UUID switch (q.segment.kind) { case "all_registered": return sql` @@ -150,7 +151,7 @@ function memberQuery(sql: Queryable, q: AudienceQuery): Promise { function guestQuery(sql: Queryable, q: AudienceQuery): Promise { const tail = guestSuppressionTail(sql, q.cleanupId, q.kind) - const after = q.after ?? "00000000-0000-0000-0000-000000000000" + const after = q.after ?? FIRST_UUID switch (q.segment.kind) { case "all_registered": case "guests_only": diff --git a/services/api/src/services/host/broadcast-audience.ts b/services/api/src/services/host/broadcast-audience.ts new file mode 100644 index 00000000..852bcd63 --- /dev/null +++ b/services/api/src/services/host/broadcast-audience.ts @@ -0,0 +1,44 @@ +import type { BroadcastKind, BroadcastSegment } from "@civfix/shared" +import type { BroadcastRepository } from "./broadcast-repository.js" + +export const AUDIENCE_PAGE_SIZE = 1000 +const AUDIENCE_MAX_PAGES = 100 +// Sorts after every real id, so a side that is already exhausted pages to an empty result. +const LAST_UUID = "ffffffff-ffff-ffff-ffff-ffffffffffff" + +export interface AudienceScope { + cleanupId: string + segment: BroadcastSegment + kind: BroadcastKind +} + +export interface AudiencePage { + members: string[] + guests: string[] +} + +export async function* audiencePages( + repo: Pick, + scope: AudienceScope, +): AsyncGenerator { + let afterMember: string | null = null + let afterGuest: string | null = null + let memberDone = false + let guestDone = false + for (let page = 0; page < AUDIENCE_MAX_PAGES; page += 1) { + const result: AudiencePage = await repo.audiencePage({ + cleanupId: scope.cleanupId, + segment: scope.segment, + kind: scope.kind, + afterMember: memberDone ? LAST_UUID : afterMember, + afterGuest: guestDone ? LAST_UUID : afterGuest, + limit: AUDIENCE_PAGE_SIZE, + }) + if (result.members.length < AUDIENCE_PAGE_SIZE) memberDone = true + else afterMember = result.members.at(-1) ?? afterMember + if (result.guests.length < AUDIENCE_PAGE_SIZE) guestDone = true + else afterGuest = result.guests.at(-1) ?? afterGuest + yield result + if (memberDone && guestDone) return + } +} diff --git a/services/api/src/services/host/broadcast-capability-token.ts b/services/api/src/services/host/broadcast-capability-token.ts index 74fed900..7f977656 100644 --- a/services/api/src/services/host/broadcast-capability-token.ts +++ b/services/api/src/services/host/broadcast-capability-token.ts @@ -1,7 +1,11 @@ import { createHmac } from "node:crypto" import { constantTimeStringEqual } from "../../auth/crypto.js" -export const UNSUBSCRIBE_TOKEN_VERSION = "u1" +const UNSUBSCRIBE_TOKEN_VERSION = "u1" +const TOKEN_PAYLOAD_VERSION = 1 +const TOKEN_PART_COUNT = 3 +const MS_PER_SECOND = 1000 +const MS_PER_DAY = 24 * 60 * 60 * MS_PER_SECOND export const UNSUBSCRIBE_TOKEN_TTL_DAYS = 400 @@ -29,11 +33,11 @@ export function mintUnsubscribeToken( signingKey: string, ): string { const payload: TokenPayload = { - v: 1, + v: TOKEN_PAYLOAD_VERSION, s: capability.subjectKind, i: capability.subjectId, e: capability.cleanupId, - x: Math.floor(capability.expiresAtMs / 1000), + x: Math.floor(capability.expiresAtMs / MS_PER_SECOND), } const body = encode(JSON.stringify(payload)) return `${UNSUBSCRIBE_TOKEN_VERSION}.${body}.${sign(body, signingKey)}` @@ -45,7 +49,7 @@ export function verifyUnsubscribeToken( nowMs: number, ): UnsubscribeCapability | null { const parts = token.split(".") - if (parts.length !== 3) return null + if (parts.length !== TOKEN_PART_COUNT) return null const [version, body, signature] = parts as [string, string, string] if (version !== UNSUBSCRIBE_TOKEN_VERSION) return null if (body.length === 0 || signature.length === 0) return null @@ -60,12 +64,12 @@ export function verifyUnsubscribeToken( } if (typeof parsed !== "object" || parsed === null) return null const p = parsed as Record - if (p.v !== 1) return null + if (p.v !== TOKEN_PAYLOAD_VERSION) return null if (typeof p.s !== "string" || !(SUBJECT_KINDS as readonly string[]).includes(p.s)) return null if (typeof p.i !== "string" || p.i.length === 0) return null if (typeof p.e !== "string" || p.e.length === 0) return null if (typeof p.x !== "number" || !Number.isFinite(p.x)) return null - const expiresAtMs = p.x * 1000 + const expiresAtMs = p.x * MS_PER_SECOND if (expiresAtMs <= nowMs) return null return { @@ -77,7 +81,7 @@ export function verifyUnsubscribeToken( } export function unsubscribeExpiryFrom(sentAtMs: number): number { - return sentAtMs + UNSUBSCRIBE_TOKEN_TTL_DAYS * 24 * 60 * 60 * 1000 + return sentAtMs + UNSUBSCRIBE_TOKEN_TTL_DAYS * MS_PER_DAY } function encode(value: string): string { diff --git a/services/api/src/services/host/broadcast-email-sender.ts b/services/api/src/services/host/broadcast-email-sender.ts new file mode 100644 index 00000000..ac978380 --- /dev/null +++ b/services/api/src/services/host/broadcast-email-sender.ts @@ -0,0 +1,400 @@ +import type { BroadcastKind, DeliveryFailureKind } from "@civfix/shared" +import type { BroadcastVarValues } from "@civfix/shared/host" +import type { Mailer } from "@civfix/shared/interfaces" +import type { FastifyBaseLogger } from "fastify" +import type { CacheClient } from "../../auth/cache.js" +import { mailFailure } from "../../adapters/mail-failure.js" +import { mapWithLimit } from "../media-presign.js" +import type { BroadcastRepository } from "./broadcast-repository.js" +import type { + BroadcastRecord, + DeliveryClaim, + DeliveryOutcome, + EventBroadcastContext, + GuestContact, + MemberContact, +} from "./broadcast-types.js" +import { CRITICAL_BROADCAST_KINDS, MAX_DELIVERY_ATTEMPTS } from "./broadcast-types.js" +import { + broadcastContentOf, + eventTemplateVars, + renderBroadcast, + templateTextOf, + usesVar, + type RenderedBroadcast, +} from "./broadcast-render.js" +import { mintUnsubscribeToken, unsubscribeExpiryFrom } from "./broadcast-capability-token.js" +import { emailHashOf, type BroadcastConfig } from "./broadcast-service.js" + +const DEDUPE_TTL_SEC = 24 * 60 * 60 +const MS_PER_SECOND = 1000 +const AUTOMATED_KINDS: ReadonlySet = new Set([ + "confirmation", + "waitlist_promoted", + "reminder", + "event_updated", + "event_cancelled", + "thank_you", +]) + +export interface BroadcastEmailSenderDeps { + repo: BroadcastRepository + config: Pick< + BroadcastConfig, + | "apiBaseUrl" + | "emailConcurrency" + | "emailRatePerSec" + | "linkAllowedHosts" + | "mailFromEvents" + | "unsubscribeSigningKey" + | "webBaseUrl" + > + mailer: Mailer + cache: CacheClient + mailDomain: string + logger?: Pick + now: () => Date +} + +export class ChunkAuthAbort extends Error { + override readonly cause: unknown + readonly releasedIds: readonly string[] + + constructor(cause: unknown, releasedIds: readonly string[]) { + super("broadcast chunk aborted: the SMTP server rejected the sender") + this.name = "ChunkAuthAbort" + this.cause = cause + this.releasedIds = releasedIds + } +} + +class TokenBucket { + private tokens: number + private lastRefillMs: number + + constructor( + private readonly ratePerSec: number, + private readonly now: () => number = () => Date.now(), + private readonly sleep: (ms: number) => Promise = (ms) => + new Promise((resolve) => setTimeout(resolve, ms)), + ) { + this.tokens = ratePerSec + this.lastRefillMs = now() + } + + async take(): Promise { + for (;;) { + const at = this.now() + const elapsed = at - this.lastRefillMs + if (elapsed > 0) { + this.tokens = Math.min( + this.ratePerSec, + this.tokens + (elapsed / MS_PER_SECOND) * this.ratePerSec, + ) + this.lastRefillMs = at + } + if (this.tokens >= 1) { + this.tokens -= 1 + return + } + await this.sleep(Math.ceil(((1 - this.tokens) / this.ratePerSec) * MS_PER_SECOND)) + } + } +} + +interface EmailChunk { + record: BroadcastRecord + event: EventBroadcastContext + members: Map + guests: Map + ticketTypes: Map + addressHashes: Map + suppressedHashes: Set + bucket: TokenBucket + critical: boolean + replyTo: string | null + sentAtMs: number + shared: BroadcastVarValues + outcomes: DeliveryOutcome[] + releasedIds: string[] + abortError: unknown +} + +function addressKey(broadcastId: string, hash: string): string { + return `bcast:addr:${broadcastId}:${hash}` +} + +function contactOf( + chunk: EmailChunk, + claim: DeliveryClaim, +): MemberContact | GuestContact | undefined { + return claim.userId !== null + ? chunk.members.get(claim.userId) + : chunk.guests.get(claim.guestId as string) +} + +function firstNameOf(chunk: EmailChunk, claim: DeliveryClaim): string { + return claim.userId !== null + ? (chunk.members.get(claim.userId)?.firstName ?? "") + : (chunk.guests.get(claim.guestId as string)?.name ?? "") +} + +function listUnsubscribeHeaders(oneClickUrl: string, kind: BroadcastKind): Record { + const headers: Record = { + "List-Unsubscribe": `<${oneClickUrl}>`, + "List-Unsubscribe-Post": "List-Unsubscribe=One-Click", + } + if (AUTOMATED_KINDS.has(kind)) headers["Auto-Submitted"] = "auto-generated" + return headers +} + +export function makeBroadcastEmailSender(deps: BroadcastEmailSenderDeps) { + const { repo, config } = deps + + async function releaseAddress(broadcastId: string, hash: string): Promise { + const key = addressKey(broadcastId, hash) + try { + await deps.cache.del(key) + await deps.cache.del(`${key}:owner`) + } catch (err) { + deps.logger?.warn( + { err }, + "broadcast: address dedupe release failed; a retry of this row may be skipped", + ) + } + } + + async function claimAddress( + broadcastId: string, + hash: string, + deliveryId: string, + ): Promise { + const key = addressKey(broadcastId, hash) + try { + const held = await deps.cache.incr(key, DEDUPE_TTL_SEC) + if (held === 1) { + await deps.cache.set(`${key}:owner`, deliveryId, DEDUPE_TTL_SEC) + return true + } + return (await deps.cache.get(`${key}:owner`)) === deliveryId + } catch (err) { + deps.logger?.warn( + { err }, + "broadcast: address dedupe unavailable; sending anyway (a duplicate beats a silence)", + ) + return true + } + } + + function unsubscribeLinks( + chunk: EmailChunk, + claim: DeliveryClaim, + subjectId: string, + ): { unsubscribeUrl: string; oneClickUrl: string } { + const token = encodeURIComponent( + mintUnsubscribeToken( + { + subjectKind: claim.userId !== null ? "user" : "guest", + subjectId, + cleanupId: chunk.record.cleanupId, + expiresAtMs: unsubscribeExpiryFrom(chunk.sentAtMs), + }, + config.unsubscribeSigningKey, + ), + ) + return { + unsubscribeUrl: `${config.webBaseUrl}/unsubscribe?t=${token}`, + oneClickUrl: `${config.apiBaseUrl}/v1/broadcasts/unsubscribe?t=${token}`, + } + } + + function renderFor( + chunk: EmailChunk, + claim: DeliveryClaim, + subjectId: string, + unsubscribeUrl: string, + ): RenderedBroadcast { + return renderBroadcast(broadcastContentOf(chunk.record), { + eventTitle: chunk.event.title, + vars: { + ...chunk.shared, + first_name: firstNameOf(chunk, claim), + ticket_type: chunk.ticketTypes.get(subjectId) ?? "", + }, + unsubscribeUrl, + replyTo: chunk.replyTo, + critical: chunk.critical, + allowedLinkHosts: config.linkAllowedHosts, + }) + } + + function holdForRetry(chunk: EmailChunk, claim: DeliveryClaim): void { + chunk.outcomes.push({ id: claim.id, status: "pending" }) + chunk.releasedIds.push(claim.id) + } + + /** Null means the claim's outcome is already recorded and nothing may be mailed for it. */ + async function screen( + chunk: EmailChunk, + claim: DeliveryClaim, + ): Promise<{ email: string; hash: string } | null> { + const contact = contactOf(chunk, claim) + if (contact === undefined) { + chunk.outcomes.push({ + id: claim.id, + status: "suppressed", + suppressionReason: claim.userId !== null ? "deleted_user" : "contact_scrubbed", + }) + return null + } + const email = contact.email + if (email === null || email.length === 0) { + chunk.outcomes.push({ id: claim.id, status: "suppressed", suppressionReason: "no_contact" }) + return null + } + const hash = chunk.addressHashes.get(claim.id) ?? emailHashOf(email) + if (chunk.suppressedHashes.has(hash)) { + chunk.outcomes.push({ + id: claim.id, + status: "suppressed", + suppressionReason: "bounce_suppressed", + }) + return null + } + if (!(await claimAddress(chunk.record.id, hash, claim.id))) { + chunk.outcomes.push({ id: claim.id, status: "skipped" }) + return null + } + return { email, hash } + } + + async function recordSendFailure( + chunk: EmailChunk, + claim: DeliveryClaim, + hash: string, + err: unknown, + ): Promise { + const failure = mailFailure(err) + deps.logger?.warn( + { + evt: "broadcast.failed", + deliveryId: claim.id, + failureKind: failure.kind, + senderRejected: failure.senderRejected, + }, + "broadcast email send failed", + ) + if (failure.kind === "auth") { + chunk.abortError = err + await releaseAddress(chunk.record.id, hash) + holdForRetry(chunk, claim) + return + } + if (failure.kind === "permanent") { + await repo.suppressEmail(hash, "hard_bounce").catch((suppressErr: unknown) => { + deps.logger?.warn( + { err: suppressErr, deliveryId: claim.id }, + "broadcast: hard-bounce suppression write failed; the address may be mailed again", + ) + }) + chunk.outcomes.push({ id: claim.id, status: "failed", failureKind: "permanent" }) + return + } + if (failure.kind === "transient" && claim.attempts < MAX_DELIVERY_ATTEMPTS) { + await releaseAddress(chunk.record.id, hash) + chunk.outcomes.push({ id: claim.id, status: "pending" }) + return + } + chunk.outcomes.push({ + id: claim.id, + status: "failed", + failureKind: failure.kind as DeliveryFailureKind, + }) + } + + async function deliver(chunk: EmailChunk, claim: DeliveryClaim): Promise { + if (chunk.abortError !== null) { + holdForRetry(chunk, claim) + return + } + const address = await screen(chunk, claim) + if (address === null) return + const subjectId = (claim.userId ?? claim.guestId) as string + const { unsubscribeUrl, oneClickUrl } = unsubscribeLinks(chunk, claim, subjectId) + const rendered = renderFor(chunk, claim, subjectId, unsubscribeUrl) + const headers = listUnsubscribeHeaders(oneClickUrl, chunk.record.kind) + + await chunk.bucket.take() + try { + const sent = await deps.mailer.sendOutbound({ + from: config.mailFromEvents, + to: address.email, + subject: rendered.subject, + text: rendered.text, + html: rendered.html, + messageId: ``, + headers, + ...(chunk.replyTo !== null ? { replyTo: chunk.replyTo } : {}), + }) + chunk.outcomes.push({ + id: claim.id, + status: "sent", + sentAt: deps.now(), + ...(sent.messageId !== undefined ? { providerMessageId: sent.messageId } : {}), + }) + } catch (err) { + await recordSendFailure(chunk, claim, address.hash, err) + } + } + + async function runEmail( + record: BroadcastRecord, + event: EventBroadcastContext, + claims: readonly DeliveryClaim[], + outcomes: DeliveryOutcome[], + ): Promise { + const relevant = claims.filter((c) => c.channel === "email") + if (relevant.length === 0) return + const memberIds = relevant.filter((c) => c.userId !== null).map((c) => c.userId as string) + const guestIds = relevant.filter((c) => c.guestId !== null).map((c) => c.guestId as string) + const [members, guests, ticketTypes] = await Promise.all([ + repo.memberContacts(memberIds), + repo.guestContacts(guestIds), + usesVar(templateTextOf(record), "ticket_type") + ? repo.ticketTypeNames({ cleanupId: record.cleanupId, userIds: memberIds, guestIds }) + : Promise.resolve(new Map()), + ]) + const addressHashes = new Map() + for (const claim of relevant) { + const contact = + claim.userId !== null ? members.get(claim.userId) : guests.get(claim.guestId as string) + const email = contact?.email + if (email == null || email.length === 0) continue + addressHashes.set(claim.id, emailHashOf(email)) + } + const suppressedHashes = await repo.suppressedEmailHashes([...addressHashes.values()]) + + const chunk: EmailChunk = { + record, + event, + members, + guests, + ticketTypes, + addressHashes, + suppressedHashes, + bucket: new TokenBucket(config.emailRatePerSec), + critical: CRITICAL_BROADCAST_KINDS.has(record.kind), + replyTo: event.replyToVerified && event.replyTo ? event.replyTo : null, + sentAtMs: deps.now().getTime(), + shared: eventTemplateVars(event, config.webBaseUrl), + outcomes, + releasedIds: [], + abortError: null, + } + await mapWithLimit([...relevant], config.emailConcurrency, (claim) => deliver(chunk, claim)) + + if (chunk.abortError !== null) throw new ChunkAuthAbort(chunk.abortError, chunk.releasedIds) + } + + return { runEmail } +} diff --git a/services/api/src/services/host/broadcast-inapp-sender.ts b/services/api/src/services/host/broadcast-inapp-sender.ts new file mode 100644 index 00000000..14621fff --- /dev/null +++ b/services/api/src/services/host/broadcast-inapp-sender.ts @@ -0,0 +1,213 @@ +import type { BroadcastKind, NotificationType } from "@civfix/shared" +import type { BroadcastVarValues } from "@civfix/shared/host" +import type { FastifyBaseLogger } from "fastify" +import { isWithinQuietHours, pushGateAllows, type PushGateMode } from "../notification-helpers.js" +import type { NotificationPrefsRecord, NotificationService } from "../notification-service.js" +import type { BroadcastRepository } from "./broadcast-repository.js" +import type { + BroadcastRecord, + DeliveryClaim, + DeliveryOutcome, + EventBroadcastContext, + MemberContact, +} from "./broadcast-types.js" +import { CRITICAL_BROADCAST_KINDS, MAX_DELIVERY_ATTEMPTS } from "./broadcast-types.js" +import { + broadcastContentOf, + eventTemplateVars, + notificationLink, + renderBroadcast, + templateTextOf, + usesPerRecipientVar, + usesVar, + verifiedReplyTo, + type RenderedBroadcast, +} from "./broadcast-render.js" +import type { BroadcastConfig } from "./broadcast-service.js" + +export const KIND_NOTIFICATION_TYPE: Record = { + host_broadcast: "event_broadcast", + thank_you: "event_broadcast", + reminder: "cleanup_reminder", + event_updated: "system", + event_cancelled: "cleanup_cancelled", + confirmation: "system", + waitlist_promoted: "system", + announcement: "event_broadcast", +} + +export interface BroadcastInAppSenderDeps { + repo: BroadcastRepository + config: Pick + notifications: NotificationService + logger?: Pick + now: () => Date +} + +interface RenderGroup { + rendering: RenderedBroadcast + userIds: string[] +} + +function pushMode(record: BroadcastRecord): PushGateMode { + if (!record.channels.includes("push")) return "never" + return CRITICAL_BROADCAST_KINDS.has(record.kind) ? "always" : "auto" +} + +function pushAllowed( + pref: NotificationPrefsRecord | undefined, + type: NotificationType, + mode: PushGateMode, + at: Date, +): boolean { + if (mode === "never") return false + return ( + pref === undefined || + (pushGateAllows(type, pref, mode) && + !isWithinQuietHours(at, pref.quietStart, pref.quietEnd, pref.tz)) + ) +} + +/** One notification call per distinct rendering, so a shared message fans out in a single batch. */ +function groupByRendering(deliverable: ReadonlyMap): RenderGroup[] { + const groups = new Map() + for (const [userId, rendering] of deliverable) { + const key = JSON.stringify([rendering.inAppTitle, rendering.inAppBody]) + const group = groups.get(key) + if (group !== undefined) group.userIds.push(userId) + else groups.set(key, { rendering, userIds: [userId] }) + } + return [...groups.values()] +} + +function requeueUndelivered( + claims: readonly DeliveryClaim[], + undelivered: ReadonlySet, + outcomes: DeliveryOutcome[], +): void { + for (const claim of claims) { + if (claim.userId === null || !undelivered.has(claim.userId)) continue + const index = outcomes.findIndex((o) => o.id === claim.id) + if (index >= 0 && outcomes[index]?.status !== "sent") continue + if (index >= 0) outcomes.splice(index, 1) + outcomes.push( + claim.attempts >= MAX_DELIVERY_ATTEMPTS + ? { id: claim.id, status: "failed", failureKind: "unknown" } + : { id: claim.id, status: "pending" }, + ) + } +} + +export function makeBroadcastInAppSender(deps: BroadcastInAppSenderDeps) { + const { repo, config } = deps + + function render( + record: BroadcastRecord, + event: EventBroadcastContext, + vars: BroadcastVarValues, + ): RenderedBroadcast { + return renderBroadcast(broadcastContentOf(record), { + eventTitle: event.title, + vars, + replyTo: verifiedReplyTo(event), + critical: CRITICAL_BROADCAST_KINDS.has(record.kind), + allowedLinkHosts: config.linkAllowedHosts, + }) + } + + async function fanOut( + record: BroadcastRecord, + event: EventBroadcastContext, + type: NotificationType, + mode: PushGateMode, + deliverable: ReadonlyMap, + ): Promise<{ undelivered: Set; fanOutError: unknown }> { + const undelivered = new Set() + let fanOutError: unknown = null + for (const group of groupByRendering(deliverable)) { + try { + const { failed } = await deps.notifications.createNotificationsReportingFailures( + group.userIds, + { + type, + title: group.rendering.inAppTitle, + body: group.rendering.inAppBody, + link: notificationLink(record, event), + push: mode, + }, + ) + for (const userId of failed) undelivered.add(userId) + } catch (err) { + fanOutError = err + for (const userId of group.userIds) undelivered.add(userId) + } + } + return { undelivered, fanOutError } + } + + async function runInAppAndPush( + record: BroadcastRecord, + event: EventBroadcastContext, + claims: readonly DeliveryClaim[], + outcomes: DeliveryOutcome[], + ): Promise { + const relevant = claims.filter( + (c) => (c.channel === "inapp" || c.channel === "push") && c.userId !== null, + ) + if (relevant.length === 0) return + const userIds = [...new Set(relevant.map((c) => c.userId as string))] + const content = templateTextOf(record) + const perRecipient = usesPerRecipientVar(content) + const [contacts, prefs, ticketTypes] = await Promise.all([ + repo.memberContacts(userIds), + repo.pushPrefs(userIds), + usesVar(content, "ticket_type") + ? repo.ticketTypeNames({ cleanupId: record.cleanupId, userIds, guestIds: [] }) + : Promise.resolve(new Map()), + ]) + const shared = eventTemplateVars(event, config.webBaseUrl) + const oneRendering = perRecipient ? null : render(record, event, shared) + const type = KIND_NOTIFICATION_TYPE[record.kind] + const mode = pushMode(record) + const at = deps.now() + + const renderingFor = (userId: string, contact: MemberContact): RenderedBroadcast => + oneRendering ?? + render(record, event, { + ...shared, + first_name: contact.firstName, + ticket_type: ticketTypes.get(userId) ?? "", + }) + + const deliverable = new Map() + for (const claim of relevant) { + const userId = claim.userId as string + const contact = contacts.get(userId) + if (contact === undefined) { + outcomes.push({ id: claim.id, status: "suppressed", suppressionReason: "deleted_user" }) + continue + } + if (claim.channel === "push") { + outcomes.push( + pushAllowed(prefs.get(userId), type, mode, at) + ? { id: claim.id, status: "sent", sentAt: at } + : { id: claim.id, status: "suppressed", suppressionReason: "prefs_off" }, + ) + continue + } + deliverable.set(userId, renderingFor(userId, contact)) + outcomes.push({ id: claim.id, status: "sent", sentAt: at }) + } + + if (deliverable.size === 0) return + const { undelivered, fanOutError } = await fanOut(record, event, type, mode, deliverable) + if (undelivered.size === 0) return + deps.logger?.error( + { err: fanOutError, broadcastId: record.id, undelivered: undelivered.size }, + "broadcast: in-app fan-out failed for some recipients; only those rows return to pending", + ) + requeueUndelivered(relevant, undelivered, outcomes) + } + + return { runInAppAndPush } +} diff --git a/services/api/src/services/host/broadcast-lanes.ts b/services/api/src/services/host/broadcast-lanes.ts index a10721b7..299d2add 100644 --- a/services/api/src/services/host/broadcast-lanes.ts +++ b/services/api/src/services/host/broadcast-lanes.ts @@ -2,18 +2,46 @@ import type { BroadcastChannel } from "@civfix/shared" import type { FastifyBaseLogger } from "fastify" import type { CounterStore } from "../../abuse/counter-store.js" import type { BroadcastRepository } from "./broadcast-repository.js" -import type { EventBroadcastContext } from "./broadcast-types.js" +import type { BroadcastCreateInput, EventBroadcastContext } from "./broadcast-types.js" +import { verifiedReplyTo } from "./broadcast-render.js" import { eventWindowOf, hasEventEnded } from "../cleanup-rules.js" export const DEFAULT_REMINDER_OFFSETS_MIN = [1440, 180] as const -export const REMINDER_STALE_HOURS = 6 +const REMINDER_STALE_HOURS = 6 -export const REMINDER_SWEEP_LIMIT = 200 +const REMINDER_SWEEP_LIMIT = 200 const AUTOMATED_CHANNELS: BroadcastChannel[] = ["inapp", "push", "email"] -export const EVENT_UPDATE_WINDOW_SEC = 3600 +const MS_PER_SECOND = 1000 +const MS_PER_HOUR = 3_600_000 +const EVENT_UPDATE_WINDOW_SEC = MS_PER_HOUR / MS_PER_SECOND +// An ISO timestamp cut to "YYYY-MM-DDTHH" names the UTC hour the throttle window counts in. +const ISO_HOUR_LENGTH = 13 + +type AutomatedKind = "event_updated" | "event_cancelled" | "reminder" + +function automatedBroadcast( + event: EventBroadcastContext, + kind: AutomatedKind, + subject: string, + bodyMd: string, + startedAt: Date, +): BroadcastCreateInput { + return { + cleanupId: event.cleanupId, + createdBy: null, + kind, + subject, + bodyMd, + segment: { kind: "all_registered" }, + channels: AUTOMATED_CHANNELS, + status: "sending", + startedAt, + replyTo: verifiedReplyTo(event), + } +} export type EventUpdateVerdict = | { status: "started"; broadcastId: string } @@ -32,35 +60,13 @@ export interface BroadcastLaneDeps { export function makeBroadcastLanes(deps: BroadcastLaneDeps) { const now = deps.now ?? (() => new Date()) - async function startLane( - event: EventBroadcastContext, - kind: "event_updated", - subject: string, - bodyMd: string, - ): Promise { - const record = await deps.repo.create({ - cleanupId: event.cleanupId, - createdBy: null, - kind, - subject, - bodyMd, - segment: { kind: "all_registered" }, - channels: AUTOMATED_CHANNELS, - status: "sending", - startedAt: now(), - replyTo: event.replyToVerified ? event.replyTo : null, - }) - await deps.enqueuePlan(record.id) - return record.id - } - function secondsToWindowEnd(at: Date): number { - const nextHourMs = Math.floor(at.getTime() / 3_600_000) * 3_600_000 + 3_600_000 - return Math.max(1, Math.ceil((nextHourMs - at.getTime()) / 1000)) + const nextHourMs = Math.floor(at.getTime() / MS_PER_HOUR) * MS_PER_HOUR + MS_PER_HOUR + return Math.max(1, Math.ceil((nextHourMs - at.getTime()) / MS_PER_SECOND)) } async function reserveEventUpdateSlot(cleanupId: string): Promise { - const hourKey = now().toISOString().slice(0, 13) + const hourKey = now().toISOString().slice(0, ISO_HOUR_LENGTH) try { const used = await deps.counters.incr( `bcast:evupd:${cleanupId}:${hourKey}`, @@ -113,13 +119,17 @@ export function makeBroadcastLanes(deps: BroadcastLaneDeps) { if (!(await reserveEventUpdateSlot(cleanupId))) { return { status: "throttled", retryAfterSec: secondsToWindowEnd(now()) } } - const broadcastId = await startLane( - event, - "event_updated", - `${event.title} has new details`, - `The details for **${event.title}** have changed. Open the event to see what is different and confirm you can still make it.`, + const record = await deps.repo.create( + automatedBroadcast( + event, + "event_updated", + `${event.title} has new details`, + `The details for **${event.title}** have changed. Open the event to see what is different and confirm you can still make it.`, + now(), + ), ) - return { status: "started", broadcastId } + await deps.enqueuePlan(record.id) + return { status: "started", broadcastId: record.id } }, async eventCancelled(cleanupId: string, reason: string | null): Promise { @@ -129,18 +139,15 @@ export function makeBroadcastLanes(deps: BroadcastLaneDeps) { reason !== null && reason.trim().length > 0 ? `**${event.title}** has been cancelled by the organizer.\n\nReason: ${reason.trim()}` : `**${event.title}** has been cancelled by the organizer.` - const record = await deps.repo.createIfAbsent({ - cleanupId: event.cleanupId, - createdBy: null, - kind: "event_cancelled", - subject: `${event.title} has been cancelled`, - bodyMd: body, - segment: { kind: "all_registered" }, - channels: AUTOMATED_CHANNELS, - status: "sending", - startedAt: now(), - replyTo: event.replyToVerified ? event.replyTo : null, - }) + const record = await deps.repo.createIfAbsent( + automatedBroadcast( + event, + "event_cancelled", + `${event.title} has been cancelled`, + body, + now(), + ), + ) if (record === null) { await replanUnplannedCancellation(cleanupId) return null @@ -153,7 +160,7 @@ export function makeBroadcastLanes(deps: BroadcastLaneDeps) { const at = now() const due = await deps.repo.listDueReminders({ now: at, - staleAfter: new Date(at.getTime() - REMINDER_STALE_HOURS * 3_600_000), + staleAfter: new Date(at.getTime() - REMINDER_STALE_HOURS * MS_PER_HOUR), defaultOffsets: DEFAULT_REMINDER_OFFSETS_MIN, limit: REMINDER_SWEEP_LIMIT, }) @@ -162,17 +169,14 @@ export function makeBroadcastLanes(deps: BroadcastLaneDeps) { const event = await deps.repo.eventContext(reminder.cleanupId) if (event === null) continue const record = await deps.repo.createIfAbsent({ - cleanupId: reminder.cleanupId, - createdBy: null, - kind: "reminder", + ...automatedBroadcast( + event, + "reminder", + `Reminder: ${event.title}`, + `**${event.title}** is coming up. Check the event page for the meeting point and what to bring.`, + at, + ), reminderOffsetMin: reminder.offsetMin, - subject: `Reminder: ${event.title}`, - bodyMd: `**${event.title}** is coming up. Check the event page for the meeting point and what to bring.`, - segment: { kind: "all_registered" }, - channels: AUTOMATED_CHANNELS, - status: "sending", - startedAt: at, - replyTo: event.replyToVerified ? event.replyTo : null, }) if (record === null) continue created += 1 diff --git a/services/api/src/services/host/broadcast-lifecycle.ts b/services/api/src/services/host/broadcast-lifecycle.ts new file mode 100644 index 00000000..b5be8d90 --- /dev/null +++ b/services/api/src/services/host/broadcast-lifecycle.ts @@ -0,0 +1,115 @@ +import type { FastifyBaseLogger } from "fastify" +import type { AdminAuditAction } from "../admin/audit.js" +import type { InsightsInvalidator } from "./host-analytics-cache.js" +import type { BroadcastRepository } from "./broadcast-repository.js" +import type { BroadcastRecord, EventBroadcastContext } from "./broadcast-types.js" +import { HOST_COMPOSED_BROADCAST_KINDS } from "./broadcast-types.js" +import type { BroadcastConfig } from "./broadcast-service.js" + +const ORG_SUSPENDED_REASON = "org_suspended" +const KILLED_AUDIT_ACTION: AdminAuditAction = "event.broadcast_killed" + +export interface BroadcastLifecycleDeps { + repo: BroadcastRepository + config: Pick + insights: InsightsInvalidator + audit: ( + action: AdminAuditAction, + actorId: string | null, + target: string, + meta: Record, + ) => Promise + logger?: Pick + now: () => Date +} + +export function makeBroadcastLifecycle(deps: BroadcastLifecycleDeps) { + const { repo, config, insights } = deps + + async function killed(record: BroadcastRecord): Promise { + if (config.killSwitch) return true + if (record.createdBy === null) return false + try { + const state = await repo.hostMessagingState(record.createdBy) + return state === null || state.suspended + } catch (err) { + deps.logger?.warn( + { err, broadcastId: record.id }, + "broadcast: kill-switch read failed; treating as suspended (fail closed)", + ) + return true + } + } + + async function killBroadcast(record: BroadcastRecord, reason: string): Promise { + const suppressed = await repo.suppressRemaining(record.id, "kill_switch") + await repo.transition(record.id, ["draft", "scheduled", "sending"], "cancelled", { + finishedAt: deps.now(), + }) + await repo.refreshCounts(record.id) + await insights.bumpInsightsGeneration(record.cleanupId) + await deps.audit(KILLED_AUDIT_ACTION, record.createdBy, `broadcast:${record.id}`, { + cleanupId: record.cleanupId, + kind: record.kind, + reason, + suppressed, + }) + deps.logger?.warn( + { evt: "broadcast.killed", broadcastId: record.id, cleanupId: record.cleanupId, suppressed }, + "broadcast killed", + ) + } + + /** + * DECISIONS §32: an operator-suspended organization's events send no host-composed message, and a + * release or a queued plan is a send just as much as the compose-time call was. Critical automated + * notices (a cancellation, a changed time) stay deliverable: attendees must still hear about them. + */ + function organizationSuspendedIn( + record: BroadcastRecord, + event: Pick | null, + ): boolean { + return HOST_COMPOSED_BROADCAST_KINDS.has(record.kind) && event?.organizationSuspended === true + } + + async function organizationSuspendedFor(record: BroadcastRecord): Promise { + if (!HOST_COMPOSED_BROADCAST_KINDS.has(record.kind)) return false + return organizationSuspendedIn(record, await repo.eventContext(record.cleanupId)) + } + + async function failForSuspendedOrganization( + record: BroadcastRecord, + from: "scheduled" | "sending", + at: Date, + ): Promise { + const moved = await repo.transition(record.id, [from], "failed", { finishedAt: at }) + if (moved === null) return false + const suppressed = await repo.suppressRemaining(record.id, "kill_switch") + await repo.refreshCounts(record.id) + await insights.bumpInsightsGeneration(record.cleanupId) + await deps.audit(KILLED_AUDIT_ACTION, record.createdBy, `broadcast:${record.id}`, { + cleanupId: record.cleanupId, + kind: record.kind, + reason: ORG_SUSPENDED_REASON, + suppressed, + }) + deps.logger?.warn( + { + evt: "broadcast.failed", + broadcastId: record.id, + cleanupId: record.cleanupId, + reason: ORG_SUSPENDED_REASON, + }, + "broadcast refused: the event's organization is suspended", + ) + return true + } + + return { + killed, + killBroadcast, + organizationSuspendedIn, + organizationSuspendedFor, + failForSuspendedOrganization, + } +} diff --git a/services/api/src/services/host/broadcast-pipeline.ts b/services/api/src/services/host/broadcast-pipeline.ts index 41ddd015..50b8b111 100644 --- a/services/api/src/services/host/broadcast-pipeline.ts +++ b/services/api/src/services/host/broadcast-pipeline.ts @@ -1,80 +1,45 @@ +import type { BroadcastKind } from "@civfix/shared" import { ANNOUNCEMENT_BROADCAST_KIND } from "@civfix/shared" -import type { BroadcastKind, DeliveryFailureKind, NotificationType } from "@civfix/shared" -import type { BroadcastVarValues } from "@civfix/shared/host" import type { Mailer } from "@civfix/shared/interfaces" import type { FastifyBaseLogger } from "fastify" import type { CacheClient } from "../../auth/cache.js" import type { AdminAuditAction } from "../admin/audit.js" -import { mailFailure } from "../../adapters/mail-failure.js" -import { isWithinQuietHours, pushGateAllows, type PushGateMode } from "../notification-helpers.js" import type { NotificationService } from "../notification-service.js" -import { mapWithLimit } from "../media-presign.js" import { makeInsightsGeneration } from "./host-analytics-cache.js" import type { BroadcastRepository } from "./broadcast-repository.js" import type { - BroadcastRecord, BroadcastRecipientKind, - DeliveryClaim, + BroadcastRecord, DeliveryOutcome, DeliveryRowInput, - EventBroadcastContext, } from "./broadcast-types.js" -import { CRITICAL_BROADCAST_KINDS, HOST_COMPOSED_BROADCAST_KINDS } from "./broadcast-types.js" -import { - eventManageUrl, - eventPath, - formatEventWhen, - renderBroadcast, - type RenderedBroadcast, -} from "./broadcast-render.js" -import { mintUnsubscribeToken, unsubscribeExpiryFrom } from "./broadcast-capability-token.js" +import { MAX_DELIVERY_ATTEMPTS } from "./broadcast-types.js" +import { audiencePages } from "./broadcast-audience.js" +import { ChunkAuthAbort, makeBroadcastEmailSender } from "./broadcast-email-sender.js" +import { makeBroadcastInAppSender } from "./broadcast-inapp-sender.js" +import { makeBroadcastLifecycle } from "./broadcast-lifecycle.js" import { - AUDIENCE_MAX_PAGES, - AUDIENCE_PAGE_SIZE, BroadcastCapError, - LAST_UUID, - emailHashOf, type BroadcastConfig, type BroadcastService, } from "./broadcast-service.js" -export const CHUNK_STALE_MS = 10 * 60 * 1000 -export const SENDING_STALE_MS = 5 * 60 * 1000 -export const MAX_DELIVERY_ATTEMPTS = 3 -const ORG_SUSPENDED_REASON = "org_suspended" -export const AUTH_ABORT_BACKOFF_SEC = [60, 300, 900] as const -export const CRITICAL_KINDS = CRITICAL_BROADCAST_KINDS -export const AUTOMATED_KINDS = new Set([ - "confirmation", - "waitlist_promoted", - "reminder", - "event_updated", - "event_cancelled", - "thank_you", -]) +export { KIND_NOTIFICATION_TYPE } from "./broadcast-inapp-sender.js" +export { announcementPath, notificationLink } from "./broadcast-render.js" -export const KIND_NOTIFICATION_TYPE: Record = { - host_broadcast: "event_broadcast", - thank_you: "event_broadcast", - reminder: "cleanup_reminder", - event_updated: "system", - event_cancelled: "cleanup_cancelled", - confirmation: "system", - waitlist_promoted: "system", - announcement: "event_broadcast", -} - -export const BUDGETED_KINDS: ReadonlySet = new Set([ +const CHUNK_STALE_MS = 10 * 60 * 1000 +const SENDING_STALE_MS = 5 * 60 * 1000 +export const AUTH_ABORT_BACKOFF_SEC = [60, 300, 900] as const +const DELIVERY_INSERT_BATCH = 500 +// A chunk holds chunkSize recipients, each with at most one row per channel (inapp, push, email, sms). +const MAX_CHANNELS_PER_RECIPIENT = 4 +const SCHEDULE_SWEEP_LIMIT = 50 +const STALE_SWEEP_LIMIT = 20 +const BUDGETED_KINDS: ReadonlySet = new Set([ "host_broadcast", ANNOUNCEMENT_BROADCAST_KIND, ]) -export function announcementPath(cleanupId: string, announcementId: string): string { - return `/cleanups/${cleanupId}/announcements/${announcementId}` -} - -export const PER_RECIPIENT_VARS = ["first_name", "ticket_type"] as const - export interface BroadcastPipelineDeps { repo: BroadcastRepository service: BroadcastService @@ -98,196 +63,126 @@ export interface BroadcastPipelineDeps { now?: () => Date } -export class ChunkAuthAbort extends Error { - override readonly cause: unknown - readonly releasedIds: readonly string[] - - constructor(cause: unknown, releasedIds: readonly string[]) { - super("broadcast chunk aborted: the SMTP server rejected the sender") - this.name = "ChunkAuthAbort" - this.cause = cause - this.releasedIds = releasedIds - } -} - -export class TokenBucket { - private tokens: number - private lastRefillMs: number - - constructor( - private readonly ratePerSec: number, - private readonly now: () => number = () => Date.now(), - private readonly sleep: (ms: number) => Promise = (ms) => - new Promise((resolve) => setTimeout(resolve, ms)), - ) { - this.tokens = ratePerSec - this.lastRefillMs = now() - } - - async take(): Promise { - for (;;) { - const at = this.now() - const elapsed = at - this.lastRefillMs - if (elapsed > 0) { - this.tokens = Math.min(this.ratePerSec, this.tokens + (elapsed / 1000) * this.ratePerSec) - this.lastRefillMs = at - } - if (this.tokens >= 1) { - this.tokens -= 1 - return - } - await this.sleep(Math.ceil(((1 - this.tokens) / this.ratePerSec) * 1000)) - } - } -} - export interface PlanOutcome { kind: "planned" | "skipped" | "killed" | "org_suspended" | "too_many" | "over_budget" | "empty" recipients?: number chunks?: number } -export function notificationLink( - record: Pick, - event: Pick, -): string { - return record.kind === ANNOUNCEMENT_BROADCAST_KIND - ? announcementPath(record.cleanupId, record.id) - : eventPath(event.pageSlug, record.cleanupId) -} - -export function usesVar(text: string, name: string): boolean { - return text.includes(`{${name}}`) -} - -export function usesPerRecipientVar(text: string): boolean { - return PER_RECIPIENT_VARS.some((name) => usesVar(text, name)) +function deliveryRowsFor( + record: BroadcastRecord, + memberIds: readonly string[], + guestIds: readonly string[], +): DeliveryRowInput[] { + const chunkSize = Math.max(1, record.chunkSize) + const rows: DeliveryRowInput[] = [] + const recipients: Array<{ kind: BroadcastRecipientKind; id: string }> = [ + ...memberIds.map((id) => ({ kind: "member" as const, id })), + ...guestIds.map((id) => ({ kind: "guest" as const, id })), + ] + recipients.forEach((recipient, index) => { + const chunkNo = Math.floor(index / chunkSize) + for (const channel of record.channels) { + if (recipient.kind === "guest" && channel !== "email" && channel !== "sms") continue + rows.push({ + broadcastId: record.id, + chunkNo, + recipientKind: recipient.kind, + userId: recipient.kind === "member" ? recipient.id : null, + guestId: recipient.kind === "guest" ? recipient.id : null, + channel, + }) + } + }) + return rows } export function makeBroadcastPipeline(deps: BroadcastPipelineDeps) { const now = deps.now ?? (() => new Date()) const { repo, config } = deps - const insights = makeInsightsGeneration({ + const optionalLogger = deps.logger !== undefined ? { logger: deps.logger } : {} + const insights = makeInsightsGeneration({ cache: deps.cache, ...optionalLogger }) + const lifecycle = makeBroadcastLifecycle({ + repo, + config, + insights, + audit: deps.audit, + now, + ...optionalLogger, + }) + const inApp = makeBroadcastInAppSender({ + repo, + config, + notifications: deps.notifications, + now, + ...optionalLogger, + }) + const email = makeBroadcastEmailSender({ + repo, + config, + mailer: deps.mailer, cache: deps.cache, - ...(deps.logger !== undefined ? { logger: deps.logger } : {}), + mailDomain: deps.mailDomain, + now, + ...optionalLogger, }) - async function killed(record: BroadcastRecord): Promise { - if (config.killSwitch) return true - if (record.createdBy === null) return false - try { - const state = await repo.hostMessagingState(record.createdBy) - return state === null || state.suspended - } catch (err) { - deps.logger?.warn( - { err, broadcastId: record.id }, - "broadcast: kill-switch read failed; treating as suspended (fail closed)", - ) - return true - } - } - - async function killBroadcast(record: BroadcastRecord, reason: string): Promise { - const suppressed = await repo.suppressRemaining(record.id, "kill_switch") - await repo.transition(record.id, ["draft", "scheduled", "sending"], "cancelled", { - finishedAt: now(), - }) - await repo.refreshCounts(record.id) - await insights.bumpInsightsGeneration(record.cleanupId) - await deps.audit("event.broadcast_killed", record.createdBy, `broadcast:${record.id}`, { + async function collectAudience( + record: BroadcastRecord, + ): Promise<{ memberIds: string[]; guestIds: string[] }> { + const memberIds: string[] = [] + const guestIds: string[] = [] + const scope = { cleanupId: record.cleanupId, + segment: record.segment ?? { kind: "all_registered" as const }, kind: record.kind, - reason, - suppressed, - }) - deps.logger?.warn( - { evt: "broadcast.killed", broadcastId: record.id, cleanupId: record.cleanupId, suppressed }, - "broadcast killed", - ) + } + for await (const page of audiencePages(repo, scope)) { + memberIds.push(...page.members) + guestIds.push(...page.guests) + if (memberIds.length + guestIds.length > config.maxRecipients) break + } + return { memberIds, guestIds } } - /** - * DECISIONS §32: an operator-suspended organization's events send no host-composed message, and a - * release or a queued plan is a send just as much as the compose-time call was. Critical automated - * notices (a cancellation, a changed time) stay deliverable: attendees must still hear about them. - */ - async function organizationSuspendedFor(record: BroadcastRecord): Promise { - if (!HOST_COMPOSED_BROADCAST_KINDS.has(record.kind)) return false - const event = await repo.eventContext(record.cleanupId) - return event?.organizationSuspended === true + async function failPlan(record: BroadcastRecord): Promise { + await repo.transition(record.id, ["sending"], "failed", { finishedAt: now() }) + await insights.bumpInsightsGeneration(record.cleanupId) } - async function failForSuspendedOrganization( + /** False when the host's daily recipient budget refused the send and the plan was failed. */ + async function chargeRecipientBudget( record: BroadcastRecord, - from: "scheduled" | "sending", - at: Date, + recipientCount: number, + reservedNow: boolean, ): Promise { - const moved = await repo.transition(record.id, [from], "failed", { finishedAt: at }) - if (moved === null) return false - const suppressed = await repo.suppressRemaining(record.id, "kill_switch") - await repo.refreshCounts(record.id) - await insights.bumpInsightsGeneration(record.cleanupId) - await deps.audit("event.broadcast_killed", record.createdBy, `broadcast:${record.id}`, { - cleanupId: record.cleanupId, - kind: record.kind, - reason: ORG_SUSPENDED_REASON, - suppressed, - }) + if (!reservedNow || record.createdBy === null || !BUDGETED_KINDS.has(record.kind)) return true + if (await deps.service.reserveRecipientBudget(record.createdBy, recipientCount)) return true + await repo.suppressRemaining(record.id, "cap") + await failPlan(record) deps.logger?.warn( - { - evt: "broadcast.failed", - broadcastId: record.id, - cleanupId: record.cleanupId, - reason: ORG_SUSPENDED_REASON, - }, - "broadcast refused: the event's organization is suspended", + { evt: "broadcast.failed", broadcastId: record.id, reason: "recipients_per_day" }, + "broadcast refused: host daily recipient budget exhausted", ) - return true + return false } async function plan(broadcastId: string): Promise { const record = await repo.findById(broadcastId) if (record === null || record.status !== "sending") return { kind: "skipped" } - if (await killed(record)) { - await killBroadcast(record, "kill_switch") + if (await lifecycle.killed(record)) { + await lifecycle.killBroadcast(record, "kill_switch") return { kind: "killed" } } - if (await organizationSuspendedFor(record)) { - const failed = await failForSuspendedOrganization(record, "sending", now()) + if (await lifecycle.organizationSuspendedFor(record)) { + const failed = await lifecycle.failForSuspendedOrganization(record, "sending", now()) return { kind: failed ? "org_suspended" : "skipped" } } - const segment = record.segment ?? { kind: "all_registered" as const } - const memberIds: string[] = [] - const guestIds: string[] = [] - let afterMember: string | null = null - let afterGuest: string | null = null - let memberDone = false - let guestDone = false - - for (let page = 0; page < AUDIENCE_MAX_PAGES; page += 1) { - const result = await repo.audiencePage({ - cleanupId: record.cleanupId, - segment, - kind: record.kind, - afterMember: memberDone ? LAST_UUID : afterMember, - afterGuest: guestDone ? LAST_UUID : afterGuest, - limit: AUDIENCE_PAGE_SIZE, - }) - memberIds.push(...result.members) - guestIds.push(...result.guests) - if (result.members.length < AUDIENCE_PAGE_SIZE) memberDone = true - else afterMember = result.members.at(-1) ?? afterMember - if (result.guests.length < AUDIENCE_PAGE_SIZE) guestDone = true - else afterGuest = result.guests.at(-1) ?? afterGuest - if (memberIds.length + guestIds.length > config.maxRecipients) break - if (memberDone && guestDone) break - } - + const { memberIds, guestIds } = await collectAudience(record) const recipientCount = memberIds.length + guestIds.length if (recipientCount > config.maxRecipients) { - await repo.transition(record.id, ["sending"], "failed", { finishedAt: now() }) - await insights.bumpInsightsGeneration(record.cleanupId) + await failPlan(record) deps.logger?.error( { evt: "broadcast.failed", broadcastId: record.id, recipientCount, reason: "too_many" }, "broadcast refused: audience above the platform cap", @@ -301,48 +196,14 @@ export function makeBroadcastPipeline(deps: BroadcastPipelineDeps) { return { kind: "empty", recipients: 0 } } - const chunkSize = Math.max(1, record.chunkSize) - - const rows: DeliveryRowInput[] = [] - const recipients: Array<{ kind: BroadcastRecipientKind; id: string }> = [ - ...memberIds.map((id) => ({ kind: "member" as const, id })), - ...guestIds.map((id) => ({ kind: "guest" as const, id })), - ] - recipients.forEach((recipient, index) => { - const chunkNo = Math.floor(index / chunkSize) - for (const channel of record.channels) { - if (recipient.kind === "guest" && channel !== "email" && channel !== "sms") continue - rows.push({ - broadcastId: record.id, - chunkNo, - recipientKind: recipient.kind, - userId: recipient.kind === "member" ? recipient.id : null, - guestId: recipient.kind === "guest" ? recipient.id : null, - channel, - }) - } - }) - - for (let i = 0; i < rows.length; i += 500) { - await repo.insertDeliveries(rows.slice(i, i + 500)) + const rows = deliveryRowsFor(record, memberIds, guestIds) + for (let i = 0; i < rows.length; i += DELIVERY_INSERT_BATCH) { + await repo.insertDeliveries(rows.slice(i, i + DELIVERY_INSERT_BATCH)) } const reservedNow = await repo.markPlanned(record.id, { recipientCount, plannedAt: now() }) - if (reservedNow && record.createdBy !== null && BUDGETED_KINDS.has(record.kind)) { - const withinBudget = await deps.service.reserveRecipientBudget( - record.createdBy, - recipientCount, - ) - if (!withinBudget) { - await repo.suppressRemaining(record.id, "cap") - await repo.transition(record.id, ["sending"], "failed", { finishedAt: now() }) - await insights.bumpInsightsGeneration(record.cleanupId) - deps.logger?.warn( - { evt: "broadcast.failed", broadcastId: record.id, reason: "recipients_per_day" }, - "broadcast refused: host daily recipient budget exhausted", - ) - return { kind: "over_budget", recipients: recipientCount } - } + if (!(await chargeRecipientBudget(record, recipientCount, reservedNow))) { + return { kind: "over_budget", recipients: recipientCount } } await insights.bumpInsightsGeneration(record.cleanupId) @@ -366,16 +227,16 @@ export function makeBroadcastPipeline(deps: BroadcastPipelineDeps) { async function runChunk(broadcastId: string, chunkNo: number, authRetry = 0): Promise { const record = await repo.findById(broadcastId) if (record === null || record.status !== "sending") return - if (await killed(record)) { - await killBroadcast(record, "kill_switch_mid_send") + if (await lifecycle.killed(record)) { + await lifecycle.killBroadcast(record, "kill_switch_mid_send") return } const event = await repo.eventContext(record.cleanupId) if (event === null) return // An operator can suspend the organization after plan() passed; every chunk rechecks so the // suspension stops what is still pending rather than only the next broadcast. - if (HOST_COMPOSED_BROADCAST_KINDS.has(record.kind) && event.organizationSuspended) { - await failForSuspendedOrganization(record, "sending", now()) + if (lifecycle.organizationSuspendedIn(record, event)) { + await lifecycle.failForSuspendedOrganization(record, "sending", now()) return } @@ -384,7 +245,7 @@ export function makeBroadcastPipeline(deps: BroadcastPipelineDeps) { chunkNo, staleBefore: new Date(now().getTime() - CHUNK_STALE_MS), maxAttempts: MAX_DELIVERY_ATTEMPTS, - limit: Math.max(1, record.chunkSize) * 4, + limit: Math.max(1, record.chunkSize) * MAX_CHANNELS_PER_RECIPIENT, }) if (claims.length === 0) { await finalizeIfDrained(record) @@ -394,8 +255,8 @@ export function makeBroadcastPipeline(deps: BroadcastPipelineDeps) { const outcomes: DeliveryOutcome[] = [] let aborted: ChunkAuthAbort | null = null try { - await runInAppAndPush(record, event, claims, outcomes) - await runEmail(record, event, claims, outcomes) + await inApp.runInAppAndPush(record, event, claims, outcomes) + await email.runEmail(record, event, claims, outcomes) } catch (err) { if (!(err instanceof ChunkAuthAbort)) throw err aborted = err @@ -461,369 +322,38 @@ export function makeBroadcastPipeline(deps: BroadcastPipelineDeps) { if (finished !== null) await insights.bumpInsightsGeneration(record.cleanupId) } - function pushMode(record: BroadcastRecord): PushGateMode { - if (!record.channels.includes("push")) return "never" - return CRITICAL_KINDS.has(record.kind) ? "always" : "auto" - } - - async function runInAppAndPush( - record: BroadcastRecord, - event: EventBroadcastContext, - claims: readonly DeliveryClaim[], - outcomes: DeliveryOutcome[], - ): Promise { - const relevant = claims.filter( - (c) => (c.channel === "inapp" || c.channel === "push") && c.userId !== null, - ) - if (relevant.length === 0) return - const userIds = [...new Set(relevant.map((c) => c.userId as string))] - const content = `${record.subject ?? ""} ${record.bodyMd ?? ""} ${record.ctaLabel ?? ""}` - const perRecipient = usesPerRecipientVar(content) - const [contacts, prefs, ticketTypes] = await Promise.all([ - repo.memberContacts(userIds), - repo.pushPrefs(userIds), - usesVar(content, "ticket_type") - ? repo.ticketTypeNames({ cleanupId: record.cleanupId, userIds, guestIds: [] }) - : Promise.resolve(new Map()), - ]) - const shared = sharedVars(event) - const oneRendering = perRecipient ? null : render(record, event, shared) - const type = KIND_NOTIFICATION_TYPE[record.kind] - const mode = pushMode(record) - const at = now() - - const deliverable = new Map() - for (const claim of relevant) { - const userId = claim.userId as string - const contact = contacts.get(userId) - if (contact === undefined) { - outcomes.push({ id: claim.id, status: "suppressed", suppressionReason: "deleted_user" }) - continue - } - if (claim.channel === "push") { - const pref = prefs.get(userId) - const allowed = - pref === undefined || - (pushGateAllows(type, pref, mode) && - !isWithinQuietHours(at, pref.quietStart, pref.quietEnd, pref.tz)) - if (mode === "never" || !allowed) { - outcomes.push({ id: claim.id, status: "suppressed", suppressionReason: "prefs_off" }) - continue - } - outcomes.push({ id: claim.id, status: "sent", sentAt: at }) - continue - } - const rendering = - oneRendering ?? - render(record, event, { - ...shared, - first_name: contact.firstName, - ticket_type: ticketTypes.get(userId) ?? "", - }) - deliverable.set(userId, rendering) - outcomes.push({ id: claim.id, status: "sent", sentAt: at }) - } - - if (deliverable.size === 0) return - const groups = new Map() - for (const [userId, rendering] of deliverable) { - const key = JSON.stringify([rendering.inAppTitle, rendering.inAppBody]) - const group = groups.get(key) - if (group !== undefined) group.userIds.push(userId) - else groups.set(key, { rendering, userIds: [userId] }) - } - const undelivered = new Set() - let fanOutError: unknown = null - for (const group of groups.values()) { - try { - const { failed } = await deps.notifications.createNotificationsReportingFailures( - group.userIds, - { - type, - title: group.rendering.inAppTitle, - body: group.rendering.inAppBody, - link: notificationLink(record, event), - push: mode, - }, - ) - for (const userId of failed) undelivered.add(userId) - } catch (err) { - fanOutError = err - for (const userId of group.userIds) undelivered.add(userId) - } - } - if (undelivered.size === 0) return - deps.logger?.error( - { err: fanOutError, broadcastId: record.id, undelivered: undelivered.size }, - "broadcast: in-app fan-out failed for some recipients; only those rows return to pending", - ) - for (const claim of relevant) { - if (claim.userId === null || !undelivered.has(claim.userId)) continue - const index = outcomes.findIndex((o) => o.id === claim.id) - if (index >= 0 && outcomes[index]?.status !== "sent") continue - if (index >= 0) outcomes.splice(index, 1) - outcomes.push( - claim.attempts >= MAX_DELIVERY_ATTEMPTS - ? { id: claim.id, status: "failed", failureKind: "unknown" } - : { id: claim.id, status: "pending" }, - ) - } - } - - async function runEmail( - record: BroadcastRecord, - event: EventBroadcastContext, - claims: readonly DeliveryClaim[], - outcomes: DeliveryOutcome[], - ): Promise { - const relevant = claims.filter((c) => c.channel === "email") - if (relevant.length === 0) return - const memberIds = relevant.filter((c) => c.userId !== null).map((c) => c.userId as string) - const guestIds = relevant.filter((c) => c.guestId !== null).map((c) => c.guestId as string) - const content = `${record.subject ?? ""} ${record.bodyMd ?? ""} ${record.ctaLabel ?? ""}` - const [members, guests, ticketTypes] = await Promise.all([ - repo.memberContacts(memberIds), - repo.guestContacts(guestIds), - usesVar(content, "ticket_type") - ? repo.ticketTypeNames({ cleanupId: record.cleanupId, userIds: memberIds, guestIds }) - : Promise.resolve(new Map()), - ]) - const addresses = new Map() - for (const claim of relevant) { - const contact = - claim.userId !== null ? members.get(claim.userId) : guests.get(claim.guestId as string) - const email = contact?.email - if (email == null || email.length === 0) continue - addresses.set(claim.id, emailHashOf(email)) - } - const suppressedHashes = await repo.suppressedEmailHashes([...addresses.values()]) - - const bucket = new TokenBucket(config.emailRatePerSec) - const critical = CRITICAL_KINDS.has(record.kind) - const replyTo = event.replyToVerified && event.replyTo ? event.replyTo : null - const sentAtMs = now().getTime() - const shared = sharedVars(event) - let abortError: unknown = null - const releasedIds: string[] = [] - - await mapWithLimit([...relevant], config.emailConcurrency, async (claim): Promise => { - if (abortError !== null) { - outcomes.push({ id: claim.id, status: "pending" }) - releasedIds.push(claim.id) - return - } - const isMember = claim.userId !== null - const contact = isMember - ? members.get(claim.userId as string) - : guests.get(claim.guestId as string) - if (contact === undefined) { - outcomes.push({ - id: claim.id, - status: "suppressed", - suppressionReason: isMember ? "deleted_user" : "contact_scrubbed", - }) - return - } - const email = contact.email - if (email === null || email.length === 0) { - outcomes.push({ id: claim.id, status: "suppressed", suppressionReason: "no_contact" }) - return - } - const hash = addresses.get(claim.id) ?? emailHashOf(email) - if (suppressedHashes.has(hash)) { - outcomes.push({ - id: claim.id, - status: "suppressed", - suppressionReason: "bounce_suppressed", - }) - return - } - const fresh = await claimAddress(record.id, hash, claim.id) - if (!fresh) { - outcomes.push({ id: claim.id, status: "skipped" }) - return - } - - const subjectId = (claim.userId ?? claim.guestId) as string - const firstName = isMember - ? (members.get(claim.userId as string)?.firstName ?? "") - : (guests.get(claim.guestId as string)?.name ?? "") - const token = encodeURIComponent( - mintUnsubscribeToken( - { - subjectKind: isMember ? "user" : "guest", - subjectId, - cleanupId: record.cleanupId, - expiresAtMs: unsubscribeExpiryFrom(sentAtMs), - }, - config.unsubscribeSigningKey, - ), - ) - const unsubscribeUrl = `${config.webBaseUrl}/unsubscribe?t=${token}` - const oneClickUrl = `${config.apiBaseUrl}/v1/broadcasts/unsubscribe?t=${token}` - const rendered = renderBroadcast( - { - subject: record.subject ?? "", - bodyMd: record.bodyMd ?? "", - ctaLabel: record.ctaLabel, - ctaUrl: record.ctaUrl, - }, - { - eventTitle: event.title, - vars: { - ...shared, - first_name: firstName, - ticket_type: ticketTypes.get(subjectId) ?? "", - }, - unsubscribeUrl, - replyTo, - critical, - allowedLinkHosts: config.linkAllowedHosts, - }, - ) - const headers: Record = { - "List-Unsubscribe": `<${oneClickUrl}>`, - "List-Unsubscribe-Post": "List-Unsubscribe=One-Click", - } - if (AUTOMATED_KINDS.has(record.kind)) headers["Auto-Submitted"] = "auto-generated" - - await bucket.take() - try { - const sent = await deps.mailer.sendOutbound({ - from: config.mailFromEvents, - to: email, - subject: rendered.subject, - text: rendered.text, - html: rendered.html, - messageId: ``, - headers, - ...(replyTo !== null ? { replyTo } : {}), - }) - outcomes.push({ - id: claim.id, - status: "sent", - sentAt: now(), - ...(sent.messageId !== undefined ? { providerMessageId: sent.messageId } : {}), - }) - } catch (err) { - const failure = mailFailure(err) - deps.logger?.warn( - { - evt: "broadcast.failed", - deliveryId: claim.id, - failureKind: failure.kind, - senderRejected: failure.senderRejected, - }, - "broadcast email send failed", - ) - if (failure.kind === "auth") { - abortError = err - await releaseAddress(record.id, hash) - outcomes.push({ id: claim.id, status: "pending" }) - releasedIds.push(claim.id) - return - } - if (failure.kind === "permanent") { - await repo.suppressEmail(hash, "hard_bounce").catch((suppressErr: unknown) => { - deps.logger?.warn( - { err: suppressErr, deliveryId: claim.id }, - "broadcast: hard-bounce suppression write failed; the address may be mailed again", - ) - }) - outcomes.push({ id: claim.id, status: "failed", failureKind: "permanent" }) - return - } - if (failure.kind === "transient" && claim.attempts < MAX_DELIVERY_ATTEMPTS) { - await releaseAddress(record.id, hash) - outcomes.push({ id: claim.id, status: "pending" }) - return - } - outcomes.push({ - id: claim.id, - status: "failed", - failureKind: failure.kind as DeliveryFailureKind, - }) - } - }) - - if (abortError !== null) throw new ChunkAuthAbort(abortError, releasedIds) - } - - function addressKey(broadcastId: string, hash: string): string { - return `bcast:addr:${broadcastId}:${hash}` - } - - async function releaseAddress(broadcastId: string, hash: string): Promise { - const key = addressKey(broadcastId, hash) - try { - await deps.cache.del(key) - await deps.cache.del(`${key}:owner`) - } catch (err) { - deps.logger?.warn( - { err }, - "broadcast: address dedupe release failed; a retry of this row may be skipped", - ) - } - } - - async function claimAddress( - broadcastId: string, - hash: string, - deliveryId: string, + /** True hands the broadcast on to plan(), which kills it; false ends the release here. */ + async function refuseAtRelease( + id: string, + cleanupId: string, + err: BroadcastCapError, + at: Date, ): Promise { - const key = addressKey(broadcastId, hash) - try { - const held = await deps.cache.incr(key, DEDUPE_TTL_SEC) - if (held === 1) { - await deps.cache.set(`${key}:owner`, deliveryId, DEDUPE_TTL_SEC) - return true - } - return (await deps.cache.get(`${key}:owner`)) === deliveryId - } catch (err) { + if (err.kind === "kill_switch" || err.kind === "suspended") return true + if (err.kind === "counter_unavailable") { + await repo.transition(id, ["sending"], "scheduled", { startedAt: null }) deps.logger?.warn( - { err }, - "broadcast: address dedupe unavailable; sending anyway (a duplicate beats a silence)", + { evt: "broadcast.release.deferred", broadcastId: id }, + "scheduled broadcast returned to scheduled: cap counters unavailable, retrying next sweep", ) - return true - } - } - - function sharedVars(event: EventBroadcastContext): BroadcastVarValues { - return { - event_title: event.title, - event_when: formatEventWhen(event.scheduledAt, event.timezone), - event_where: event.address ?? "the meeting point", - manage_link: eventManageUrl(config.webBaseUrl, event.pageSlug, event.cleanupId), + return false } - } - - function render( - record: BroadcastRecord, - event: EventBroadcastContext, - vars: BroadcastVarValues, - ): RenderedBroadcast { - return renderBroadcast( - { - subject: record.subject ?? "", - bodyMd: record.bodyMd ?? "", - ctaLabel: record.ctaLabel, - ctaUrl: record.ctaUrl, - }, - { - eventTitle: event.title, - vars, - replyTo: event.replyToVerified ? event.replyTo : null, - critical: CRITICAL_KINDS.has(record.kind), - allowedLinkHosts: config.linkAllowedHosts, - }, + await repo.suppressRemaining(id, "cap") + await repo.transition(id, ["sending"], "failed", { finishedAt: at }) + await repo.refreshCounts(id) + await insights.bumpInsightsGeneration(cleanupId) + deps.logger?.warn( + { evt: "broadcast.failed", broadcastId: id, reason: err.kind }, + "scheduled broadcast refused at release: send-slot cap", ) + return false } async function releaseScheduled(id: string, at: Date): Promise { const due = await repo.findById(id) if (due === null || due.status !== "scheduled") return false - if (await organizationSuspendedFor(due)) { - await failForSuspendedOrganization(due, "scheduled", at) + if (await lifecycle.organizationSuspendedFor(due)) { + await lifecycle.failForSuspendedOrganization(due, "scheduled", at) return false } const moved = await repo.transition(id, ["scheduled"], "sending", { startedAt: at }) @@ -833,59 +363,45 @@ export function makeBroadcastPipeline(deps: BroadcastPipelineDeps) { await deps.service.reserveSendSlot(moved.cleanupId, moved.createdBy) } catch (err) { if (!(err instanceof BroadcastCapError)) throw err - if (err.kind === "kill_switch" || err.kind === "suspended") { - await plan(id) - return true - } - if (err.kind === "counter_unavailable") { - await repo.transition(id, ["sending"], "scheduled", { startedAt: null }) - deps.logger?.warn( - { evt: "broadcast.release.deferred", broadcastId: id }, - "scheduled broadcast returned to scheduled: cap counters unavailable, retrying next sweep", - ) - return false - } - await repo.suppressRemaining(id, "cap") - await repo.transition(id, ["sending"], "failed", { finishedAt: at }) - await repo.refreshCounts(id) - await insights.bumpInsightsGeneration(moved.cleanupId) - deps.logger?.warn( - { evt: "broadcast.failed", broadcastId: id, reason: err.kind }, - "scheduled broadcast refused at release: send-slot cap", - ) - return false + if (!(await refuseAtRelease(id, moved.cleanupId, err, at))) return false } } await plan(id) return true } + async function resumeStale(id: string): Promise { + const record = await repo.findById(id) + if (record === null) return false + if (record.plannedAt === null) { + await plan(id) + return true + } + const counts = await repo.deliveryCounts(id) + if (counts.pending === 0) { + await finalizeIfDrained(record) + return false + } + for (const chunkNo of await repo.listPendingChunks(id)) { + await deps.enqueueChunk(id, chunkNo) + } + return true + } + async function sweep(): Promise<{ released: number; resumed: number }> { const at = now() - const due = await repo.listDueScheduled(at, 50) + const due = await repo.listDueScheduled(at, SCHEDULE_SWEEP_LIMIT) let released = 0 for (const id of due) { if (await releaseScheduled(id, at)) released += 1 } - const stale = await repo.listStaleSending(new Date(at.getTime() - SENDING_STALE_MS), 20) + const stale = await repo.listStaleSending( + new Date(at.getTime() - SENDING_STALE_MS), + STALE_SWEEP_LIMIT, + ) let resumed = 0 for (const id of stale) { - const record = await repo.findById(id) - if (record === null) continue - if (record.plannedAt === null) { - await plan(id) - resumed += 1 - continue - } - const counts = await repo.deliveryCounts(id) - if (counts.pending === 0) { - await finalizeIfDrained(record) - continue - } - for (const chunkNo of await repo.listPendingChunks(id)) { - await deps.enqueueChunk(id, chunkNo) - } - resumed += 1 + if (await resumeStale(id)) resumed += 1 } return { released, resumed } } @@ -893,6 +409,4 @@ export function makeBroadcastPipeline(deps: BroadcastPipelineDeps) { return { plan, runChunk, sweep, finalizeIfDrained } } -const DEDUPE_TTL_SEC = 24 * 60 * 60 - export type BroadcastPipeline = ReturnType diff --git a/services/api/src/services/host/broadcast-render.ts b/services/api/src/services/host/broadcast-render.ts index bdc6e50d..1fb0df41 100644 --- a/services/api/src/services/host/broadcast-render.ts +++ b/services/api/src/services/host/broadcast-render.ts @@ -1,4 +1,4 @@ -import { AppError, MAX_BROADCAST_BODY } from "@civfix/shared" +import { ANNOUNCEMENT_BROADCAST_KIND, AppError, MAX_BROADCAST_BODY } from "@civfix/shared" import { BroadcastLinkError, assertSafeBroadcastLinks, @@ -14,8 +14,15 @@ import { import { markdownToPlainText, parseMarkdownSubset } from "@civfix/shared/markdown" import { button, richList, richParagraph, type EmailBlock } from "../../adapters/email-blocks.js" import { eventFooter, renderEmailBody } from "../../adapters/email-layout.js" +import type { BroadcastRecord, EventBroadcastContext } from "./broadcast-types.js" import { DEFAULT_EVENT_TIME_ZONE } from "./event-fields.js" +const DEFAULT_CTA_LABEL = "Open" +const DEFAULT_EVENT_WHERE = "the meeting point" +const EVENT_WHEN_LOCALE = "en-US" +const BROADCAST_LINK_FIELD = "bodyMd" +const PER_RECIPIENT_VARS = ["first_name", "ticket_type"] as const + export interface BroadcastContent { subject: string bodyMd: string @@ -58,7 +65,7 @@ export function renderBroadcast( node.type === "list" ? richList(node) : richParagraph(node.children), ) if (ctaUrl.length > 0) { - blocks.push(button(ctaUrl, ctaLabel.length > 0 ? ctaLabel : "Open")) + blocks.push(button(ctaUrl, ctaLabel.length > 0 ? ctaLabel : DEFAULT_CTA_LABEL)) } const footerOpts = { @@ -89,7 +96,7 @@ export function renderBroadcast( export function formatEventWhen(scheduledAt: Date, timezone: string | null): string { try { - return new Intl.DateTimeFormat("en-US", { + return new Intl.DateTimeFormat(EVENT_WHEN_LOCALE, { timeZone: timezone ?? DEFAULT_EVENT_TIME_ZONE, weekday: "long", month: "long", @@ -118,7 +125,61 @@ export function eventManageUrl( return `${webBaseUrl}${eventPath(pageSlug, cleanupId)}` } -export const BROADCAST_LINK_FIELD = "bodyMd" +export function announcementPath(cleanupId: string, announcementId: string): string { + return `/cleanups/${cleanupId}/announcements/${announcementId}` +} + +export function notificationLink( + record: Pick, + event: Pick, +): string { + return record.kind === ANNOUNCEMENT_BROADCAST_KIND + ? announcementPath(record.cleanupId, record.id) + : eventPath(event.pageSlug, record.cleanupId) +} + +export function eventTemplateVars( + event: EventBroadcastContext, + webBaseUrl: string, +): BroadcastVarValues { + return { + event_title: event.title, + event_when: formatEventWhen(event.scheduledAt, event.timezone), + event_where: event.address ?? DEFAULT_EVENT_WHERE, + manage_link: eventManageUrl(webBaseUrl, event.pageSlug, event.cleanupId), + } +} + +export function broadcastContentOf( + record: Pick, +): BroadcastContent { + return { + subject: record.subject ?? "", + bodyMd: record.bodyMd ?? "", + ctaLabel: record.ctaLabel, + ctaUrl: record.ctaUrl, + } +} + +export function templateTextOf( + record: Pick, +): string { + return `${record.subject ?? ""} ${record.bodyMd ?? ""} ${record.ctaLabel ?? ""}` +} + +export function usesVar(text: string, name: string): boolean { + return text.includes(`{${name}}`) +} + +export function usesPerRecipientVar(text: string): boolean { + return PER_RECIPIENT_VARS.some((name) => usesVar(text, name)) +} + +export function verifiedReplyTo( + event: Pick, +): string | null { + return event.replyToVerified ? event.replyTo : null +} export function assertBroadcastLinkPolicy( text: string, @@ -160,7 +221,6 @@ function issueCopy(issue: BroadcastLinkIssue): string { case "too_many": return `too many links (${issue.count}; the limit is ${issue.max})` case "insecure_scheme": - return "links must start with https://" case "scheme_relative": return "links must start with https://" case "userinfo": diff --git a/services/api/src/services/host/broadcast-repository.drizzle.ts b/services/api/src/services/host/broadcast-repository.drizzle.ts index 4bea3473..42979ac1 100644 --- a/services/api/src/services/host/broadcast-repository.drizzle.ts +++ b/services/api/src/services/host/broadcast-repository.drizzle.ts @@ -41,7 +41,7 @@ import type { HostMessagingState, MemberContact, } from "./broadcast-types.js" -import { ANNOUNCEMENT_VISIBLE_STATUSES } from "./broadcast-types.js" +import { ANNOUNCEMENT_VISIBLE_STATUSES, DEFAULT_BROADCAST_CHUNK_SIZE } from "./broadcast-types.js" interface BroadcastRowSelect { id: string @@ -109,6 +109,13 @@ function toKeysetRecord(row: CursorRowSelect): KeysetRow { return { ...toRecord(row), cursorAt: row.cursor_at } } +function broadcastColumns(tag: Queryable) { + return tag`id, cleanup_id, created_by, kind, reminder_offset_min, status, subject, body_md, + cta_label, cta_url, segment, channels, reply_to, scheduled_at, planned_at, started_at, + finished_at, chunk_size, chunk_count, recipient_count, sent_count, failed_count, + suppressed_count, content_scrubbed_at, created_at, updated_at` +} + function joinSet(sql: Sql, fragments: Array>): ReturnType { const [first, ...rest] = fragments if (first === undefined) throw new Error("broadcast UPDATE needs at least one column to set") @@ -131,10 +138,7 @@ function firstName(displayName: string): string { export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { async function selectById(broadcastId: string): Promise { const rows = await sql` - SELECT id, cleanup_id, created_by, kind, reminder_offset_min, status, subject, body_md, - cta_label, cta_url, segment, channels, reply_to, scheduled_at, planned_at, started_at, - finished_at, chunk_size, chunk_count, recipient_count, sent_count, failed_count, - suppressed_count, content_scrubbed_at, created_at, updated_at FROM broadcasts WHERE id = ${broadcastId} LIMIT 1` + SELECT ${broadcastColumns(sql)} FROM broadcasts WHERE id = ${broadcastId} LIMIT 1` return rows[0] ? toRecord(rows[0]) : null } @@ -151,12 +155,9 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { ${input.status ?? "draft"}, ${input.subject}, ${input.bodyMd}, ${input.ctaLabel ?? null}, ${input.ctaUrl ?? null}, ${jsonParam(sql, input.segment)}, ${input.channels}::text[], ${input.replyTo ?? null}, ${input.scheduledAt ?? null}, - ${input.startedAt ?? null}, ${input.chunkSize ?? 200} + ${input.startedAt ?? null}, ${input.chunkSize ?? DEFAULT_BROADCAST_CHUNK_SIZE} ) - RETURNING id, cleanup_id, created_by, kind, reminder_offset_min, status, subject, body_md, - cta_label, cta_url, segment, channels, reply_to, scheduled_at, planned_at, started_at, - finished_at, chunk_size, chunk_count, recipient_count, sent_count, failed_count, - suppressed_count, content_scrubbed_at, created_at, updated_at` + RETURNING ${broadcastColumns(sql)}` return toRecord(rows[0]!) } @@ -193,10 +194,7 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { ${input.replyTo ?? null}, ${input.startedAt ?? null} ) ON CONFLICT DO NOTHING - RETURNING id, cleanup_id, created_by, kind, reminder_offset_min, status, subject, body_md, - cta_label, cta_url, segment, channels, reply_to, scheduled_at, planned_at, started_at, - finished_at, chunk_size, chunk_count, recipient_count, sent_count, failed_count, - suppressed_count, content_scrubbed_at, created_at, updated_at` + RETURNING ${broadcastColumns(sql)}` return rows[0] ? toRecord(rows[0]) : null }, @@ -204,10 +202,7 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { async findForEvent(cleanupId: string, broadcastId: string): Promise { const rows = await sql` - SELECT id, cleanup_id, created_by, kind, reminder_offset_min, status, subject, body_md, - cta_label, cta_url, segment, channels, reply_to, scheduled_at, planned_at, started_at, - finished_at, chunk_size, chunk_count, recipient_count, sent_count, failed_count, - suppressed_count, content_scrubbed_at, created_at, updated_at FROM broadcasts + SELECT ${broadcastColumns(sql)} FROM broadcasts WHERE id = ${broadcastId} AND cleanup_id = ${cleanupId} LIMIT 1` return rows[0] ? toRecord(rows[0]) : null @@ -215,10 +210,7 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { async findEventCancellation(cleanupId: string): Promise { const rows = await sql` - SELECT id, cleanup_id, created_by, kind, reminder_offset_min, status, subject, body_md, - cta_label, cta_url, segment, channels, reply_to, scheduled_at, planned_at, started_at, - finished_at, chunk_size, chunk_count, recipient_count, sent_count, failed_count, - suppressed_count, content_scrubbed_at, created_at, updated_at FROM broadcasts + SELECT ${broadcastColumns(sql)} FROM broadcasts WHERE cleanup_id = ${cleanupId} AND kind = 'event_cancelled' LIMIT 1` return rows[0] ? toRecord(rows[0]) : null @@ -231,10 +223,7 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { ? sql`AND ${keysetPredicate(sql, sql`created_at`, sql`id`, query.cursor)}` : sql`` const rows = await sql` - SELECT id, cleanup_id, created_by, kind, reminder_offset_min, status, subject, body_md, - cta_label, cta_url, segment, channels, reply_to, scheduled_at, planned_at, started_at, - finished_at, chunk_size, chunk_count, recipient_count, sent_count, failed_count, - suppressed_count, content_scrubbed_at, created_at, updated_at, + SELECT ${broadcastColumns(sql)}, ${keysetInstant(sql, sql`created_at`)} AS cursor_at FROM broadcasts WHERE cleanup_id = ${query.cleanupId} @@ -251,10 +240,7 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { ? sql`AND ${keysetPredicate(sql, sql`created_at`, sql`id`, query.cursor)}` : sql`` const rows = await sql` - SELECT id, cleanup_id, created_by, kind, reminder_offset_min, status, subject, body_md, - cta_label, cta_url, segment, channels, reply_to, scheduled_at, planned_at, started_at, - finished_at, chunk_size, chunk_count, recipient_count, sent_count, failed_count, - suppressed_count, content_scrubbed_at, created_at, updated_at, + SELECT ${broadcastColumns(sql)}, ${keysetInstant(sql, sql`created_at`)} AS cursor_at FROM broadcasts WHERE cleanup_id = ${query.cleanupId} @@ -456,10 +442,7 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { const rows = await sql` UPDATE broadcasts SET ${joinSet(sql, sets)}, updated_at = now() WHERE id = ${broadcastId} AND cleanup_id = ${cleanupId} AND status = 'draft' - RETURNING id, cleanup_id, created_by, kind, reminder_offset_min, status, subject, body_md, - cta_label, cta_url, segment, channels, reply_to, scheduled_at, planned_at, started_at, - finished_at, chunk_size, chunk_count, recipient_count, sent_count, failed_count, - suppressed_count, content_scrubbed_at, created_at, updated_at` + RETURNING ${broadcastColumns(sql)}` return rows[0] ? toRecord(rows[0]) : null }, @@ -486,10 +469,7 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { const rows = await sql` UPDATE broadcasts SET ${joinSet(sql, sets)}, updated_at = now() WHERE id = ${broadcastId} AND status = ANY(${[...from]}::text[]) - RETURNING id, cleanup_id, created_by, kind, reminder_offset_min, status, subject, body_md, - cta_label, cta_url, segment, channels, reply_to, scheduled_at, planned_at, started_at, - finished_at, chunk_size, chunk_count, recipient_count, sent_count, failed_count, - suppressed_count, content_scrubbed_at, created_at, updated_at` + RETURNING ${broadcastColumns(sql)}` return rows[0] ? toRecord(rows[0]) : null }, @@ -688,10 +668,7 @@ export function makeDrizzleBroadcastRepository(sql: Sql): BroadcastRepository { FROM broadcast_deliveries WHERE broadcast_id = ${broadcastId} ) AS c WHERE b.id = ${broadcastId} - RETURNING id, cleanup_id, created_by, kind, reminder_offset_min, status, subject, body_md, - cta_label, cta_url, segment, channels, reply_to, scheduled_at, planned_at, started_at, - finished_at, chunk_size, chunk_count, recipient_count, sent_count, failed_count, - suppressed_count, content_scrubbed_at, created_at, updated_at` + RETURNING ${broadcastColumns(sql)}` return rows[0] ? toRecord(rows[0]) : null }, diff --git a/services/api/src/services/host/broadcast-repository.memory.ts b/services/api/src/services/host/broadcast-repository.memory.ts index 99ce3303..c6cc0bb1 100644 --- a/services/api/src/services/host/broadcast-repository.memory.ts +++ b/services/api/src/services/host/broadcast-repository.memory.ts @@ -36,6 +36,7 @@ import type { import { ANNOUNCEMENT_VISIBLE_STATUSES, CRITICAL_BROADCAST_KINDS, + DEFAULT_BROADCAST_CHUNK_SIZE, HOST_COMPOSED_BROADCAST_KINDS, } from "./broadcast-types.js" @@ -101,6 +102,27 @@ function withCursorAt(row: T): KeysetRow { return { ...row, cursorAt: row.createdAt.toISOString() } } +function isOpenDelivery(row: DeliveryRow): boolean { + return row.status === "pending" || row.status === "in_flight" +} + +function deliveryKey(row: DeliveryRowInput): string { + return `${row.broadcastId}|${row.channel}|${row.recipientKind}|${row.userId ?? row.guestId ?? ""}` +} + +function unsubscribeKey( + scope: "event" | "global", + cleanupId: string | null, + subjectKind: "user" | "guest", + subjectId: string, +): string { + return `${scope}|${cleanupId ?? ""}|${subjectKind}|${subjectId}` +} + +function muteKey(cleanupId: string, userId: string): string { + return `${cleanupId}|${userId}` +} + export class InMemoryBroadcastRepository implements BroadcastRepository { private readonly broadcasts = new Map() private readonly deliveries = new Map() @@ -200,7 +222,7 @@ export class InMemoryBroadcastRepository implements BroadcastRepository { plannedAt: null, startedAt: input.startedAt ?? null, finishedAt: null, - chunkSize: input.chunkSize ?? 200, + chunkSize: input.chunkSize ?? DEFAULT_BROADCAST_CHUNK_SIZE, chunkCount: 0, recipientCount: 0, sentCount: 0, @@ -435,7 +457,7 @@ export class InMemoryBroadcastRepository implements BroadcastRepository { for (const id of deliveryIds) { const row = this.deliveries.get(id) if (row === undefined) continue - if (row.status !== "pending" && row.status !== "in_flight") continue + if (!isOpenDelivery(row)) continue row.status = "pending" row.attempts = Math.max(row.attempts - 1, 0) row.updatedAt = new Date() @@ -447,7 +469,7 @@ export class InMemoryBroadcastRepository implements BroadcastRepository { const chunks = new Set() for (const row of this.deliveries.values()) { if (row.broadcastId !== broadcastId) continue - if (row.status !== "pending" && row.status !== "in_flight") continue + if (!isOpenDelivery(row)) continue chunks.add(row.chunkNo) } return Promise.resolve([...chunks].sort((a, b) => a - b)) @@ -481,11 +503,8 @@ export class InMemoryBroadcastRepository implements BroadcastRepository { insertDeliveries(rows: readonly DeliveryRowInput[]): Promise { let inserted = 0 for (const row of rows) { - const key = `${row.broadcastId}|${row.channel}|${row.recipientKind}|${row.userId ?? row.guestId ?? ""}` - const clash = [...this.deliveries.values()].some( - (d) => - `${d.broadcastId}|${d.channel}|${d.recipientKind}|${d.userId ?? d.guestId ?? ""}` === key, - ) + const key = deliveryKey(row) + const clash = [...this.deliveries.values()].some((d) => deliveryKey(d) === key) if (clash) continue const now = new Date() const id = randomUUID() @@ -555,7 +574,7 @@ export class InMemoryBroadcastRepository implements BroadcastRepository { let n = 0 for (const row of this.deliveries.values()) { if (row.broadcastId !== broadcastId) continue - if (row.status !== "pending" && row.status !== "in_flight") continue + if (!isOpenDelivery(row)) continue if (row.attempts < maxAttempts) continue row.status = "failed" row.failureKind = "unknown" @@ -569,7 +588,7 @@ export class InMemoryBroadcastRepository implements BroadcastRepository { let n = 0 for (const row of this.deliveries.values()) { if (row.broadcastId !== broadcastId) continue - if (row.status !== "pending" && row.status !== "in_flight") continue + if (!isOpenDelivery(row)) continue row.status = "suppressed" row.suppressionReason = reason row.updatedAt = new Date() @@ -582,7 +601,7 @@ export class InMemoryBroadcastRepository implements BroadcastRepository { const counts: DeliveryCounts = { pending: 0, sent: 0, failed: 0, suppressed: 0, skipped: 0 } for (const row of this.deliveries.values()) { if (row.broadcastId !== broadcastId) continue - if (row.status === "pending" || row.status === "in_flight") counts.pending += 1 + if (isOpenDelivery(row)) counts.pending += 1 else if (row.status === "sent") counts.sent += 1 else if (row.status === "failed") counts.failed += 1 else if (row.status === "suppressed") counts.suppressed += 1 @@ -743,20 +762,20 @@ export class InMemoryBroadcastRepository implements BroadcastRepository { subjectId: string }): Promise { this.unsubscribes.add( - `${args.scope}|${args.cleanupId ?? ""}|${args.subjectKind}|${args.subjectId}`, + unsubscribeKey(args.scope, args.cleanupId, args.subjectKind, args.subjectId), ) return Promise.resolve() } setEventMute(cleanupId: string, userId: string, muted: boolean): Promise { - const key = `${cleanupId}|${userId}` + const key = muteKey(cleanupId, userId) if (muted) this.mutes.add(key) else this.mutes.delete(key) return Promise.resolve() } isEventMuted(cleanupId: string, userId: string): Promise { - return Promise.resolve(this.mutes.has(`${cleanupId}|${userId}`)) + return Promise.resolve(this.mutes.has(muteKey(cleanupId, userId))) } listDueReminders(args: { limit: number }): Promise { @@ -829,14 +848,18 @@ export class InMemoryBroadcastRepository implements BroadcastRepository { private memberAudible(cleanupId: string, member: MemoryMember, kind: BroadcastKind): boolean { if (CRITICAL_BROADCAST_KINDS.has(kind)) return true if (HOST_COMPOSED_BROADCAST_KINDS.has(kind) && member.hostBroadcastsPref === false) return false - if (this.mutes.has(`${cleanupId}|${member.userId}`)) return false - if (this.unsubscribes.has(`event|${cleanupId}|user|${member.userId}`)) return false - return !this.unsubscribes.has(`global||user|${member.userId}`) + if (this.mutes.has(muteKey(cleanupId, member.userId))) return false + if (this.unsubscribes.has(unsubscribeKey("event", cleanupId, "user", member.userId))) { + return false + } + return !this.unsubscribes.has(unsubscribeKey("global", null, "user", member.userId)) } private guestOptedOut(cleanupId: string, guest: MemoryGuest): boolean { - if (this.unsubscribes.has(`event|${cleanupId}|guest|${guest.guestId}`)) return true - return this.unsubscribes.has(`global||guest|${guest.guestId}`) + if (this.unsubscribes.has(unsubscribeKey("event", cleanupId, "guest", guest.guestId))) { + return true + } + return this.unsubscribes.has(unsubscribeKey("global", null, "guest", guest.guestId)) } private memberMatchesSegment(member: MemoryMember, query: AudiencePageQuery): boolean { diff --git a/services/api/src/services/host/broadcast-service.ts b/services/api/src/services/host/broadcast-service.ts index 05da2339..00466f1b 100644 --- a/services/api/src/services/host/broadcast-service.ts +++ b/services/api/src/services/host/broadcast-service.ts @@ -8,12 +8,12 @@ import type { BroadcastSegment, BroadcastStatus, CreateEventBroadcastRequest, - HostBroadcastChannel, ListBroadcastDeliveriesRequest, ListEventBroadcastsRequest, PreviewEventBroadcastRequest, UpdateEventBroadcastRequest, } from "@civfix/shared" +import type { BroadcastVarValues } from "@civfix/shared/host" import type { Mailer } from "@civfix/shared/interfaces" import type { FastifyBaseLogger } from "fastify" import { assertNoSlur } from "../../abuse/slur-filter.js" @@ -28,20 +28,25 @@ import type { import { CRITICAL_BROADCAST_KINDS } from "./broadcast-types.js" import { assertBroadcastLinkPolicy, + broadcastContentOf, broadcastLinkWarnings, - eventManageUrl, - formatEventWhen, + eventTemplateVars, renderBroadcast, + verifiedReplyTo, } from "./broadcast-render.js" import { verifyUnsubscribeToken } from "./broadcast-capability-token.js" +import { audiencePages } from "./broadcast-audience.js" -export const BROADCAST_DEFAULT_LIMIT = 20 -export const BROADCAST_TEST_SENDS_PER_HOUR = 5 -export const TEST_SEND_WINDOW_SEC = 60 * 60 +const BROADCAST_DEFAULT_LIMIT = 20 +const BROADCAST_TEST_SENDS_PER_HOUR = 5 +const TEST_SEND_WINDOW_SEC = 60 * 60 export const DAY_SECONDS = 24 * 60 * 60 -export const AUDIENCE_PAGE_SIZE = 1000 -export const AUDIENCE_MAX_PAGES = 100 -export const LAST_UUID = "ffffffff-ffff-ffff-ffff-ffffffffffff" +const MS_PER_HOUR = 3_600_000 +const ISO_DAY_LENGTH = 10 +const COOLDOWN_SENDS = 1 +const PREVIEW_SAMPLE_FIRST_NAME = "Alex" +const PREVIEW_SAMPLE_TICKET_TYPE = "General" +const TEST_SUBJECT_PREFIX = "[Test] " export interface BroadcastConfig { killSwitch: boolean @@ -119,10 +124,19 @@ export function emailHashOf(email: string): string { } export function utcDayKey(at: Date): string { - return at.toISOString().slice(0, 10) + return at.toISOString().slice(0, ISO_DAY_LENGTH) } -export function toBroadcastDTO(record: BroadcastRecord): BroadcastDTO { +export async function withCaps(fn: () => Promise): Promise { + try { + return await fn() + } catch (err) { + if (err instanceof BroadcastCapError) throw capError(err.kind) + throw err + } +} + +function toBroadcastDTO(record: BroadcastRecord): BroadcastDTO { return { id: record.id, cleanupId: record.cleanupId, @@ -220,7 +234,7 @@ export function makeBroadcastService(deps: BroadcastServiceDeps): BroadcastServi if (!state.emailVerified) { throw new BroadcastCapError("unverified_email", CAP_COPY.unverified_email) } - const ageHours = (now().getTime() - state.accountCreatedAt.getTime()) / 3_600_000 + const ageHours = (now().getTime() - state.accountCreatedAt.getTime()) / MS_PER_HOUR if (ageHours < config.minAccountAgeHours) { throw new BroadcastCapError("account_too_new", CAP_COPY.account_too_new) } @@ -268,7 +282,7 @@ export function makeBroadcastService(deps: BroadcastServiceDeps): BroadcastServi async function reserveSendCounters(cleanupId: string, actorId: string): Promise { const cooldownKey = `bcast:cool:${actorId}:${cleanupId}` - await reserve(cooldownKey, config.cooldownSec, 1, "cooldown") + await reserve(cooldownKey, config.cooldownSec, COOLDOWN_SENDS, "cooldown") try { await reserve( `bcast:event:${cleanupId}:${utcDayKey(now())}`, @@ -339,7 +353,7 @@ export function makeBroadcastService(deps: BroadcastServiceDeps): BroadcastServi const event = await requireEventOrgNotSuspended(cleanupId) const moved = await repo.transition(broadcastId, ["draft", "scheduled"], "sending", { startedAt: now(), - replyTo: event.replyToVerified ? event.replyTo : null, + replyTo: verifiedReplyTo(event), }) if (moved === null) throw AppError.conflict("That message is already sending.") if (!skipEventSendCounters) { @@ -480,7 +494,7 @@ export function makeBroadcastService(deps: BroadcastServiceDeps): BroadcastServi eventTitle: event.title, vars: previewVars(event, config.webBaseUrl), unsubscribeUrl: `${config.webBaseUrl}/unsubscribe`, - replyTo: event.replyToVerified ? event.replyTo : null, + replyTo: verifiedReplyTo(event), allowedLinkHosts: config.linkAllowedHosts, }, ) @@ -517,28 +531,20 @@ export function makeBroadcastService(deps: BroadcastServiceDeps): BroadcastServi if (contact?.email == null) { throw AppError.validation({ email: "missing" }, "Add an email address to send a test.") } - const rendered = renderBroadcast( - { - subject: record.subject ?? "", - bodyMd: record.bodyMd ?? "", - ctaLabel: record.ctaLabel, - ctaUrl: record.ctaUrl, - }, - { - eventTitle: event.title, - vars: { - ...previewVars(event, config.webBaseUrl), - first_name: contact.firstName, - }, - unsubscribeUrl: `${config.webBaseUrl}/unsubscribe`, - replyTo: event.replyToVerified ? event.replyTo : null, - allowedLinkHosts: config.linkAllowedHosts, + const rendered = renderBroadcast(broadcastContentOf(record), { + eventTitle: event.title, + vars: { + ...previewVars(event, config.webBaseUrl), + first_name: contact.firstName, }, - ) + unsubscribeUrl: `${config.webBaseUrl}/unsubscribe`, + replyTo: verifiedReplyTo(event), + allowedLinkHosts: config.linkAllowedHosts, + }) await deps.mailer.sendOutbound({ from: config.mailFromEvents, to: contact.email, - subject: `[Test] ${rendered.subject}`, + subject: `${TEST_SUBJECT_PREFIX}${rendered.subject}`, text: rendered.text, html: rendered.html, headers: { "Auto-Submitted": "auto-generated" }, @@ -676,14 +682,11 @@ export function makeBroadcastService(deps: BroadcastServiceDeps): BroadcastServi } } -function previewVars(event: EventBroadcastContext, webBaseUrl: string): Record { +function previewVars(event: EventBroadcastContext, webBaseUrl: string): BroadcastVarValues { return { - first_name: "Alex", - event_title: event.title, - event_when: formatEventWhen(event.scheduledAt, event.timezone), - event_where: event.address ?? "the meeting point", - ticket_type: "General", - manage_link: eventManageUrl(webBaseUrl, event.pageSlug, event.cleanupId), + ...eventTemplateVars(event, webBaseUrl), + first_name: PREVIEW_SAMPLE_FIRST_NAME, + ticket_type: PREVIEW_SAMPLE_TICKET_TYPE, } } @@ -693,28 +696,11 @@ async function countAudience( segment: BroadcastSegment, cap: number, ): Promise { + if (cap <= 0) return cap let total = 0 - let afterMember: string | null = null - let afterGuest: string | null = null - let memberDone = false - let guestDone = false - for (let page = 0; page < AUDIENCE_MAX_PAGES && total < cap; page += 1) { - const result: { members: string[]; guests: string[] } = await repo.audiencePage({ - cleanupId, - segment, - kind: "host_broadcast", - afterMember: memberDone ? LAST_UUID : afterMember, - afterGuest: guestDone ? LAST_UUID : afterGuest, - limit: AUDIENCE_PAGE_SIZE, - }) - total += result.members.length + result.guests.length - if (result.members.length < AUDIENCE_PAGE_SIZE) memberDone = true - else afterMember = result.members.at(-1) ?? afterMember - if (result.guests.length < AUDIENCE_PAGE_SIZE) guestDone = true - else afterGuest = result.guests.at(-1) ?? afterGuest - if (memberDone && guestDone) break + for await (const page of audiencePages(repo, { cleanupId, segment, kind: "host_broadcast" })) { + total += page.members.length + page.guests.length + if (total >= cap) break } return Math.min(total, cap) } - -export type { HostBroadcastChannel, BroadcastStatus } diff --git a/services/api/src/services/host/broadcast-types.ts b/services/api/src/services/host/broadcast-types.ts index 68396f38..c7aec817 100644 --- a/services/api/src/services/host/broadcast-types.ts +++ b/services/api/src/services/host/broadcast-types.ts @@ -180,6 +180,11 @@ export interface AdminHostListParams { limit: number } +// Matches the broadcasts.chunk_size column default, which createIfAbsent relies on. +export const DEFAULT_BROADCAST_CHUNK_SIZE = 200 + +export const MAX_DELIVERY_ATTEMPTS = 3 + export const CRITICAL_BROADCAST_KINDS: ReadonlySet = new Set([ "event_updated", "event_cancelled", diff --git a/services/api/src/services/host/comms-jobs.ts b/services/api/src/services/host/comms-jobs.ts index 573af2d5..9145e25b 100644 --- a/services/api/src/services/host/comms-jobs.ts +++ b/services/api/src/services/host/comms-jobs.ts @@ -21,10 +21,15 @@ import { makeDrizzleHostTeamRepository } from "./host-team-repository.drizzle.js import { TEAM_INVITE_EMAIL_SCRUB_DELAY_MS } from "./host-team-service.js" import { runRegistrationRetentionLanes } from "./registration-retention.js" -export const DELIVERY_RETENTION_DAYS = 180 -export const BROADCAST_CONTENT_RETENTION_DAYS = 180 -export const EXPORT_ROW_RETENTION_DAYS = 90 -export const EXPORT_REAP_LIMIT = 200 +const DELIVERY_RETENTION_DAYS = 180 +const BROADCAST_CONTENT_RETENTION_DAYS = 180 +const EXPORT_ROW_RETENTION_DAYS = 90 +const EXPORT_REAP_LIMIT = 200 +const MS_PER_DAY = 86_400_000 + +function daysBefore(now: Date, days: number): Date { + return new Date(now.getTime() - days * MS_PER_DAY) +} export async function registerCommsJobs( container: Container, @@ -33,25 +38,30 @@ export async function registerCommsJobs( const env = container.env const runtime = () => makeCommsRuntime(container, logger) + async function workParsed( + jobName: string, + parseJob: (data: unknown) => T | null, + run: (data: T) => Promise, + ): Promise { + await container.jobs.work(jobName, async (job) => { + const data = parseJob(job.data) + if (data === null) { + logger?.warn({ jobId: job.id }, `${jobName}: malformed job data (skipped)`) + return + } + await run(data) + }) + } + registerHostRetentionLanes() - await container.jobs.work(BROADCAST_PLAN_JOB, async (job) => { - const data = parseBroadcastPlanJob(job.data) - if (data === null) { - logger?.warn({ jobId: job.id }, "broadcast.plan: malformed job data (skipped)") - return - } - await runtime().pipeline.plan(data.broadcastId) - }) + await workParsed(BROADCAST_PLAN_JOB, parseBroadcastPlanJob, (data) => + runtime().pipeline.plan(data.broadcastId), + ) - await container.jobs.work(BROADCAST_CHUNK_JOB, async (job) => { - const data = parseBroadcastChunkJob(job.data) - if (data === null) { - logger?.warn({ jobId: job.id }, "broadcast.chunk: malformed job data (skipped)") - return - } - await runtime().pipeline.runChunk(data.broadcastId, data.chunkNo, data.authRetry) - }) + await workParsed(BROADCAST_CHUNK_JOB, parseBroadcastChunkJob, (data) => + runtime().pipeline.runChunk(data.broadcastId, data.chunkNo, data.authRetry), + ) await container.jobs.schedule(BROADCAST_SCHEDULE_SWEEP_JOB, env.BROADCAST_SWEEP_CRON) await container.jobs.work(BROADCAST_SCHEDULE_SWEEP_JOB, async () => { @@ -75,14 +85,9 @@ export async function registerCommsJobs( ) }) - await container.jobs.work(HOST_EXPORT_JOB, async (job) => { - const data = parseHostExportJob(job.data) - if (data === null) { - logger?.warn({ jobId: job.id }, "host.export: malformed job data (skipped)") - return - } - await runtime().exports.run(data.exportId) - }) + await workParsed(HOST_EXPORT_JOB, parseHostExportJob, (data) => + runtime().exports.run(data.exportId), + ) await container.jobs.schedule(HOST_EXPORT_REAP_JOB, env.HOST_EXPORT_REAP_CRON) await container.jobs.work(HOST_EXPORT_REAP_JOB, async () => { @@ -99,19 +104,19 @@ export async function registerCommsJobs( export function registerHostRetentionLanes(): void { registerRetentionLane("broadcast_deliveries", async (sql, now) => { const repo = makeDrizzleBroadcastRepository(sql) - const cutoff = new Date(now.getTime() - DELIVERY_RETENTION_DAYS * 86_400_000) + const cutoff = daysBefore(now, DELIVERY_RETENTION_DAYS) return drainTable((batchSize) => repo.deleteOldDeliveries(cutoff, batchSize)) }) registerRetentionLane("broadcast_content", async (sql, now) => { const repo = makeDrizzleBroadcastRepository(sql) - const cutoff = new Date(now.getTime() - BROADCAST_CONTENT_RETENTION_DAYS * 86_400_000) + const cutoff = daysBefore(now, BROADCAST_CONTENT_RETENTION_DAYS) return drainTable((batchSize) => repo.scrubBroadcastContent(cutoff, batchSize)) }) registerRetentionLane("host_exports", async (sql, now) => { const repo = makeDrizzleHostExportRepository(sql) - const cutoff = new Date(now.getTime() - EXPORT_ROW_RETENTION_DAYS * 86_400_000) + const cutoff = daysBefore(now, EXPORT_ROW_RETENTION_DAYS) return drainTable((batchSize) => repo.deleteOlderThan(cutoff, batchSize)) }) diff --git a/services/api/src/services/host/comms-wiring.ts b/services/api/src/services/host/comms-wiring.ts index 43cbf2b5..4139f2d0 100644 --- a/services/api/src/services/host/comms-wiring.ts +++ b/services/api/src/services/host/comms-wiring.ts @@ -34,12 +34,15 @@ import { makeMetricsService, type MetricsService } from "./metrics-service.js" import { makeDrizzleHostExportRepository } from "./export-repository.drizzle.js" import { DEFAULT_EXPORT_MAX_BYTES, + EXPORT_NOT_FOUND, makeHostExportService, type HostExportService, } from "./export-service.js" import { registerEventExportBuilders } from "./host-export-builders.js" import { requireCapability } from "./authz.js" +const BROADCAST_JOB_RETRY_LIMIT = 3 + export type CommsLogger = Pick export interface CommsRuntime { @@ -118,6 +121,7 @@ export function makeCommsRuntime(container: Container, logger?: CommsLogger): Co const env = container.env const sql = container.getDb().sql const config = broadcastConfigOf(container) + const optionalLogger = logger !== undefined ? { logger } : {} if (!buildersRegistered) { registerEventExportBuilders(() => container.getDb().sql) @@ -127,19 +131,21 @@ export function makeCommsRuntime(container: Container, logger?: CommsLogger): Co const repo = makeDrizzleBroadcastRepository(sql) const metricsRepo = makeDrizzleMetricsRepository(sql) + const enqueuePlan = async (broadcastId: string): Promise => { + await container.jobs.enqueue( + BROADCAST_PLAN_JOB, + { broadcastId }, + { singletonKey: `plan:${broadcastId}`, retryLimit: BROADCAST_JOB_RETRY_LIMIT }, + ) + } + const broadcasts = makeBroadcastService({ repo, counters: container.getCounterStore(), config, mailer: container.mailer, - enqueuePlan: async (broadcastId) => { - await container.jobs.enqueue( - BROADCAST_PLAN_JOB, - { broadcastId }, - { singletonKey: `plan:${broadcastId}`, retryLimit: 3 }, - ) - }, - ...(logger !== undefined ? { logger } : {}), + enqueuePlan, + ...optionalLogger, }) const announcements = makeAnnouncementService({ @@ -149,7 +155,7 @@ export function makeCommsRuntime(container: Container, logger?: CommsLogger): Co ), broadcasts, config, - ...(logger !== undefined ? { logger } : {}), + ...optionalLogger, }) const pipeline = makeBroadcastPipeline({ @@ -166,28 +172,22 @@ export function makeCommsRuntime(container: Container, logger?: CommsLogger): Co { broadcastId, chunkNo, authRetry: opts?.authRetry ?? 0 }, { singletonKey: `chunk:${broadcastId}:${chunkNo}`, - retryLimit: 3, + retryLimit: BROADCAST_JOB_RETRY_LIMIT, ...(opts?.startAfterSec !== undefined ? { startAfter: opts.startAfterSec } : {}), }, ) }, audit: (action, actorId, target, meta) => auditBestEffort(sql, { action, actorId, target, meta }, logger), - ...(logger !== undefined ? { logger } : {}), + ...optionalLogger, }) const lanes = makeBroadcastLanes({ repo, counters: container.getCounterStore(), perEventPerHour: config.eventUpdatePerEventPerHour, - enqueuePlan: async (broadcastId) => { - await container.jobs.enqueue( - BROADCAST_PLAN_JOB, - { broadcastId }, - { singletonKey: `plan:${broadcastId}`, retryLimit: 3 }, - ) - }, - ...(logger !== undefined ? { logger } : {}), + enqueuePlan, + ...optionalLogger, }) const metrics = makeMetricsService({ @@ -195,14 +195,14 @@ export function makeCommsRuntime(container: Container, logger?: CommsLogger): Co cache: container.getCache(), selfHosts: selfHostsOf(container.env.WEB_ORIGINS), lookbackDays: env.METRICS_ROLLUP_LOOKBACK_DAYS, - ...(logger !== undefined ? { logger } : {}), + ...optionalLogger, }) const analyticsRepo = makeDrizzleAnalyticsRepository(sql) const analyticsCache = makeHostAnalyticsCache({ cache: container.getCache(), ttlSeconds: env.HOST_ANALYTICS_CACHE_TTL_SEC, - ...(logger !== undefined ? { logger } : {}), + ...optionalLogger, }) const analytics = makeAnalyticsService({ @@ -233,10 +233,10 @@ export function makeCommsRuntime(container: Container, logger?: CommsLogger): Co ttlHours: env.HOST_EXPORT_TTL_HOURS, }, authorize: async (record) => { - if (record.cleanupId === null) throw AppError.notFound("Export not found") + if (record.cleanupId === null) throw AppError.notFound(EXPORT_NOT_FOUND) await requireCapability(sql, record.cleanupId, record.requestedBy, "export") }, - ...(logger !== undefined ? { logger } : {}), + ...optionalLogger, }) return { diff --git a/services/api/src/services/host/event-analytics-repository.drizzle.ts b/services/api/src/services/host/event-analytics-repository.drizzle.ts index baff633d..9c28e36d 100644 --- a/services/api/src/services/host/event-analytics-repository.drizzle.ts +++ b/services/api/src/services/host/event-analytics-repository.drizzle.ts @@ -4,7 +4,7 @@ import type { Sql } from "../../db/client.js" type Fragment = postgres.Fragment -export const EVENT_ANALYTICS_SLOT_ROW_LIMIT = 50 +const EVENT_ANALYTICS_SLOT_ROW_LIMIT = 50 export interface EventAnalyticsFacts { walkUps: number @@ -13,7 +13,7 @@ export interface EventAnalyticsFacts { postsCreated: number } -export const ZERO_EVENT_ANALYTICS_FACTS: EventAnalyticsFacts = Object.freeze({ +const ZERO_EVENT_ANALYTICS_FACTS: EventAnalyticsFacts = Object.freeze({ walkUps: 0, reportsLinked: 0, reportsResolved: 0, @@ -42,6 +42,18 @@ export interface EventAnalyticsRepository { comparisonMedians(cleanupIds: readonly string[]): Promise } +const EMPTY_COMPARISON_MEDIANS: EventComparisonMedians = Object.freeze({ + sampleSize: 0, + signups: null, + checkInRate: null, + hoursPerVolunteer: null, + fillRate: null, +}) + +function keyCounts(rows: { key: string; n: number }[]): KeyCount[] { + return rows.map((row) => ({ key: row.key, count: row.n })) +} + export function makeDrizzleEventAnalyticsRepository(sql: Sql): EventAnalyticsRepository { return { async previousCompletedEventIds(args) { @@ -123,7 +135,7 @@ export function makeDrizzleEventAnalyticsRepository(sql: Sql): EventAnalyticsRep GROUP BY sl.title ORDER BY count(*) DESC, sl.title ASC LIMIT ${EVENT_ANALYTICS_SLOT_ROW_LIMIT}` - return rows.map((row) => ({ key: row.key, count: row.n })) + return keyCounts(rows) }, async reportStatuses(cleanupId) { @@ -135,7 +147,7 @@ export function makeDrizzleEventAnalyticsRepository(sql: Sql): EventAnalyticsRep GROUP BY rep.status ORDER BY count(*) DESC, rep.status ASC LIMIT ${EVENT_ANALYTICS_SLOT_ROW_LIMIT}` - return rows.map((row) => ({ key: row.key, count: row.n })) + return keyCounts(rows) }, async hoursBuckets(cleanupId) { @@ -157,19 +169,11 @@ export function makeDrizzleEventAnalyticsRepository(sql: Sql): EventAnalyticsRep count(*)::int AS n FROM per_person GROUP BY 1` - return rows.map((row) => ({ key: row.key, count: row.n })) + return keyCounts(rows) }, async comparisonMedians(cleanupIds) { - if (cleanupIds.length === 0) { - return { - sampleSize: 0, - signups: null, - checkInRate: null, - hoursPerVolunteer: null, - fillRate: null, - } - } + if (cleanupIds.length === 0) return { ...EMPTY_COMPARISON_MEDIANS } const rows = await sql< { sample_size: number diff --git a/services/api/src/services/host/event-analytics-service.ts b/services/api/src/services/host/event-analytics-service.ts index 45044911..b475d3f9 100644 --- a/services/api/src/services/host/event-analytics-service.ts +++ b/services/api/src/services/host/event-analytics-service.ts @@ -11,39 +11,61 @@ import { type BreakdownRow, type EventAnalyticsPhase, type EventAnalyticsScope, - type FunnelStep, type GetEventAnalyticsResponse, type Panel, type SeriesPoint, - type SuppressedRate, } from "@civfix/shared" import { breakdown, dailySeries, - eventPhase, funnel, seriesClosure, - suppressRate, type DayCount, type DayRange, type DerivedBreakdownRow, type DerivedPanel, type KeyCount, - type SeriesClosure, } from "@civfix/shared/host" -import type { AnalyticsRepository, EventClockRecord } from "./analytics-repository.drizzle.js" -import type { EventAnalyticsRepository } from "./event-analytics-repository.drizzle.js" +import type { + AnalyticsRepository, + EventClockRecord, + EventHoursTotals, + EventKpiRow, + SourceSeats, +} from "./analytics-repository.drizzle.js" +import type { + EventAnalyticsFacts, + EventAnalyticsRepository, +} from "./event-analytics-repository.drizzle.js" import type { MetricRow, MetricsRepository } from "./metrics-repository.drizzle.js" -import { hostAnalyticsCacheKey, type HostAnalyticsCache } from "./host-analytics-cache.js" +import { + hostAnalyticsCacheKey, + perViewerScope, + type HostAnalyticsCache, +} from "./host-analytics-cache.js" import { eventDayKey } from "./event-day.js" import { DEFAULT_EVENT_TIME_ZONE } from "./event-fields.js" import { ARRIVAL_SAMPLE_LIMIT } from "./analytics-service.js" +import { METRIC_DONATION_CLICKS, METRIC_PAGE_VIEWS } from "./event-metric-names.js" +import { + DAY_MS, + clockPhase, + closureAllowsTotal, + emptyRate, + metricTotal, + seriesOf, + shiftDayKey, + toFunnelSteps, + toRate, + toSeries, +} from "./host-analytics-shaping.js" -export const EVENT_ANALYTICS_LIFECYCLE_DAYS = MAX_EVENT_ANALYTICS_SERIES_POINTS -export const EVENT_ANALYTICS_ARCHIVE_DAYS = 30 -export const EVENT_ANALYTICS_ARRIVAL_BUCKET_MIN = 15 -export const EVENT_ANALYTICS_DELTA_DAYS = 7 -export const DAY_MS = 86_400_000 +const EVENT_ANALYTICS_LIFECYCLE_DAYS = MAX_EVENT_ANALYTICS_SERIES_POINTS +const EVENT_ANALYTICS_ARCHIVE_DAYS = 30 +const EVENT_ANALYTICS_ARRIVAL_BUCKET_MIN = 15 +const EVENT_ANALYTICS_DELTA_DAYS = 7 +const EVENT_ANALYTICS_CACHE_ENDPOINT = "event-analytics" +const FULL_SCOPE: EventAnalyticsScope = "full" export interface EventAnalyticsServiceDeps { analytics: AnalyticsRepository @@ -67,36 +89,6 @@ export interface EventAnalyticsService { ): Promise } -function shiftDayKey(day: string, deltaDays: number): string { - const [year, month, date] = day.split("-").map(Number) - const shifted = new Date(Date.UTC(year ?? 1970, (month ?? 1) - 1, date ?? 1) + deltaDays * DAY_MS) - return shifted.toISOString().slice(0, 10) -} - -function emptyRate(): SuppressedRate { - return { value: null, numerator: null, denominator: null, suppressed: true } -} - -function toRate(numerator: number, denominator: number): SuppressedRate { - const ratio = suppressRate(numerator, denominator) - return { - value: ratio.value, - numerator: ratio.suppressed ? null : numerator, - denominator: ratio.suppressed ? null : denominator, - suppressed: ratio.suppressed, - } -} - -function toSeries( - points: readonly { day: string; value: number | null; suppressed: boolean }[], -): SeriesPoint[] { - return points.map((point) => ({ - day: point.day, - value: point.value, - suppressed: point.suppressed, - })) -} - function toPanel(panel: DerivedPanel): Panel { return { panelSuppressed: panel.panelSuppressed, @@ -111,32 +103,6 @@ function toPanel(panel: DerivedPanel): Panel { } } -function toFunnelSteps( - panel: DerivedPanel<{ key: string; value: number | null; suppressed: boolean }>, -): FunnelStep[] { - return panel.rows.map((row) => ({ - step: row.key, - label: row.key, - value: row.value, - suppressed: row.suppressed, - })) -} - -function metricTotal(rows: readonly MetricRow[], metric: string): number | null { - const matching = rows.filter((row) => row.metric === metric) - if (matching.length === 0) return null - return matching.reduce((acc, row) => acc + row.value, 0) -} - -function seriesOf(rows: readonly MetricRow[], metric: string): DayCount[] { - const byDay = new Map() - for (const row of rows) { - if (row.metric !== metric) continue - byDay.set(row.day, (byDay.get(row.day) ?? 0) + row.value) - } - return [...byDay].map(([day, count]) => ({ day, count })) -} - function tailSum(points: readonly { value: number | null }[], days: number): number | null { const tail = points.slice(-days) let total = 0 @@ -147,20 +113,18 @@ function tailSum(points: readonly { value: number | null }[], days: number): num return total } -function closureAllowsTotal(closure: SeriesClosure): boolean { - return closure.panelSuppressed || closure.totalPublishable +function cardSlotPanel( + rows: readonly KeyCount[], + registeredPublishable: boolean, + full: boolean, +): Panel { + const panel = toPanel(breakdown(rows, { totalPublishable: registeredPublishable })) + if (full) return panel + return { ...panel, rows: panel.rows.slice(0, EVENT_ANALYTICS_CARD_SLOT_ROWS) } } export function analyticsPhaseOf(clock: EventClockRecord, at: Date): EventAnalyticsPhase { - const phase = eventPhase( - { - status: clock.status, - scheduledAt: clock.scheduledAt.toISOString(), - endsAt: clock.endsAt?.toISOString() ?? null, - completedAt: clock.completedAt?.toISOString() ?? null, - }, - at.getTime(), - ) + const phase = clockPhase(clock, at) if (phase === "live") return "day_of" if (phase !== "ended") return "upcoming" const endedAt = clock.completedAt ?? clock.endsAt ?? clock.scheduledAt @@ -185,6 +149,153 @@ export function arrivalBuckets(offsets: readonly number[]): SeriesPoint[] { })) } +interface EventAnalyticsInputs { + kpis: EventKpiRow + facts: EventAnalyticsFacts + metricRows: MetricRow[] + registrationDays: DayCount[] + cancellationDays: DayCount[] + offsets: number[] + waitlist: { promoted: number; joined: number } + hours: EventHoursTotals + bySource: SourceSeats[] + registrationsBySlot: KeyCount[] + checkinsBySlot: KeyCount[] + hoursBuckets: KeyCount[] + reportStatuses: KeyCount[] + comparison: GetEventAnalyticsResponse["comparison"] +} + +interface EventAnalyticsContext { + scope: EventAnalyticsScope + phase: EventAnalyticsPhase + clock: EventClockRecord + at: Date + window: DayRange + full: boolean +} + +function kpisOf( + inputs: EventAnalyticsInputs, + pageViews: number | null, + donationClicks: number | null, +): GetEventAnalyticsResponse["kpis"] { + const { kpis, facts, hours } = inputs + return { + signups: kpis.registered, + capacity: kpis.capacity, + waitlisted: kpis.waitlisted, + cancelled: kpis.cancelled, + checkedIn: kpis.checkedIn, + noShow: kpis.noShow, + walkUps: facts.walkUps, + pageViews, + uniqueViewers: null, + shares: null, + donationClicks, + hoursTotal: hours.credited, + hoursVolunteers: hours.attendeesCredited, + reportsLinked: facts.reportsLinked, + reportsResolved: facts.reportsResolved, + postsCreated: facts.postsCreated, + } +} + +function ratesOf( + inputs: EventAnalyticsInputs, + pageViews: number | null, + registeredPublishable: boolean, +): GetEventAnalyticsResponse["rates"] { + const { kpis, waitlist } = inputs + return { + checkIn: registeredPublishable ? toRate(kpis.checkedIn, kpis.registered) : emptyRate(), + noShow: registeredPublishable ? toRate(kpis.noShow, kpis.registered) : emptyRate(), + fill: + kpis.capacity === null || kpis.capacity === 0 || !registeredPublishable + ? emptyRate() + : toRate(kpis.registered, kpis.capacity), + viewToSignup: + pageViews === null || pageViews === 0 || !registeredPublishable + ? emptyRate() + : toRate(kpis.registered, pageViews), + waitlistConversion: toRate(waitlist.promoted, waitlist.joined), + } +} + +function shapeResponse( + inputs: EventAnalyticsInputs, + ctx: EventAnalyticsContext, +): GetEventAnalyticsResponse { + const { window, full, clock } = ctx + const closure = seriesClosure(inputs.registrationDays, window, { suppressPoints: true }) + const registeredPublishable = closureAllowsTotal(closure) + const cancellations = dailySeries(inputs.cancellationDays, window, { suppressPoints: true }) + const viewsSeries = dailySeries(seriesOf(inputs.metricRows, METRIC_PAGE_VIEWS), window) + const pageViews = metricTotal(inputs.metricRows, METRIC_PAGE_VIEWS) + const donationClicks = metricTotal(inputs.metricRows, METRIC_DONATION_CLICKS) + + const steps: KeyCount[] = [ + { key: "signups", count: inputs.kpis.registered }, + { key: "checked_in", count: inputs.kpis.checkedIn }, + { key: "logged_hours", count: inputs.hours.attendeesCredited }, + ] + + const cumulative = toSeries(closure.cumulative) + const daily = toSeries(closure.daily) + const viewsDaily = toSeries(viewsSeries.points) + const arrivals = arrivalBuckets(inputs.offsets) + + return { + generatedAt: ctx.at.toISOString(), + k: ANALYTICS_SUPPRESSION_K, + scope: ctx.scope, + phase: ctx.phase, + lifecycle: { + createdAt: clock.createdAt.toISOString(), + startAt: clock.scheduledAt.toISOString(), + endAt: clock.endsAt?.toISOString() ?? null, + completedAt: clock.completedAt?.toISOString() ?? null, + }, + kpis: kpisOf(inputs, pageViews, donationClicks), + rates: ratesOf(inputs, pageViews, registeredPublishable), + deltas: { + signups7d: tailSum(daily, EVENT_ANALYTICS_DELTA_DAYS), + views7d: tailSum(viewsDaily, EVENT_ANALYTICS_DELTA_DAYS), + }, + signups: { + cumulative: full ? cumulative : cumulative.slice(-EVENT_ANALYTICS_CARD_SERIES_POINTS), + daily: full ? daily : [], + cancellations: full ? toSeries(cancellations.points) : [], + bySlot: cardSlotPanel(inputs.registrationsBySlot, registeredPublishable, full), + ...(full + ? { + bySource: toPanel( + breakdown( + inputs.bySource.map((row) => ({ key: row.source, count: row.seats })), + { totalPublishable: registeredPublishable }, + ), + ), + } + : {}), + }, + reach: { + viewsDaily: full ? viewsDaily : viewsDaily.slice(-EVENT_ANALYTICS_CARD_SERIES_POINTS), + funnel: full ? toFunnelSteps(funnel(steps)) : [], + }, + eventDay: { + arrivals: full ? arrivals : arrivals.slice(-EVENT_ANALYTICS_CARD_SERIES_POINTS), + ...(full ? { bySlot: toPanel(breakdown(inputs.checkinsBySlot)) } : {}), + }, + impact: full + ? { + hoursBuckets: toPanel(breakdown(inputs.hoursBuckets)), + reportStatuses: toPanel(breakdown(inputs.reportStatuses)), + } + : {}, + comparison: inputs.comparison, + } +} + export function makeEventAnalyticsService(deps: EventAnalyticsServiceDeps): EventAnalyticsService { const now = deps.now ?? (() => new Date()) @@ -218,18 +329,13 @@ export function makeEventAnalyticsService(deps: EventAnalyticsServiceDeps): Even } } - async function compute( + async function loadInputs( cleanupId: string, - scope: EventAnalyticsScope, viewer: EventAnalyticsViewer, - clock: EventClockRecord, - phase: EventAnalyticsPhase, - at: Date, - ): Promise { - const timezone = clock.timezone ?? DEFAULT_EVENT_TIME_ZONE - const window = lifecycleWindow(clock, timezone) - const full = scope === "full" - + timezone: string, + window: DayRange, + full: boolean, + ): Promise { const [ kpis, facts, @@ -248,7 +354,12 @@ export function makeEventAnalyticsService(deps: EventAnalyticsServiceDeps): Even ] = await Promise.all([ deps.analytics.eventKpis(cleanupId), deps.events.facts(cleanupId), - deps.metrics.read(cleanupId, ["page_views", "donation_clicks"], window.from, window.to), + deps.metrics.read( + cleanupId, + [METRIC_PAGE_VIEWS, METRIC_DONATION_CLICKS], + window.from, + window.to, + ), deps.analytics.registrationsByDay(cleanupId, timezone, window.from, window.to), deps.analytics.cancellationsByDay(cleanupId, timezone, window.from, window.to), deps.analytics.arrivalOffsets(cleanupId, ARRIVAL_SAMPLE_LIMIT), @@ -261,114 +372,37 @@ export function makeEventAnalyticsService(deps: EventAnalyticsServiceDeps): Even full ? deps.events.reportStatuses(cleanupId) : Promise.resolve([]), full ? comparisonOf(cleanupId, viewer) : Promise.resolve(null), ]) - - const closure = seriesClosure(registrationDays, window, { suppressPoints: true }) - const registeredPublishable = closureAllowsTotal(closure) - const cancellations = dailySeries(cancellationDays, window, { suppressPoints: true }) - const viewsSeries = dailySeries(seriesOf(metricRows, "page_views"), window) - const pageViews = metricTotal(metricRows, "page_views") - const donationClicks = metricTotal(metricRows, "donation_clicks") - - const steps: KeyCount[] = [ - { key: "signups", count: kpis.registered }, - { key: "checked_in", count: kpis.checkedIn }, - { key: "logged_hours", count: hours.attendeesCredited }, - ] - - const cumulative = toSeries(closure.cumulative) - const daily = toSeries(closure.daily) - const viewsDaily = toSeries(viewsSeries.points) - const arrivals = arrivalBuckets(offsets) - - const response: GetEventAnalyticsResponse = { - generatedAt: at.toISOString(), - k: ANALYTICS_SUPPRESSION_K, - scope, - phase, - lifecycle: { - createdAt: clock.createdAt.toISOString(), - startAt: clock.scheduledAt.toISOString(), - endAt: clock.endsAt?.toISOString() ?? null, - completedAt: clock.completedAt?.toISOString() ?? null, - }, - kpis: { - signups: kpis.registered, - capacity: kpis.capacity, - waitlisted: kpis.waitlisted, - cancelled: kpis.cancelled, - checkedIn: kpis.checkedIn, - noShow: kpis.noShow, - walkUps: facts.walkUps, - pageViews, - uniqueViewers: null, - shares: null, - donationClicks, - hoursTotal: hours.credited, - hoursVolunteers: hours.attendeesCredited, - reportsLinked: facts.reportsLinked, - reportsResolved: facts.reportsResolved, - postsCreated: facts.postsCreated, - }, - rates: { - checkIn: registeredPublishable ? toRate(kpis.checkedIn, kpis.registered) : emptyRate(), - noShow: registeredPublishable ? toRate(kpis.noShow, kpis.registered) : emptyRate(), - fill: - kpis.capacity === null || kpis.capacity === 0 || !registeredPublishable - ? emptyRate() - : toRate(kpis.registered, kpis.capacity), - viewToSignup: - pageViews === null || pageViews === 0 || !registeredPublishable - ? emptyRate() - : toRate(kpis.registered, pageViews), - waitlistConversion: toRate(waitlist.promoted, waitlist.joined), - }, - deltas: { - signups7d: tailSum(daily, EVENT_ANALYTICS_DELTA_DAYS), - views7d: tailSum(viewsDaily, EVENT_ANALYTICS_DELTA_DAYS), - }, - signups: { - cumulative: full ? cumulative : cumulative.slice(-EVENT_ANALYTICS_CARD_SERIES_POINTS), - daily: full ? daily : [], - cancellations: full ? toSeries(cancellations.points) : [], - bySlot: cardSlotPanel(registrationsBySlot, registeredPublishable, full), - ...(full - ? { - bySource: toPanel( - breakdown( - bySource.map((row) => ({ key: row.source, count: row.seats })), - { totalPublishable: registeredPublishable }, - ), - ), - } - : {}), - }, - reach: { - viewsDaily: full ? viewsDaily : viewsDaily.slice(-EVENT_ANALYTICS_CARD_SERIES_POINTS), - funnel: full ? toFunnelSteps(funnel(steps)) : [], - }, - eventDay: { - arrivals: full ? arrivals : arrivals.slice(-EVENT_ANALYTICS_CARD_SERIES_POINTS), - ...(full ? { bySlot: toPanel(breakdown(checkinsBySlot)) } : {}), - }, - impact: full - ? { - hoursBuckets: toPanel(breakdown(hoursBuckets)), - reportStatuses: toPanel(breakdown(reportStatuses)), - } - : {}, + return { + kpis, + facts, + metricRows, + registrationDays, + cancellationDays, + offsets, + waitlist, + hours, + bySource, + registrationsBySlot, + checkinsBySlot, + hoursBuckets, + reportStatuses, comparison, } - return response } - function cardSlotPanel( - rows: readonly KeyCount[], - registeredPublishable: boolean, - full: boolean, - ): Panel { - const panel = toPanel(breakdown(rows, { totalPublishable: registeredPublishable })) - if (full) return panel - return { ...panel, rows: panel.rows.slice(0, EVENT_ANALYTICS_CARD_SLOT_ROWS) } + async function compute( + cleanupId: string, + scope: EventAnalyticsScope, + viewer: EventAnalyticsViewer, + clock: EventClockRecord, + phase: EventAnalyticsPhase, + at: Date, + ): Promise { + const timezone = clock.timezone ?? DEFAULT_EVENT_TIME_ZONE + const window = lifecycleWindow(clock, timezone) + const full = scope === FULL_SCOPE + const inputs = await loadInputs(cleanupId, viewer, timezone, window, full) + return shapeResponse(inputs, { scope, phase, clock, at, window, full }) } return { @@ -380,10 +414,10 @@ export function makeEventAnalyticsService(deps: EventAnalyticsServiceDeps): Even const generation = await deps.cache.generationOf(cleanupId) return deps.cache.getOrSet( hostAnalyticsCacheKey({ - endpoint: "event-analytics", + endpoint: EVENT_ANALYTICS_CACHE_ENDPOINT, scope: cleanupId, range: `${scope}:${phase}`, - viewerScope: `${viewer.viewerScope}:${viewer.userId}`, + viewerScope: perViewerScope(viewer), generation, }), () => compute(cleanupId, scope, viewer, clock, phase, at), diff --git a/services/api/src/services/host/event-metric-names.ts b/services/api/src/services/host/event-metric-names.ts new file mode 100644 index 00000000..4a224fd6 --- /dev/null +++ b/services/api/src/services/host/event-metric-names.ts @@ -0,0 +1,12 @@ +// The rollup SQL in metrics-repository.drizzle.ts writes the same names as literals; keep them in step. +export const METRIC_PAGE_VIEWS = "page_views" +export const METRIC_SOURCE = "source" +export const METRIC_DONATION_CLICKS = "donation_clicks" +export const METRIC_REGISTRATIONS = "registrations" +export const METRIC_BROADCAST_RECIPIENTS = "broadcast_recipients" +export const METRIC_BROADCAST_SENT = "broadcast_sent" +export const METRIC_BROADCAST_FAILED = "broadcast_failed" +export const METRIC_BROADCAST_SUPPRESSED = "broadcast_suppressed" +export const METRIC_UNSUBSCRIBES = "unsubscribes" + +export const NO_BUCKET = "" diff --git a/services/api/src/services/host/export-service.ts b/services/api/src/services/host/export-service.ts index 848ad126..26341b6a 100644 --- a/services/api/src/services/host/export-service.ts +++ b/services/api/src/services/host/export-service.ts @@ -3,15 +3,27 @@ import type { HostExportDTO, HostExportFilters, HostExportKind } from "@civfix/s import type { Storage } from "@civfix/shared/interfaces" import type { FastifyBaseLogger } from "fastify" import { csvProvenanceRow, csvRow } from "./export-csv.js" -import { hostExportBuilder, type HostExportContext } from "./export-builders.js" +import { + hostExportBuilder, + type HostExportBuilder, + type HostExportContext, +} from "./export-builders.js" import type { HostExportRecord, HostExportRepository } from "./export-repository.drizzle.js" import type { WriteAuditInput } from "../admin/audit.js" -export const EXPORT_DOWNLOAD_URL_TTL_SEC = 300 -export const EXPORT_LIST_LIMIT = 50 -export const EXPORT_YIELD_EVERY_ROWS = 1000 +const MS_PER_SECOND = 1000 +const MS_PER_HOUR = 3_600_000 +const EXPORT_DOWNLOAD_URL_TTL_SEC = 300 +const EXPORT_LIST_LIMIT = 50 +const EXPORT_YIELD_EVERY_ROWS = 1000 export const EXPORT_RUN_STALE_MS = 10 * 60 * 1000 export const EXPORT_ABANDON_AFTER_MS = 60 * 60 * 1000 +const EXPORT_CONTENT_TYPE = "text/csv; charset=utf-8" +const FALLBACK_EXPORT_FILENAME = "civfix-export.csv" +const ERROR_BUILD_FAILED = "build_failed" +const ERROR_FORBIDDEN = "forbidden" +const ERROR_NOT_STARTED = "not_started" +export const EXPORT_NOT_FOUND = "Export not found" export interface HostExportConfig { maxRows: number @@ -27,6 +39,13 @@ interface PresignStorage { delete(key: string): Promise } +interface RenderedCsv { + parts: Buffer[] + bytes: number + rowCount: number + truncated: boolean +} + export interface HostExportServiceDeps { repo: HostExportRepository storage: PresignStorage @@ -93,7 +112,7 @@ export function makeHostExportService(deps: HostExportServiceDeps): HostExportSe await deps.storage.delete(key) await deps.repo.releaseObject( { id: claimed.id, status: "failed", runToken: claimed.runToken }, - "build_failed", + ERROR_BUILD_FAILED, ) } catch (err) { deps.logger?.warn( @@ -103,6 +122,133 @@ export function makeHostExportService(deps: HostExportServiceDeps): HostExportSe } } + async function claimIsAuthorized(claimed: HostExportRecord): Promise { + if (deps.authorize === undefined) return true + try { + await deps.authorize(claimed) + return true + } catch (err) { + deps.logger?.warn( + { err, exportId: claimed.id }, + "host export refused: the requester no longer holds the capability", + ) + await deps.repo.markFailed(claimed.id, ERROR_FORBIDDEN, claimed.runToken) + return false + } + } + + async function renderCsv( + builder: HostExportBuilder, + ctx: HostExportContext, + ): Promise { + const provenance = await builder.provenance(ctx) + const header = await builder.header(ctx) + const parts: Buffer[] = [] + let bytes = 0 + const append = (text: string): void => { + const buf = Buffer.from(text, "utf8") + parts.push(buf) + bytes += buf.byteLength + } + for (const line of provenance) append(csvProvenanceRow(line)) + append(csvRow(header)) + + let rowCount = 0 + let truncated = false + for await (const row of builder.rows(ctx)) { + if (rowCount >= deps.config.maxRows || bytes >= deps.config.maxBytes) { + truncated = true + break + } + append(csvRow(row)) + rowCount += 1 + if (rowCount % EXPORT_YIELD_EVERY_ROWS === 0) { + await new Promise((resolve) => setImmediate(resolve)) + } + } + if (truncated) { + append( + csvProvenanceRow( + `truncated: this file stops at ${rowCount} rows (${deps.config.maxRows} row / ${deps.config.maxBytes} byte cap)`, + ), + ) + } + return { parts, bytes, rowCount, truncated } + } + + async function discardSupersededObject(claimed: HostExportRecord, key: string): Promise { + await deps.storage.delete(key).catch((err: unknown) => { + deps.logger?.warn( + { err, exportId: claimed.id, key }, + "host export: losing run could not delete its own object", + ) + }) + deps.logger?.warn( + { evt: "host.export.superseded", exportId: claimed.id }, + "host export run superseded by a newer claim; its object was discarded", + ) + } + + async function reapExpired(limit: number): Promise { + const expired = await deps.repo.listExpired(now(), limit) + let reaped = 0 + for (const record of expired) { + if (record.r2Key !== null) { + try { + await deps.storage.delete(record.r2Key) + } catch (err) { + deps.logger?.warn( + { err, exportId: record.id }, + "host export reap: object delete failed; row left ready for the next pass", + ) + continue + } + } + await deps.repo.markExpired(record.id) + reaped += 1 + } + return reaped + } + + async function reapOrphaned(limit: number): Promise { + let reaped = 0 + const orphaned = await deps.repo.listOrphaned({ + staleBefore: new Date(now().getTime() - EXPORT_ABANDON_AFTER_MS), + limit, + }) + for (const record of orphaned) { + const errorCode = record.status === "queued" ? ERROR_NOT_STARTED : ERROR_BUILD_FAILED + // Fence first: failing the row under its run token makes a still-live run's markReady miss, so + // no ready row can end up naming the object deleted below. The key stays on the failed row + // until the delete succeeds. + if ( + record.status !== "failed" && + !(await deps.repo.markFailed(record.id, errorCode, record.runToken)) + ) { + continue + } + if (record.r2Key === null) { + reaped += 1 + continue + } + try { + await deps.storage.delete(record.r2Key) + } catch (err) { + deps.logger?.warn( + { err, exportId: record.id }, + "host export reap: orphaned object delete failed; row kept for the next pass", + ) + continue + } + const released = await deps.repo.releaseObject( + { id: record.id, status: "failed", runToken: record.runToken }, + errorCode, + ) + if (released) reaped += 1 + } + return reaped + } + return { async request(args) { const record = await deps.repo.create( @@ -130,7 +276,7 @@ export function makeHostExportService(deps: HostExportServiceDeps): HostExportSe async get(exportId) { const record = await deps.repo.findById(exportId) - if (record === null) throw AppError.notFound("Export not found") + if (record === null) throw AppError.notFound(EXPORT_NOT_FOUND) return record }, @@ -141,18 +287,7 @@ export function makeHostExportService(deps: HostExportServiceDeps): HostExportSe new Date(at.getTime() - EXPORT_RUN_STALE_MS), ) if (claimed === null) return { status: "skipped" } - if (deps.authorize !== undefined) { - try { - await deps.authorize(claimed) - } catch (err) { - deps.logger?.warn( - { err, exportId: claimed.id }, - "host export refused: the requester no longer holds the capability", - ) - await deps.repo.markFailed(claimed.id, "forbidden", claimed.runToken) - return { status: "failed" } - } - } + if (!(await claimIsAuthorized(claimed))) return { status: "failed" } let key: string | null = null const ctx: HostExportContext = { exportId: claimed.id, @@ -164,44 +299,7 @@ export function makeHostExportService(deps: HostExportServiceDeps): HostExportSe } try { const builder = hostExportBuilder(claimed.kind) - const provenance = await builder.provenance(ctx) - const header = await builder.header(ctx) - const parts: Buffer[] = [] - let bytes = 0 - for (const line of provenance) { - const buf = Buffer.from(csvProvenanceRow(line), "utf8") - parts.push(buf) - bytes += buf.byteLength - } - const headerBuf = Buffer.from(csvRow(header), "utf8") - parts.push(headerBuf) - bytes += headerBuf.byteLength - - let rowCount = 0 - let truncated = false - for await (const row of builder.rows(ctx)) { - if (rowCount >= deps.config.maxRows || bytes >= deps.config.maxBytes) { - truncated = true - break - } - const buf = Buffer.from(csvRow(row), "utf8") - parts.push(buf) - bytes += buf.byteLength - rowCount += 1 - if (rowCount % EXPORT_YIELD_EVERY_ROWS === 0) { - await new Promise((resolve) => setImmediate(resolve)) - } - } - if (truncated) { - const note = Buffer.from( - csvProvenanceRow( - `truncated: this file stops at ${rowCount} rows (${deps.config.maxRows} row / ${deps.config.maxBytes} byte cap)`, - ), - "utf8", - ) - parts.push(note) - bytes += note.byteLength - } + const { parts, bytes, rowCount, truncated } = await renderCsv(builder, ctx) // A re-claimed row still names the crashed run's object. It is deleted before this run's key // replaces it; if the delete fails the row fails holding that key, so the reaper finishes it. @@ -220,10 +318,10 @@ export function makeHostExportService(deps: HostExportServiceDeps): HostExportSe return { status: "skipped" } } await deps.storage.put(key, Buffer.concat(parts), { - contentType: "text/csv; charset=utf-8", + contentType: EXPORT_CONTENT_TYPE, contentDisposition: `attachment; filename="${builder.filename(ctx)}"`, }) - const expiresAt = new Date(at.getTime() + deps.config.ttlHours * 3_600_000) + const expiresAt = new Date(at.getTime() + deps.config.ttlHours * MS_PER_HOUR) const ready = await deps.repo.markReady(claimed.id, { r2Key: key, rowCount, @@ -233,16 +331,7 @@ export function makeHostExportService(deps: HostExportServiceDeps): HostExportSe runToken: claimed.runToken, }) if (ready === null) { - await deps.storage.delete(key).catch((err: unknown) => { - deps.logger?.warn( - { err, exportId: claimed.id, key }, - "host export: losing run could not delete its own object", - ) - }) - deps.logger?.warn( - { evt: "host.export.superseded", exportId: claimed.id }, - "host export run superseded by a newer claim; its object was discarded", - ) + await discardSupersededObject(claimed, key) return { status: "skipped" } } deps.logger?.info( @@ -252,7 +341,7 @@ export function makeHostExportService(deps: HostExportServiceDeps): HostExportSe return { status: "ready" } } catch (err) { deps.logger?.error({ err, exportId: claimed.id }, "host export failed") - await deps.repo.markFailed(claimed.id, "build_failed", claimed.runToken) + await deps.repo.markFailed(claimed.id, ERROR_BUILD_FAILED, claimed.runToken) if (key !== null) await discardFailedObject(claimed, key) return { status: "failed" } } @@ -268,66 +357,18 @@ export function makeHostExportService(deps: HostExportServiceDeps): HostExportSe const url = await deps.storage.presignGet(record.r2Key, EXPORT_DOWNLOAD_URL_TTL_SEC, { forceSigned: true, }) - const filename = record.r2Key.split("/").pop() ?? "civfix-export.csv" + const filename = record.r2Key.split("/").pop() ?? FALLBACK_EXPORT_FILENAME return { url, - expiresAt: new Date(now().getTime() + EXPORT_DOWNLOAD_URL_TTL_SEC * 1000).toISOString(), + expiresAt: new Date( + now().getTime() + EXPORT_DOWNLOAD_URL_TTL_SEC * MS_PER_SECOND, + ).toISOString(), filename, } }, async reap(limit) { - const expired = await deps.repo.listExpired(now(), limit) - let reaped = 0 - for (const record of expired) { - if (record.r2Key !== null) { - try { - await deps.storage.delete(record.r2Key) - } catch (err) { - deps.logger?.warn( - { err, exportId: record.id }, - "host export reap: object delete failed; row left ready for the next pass", - ) - continue - } - } - await deps.repo.markExpired(record.id) - reaped += 1 - } - const orphaned = await deps.repo.listOrphaned({ - staleBefore: new Date(now().getTime() - EXPORT_ABANDON_AFTER_MS), - limit, - }) - for (const record of orphaned) { - const errorCode = record.status === "queued" ? "not_started" : "build_failed" - // Fence first: failing the row under its run token makes a still-live run's markReady miss, so - // no ready row can end up naming the object deleted below. The key stays on the failed row - // until the delete succeeds. - if ( - record.status !== "failed" && - !(await deps.repo.markFailed(record.id, errorCode, record.runToken)) - ) { - continue - } - if (record.r2Key === null) { - reaped += 1 - continue - } - try { - await deps.storage.delete(record.r2Key) - } catch (err) { - deps.logger?.warn( - { err, exportId: record.id }, - "host export reap: orphaned object delete failed; row kept for the next pass", - ) - continue - } - const released = await deps.repo.releaseObject( - { id: record.id, status: "failed", runToken: record.runToken }, - errorCode, - ) - if (released) reaped += 1 - } + const reaped = (await reapExpired(limit)) + (await reapOrphaned(limit)) return { reaped } }, } diff --git a/services/api/src/services/host/host-analytics-cache.ts b/services/api/src/services/host/host-analytics-cache.ts index 9b248170..705f2915 100644 --- a/services/api/src/services/host/host-analytics-cache.ts +++ b/services/api/src/services/host/host-analytics-cache.ts @@ -1,11 +1,11 @@ import type { FastifyBaseLogger } from "fastify" import type { CacheClient } from "../../auth/cache.js" -export const HOST_ANALYTICS_CACHE_PREFIX = "hostan:v1" +const HOST_ANALYTICS_CACHE_PREFIX = "hostan:v1" -export const INSIGHTS_GENERATION_TTL_SEC = 86_400 +const INSIGHTS_GENERATION_TTL_SEC = 86_400 -export const INSIGHTS_GENERATION_ZERO = "0" +const INSIGHTS_GENERATION_ZERO = "0" export interface HostAnalyticsCacheDeps { cache: CacheClient @@ -42,6 +42,12 @@ export function hostAnalyticsCacheKey(args: { return args.generation === undefined ? key : `${key}:g${args.generation}` } +// Comparison and returning-attendee figures come from the viewer's own hosted events, so the user id +// is part of the key. +export function perViewerScope(viewer: { viewerScope: string; userId: string }): string { + return `${viewer.viewerScope}:${viewer.userId}` +} + export function insightsGenerationKey(cleanupId: string): string { return `${HOST_ANALYTICS_CACHE_PREFIX}:insights:gen:${cleanupId}` } diff --git a/services/api/src/services/host/host-analytics-shaping.ts b/services/api/src/services/host/host-analytics-shaping.ts new file mode 100644 index 00000000..7cf76b11 --- /dev/null +++ b/services/api/src/services/host/host-analytics-shaping.ts @@ -0,0 +1,100 @@ +import type { EventPhase, FunnelStep, SeriesPoint, SuppressedRate } from "@civfix/shared" +import { + eventPhase, + suppressRate, + type DayCount, + type DerivedPanel, + type SeriesClosure, +} from "@civfix/shared/host" +import type { EventClockRecord } from "./analytics-repository.drizzle.js" +import type { MetricRow } from "./metrics-repository.drizzle.js" + +export const DAY_MS = 86_400_000 + +const ISO_DAY_LENGTH = 10 +const EPOCH_YEAR = 1970 + +export function isoDayOf(at: Date): string { + return at.toISOString().slice(0, ISO_DAY_LENGTH) +} + +export function shiftDayKey(day: string, deltaDays: number): string { + const [year, month, date] = day.split("-").map(Number) + const shifted = new Date( + Date.UTC(year ?? EPOCH_YEAR, (month ?? 1) - 1, date ?? 1) + deltaDays * DAY_MS, + ) + return isoDayOf(shifted) +} + +export function clockPhase(clock: EventClockRecord, at: Date): EventPhase { + return eventPhase( + { + status: clock.status, + scheduledAt: clock.scheduledAt.toISOString(), + endsAt: clock.endsAt?.toISOString() ?? null, + completedAt: clock.completedAt?.toISOString() ?? null, + }, + at.getTime(), + ) +} + +export function emptyRate(): SuppressedRate { + return { value: null, numerator: null, denominator: null, suppressed: true } +} + +export function toSuppressedRate( + ratio: { value: number | null; suppressed: boolean }, + numerator: number, + denominator: number, +): SuppressedRate { + return { + value: ratio.value, + numerator: ratio.suppressed ? null : numerator, + denominator: ratio.suppressed ? null : denominator, + suppressed: ratio.suppressed, + } +} + +export function toRate(numerator: number, denominator: number): SuppressedRate { + return toSuppressedRate(suppressRate(numerator, denominator), numerator, denominator) +} + +export function toSeries( + points: readonly { day: string; value: number | null; suppressed: boolean }[], +): SeriesPoint[] { + return points.map((point) => ({ + day: point.day, + value: point.value, + suppressed: point.suppressed, + })) +} + +export function toFunnelSteps( + panel: DerivedPanel<{ key: string; value: number | null; suppressed: boolean }>, +): FunnelStep[] { + return panel.rows.map((row) => ({ + step: row.key, + label: row.key, + value: row.value, + suppressed: row.suppressed, + })) +} + +export function closureAllowsTotal(closure: SeriesClosure): boolean { + return closure.panelSuppressed || closure.totalPublishable +} + +export function metricTotal(rows: readonly MetricRow[], metric: string): number | null { + const matching = rows.filter((row) => row.metric === metric) + if (matching.length === 0) return null + return matching.reduce((acc, row) => acc + row.value, 0) +} + +export function seriesOf(rows: readonly MetricRow[], metric: string): DayCount[] { + const byDay = new Map() + for (const row of rows) { + if (row.metric !== metric) continue + byDay.set(row.day, (byDay.get(row.day) ?? 0) + row.value) + } + return [...byDay].map(([day, count]) => ({ day, count })) +} diff --git a/services/api/src/services/host/host-export-builders.ts b/services/api/src/services/host/host-export-builders.ts index 4d846aba..7ba0bde0 100644 --- a/services/api/src/services/host/host-export-builders.ts +++ b/services/api/src/services/host/host-export-builders.ts @@ -1,7 +1,10 @@ import type { Sql } from "../../db/client.js" import { registerHostExportBuilder, type HostExportContext } from "./export-builders.js" -export const EXPORT_PAGE_SIZE = 1000 +const EXPORT_PAGE_SIZE = 1000 +const SHORT_REF_LENGTH = 8 +const ISO_DAY_LENGTH = 10 +const SHORT_REF_FALLBACK = "event" interface RosterRow { registration_id: string @@ -87,10 +90,23 @@ export function registerEventExportBuilders(getSql: () => Sql): void { }) } -async function* rosterRows(sql: Sql, ctx: HostExportContext): AsyncIterable { +async function* keysetPages( + fetchPage: (after: string) => Promise, + keyOf: (row: Row) => string, +): AsyncIterable { let after = "" for (;;) { - const rows = await sql` + const rows = await fetchPage(after) + if (rows.length === 0) return + yield* rows + after = keyOf(rows[rows.length - 1]!) + if (rows.length < EXPORT_PAGE_SIZE) return + } +} + +async function* rosterRows(sql: Sql, ctx: HostExportContext): AsyncIterable { + const pages = keysetPages( + (after) => sql` SELECT r.id AS registration_id, COALESCE(NULLIF(min(s.attendee_name), ''), g.name, u.display_name) AS attendee_name, CASE WHEN r.user_id IS NOT NULL THEN 'member' ELSE 'guest' END AS attendee_kind, @@ -116,33 +132,30 @@ async function* rosterRows(sql: Sql, ctx: HostExportContext): AsyncIterable row.registration_id, + ) + for await (const row of pages) { + yield [ + row.registration_id, + row.attendee_name ?? "", + row.attendee_kind, + row.ticket_type ?? "", + String(row.seats), + row.slot ?? "", + row.status, + iso(row.registered_at), + iso(row.checked_in_at), + row.checkin_method ?? "", + row.guest_email ?? "", + row.guest_phone ?? "", + ] } } async function* checkinRows(sql: Sql, ctx: HostExportContext): AsyncIterable { - let after = "" - for (;;) { - const rows = await sql< + const pages = keysetPages( + (after) => sql< { id: string attendee_name: string | null @@ -166,28 +179,25 @@ async function* checkinRows(sql: Sql, ctx: HostExportContext): AsyncIterable ${after}::uuid) ORDER BY s.id - LIMIT ${EXPORT_PAGE_SIZE}` - if (rows.length === 0) return - for (const row of rows) { - yield [ - row.id, - row.attendee_name ?? "", - row.attendee_kind, - row.ticket_type ?? "", - iso(row.checked_in_at), - row.checkin_method ?? "", - iso(row.no_show_at), - ] - } - after = rows[rows.length - 1]!.id - if (rows.length < EXPORT_PAGE_SIZE) return + LIMIT ${EXPORT_PAGE_SIZE}`, + (row) => row.id, + ) + for await (const row of pages) { + yield [ + row.id, + row.attendee_name ?? "", + row.attendee_kind, + row.ticket_type ?? "", + iso(row.checked_in_at), + row.checkin_method ?? "", + iso(row.no_show_at), + ] } } async function* answerRows(sql: Sql, ctx: HostExportContext): AsyncIterable { - let after = "" - for (;;) { - const rows = await sql< + const pages = keysetPages( + (after) => sql< { id: string registration_id: string @@ -208,24 +218,22 @@ async function* answerRows(sql: Sql, ctx: HostExportContext): AsyncIterable ${after}::uuid) ORDER BY a.id - LIMIT ${EXPORT_PAGE_SIZE}` - if (rows.length === 0) return - for (const row of rows) { - const answer = - row.scrubbed_at !== null - ? "" - : (row.value_text ?? (row.value_json === null ? "" : JSON.stringify(row.value_json))) - yield [row.registration_id, row.attendee_kind, row.prompt, answer, iso(row.created_at)] - } - after = rows[rows.length - 1]!.id - if (rows.length < EXPORT_PAGE_SIZE) return + LIMIT ${EXPORT_PAGE_SIZE}`, + (row) => row.id, + ) + for await (const row of pages) { + const answer = + row.scrubbed_at !== null + ? "" + : (row.value_text ?? (row.value_json === null ? "" : JSON.stringify(row.value_json))) + yield [row.registration_id, row.attendee_kind, row.prompt, answer, iso(row.created_at)] } } function shortRef(ctx: HostExportContext): string { - return (ctx.cleanupId ?? ctx.organizationId ?? "event").slice(0, 8) + return (ctx.cleanupId ?? ctx.organizationId ?? SHORT_REF_FALLBACK).slice(0, SHORT_REF_LENGTH) } function dayOf(ctx: HostExportContext): string { - return ctx.now.toISOString().slice(0, 10) + return ctx.now.toISOString().slice(0, ISO_DAY_LENGTH) } diff --git a/services/api/src/services/host/insights-service.ts b/services/api/src/services/host/insights-service.ts index a5ebb6f7..1873b888 100644 --- a/services/api/src/services/host/insights-service.ts +++ b/services/api/src/services/host/insights-service.ts @@ -16,13 +16,17 @@ import { type LeaderboardEntryDTO, type SeatPoint, } from "@civfix/shared" -import { eventPhase, DEFAULT_DURATION_MS } from "@civfix/shared/host" +import { DEFAULT_DURATION_MS } from "@civfix/shared/host" import type { AnalyticsRepository, EventClockRecord, SeatTrendPoint, } from "./analytics-repository.drizzle.js" -import { hostAnalyticsCacheKey, type HostAnalyticsCache } from "./host-analytics-cache.js" +import { + hostAnalyticsCacheKey, + perViewerScope, + type HostAnalyticsCache, +} from "./host-analytics-cache.js" import { leaderboardEntryOf } from "../volunteer-hours-service.js" import type { CheckinCountersRecord, @@ -30,20 +34,25 @@ import type { } from "./registration-repository.types.js" import { CHECKIN_COARSEN_DAYS } from "./registration-retention.js" import { DEFAULT_EVENT_TIME_ZONE } from "./event-fields.js" +import { DAY_MS, clockPhase } from "./host-analytics-shaping.js" export const INSIGHTS_LIVE_CACHE_TTL_SEC = 15 export const INSIGHTS_CACHE_TTL_SEC = 300 -export const INSIGHTS_PORTFOLIO_EVENT_LIMIT = 200 +const INSIGHTS_PORTFOLIO_EVENT_LIMIT = 200 + +const INSIGHTS_RETURNING_MIN_EVENTS = 2 -export const INSIGHTS_RETURNING_MIN_EVENTS = 2 +const INSIGHTS_ARRIVAL_MIN_OFFSET_MIN = -120 -export const INSIGHTS_ARRIVAL_MIN_OFFSET_MIN = -120 +const INSIGHTS_ARRIVAL_MAX_OFFSET_MIN = 240 -export const INSIGHTS_ARRIVAL_MAX_OFFSET_MIN = 240 +const MINUTE_MS = 60_000 -const DAY_MS = 86_400_000 +const INSIGHTS_CACHE_ENDPOINT = "insights" +const ENDED_CACHE_RANGE = "event:ended" +const OPEN_CACHE_RANGE = "event" type InsightsAnalyticsRepository = Pick< AnalyticsRepository, @@ -98,7 +107,7 @@ function arrivalBuckets( ): ArrivalOffsetBucket[] { const byOffset = new Map() for (const bucket of arrivals) { - const offsetMin = Math.round((bucket.at.getTime() - startsAt.getTime()) / 60_000) + const offsetMin = Math.round((bucket.at.getTime() - startsAt.getTime()) / MINUTE_MS) if (offsetMin < INSIGHTS_ARRIVAL_MIN_OFFSET_MIN) continue if (offsetMin > INSIGHTS_ARRIVAL_MAX_OFFSET_MIN) continue byOffset.set(offsetMin, (byOffset.get(offsetMin) ?? 0) + bucket.count) @@ -226,22 +235,14 @@ export function makeInsightsService(deps: InsightsServiceDeps): InsightsService const clock = await deps.analytics.eventClock(cleanupId) if (clock === null) throw AppError.notFound("Cleanup not found") const at = now() - const phase = eventPhase( - { - status: clock.status, - scheduledAt: clock.scheduledAt.toISOString(), - endsAt: clock.endsAt?.toISOString() ?? null, - completedAt: clock.completedAt?.toISOString() ?? null, - }, - at.getTime(), - ) + const phase = clockPhase(clock, at) const generation = await deps.cache.generationOf(cleanupId) const rollups = await deps.cache.getOrSet( hostAnalyticsCacheKey({ - endpoint: "insights", + endpoint: INSIGHTS_CACHE_ENDPOINT, scope: cleanupId, - range: phase === "ended" ? "event:ended" : "event", - viewerScope: `${viewer.viewerScope}:${viewer.userId}`, + range: phase === "ended" ? ENDED_CACHE_RANGE : OPEN_CACHE_RANGE, + viewerScope: perViewerScope(viewer), generation, }), () => compute(cleanupId, viewer, clock, phase, at), diff --git a/services/api/src/services/host/metrics-repository.drizzle.ts b/services/api/src/services/host/metrics-repository.drizzle.ts index 919ab662..cca5b7df 100644 --- a/services/api/src/services/host/metrics-repository.drizzle.ts +++ b/services/api/src/services/host/metrics-repository.drizzle.ts @@ -36,6 +36,17 @@ export interface MetricsRepository { ): Promise } +interface MetricRowSelect { + day: string + metric: string + bucket: string + value: string +} + +function toMetricRow(row: MetricRowSelect): MetricRow { + return { day: row.day, metric: row.metric, bucket: row.bucket, value: Number(row.value) } +} + export function makeDrizzleMetricsRepository(sql: Sql): MetricsRepository { return { async resolveSlug(slug: string) { @@ -169,7 +180,7 @@ export function makeDrizzleMetricsRepository(sql: Sql): MetricsRepository { }, async read(cleanupId: string, metrics: readonly string[], from: string, to: string) { - const rows = await sql<{ day: string; metric: string; bucket: string; value: string }[]>` + const rows = await sql` SELECT to_char(day, 'YYYY-MM-DD') AS day, metric, bucket, value::text FROM event_metrics_daily WHERE cleanup_id = ${cleanupId} @@ -177,12 +188,7 @@ export function makeDrizzleMetricsRepository(sql: Sql): MetricsRepository { AND day BETWEEN ${from}::date AND ${to}::date ORDER BY day LIMIT 20000` - return rows.map((row) => ({ - day: row.day, - metric: row.metric, - bucket: row.bucket, - value: Number(row.value), - })) + return rows.map(toMetricRow) }, async readMany( @@ -192,7 +198,7 @@ export function makeDrizzleMetricsRepository(sql: Sql): MetricsRepository { to: string, ) { if (cleanupIds.length === 0) return [] - const rows = await sql<{ day: string; metric: string; bucket: string; value: string }[]>` + const rows = await sql` SELECT to_char(day, 'YYYY-MM-DD') AS day, metric, bucket, sum(value)::text AS value FROM event_metrics_daily WHERE cleanup_id = ANY(${[...cleanupIds]}::uuid[]) @@ -201,12 +207,7 @@ export function makeDrizzleMetricsRepository(sql: Sql): MetricsRepository { GROUP BY 1, 2, 3 ORDER BY day LIMIT 20000` - return rows.map((row) => ({ - day: row.day, - metric: row.metric, - bucket: row.bucket, - value: Number(row.value), - })) + return rows.map(toMetricRow) }, } } diff --git a/services/api/src/services/host/metrics-service.ts b/services/api/src/services/host/metrics-service.ts index 2c1221d2..587aa92b 100644 --- a/services/api/src/services/host/metrics-service.ts +++ b/services/api/src/services/host/metrics-service.ts @@ -4,21 +4,34 @@ import type { CacheClient } from "../../auth/cache.js" import type { MetricUpsert, MetricsRepository } from "./metrics-repository.drizzle.js" import { eventDayKey } from "./event-day.js" import { DEFAULT_EVENT_TIME_ZONE } from "./event-fields.js" +import { + METRIC_DONATION_CLICKS, + METRIC_PAGE_VIEWS, + METRIC_SOURCE, + NO_BUCKET, +} from "./event-metric-names.js" +import { DAY_MS, isoDayOf } from "./host-analytics-shaping.js" export { eventDayKey } -export const METRIC_PAGE_VIEWS = "page_views" -export const METRIC_SOURCE = "source" -export const METRIC_DONATION_CLICKS = "donation_clicks" +const COUNTER_TTL_SEC = 4 * 24 * 60 * 60 -export const COUNTER_TTL_SEC = 4 * 24 * 60 * 60 +const ROLLUP_PAGE_SIZE = 500 -export const ROLLUP_PAGE_SIZE = 500 +const FLUSH_READ_BATCH = 200 -const DAY_MS = 86_400_000 +const FLUSH_UPSERT_BATCH = 500 + +// One day ahead: a bump keys its dirty set by the event-local day, which east of UTC is already +// tomorrow. +const FLUSH_LEAD_DAYS = 1 const COUNTER_PREFIX = "evm:v1" +const COUNTER_KEY_PARTS = 6 + +const DAY_KEY_LENGTH = 10 + const BOT_UA_RE = /bot|crawler|spider|crawling|slurp|facebookexternalhit|embedly|quora link preview|whatsapp|telegram|discordbot|preview|monitor|curl|wget|python-requests|headless|lighthouse|pingdom|uptime/i @@ -134,6 +147,14 @@ export function makeMetricsService(deps: MetricsServiceDeps): MetricsService { return `${COUNTER_PREFIX}:dirty:${day}` } + function flushDays(): string[] { + const days: string[] = [] + for (let i = -FLUSH_LEAD_DAYS; i <= deps.lookbackDays; i += 1) { + days.push(isoDayOf(new Date(now().getTime() - i * DAY_MS))) + } + return days + } + async function bump( cleanupId: string, timezone: string | null, @@ -165,24 +186,19 @@ export function makeMetricsService(deps: MetricsServiceDeps): MetricsService { ...(input.referrer !== undefined ? { referrer: input.referrer } : {}), selfHosts: deps.selfHosts, }) - await bump(resolved.cleanupId, resolved.timezone, METRIC_PAGE_VIEWS, "") + await bump(resolved.cleanupId, resolved.timezone, METRIC_PAGE_VIEWS, NO_BUCKET) await bump(resolved.cleanupId, resolved.timezone, METRIC_SOURCE, source) return { ok: true } }, async recordDonationClick(cleanupId: string) { const timezone = await deps.repo.eventTimezone(cleanupId) - await bump(cleanupId, timezone, METRIC_DONATION_CLICKS, "") + await bump(cleanupId, timezone, METRIC_DONATION_CLICKS, NO_BUCKET) }, async flushCounters() { - // A bump keys its dirty set by the event-local day, which east of UTC is already tomorrow. - const days: string[] = [] - for (let i = -1; i <= deps.lookbackDays; i += 1) { - days.push(new Date(now().getTime() - i * DAY_MS).toISOString().slice(0, 10)) - } const upserts: MetricUpsert[] = [] - for (const day of days) { + for (const day of flushDays()) { let keys: string[] try { keys = await deps.cache.smembers(dirtyKey(day)) @@ -190,21 +206,10 @@ export function makeMetricsService(deps: MetricsServiceDeps): MetricsService { deps.logger?.warn({ err, day }, "event metrics: dirty-set read failed (skipping day)") continue } - for (let i = 0; i < keys.length; i += 200) { - const batch = keys.slice(i, i + 200) - const values = await readMany(deps.cache, batch) - batch.forEach((key, index) => { - const parsed = parseCounterKey(key) - const raw = values[index] - if (parsed === null || raw === null || raw === undefined) return - const value = Number(raw) - if (!Number.isFinite(value) || value <= 0) return - upserts.push({ ...parsed, value }) - }) - } + await collectCounters(deps.cache, keys, upserts) } - for (let i = 0; i < upserts.length; i += 500) { - await deps.repo.upsertGreatest(upserts.slice(i, i + 500)) + for (let i = 0; i < upserts.length; i += FLUSH_UPSERT_BATCH) { + await deps.repo.upsertGreatest(upserts.slice(i, i + FLUSH_UPSERT_BATCH)) } return { flushed: upserts.length } }, @@ -238,11 +243,30 @@ async function readMany(cache: CacheClient, keys: readonly string[]): Promise<(s return Promise.all(keys.map((key) => cache.get(key))) } +async function collectCounters( + cache: CacheClient, + keys: readonly string[], + upserts: MetricUpsert[], +): Promise { + for (let i = 0; i < keys.length; i += FLUSH_READ_BATCH) { + const batch = keys.slice(i, i + FLUSH_READ_BATCH) + const values = await readMany(cache, batch) + batch.forEach((key, index) => { + const parsed = parseCounterKey(key) + const raw = values[index] + if (parsed === null || raw === null || raw === undefined) return + const value = Number(raw) + if (!Number.isFinite(value) || value <= 0) return + upserts.push({ ...parsed, value }) + }) + } +} + export function parseCounterKey( key: string, ): { cleanupId: string; day: string; metric: string; bucket: string } | null { const parts = key.split(":") - if (parts.length !== 6) return null + if (parts.length !== COUNTER_KEY_PARTS) return null const [prefix, version, cleanupId, day, metric, bucket] = parts as [ string, string, @@ -252,6 +276,6 @@ export function parseCounterKey( string, ] if (`${prefix}:${version}` !== COUNTER_PREFIX) return null - if (cleanupId.length === 0 || day.length !== 10 || metric.length === 0) return null + if (cleanupId.length === 0 || day.length !== DAY_KEY_LENGTH || metric.length === 0) return null return { cleanupId, day, metric, bucket } } diff --git a/services/api/src/services/host/retention-lanes.ts b/services/api/src/services/host/retention-lanes.ts index 5914650f..cd0015e0 100644 --- a/services/api/src/services/host/retention-lanes.ts +++ b/services/api/src/services/host/retention-lanes.ts @@ -1,6 +1,9 @@ import type { FastifyBaseLogger } from "fastify" import type { Sql } from "../../db/client.js" +const RETENTION_BATCH_SIZE = 1000 +const RETENTION_MAX_PAGES = 50 + export interface RetentionLaneResult { lane: string processed: number @@ -47,9 +50,6 @@ export async function runRetentionLanes( return results } -export const RETENTION_BATCH_SIZE = 1000 -export const RETENTION_MAX_PAGES = 50 - export async function drainTable( page: (batchSize: number) => Promise, batchSize = RETENTION_BATCH_SIZE,