diff --git a/services/api/src/abuse/gps-sanity.ts b/services/api/src/abuse/gps-sanity.ts index 223522b0..9af73093 100644 --- a/services/api/src/abuse/gps-sanity.ts +++ b/services/api/src/abuse/gps-sanity.ts @@ -1,10 +1,13 @@ import type { AbuseChecks } from "@civfix/shared/interfaces" import type { LatLng } from "@civfix/shared" -export const CF_LAT_HEADER = "cf-iplatitude" -export const CF_LNG_HEADER = "cf-iplongitude" +const CF_LAT_HEADER = "cf-iplatitude" +const CF_LNG_HEADER = "cf-iplongitude" export const CF_CITY_HEADER = "cf-ipcity" +const MAX_ABS_LATITUDE = 90 +const MAX_ABS_LONGITUDE = 180 + export type HeaderBag = Record export function headerValue(headers: HeaderBag, name: string): string | null { @@ -22,7 +25,7 @@ export function parseCfGeo(headers: HeaderBag): LatLng | null { const lat = Number.parseFloat(latRaw) const lng = Number.parseFloat(lngRaw) if (!Number.isFinite(lat) || !Number.isFinite(lng)) return null - if (lat < -90 || lat > 90 || lng < -180 || lng > 180) return null + if (Math.abs(lat) > MAX_ABS_LATITUDE || Math.abs(lng) > MAX_ABS_LONGITUDE) return null return { lat, lng } } diff --git a/services/api/src/abuse/h3-cap.ts b/services/api/src/abuse/h3-cap.ts index d87e5ada..e0d1f247 100644 --- a/services/api/src/abuse/h3-cap.ts +++ b/services/api/src/abuse/h3-cap.ts @@ -15,7 +15,7 @@ export const ABUSE_H3_RES = 10 export const H3_CELL_LIMIT_PER_HOUR = 30 -export const H3_WINDOW_SECONDS = 60 * 60 +const H3_WINDOW_SECONDS = 60 * 60 const H3_COUNTER_PREFIX = "abuse:h3:" diff --git a/services/api/src/adapters/jobs.pgboss.ts b/services/api/src/adapters/jobs.pgboss.ts index 375edb9c..d7fa2040 100644 --- a/services/api/src/adapters/jobs.pgboss.ts +++ b/services/api/src/adapters/jobs.pgboss.ts @@ -9,7 +9,7 @@ import { OUTREACH_DIGEST_JOB } from "../services/admin/jurisdiction-contacts-typ import { INBOUND_SWEEP_JOB } from "../services/admin/inbound-jobs.js" import { REPORT_AUTOFORWARD_JOB } from "../services/report-service.types.js" import { DATA_EXPORT_JOB } from "../services/data-export-jobs.js" -import { CLEANUP_CANCEL_FANOUT_JOB } from "../services/cleanup-service.js" +import { CLEANUP_CANCEL_FANOUT_JOB } from "../services/cleanup-notifications.js" import { CLEANUP_GUEST_UPDATE_FANOUT_JOB, GUEST_RETENTION_SWEEP_JOB, diff --git a/services/api/src/routes/chat-gateway-wiring.ts b/services/api/src/routes/chat-gateway-wiring.ts index d1ce8eef..4fb932a6 100644 --- a/services/api/src/routes/chat-gateway-wiring.ts +++ b/services/api/src/routes/chat-gateway-wiring.ts @@ -1,5 +1,5 @@ import { ErrorCode } from "@civfix/shared" -import type { FastifyInstance } from "fastify" +import type { FastifyBaseLogger, FastifyInstance } from "fastify" import type { Container } from "../di.js" import { makeWsTicketStore } from "../auth/ws-ticket.js" import { @@ -25,9 +25,10 @@ import { type OnChatReply, type OnGroupMessage, type OnReportMessage, + type ReportVisibleFn, type ThreadRecipientsOf, } from "../ws/gateway.js" -import { resolveMentionTargets } from "../services/social-repository.drizzle.js" +import { resolveMentionTargets } from "../services/mention-resolver.drizzle.js" import { makeChatMentionResolver } from "../services/chat-mention-resolver.js" import { makeDrizzleCleanupRepository } from "../services/cleanup-repository.drizzle.js" import { makeDrizzleDiscussionRepository } from "../services/discussion-repository.drizzle.js" @@ -41,12 +42,11 @@ import { makeChatGroupRepository, type ChatGroupRepository, } from "../services/chat-group-repository.drizzle.js" -import type { GatewayGroupChat } from "../ws/types.js" +import { WS_ROUTE, WS_SEND_LIMITS, type GatewayGroupChat } from "../ws/types.js" import { makeCityForwardThrottle } from "../services/report-city-forward.js" import { makeContainerReportCityForward } from "../services/report-city-forward-wiring.js" import { isReportVisibleTo } from "../services/report-visibility.js" import { makeTokenBucketLimiter, type RateLimiter } from "../ws/report-rate-limit.js" -import type { ReportVisibleFn } from "../ws/gateway.js" import { THREAD_SIGNAL_MEMBER_CAP } from "../services/cleanup-service.js" import { makeDrizzleChatRepository, @@ -92,11 +92,21 @@ import { InMemoryChatReadState, type ChatReadState } from "../services/threads-s import { makeMarkRoomRead, type MarkRoomRead } from "../services/room-read-service.js" import type { ChatGatewayOverrides } from "./chat.routes.js" -const WS_UPGRADE_ROUTE = "/ws" - const WS_UPGRADE_RATE_LIMIT = { max: 60, timeWindow: "1 minute" } as const -const REPORT_SEND_LIMIT = { capacity: 30, refillPerSec: 0.5 } as const +const CLEANUP_MEMBERS_TABLE = "cleanup_members" + +const CHAT_MESSAGES_TABLE = "chat_messages" + +type FanoutRoomKind = "report" | "group" + +type CleanupRepository = ReturnType + +type DiscussionRepository = ReturnType + +type BatchIdLookup = (ownerId: string, candidateIds: string[]) => Promise> + +type ClearBell = (kind: ConversationBellKind, id: string, userId: string) => Promise export type ChatMentionSeam = Pick< GatewayChatMentions, @@ -132,6 +142,25 @@ export function chatMentionDeps(app: FastifyInstance, container: Container): Cha return seam } +// A bell that fails to clear leaves a stale badge, never a failed read, so the error is logged only. +function makeClearBell( + getService: () => NotificationService | undefined, + log: Pick, +): ClearBell { + return async (kind, id, userId) => { + const service = getService() + if (!service) return + try { + await clearConversationBellFor(service, kind, id, userId) + } catch (err) { + log.warn( + { err, kind, id, userId }, + "read: clear conversation notifications failed (suppressed)", + ) + } + } +} + export interface ConversationReadSeam { readState: ChatReadState markRoomRead: MarkRoomRead @@ -161,7 +190,7 @@ export function conversationReadSeam( ? overrides.reportChat : useFakeChat ? undefined - : (reportChat ??= makeReportChatRepository(container.getDb().sql, presignMedia)) + : (reportChat ??= makeReportChatRepository(container.getDb().sql)) let groups: ChatGroupRepository | undefined const getGroups = (): ChatGroupRepository | undefined => @@ -186,18 +215,7 @@ export function conversationReadSeam( group: async (id, userId, at) => { await getGroups()?.markRead(id, userId, at) }, - clearBell: async (kind, id, userId) => { - const service = notifications() - if (!service) return - try { - await clearConversationBellFor(service, kind, id, userId) - } catch (err) { - app.log.warn( - { err, kind, id, userId }, - "read: clear conversation notifications failed (suppressed)", - ) - } - }, + clearBell: makeClearBell(notifications, app.log), }), } readSeams.set(app, seam) @@ -208,11 +226,9 @@ export interface ChatWiring { readState: ChatReadState isMember: IsMemberFn dmRepo: DmRepository - blocksRepo: BlocksRepository isBlockedEitherWay: IsBlockedEitherWayFn dmPeerOf: (threadId: string, userId: string) => Promise getChatRepo(): ChatRepository - getReportChatRepo(): ReportChatRepository listDmThreadsFor: DmRepository["listThreadsForUser"] } @@ -225,7 +241,7 @@ export function applyWsUpgradeRateLimit(app: FastifyInstance): void { app.addHook("onRequest", async (request, reply) => { // The router matches on the percent-decoded path, so only the matched pattern catches every // spelling of /ws that reaches the upgrade handler. - if (request.routeOptions.url !== WS_UPGRADE_ROUTE) return + if (request.routeOptions.url !== WS_ROUTE) return const result = await limiter(request) if (!result.isAllowed && result.isExceeded) { applyRateLimitHeaders(reply, result) @@ -236,18 +252,37 @@ export function applyWsUpgradeRateLimit(app: FastifyInstance): void { }) } -export function wireChatGateway(app: FastifyInstance, container: Container): ChatWiring { - const overrides: ChatGatewayOverrides | undefined = app.chatOverrides - const useFakeChat = container.env.USE_FAKE_CHAT - - const { readState, markRoomRead } = conversationReadSeam(app, container) +interface WiringContext { + app: FastifyInstance + container: Container + overrides: ChatGatewayOverrides | undefined + useFakeChat: boolean +} - const presence: ChatPresence = +function buildPresence({ container, overrides, useFakeChat }: WiringContext): ChatPresence { + return ( overrides?.presence ?? (useFakeChat ? new InMemoryChatPresence() : new RedisChatPresence(container.getRedis())) + ) +} + +interface ChatRepos { + getCleanupRepo(): CleanupRepository + isMember: IsMemberFn + blocksRepo: BlocksRepository + dmRepo: DmRepository + isBlockedEitherWay: IsBlockedEitherWayFn + blockedIdsFor: BatchIdLookup | undefined + getChatRepo(): ChatRepository + getReportChatRepo(): ReportChatRepository + getGroupsRepo(): ChatGroupRepository | undefined + groupWired: boolean + getReportRepo(): DiscussionRepository +} - let cleanupRepo: ReturnType | undefined - const getCleanupRepo = (): ReturnType => +function buildChatRepos({ container, overrides, useFakeChat }: WiringContext): ChatRepos { + let cleanupRepo: CleanupRepository | undefined + const getCleanupRepo = (): CleanupRepository => (cleanupRepo ??= makeDrizzleCleanupRepository(container.getDb().sql)) const isMember: IsMemberFn = overrides ? overrides.isMember @@ -255,70 +290,92 @@ export function wireChatGateway(app: FastifyInstance, container: Container): Cha const blocksRepo: BlocksRepository = overrides?.blocksRepo ?? container.getBlocksRepo() const dmRepo: DmRepository = overrides?.dmRepo ?? container.getDmRepo() - const isBlockedEitherWay: IsBlockedEitherWayFn = (a, b) => blocksRepo.isBlockedEitherWay(a, b) - - const blockedIdsForCandidates = ((): - | ((actorId: string, candidateIds: string[]) => Promise>) - | undefined => { - const repo = blocksRepo - const batch = repo.blockedIdsAmong - if (!batch) return undefined - return (actorId, candidateIds) => batch.call(repo, actorId, candidateIds) - })() + const batchBlocked = blocksRepo.blockedIdsAmong + const blockedIdsFor: BatchIdLookup | undefined = batchBlocked + ? (actorId, candidateIds) => batchBlocked.call(blocksRepo, actorId, candidateIds) + : undefined const presignMedia = makePrivateMediaPresigner(container.storage) let chatRepo: ChatRepository | undefined - const getChatRepo = (): ChatRepository => - overrides?.chatRepo ?? - (chatRepo ??= makeDrizzleChatRepository(container.getDb().sql, presignMedia)) - let reportChatRepo: ReportChatRepository | undefined - const getReportChatRepo = (): ReportChatRepository => - overrides?.reportChat ?? - (reportChatRepo ??= makeReportChatRepository(container.getDb().sql, presignMedia)) + let groupsRepo: ChatGroupRepository | undefined + let reportRepo: DiscussionRepository | undefined - let lazyGroupsRepo: ChatGroupRepository | undefined - const getGroupsRepo = (): ChatGroupRepository | undefined => - overrides - ? overrides.groups - : useFakeChat - ? undefined - : (lazyGroupsRepo ??= makeChatGroupRepository(container.getDb().sql, presignMedia)) - const groupWired = overrides ? overrides.groups !== undefined : !useFakeChat - const groupChat: GatewayGroupChat | undefined = groupWired - ? { - isMember: async (groupId, userId) => - (await getGroupsRepo()!.roleOf(groupId, userId)) !== null, - access: async (groupId, userId) => { - const a = await getGroupsRepo()!.accessOf(groupId, userId) - if (a === null) return null - return { isMember: a.role !== null, canPost: canPostToGroup(a), visibility: a.visibility } - }, - advanceReadWatermark: (groupId, userId, upToId) => - getGroupsRepo()!.advanceReadWatermark(groupId, userId, upToId), - } - : undefined + return { + getCleanupRepo, + isMember, + blocksRepo, + dmRepo, + isBlockedEitherWay: (a, b) => blocksRepo.isBlockedEitherWay(a, b), + blockedIdsFor, + getChatRepo: () => + overrides?.chatRepo ?? + (chatRepo ??= makeDrizzleChatRepository(container.getDb().sql, presignMedia)), + getReportChatRepo: () => + overrides?.reportChat ?? (reportChatRepo ??= makeReportChatRepository(container.getDb().sql)), + getGroupsRepo: () => + overrides + ? overrides.groups + : useFakeChat + ? undefined + : (groupsRepo ??= makeChatGroupRepository(container.getDb().sql, presignMedia)), + groupWired: overrides ? overrides.groups !== undefined : !useFakeChat, + getReportRepo: () => (reportRepo ??= makeDrizzleDiscussionRepository(container.getDb().sql)), + } +} + +function buildGroupChat(repos: ChatRepos): GatewayGroupChat | undefined { + if (!repos.groupWired) return undefined + const groups = (): ChatGroupRepository => repos.getGroupsRepo()! + return { + isMember: async (groupId, userId) => (await groups().roleOf(groupId, userId)) !== null, + access: async (groupId, userId) => { + const a = await groups().accessOf(groupId, userId) + if (a === null) return null + return { isMember: a.role !== null, canPost: canPostToGroup(a), visibility: a.visibility } + }, + advanceReadWatermark: (groupId, userId, upToId) => + groups().advanceReadWatermark(groupId, userId, upToId), + } +} - const groupMembersInFlight = new Map>() - const listGroupMembersShared = ( - groupId: string, - limit: number = GROUP_MEMBER_SCAN_CAP, - ): Promise => { +// One map per wiring: it dedupes concurrent member scans between the thread signal and the group +// fan-out for the same message. +function makeSharedGroupMemberList( + getGroupsRepo: () => ChatGroupRepository | undefined, +): (groupId: string, limit?: number) => Promise { + const inFlightByKey = new Map>() + return (groupId, limit = GROUP_MEMBER_SCAN_CAP) => { const repo = getGroupsRepo() if (!repo) return Promise.resolve([]) const key = `${groupId}:${limit}` - const inFlight = groupMembersInFlight.get(key) + const inFlight = inFlightByKey.get(key) if (inFlight) return inFlight const query = repo.listMemberIds(groupId, limit) - groupMembersInFlight.set(key, query) + inFlightByKey.set(key, query) // Only evicts the shared entry; each caller awaits `query` itself and sees the rejection. - void query.catch(() => {}).then(() => groupMembersInFlight.delete(key)) + void query.catch(() => {}).then(() => inFlightByKey.delete(key)) return query } +} - const dmPeerOf = makeDmPeerOf(dmRepo) +interface ChatNotifications { + notificationService: NotificationService | undefined + conversationMutes: ConversationMutesRepository | undefined + isMutedFor: ReturnType + mutedUserIdsFor: Record< + FanoutRoomKind, + ((roomId: string, userIds: string[]) => Promise>) | undefined + > +} +function buildChatNotifications({ + app, + container, + overrides, + useFakeChat, +}: WiringContext): ChatNotifications { const notificationService: NotificationService | undefined = overrides?.notificationService ?? (useFakeChat @@ -337,124 +394,108 @@ export function wireChatGateway(app: FastifyInstance, container: Container): Cha const isMutedFor = makeFailOpenMuteCheck(conversationMutes, app.log) const mutedUserIdsForRoom = ( - kind: "report" | "group", + kind: FanoutRoomKind, ): ((roomId: string, userIds: string[]) => Promise>) | undefined => { const repo = conversationMutes const batch = repo?.mutedUserIdsFor if (!repo || !batch) return undefined return (roomId, userIds) => batch.call(repo, kind, roomId, userIds) } - const reportMutedUserIdsFor = mutedUserIdsForRoom("report") - const groupMutedUserIdsFor = mutedUserIdsForRoom("group") - - const clearConversationBell = async ( - kind: ConversationBellKind, - id: string, - userId: string, - ): Promise => { - if (!notificationService) return - try { - await clearConversationBellFor(notificationService, kind, id, userId) - } catch (err) { - app.log.warn( - { err, kind, id, userId }, - "read: clear conversation notifications failed (suppressed)", - ) - } - } - const dmGatewayDeps: GatewayDmDeps = { - peerOf: dmPeerOf, - persist: (input) => dmRepo.persist(input), - markRead: makeDmAckMarkRead(dmRepo, (threadId, userId) => - clearConversationBell("dm", threadId, userId), - ), + return { + notificationService, + conversationMutes, + isMutedFor, + mutedUserIdsFor: { report: mutedUserIdsForRoom("report"), group: mutedUserIdsForRoom("group") }, } +} - const advanceCleanupWatermark: ( - cleanupId: string, - userId: string, - upToId: string, - ) => Promise = useFakeChat - ? (cleanupId, userId) => readState.markRead(cleanupId, userId, new Date()) - : (cleanupId, userId, upToId) => - monotonicReadWatermarkUpdate( - container.getDb().sql, - "cleanup_members", - { cleanup_id: cleanupId, user_id: userId }, - { - messagesTable: "chat_messages", - messageId: upToId, - scopeColumn: "cleanup_id", - scopeId: cleanupId, - }, - ) +function makeCleanupWatermark( + { container, useFakeChat }: WiringContext, + readState: ChatReadState, +): (cleanupId: string, userId: string, upToId: string) => Promise { + if (useFakeChat) return (cleanupId, userId) => readState.markRead(cleanupId, userId, new Date()) + return (cleanupId, userId, upToId) => + monotonicReadWatermarkUpdate( + container.getDb().sql, + CLEANUP_MEMBERS_TABLE, + { cleanup_id: cleanupId, user_id: userId }, + { + messagesTable: CHAT_MESSAGES_TABLE, + messageId: upToId, + scopeColumn: "cleanup_id", + scopeId: cleanupId, + }, + ) +} - const threadRecipientsOf: ThreadRecipientsOf = async (kind, id, senderId) => { +function makeThreadRecipientsOf( + { useFakeChat }: WiringContext, + repos: ChatRepos, + dmPeerOf: (threadId: string, userId: string) => Promise, + listGroupMembers: (groupId: string) => Promise, +): ThreadRecipientsOf { + return async (kind, id, senderId) => { if (kind === "dm") { const peer = await dmPeerOf(id, senderId) return peer !== null ? [peer] : [] } if (kind === "report") return [] if (kind === "group") { - const members = await listGroupMembersShared(id) + const members = await listGroupMembers(id) return members.filter((m) => m !== senderId).slice(0, THREAD_SIGNAL_MEMBER_CAP) } if (useFakeChat) return [] - const members = await getCleanupRepo().listMemberIds(id, THREAD_SIGNAL_MEMBER_CAP) + const members = await repos.getCleanupRepo().listMemberIds(id, THREAD_SIGNAL_MEMBER_CAP) return members.filter((m) => m !== senderId) } +} - const bellDeps: ChatBellDeps | undefined = - useFakeChat || !notificationService - ? undefined - : { - notificationService, - isMutedFor, - isCleanupMember: isMember, - isReportChatMember: (reportId, userId) => getReportChatRepo().isMember(reportId, userId), - isChatGroupMember: async (groupId, userId) => - ((await getGroupsRepo()?.roleOf(groupId, userId)) ?? null) !== null, - isBlockedEitherWay, - presence, - roomKeyFor, - } - - const chatMentions: GatewayChatMentions | undefined = - overrides?.chatMentions ?? - (!bellDeps - ? undefined - : { - ...chatMentionDeps(app, container), - notifyChatMention: makeChatMentionNotifier(bellDeps), - }) - - const onChatReply: OnChatReply | undefined = bellDeps - ? makeChatReplyNotifier(bellDeps) - : undefined - - let reportRepo: ReturnType | undefined - const getReportRepo = (): ReturnType => - (reportRepo ??= makeDrizzleDiscussionRepository(container.getDb().sql)) - - const reportVisible: ReportVisibleFn | undefined = - overrides?.reportVisible ?? - (useFakeChat - ? undefined - : async (reportId, userId) => - isReportVisibleTo(await getReportRepo().findReportForDiscussion(reportId), userId)) +function buildBellDeps( + { useFakeChat }: WiringContext, + repos: ChatRepos, + notifications: ChatNotifications, + presence: ChatPresence, +): ChatBellDeps | undefined { + const { notificationService, isMutedFor } = notifications + if (useFakeChat || !notificationService) return undefined + return { + notificationService, + isMutedFor, + isCleanupMember: repos.isMember, + isReportChatMember: (reportId, userId) => repos.getReportChatRepo().isMember(reportId, userId), + isChatGroupMember: async (groupId, userId) => + ((await repos.getGroupsRepo()?.roleOf(groupId, userId)) ?? null) !== null, + isBlockedEitherWay: repos.isBlockedEitherWay, + presence, + roomKeyFor, + } +} - const reportSendLimiter: RateLimiter = makeTokenBucketLimiter(REPORT_SEND_LIMIT) +function makeReportVisible( + { overrides, useFakeChat }: WiringContext, + repos: ChatRepos, +): ReportVisibleFn | undefined { + if (overrides?.reportVisible) return overrides.reportVisible + if (useFakeChat) return undefined + return async (reportId, userId) => + isReportVisibleTo(await repos.getReportRepo().findReportForDiscussion(reportId), userId) +} +function makeRoomFanoutHandoff({ + app, + container, + useFakeChat, +}: WiringContext): ( + kind: FanoutRoomKind, +) => Pick { const fanoutMode = roomFanoutMode({ useFakeChat, useFakeJobs: container.env.USE_FAKE_JOBS, usesRealRedis: container.usesRealRedis === true, }) const roomFanoutClaim = makeWindowClaim(container, app.log) - const roomFanoutHandoff = ( - kind: "report" | "group", - ): Pick => ({ + return (kind) => ({ ...(fanoutMode.queued ? { dispatchToJob: makeRoomFanoutDispatcher(container.jobs, kind) } : {}), ...(fanoutMode.claimed ? { @@ -463,23 +504,32 @@ export function wireChatGateway(app: FastifyInstance, container: Container): Cha } : {}), }) +} +function makeLazySendDedupe({ container, useFakeChat }: WiringContext): SendDedupeStore { let dedupeStore: SendDedupeStore | undefined const resolveDedupeStore = (): SendDedupeStore => (dedupeStore ??= useFakeChat ? new InMemorySendDedupeStore() : new RedisSendDedupeStore(container.getRedis())) - const sendDedupe: SendDedupeStore = { + return { reserve: (key) => resolveDedupeStore().reserve(key), commit: (key, messageId) => resolveDedupeStore().commit(key, messageId), release: (key) => resolveDedupeStore().release(key), } +} +function withSendResilience( + { app, container }: WiringContext, + repos: ChatRepos, + dedupe: SendDedupeStore, +): GatewayChatService { const baseChat = container.chatService as GatewayChatService & { deliverLocal?: LocalDeliver } + const { dmRepo, getChatRepo } = repos const sendResilience = makeSendResilience({ - dedupe: sendDedupe, + dedupe, findRoomMessage: (kind, roomId, messageId, viewerUserId) => { if (kind === "dm") return dmRepo.findMessage(roomId, messageId, viewerUserId) if (kind === "report") return getChatRepo().findReportMessage(roomId, messageId, viewerUserId) @@ -492,7 +542,7 @@ export function wireChatGateway(app: FastifyInstance, container: Container): Cha app.log.error({ ...info, component: "chat-broadcast" }, "chat: room broadcast not published"), logger: app.log, }) - const chatWithResilience: GatewayChatService = { + return { joinRoom: (room, conn, userId) => baseChat.joinRoom(room, conn, userId), leaveRoom: (room, conn) => baseChat.leaveRoom(room, conn), persist: (input) => baseChat.persist(input), @@ -506,8 +556,13 @@ export function wireChatGateway(app: FastifyInstance, container: Container): Cha : {}), sendResilience, } +} - const canForwardCity = makeCityForwardThrottle( +function makeContainerCityForwardThrottle({ + app, + container, +}: WiringContext): ReturnType { + return makeCityForwardThrottle( { incr: (key, ttlSeconds) => container.getCounterStore().incr(key, ttlSeconds), incrBy: (key, by, ttlSeconds) => container.getCounterStore().incrBy(key, by, ttlSeconds), @@ -515,73 +570,159 @@ export function wireChatGateway(app: FastifyInstance, container: Container): Cha }, app.log, ) +} + +interface RoomMemberNotifiers { + notifyReportChatMembers: ReturnType | undefined + onGroupMessage: OnGroupMessage | undefined +} + +function buildRoomMemberNotifiers( + { app }: WiringContext, + repos: ChatRepos, + notifications: ChatNotifications, + presence: ChatPresence, + listGroupMembers: (groupId: string, limit?: number) => Promise, + fanoutHandoff: ReturnType, +): RoomMemberNotifiers { + const { notificationService, conversationMutes, mutedUserIdsFor } = notifications + const { blockedIdsFor, isBlockedEitherWay } = repos const notifyReportChatMembers = notificationService && conversationMutes ? makeReportChatNotifier({ notificationService, reportChatRepo: { - listMemberIds: (reportId, limit) => getReportChatRepo().listMemberIds(reportId, limit), + listMemberIds: (reportId, limit) => + repos.getReportChatRepo().listMemberIds(reportId, limit), }, isMuted: (userId, roomId) => conversationMutes.isMuted(userId, "report", roomId), - ...(reportMutedUserIdsFor ? { mutedUserIdsFor: reportMutedUserIdsFor } : {}), + ...(mutedUserIdsFor.report ? { mutedUserIdsFor: mutedUserIdsFor.report } : {}), presence, roomKeyFor, isBlockedEitherWay, - ...(blockedIdsForCandidates ? { blockedIdsFor: blockedIdsForCandidates } : {}), - ...roomFanoutHandoff("report"), + ...(blockedIdsFor ? { blockedIdsFor } : {}), + ...fanoutHandoff("report"), logger: app.log, }) : undefined const notifyGroupChatMembers = - notificationService && conversationMutes && groupWired + notificationService && conversationMutes && repos.groupWired ? makeGroupChatNotifier({ notificationService, groupRepo: { - listMemberIds: (groupId, limit) => listGroupMembersShared(groupId, limit), + listMemberIds: (groupId, limit) => listGroupMembers(groupId, limit), }, isMuted: (userId, roomId) => conversationMutes.isMuted(userId, "group", roomId), - ...(groupMutedUserIdsFor ? { mutedUserIdsFor: groupMutedUserIdsFor } : {}), + ...(mutedUserIdsFor.group ? { mutedUserIdsFor: mutedUserIdsFor.group } : {}), presence, roomKeyFor, isBlockedEitherWay, - ...(blockedIdsForCandidates ? { blockedIdsFor: blockedIdsForCandidates } : {}), - ...roomFanoutHandoff("group"), + ...(blockedIdsFor ? { blockedIdsFor } : {}), + ...fanoutHandoff("group"), logger: app.log, }) : undefined - const onGroupMessage: OnGroupMessage | undefined = notifyGroupChatMembers - ? async (groupId, message) => { - void notifyGroupChatMembers(groupId, message).catch(() => {}) - } - : undefined - const forwardCityMention = makeContainerReportCityForward(container, { - getReportRepo, - canForward: canForwardCity, - logger: app.log, - }) - const onReportMessage: OnReportMessage | undefined = useFakeChat - ? undefined - : async (reportId, message, actorUserId) => { - if (notifyReportChatMembers) void notifyReportChatMembers(reportId, message).catch(() => {}) - await forwardCityMention(reportId, message, actorUserId) - } + return { + notifyReportChatMembers, + onGroupMessage: notifyGroupChatMembers + ? async (groupId, message) => { + void notifyGroupChatMembers(groupId, message).catch(() => {}) + } + : undefined, + } +} - applyWsUpgradeRateLimit(app) +function makeOnReportMessage( + ctx: WiringContext, + repos: ChatRepos, + notifyReportChatMembers: RoomMemberNotifiers["notifyReportChatMembers"], +): OnReportMessage | undefined { + const forwardCityMention = makeContainerReportCityForward(ctx.container, { + getReportRepo: repos.getReportRepo, + canForward: makeContainerCityForwardThrottle(ctx), + logger: ctx.app.log, + }) + if (ctx.useFakeChat) return undefined + return async (reportId, message, actorUserId) => { + if (notifyReportChatMembers) void notifyReportChatMembers(reportId, message).catch(() => {}) + await forwardCityMention(reportId, message, actorUserId) + } +} - const reportChatSource = overrides +function buildGatewayReportChat( + { overrides, useFakeChat }: WiringContext, + repos: ChatRepos, + clearBell: ClearBell, +): GatewayReportChat | undefined { + const source = overrides ? overrides.reportChat : useFakeChat ? undefined - : getReportChatRepo() - const reportChat: GatewayReportChat | undefined = reportChatSource - ? makeGatewayReportChat(reportChatSource, (reportId, userId) => - clearConversationBell("report", reportId, userId), - ) + : repos.getReportChatRepo() + if (!source) return undefined + return makeGatewayReportChat(source, (reportId, userId) => clearBell("report", reportId, userId)) +} + +export function wireChatGateway(app: FastifyInstance, container: Container): ChatWiring { + const ctx: WiringContext = { + app, + container, + overrides: app.chatOverrides, + useFakeChat: container.env.USE_FAKE_CHAT, + } + + const { readState, markRoomRead } = conversationReadSeam(app, container) + const presence = buildPresence(ctx) + const repos = buildChatRepos(ctx) + const { isMember, dmRepo, isBlockedEitherWay } = repos + const groupChat = buildGroupChat(repos) + const listGroupMembers = makeSharedGroupMemberList(repos.getGroupsRepo) + const dmPeerOf = makeDmPeerOf(dmRepo) + + const notifications = buildChatNotifications(ctx) + const { notificationService, isMutedFor } = notifications + const clearBell = makeClearBell(() => notificationService, app.log) + + const dmGatewayDeps: GatewayDmDeps = { + peerOf: dmPeerOf, + persist: (input) => dmRepo.persist(input), + markRead: makeDmAckMarkRead(dmRepo, (threadId, userId) => clearBell("dm", threadId, userId)), + } + const advanceCleanupWatermark = makeCleanupWatermark(ctx, readState) + const threadRecipientsOf = makeThreadRecipientsOf(ctx, repos, dmPeerOf, listGroupMembers) + + const bellDeps = buildBellDeps(ctx, repos, notifications, presence) + const chatMentions: GatewayChatMentions | undefined = + ctx.overrides?.chatMentions ?? + (bellDeps + ? { ...chatMentionDeps(app, container), notifyChatMention: makeChatMentionNotifier(bellDeps) } + : undefined) + const onChatReply: OnChatReply | undefined = bellDeps + ? makeChatReplyNotifier(bellDeps) : undefined + const reportVisible = makeReportVisible(ctx, repos) + const reportSendLimiter: RateLimiter = makeTokenBucketLimiter(WS_SEND_LIMITS.report) + const fanoutHandoff = makeRoomFanoutHandoff(ctx) + const chatWithResilience = withSendResilience(ctx, repos, makeLazySendDedupe(ctx)) + + const { notifyReportChatMembers, onGroupMessage } = buildRoomMemberNotifiers( + ctx, + repos, + notifications, + presence, + listGroupMembers, + fanoutHandoff, + ) + const onReportMessage = makeOnReportMessage(ctx, repos, notifyReportChatMembers) + + applyWsUpgradeRateLimit(app) + + const reportChat = buildGatewayReportChat(ctx, repos, clearBell) + const wsTicketCache = app.authServices?.cache registerChatGateway(app, { chat: chatWithResilience, @@ -592,7 +733,7 @@ export function wireChatGateway(app: FastifyInstance, container: Container): Cha : {}), markRead: async (cleanupId, userId, upToId) => { await advanceCleanupWatermark(cleanupId, userId, upToId) - await clearConversationBell("cleanup", cleanupId, userId) + await clearBell("cleanup", cleanupId, userId) }, presence, dm: dmGatewayDeps, @@ -618,11 +759,9 @@ export function wireChatGateway(app: FastifyInstance, container: Container): Cha readState, isMember, dmRepo, - blocksRepo, isBlockedEitherWay, dmPeerOf, - getChatRepo, - getReportChatRepo, + getChatRepo: repos.getChatRepo, listDmThreadsFor: (userId, limit, cursor) => dmRepo.listThreadsForUser(userId, limit, cursor), } } diff --git a/services/api/src/routes/chat-groups.routes.ts b/services/api/src/routes/chat-groups.routes.ts index 7a35d185..ca94596e 100644 --- a/services/api/src/routes/chat-groups.routes.ts +++ b/services/api/src/routes/chat-groups.routes.ts @@ -21,7 +21,11 @@ import type { Container } from "../di.js" import { requireAuth } from "../auth/context.js" import { parse, trimTextFields } from "./_validate.js" import { route } from "../versioning/route.js" -import { chatHistoryPayload, deleteMessageWithPowers } from "./chat-route-helpers.js" +import { + chatHistoryPayload, + clampChatHistoryLimit, + deleteMessageWithPowers, +} from "./chat-route-helpers.js" import { makePrivateMediaPresigner } from "../services/media-presign.js" import { makeChatGroupRepository, @@ -53,9 +57,6 @@ const GroupIdParamsSchema = z.object({ id: IdSchema }).strict() const GroupMemberParamsSchema = z.object({ id: IdSchema, userId: IdSchema }).strict() const GroupMessageParamsSchema = z.object({ id: IdSchema, messageId: IdSchema }).strict() -const GROUP_HISTORY_DEFAULT = 30 -const GROUP_HISTORY_MAX = 50 - /** Creating rooms is rare and deliberate; 10/hour bounds scripted room spam per user/IP key. */ export const CREATE_GROUP_RATE_LIMIT = perIdentity({ max: 10, timeWindow: "1 hour" }) /** Bulk invites: bounded so a hijacked session can't blast invite sweeps; ample for normal use. */ @@ -222,7 +223,7 @@ export async function registerChatGroupRoutes( const { id } = parse(GroupIdParamsSchema, request.params) const q = parse(GroupHistoryRequestSchema, { ...(request.query as object), id }) await svc().requireReadable(userId, id) - const limit = Math.min(Math.max(q.limit ?? GROUP_HISTORY_DEFAULT, 1), GROUP_HISTORY_MAX) + const limit = clampChatHistoryLimit(q.limit) const payload: ChatHistoryResponse = await chatHistoryPayload( { history: (before, pageLimit, around) => diff --git a/services/api/src/routes/chat-notify-copy.ts b/services/api/src/routes/chat-notify-copy.ts index 1b9d6bf4..55603a8f 100644 --- a/services/api/src/routes/chat-notify-copy.ts +++ b/services/api/src/routes/chat-notify-copy.ts @@ -1,16 +1,19 @@ import type { ChatMessageDTO } from "@civfix/shared" -const PREVIEW_MAX = 80 +const PREVIEW_MAX_CHARS = 80 -/** Display name first so a message notification names the person, matching the thread title. */ -export function authorDisplay(message: ChatMessageDTO, fallback: string): string { +/** + * Display name first so a message notification names the person, matching the thread title. An empty + * string means no usable author, and each bell picks its own localized fallback. + */ +export function messageAuthorName(message: ChatMessageDTO): string { const from = message.from - // A sender-less SYSTEM message never raises a dm or mention bell, so a missing author just falls back - // like a blank name would. - if (!from) return fallback + // A sender-less SYSTEM message never raises a dm or mention bell, so a missing author reads like a + // blank name. + if (!from) return "" if (from.name.trim() !== "") return from.name if (from.handle) return `@${from.handle}` - return fallback + return "" } /** @@ -22,10 +25,9 @@ export function textPreview(message: ChatMessageDTO): string | null { const body = message.body if (message.kind === "text" && typeof body === "string" && body.trim() !== "") { const trimmed = body.trim() - return trimmed.length > PREVIEW_MAX ? `${trimmed.slice(0, PREVIEW_MAX - 1)}…` : trimmed + return trimmed.length > PREVIEW_MAX_CHARS + ? `${trimmed.slice(0, PREVIEW_MAX_CHARS - 1)}…` + : trimmed } return null } - -export const dmAuthorName = (message: ChatMessageDTO): string => authorDisplay(message, "") -export const mentionAuthorName = (message: ChatMessageDTO): string => authorDisplay(message, "") diff --git a/services/api/src/routes/chat-powers-wiring.ts b/services/api/src/routes/chat-powers-wiring.ts index 3f01f140..5fb1c2dd 100644 --- a/services/api/src/routes/chat-powers-wiring.ts +++ b/services/api/src/routes/chat-powers-wiring.ts @@ -35,7 +35,7 @@ type GlobalRole = (typeof ROLE_VALUES)[number] * A thread the caller is not in resolves to no peer and answers false: isDmParticipant already gates * that case, and this helper's contract stays about blocks alone. */ -export function makeIsDmBlocked( +function makeIsDmBlocked( dm: DmRepository, blocks: BlocksRepository, ): (threadId: string, userId: string) => Promise { diff --git a/services/api/src/routes/chat-route-helpers.ts b/services/api/src/routes/chat-route-helpers.ts index 2c019d90..320b6fe8 100644 --- a/services/api/src/routes/chat-route-helpers.ts +++ b/services/api/src/routes/chat-route-helpers.ts @@ -10,6 +10,15 @@ export { neutralizeChatViewerFields } export type ChatRoomKind = "cleanup" | "report" | "group" export const DELETE_MESSAGE_FORBIDDEN = "You can't delete this message." +export const MESSAGE_ALREADY_DELETED = "This message was already deleted." +export const REPORT_NOT_FOUND = "Report not found" + +export const CHAT_HISTORY_DEFAULT_LIMIT = 30 +const CHAT_HISTORY_MAX_LIMIT = 50 + +export function clampChatHistoryLimit(requested: number | undefined): number { + return Math.min(Math.max(requested ?? CHAT_HISTORY_DEFAULT_LIMIT, 1), CHAT_HISTORY_MAX_LIMIT) +} export interface ChatHistorySource { history( @@ -79,13 +88,13 @@ export async function deleteMessageWithPowers( if (tombstone === null) { const state = input.senderPath ? await stateInRoom() : null if (state !== null && state.deletedAt !== null && state.senderId === userId) { - throw AppError.conflict("This message was already deleted.") + throw AppError.conflict(MESSAGE_ALREADY_DELETED) } const powers = await input.resolveChatPowers({ roomKind, roomId, userId }) if (!powers.canDeleteOthers) throw AppError.forbidden(DELETE_MESSAGE_FORBIDDEN) const current = state ?? (await stateInRoom()) if (current !== null && current.deletedAt !== null) { - throw AppError.conflict("This message was already deleted.") + throw AppError.conflict(MESSAGE_ALREADY_DELETED) } tombstone = await input.softDelete({ bypassSenderGate: true }) if (tombstone === null) throw AppError.forbidden(DELETE_MESSAGE_FORBIDDEN) diff --git a/services/api/src/routes/chat.routes.ts b/services/api/src/routes/chat.routes.ts index 94d46b02..29c1f32f 100644 --- a/services/api/src/routes/chat.routes.ts +++ b/services/api/src/routes/chat.routes.ts @@ -14,18 +14,19 @@ import type { Container } from "../di.js" import { requireAuth } from "../auth/context.js" import { parse } from "./_validate.js" import { route } from "../versioning/route.js" -import { roomKeyFor } from "../ws/gateway.js" import { wireChatGateway } from "./chat-gateway-wiring.js" -import { - deleteMessageWithPowers, - DELETE_MESSAGE_FORBIDDEN, - neutralizeChatViewerFields, -} from "./chat-route-helpers.js" +import { deleteMessageWithPowers, DELETE_MESSAGE_FORBIDDEN } from "./chat-route-helpers.js" +import { neutralizeChatViewerFields } from "../services/chat-viewer-fields.js" import { makeChatReactionService } from "../services/chat-reaction-service.js" import type { ChatRepository } from "../services/chat-repository.drizzle.js" import type { ReportChatRepository } from "../services/report-chat-repository.drizzle.js" import type { ChatPollRepository } from "../services/chat-poll-repository.drizzle.js" -import { type GatewayChatMentions, type IsMemberFn, type ReportVisibleFn } from "../ws/gateway.js" +import { + roomKeyFor, + type GatewayChatMentions, + type IsMemberFn, + type ReportVisibleFn, +} from "../ws/gateway.js" import { makeDrizzleGroupThreadsSource, makeDrizzleReportThreadsSource, @@ -79,6 +80,8 @@ declare module "fastify" { } } +const WS_MAX_PAYLOAD_BYTES = 64 * 1024 + const ThreadMessageParamsSchema = z.object({ cleanupId: IdSchema, messageId: IdSchema }).strict() export const CHAT_REACTION_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute" }) @@ -91,7 +94,7 @@ export async function registerChatRoutes( ): Promise { const csrfProtect = container.csrf.protect - await app.register(fastifyWebsocket, { options: { maxPayload: 64 * 1024 } }) + await app.register(fastifyWebsocket, { options: { maxPayload: WS_MAX_PAYLOAD_BYTES } }) const overrides = app.chatOverrides const wiring = wireChatGateway(app, container) diff --git a/services/api/src/routes/cleanups.routes.ts b/services/api/src/routes/cleanups.routes.ts index 1b171c3f..4e3d0cef 100644 --- a/services/api/src/routes/cleanups.routes.ts +++ b/services/api/src/routes/cleanups.routes.ts @@ -168,6 +168,58 @@ export const RequestEventResourcesBodySchema = trimTextFields( "message", ) +type ContainerCleanupDeps = Omit + +function productionCleanupDeps(app: FastifyInstance, container: Container): ContainerCleanupDeps { + return { + presignThumb: (thumbKey: string) => + container.storage.presignGet(thumbKey, MEDIA_GET_URL_TTL_SEC), + resolveJurisdictionGeoid: makeGeoidResolver(container), + resolveAddress: makeCachedAddressResolver(container), + resolveJurisdictionCode: (geoid: string | null) => + resolveJurisdictionCode(container.getDb().sql, geoid), + outboundMail: makeOutboundMailService({ + repo: makeDrizzleMailRepository(container.getDb().sql), + mailer: container.mailer, + env: { + MAIL_FROM_OUTREACH: container.env.MAIL_FROM_OUTREACH, + MAIL_REPLY_DOMAIN: container.env.MAIL_REPLY_DOMAIN, + }, + }), + affiliations: container.getAffiliationLoader(), + notifier: makeRouteNotificationService(container, app.log), + attendeeNotifier: makeCommsRuntime(container, app.log).lanes, + insightsInvalidator: makeInsightsGeneration({ + cache: container.getCache(), + logger: app.log, + }), + counters: container.getCounterStore(), + jobs: container.jobs, + presignEventMedia: makeEventMediaPresigner(container.storage), + audit: makeHostAuditSink(container.getDb().sql, app.log), + enrichDTOs: (dtos, viewerUserId) => enrichCleanupDTOs(container, dtos, viewerUserId), + } +} + +// Under test overrides the service runs on the injected repository plus only the seams the test names; +// the ticket signer still comes from the container. +function overriddenCleanupDeps(overrides: CleanupServiceOverrides): ContainerCleanupDeps { + return { + ...(overrides.presignThumb !== undefined ? { presignThumb: overrides.presignThumb } : {}), + ...(overrides.newId !== undefined ? { newId: overrides.newId } : {}), + ...(overrides.outboundMail !== undefined ? { outboundMail: overrides.outboundMail } : {}), + ...(overrides.notifier !== undefined ? { notifier: overrides.notifier } : {}), + ...(overrides.attendeeNotifier !== undefined + ? { attendeeNotifier: overrides.attendeeNotifier } + : {}), + ...(overrides.counters !== undefined ? { counters: overrides.counters } : {}), + ...(overrides.presignEventMedia !== undefined + ? { presignEventMedia: overrides.presignEventMedia } + : {}), + ...(overrides.audit !== undefined ? { audit: overrides.audit } : {}), + } +} + export function makeContainerCleanupService( app: FastifyInstance, container: Container, @@ -175,59 +227,12 @@ export function makeContainerCleanupService( const overrides = app.cleanupOverrides const repo: CleanupRepository = overrides !== undefined ? overrides.repo : makeDrizzleCleanupRepository(container.getDb().sql) - - return makeCleanupService({ - repo, - tickets: container.getTicketTokenSigner(), - ...(overrides?.presignThumb !== undefined - ? { presignThumb: overrides.presignThumb } - : overrides - ? {} - : { - presignThumb: (thumbKey: string) => - container.storage.presignGet(thumbKey, MEDIA_GET_URL_TTL_SEC), - }), - ...(overrides - ? {} - : { - resolveJurisdictionGeoid: makeGeoidResolver(container), - resolveAddress: makeCachedAddressResolver(container), - resolveJurisdictionCode: (geoid: string | null) => - resolveJurisdictionCode(container.getDb().sql, geoid), - outboundMail: makeOutboundMailService({ - repo: makeDrizzleMailRepository(container.getDb().sql), - mailer: container.mailer, - env: { - MAIL_FROM_OUTREACH: container.env.MAIL_FROM_OUTREACH, - MAIL_REPLY_DOMAIN: container.env.MAIL_REPLY_DOMAIN, - }, - }), - affiliations: container.getAffiliationLoader(), - notifier: makeRouteNotificationService(container, app.log), - attendeeNotifier: makeCommsRuntime(container, app.log).lanes, - insightsInvalidator: makeInsightsGeneration({ - cache: container.getCache(), - logger: app.log, - }), - counters: container.getCounterStore(), - jobs: container.jobs, - presignEventMedia: makeEventMediaPresigner(container.storage), - audit: makeHostAuditSink(container.getDb().sql, app.log), - enrichDTOs: (dtos, viewerUserId) => enrichCleanupDTOs(container, dtos, viewerUserId), - }), - ...(overrides?.newId !== undefined ? { newId: overrides.newId } : {}), - ...(overrides?.outboundMail !== undefined ? { outboundMail: overrides.outboundMail } : {}), - ...(overrides?.notifier !== undefined ? { notifier: overrides.notifier } : {}), - ...(overrides?.attendeeNotifier !== undefined - ? { attendeeNotifier: overrides.attendeeNotifier } - : {}), - ...(overrides?.counters !== undefined ? { counters: overrides.counters } : {}), - ...(overrides?.presignEventMedia !== undefined - ? { presignEventMedia: overrides.presignEventMedia } - : {}), - ...(overrides?.audit !== undefined ? { audit: overrides.audit } : {}), - logger: app.log, - }) + const tickets = container.getTicketTokenSigner() + const deps = + overrides !== undefined + ? overriddenCleanupDeps(overrides) + : productionCleanupDeps(app, container) + return makeCleanupService({ repo, tickets, ...deps, logger: app.log }) } export async function registerCleanupRoutes( diff --git a/services/api/src/routes/conversations.routes.ts b/services/api/src/routes/conversations.routes.ts index a70679ea..ff8568a9 100644 --- a/services/api/src/routes/conversations.routes.ts +++ b/services/api/src/routes/conversations.routes.ts @@ -114,6 +114,12 @@ export async function registerConversationRoutes( const getMarkRoomRead = (): MarkRoomRead => overrides?.markRoomRead ?? (markRoomRead ??= conversationReadSeam(app, container).markRoomRead) + const nudgeThread = (userId: string, roomId: string): void => { + void Promise.resolve( + container.userChannel?.publishToUser(userId, { topic: "threads", id: roomId }), + ).catch(() => {}) + } + route( app, "toggleConversationMute", @@ -147,9 +153,7 @@ export async function registerConversationRoutes( throw AppError.forbidden("You can't change this conversation.") } await getHidesRepo().setHidden(userId, body.roomKind, body.roomId, body.hidden) - void Promise.resolve( - container.userChannel?.publishToUser(userId, { topic: "threads", id: body.roomId }), - ).catch(() => {}) + nudgeThread(userId, body.roomId) const payload: ToggleHiddenResponse = { hidden: body.hidden } reply.status(200).send(payload) }, @@ -166,9 +170,7 @@ export async function registerConversationRoutes( throw AppError.forbidden("You can't open this conversation.") } await getMarkRoomRead()(body.roomKind, body.roomId, userId) - void Promise.resolve( - container.userChannel?.publishToUser(userId, { topic: "threads", id: body.roomId }), - ).catch(() => {}) + nudgeThread(userId, body.roomId) const payload: MarkThreadReadResponse = { ok: true } reply.status(200).send(payload) }, diff --git a/services/api/src/routes/dm.routes.ts b/services/api/src/routes/dm.routes.ts index 6c5d9c0d..6f8143dd 100644 --- a/services/api/src/routes/dm.routes.ts +++ b/services/api/src/routes/dm.routes.ts @@ -27,7 +27,13 @@ import { makeConversationMutesRepository, type ConversationMutesRepository, } from "../services/conversation-mutes-repository.drizzle.js" -import { chatHistoryPayload, neutralizeChatViewerFields } from "./chat-route-helpers.js" +import { + CHAT_HISTORY_DEFAULT_LIMIT, + chatHistoryPayload, + DELETE_MESSAGE_FORBIDDEN, + MESSAGE_ALREADY_DELETED, +} from "./chat-route-helpers.js" +import { neutralizeChatViewerFields } from "../services/chat-viewer-fields.js" import { chatMentionDeps, type ChatMentionSeam } from "./chat-gateway-wiring.js" const DmIdParamsSchema = z.object({ id: IdSchema }).strict() @@ -41,8 +47,6 @@ export const DM_OPEN_RATE_LIMIT = perIdentity({ max: 20, timeWindow: "1 minute" export const DM_REACTION_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute" }) export const DM_MESSAGE_MUTATION_RATE_LIMIT = perIdentity({ max: 30, timeWindow: "1 minute" }) -const DM_HISTORY_DEFAULT_LIMIT = 30 - export async function registerDmRoutes(app: FastifyInstance, container: Container): Promise { const csrfProtect = container.csrf.protect @@ -120,7 +124,7 @@ export async function registerDmRoutes(app: FastifyInstance, container: Containe const q = parse(DmHistoryQuerySchema, request.query) await authorizePeer(id, userId, "You can't view this conversation.") - const limit = q.limit ?? DM_HISTORY_DEFAULT_LIMIT + const limit = q.limit ?? CHAT_HISTORY_DEFAULT_LIMIT const payload: ChatHistoryResponse = await chatHistoryPayload( { history: (before, pageLimit, around) => @@ -210,7 +214,7 @@ export async function registerDmRoutes(app: FastifyInstance, container: Containe async (request, reply) => { const userId = requireAuth(request) const { threadId, messageId } = parse(ThreadMessageParamsSchema, request.params) - await authorizePeer(threadId, userId, "You can't delete this message.") + await authorizePeer(threadId, userId, DELETE_MESSAGE_FORBIDDEN) const tombstone: ChatMessageDTO | null = await dmRepo().softDelete( threadId, @@ -225,9 +229,9 @@ export async function registerDmRoutes(app: FastifyInstance, container: Containe meta.deletedAt !== null && meta.senderId === userId ) { - throw AppError.conflict("This message was already deleted.") + throw AppError.conflict(MESSAGE_ALREADY_DELETED) } - throw AppError.forbidden("You can't delete this message.") + throw AppError.forbidden(DELETE_MESSAGE_FORBIDDEN) } const roomView = neutralizeChatViewerFields(tombstone) diff --git a/services/api/src/routes/forms.routes.ts b/services/api/src/routes/forms.routes.ts index 5c268dd8..39197857 100644 --- a/services/api/src/routes/forms.routes.ts +++ b/services/api/src/routes/forms.routes.ts @@ -13,13 +13,18 @@ import { sanitizeHeaderValue } from "../adapters/mail-text.js" import { parse } from "./_validate.js" import { exposeMessage } from "../errors/exposed-message.js" -export const HOME_TURF_RATE_LIMIT = perHost({ max: 5, timeWindow: "1 minute" }) +const HOME_TURF_REQUESTS_PER_MINUTE = 5 -export const HOME_TURF_BODY_LIMIT = 16384 +export const HOME_TURF_RATE_LIMIT = perHost({ + max: HOME_TURF_REQUESTS_PER_MINUTE, + timeWindow: "1 minute", +}) + +const HOME_TURF_BODY_LIMIT = 16384 export const HOME_TURF_IP_LIMIT_PER_HOUR = 10 -export const HOME_TURF_IP_WINDOW_SECONDS = 60 * 60 +const HOME_TURF_IP_WINDOW_SECONDS = 60 * 60 const HOME_TURF_IP_COUNTER_PREFIX = "abuse:home-turf:ip:" @@ -27,7 +32,14 @@ const HOME_TURF_EMAIL_COUNTER_PREFIX = "abuse:home-turf:email:" export const HOME_TURF_EMAIL_LIMIT_PER_DAY = 1 -export const HOME_TURF_EMAIL_WINDOW_SECONDS = 24 * 60 * 60 +const HOME_TURF_EMAIL_WINDOW_SECONDS = 24 * 60 * 60 + +const MAX_EMAIL_LENGTH = 254 +const MAX_NOTES_LENGTH = 2000 +const MAX_TOKEN_LENGTH = 4096 +const GMAIL_DOMAIN = "gmail.com" +const GOOGLEMAIL_DOMAIN = "googlemail.com" +const OK_BODY = { ok: true } as const export interface HomeTurfOverrides { counters?: CounterStore @@ -46,11 +58,11 @@ const HomeTurfFormSchema = z school: z.string().trim().min(1).max(160), city: z.string().trim().min(1).max(120), teamSize: z.string().trim().min(1).max(30), - email: z.string().trim().max(254).email(), + email: z.string().trim().max(MAX_EMAIL_LENGTH).email(), phone: z.string().trim().min(1).max(40), - notes: z.string().trim().max(2000).optional(), - turnstileToken: z.string().min(1).max(4096), - honeypot: z.string().max(4096).optional(), + notes: z.string().trim().max(MAX_NOTES_LENGTH).optional(), + turnstileToken: z.string().min(1).max(MAX_TOKEN_LENGTH), + honeypot: z.string().max(MAX_TOKEN_LENGTH).optional(), }) .strict() @@ -67,7 +79,7 @@ export async function enforceHomeTurfIpCap( } } -export function canonicalizeHomeTurfEmail(email: string): string { +function canonicalizeHomeTurfEmail(email: string): string { const trimmed = email.trim().toLowerCase() const at = trimmed.lastIndexOf("@") if (at <= 0 || at === trimmed.length - 1) return trimmed @@ -75,8 +87,8 @@ export function canonicalizeHomeTurfEmail(email: string): string { let domain = trimmed.slice(at + 1) const plus = local.indexOf("+") if (plus !== -1) local = local.slice(0, plus) - if (domain === "googlemail.com") domain = "gmail.com" - if (domain === "gmail.com") local = local.replace(/\./g, "") + if (domain === GOOGLEMAIL_DOMAIN) domain = GMAIL_DOMAIN + if (domain === GMAIL_DOMAIN) local = local.replace(/\./g, "") return `${local}@${domain}` } @@ -154,7 +166,7 @@ export async function registerHomeTurfRoutes( { ip: request.ip }, "home-turf form: honeypot tripped; fake success, no mail", ) - return reply.status(200).send({ ok: true }) + return reply.status(200).send(OK_BODY) } const notifyTo = requireNotifyTo() @@ -169,7 +181,7 @@ export async function registerHomeTurfRoutes( request.log.info( "home-turf form: recipient confirmation cap reached; staff notified, confirmation skipped", ) - return reply.status(200).send({ ok: true }) + return reply.status(200).send(OK_BODY) } try { @@ -181,7 +193,7 @@ export async function registerHomeTurfRoutes( ) } - return reply.status(200).send({ ok: true }) + return reply.status(200).send(OK_BODY) }, ) } diff --git a/services/api/src/routes/geo.routes.ts b/services/api/src/routes/geo.routes.ts index aeabc8c2..23bbca07 100644 --- a/services/api/src/routes/geo.routes.ts +++ b/services/api/src/routes/geo.routes.ts @@ -10,6 +10,7 @@ import { route } from "../versioning/route.js" import { parse } from "./_validate.js" const APPROXIMATE_LOCATION_RATE_LIMIT = { max: 60, timeWindow: "1 minute" } as const +const APPROXIMATE_LOCATION_CACHE_CONTROL = "private, max-age=300" const ApproximateLocationResponseJsonSchema = { type: "object", @@ -43,7 +44,7 @@ export async function registerGeoRoutes(app: FastifyInstance, container: Contain }, ) - reply.header("Cache-Control", "private, max-age=300") + reply.header("Cache-Control", APPROXIMATE_LOCATION_CACHE_CONTROL) reply.status(200).send(payload) }, ) diff --git a/services/api/src/routes/guest-rsvp.routes.ts b/services/api/src/routes/guest-rsvp.routes.ts index 3a43fda9..8b1a908e 100644 --- a/services/api/src/routes/guest-rsvp.routes.ts +++ b/services/api/src/routes/guest-rsvp.routes.ts @@ -40,7 +40,7 @@ declare module "fastify" { const CleanupIdParamsSchema = z.object({ id: IdSchema }).strict() -export const GuestRsvpRequestBodySchema = trimTextFields(GuestRsvpRequestRequestSchema, "name") +const GuestRsvpRequestBodySchema = trimTextFields(GuestRsvpRequestRequestSchema, "name") export async function registerGuestRsvpRoutes( app: FastifyInstance, diff --git a/services/api/src/routes/local-storage.routes.ts b/services/api/src/routes/local-storage.routes.ts index be9e4c3d..3a9bba15 100644 --- a/services/api/src/routes/local-storage.routes.ts +++ b/services/api/src/routes/local-storage.routes.ts @@ -1,7 +1,7 @@ import { Transform, type Readable } from "node:stream" import { AppError, MAX_VIDEO_BYTES } from "@civfix/shared" import { z } from "zod" -import type { FastifyInstance, FastifyRequest, RequestPayload } from "fastify" +import type { FastifyInstance, FastifyReply, FastifyRequest, RequestPayload } from "fastify" import { isSafeObjectKey, LOCAL_STORAGE_ROUTE_PREFIX, @@ -24,6 +24,9 @@ const OBJECT_ROUTE = `${LOCAL_STORAGE_ROUTE_PREFIX}/:namespace/*` const LOCAL_STORAGE_RATE_LIMIT = { max: 600, timeWindow: "1 minute" } as const const CROSS_ORIGIN_RESOURCE_POLICY_FOR_EMBEDDABLE_MEDIA = "cross-origin" +const OBJECT_CACHE_CONTROL = "private, max-age=60" +const OBJECT_NOT_FOUND = "Object not found" +const SIZE_MISMATCH = "Upload size does not match the presigned Content-Length" const MEDIA_TYPE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]*\/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]*$/ @@ -96,7 +99,7 @@ export async function registerLocalStorageRoutes( throw AppError.mediaRejected("Upload body is missing") } if (body.byteLength !== grant.byteSize) { - throw AppError.mediaRejected("Upload size does not match the presigned Content-Length") + throw AppError.mediaRejected(SIZE_MISMATCH) } await grant.storage.put(grant.key, body, { contentType: grant.contentType }) reply.status(200).send({ ok: true }) @@ -110,18 +113,9 @@ export async function registerLocalStorageRoutes( const { storage, key } = resolveGetGrant(request, byNamespace) const head = await storage.head(key) if (head === null) { - throw AppError.notFound("Object not found") + throw AppError.notFound(OBJECT_NOT_FOUND) } - - reply.header("accept-ranges", "bytes") - reply.header("cache-control", "private, max-age=60") - if (head.etag !== undefined) { - reply.header("etag", `"${head.etag}"`) - } - reply.header( - "cross-origin-resource-policy", - CROSS_ORIGIN_RESOURCE_POLICY_FOR_EMBEDDABLE_MEDIA, - ) + setObjectHeaders(reply, head) const range = parseRange(request.headers.range, head.size) if (range === "unsatisfiable") { @@ -145,6 +139,15 @@ export async function registerLocalStorageRoutes( }) } +function setObjectHeaders(reply: FastifyReply, head: { etag?: string | undefined }): void { + reply.header("accept-ranges", "bytes") + reply.header("cache-control", OBJECT_CACHE_CONTROL) + if (head.etag !== undefined) { + reply.header("etag", `"${head.etag}"`) + } + reply.header("cross-origin-resource-policy", CROSS_ORIGIN_RESOURCE_POLICY_FOR_EMBEDDABLE_MEDIA) +} + function resolvePutGrant( request: FastifyRequest, byNamespace: ReadonlyMap, @@ -221,7 +224,7 @@ function assertDeclaredUploadMatchesGrant(request: FastifyRequest, grant: Resolv } const declaredLength = request.headers["content-length"] if (declaredLength !== undefined && Number(declaredLength) !== grant.byteSize) { - throw AppError.mediaRejected("Upload size does not match the presigned Content-Length") + throw AppError.mediaRejected(SIZE_MISMATCH) } } @@ -232,7 +235,7 @@ function storageOf( const namespace = (request.params as Record).namespace const storage = typeof namespace === "string" ? byNamespace.get(namespace) : undefined if (storage === undefined) { - throw AppError.notFound("Object not found") + throw AppError.notFound(OBJECT_NOT_FOUND) } return storage } @@ -241,7 +244,7 @@ function objectKeyOf(request: FastifyRequest): string { const wildcard = (request.params as Record)["*"] const key = typeof wildcard === "string" ? wildcard : "" if (!isSafeObjectKey(key)) { - throw AppError.notFound("Object not found") + throw AppError.notFound(OBJECT_NOT_FOUND) } return key } diff --git a/services/api/src/routes/map.routes.ts b/services/api/src/routes/map.routes.ts index 00314897..11c85fc8 100644 --- a/services/api/src/routes/map.routes.ts +++ b/services/api/src/routes/map.routes.ts @@ -30,8 +30,12 @@ import { route } from "../versioning/route.js" export { MAP_CLEANUPS_LIMIT } -export const CARTO_VOYAGER_RASTER_URL = +const CARTO_VOYAGER_RASTER_URL = "https://a.basemaps.cartocdn.com/rastertiles/voyager/{z}/{x}/{y}{r}.png" +const CARTO_ATTRIBUTION = "(c) OpenStreetMap contributors, (c) CARTO" +const TILE_INFO_CACHE_CONTROL = "public, max-age=3600" +const MAP_CLEANUPS_CACHE_CONTROL = "public, max-age=60" +const CONTACT_SUGGESTED_AUDIT_ACTION = "discovery.contact_suggested" const CleanupsQuerySchema = z.object({ bbox: CappedBBoxQueryParam, @@ -108,12 +112,12 @@ export async function registerMapRoutes(app: FastifyInstance, container: Contain const payload: TileInfoResponse = { pmtilesUrl: "", rasterUrl: env.TILES_RASTER_URL ?? CARTO_VOYAGER_RASTER_URL, - attribution: "(c) OpenStreetMap contributors, (c) CARTO", + attribution: CARTO_ATTRIBUTION, minZoom: env.TILES_MIN_ZOOM, maxZoom: env.TILES_MAX_ZOOM, bounds: env.TILES_BOUNDS, } - reply.header("Cache-Control", "public, max-age=3600") + reply.header("Cache-Control", TILE_INFO_CACHE_CONTROL) reply.status(200).send(payload) }) @@ -199,7 +203,7 @@ export async function registerMapRoutes(app: FastifyInstance, container: Contain const repo = makeCleanupMapRepository(container.getDb().sql) const pins = await repo.listCleanupPins(bbox, when) const payload: MapCleanupsResponse = { pins } - reply.header("Cache-Control", "public, max-age=60") + reply.header("Cache-Control", MAP_CLEANUPS_CACHE_CONTROL) reply.status(200).send(payload) }, ) @@ -222,7 +226,7 @@ export async function registerMapRoutes(app: FastifyInstance, container: Contain await writeAudit(container.getDb().sql, { actorId: null, - action: "discovery.contact_suggested", + action: CONTACT_SUGGESTED_AUDIT_ACTION, target: `jurisdiction:${geoid}`, meta: { email: body.email ?? null, diff --git a/services/api/src/routes/messages.routes.ts b/services/api/src/routes/messages.routes.ts index 56f5062b..560f8257 100644 --- a/services/api/src/routes/messages.routes.ts +++ b/services/api/src/routes/messages.routes.ts @@ -8,6 +8,7 @@ import { ToggleMessageReactionRequestSchema, VotePollRequestSchema, type ChatMessageDTO, + type SetMessagePinnedRequest, } from "@civfix/shared" import { randomUUID } from "node:crypto" import { z } from "zod" @@ -37,7 +38,8 @@ import { makeDrizzleDiscussionRepository } from "../services/discussion-reposito import type { DiscussionRepository } from "../services/discussion-types.js" import { isReportVisibleTo } from "../services/report-visibility.js" import { makeDmPeerOf } from "../services/dm-peer.js" -import { messageRoomMatches, neutralizeChatViewerFields } from "./chat-route-helpers.js" +import { messageRoomMatches, REPORT_NOT_FOUND } from "./chat-route-helpers.js" +import { neutralizeChatViewerFields } from "../services/chat-viewer-fields.js" import { makePrivateMediaPresigner } from "../services/media-presign.js" import { chatMentionDeps, type ChatMentionSeam } from "./chat-gateway-wiring.js" import type { DmRepository } from "../services/dm-repository.drizzle.js" @@ -79,12 +81,26 @@ export const VOTE_POLL_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute export const CLOSE_POLL_RATE_LIMIT = perIdentity({ max: 30, timeWindow: "1 minute" }) -export async function registerMessagesRoutes( - app: FastifyInstance, - container: Container, -): Promise { - const csrfProtect = container.csrf.protect +const MESSAGE_NOT_FOUND = "Message not found" +const MESSAGE_DELETED = "This message was deleted." +const PIN_FORBIDDEN_FIELD_CODE = "pin_forbidden" + +interface MessagesDeps { + getChatRepo(): ChatRepository + getReportChatRepo(): ReportChatRepository + dmRepo(): DmRepository + isBlockedEitherWay(a: string, b: string): Promise + dmPeerOf: ReturnType + isCleanupMember: IsRoomMemberFn + isGroupMember: IsRoomMemberFn + canSendGroup: IsRoomMemberFn + isReportChatMember: IsRoomMemberFn + isReportVisible(reportId: string, userId: string): Promise + requireVisibleReport(reportId: string, userId: string): Promise + chatMentions: ChatMentionSeam | undefined +} +function makeMessagesDeps(app: FastifyInstance, container: Container): MessagesDeps { const overrides = app.chatOverrides const useFakeChat = container.env.USE_FAKE_CHAT @@ -98,11 +114,7 @@ export async function registerMessagesRoutes( let reportChatRepo: ReportChatRepository | undefined const getReportChatRepo = (): ReportChatRepository => - overrides?.reportChat ?? - (reportChatRepo ??= makeReportChatRepository( - container.getDb().sql, - makePrivateMediaPresigner(container.storage), - )) + overrides?.reportChat ?? (reportChatRepo ??= makeReportChatRepository(container.getDb().sql)) let cleanupRepo: ReturnType | undefined const getCleanupRepo = (): ReturnType => @@ -134,8 +146,6 @@ export async function registerMessagesRoutes( const dmRepo = (): DmRepository => overrides?.dmRepo ?? container.getDmRepo() const blocksRepo = (): BlocksRepository => overrides?.blocksRepo ?? container.getBlocksRepo() - const dmPeerOf = makeDmPeerOf({ getThread: (threadId) => dmRepo().getThread(threadId) }) - let discussionRepo: DiscussionRepository | undefined const getReportLookup = (): DiscussionRepository | undefined => app.discussionOverrides?.repo ?? @@ -151,12 +161,79 @@ export async function registerMessagesRoutes( return isReportVisibleTo(await repo.findReportForDiscussion(reportId), userId) } - const requireVisibleReport = async (reportId: string, userId: string): Promise => { - if (!(await isReportVisible(reportId, userId))) throw AppError.notFound("Report not found") + return { + getChatRepo, + getReportChatRepo, + dmRepo, + isBlockedEitherWay: (a, b) => blocksRepo().isBlockedEitherWay(a, b), + dmPeerOf: makeDmPeerOf({ getThread: (threadId) => dmRepo().getThread(threadId) }), + isCleanupMember, + isGroupMember, + canSendGroup, + isReportChatMember: (roomId, userId) => getReportChatRepo().isMember(roomId, userId), + isReportVisible, + requireVisibleReport: async (reportId, userId) => { + if (!(await isReportVisible(reportId, userId))) throw AppError.notFound(REPORT_NOT_FOUND) + }, + chatMentions: + overrides?.chatMentions ?? (useFakeChat ? undefined : chatMentionDeps(app, container)), } +} - const chatMentions: ChatMentionSeam | undefined = - overrides?.chatMentions ?? (useFakeChat ? undefined : chatMentionDeps(app, container)) +type PinRoomKind = SetMessagePinnedRequest["roomKind"] + +async function loadPinTarget( + deps: MessagesDeps, + roomKind: PinRoomKind, + roomId: string, + messageId: string, +): Promise<{ kind: string; deletedAt: Date | null }> { + if (roomKind === "dm") { + const meta = await deps.dmRepo().findMessageMeta(messageId) + if (meta === null || meta.threadId !== roomId) throw AppError.notFound(MESSAGE_NOT_FOUND) + return { kind: meta.kind, deletedAt: meta.deletedAt } + } + const meta = await deps.getChatRepo().findMessageMeta(messageId) + if (!messageRoomMatches(meta, roomKind, roomId)) throw AppError.notFound(MESSAGE_NOT_FOUND) + return { kind: meta.kind, deletedAt: meta.deletedAt } +} + +function setPinnedIn( + deps: MessagesDeps, + roomKind: PinRoomKind, + roomId: string, + messageId: string, + userId: string, + pinned: boolean, +): Promise { + if (roomKind === "dm") return deps.dmRepo().setPinned(roomId, messageId, userId, pinned) + const chat = deps.getChatRepo() + if (roomKind === "report") return chat.setReportPinned(roomId, messageId, userId, pinned) + if (roomKind === "group") return chat.setGroupPinned(roomId, messageId, userId, pinned) + return chat.setPinned(roomId, messageId, userId, pinned) +} + +export async function registerMessagesRoutes( + app: FastifyInstance, + container: Container, +): Promise { + const csrfProtect = container.csrf.protect + const overrides = app.chatOverrides + const deps = makeMessagesDeps(app, container) + const { + getChatRepo, + getReportChatRepo, + dmRepo, + isBlockedEitherWay, + dmPeerOf, + isCleanupMember, + isGroupMember, + canSendGroup, + isReportChatMember, + isReportVisible, + requireVisibleReport, + chatMentions, + } = deps route( app, @@ -175,7 +252,7 @@ export async function registerMessagesRoutes( isReportVisible, isGroupMember: canSendGroup, dmPeerOf, - isBlockedEitherWay: (a, b) => blocksRepo().isBlockedEitherWay(a, b), + isBlockedEitherWay, ...(chatMentions ? { chatMentions } : {}), broadcastEvent: (roomKey, frame) => container.chatService.broadcastEvent?.(roomKey, frame), }) @@ -208,37 +285,18 @@ export async function registerMessagesRoutes( const powers = await resolveChatPowers({ roomKind, roomId, userId }) if (!powers.canPin) { throw new AppError(ErrorCode.FORBIDDEN, "You can't pin messages in this chat.", { - fields: { code: "pin_forbidden" }, + fields: { code: PIN_FORBIDDEN_FIELD_CODE }, }) } - let kind: string - let deletedAt: Date | null - if (roomKind === "dm") { - const meta = await dmRepo().findMessageMeta(messageId) - if (meta === null || meta.threadId !== roomId) throw AppError.notFound("Message not found") - ;({ kind, deletedAt } = meta) - } else { - const meta = await getChatRepo().findMessageMeta(messageId) - if (!messageRoomMatches(meta, roomKind, roomId)) { - throw AppError.notFound("Message not found") - } - ;({ kind, deletedAt } = meta) + const { kind, deletedAt } = await loadPinTarget(deps, roomKind, roomId, messageId) + if (kind === "system") { + throw AppError.validation({ messageId: "System messages can't be pinned." }) } + if (deletedAt !== null) throw AppError.validation({ messageId: MESSAGE_DELETED }) - if (kind === "system") - throw AppError.validation({ messageId: "System messages can't be pinned." }) - if (deletedAt !== null) throw AppError.validation({ messageId: "This message was deleted." }) - - const updated: ChatMessageDTO | null = - roomKind === "dm" - ? await dmRepo().setPinned(roomId, messageId, userId, pinned) - : roomKind === "report" - ? await getChatRepo().setReportPinned(roomId, messageId, userId, pinned) - : roomKind === "group" - ? await getChatRepo().setGroupPinned(roomId, messageId, userId, pinned) - : await getChatRepo().setPinned(roomId, messageId, userId, pinned) - if (updated === null) throw AppError.validation({ messageId: "This message was deleted." }) + const updated = await setPinnedIn(deps, roomKind, roomId, messageId, userId, pinned) + if (updated === null) throw AppError.validation({ messageId: MESSAGE_DELETED }) broadcastMessageUpdate( container.chatService, @@ -267,7 +325,7 @@ export async function registerMessagesRoutes( isReportChatMember: (reportId, uid) => getReportChatRepo().isMember(reportId, uid), isChatGroupMember: isGroupMember, dmPeerOf, - isBlockedEitherWay: (a, b) => blocksRepo().isBlockedEitherWay(a, b), + isBlockedEitherWay, isReportVisible, }) const updated: ChatMessageDTO = await reactions.toggleReaction({ @@ -294,8 +352,6 @@ export async function registerMessagesRoutes( let pollService: ReturnType | undefined const pollNotifier = makeContainerPollNotifier(container, app.log) - const isReportChatMember = (roomId: string, userId: string): Promise => - getReportChatRepo().isMember(roomId, userId) const getPollService = (): ReturnType => (pollService ??= makeChatPollService({ chat: getChatRepo(), diff --git a/services/api/src/routes/report-chat.routes.ts b/services/api/src/routes/report-chat.routes.ts index deb9fc8e..d38e0536 100644 --- a/services/api/src/routes/report-chat.routes.ts +++ b/services/api/src/routes/report-chat.routes.ts @@ -20,9 +20,11 @@ import { route } from "../versioning/route.js" import { roomKeyFor } from "../ws/gateway.js" import { chatHistoryPayload, + clampChatHistoryLimit, deleteMessageWithPowers, - neutralizeChatViewerFields, + REPORT_NOT_FOUND, } from "./chat-route-helpers.js" +import { neutralizeChatViewerFields } from "../services/chat-viewer-fields.js" import { makeDrizzleChatRepository, type ChatRepository, @@ -52,9 +54,6 @@ declare module "fastify" { const ReportChatIdParamsSchema = z.object({ id: IdSchema }).strict() const ReportChatMessageParamsSchema = z.object({ id: IdSchema, messageId: IdSchema }).strict() -const REPORT_CHAT_HISTORY_DEFAULT = 30 -const REPORT_CHAT_HISTORY_MAX = 50 - export const REPORT_REACTION_RATE_LIMIT = perIdentity({ max: 60, timeWindow: "1 minute" }) export const REPORT_DELETE_RATE_LIMIT = perIdentity({ max: 30, timeWindow: "1 minute" }) @@ -78,10 +77,7 @@ export async function registerReportChatRoutes( let reportChatRepo: ReportChatRepository | undefined const getReportChatRepo = (): ReportChatRepository => app.chatOverrides?.reportChat ?? - (reportChatRepo ??= makeReportChatRepository( - container.getDb().sql, - makePrivateMediaPresigner(container.storage), - )) + (reportChatRepo ??= makeReportChatRepository(container.getDb().sql)) let discussionRepo: DiscussionRepository | undefined const getReportRepo = (): DiscussionRepository => @@ -93,16 +89,19 @@ export async function registerReportChatRoutes( viewerUserId: string | null, ): Promise => { const report = await getReportRepo().findReportForDiscussion(reportId) - if (!isReportVisibleTo(report, viewerUserId)) throw AppError.notFound("Report not found") + if (!isReportVisibleTo(report, viewerUserId)) throw AppError.notFound(REPORT_NOT_FOUND) + } + + const nudgeThreads = (userId: string): void => { + void Promise.resolve(container.userChannel?.publishToUser(userId, { topic: "threads" })).catch( + () => {}, + ) } route(app, "reportMessages", async (request, reply) => { const { id } = parse(ReportChatIdParamsSchema, request.params) const q = parse(ReportChatHistoryRequestSchema, { ...(request.query as object), id }) - const limit = Math.min( - Math.max(q.limit ?? REPORT_CHAT_HISTORY_DEFAULT, 1), - REPORT_CHAT_HISTORY_MAX, - ) + const limit = clampChatHistoryLimit(q.limit) const viewerUserId = request.auth?.userId ?? null await requireVisibleReport(id, viewerUserId) const payload: ChatHistoryResponse = await chatHistoryPayload( @@ -191,9 +190,7 @@ export async function registerReportChatRoutes( parse(JoinReportChatRequestSchema, { id }) await requireVisibleReport(id, userId) await getReportChatRepo().join(id, userId, "member") - void Promise.resolve( - container.userChannel?.publishToUser(userId, { topic: "threads" }), - ).catch(() => {}) + nudgeThreads(userId) reply.status(200).send({ ok: true }) }, ) @@ -231,9 +228,7 @@ export async function registerReportChatRoutes( const { id } = parse(ReportChatIdParamsSchema, request.params) parse(LeaveReportChatRequestSchema, { id }) await getReportChatRepo().leave(id, userId) - void Promise.resolve( - container.userChannel?.publishToUser(userId, { topic: "threads" }), - ).catch(() => {}) + nudgeThreads(userId) reply.status(200).send({ ok: true }) }, ) diff --git a/services/api/src/routes/report-content.routes.ts b/services/api/src/routes/report-content.routes.ts index 90d79ff7..f3a56986 100644 --- a/services/api/src/routes/report-content.routes.ts +++ b/services/api/src/routes/report-content.routes.ts @@ -24,6 +24,11 @@ export const REPORT_CONTENT_RATE_LIMIT = perIdentity({ hostMax: 60, }) +const OWNER_TAKEDOWN_FLAG = "Owner takedown request" +const USER_REPORT_FLAG = "User report" +const FALLBACK_REPORTER_LABEL = "User" +const TAKEDOWN_REQUESTED_AUDIT_ACTION = "report.takedown_requested" + declare module "fastify" { interface FastifyInstance { contentSubjectGate?: ContentSubjectGate @@ -67,11 +72,7 @@ export async function registerReportContentRoutes( await subjectGate().assertReportable(body.subjectType, body.subjectId, userId) const store = app.authServices?.users - const reporterUser = store ? await store.findById(userId) : null - const reporter = - reporterUser?.handle != null && reporterUser.handle !== "" - ? `@${reporterUser.handle}` - : (reporterUser?.displayName ?? "User") + const reporter = reporterLabel(store ? await store.findById(userId) : null) const isOwnerTakedown = body.subjectType === "report" && @@ -81,7 +82,7 @@ export async function registerReportContentRoutes( kind: "user_report", subjectType: body.subjectType, subjectId: body.subjectId, - flag: isOwnerTakedown ? "Owner takedown request" : "User report", + flag: isOwnerTakedown ? OWNER_TAKEDOWN_FLAG : USER_REPORT_FLAG, reason: body.reason, reporter, reporterUserId: userId, @@ -93,7 +94,7 @@ export async function registerReportContentRoutes( if (isOwnerTakedown) { await writeAudit(container.getDb().sql, { actorId: userId, - action: "report.takedown_requested", + action: TAKEDOWN_REQUESTED_AUDIT_ACTION, target: `report:${body.subjectId}`, meta: { reason: body.reason, via: "content-reports" }, }) @@ -105,6 +106,14 @@ export async function registerReportContentRoutes( ) } +function reporterLabel( + user: { handle?: string | null; displayName?: string | null } | null | undefined, +): string { + return user?.handle != null && user.handle !== "" + ? `@${user.handle}` + : (user?.displayName ?? FALLBACK_REPORTER_LABEL) +} + async function isOwnerTakedownReport( container: Container, reportId: string, diff --git a/services/api/src/routes/reports.routes.ts b/services/api/src/routes/reports.routes.ts index 725f6904..15d2b7d3 100644 --- a/services/api/src/routes/reports.routes.ts +++ b/services/api/src/routes/reports.routes.ts @@ -26,6 +26,9 @@ import { makeMediaPresigner, makePrivateMediaPresigner } from "../services/media import { perIdentity } from "../plugins/rate-limit.js" import { makeReportService, + type ReportChatMeta, + type ReportDiscussionMeta, + type ReportOwner, type ReportRepository, type ReportService, type ReportServiceDeps, @@ -50,6 +53,9 @@ export const CREATE_REPORT_RATE_LIMIT = perIdentity({ const MAP_REPORTS_RATE_LIMIT = { max: 60, timeWindow: "1 minute" } as const const SEARCH_REPORTS_RATE_LIMIT = { max: 30, timeWindow: "1 minute" } as const +const MAP_REPORTS_CACHE_CONTROL = "public, max-age=60" +const MAX_MAP_ZOOM = 22 +const MAX_SEARCH_LIMIT = 50 export interface ReportServiceOverrides { repo: ReportRepository @@ -84,7 +90,7 @@ export const CreateReportBodySchema = trimTextFields( const ReportRefOrIdParamsSchema = z.object({ id: ReportRefOrIdSchema }).strict() -const ZoomQueryParam = z.coerce.number().int().min(0).max(22) +const ZoomQueryParam = z.coerce.number().int().min(0).max(MAX_MAP_ZOOM) const MapReportsQuerySchema = z .object({ @@ -107,7 +113,7 @@ const SearchReportsQuerySchema = z categories: CategoriesQueryParam.optional(), types: TypesQueryParam.optional(), cursor: z.string().optional(), - limit: z.coerce.number().int().positive().max(50).optional(), + limit: z.coerce.number().int().positive().max(MAX_SEARCH_LIMIT).optional(), }) .transform((q) => ({ ...(q.q !== undefined ? { q: q.q } : {}), @@ -176,120 +182,8 @@ export async function registerReportRoutes( function service(): ReportService { const overrides = app.reportOverrides - if (overrides) { - return makeReportService({ - repo: overrides.repo, - resolveJurisdictionGeoid: - overrides.resolveJurisdictionGeoid ?? (() => Promise.resolve(null)), - ...(overrides.resolveJurisdictionCode !== undefined - ? { resolveJurisdictionCode: overrides.resolveJurisdictionCode } - : {}), - presignMedia: overrides.presignMedia ?? makeMediaPresigner(container.storage), - presignPrivateMedia: - overrides.presignPrivateMedia ?? makePrivateMediaPresigner(container.storage), - ...(overrides.resolveAddress !== undefined - ? { resolveAddress: overrides.resolveAddress } - : {}), - ...(overrides.loadLinkedEventsForReports !== undefined - ? { loadLinkedEventsForReports: overrides.loadLinkedEventsForReports } - : {}), - ...(overrides.loadDiscussionMeta !== undefined - ? { loadDiscussionMeta: overrides.loadDiscussionMeta } - : {}), - ...(overrides.loadReportChatMeta !== undefined - ? { loadReportChatMeta: overrides.loadReportChatMeta } - : {}), - ...(overrides.joinReportChatAsOwner !== undefined - ? { joinReportChatAsOwner: overrides.joinReportChatAsOwner } - : {}), - ...(overrides.reportChatEmitter !== undefined - ? { reportChatEmitter: overrides.reportChatEmitter } - : {}), - ...(overrides.newId !== undefined ? { newId: overrides.newId } : {}), - ...(overrides.now !== undefined ? { now: overrides.now } : {}), - }) - } - - const sql = container.getDb().sql - const repo: ReportRepository = makeDrizzleReportRepository(sql) - const cleanupRepo = makeDrizzleCleanupRepository(sql) - const discussionRepo = makeDrizzleDiscussionRepository(sql) - const chatRepo = makeDrizzleChatRepository(sql) - const reportChatRepo = makeReportChatRepository(sql) - return makeReportService({ - repo, - loadLinkedEventsForReports: (reportIds) => cleanupRepo.loadLinkedEventsForReports(reportIds), - loadDiscussionMeta: async (reportId) => { - const [count, report] = await Promise.all([ - chatRepo.countReportMessages(reportId), - discussionRepo.findReportForDiscussion(reportId), - ]) - const jurisdiction = report?.jurisdiction ?? null - return { - discussionCount: count, - cityHandle: - jurisdiction !== null - ? effectiveJurisdictionHandle({ - handle: jurisdiction.handle, - name: jurisdiction.name, - }) - : null, - cityName: jurisdiction?.name ?? null, - canForwardToCity: - container.env.REPORT_AUTOFORWARD_ENABLED && - jurisdiction !== null && - jurisdiction.contactEmail !== null && - jurisdiction.contactEmail !== "", - } - }, - loadReportChatMeta: async (reportId, viewerUserId) => { - const rows = await sql< - { joined: boolean; member_count: number; message_count: number; unread: number }[] - >` - SELECT - EXISTS ( - SELECT 1 FROM report_chat_members m - WHERE m.report_id = ${reportId} AND m.user_id = ${viewerUserId} - ) AS joined, - ( - SELECT count(*)::int FROM report_chat_members m WHERE m.report_id = ${reportId} - ) AS member_count, - ( - SELECT count(*)::int - FROM chat_messages cm - WHERE cm.report_id = ${reportId} AND cm.deleted_at IS NULL - ) AS message_count, - COALESCE(( - SELECT count(*)::int - FROM report_chat_members mem - JOIN chat_messages cm ON cm.report_id = mem.report_id - WHERE mem.report_id = ${reportId} - AND mem.user_id = ${viewerUserId} - AND cm.deleted_at IS NULL - AND cm.sender_id IS DISTINCT FROM ${viewerUserId} - AND cm.created_at > GREATEST(mem.joined_at, COALESCE(mem.last_read_at, to_timestamp(0))) - ), 0) AS unread - ` - const row = rows[0] - return { - joined: row?.joined ?? false, - memberCount: row?.member_count ?? 0, - messageCount: row?.message_count ?? 0, - unread: row?.unread ?? 0, - } - }, - resolveJurisdictionGeoid: makeGeoidResolver(container), - resolveJurisdictionCode: (geoid) => resolveJurisdictionCode(sql, geoid), - resolveAddress: makeCachedAddressResolver(container), - presignMedia: makeMediaPresigner(container.storage), - presignPrivateMedia: makePrivateMediaPresigner(container.storage), - jobs: container.jobs, - autoForwardEnabled: container.env.REPORT_AUTOFORWARD_ENABLED, - isReportVerified: (userId) => isReportVerified(sql, userId), - joinReportChatAsOwner: (reportId, userId) => reportChatRepo.join(reportId, userId, "owner"), - reportChatEmitter: makeContainerReportChatEmitter(container, app.log), - logger: app.log, - }) + if (overrides) return makeOverriddenReportService(overrides, container) + return makeContainerReportService(container, app.log) } route( @@ -335,7 +229,7 @@ export async function registerReportRoutes( validated.types ?? null, validated.zoom, ) - reply.header("Cache-Control", "public, max-age=60") + reply.header("Cache-Control", MAP_REPORTS_CACHE_CONTROL) reply.status(200).send(payload) }, ) @@ -368,6 +262,139 @@ export async function registerReportRoutes( }) } +function makeOverriddenReportService( + overrides: ReportServiceOverrides, + container: Container, +): ReportService { + return makeReportService({ + repo: overrides.repo, + resolveJurisdictionGeoid: overrides.resolveJurisdictionGeoid ?? (() => Promise.resolve(null)), + ...(overrides.resolveJurisdictionCode !== undefined + ? { resolveJurisdictionCode: overrides.resolveJurisdictionCode } + : {}), + presignMedia: overrides.presignMedia ?? makeMediaPresigner(container.storage), + presignPrivateMedia: + overrides.presignPrivateMedia ?? makePrivateMediaPresigner(container.storage), + ...(overrides.resolveAddress !== undefined ? { resolveAddress: overrides.resolveAddress } : {}), + ...(overrides.loadLinkedEventsForReports !== undefined + ? { loadLinkedEventsForReports: overrides.loadLinkedEventsForReports } + : {}), + ...(overrides.loadDiscussionMeta !== undefined + ? { loadDiscussionMeta: overrides.loadDiscussionMeta } + : {}), + ...(overrides.loadReportChatMeta !== undefined + ? { loadReportChatMeta: overrides.loadReportChatMeta } + : {}), + ...(overrides.joinReportChatAsOwner !== undefined + ? { joinReportChatAsOwner: overrides.joinReportChatAsOwner } + : {}), + ...(overrides.reportChatEmitter !== undefined + ? { reportChatEmitter: overrides.reportChatEmitter } + : {}), + ...(overrides.newId !== undefined ? { newId: overrides.newId } : {}), + ...(overrides.now !== undefined ? { now: overrides.now } : {}), + }) +} + +function makeContainerReportService( + container: Container, + log: FastifyInstance["log"], +): ReportService { + const sql = container.getDb().sql + const repo: ReportRepository = makeDrizzleReportRepository(sql) + const cleanupRepo = makeDrizzleCleanupRepository(sql) + const reportChatRepo = makeReportChatRepository(sql) + return makeReportService({ + repo, + loadLinkedEventsForReports: (reportIds) => cleanupRepo.loadLinkedEventsForReports(reportIds), + loadDiscussionMeta: makeDiscussionMetaLoader(container, sql), + loadReportChatMeta: (reportId, viewerUserId) => loadReportChatMeta(sql, reportId, viewerUserId), + resolveJurisdictionGeoid: makeGeoidResolver(container), + resolveJurisdictionCode: (geoid) => resolveJurisdictionCode(sql, geoid), + resolveAddress: makeCachedAddressResolver(container), + presignMedia: makeMediaPresigner(container.storage), + presignPrivateMedia: makePrivateMediaPresigner(container.storage), + jobs: container.jobs, + autoForwardEnabled: container.env.REPORT_AUTOFORWARD_ENABLED, + isReportVerified: (userId) => isReportVerified(sql, userId), + joinReportChatAsOwner: (reportId, userId) => reportChatRepo.join(reportId, userId, "owner"), + reportChatEmitter: makeContainerReportChatEmitter(container, log), + logger: log, + }) +} + +function makeDiscussionMetaLoader( + container: Container, + sql: Sql, +): (reportId: string) => Promise { + const discussionRepo = makeDrizzleDiscussionRepository(sql) + const chatRepo = makeDrizzleChatRepository(sql) + return async (reportId) => { + const [count, report] = await Promise.all([ + chatRepo.countReportMessages(reportId), + discussionRepo.findReportForDiscussion(reportId), + ]) + const jurisdiction = report?.jurisdiction ?? null + return { + discussionCount: count, + cityHandle: + jurisdiction !== null + ? effectiveJurisdictionHandle({ + handle: jurisdiction.handle, + name: jurisdiction.name, + }) + : null, + cityName: jurisdiction?.name ?? null, + canForwardToCity: + container.env.REPORT_AUTOFORWARD_ENABLED && + jurisdiction !== null && + jurisdiction.contactEmail !== null && + jurisdiction.contactEmail !== "", + } + } +} + +async function loadReportChatMeta( + sql: Sql, + reportId: string, + viewerUserId: string | null, +): Promise { + const rows = await sql< + { joined: boolean; member_count: number; message_count: number; unread: number }[] + >` + SELECT + EXISTS ( + SELECT 1 FROM report_chat_members m + WHERE m.report_id = ${reportId} AND m.user_id = ${viewerUserId} + ) AS joined, + ( + SELECT count(*)::int FROM report_chat_members m WHERE m.report_id = ${reportId} + ) AS member_count, + ( + SELECT count(*)::int + FROM chat_messages cm + WHERE cm.report_id = ${reportId} AND cm.deleted_at IS NULL + ) AS message_count, + COALESCE(( + SELECT count(*)::int + FROM report_chat_members mem + JOIN chat_messages cm ON cm.report_id = mem.report_id + WHERE mem.report_id = ${reportId} + AND mem.user_id = ${viewerUserId} + AND cm.deleted_at IS NULL + AND cm.sender_id IS DISTINCT FROM ${viewerUserId} + AND cm.created_at > GREATEST(mem.joined_at, COALESCE(mem.last_read_at, to_timestamp(0))) + ), 0) AS unread + ` + const row = rows[0] + return { + joined: row?.joined ?? false, + memberCount: row?.member_count ?? 0, + messageCount: row?.message_count ?? 0, + unread: row?.unread ?? 0, + } +} + async function isReportVerified(sql: Sql, userId: string): Promise { const rows = await sql<{ report_verified: boolean }[]>` SELECT report_verified FROM user_moderation WHERE user_id = ${userId} LIMIT 1 @@ -375,13 +402,6 @@ async function isReportVerified(sql: Sql, userId: string): Promise { return rows[0]?.report_verified ?? false } -function ownerOf(request: FastifyRequest): { - userId?: string | undefined - anonSessionId?: string | undefined -} { - const auth = request.auth - return { - userId: auth?.userId ?? undefined, - anonSessionId: auth?.anonSessionId ?? undefined, - } +function ownerOf(request: FastifyRequest): ReportOwner { + return { userId: request.auth?.userId ?? undefined } } diff --git a/services/api/src/routes/service-hours-certificates.routes.ts b/services/api/src/routes/service-hours-certificates.routes.ts index 5aded156..2467996e 100644 --- a/services/api/src/routes/service-hours-certificates.routes.ts +++ b/services/api/src/routes/service-hours-certificates.routes.ts @@ -49,8 +49,18 @@ declare module "fastify" { } } +const CODE_PARAM_MAX_CHARS = 32 + /** Validates the raw path param before the shared schema's normalizing transform runs. */ -const CodeParamsSchema = z.object({ code: z.string().min(1).max(32) }).strict() +const CodeParamsSchema = z.object({ code: z.string().min(1).max(CODE_PARAM_MAX_CHARS) }).strict() + +/** + * An overrides object with no ledger twin means "no hours": every issue then 409s, which is exactly + * what an offline harness wants, and it is never a silent reach for the database. + */ +const EMPTY_LEDGER: Pick = { + entriesForCertificate: () => Promise.resolve({ items: [], totalHours: 0, entryCount: 0 }), +} /** * ISSUE renders a PDF synchronously and mints a durable public artifact from personal data; 6/hour caps @@ -78,16 +88,7 @@ export async function registerServiceHoursCertificateRoutes( const overrides = app.certificateOverrides return makeCertificateService({ repo: overrides ? overrides.repo : container.getCertificateRepo(), - hours: - overrides?.hours ?? - (overrides - ? // An overrides object with no ledger twin means "no hours": every issue then 409s, which is - // exactly what an offline harness wants, and it is never a silent reach for the database. - { - entriesForCertificate: () => - Promise.resolve({ items: [], totalHours: 0, entryCount: 0 }), - } - : container.getVolunteerHoursRepo()), + hours: overrides ? (overrides.hours ?? EMPTY_LEDGER) : container.getVolunteerHoursRepo(), storage: overrides?.storage ?? container.storage, verifyBaseUrl: `${webBaseUrlOf(container.env)}${CERTIFICATE_VERIFY_PATH}`, logger: app.log, diff --git a/services/api/src/routes/volunteer-hours.routes.ts b/services/api/src/routes/volunteer-hours.routes.ts index 6c67fe7a..31ce3c1d 100644 --- a/services/api/src/routes/volunteer-hours.routes.ts +++ b/services/api/src/routes/volunteer-hours.routes.ts @@ -49,8 +49,14 @@ declare module "fastify" { } } +const GEOID_MAX_CHARS = 64 + +const ANONYMOUS_CACHE_CONTROL = "public, max-age=60" + +const VIEWER_CACHE_CONTROL = "private, max-age=0, no-store" + const CleanupIdParamsSchema = z.object({ id: IdSchema }).strict() -const GeoidParamsSchema = z.object({ geoid: z.string().min(1).max(64) }).strict() +const GeoidParamsSchema = z.object({ geoid: z.string().min(1).max(GEOID_MAX_CHARS) }).strict() const UserIdParamsSchema = z.object({ id: IdSchema }).strict() const LEADERBOARD_RATE_LIMIT = { max: 60, timeWindow: "1 minute" } as const @@ -76,6 +82,12 @@ function appendVary(reply: FastifyReply, ...fields: readonly string[]): void { reply.header("Vary", current.join(", ")) } +// The body differs per viewer (blocks, own rank), so a shared cache may only hold the anonymous copy. +function setViewerCacheHeaders(reply: FastifyReply, viewerId: string | null): void { + appendVary(reply, "Cookie", "Authorization") + reply.header("Cache-Control", viewerId === null ? ANONYMOUS_CACHE_CONTROL : VIEWER_CACHE_CONTROL) +} + export async function registerVolunteerHoursRoutes( app: FastifyInstance, container: Container, @@ -198,11 +210,7 @@ export async function registerVolunteerHoursRoutes( const viewerId = request.auth?.userId ?? null const payload: PublicVolunteerHoursResponse = await service().getPublicHours(query, viewerId) - appendVary(reply, "Cookie", "Authorization") - reply.header( - "Cache-Control", - viewerId === null ? "public, max-age=60" : "private, max-age=0, no-store", - ) + setViewerCacheHeaders(reply, viewerId) reply.status(200).send(payload) }, ) @@ -248,11 +256,7 @@ export async function registerVolunteerHoursRoutes( const viewerId = request.auth?.userId ?? null const payload: LeaderboardResponse = await service().leaderboard(geoid, query, viewerId) - appendVary(reply, "Cookie", "Authorization") - reply.header( - "Cache-Control", - viewerId === null ? "public, max-age=60" : "private, max-age=0, no-store", - ) + setViewerCacheHeaders(reply, viewerId) reply.status(200).send(payload) }, ) diff --git a/services/api/src/services/address-resolver.ts b/services/api/src/services/address-resolver.ts index c2a52241..79724446 100644 --- a/services/api/src/services/address-resolver.ts +++ b/services/api/src/services/address-resolver.ts @@ -64,6 +64,17 @@ async function orNull(p: Promise): Promise { } } +function topRung( + hit: { line: string; precision: AddressPrecision } | null, + cityStateLabel: string, +): ResolvedAddress { + if (hit !== null) return { address: hit.line, precision: hit.precision, cityStateLabel } + if (cityStateLabel.length > 0) { + return { address: cityStateLabel, precision: "locality", cityStateLabel } + } + return { address: null, precision: null, cityStateLabel } +} + export function makeAddressResolver(deps: AddressResolverDeps): AddressResolver { const cache = deps.cache ?? NO_GEOCODE_CACHE @@ -87,12 +98,7 @@ export function makeAddressResolver(deps: AddressResolverDeps): AddressResolver ]) const cityStateLabel = label ?? "" - const resolved: ResolvedAddress = - hit !== null - ? { address: hit.line, precision: hit.precision, cityStateLabel } - : cityStateLabel.length > 0 - ? { address: cityStateLabel, precision: "locality", cityStateLabel } - : { address: null, precision: null, cityStateLabel } + const resolved = topRung(hit, cityStateLabel) if (hit !== null || !chainAlreadyMissed) { await orNull( diff --git a/services/api/src/services/anon-hold-release-repo.drizzle.ts b/services/api/src/services/anon-hold-release-repo.drizzle.ts index 2cf5c056..f5efc9c1 100644 --- a/services/api/src/services/anon-hold-release-repo.drizzle.ts +++ b/services/api/src/services/anon-hold-release-repo.drizzle.ts @@ -1,6 +1,29 @@ -import type { Sql } from "../db/client.js" +import type { Queryable, Sql } from "../db/client.js" import type { AnonHoldReleaseRepo, HeldReportView, ReleaseMediaView } from "./anon-hold-release.js" +const RELEASE_TIMELINE_NOTE = "Released after automated review" + +async function countOpenFlags( + tag: Queryable, + reportId: string, + mediaIds: string[], +): Promise { + const mediaClause = + mediaIds.length > 0 + ? tag`OR (subject_type = 'media' AND subject_id IN ${tag(mediaIds)})` + : tag`` + const rows = await tag<{ n: number }[]>` + SELECT COUNT(*)::int AS n + FROM abuse_flags + WHERE resolved_at IS NULL + AND ( + (subject_type = 'report' AND subject_id = ${reportId}) + ${mediaClause} + ) + ` + return rows[0]?.n ?? 0 +} + export function makeDrizzleAnonHoldReleaseRepo(sql: Sql): AnonHoldReleaseRepo { return { async findReport(reportId: string): Promise { @@ -43,21 +66,8 @@ export function makeDrizzleAnonHoldReleaseRepo(sql: Sql): AnonHoldReleaseRepo { return rows.map((m) => ({ id: m.id, status: m.status })) }, - async countOpenAbuseFlags(reportId: string, mediaIds: string[]): Promise { - const mediaClause = - mediaIds.length > 0 - ? sql`OR (subject_type = 'media' AND subject_id IN ${sql(mediaIds)})` - : sql`` - const rows = await sql<{ n: number }[]>` - SELECT COUNT(*)::int AS n - FROM abuse_flags - WHERE resolved_at IS NULL - AND ( - (subject_type = 'report' AND subject_id = ${reportId}) - ${mediaClause} - ) - ` - return rows[0]?.n ?? 0 + countOpenAbuseFlags(reportId: string, mediaIds: string[]): Promise { + return countOpenFlags(sql, reportId, mediaIds) }, async publishHeldReport( @@ -79,20 +89,7 @@ export function makeDrizzleAnonHoldReleaseRepo(sql: Sql): AnonHoldReleaseRepo { ` if ((notReady[0]?.n ?? 0) > 0) return false - const mediaClause = - mediaIds.length > 0 - ? tx`OR (subject_type = 'media' AND subject_id IN ${tx(mediaIds)})` - : tx`` - const openFlags = await tx<{ n: number }[]>` - SELECT COUNT(*)::int AS n - FROM abuse_flags - WHERE resolved_at IS NULL - AND ( - (subject_type = 'report' AND subject_id = ${reportId}) - ${mediaClause} - ) - ` - if ((openFlags[0]?.n ?? 0) > 0) return false + if ((await countOpenFlags(tx, reportId, mediaIds)) > 0) return false await tx` UPDATE reports @@ -101,7 +98,7 @@ export function makeDrizzleAnonHoldReleaseRepo(sql: Sql): AnonHoldReleaseRepo { ` await tx` INSERT INTO report_timeline (report_id, status, note, actor_id) - VALUES (${reportId}, ${"published"}, ${"Released after automated review"}, ${null}) + VALUES (${reportId}, ${"published"}, ${RELEASE_TIMELINE_NOTE}, ${null}) ` await tx` UPDATE moderation_items diff --git a/services/api/src/services/avatar-media.ts b/services/api/src/services/avatar-media.ts index 7fbf0f92..f3c3ae86 100644 --- a/services/api/src/services/avatar-media.ts +++ b/services/api/src/services/avatar-media.ts @@ -20,7 +20,8 @@ export interface AvatarMediaRow extends Record { served_key: string | null } -export const AVATAR_CLAIM_WINDOW_SECONDS = UNBOUND_GRACE_MS / 1000 +const MS_PER_SECOND = 1000 +const AVATAR_CLAIM_WINDOW_SECONDS = UNBOUND_GRACE_MS / MS_PER_SECOND type SqlTemplateTag = (strings: TemplateStringsArray, ...values: (string | number | null)[]) => Q diff --git a/services/api/src/services/certificate-fonts.ts b/services/api/src/services/certificate-fonts.ts index 0d2bd0fa..a2ffff6d 100644 --- a/services/api/src/services/certificate-fonts.ts +++ b/services/api/src/services/certificate-fonts.ts @@ -95,8 +95,6 @@ export const FONT = { cjk: "NotoSansKR-Regular.otf", } as const -export type FontRole = keyof typeof FONT - /** * Codepoint ranges the Latin brand faces cannot cover, i.e. roughly what Noto Sans KR provides. * @@ -117,7 +115,7 @@ const CJK_RANGES: readonly (readonly [number, number])[] = [ [0xff00, 0xffef], // Halfwidth and fullwidth forms ] -export function needsCjk(text: string): boolean { +function needsCjk(text: string): boolean { for (const ch of text) { const cp = ch.codePointAt(0) ?? 0 for (const [lo, hi] of CJK_RANGES) { diff --git a/services/api/src/services/certificate-model.ts b/services/api/src/services/certificate-model.ts index 1e038d37..f7e90526 100644 --- a/services/api/src/services/certificate-model.ts +++ b/services/api/src/services/certificate-model.ts @@ -104,6 +104,8 @@ export interface BuildTranscriptModelInput { // An en dash, not a hyphen: it reads as "no value" in print and matches the period range glyph. export const EMPTY_VALUE = "–" +const COMMUNITIES_NAMED_ON_TILE = 2 + function toIso(value: Date | string): string { return typeof value === "string" ? new Date(value).toISOString() : value.toISOString() } @@ -196,8 +198,8 @@ export function communitiesLabel( jurisdictions: readonly string[], t: CertificateTranslator, ): string { - const shown = jurisdictions.slice(0, 2).join(", ") - const extra = jurisdictions.length - 2 + const shown = jurisdictions.slice(0, COMMUNITIES_NAMED_ON_TILE).join(", ") + const extra = jurisdictions.length - COMMUNITIES_NAMED_ON_TILE if (extra <= 0) return shown return `${shown} ${t("certificate.summary.more", { count: extra })}`.trim() } diff --git a/services/api/src/services/certificate-pdf.ts b/services/api/src/services/certificate-pdf.ts index 159eece0..c503b232 100644 --- a/services/api/src/services/certificate-pdf.ts +++ b/services/api/src/services/certificate-pdf.ts @@ -17,7 +17,7 @@ import { export const CERTIFICATE_VERIFY_PATH = "/service-record" -export const CERTIFICATE_VERIFY_BASE_URL = `https://civfix.org${CERTIFICATE_VERIFY_PATH}` +const CERTIFICATE_VERIFY_BASE_URL = `https://civfix.org${CERTIFICATE_VERIFY_PATH}` export interface ServiceHoursPdfInput { model: TranscriptModel @@ -28,6 +28,20 @@ export interface ServiceHoursPdfInput { t?: CertificateTranslator } +const BRAND = "civfix" +const SEAL_WORDMARK = "CIVFIX" +const DOC_MARGINS = { top: 48, bottom: 54, left: 54, right: 54 } as const +const PDF_VERSION = "1.7" + +const URL_SCHEME = /^https?:\/\// +const ISO_MILLISECONDS = /\.\d{3}Z$/ +const FINGERPRINT_PRINTED_CHARS = 16 + +const QR_TYPE_NUMBER_AUTO = 0 +const QR_ERROR_CORRECTION = "M" +/** ISO/IEC 18004 asks for a four-module light margin around the symbol. */ +const QR_QUIET_ZONE_MODULES = 4 + const PAGE = { left: 54, right: 558, @@ -57,17 +71,70 @@ const COL = { creditedBy: { x: 460, w: 98 }, } as const +const HAIRLINE_WIDTH = 0.75 +const RULE_WIDTH = 1 +const CARD_RADIUS = 10 + +const ACCENT_BAR = { y: 36, height: 5 } as const +const LETTERHEAD_RULE_Y = 112 +const RUNNING_HEADER_RULE_Y = 70 +const HOLDER_CARD = { + top: 126, + height: 88, + inset: 16, + width: 290, + asideX: 366, + asideWidth: 176, +} as const +const SUMMARY_TILE = { top: 228, width: 160, gap: 12, height: 78 } as const +const COLUMN_BAND = { firstPageY: 322, continuationY: 78, height: 22 } as const +const CELL_INSET = 6 + const ROW_MIN_HEIGHT = 20 const ROW_PADDING = 8 +const ACTIVITY_MAX_LINES = 2 + +const TOTALS_ROW = { ruleAbove: 4, textOffset: 10, ruleBelow: 26, advance: 30 } as const +const TRUNCATION_BANNER_HEIGHT = 22 +const ISSUER_GAP = 24 +const ATTESTATION_WIDTH = 460 +const SEAL = { radius: 34, innerRadius: 29 } as const +const QR_BOX = { x: 474, size: 84 } as const +const VERIFY_TEXT = { x: 306, width: 156 } as const +const FOOTER_COLUMN_WIDTH = 200 type Doc = PDFKit.PDFDocument +type QrCodeFactory = typeof import("qrcode-generator") +type TranscriptRow = TranscriptModel["rows"][number] +interface Column { + x: number + w: number +} -function useFont(doc: Doc, registered: Set, file: string, size: number): Doc { - if (!registered.has(file)) { - doc.registerFont(file, fontBuffer(file)) - registered.add(file) +interface PdfContext { + doc: Doc + registeredFonts: Set + qrcode: QrCodeFactory + t: CertificateTranslator + locale: string + model: TranscriptModel + holderName: string + displayCode: string + issuedAt: Date + issuedLabel: string + verifyUrl: string + verifyLabel: string + fingerprint: string | null | undefined + /** Read by the page-added hook: once the table has ended, a new page gets no column band. */ + tableContinues: boolean +} + +function useFont(ctx: PdfContext, file: string, size: number): Doc { + if (!ctx.registeredFonts.has(file)) { + ctx.doc.registerFont(file, fontBuffer(file)) + ctx.registeredFonts.add(file) } - return doc.font(file).fontSize(size) + return ctx.doc.font(file).fontSize(size) } function displayFont(text: string): string { @@ -78,496 +145,556 @@ function toDate(value: Date | string): Date { return typeof value === "string" ? new Date(value) : value } -export async function buildServiceHoursPdf(input: ServiceHoursPdfInput): Promise { - const PDFDocument = (await import("pdfkit")).default - const qrcode = (await import("qrcode-generator")).default +function line( + ctx: PdfContext, + file: string, + size: number, + color: string, + text: string, + x: number, + y: number, + extra: PDFKit.Mixins.TextOptions = {}, +): void { + useFont(ctx, file, size).fillColor(color) + ctx.doc.text(text, x, y, { + lineBreak: false, + ellipsis: true, + height: ctx.doc.currentLineHeight() + 0.5, + ...extra, + }) +} - const { model } = input - const t = input.t ?? certificateTranslator(model.locale) - const locale = model.locale - const displayCode = formatCertificateCode(input.code) - const issuedAt = toDate(input.issuedAt) - const verifyBaseUrl = input.verifyBaseUrl ?? CERTIFICATE_VERIFY_BASE_URL - const verifyUrl = `${verifyBaseUrl}/${displayCode}` - const verifyLabel = verifyBaseUrl.replace(/^https?:\/\//, "") - const holderName = model.holder.displayName - const issuedLabel = formatDate(issuedAt, locale) +function horizontalRule(ctx: PdfContext, y: number, width: number, color: string): void { + ctx.doc.moveTo(PAGE.left, y).lineTo(PAGE.right, y).lineWidth(width).strokeColor(color).stroke() +} - const doc = new PDFDocument({ - size: "LETTER", - margins: { top: 48, bottom: 54, left: 54, right: 54 }, - bufferPages: true, - autoFirstPage: false, - pdfVersion: "1.7", - lang: locale, - displayTitle: true, - info: { - Title: t("certificate.doc.pdf_title", { name: holderName, code: displayCode }), - Author: "civfix", - Subject: t("certificate.doc.title"), - Keywords: displayCode, - Creator: "civfix", - Producer: "civfix", - CreationDate: issuedAt, - ModDate: issuedAt, - }, +function hairline(ctx: PdfContext, y: number): void { + horizontalRule(ctx, y, HAIRLINE_WIDTH, COLOR.hairline) +} + +function drawFirstPageChrome(ctx: PdfContext): void { + const { doc, t } = ctx + doc.rect(PAGE.left, ACCENT_BAR.y, PAGE.contentWidth, ACCENT_BAR.height).fill(COLOR.accent) + + line(ctx, FONT.wordmark, 26, COLOR.ink, BRAND, PAGE.left, 54) + line(ctx, FONT.display, 8, COLOR.ink3, t("certificate.doc.title").toUpperCase(), PAGE.left, 86, { + width: 260, + characterSpacing: 0.9, }) - const chunks: Buffer[] = [] - doc.on("data", (chunk: Buffer) => chunks.push(chunk)) - const finished = new Promise((resolve, reject) => { - doc.on("end", () => resolve()) - doc.on("error", (err: Error) => reject(err)) + line(ctx, FONT.display, 7, COLOR.ink3, t("certificate.header.number").toUpperCase(), 330, 56, { + width: 228, + align: "right", + characterSpacing: 0.6, }) + line(ctx, FONT.mono, 12, COLOR.ink, ctx.displayCode, 330, 68, { width: 228, align: "right" }) - const registered = new Set() - const font = (file: string, size: number) => useFont(doc, registered, file, size) - - function line( - file: string, - size: number, - color: string, - text: string, - x: number, - y: number, - extra: PDFKit.Mixins.TextOptions = {}, - ): void { - font(file, size).fillColor(color) - doc.text(text, x, y, { - lineBreak: false, - ellipsis: true, - height: doc.currentLineHeight() + 0.5, - ...extra, - }) - } + hairline(ctx, LETTERHEAD_RULE_Y) + drawHolderCard(ctx) + drawSummaryTiles(ctx) + drawColumnBand(ctx, COLUMN_BAND.firstPageY) +} - let pageNumber = 0 - let tableContinues = true - doc.on("pageAdded", () => { - pageNumber += 1 - if (pageNumber === 1) drawFirstPageChrome() - else drawContinuationChrome() +function drawContinuationChrome(ctx: PdfContext): void { + line(ctx, FONT.wordmark, 11, COLOR.ink, BRAND, PAGE.left, 40) + const trail = `${ctx.t("certificate.doc.title")} · ${ctx.holderName} · ${ctx.displayCode}` + line(ctx, fontFor(trail, "regular"), 8, COLOR.ink3, trail, 150, 42, { + width: 408, + align: "right", }) + hairline(ctx, RUNNING_HEADER_RULE_Y) + if (ctx.tableContinues) drawColumnBand(ctx, COLUMN_BAND.continuationY) +} - function drawFirstPageChrome(): void { - doc.rect(PAGE.left, 36, PAGE.contentWidth, 5).fill(COLOR.accent) - - line(FONT.wordmark, 26, COLOR.ink, "civfix", PAGE.left, 54) - line(FONT.display, 8, COLOR.ink3, t("certificate.doc.title").toUpperCase(), PAGE.left, 86, { - width: 260, - characterSpacing: 0.9, - }) +function holderPeriodLabel(ctx: PdfContext): string { + const { periodStart, periodEnd } = ctx.model + if (!periodStart || !periodEnd) return EMPTY_VALUE + return `${formatDate(new Date(periodStart), ctx.locale)} – ${formatDate(new Date(periodEnd), ctx.locale)}` +} - line(FONT.display, 7, COLOR.ink3, t("certificate.header.number").toUpperCase(), 330, 56, { - width: 228, - align: "right", - characterSpacing: 0.6, - }) - line(FONT.mono, 12, COLOR.ink, displayCode, 330, 68, { width: 228, align: "right" }) +function drawHolderCard(ctx: PdfContext): void { + const { doc, t, model, holderName } = ctx + const { top, width } = HOLDER_CARD + doc + .roundedRect(PAGE.left, top, PAGE.contentWidth, HOLDER_CARD.height, CARD_RADIUS) + .lineWidth(HAIRLINE_WIDTH) + .fillAndStroke(COLOR.card, COLOR.hairline) - hairline(112) - drawHolderCard() - drawSummaryTiles() - drawColumnBand(322) - } + const x = PAGE.left + HOLDER_CARD.inset + line( + ctx, + FONT.display, + 8, + COLOR.ink3, + t("certificate.holder.eyebrow").toUpperCase(), + x, + top + 14, + { + width, + characterSpacing: 0.8, + }, + ) + line(ctx, displayFont(holderName), 20, COLOR.ink, holderName, x, top + 28, { width }) - function drawContinuationChrome(): void { - line(FONT.wordmark, 11, COLOR.ink, "civfix", PAGE.left, 40) - const trail = `${t("certificate.doc.title")} · ${holderName} · ${displayCode}` - line(fontFor(trail, "regular"), 8, COLOR.ink3, trail, 150, 42, { width: 408, align: "right" }) - hairline(70) - if (tableContinues) drawColumnBand(78) + const handle = model.holder.handle + if (handle) { + line(ctx, fontFor(handle, "regular"), 10, COLOR.ink2, `@${handle}`, x, top + 56, { width }) } + const { asideX, asideWidth } = HOLDER_CARD + labelledValue( + ctx, + asideX, + top + 14, + asideWidth, + t("certificate.holder.period"), + holderPeriodLabel(ctx), + ) + labelledValue(ctx, asideX, top + 48, asideWidth, t("certificate.holder.issued"), ctx.issuedLabel) +} - function hairline(y: number): void { - doc - .moveTo(PAGE.left, y) - .lineTo(PAGE.right, y) - .lineWidth(0.75) - .strokeColor(COLOR.hairline) - .stroke() - } +function labelledValue( + ctx: PdfContext, + x: number, + y: number, + w: number, + label: string, + value: string, +): void { + line(ctx, FONT.display, 7, COLOR.ink3, label.toUpperCase(), x, y, { + width: w, + align: "right", + characterSpacing: 0.6, + }) + line(ctx, fontFor(value, "regular"), 9.5, COLOR.ink, value, x, y + 11, { + width: w, + align: "right", + }) +} - function drawHolderCard(): void { - const top = 126 - doc - .roundedRect(PAGE.left, top, PAGE.contentWidth, 88, 10) - .lineWidth(0.75) - .fillAndStroke(COLOR.card, COLOR.hairline) +interface SummaryTile { + label: string + value: string + note?: string +} - const x = PAGE.left + 16 - line(FONT.display, 8, COLOR.ink3, t("certificate.holder.eyebrow").toUpperCase(), x, top + 14, { - width: 290, - characterSpacing: 0.8, - }) - line(displayFont(holderName), 20, COLOR.ink, holderName, x, top + 28, { width: 290 }) - - let y = top + 56 - if (model.holder.handle) { - line( - fontFor(model.holder.handle, "regular"), - 10, - COLOR.ink2, - `@${model.holder.handle}`, - x, - y, - { - width: 290, - }, - ) - y += 14 - } - const rx = 366 - const rw = 176 - const period = - model.periodStart && model.periodEnd - ? `${formatDate(new Date(model.periodStart), locale)} – ${formatDate(new Date(model.periodEnd), locale)}` - : EMPTY_VALUE - labelledValue(rx, top + 14, rw, t("certificate.holder.period"), period) - labelledValue(rx, top + 48, rw, t("certificate.holder.issued"), issuedLabel) - } +function summaryTiles(ctx: PdfContext): SummaryTile[] { + const { t, model, locale } = ctx + const communities = model.jurisdictions + return [ + { label: t("certificate.summary.total_hours"), value: formatNumber(model.totalHours, locale) }, + { label: t("certificate.summary.activities"), value: formatNumber(model.entryCount, locale) }, + { + label: t("certificate.summary.communities"), + value: formatNumber(communities.length, locale), + note: communitiesLabel(communities, t), + }, + ] +} - function labelledValue(x: number, y: number, w: number, label: string, value: string): void { - line(FONT.display, 7, COLOR.ink3, label.toUpperCase(), x, y, { - width: w, - align: "right", - characterSpacing: 0.6, +function drawSummaryTiles(ctx: PdfContext): void { + const { doc } = ctx + const { top, width: w, gap } = SUMMARY_TILE + summaryTiles(ctx).forEach((tile, index) => { + const x = PAGE.left + index * (w + gap) + doc.roundedRect(x, top, w, SUMMARY_TILE.height, CARD_RADIUS).fill(COLOR.tile) + if (index === 0) doc.rect(x, top + 10, 3, 58).fill(COLOR.accent) + line(ctx, FONT.display, 7.5, COLOR.ink3, tile.label.toUpperCase(), x + 14, top + 14, { + width: w - 24, + characterSpacing: 0.7, }) - line(fontFor(value, "regular"), 9.5, COLOR.ink, value, x, y + 11, { width: w, align: "right" }) - } - - function drawSummaryTiles(): void { - const top = 228 - const w = 160 - const gap = 12 - const communities = model.jurisdictions - const tiles: { label: string; value: string; note?: string }[] = [ - { - label: t("certificate.summary.total_hours"), - value: formatNumber(model.totalHours, locale), - }, - { - label: t("certificate.summary.activities"), - value: formatNumber(model.entryCount, locale), - }, - { - label: t("certificate.summary.communities"), - value: formatNumber(communities.length, locale), - note: communitiesLabel(communities, t), - }, - ] - - tiles.forEach((tile, index) => { - const x = PAGE.left + index * (w + gap) - doc.roundedRect(x, top, w, 78, 10).fill(COLOR.tile) - if (index === 0) doc.rect(x, top + 10, 3, 58).fill(COLOR.accent) - line(FONT.display, 7.5, COLOR.ink3, tile.label.toUpperCase(), x + 14, top + 14, { + line(ctx, FONT.display, 30, COLOR.ink, tile.value, x + 14, top + 28, { width: w - 24 }) + if (tile.note) { + line(ctx, fontFor(tile.note, "regular"), 8, COLOR.ink3, tile.note, x + 14, top + 62, { width: w - 24, - characterSpacing: 0.7, }) - line(FONT.display, 30, COLOR.ink, tile.value, x + 14, top + 28, { width: w - 24 }) - if (tile.note) { - line(fontFor(tile.note, "regular"), 8, COLOR.ink3, tile.note, x + 14, top + 62, { - width: w - 24, - }) - } - }) - } - - function drawColumnBand(y: number): void { - doc.rect(PAGE.left, y, PAGE.contentWidth, 22).fill(COLOR.band) - const labels: [{ x: number; w: number }, string, "left" | "right"][] = [ - [COL.date, t("certificate.table.date"), "left"], - [COL.activity, t("certificate.table.activity"), "left"], - [COL.community, t("certificate.table.community"), "left"], - [COL.hours, t("certificate.table.hours"), "right"], - [COL.creditedBy, t("certificate.table.credited_by"), "left"], - ] - for (const [col, label, align] of labels) { - line( - FONT.display, - 7.5, - COLOR.ink2, - label.toUpperCase(), - col.x + (align === "left" ? 6 : 0), - y + 7, - { width: col.w - 6, align, characterSpacing: 0.6 }, - ) } - doc - .moveTo(PAGE.left, y + 22) - .lineTo(PAGE.right, y + 22) - .lineWidth(0.75) - .strokeColor(COLOR.rule) - .stroke() + }) +} + +function drawColumnBand(ctx: PdfContext, y: number): void { + const { doc, t } = ctx + doc.rect(PAGE.left, y, PAGE.contentWidth, COLUMN_BAND.height).fill(COLOR.band) + const labels: [Column, string, "left" | "right"][] = [ + [COL.date, t("certificate.table.date"), "left"], + [COL.activity, t("certificate.table.activity"), "left"], + [COL.community, t("certificate.table.community"), "left"], + [COL.hours, t("certificate.table.hours"), "right"], + [COL.creditedBy, t("certificate.table.credited_by"), "left"], + ] + for (const [col, label, align] of labels) { + line( + ctx, + FONT.display, + 7.5, + COLOR.ink2, + label.toUpperCase(), + col.x + (align === "left" ? CELL_INSET : 0), + y + 7, + { width: col.w - CELL_INSET, align, characterSpacing: 0.6 }, + ) } + horizontalRule(ctx, y + COLUMN_BAND.height, HAIRLINE_WIDTH, COLOR.rule) +} - const rowHeights = model.rows.map((row) => { - font(fontFor(row.activity, "regular"), 9.5) - const twoLines = doc.currentLineHeight() * 2 +function measureRowHeights(ctx: PdfContext): number[] { + return ctx.model.rows.map((row) => { + useFont(ctx, fontFor(row.activity, "regular"), 9.5) + const maxActivityHeight = ctx.doc.currentLineHeight() * ACTIVITY_MAX_LINES const measured = Math.min( - doc.heightOfString(row.activity, { width: COL.activity.w - 12 }), - twoLines, + ctx.doc.heightOfString(row.activity, { width: COL.activity.w - 2 * CELL_INSET }), + maxActivityHeight, ) return Math.max(ROW_MIN_HEIGHT, measured + ROW_PADDING) }) - const pages = planPages(rowHeights) +} - doc.addPage() +function drawRow(ctx: PdfContext, row: TranscriptRow, y: number, height: number): void { + const textY = y + CELL_INSET + line( + ctx, + fontFor(row.dateLabel, "regular"), + 9, + COLOR.ink2, + row.dateLabel, + COL.date.x + CELL_INSET, + textY, + { + width: COL.date.w - 8, + }, + ) + useFont(ctx, fontFor(row.activity, "regular"), 9.5).fillColor(COLOR.ink) + ctx.doc.text(row.activity, COL.activity.x + CELL_INSET, textY, { + width: COL.activity.w - 2 * CELL_INSET, + height: height - CELL_INSET, + ellipsis: true, + }) + line( + ctx, + fontFor(row.community, "regular"), + 9, + COLOR.ink2, + row.community, + COL.community.x + CELL_INSET, + textY, + { width: COL.community.w - 2 * CELL_INSET }, + ) + line(ctx, FONT.mono, 9.5, COLOR.ink, row.hours.toFixed(2), COL.hours.x, textY, { + width: COL.hours.w - 4, + align: "right", + }) + line( + ctx, + fontFor(row.creditedBy, "regular"), + 9, + COLOR.ink2, + row.creditedBy, + COL.creditedBy.x + CELL_INSET, + textY, + { width: COL.creditedBy.w - 8 }, + ) +} +function drawLedgerTable(ctx: PdfContext, rowHeights: readonly number[]): number { let cursorY = 0 - for (const plan of pages) { - if (plan.page > 1) doc.addPage() + for (const plan of planPages(rowHeights)) { + if (plan.page > 1) ctx.doc.addPage() let y = plan.top for (let i = plan.startIndex; i < plan.endIndex; i++) { - const row = model.rows[i] + const row = ctx.model.rows[i] const height = rowHeights[i] ?? ROW_MIN_HEIGHT if (!row) continue - if (i % 2 === 0) doc.rect(PAGE.left, y, PAGE.contentWidth, height).fill(COLOR.card) - drawRow(row, y, height) + if (i % 2 === 0) ctx.doc.rect(PAGE.left, y, PAGE.contentWidth, height).fill(COLOR.card) + drawRow(ctx, row, y, height) y += height } cursorY = y } + return cursorY +} - function drawRow(row: TranscriptModel["rows"][number], y: number, height: number): void { - const textY = y + 6 - line(fontFor(row.dateLabel, "regular"), 9, COLOR.ink2, row.dateLabel, COL.date.x + 6, textY, { - width: COL.date.w - 8, - }) - font(fontFor(row.activity, "regular"), 9.5).fillColor(COLOR.ink) - doc.text(row.activity, COL.activity.x + 6, textY, { - width: COL.activity.w - 12, - height: height - 6, - ellipsis: true, - }) - line( - fontFor(row.community, "regular"), - 9, - COLOR.ink2, - row.community, - COL.community.x + 6, - textY, - { - width: COL.community.w - 12, - }, - ) - line(FONT.mono, 9.5, COLOR.ink, row.hours.toFixed(2), COL.hours.x, textY, { - width: COL.hours.w - 4, - align: "right", - }) - line( - fontFor(row.creditedBy, "regular"), - 9, - COLOR.ink2, - row.creditedBy, - COL.creditedBy.x + 6, - textY, - { width: COL.creditedBy.w - 8 }, - ) - } +function startTrailingPage(ctx: PdfContext): number { + ctx.tableContinues = false + ctx.doc.addPage() + return DEFAULT_PAGE_PLAN_OPTIONS.continuationTop +} - if (!totalsFitsOnPage(cursorY)) { - tableContinues = false - doc.addPage() - cursorY = DEFAULT_PAGE_PLAN_OPTIONS.continuationTop - } - doc - .moveTo(PAGE.left, cursorY + 4) - .lineTo(PAGE.right, cursorY + 4) - .lineWidth(1) - .strokeColor(COLOR.rule) - .stroke() +function drawTotals(ctx: PdfContext, tableEndY: number): number { + const cursorY = totalsFitsOnPage(tableEndY) ? tableEndY : startTrailingPage(ctx) + horizontalRule(ctx, cursorY + TOTALS_ROW.ruleAbove, RULE_WIDTH, COLOR.rule) line( + ctx, FONT.bodyBold, 9.5, COLOR.ink, - t("certificate.table.total").toUpperCase(), - COL.activity.x + 6, - cursorY + 10, + ctx.t("certificate.table.total").toUpperCase(), + COL.activity.x + CELL_INSET, + cursorY + TOTALS_ROW.textOffset, { width: 240, characterSpacing: 0.5 }, ) - line(FONT.mono, 10, COLOR.ink, model.totalHours.toFixed(2), COL.hours.x, cursorY + 10, { - width: COL.hours.w - 4, - align: "right", + line( + ctx, + FONT.mono, + 10, + COLOR.ink, + ctx.model.totalHours.toFixed(2), + COL.hours.x, + cursorY + TOTALS_ROW.textOffset, + { width: COL.hours.w - 4, align: "right" }, + ) + horizontalRule(ctx, cursorY + TOTALS_ROW.ruleBelow, RULE_WIDTH, COLOR.rule) + return cursorY + TOTALS_ROW.advance +} + +function drawTruncationBanner(ctx: PdfContext, cursorY: number): number { + const { model, locale } = ctx + if (!model.truncated) return cursorY + const banner = ctx.t("certificate.table.truncated", { + shown: formatNumber(model.includedCount, locale), + total: formatNumber(model.entryCount, locale), }) - doc - .moveTo(PAGE.left, cursorY + 26) - .lineTo(PAGE.right, cursorY + 26) - .lineWidth(1) - .strokeColor(COLOR.rule) - .stroke() - cursorY += 30 - - if (model.truncated) { - const banner = t("certificate.table.truncated", { - shown: formatNumber(model.includedCount, locale), - total: formatNumber(model.entryCount, locale), + useFont(ctx, fontFor(banner, "regular"), 8.5) + .fillColor(COLOR.ink3) + .text(banner, PAGE.left, cursorY, { + width: PAGE.contentWidth, + height: TRUNCATION_BANNER_HEIGHT, }) - font(fontFor(banner, "regular"), 8.5) - .fillColor(COLOR.ink3) - .text(banner, PAGE.left, cursorY, { width: PAGE.contentWidth, height: 22 }) - cursorY += 22 - } + return cursorY + TRUNCATION_BANNER_HEIGHT +} - if (issuerNeedsNewPage(cursorY)) { - tableContinues = false - doc.addPage() - cursorY = DEFAULT_PAGE_PLAN_OPTIONS.continuationTop - } else { - cursorY += 24 - } - drawIssuerBlock(cursorY) - - function drawIssuerBlock(top: number): void { - const attestation = t("certificate.attestation.body") - font(fontFor(attestation, "regular"), 9.5).fillColor(COLOR.ink2) - const paragraphHeight = doc.heightOfString(attestation, { width: 460 }) - doc.text(attestation, PAGE.left, top, { width: 460 }) - - const blockTop = top + paragraphHeight + 22 - - const sealX = PAGE.left - const cx = sealX + 34 - const cy = blockTop + 34 - doc.circle(cx, cy, 34).lineWidth(1.5).strokeColor(COLOR.accent).stroke() - doc.circle(cx, cy, 29).lineWidth(1).strokeColor(COLOR.accent).stroke() - line(FONT.display, 9, COLOR.ink, "CIVFIX", sealX, cy - 16, { - width: 68, - align: "center", - characterSpacing: 1.2, - }) - line(FONT.display, 6, COLOR.ink2, t("certificate.seal.line").toUpperCase(), sealX, cy - 2, { - width: 64, - align: "center", - characterSpacing: 0.2, - }) - line(FONT.mono, 8, COLOR.ink3, formatYear(issuedAt), sealX, cy + 8, { - width: 68, - align: "center", - }) +function issuerBlockTop(ctx: PdfContext, cursorY: number): number { + return issuerNeedsNewPage(cursorY) ? startTrailingPage(ctx) : cursorY + ISSUER_GAP +} - const issuerLine = t("certificate.issuer.line") - line(fontFor(issuerLine, "bold"), 9, COLOR.ink, issuerLine, sealX, blockTop + 78, { - width: 260, - }) - line( - FONT.mono, - 7.5, - COLOR.ink3, - t("certificate.issuer.generated", { - timestamp: issuedAt.toISOString().replace(/\.\d{3}Z$/, "Z"), - }), - sealX, - blockTop + 90, - { width: 260 }, - ) +function drawSeal(ctx: PdfContext, x: number, top: number): void { + const { doc } = ctx + const cx = x + SEAL.radius + const cy = top + SEAL.radius + doc.circle(cx, cy, SEAL.radius).lineWidth(1.5).strokeColor(COLOR.accent).stroke() + doc.circle(cx, cy, SEAL.innerRadius).lineWidth(1).strokeColor(COLOR.accent).stroke() + line(ctx, FONT.display, 9, COLOR.ink, SEAL_WORDMARK, x, cy - 16, { + width: 68, + align: "center", + characterSpacing: 1.2, + }) + line(ctx, FONT.display, 6, COLOR.ink2, ctx.t("certificate.seal.line").toUpperCase(), x, cy - 2, { + width: 64, + align: "center", + characterSpacing: 0.2, + }) + line(ctx, FONT.mono, 8, COLOR.ink3, formatYear(ctx.issuedAt), x, cy + 8, { + width: 68, + align: "center", + }) +} - drawQr(verifyUrl, 474, blockTop, 84) - const textX = 306 - const textW = 156 - const prompt = t("certificate.verify.prompt", { url: verifyLabel }) - font(fontFor(prompt, "regular"), 8).fillColor(COLOR.ink2) - doc.text(prompt, textX, blockTop + 4, { - width: textW, - align: "right", - height: 24, - ellipsis: true, - }) - line(FONT.mono, 11, COLOR.ink, displayCode, textX, blockTop + 34, { - width: textW, +function drawIssuerLines(ctx: PdfContext, x: number, blockTop: number): void { + const issuerLine = ctx.t("certificate.issuer.line") + line(ctx, fontFor(issuerLine, "bold"), 9, COLOR.ink, issuerLine, x, blockTop + 78, { + width: 260, + }) + line( + ctx, + FONT.mono, + 7.5, + COLOR.ink3, + ctx.t("certificate.issuer.generated", { + timestamp: ctx.issuedAt.toISOString().replace(ISO_MILLISECONDS, "Z"), + }), + x, + blockTop + 90, + { width: 260 }, + ) +} + +function drawVerifyText(ctx: PdfContext, blockTop: number): void { + const { x, width } = VERIFY_TEXT + const prompt = ctx.t("certificate.verify.prompt", { url: ctx.verifyLabel }) + useFont(ctx, fontFor(prompt, "regular"), 8).fillColor(COLOR.ink2) + ctx.doc.text(prompt, x, blockTop + 4, { width, align: "right", height: 24, ellipsis: true }) + line(ctx, FONT.mono, 11, COLOR.ink, ctx.displayCode, x, blockTop + 34, { width, align: "right" }) + if (!ctx.fingerprint) return + const fingerprintLabel = ctx.t("certificate.verify.fingerprint") + line( + ctx, + fontFor(fingerprintLabel, "regular"), + 7, + COLOR.ink3, + fingerprintLabel, + x, + blockTop + 50, + { + width, align: "right", - }) - if (input.fingerprint) { - const fingerprintLabel = t("certificate.verify.fingerprint") - line( - fontFor(fingerprintLabel, "regular"), - 7, - COLOR.ink3, - fingerprintLabel, - textX, - blockTop + 50, - { - width: textW, - align: "right", - }, - ) - line(FONT.mono, 7.5, COLOR.ink3, input.fingerprint.slice(0, 16), textX, blockTop + 59, { - width: textW, - align: "right", - }) - } - } + }, + ) + line( + ctx, + FONT.mono, + 7.5, + COLOR.ink3, + ctx.fingerprint.slice(0, FINGERPRINT_PRINTED_CHARS), + x, + blockTop + 59, + { width, align: "right" }, + ) +} + +function drawIssuerBlock(ctx: PdfContext, top: number): void { + const { doc } = ctx + const attestation = ctx.t("certificate.attestation.body") + useFont(ctx, fontFor(attestation, "regular"), 9.5).fillColor(COLOR.ink2) + const paragraphHeight = doc.heightOfString(attestation, { width: ATTESTATION_WIDTH }) + doc.text(attestation, PAGE.left, top, { width: ATTESTATION_WIDTH }) + + const blockTop = top + paragraphHeight + 22 + drawSeal(ctx, PAGE.left, blockTop) + drawIssuerLines(ctx, PAGE.left, blockTop) + drawQr(ctx, ctx.verifyUrl, QR_BOX.x, blockTop, QR_BOX.size) + drawVerifyText(ctx, blockTop) +} - function drawQr(url: string, x: number, y: number, box: number): void { - const qr = qrcode(0, "M") - qr.addData(url) - qr.make() - const count = qr.getModuleCount() - const cell = box / (count + 8) - const originX = x + cell * 4 - const originY = y + cell * 4 - doc.fillColor(COLOR.ink) - for (let row = 0; row < count; row++) { - for (let col = 0; col < count; col++) { - if (qr.isDark(row, col)) { - doc.rect(originX + col * cell, originY + row * cell, cell, cell).fill(COLOR.ink) - } +function drawQr(ctx: PdfContext, url: string, x: number, y: number, box: number): void { + const { doc } = ctx + const qr = ctx.qrcode(QR_TYPE_NUMBER_AUTO, QR_ERROR_CORRECTION) + qr.addData(url) + qr.make() + const count = qr.getModuleCount() + const cell = box / (count + 2 * QR_QUIET_ZONE_MODULES) + const originX = x + cell * QR_QUIET_ZONE_MODULES + const originY = y + cell * QR_QUIET_ZONE_MODULES + doc.fillColor(COLOR.ink) + for (let row = 0; row < count; row++) { + for (let col = 0; col < count; col++) { + if (qr.isDark(row, col)) { + doc.rect(originX + col * cell, originY + row * cell, cell, cell).fill(COLOR.ink) } } } +} + +function drawFooter(ctx: PdfContext, page: number, total: number): void { + hairline(ctx, PAGE.footerRule) + line( + ctx, + FONT.body, + 7.5, + COLOR.ink3, + `${ctx.displayCode} · ${ctx.issuedLabel}`, + PAGE.left, + PAGE.footerText, + { width: FOOTER_COLUMN_WIDTH }, + ) + line(ctx, FONT.body, 7.5, COLOR.ink3, ctx.verifyLabel, 206, PAGE.footerText, { + width: FOOTER_COLUMN_WIDTH, + align: "center", + }) + line( + ctx, + FONT.body, + 7.5, + COLOR.ink3, + ctx.t("certificate.footer.page", { page, total }), + 358, + PAGE.footerText, + { width: FOOTER_COLUMN_WIDTH, align: "right" }, + ) + if (page === 1) { + const footnote = ctx.t("certificate.footer.timezone") + line(ctx, fontFor(footnote, "regular"), 7, COLOR.ink3, footnote, PAGE.left, PAGE.footnote, { + width: PAGE.contentWidth, + }) + } +} +/** The footer sits below the bottom margin, so the margin is lifted while it is drawn. */ +function drawFooters(ctx: PdfContext): void { + const { doc } = ctx const range = doc.bufferedPageRange() for (let i = range.start; i < range.start + range.count; i++) { doc.switchToPage(i) const savedBottom = doc.page.margins.bottom doc.page.margins.bottom = 0 - drawFooter(i - range.start + 1, range.count) + drawFooter(ctx, i - range.start + 1, range.count) doc.page.margins.bottom = savedBottom } +} - function drawFooter(page: number, total: number): void { - doc - .moveTo(PAGE.left, PAGE.footerRule) - .lineTo(PAGE.right, PAGE.footerRule) - .lineWidth(0.75) - .strokeColor(COLOR.hairline) - .stroke() - line( - FONT.body, - 7.5, - COLOR.ink3, - `${displayCode} · ${issuedLabel}`, - PAGE.left, - PAGE.footerText, - { - width: 200, - }, - ) - line(FONT.body, 7.5, COLOR.ink3, verifyLabel, 206, PAGE.footerText, { - width: 200, - align: "center", - }) - line( - FONT.body, - 7.5, - COLOR.ink3, - t("certificate.footer.page", { page, total }), - 358, - PAGE.footerText, - { width: 200, align: "right" }, - ) - if (page === 1) { - const footnote = t("certificate.footer.timezone") - line(fontFor(footnote, "regular"), 7, COLOR.ink3, footnote, PAGE.left, PAGE.footnote, { - width: PAGE.contentWidth, - }) - } +function collectBytes(doc: Doc): Promise { + const chunks: Buffer[] = [] + doc.on("data", (chunk: Buffer) => chunks.push(chunk)) + return new Promise((resolve, reject) => { + doc.on("end", () => resolve(new Uint8Array(Buffer.concat(chunks)))) + doc.on("error", (err: Error) => reject(err)) + }) +} + +export async function buildServiceHoursPdf(input: ServiceHoursPdfInput): Promise { + const PDFDocument = (await import("pdfkit")).default + const qrcode = (await import("qrcode-generator")).default + + const { model } = input + const t = input.t ?? certificateTranslator(model.locale) + const locale = model.locale + const displayCode = formatCertificateCode(input.code) + const issuedAt = toDate(input.issuedAt) + const verifyBaseUrl = input.verifyBaseUrl ?? CERTIFICATE_VERIFY_BASE_URL + const holderName = model.holder.displayName + + const doc = new PDFDocument({ + size: "LETTER", + margins: DOC_MARGINS, + bufferPages: true, + autoFirstPage: false, + pdfVersion: PDF_VERSION, + lang: locale, + displayTitle: true, + info: { + Title: t("certificate.doc.pdf_title", { name: holderName, code: displayCode }), + Author: BRAND, + Subject: t("certificate.doc.title"), + Keywords: displayCode, + Creator: BRAND, + Producer: BRAND, + CreationDate: issuedAt, + ModDate: issuedAt, + }, + }) + const bytes = collectBytes(doc) + + const ctx: PdfContext = { + doc, + registeredFonts: new Set(), + qrcode, + t, + locale, + model, + holderName, + displayCode, + issuedAt, + issuedLabel: formatDate(issuedAt, locale), + verifyUrl: `${verifyBaseUrl}/${displayCode}`, + verifyLabel: verifyBaseUrl.replace(URL_SCHEME, ""), + fingerprint: input.fingerprint, + tableContinues: true, } + let pageNumber = 0 + doc.on("pageAdded", () => { + pageNumber += 1 + if (pageNumber === 1) drawFirstPageChrome(ctx) + else drawContinuationChrome(ctx) + }) + + const rowHeights = measureRowHeights(ctx) + doc.addPage() + const tableEndY = drawLedgerTable(ctx, rowHeights) + const afterTotals = drawTotals(ctx, tableEndY) + const afterBanner = drawTruncationBanner(ctx, afterTotals) + drawIssuerBlock(ctx, issuerBlockTop(ctx, afterBanner)) + drawFooters(ctx) + doc.end() - await finished - return new Uint8Array(Buffer.concat(chunks)) + return bytes } function formatDate(value: Date, locale: string): string { diff --git a/services/api/src/services/certificate-repository.drizzle.ts b/services/api/src/services/certificate-repository.drizzle.ts index 0b093064..6fb05239 100644 --- a/services/api/src/services/certificate-repository.drizzle.ts +++ b/services/api/src/services/certificate-repository.drizzle.ts @@ -18,6 +18,7 @@ import type { Sql } from "../db/client.js" import { CertificateConflictError, + type CertificateConflictKind, type CertificateHolder, type CertificateInsert, type CertificateRepository, @@ -31,21 +32,24 @@ const PG_UNIQUE_VIOLATION = "23505" const FINGERPRINT_INDEX = "service_hours_certificates_live_fp_uidx" /** The `(code)` index (0064). */ const CODE_INDEX = "service_hours_certificates_code_uidx" +/** Substrings of the driver's `detail` text naming each index's key columns. */ +const FINGERPRINT_DETAIL_MARKER = "ledger_fingerprint" +const CODE_DETAIL_MARKER = "(code)" /** * postgres.js surfaces the violated index/constraint name on `constraint_name`. The `detail` fallback is * belt-and-braces for a driver that ever stops populating it: misclassifying a fingerprint conflict as a * code conflict would burn all five mint attempts and then 500 on a race the design says must succeed. */ -function conflictKind(err: unknown): "code" | "fingerprint" | null { +function conflictKind(err: unknown): CertificateConflictKind | null { if (typeof err !== "object" || err === null) return null const e = err as { code?: unknown; constraint_name?: unknown; detail?: unknown } if (e.code !== PG_UNIQUE_VIOLATION) return null const constraint = typeof e.constraint_name === "string" ? e.constraint_name : "" const detail = typeof e.detail === "string" ? e.detail : "" - if (constraint === FINGERPRINT_INDEX || detail.includes("ledger_fingerprint")) + if (constraint === FINGERPRINT_INDEX || detail.includes(FINGERPRINT_DETAIL_MARKER)) return "fingerprint" - if (constraint === CODE_INDEX || detail.includes("(code)")) return "code" + if (constraint === CODE_INDEX || detail.includes(CODE_DETAIL_MARKER)) return "code" return null } diff --git a/services/api/src/services/certificate-repository.memory.ts b/services/api/src/services/certificate-repository.memory.ts index 26f884ce..48a64364 100644 --- a/services/api/src/services/certificate-repository.memory.ts +++ b/services/api/src/services/certificate-repository.memory.ts @@ -18,7 +18,6 @@ * `snapshot` is stored but never read back, exactly as in production. */ -import { randomUUID } from "node:crypto" import { CertificateConflictError, type CertificateHolder, @@ -29,6 +28,9 @@ import { } from "./certificate-service.js" import type { TranscriptModel } from "./certificate-model.js" +/** Mirrors the `users.locale` column default. */ +const DEFAULT_HOLDER_LOCALE = "en" + /** What a test registers so `findHolder` can answer, mirroring the users read. */ export interface MemoryCertificateHolder { displayName: string @@ -148,7 +150,7 @@ export class InMemoryCertificateRepository implements CertificateRepository { userId, displayName: holder.displayName, handle: holder.handle ?? null, - locale: holder.locale ?? "en", + locale: holder.locale ?? DEFAULT_HOLDER_LOCALE, }) } } @@ -176,8 +178,3 @@ function clone(row: StoredCertificate): CertificateRow { revokedReason: row.revokedReason, } } - -/** Convenience for tests that need a plausible id without importing node:crypto. */ -export function newCertificateId(): string { - return randomUUID() -} diff --git a/services/api/src/services/certificate-service.ts b/services/api/src/services/certificate-service.ts index d1284a8c..dc9e697d 100644 --- a/services/api/src/services/certificate-service.ts +++ b/services/api/src/services/certificate-service.ts @@ -49,6 +49,17 @@ import type { VolunteerHoursRepository, } from "./volunteer-hours-service.js" +const CERTIFICATE_KEY_PREFIX = "certificates/service-hours" +const PDF_CONTENT_TYPE = "application/pdf" +const MS_PER_SECOND = 1000 +const HOLDER_REVOKED_REASON = "holder" +const TOMBSTONE_REVOKED_REASON = "account_closed" + +interface StoredDocument { + documentSha256: string + byteSize: number +} + /** * Structural slice of the storage adapter, declared locally for the same reason as in * `services/media-presign.ts`: the shared `Storage` interface declares `presignGet(key, ttlSec)` with only TWO parameters. The third @@ -204,10 +215,10 @@ export interface CertificateService { * `gen_random_uuid()` and is NEVER disclosed by the public verify endpoint, so even * `/` is unguessable: the same defence-in-depth posture as `buildR2Key(uploadId, now)` in media-intake-service.ts. */ -export function certificateObjectKey(id: string, issuedAt: Date): string { +function certificateObjectKey(id: string, issuedAt: Date): string { const year = issuedAt.getUTCFullYear().toString().padStart(4, "0") const month = (issuedAt.getUTCMonth() + 1).toString().padStart(2, "0") - return `certificates/service-hours/${year}/${month}/${id}.pdf` + return `${CERTIFICATE_KEY_PREFIX}/${year}/${month}/${id}.pdf` } /** @@ -216,7 +227,7 @@ export function certificateObjectKey(id: string, issuedAt: Date): string { * which already offers Download and Print. On mobile it is what makes Safari/Chrome show the document * with a Share affordance. */ -export function certificateContentDisposition(code: string): string { +function certificateContentDisposition(code: string): string { return `inline; filename="civfix-service-hours-${formatCertificateCode(code)}.pdf"` } @@ -284,23 +295,39 @@ export function makeCertificateService(deps: CertificateServiceDeps): Certificat url, urlExpiresAt: url !== null - ? new Date(at.getTime() + CERTIFICATE_GET_URL_TTL_SEC * 1000).toISOString() + ? new Date(at.getTime() + CERTIFICATE_GET_URL_TTL_SEC * MS_PER_SECOND).toISOString() : null, revokedAt: row.revokedAt?.toISOString() ?? null, } } - async function issue( - userId: string, - requestedLocale?: string, - ): Promise { - const at = now() - const holder = await repo.findHolder(userId) - // Only reachable for a session whose user row is gone/tombstoned; 404 rather than 500. - if (holder === null) throw AppError.notFound() - - const locale = resolveLocale(requestedLocale ?? holder.locale) + async function renderAndStore( + model: TranscriptModel, + fingerprint: string, + key: string, + code: string, + issuedAt: Date, + ): Promise { + const bytes = await buildServiceHoursPdf({ + model, + code, + issuedAt, + fingerprint, + ...(deps.verifyBaseUrl !== undefined ? { verifyBaseUrl: deps.verifyBaseUrl } : {}), + }) + const documentSha256 = sha256Hex(bytes) + await storage.put(key, bytes, { + contentType: PDF_CONTENT_TYPE, + contentDisposition: certificateContentDisposition(code), + }) + return { documentSha256, byteSize: bytes.byteLength } + } + async function buildLedgerModel( + userId: string, + holder: CertificateHolder, + locale: string, + ): Promise { // v1 issues over the WHOLE ledger: no geoid / from / to filters. `entryCount` is the full matching // count; `totalHours` is the sum of the RETURNED rows, because a printed total that does not equal // the sum of the printed lines is a self-contradicting document. @@ -317,7 +344,7 @@ export function makeCertificateService(deps: CertificateServiceDeps): Certificat throw AppError.conflict(certificateTranslator(locale)("certificate.error.no_hours")) } - const model = buildTranscriptModel({ + return buildTranscriptModel({ holder: { userId, displayName: holder.displayName, @@ -327,54 +354,57 @@ export function makeCertificateService(deps: CertificateServiceDeps): Certificat totals: { entryCount: page.entryCount, totalHours: page.totalHours }, locale, }) - const fingerprint = ledgerFingerprint(model) + } - const existing = await repo.findLiveByFingerprint(userId, fingerprint) - if (existing !== null) { - const head = await storage.head(existing.r2Key) - if (head !== null) { - // The common repeat call renders NOTHING: a row lookup, a HEAD and a presign. - return { - certificate: toCertificateDTO(existing, await presign(existing.r2Key), at), - reused: true, - } - } - // Expected to be exercised approximately never (operator error / a bucket incident). The row is - // the record of truth, so re-render THIS document (same code, same issue date, same key) rather than minting a second certificate over the same ledger, which the partial unique index - // would reject anyway. The model rebuilt above has the same fingerprint as the stored snapshot, so - // it is used directly instead of detoasting `snapshot` on a path that is otherwise free. - const bytes = await buildServiceHoursPdf({ - model, - code: existing.code, - issuedAt: existing.issuedAt, - fingerprint, - ...(deps.verifyBaseUrl !== undefined ? { verifyBaseUrl: deps.verifyBaseUrl } : {}), - }) - const documentSha256 = sha256Hex(bytes) - await storage.put(existing.r2Key, bytes, { - contentType: "application/pdf", - contentDisposition: certificateContentDisposition(existing.code), - }) - await repo.markRegenerated({ - id: existing.id, - documentSha256, - byteSize: bytes.byteLength, - at, - }) - deps.logger?.warn( - { certificateId: existing.id, r2Key: existing.r2Key }, - "certificate object missing for a live row; re-rendered from the ledger", - ) + async function reuseLive( + existing: CertificateRow, + model: TranscriptModel, + fingerprint: string, + at: Date, + ): Promise { + const head = await storage.head(existing.r2Key) + if (head !== null) { + // The common repeat call renders NOTHING: a row lookup, a HEAD and a presign. return { - certificate: toCertificateDTO( - { ...existing, documentSha256, byteSize: bytes.byteLength, regeneratedAt: at }, - await presign(existing.r2Key), - at, - ), + certificate: toCertificateDTO(existing, await presign(existing.r2Key), at), reused: true, } } + // Expected to be exercised approximately never (operator error / a bucket incident). The row is + // the record of truth, so re-render THIS document (same code, same issue date, same key) rather + // than minting a second certificate over the same ledger, which the partial unique index would + // reject anyway. The freshly built model has the same fingerprint as the stored snapshot, so it is + // used directly instead of detoasting `snapshot` on a path that is otherwise free. + const { documentSha256, byteSize } = await renderAndStore( + model, + fingerprint, + existing.r2Key, + existing.code, + existing.issuedAt, + ) + await repo.markRegenerated({ id: existing.id, documentSha256, byteSize, at }) + deps.logger?.warn( + { certificateId: existing.id, r2Key: existing.r2Key }, + "certificate object missing for a live row; re-rendered from the ledger", + ) + return { + certificate: toCertificateDTO( + { ...existing, documentSha256, byteSize, regeneratedAt: at }, + await presign(existing.r2Key), + at, + ), + reused: true, + } + } + async function mintNew( + userId: string, + holder: CertificateHolder, + locale: string, + model: TranscriptModel, + fingerprint: string, + at: Date, + ): Promise { const id = newId() const r2Key = certificateObjectKey(id, at) @@ -382,18 +412,7 @@ export function makeCertificateService(deps: CertificateServiceDeps): Certificat // The code is PRINTED on the document, so a re-mint must re-render. The key is derived from the // row id, which does not change across attempts, so the re-put overwrites rather than orphaning. const code = mintCode() - const bytes = await buildServiceHoursPdf({ - model, - code, - issuedAt: at, - fingerprint, - ...(deps.verifyBaseUrl !== undefined ? { verifyBaseUrl: deps.verifyBaseUrl } : {}), - }) - const documentSha256 = sha256Hex(bytes) - await storage.put(r2Key, bytes, { - contentType: "application/pdf", - contentDisposition: certificateContentDisposition(code), - }) + const { documentSha256, byteSize } = await renderAndStore(model, fingerprint, r2Key, code, at) try { const row = await repo.insert({ @@ -412,7 +431,7 @@ export function makeCertificateService(deps: CertificateServiceDeps): Certificat snapshot: model, r2Key, documentSha256, - byteSize: bytes.byteLength, + byteSize, issuedAt: at, }) return { certificate: toCertificateDTO(row, await presign(r2Key), at), reused: false } @@ -442,6 +461,24 @@ export function makeCertificateService(deps: CertificateServiceDeps): Certificat throw AppError.internal("Could not mint a unique certificate code") } + async function issue( + userId: string, + requestedLocale?: string, + ): Promise { + const at = now() + const holder = await repo.findHolder(userId) + // Only reachable for a session whose user row is gone/tombstoned; 404 rather than 500. + if (holder === null) throw AppError.notFound() + + const locale = resolveLocale(requestedLocale ?? holder.locale) + const model = await buildLedgerModel(userId, holder, locale) + const fingerprint = ledgerFingerprint(model) + + const existing = await repo.findLiveByFingerprint(userId, fingerprint) + if (existing !== null) return reuseLive(existing, model, fingerprint, at) + return mintNew(userId, holder, locale, model, fingerprint, at) + } + async function list(userId: string): Promise { const at = now() const rows = await repo.listFor(userId) @@ -453,7 +490,7 @@ export function makeCertificateService(deps: CertificateServiceDeps): Certificat async function revoke(userId: string, code: string): Promise { const at = now() - const row = await repo.revoke(userId, code, "holder", at) + const row = await repo.revoke(userId, code, HOLDER_REVOKED_REASON, at) // Someone else's code is 404, NOT 403: a 403 would confirm that the code exists (the existence-oracle // rule documented in media.routes.ts). if (row === null) throw AppError.notFound() @@ -507,7 +544,7 @@ export function makeCertificateService(deps: CertificateServiceDeps): Certificat revokedAt: row.revokedAt?.toISOString() ?? null, // Hardcoded rather than echoing `row.revokedReason`: the tombstone is the authoritative answer // even for a row the holder had already revoked for their own reason. - revokedReason: "account_closed", + revokedReason: TOMBSTONE_REVOKED_REASON, } } diff --git a/services/api/src/services/chat-attachments.drizzle.ts b/services/api/src/services/chat-attachments.drizzle.ts index ca9a5a17..752c59e5 100644 --- a/services/api/src/services/chat-attachments.drizzle.ts +++ b/services/api/src/services/chat-attachments.drizzle.ts @@ -1,7 +1,13 @@ import type { Queryable } from "../db/client.js" import type { MediaDTO } from "@civfix/shared" import type { PresignMedia } from "./media-presign.js" -import { loadServableAttachmentsFor, makeAttachmentRepo } from "./message-attachments.drizzle.js" +import { + loadServableAttachmentsFor, + makeAttachmentRepo, + type MessageMediaColumn, +} from "./message-attachments.drizzle.js" + +const CHAT_MESSAGE_COLUMN: MessageMediaColumn = "chat_message_id" export function attachChatMedia( sql: Queryable, @@ -10,7 +16,7 @@ export function attachChatMedia( messageCreatedAt: Date, senderId: string, ): Promise { - return makeAttachmentRepo("chat_message_id").attach( + return makeAttachmentRepo(CHAT_MESSAGE_COLUMN).attach( sql, messageId, uploadIds, @@ -25,5 +31,5 @@ export function loadChatAttachments( presign: PresignMedia, viewerUserId: string | null, ): Promise> { - return loadServableAttachmentsFor(sql, "chat_message_id", messageIds, presign, viewerUserId) + return loadServableAttachmentsFor(sql, CHAT_MESSAGE_COLUMN, messageIds, presign, viewerUserId) } diff --git a/services/api/src/services/chat-bells.ts b/services/api/src/services/chat-bells.ts index 9b567f0d..336c20a4 100644 --- a/services/api/src/services/chat-bells.ts +++ b/services/api/src/services/chat-bells.ts @@ -7,18 +7,34 @@ * - dm reply vs dm delivered bell: makeDmBellNotifier is the single dm bell site, so one bell by * construction. */ -import type { NotificationType, RoomKind } from "@civfix/shared" +import type { ChatMessageDTO, NotificationType, RoomKind } from "@civfix/shared" import type { NotificationService } from "./notification-service.js" import { CONVERSATION_BELL } from "./conversation-bell.js" -import { dmAuthorName, mentionAuthorName, textPreview } from "../routes/chat-notify-copy.js" +import { messageAuthorName, textPreview } from "../routes/chat-notify-copy.js" +import type { MessageKey } from "../i18n/renderMessage.js" import type { GatewayChatMentions, OnChatReply, OnDmDelivered } from "../ws/types.js" +type GroupRoomKind = "cleanup" | "report" | "group" + +const NO_PREVIEW_BODY_KEY = "notification.message.no_preview" satisfies MessageKey + +const CHAT_MENTION_TITLE_KEY = "notification.chat_mention.title" satisfies MessageKey + +const CHAT_REPLY_TITLE_KEY = "notification.chat_reply.title" satisfies MessageKey + +const DM_TITLE_FALLBACK_KEY = "notification.dm.title_fallback" satisfies MessageKey + +function previewBody(message: ChatMessageDTO): { body: string } | { bodyKey: MessageKey } { + const preview = textPreview(message) + return preview !== null ? { body: preview } : { bodyKey: NO_PREVIEW_BODY_KEY } +} + /** * Straight off CONVERSATION_BELL: clear-on-open (clearByTypeAndLink) matches on exactly these two * strings, so a second hand-written copy of the map would silently strand bells once either side changed. */ function groupBellRoute( - kind: "cleanup" | "report" | "group", + kind: GroupRoomKind, roomId: string, ): { type: NotificationType; link: string } { const spec = CONVERSATION_BELL[kind] @@ -45,7 +61,7 @@ export interface ChatBellDeps { async function isGroupMember( deps: ChatBellDeps, - kind: "cleanup" | "report" | "group", + kind: GroupRoomKind, roomId: string, userId: string, ): Promise { @@ -54,6 +70,34 @@ async function isGroupMember( return deps.isCleanupMember(roomId, userId) } +async function bellGroupMember( + deps: ChatBellDeps, + recipientId: string, + kind: GroupRoomKind, + roomId: string, + message: ChatMessageDTO, + titleKey: MessageKey, +): Promise { + const name = messageAuthorName(message) + const { type, link } = groupBellRoute(kind, roomId) + await deps.notificationService.createNotification(recipientId, { + type, + titleKey, + vars: { name }, + ...previewBody(message), + link, + }) +} + +function dmBellTitle( + name: string, + isReplyToRecipient: boolean, +): { titleKey: MessageKey; vars?: { name: string } } | { title: string } { + if (name === "") return { titleKey: DM_TITLE_FALLBACK_KEY } + if (isReplyToRecipient) return { titleKey: CHAT_REPLY_TITLE_KEY, vars: { name } } + return { title: name } +} + /** A dm message already bells through makeDmBellNotifier, so dm mentions are a no-op here. */ export function makeChatMentionNotifier( deps: ChatBellDeps, @@ -65,16 +109,7 @@ export function makeChatMentionNotifier( if (!(await isGroupMember(deps, kind, roomId, mentionedUserId))) return if (await deps.isBlockedEitherWay(actorUserId, mentionedUserId)) return if (!(await deps.notificationService.getPrefs(mentionedUserId)).mentions) return - const name = mentionAuthorName(message) - const preview = textPreview(message) - const { type, link } = groupBellRoute(kind, roomId) - await deps.notificationService.createNotification(mentionedUserId, { - type, - titleKey: "notification.chat_mention.title", - vars: { name }, - ...(preview !== null ? { body: preview } : { bodyKey: "notification.message.no_preview" }), - link, - }) + await bellGroupMember(deps, mentionedUserId, kind, roomId, message, CHAT_MENTION_TITLE_KEY) } } @@ -99,16 +134,7 @@ export function makeChatReplyNotifier(deps: ChatBellDeps): OnChatReply { } if (online.includes(targetUserId)) return } - const name = mentionAuthorName(message) - const preview = textPreview(message) - const { type, link } = groupBellRoute(kind, roomId) - await deps.notificationService.createNotification(targetUserId, { - type, - titleKey: "notification.chat_reply.title", - vars: { name }, - ...(preview !== null ? { body: preview } : { bodyKey: "notification.message.no_preview" }), - link, - }) + await bellGroupMember(deps, targetUserId, kind, roomId, message, CHAT_REPLY_TITLE_KEY) } } @@ -127,17 +153,12 @@ export function makeDmBellNotifier(deps: DmBellDeps): OnDmDelivered { if (!isReplyToRecipient) return if (!(await deps.notificationService.getPrefs(recipientId)).mentions) return } - const name = dmAuthorName(message) - const preview = textPreview(message) + const bell = CONVERSATION_BELL.dm await deps.notificationService.createNotification(recipientId, { - type: "dm", - ...(isReplyToRecipient && name !== "" - ? { titleKey: "notification.chat_reply.title" as const, vars: { name } } - : name !== "" - ? { title: name } - : { titleKey: "notification.dm.title_fallback" as const }), - ...(preview !== null ? { body: preview } : { bodyKey: "notification.message.no_preview" }), - link: `/messages/dm/${threadId}`, + type: bell.type, + ...dmBellTitle(messageAuthorName(message), isReplyToRecipient), + ...previewBody(message), + link: bell.link(threadId), }) } } diff --git a/services/api/src/services/chat-edit-service.ts b/services/api/src/services/chat-edit-service.ts index d347011d..9289cd3d 100644 --- a/services/api/src/services/chat-edit-service.ts +++ b/services/api/src/services/chat-edit-service.ts @@ -14,10 +14,21 @@ import { parseUserMentions } from "./discussion-mentions.js" import { broadcastMessageUpdate } from "../ws/frame-handler.js" import { neutralizeChatViewerFields } from "./chat-viewer-fields.js" import type { GatewayChatMentions } from "../ws/types.js" -import type { ChatRepository } from "./chat-repository.drizzle.js" +import type { ChatMessageMeta, ChatRepository } from "./chat-repository.drizzle.js" import type { DmRepository } from "./dm-repository.drizzle.js" -export const CHAT_EDIT_FORBIDDEN = "You can't edit this message." +const CHAT_EDIT_FORBIDDEN = "You can't edit this message." + +const MESSAGE_DELETED = "This message was deleted." + +const MS_PER_HOUR = 3_600_000 + +const EDIT_WINDOW_MS = EDIT_WINDOW_HOURS * MS_PER_HOUR + +const EDIT_ERROR_CODE = { + notSender: "not_sender", + editWindowExpired: "edit_window_expired", +} as const export type IsRoomMemberFn = (roomId: string, userId: string) => Promise @@ -56,11 +67,13 @@ export interface ChatEditService { } const notSender = () => - new AppError(ErrorCode.FORBIDDEN, CHAT_EDIT_FORBIDDEN, { fields: { code: "not_sender" } }) + new AppError(ErrorCode.FORBIDDEN, CHAT_EDIT_FORBIDDEN, { + fields: { code: EDIT_ERROR_CODE.notSender }, + }) const editWindowExpired = () => new AppError(ErrorCode.FORBIDDEN, "This message can no longer be edited.", { - fields: { code: "edit_window_expired" }, + fields: { code: EDIT_ERROR_CODE.editWindowExpired }, }) function assertEditable( @@ -68,16 +81,38 @@ function assertEditable( userId: string, ): void { if (meta.senderId !== null && meta.senderId !== userId) throw notSender() - if (meta.deletedAt !== null) throw AppError.conflict("This message was deleted.") + if (meta.deletedAt !== null) throw AppError.conflict(MESSAGE_DELETED) // A sender-less SYSTEM row lands here too (kind "system"), so it 422s rather than 403s. if (meta.kind !== "text") { throw AppError.validation({ kind: "Only text messages can be edited." }) } - if (Date.now() - meta.createdAt.getTime() > EDIT_WINDOW_HOURS * 3_600_000) { + if (Date.now() - meta.createdAt.getTime() > EDIT_WINDOW_MS) { throw editWindowExpired() } } +function roomRefOf( + meta: Pick, + roomKind: RoomKind, +): string | null { + if (roomKind === "report") return meta.reportId + if (roomKind === "group") return meta.groupId + return meta.cleanupId +} + +function editInRoom( + chat: ChatRepository, + roomKind: RoomKind, + roomId: string, + messageId: string, + userId: string, + body: string, +): Promise { + if (roomKind === "report") return chat.editReportMessage(roomId, messageId, userId, body) + if (roomKind === "group") return chat.editGroupMessage(roomId, messageId, userId, body) + return chat.editMessage(roomId, messageId, userId, body) +} + export function makeChatEditService(deps: ChatEditServiceDeps): ChatEditService { async function rerecordMentions( kind: RoomKind, @@ -140,56 +175,56 @@ export function makeChatEditService(deps: ChatEditServiceDeps): ChatEditService // message no reader ever hydrates. await rerecordMentions("dm", roomId, userId, messageId, body, input.mentionedUserIds) const updated = await dm.editMessage(roomId, messageId, userId, body) - if (updated === null) throw AppError.conflict("This message was deleted.") + if (updated === null) throw AppError.conflict(MESSAGE_DELETED) fireMessageUpdate("dm", roomId, updated) return updated } - async function editRoomMessage(input: EditMessageInput): Promise { - const { roomKind, roomId, messageId, userId, body } = input - const chat = deps.chat - if (!chat) throw new Error("chat-edit-service: chat deps not wired") - const isReport = roomKind === "report" - const isGroup = roomKind === "group" - if (isReport) { - // Visibility first, so a held or unlisted report answers 404 rather than leaking a 403 keyed on a - // stale membership row. - if (deps.isReportVisible && !(await deps.isReportVisible(roomId, userId))) { - throw AppError.notFound("Report not found") - } - const isReportMember = deps.isReportMember - if (!isReportMember) throw new Error("chat-edit-service: report deps not wired") - if (!(await isReportMember(roomId, userId))) throw AppError.forbidden(CHAT_EDIT_FORBIDDEN) - } else if (isGroup) { + async function requireReportMember(reportId: string, userId: string): Promise { + // Visibility first, so a held or unlisted report answers 404 rather than leaking a 403 keyed on a + // stale membership row. + if (deps.isReportVisible && !(await deps.isReportVisible(reportId, userId))) { + throw AppError.notFound("Report not found") + } + const isReportMember = deps.isReportMember + if (!isReportMember) throw new Error("chat-edit-service: report deps not wired") + if (!(await isReportMember(reportId, userId))) throw AppError.forbidden(CHAT_EDIT_FORBIDDEN) + } + + async function requireRoomMember( + roomKind: RoomKind, + roomId: string, + userId: string, + ): Promise { + if (roomKind === "report") return requireReportMember(roomId, userId) + if (roomKind === "group") { const isGroupMember = deps.isGroupMember if (!isGroupMember) throw new Error("chat-edit-service: group deps not wired") if (!(await isGroupMember(roomId, userId))) throw AppError.forbidden(CHAT_EDIT_FORBIDDEN) - } else { - const isCleanupMember = deps.isCleanupMember - if (!isCleanupMember) throw new Error("chat-edit-service: cleanup deps not wired") - if (!(await isCleanupMember(roomId, userId))) throw AppError.forbidden(CHAT_EDIT_FORBIDDEN) + return } + const isCleanupMember = deps.isCleanupMember + if (!isCleanupMember) throw new Error("chat-edit-service: cleanup deps not wired") + if (!(await isCleanupMember(roomId, userId))) throw AppError.forbidden(CHAT_EDIT_FORBIDDEN) + } + + async function editRoomMessage(input: EditMessageInput): Promise { + const { roomKind, roomId, messageId, userId, body } = input + const chat = deps.chat + if (!chat) throw new Error("chat-edit-service: chat deps not wired") + await requireRoomMember(roomKind, roomId, userId) const meta = await chat.findMessageMeta(messageId) - const roomMatches = - meta !== null && - (isReport - ? meta.reportId === roomId - : isGroup - ? meta.groupId === roomId - : meta.cleanupId === roomId) - if (meta === null || !roomMatches) throw AppError.notFound("Message not found") + if (meta === null || roomRefOf(meta, roomKind) !== roomId) { + throw AppError.notFound("Message not found") + } assertEditable(meta, userId) assertNoSlur(body, "body") // Mentions are replaced before the sender-gated UPDATE so the re-read DTO carries them; a lost race // leaves the residue on a tombstoned row no reader hydrates. await rerecordMentions(roomKind, roomId, userId, messageId, body, input.mentionedUserIds) - const updated = isReport - ? await chat.editReportMessage(roomId, messageId, userId, body) - : isGroup - ? await chat.editGroupMessage(roomId, messageId, userId, body) - : await chat.editMessage(roomId, messageId, userId, body) - if (updated === null) throw AppError.conflict("This message was deleted.") + const updated = await editInRoom(chat, roomKind, roomId, messageId, userId, body) + if (updated === null) throw AppError.conflict(MESSAGE_DELETED) fireMessageUpdate(roomKind, roomId, updated) return updated } diff --git a/services/api/src/services/chat-fanout-jobs.ts b/services/api/src/services/chat-fanout-jobs.ts index 6d4d1015..f932a0a3 100644 --- a/services/api/src/services/chat-fanout-jobs.ts +++ b/services/api/src/services/chat-fanout-jobs.ts @@ -93,7 +93,7 @@ export async function runChatRoomFanout( await runRoomFanout(ROOM_FANOUT_SPEC[data.kind], deps.fanoutDeps[data.kind], data.roomId, message) } -export async function runChatRoomFanoutJob( +async function runChatRoomFanoutJob( container: Container, data: ChatRoomFanoutJob, logger?: RoomFanoutLogger, diff --git a/services/api/src/services/chat-group-repository.drizzle.ts b/services/api/src/services/chat-group-repository.drizzle.ts index 6242dcc4..3dba5e5d 100644 --- a/services/api/src/services/chat-group-repository.drizzle.ts +++ b/services/api/src/services/chat-group-repository.drizzle.ts @@ -3,8 +3,8 @@ import type { MediaDTO, MediaKind, MediaStatus, PersonDTO } from "@civfix/shared import type { ChatGroupKind, ChatGroupVisibility } from "../db/schema/chat-groups.js" import type { GROUP_MEMBER_ROLE_VALUES } from "../db/schema/types.js" import type { PresignMedia } from "./media-presign.js" -import { publicAuthorIdentity } from "./public-author.js" -import { blockedPairExpr, hiddenIdentity } from "./hidden-identity.js" +import { blockedPairExpr } from "./hidden-identity.js" +import { toRoomMemberPerson, type RoomMemberIdentityRow } from "./room-member-person.js" import { resolveAvatarMediaOrThrow } from "./avatar-media.js" import { userUploader } from "./media-uploader.js" import { monotonicReadWatermarkUpdate } from "./chat-read-state.drizzle.js" @@ -111,64 +111,35 @@ interface GroupRowSelect { avatar_height: number | null } -export interface MemberRowSelect { - user_id: string +export interface MemberRowSelect extends RoomMemberIdentityRow { role: GroupMemberRole joined_at: Date - display_name: string | null - handle: string | null - bio: string | null - avatar_url: string | null - user_deleted_at: Date | null - is_following: boolean - blocked_pair: boolean } export function toMemberView(r: MemberRowSelect): GroupMemberView { - const author = publicAuthorIdentity({ - id: r.user_id, - displayName: r.display_name ?? "", - handle: r.handle, - avatarUrl: r.avatar_url, - deletedAt: r.user_deleted_at, - }) - const hidden = r.blocked_pair && !author.deleted ? hiddenIdentity(r.user_id) : null - const user: PersonDTO = { - id: r.user_id, - name: hidden?.name ?? author.name, - handle: hidden !== null ? null : author.handle, - bio: author.deleted || hidden !== null ? null : r.bio, - avatar: author.avatar, - ...(hidden === null && author.avatarUrl !== undefined ? { avatarUrl: author.avatarUrl } : {}), - followers: 0, - following: 0, - isFollowing: r.is_following, - ...(author.deleted ? { deleted: true } : {}), + return { user: toRoomMemberPerson(r), role: r.role, joinedAt: r.joined_at } +} + +async function toGroupAvatar(r: GroupRowSelect, presign?: PresignMedia): Promise { + if (!presign || r.avatar_id === null || r.avatar_status !== "ready" || r.avatar_r2_key === null) { + return null + } + const { url, thumbUrl } = await presign(r.avatar_r2_key, r.avatar_thumb_key) + return { + id: r.avatar_id, + kind: r.avatar_kind ?? "image", + codec: r.avatar_codec, + url, + ...(thumbUrl !== undefined ? { thumbUrl } : {}), + width: r.avatar_width, + height: r.avatar_height, + status: r.avatar_status, } - return { user, role: r.role, joinedAt: r.joined_at } } export function makeChatGroupRepository(sql: Sql, presign?: PresignMedia): ChatGroupRepository { async function toGroupView(r: GroupRowSelect): Promise { - let avatar: MediaDTO | null = null - if ( - presign && - r.avatar_id !== null && - r.avatar_status === "ready" && - r.avatar_r2_key !== null - ) { - const { url, thumbUrl } = await presign(r.avatar_r2_key, r.avatar_thumb_key) - avatar = { - id: r.avatar_id, - kind: r.avatar_kind ?? "image", - codec: r.avatar_codec, - url, - ...(thumbUrl !== undefined ? { thumbUrl } : {}), - width: r.avatar_width, - height: r.avatar_height, - status: r.avatar_status, - } - } + const avatar = await toGroupAvatar(r, presign) return { id: r.id, kind: r.kind, diff --git a/services/api/src/services/chat-group-service.ts b/services/api/src/services/chat-group-service.ts index e7c21e0e..42ff59c5 100644 --- a/services/api/src/services/chat-group-service.ts +++ b/services/api/src/services/chat-group-service.ts @@ -19,10 +19,25 @@ import { isOfficialAccount } from "../auth/official-account.js" import { NO_AFFILIATIONS, withAffiliation, type AffiliationLoader } from "./affiliation.js" export const GROUP_MEMBERS_DEFAULT_LIMIT = 25 -export const GROUP_MEMBERS_MAX_LIMIT = 50 +const GROUP_MEMBERS_MAX_LIMIT = 50 const INVITE_BLOCK_SCAN_MEMBERS = 200 +const NOT_A_GROUP_MEMBER = "That user isn't a member of this group." + +const OWNER_ROLE_FIXED = "The owner's role can't be changed." + +const GROUP_ERROR_CODE = { + notAMember: "not_a_member", + updateForbidden: "update_forbidden", + visibilityOwnerOnly: "visibility_owner_only", + addMembersForbidden: "add_members_forbidden", + ownerMustStay: "owner_must_stay", + removeForbidden: "remove_forbidden", + roleOwnerOnly: "role_owner_only", + notPublic: "not_public", +} as const + export interface ChatGroupServiceDeps { groups: ChatGroupRepository isMutedFor?: (userId: string, groupId: string) => Promise @@ -34,6 +49,12 @@ const forbidden = (message: string, code: string): AppError => const groupNotFound = (): AppError => AppError.notFound("Group not found") +const notOpenToJoin = (): AppError => + forbidden("This group isn't open to join.", GROUP_ERROR_CODE.notPublic) + +const isOwnerOrAdmin = (role: GroupMemberRole | null): boolean => + role === "owner" || role === "admin" + function toMemberDTO(view: GroupMemberView): GroupMemberDTO { return { user: view.user, role: view.role, joinedAt: view.joinedAt.toISOString() } } @@ -141,7 +162,7 @@ export function makeChatGroupService(deps: ChatGroupServiceDeps): ChatGroupServi groups.roleOf(groupId, viewerId), ]) if (view === null || (role === null && view.visibility === "private")) { - throw forbidden("You aren't a member of this group.", "not_a_member") + throw forbidden("You aren't a member of this group.", GROUP_ERROR_CODE.notAMember) } return { view, role } } @@ -192,8 +213,11 @@ export function makeChatGroupService(deps: ChatGroupServiceDeps): ChatGroupServi async updateGroup(userId, req) { const role = await groups.roleOf(req.id, userId) - if (role !== "owner" && role !== "admin") { - throw forbidden("Only the owner or an admin can update this group.", "update_forbidden") + if (!isOwnerOrAdmin(role)) { + throw forbidden( + "Only the owner or an admin can update this group.", + GROUP_ERROR_CODE.updateForbidden, + ) } assertNoSlur(req.name ?? null, "name") assertNoSlur(req.description ?? null, "description") @@ -201,7 +225,7 @@ export function makeChatGroupService(deps: ChatGroupServiceDeps): ChatGroupServi if (req.visibility !== undefined && req.visibility !== view.visibility && role !== "owner") { throw forbidden( "Only the owner can change this group's visibility.", - "visibility_owner_only", + GROUP_ERROR_CODE.visibilityOwnerOnly, ) } await groups.update( @@ -222,8 +246,11 @@ export function makeChatGroupService(deps: ChatGroupServiceDeps): ChatGroupServi async addMembers(userId, req) { const role = await groups.roleOf(req.id, userId) - if (role !== "owner" && role !== "admin") { - throw forbidden("Only the owner or an admin can add members.", "add_members_forbidden") + if (!isOwnerOrAdmin(role)) { + throw forbidden( + "Only the owner or an admin can add members.", + GROUP_ERROR_CODE.addMembersForbidden, + ) } await requireGroup(req.id) const invitees = await filterInviteesForRoom(userId, req.id, req.memberIds) @@ -238,22 +265,25 @@ export function makeChatGroupService(deps: ChatGroupServiceDeps): ChatGroupServi ]) if (actorId === targetId) { - if (targetRole === null) throw AppError.notFound("That user isn't a member of this group.") + if (targetRole === null) throw AppError.notFound(NOT_A_GROUP_MEMBER) if (actorRole === "owner") { throw new AppError(ErrorCode.CONFLICT, "The owner can't leave their own group.", { - fields: { code: "owner_must_stay" }, + fields: { code: GROUP_ERROR_CODE.ownerMustStay }, }) } await groups.removeMember(groupId, targetId) return } - if (actorRole !== "owner" && actorRole !== "admin") { - throw forbidden("Only the owner or an admin can remove members.", "remove_forbidden") + if (!isOwnerOrAdmin(actorRole)) { + throw forbidden( + "Only the owner or an admin can remove members.", + GROUP_ERROR_CODE.removeForbidden, + ) } - if (targetRole === null) throw AppError.notFound("That user isn't a member of this group.") + if (targetRole === null) throw AppError.notFound(NOT_A_GROUP_MEMBER) if (targetRole === "owner" || (targetRole === "admin" && actorRole !== "owner")) { - throw forbidden("You can't remove this member.", "remove_forbidden") + throw forbidden("You can't remove this member.", GROUP_ERROR_CODE.removeForbidden) } await groups.banMember(groupId, targetId, actorId) }, @@ -261,19 +291,19 @@ export function makeChatGroupService(deps: ChatGroupServiceDeps): ChatGroupServi async setMemberRole(actorId, groupId, targetId, role) { const actorRole = await groups.roleOf(groupId, actorId) if (actorRole !== "owner") { - throw forbidden("Only the owner can change member roles.", "role_owner_only") + throw forbidden("Only the owner can change member roles.", GROUP_ERROR_CODE.roleOwnerOnly) } if (targetId === actorId) { - throw AppError.validation({ userId: "The owner's role can't be changed." }) + throw AppError.validation({ userId: OWNER_ROLE_FIXED }) } const targetRole = await groups.roleOf(groupId, targetId) - if (targetRole === null) throw AppError.notFound("That user isn't a member of this group.") + if (targetRole === null) throw AppError.notFound(NOT_A_GROUP_MEMBER) if (targetRole === "owner") { - throw AppError.validation({ userId: "The owner's role can't be changed." }) + throw AppError.validation({ userId: OWNER_ROLE_FIXED }) } await groups.setRole(groupId, targetId, role) const member = await groups.findMember(groupId, targetId, actorId) - if (member === null) throw AppError.notFound("That user isn't a member of this group.") + if (member === null) throw AppError.notFound(NOT_A_GROUP_MEMBER) return toMemberDTO(member) }, @@ -301,18 +331,14 @@ export function makeChatGroupService(deps: ChatGroupServiceDeps): ChatGroupServi async joinGroup(userId, groupId) { const view = await groups.findById(groupId) - if (view === null || view.visibility !== "public") { - throw forbidden("This group isn't open to join.", "not_public") - } + if (view === null || view.visibility !== "public") throw notOpenToJoin() const role = await groups.roleOf(groupId, userId) if (role !== null) { return toGroupDTO(view, userId, role) } if (!(await groups.joinUnlessBanned(groupId, userId))) { const concurrentRole = await groups.roleOf(groupId, userId) - if (concurrentRole === null) { - throw forbidden("This group isn't open to join.", "not_public") - } + if (concurrentRole === null) throw notOpenToJoin() return toGroupDTO(view, userId, concurrentRole) } const refreshed = await requireGroup(groupId) diff --git a/services/api/src/services/chat-mention-resolver.ts b/services/api/src/services/chat-mention-resolver.ts index 00e72cc4..604c82f2 100644 --- a/services/api/src/services/chat-mention-resolver.ts +++ b/services/api/src/services/chat-mention-resolver.ts @@ -67,6 +67,17 @@ export interface ChatMentionResolverDeps { listGroupMemberIds(groupId: string, candidateIds: string[]): Promise } +function mentionableMemberIds( + deps: ChatMentionResolverDeps, + kind: Exclude, + roomId: string, + candidateIds: string[], +): Promise { + if (kind === "report") return deps.listReportChatMemberIds(roomId, candidateIds) + if (kind === "group") return deps.listGroupMemberIds(roomId, candidateIds) + return deps.listCleanupMemberIds(roomId, candidateIds) +} + export function makeChatMentionResolver( deps: ChatMentionResolverDeps, ): GatewayChatMentions["resolveChatMentions"] { @@ -82,13 +93,9 @@ export function makeChatMentionResolver( return peer !== null ? resolved.filter((m) => m.id === peer) : [] } const candidateIds = resolved.map((m) => m.id) - const members = - input.kind === "report" - ? await deps.listReportChatMemberIds(input.roomId, candidateIds) - : input.kind === "group" - ? await deps.listGroupMemberIds(input.roomId, candidateIds) - : await deps.listCleanupMemberIds(input.roomId, candidateIds) - const memberIds = new Set(members) + const memberIds = new Set( + await mentionableMemberIds(deps, input.kind, input.roomId, candidateIds), + ) return resolved.filter((m) => memberIds.has(m.id)) } } diff --git a/services/api/src/services/chat-poll-service.ts b/services/api/src/services/chat-poll-service.ts index e1e44cb1..d421c02f 100644 --- a/services/api/src/services/chat-poll-service.ts +++ b/services/api/src/services/chat-poll-service.ts @@ -47,8 +47,24 @@ const ROOM_COLUMN: Record = { group: "group_id", } +const POLL_ERROR_CODE = { + closed: "poll_closed", + forbidden: "poll_forbidden", + notMember: "poll_not_member", + closeForbidden: "poll_close_forbidden", +} as const + +const pollNotFound = (): AppError => AppError.notFound("Poll not found") + const pollClosed = (): AppError => - new AppError(ErrorCode.CONFLICT, "This poll is closed.", { fields: { code: "poll_closed" } }) + new AppError(ErrorCode.CONFLICT, "This poll is closed.", { + fields: { code: POLL_ERROR_CODE.closed }, + }) + +const pollCloseForbidden = (): AppError => + new AppError(ErrorCode.FORBIDDEN, "You can't close this poll.", { + fields: { code: POLL_ERROR_CODE.closeForbidden }, + }) export interface ChatPollService { createPoll(input: CreatePollInput): Promise @@ -66,19 +82,25 @@ export function makeChatPollService(deps: ChatPollServiceDeps): ChatPollService if (!(await deps.isReportVisible(roomId, userId))) throw AppError.notFound("Report not found") } + function findRoomMessage( + roomKind: PollRoomKind, + roomId: string, + messageId: string, + viewerUserId: string | null, + ): Promise { + if (roomKind === "report") return deps.chat.findReportMessage(roomId, messageId, viewerUserId) + if (roomKind === "group") return deps.chat.findGroupMessage(roomId, messageId, viewerUserId) + return deps.chat.findMessage(roomId, messageId, viewerUserId) + } + async function readMessage( roomKind: PollRoomKind, roomId: string, messageId: string, viewerUserId: string | null, ): Promise { - const dto = - roomKind === "report" - ? await deps.chat.findReportMessage(roomId, messageId, viewerUserId) - : roomKind === "group" - ? await deps.chat.findGroupMessage(roomId, messageId, viewerUserId) - : await deps.chat.findMessage(roomId, messageId, viewerUserId) - if (dto === null) throw AppError.notFound("Poll not found") + const dto = await findRoomMessage(roomKind, roomId, messageId, viewerUserId) + if (dto === null) throw pollNotFound() return dto } @@ -87,20 +109,35 @@ export function makeChatPollService(deps: ChatPollServiceDeps): ChatPollService ): Promise<{ roomKind: PollRoomKind; roomId: string }> { const meta = await deps.chat.findMessageMeta(messageId) if (meta === null || meta.kind !== "poll" || meta.deletedAt !== null) { - throw AppError.notFound("Poll not found") + throw pollNotFound() } if (meta.reportId !== null) return { roomKind: "report", roomId: meta.reportId } if (meta.groupId !== null) return { roomKind: "group", roomId: meta.groupId } return { roomKind: "cleanup", roomId: meta.cleanupId! } } + /** Everyone else gets the viewer-neutral read; the caller gets their own vote state back. */ + async function rereadAndBroadcastUpdate( + roomKind: PollRoomKind, + roomId: string, + messageId: string, + userId: string, + ): Promise { + const [message, roomView] = await Promise.all([ + readMessage(roomKind, roomId, messageId, userId), + readMessage(roomKind, roomId, messageId, null), + ]) + deps.broadcastUpdate(roomKind, roomId, roomView) + return message + } + return { async createPoll(input: CreatePollInput): Promise { const { roomKind, roomId, userId } = input await requireVisibleRoom(roomKind, roomId, userId) if (!(await deps.canSend(roomKind, roomId, userId))) { throw new AppError(ErrorCode.FORBIDDEN, "You can't create a poll in this chat.", { - fields: { code: "poll_forbidden" }, + fields: { code: POLL_ERROR_CODE.forbidden }, }) } assertNoSlur(input.question, "question") @@ -131,11 +168,11 @@ export function makeChatPollService(deps: ChatPollServiceDeps): ChatPollService await requireVisibleRoom(roomKind, roomId, userId) if (!(await deps.isMember(roomKind, roomId, userId))) { throw new AppError(ErrorCode.FORBIDDEN, "You must be a member to vote.", { - fields: { code: "poll_not_member" }, + fields: { code: POLL_ERROR_CODE.notMember }, }) } const pollMeta = await deps.chatPolls.findPollMeta(messageId) - if (pollMeta === null) throw AppError.notFound("Poll not found") + if (pollMeta === null) throw pollNotFound() if (pollMeta.closedAt !== null) throw pollClosed() if (optionIdxs.length > 1 && !pollMeta.allowMultiple) { throw AppError.validation({ optionIdxs: "This poll allows only one choice." }) @@ -145,12 +182,7 @@ export function makeChatPollService(deps: ChatPollServiceDeps): ChatPollService throw AppError.validation({ optionIdxs: "Unknown poll option." }) } await deps.chatPolls.replaceVotes(messageId, userId, optionIdxs) - const [message, roomView] = await Promise.all([ - readMessage(roomKind, roomId, messageId, userId), - readMessage(roomKind, roomId, messageId, null), - ]) - deps.broadcastUpdate(roomKind, roomId, roomView) - return message + return rereadAndBroadcastUpdate(roomKind, roomId, messageId, userId) }, async closePoll(input: ClosePollInput): Promise { @@ -158,13 +190,9 @@ export function makeChatPollService(deps: ChatPollServiceDeps): ChatPollService const { roomKind, roomId } = await resolvePollRoom(messageId) await requireVisibleRoom(roomKind, roomId, userId) const pollMeta = await deps.chatPolls.findPollMeta(messageId) - if (pollMeta === null) throw AppError.notFound("Poll not found") + if (pollMeta === null) throw pollNotFound() const isAuthor = pollMeta.createdBy === userId const isModerator = await deps.isModerator(roomKind, roomId, userId) - const pollCloseForbidden = (): AppError => - new AppError(ErrorCode.FORBIDDEN, "You can't close this poll.", { - fields: { code: "poll_close_forbidden" }, - }) if (!(await deps.isMember(roomKind, roomId, userId)) && !isModerator) { throw pollCloseForbidden() } @@ -172,12 +200,7 @@ export function makeChatPollService(deps: ChatPollServiceDeps): ChatPollService throw pollCloseForbidden() } await deps.chatPolls.close(messageId) - const [message, roomView] = await Promise.all([ - readMessage(roomKind, roomId, messageId, userId), - readMessage(roomKind, roomId, messageId, null), - ]) - deps.broadcastUpdate(roomKind, roomId, roomView) - return message + return rereadAndBroadcastUpdate(roomKind, roomId, messageId, userId) }, } } diff --git a/services/api/src/services/chat-reaction-service.ts b/services/api/src/services/chat-reaction-service.ts index 1fa79a4d..087f49a1 100644 --- a/services/api/src/services/chat-reaction-service.ts +++ b/services/api/src/services/chat-reaction-service.ts @@ -13,13 +13,13 @@ import type { ChatMessageDTO, ReactionEmoji } from "@civfix/shared" import type { ChatMessageMeta, ChatRepository } from "./chat-repository.drizzle.js" import type { DmRepository } from "./dm-repository.drizzle.js" -export const CHAT_REACTION_FORBIDDEN = "You can't react in this conversation." +const CHAT_REACTION_FORBIDDEN = "You can't react in this conversation." /** * Report chat is view-only until you Join, which is actionable, so the refusal says how to fix it. It * lives beside the gate so the legacy and unified routes never answer the same refusal differently. */ -export const REPORT_CHAT_REACTION_FORBIDDEN = "Join the chat to react to messages." +const REPORT_CHAT_REACTION_FORBIDDEN = "Join the chat to react to messages." export type IsCleanupMemberFn = (cleanupId: string, userId: string) => Promise diff --git a/services/api/src/services/chat-reply-hydration.ts b/services/api/src/services/chat-reply-hydration.ts index b0e7b2f7..7d249831 100644 --- a/services/api/src/services/chat-reply-hydration.ts +++ b/services/api/src/services/chat-reply-hydration.ts @@ -24,7 +24,7 @@ export interface ReplyRoomScope { export const REPLY_EXCERPT_MAX = 120 -export interface ReplyTargetRow { +interface ReplyTargetRow { id: string room_ref?: string | null body: string | null @@ -48,21 +48,21 @@ export const replyDeletedTarget = (): AppError => /** A bar-chart glyph so a quoted poll reads as a poll. */ const POLL_EXCERPT_PREFIX = "\u{1F4CA} " -export function toReplyToDTO(row: ReplyTargetRow): ReplyToDTO { +function liveExcerpt(row: Pick): string { + const text = row.kind === "poll" ? POLL_EXCERPT_PREFIX + (row.body ?? "") : (row.body ?? "") + return text.slice(0, REPLY_EXCERPT_MAX) +} + +function toReplyToDTO(row: ReplyTargetRow): ReplyToDTO { const deleted = row.deleted_at !== null const from = row.sender_id !== null && row.sender_deleted_at === null ? { id: row.sender_id, displayName: row.sender_display_name ?? "" } : null - const excerpt = deleted - ? "" - : row.kind === "poll" - ? (POLL_EXCERPT_PREFIX + (row.body ?? "")).slice(0, REPLY_EXCERPT_MAX) - : (row.body ?? "").slice(0, REPLY_EXCERPT_MAX) return { id: row.id, from, - excerpt, + excerpt: deleted ? "" : liveExcerpt(row), kind: row.kind, ...(deleted ? { deleted: true } : {}), } diff --git a/services/api/src/services/chat-repository.drizzle.ts b/services/api/src/services/chat-repository.drizzle.ts index 45876067..2d4413dd 100644 --- a/services/api/src/services/chat-repository.drizzle.ts +++ b/services/api/src/services/chat-repository.drizzle.ts @@ -194,41 +194,34 @@ interface ChatRowSelect { forwarded_to_city?: boolean } -function toMessageDTO( +interface MessageExtras { + reactions?: ReactionSummaryDTO[] + mentions?: UserMentionDTO[] + viewerUserId?: string | null + clientId?: string + attachments?: MediaDTO[] + reportCity?: ReportCityContext | null + replyTo?: ReplyToDTO | null + poll?: PollDTO | null +} + +function toMessageDTO(r: ChatRowSelect, extras: MessageExtras = {}): ChatMessageDTO { + const dto = buildMessageDTO(r, extras) + return r.deleted_at !== null ? toTombstoneDTO(dto, r.deleted_at) : dto +} + +function buildMessageDTO( r: ChatRowSelect, - reactions: ReactionSummaryDTO[], - mentions: UserMentionDTO[], - viewerUserId?: string | null, - clientId?: string, - attachments: MediaDTO[] = [], - reportCity?: ReportCityContext | null, - replyTo?: ReplyToDTO | null, - poll?: PollDTO | null, -): ChatMessageDTO { - const dto = buildMessageDTO( - r, - reactions, - mentions, + { + reactions = [], + mentions = [], viewerUserId, clientId, - attachments, + attachments = [], reportCity, replyTo, poll, - ) - return r.deleted_at !== null ? toTombstoneDTO(dto, r.deleted_at) : dto -} - -function buildMessageDTO( - r: ChatRowSelect, - reactions: ReactionSummaryDTO[], - mentions: UserMentionDTO[], - viewerUserId?: string | null, - clientId?: string, - attachments: MediaDTO[] = [], - reportCity?: ReportCityContext | null, - replyTo?: ReplyToDTO | null, - poll?: PollDTO | null, + }: MessageExtras, ): ChatMessageDTO { if (r.sender_id === null && r.report_id !== null) { return mapSystemRow({ @@ -303,6 +296,13 @@ export function cityForwardFields( return { forwardedToCity, cityMention } } +function replyFor( + row: Pick, + replyByTarget: Map, +): ReplyToDTO | null { + return row.reply_to_id !== null ? (replyByTarget.get(row.reply_to_id) ?? null) : null +} + function forwardedColumn(sql: Queryable, alias: string) { return sql`EXISTS ( SELECT 1 FROM report_message_forwards f @@ -411,17 +411,15 @@ export function makeDrizzleChatRepository(sql: Sql, presign?: PresignMedia): Cha loadPollsFor(sql, pollIds, viewerUserId), ]) return page.map((r) => - toMessageDTO( - r, - reactionsByMessage.get(r.id) ?? [], - mentionsByMessage.get(r.id) ?? [], + toMessageDTO(r, { + reactions: reactionsByMessage.get(r.id) ?? [], + mentions: mentionsByMessage.get(r.id) ?? [], viewerUserId, - undefined, - attachmentsByMessage.get(r.id) ?? [], + attachments: attachmentsByMessage.get(r.id) ?? [], reportCity, - r.reply_to_id !== null ? (replyByTarget.get(r.reply_to_id) ?? null) : null, - pollsByMessage.get(r.id) ?? null, - ), + replyTo: replyFor(r, replyByTarget), + poll: pollsByMessage.get(r.id) ?? null, + }), ) } @@ -443,17 +441,15 @@ export function makeDrizzleChatRepository(sql: Sql, presign?: PresignMedia): Cha replyMapForRows(sql, "chat_messages", [row]), loadPollsFor(sql, pollIds, viewerUserId), ]) - return toMessageDTO( - row, + return toMessageDTO(row, { reactions, mentions, viewerUserId, - undefined, - attachmentsByMessage.get(row.id) ?? [], + attachments: attachmentsByMessage.get(row.id) ?? [], reportCity, - row.reply_to_id !== null ? (replyByTarget.get(row.reply_to_id) ?? null) : null, - pollsByMessage.get(row.id) ?? null, - ) + replyTo: replyFor(row, replyByTarget), + poll: pollsByMessage.get(row.id) ?? null, + }) } function roomSql(scope: RoomScope): RoomScopeSql { @@ -554,16 +550,11 @@ export function makeDrizzleChatRepository(sql: Sql, presign?: PresignMedia): Cha const row = rows[0] if (!row) return null const replyByTarget = await replyMapForRows(sql, "chat_messages", [row]) - return toMessageDTO( - row, - [], - [], - senderId, - undefined, - [], + return toMessageDTO(row, { + viewerUserId: senderId, reportCity, - row.reply_to_id !== null ? (replyByTarget.get(row.reply_to_id) ?? null) : null, - ) + replyTo: replyFor(row, replyByTarget), + }) } return { @@ -628,16 +619,13 @@ export function makeDrizzleChatRepository(sql: Sql, presign?: PresignMedia): Cha const attachments = wantsMedia ? ((await loadChatAttachments(sql, [id], presign!, input.userId)).get(id) ?? []) : [] - return toMessageDTO( - rows[0]!, - [], - [], - input.userId, - input.clientId, + return toMessageDTO(rows[0]!, { + viewerUserId: input.userId, + clientId: input.clientId, attachments, reportCity, replyTo, - ) + }) }, async findMessageMeta(messageId: string): Promise { diff --git a/services/api/src/services/chat-room-fanout-notifier.ts b/services/api/src/services/chat-room-fanout-notifier.ts index ad891afd..40379b96 100644 --- a/services/api/src/services/chat-room-fanout-notifier.ts +++ b/services/api/src/services/chat-room-fanout-notifier.ts @@ -27,6 +27,8 @@ export const ROOM_FANOUT_THROTTLE_MS = 15 * 1000 const FANOUT_MARKER_SWEEP_THRESHOLD = 5000 +const NO_PREVIEW_BODY_KEY = "notification.message.no_preview" satisfies MessageKey + export interface RoomFanoutNotifierDeps { notificationService: Pick listMemberIds: (roomId: string, limit: number) => Promise @@ -138,6 +140,20 @@ async function mutedIds( return new Set(candidates.filter((_, i) => verdicts[i] === true)) } +async function presentIds( + deps: RoomFanoutNotifierDeps, + kind: RoomFanoutKind, + roomId: string, +): Promise { + if (!deps.presence) return [] + try { + return await deps.presence.online(deps.roomKey(roomId)) + } catch (err) { + deps.logger?.warn({ err, kind }, "room fan-out presence lookup failed") + return [] + } +} + export function makeRoomFanoutNotifier( spec: RoomFanoutSpec, deps: RoomFanoutNotifierDeps, @@ -198,16 +214,7 @@ export async function runRoomFanout( const cap = MEMBER_CAP_BY_KIND[spec.kind] const memberIds = (await deps.listMemberIds(roomId, cap)).slice(0, cap) - let present: string[] = [] - if (deps.presence) { - try { - present = await deps.presence.online(deps.roomKey(roomId)) - } catch (err) { - deps.logger?.warn({ err, kind: spec.kind }, "room fan-out presence lookup failed") - present = [] - } - } - const presentSet = new Set(present) + const presentSet = new Set(await presentIds(deps, spec.kind, roomId)) const replyTargetId = message.replyTo?.from?.id ?? null const mentionedIds = new Set(message.mentions.map((m) => m.id)) @@ -232,7 +239,7 @@ export async function runRoomFanout( { type: bell.type, ...(name !== null ? { title: name } : { titleKey: spec.titleFallbackKey }), - ...(preview !== null ? { body: preview } : { bodyKey: "notification.message.no_preview" }), + ...(preview !== null ? { body: preview } : { bodyKey: NO_PREVIEW_BODY_KEY }), link: bell.link(roomId), coalesceWindowMs, }, diff --git a/services/api/src/services/chat-room-notifier-wiring.ts b/services/api/src/services/chat-room-notifier-wiring.ts index 1a99841f..f1824d53 100644 --- a/services/api/src/services/chat-room-notifier-wiring.ts +++ b/services/api/src/services/chat-room-notifier-wiring.ts @@ -17,6 +17,8 @@ export type RoomFanoutLogger = Pick export const ROOM_FANOUT_WINDOW_CLAIM_PREFIX = "chatfanout" +const MS_PER_SECOND = 1000 + export type ContainerRoomFanoutDeps = Record export function makeContainerRoomFanoutDeps( @@ -89,7 +91,7 @@ export function makeWindowClaim( logger?: RoomFanoutLogger, ): (kind: RoomFanoutKind, roomId: string, windowMs: number) => Promise { return async (kind, roomId, windowMs) => { - const ttlSeconds = Math.max(1, Math.ceil((windowMs || ROOM_FANOUT_THROTTLE_MS) / 1000)) + const ttlSeconds = Math.max(1, Math.ceil((windowMs || ROOM_FANOUT_THROTTLE_MS) / MS_PER_SECOND)) try { const hits = await container .getCounterStore() diff --git a/services/api/src/services/chat-room-scope.drizzle.ts b/services/api/src/services/chat-room-scope.drizzle.ts index bc24cccb..7a792deb 100644 --- a/services/api/src/services/chat-room-scope.drizzle.ts +++ b/services/api/src/services/chat-room-scope.drizzle.ts @@ -27,6 +27,8 @@ export type RoomAlias = "cm" | "dm" export const PIN_LIST_CAP = 25 +const MESSAGE_NOT_FOUND = "Message not found" + /** Everything else about the row shape belongs to the owning repository's hydrator. */ export interface RoomScopeRow { id: string @@ -129,13 +131,13 @@ export async function roomHistoryAround( viewerUserId: string | null, ): Promise { // A non-uuid id can never match; the short-circuit avoids a 22P02 cast error (a 500). - if (!isUuid(around)) throw AppError.notFound("Message not found") + if (!isUuid(around)) throw AppError.notFound(MESSAGE_NOT_FOUND) const anchorRows = await sql<{ id: string }[]>` SELECT id FROM ${sql(spec.table)} WHERE id = ${around} AND ${spec.scope(null)} LIMIT 1 ` - if (!anchorRows[0]) throw AppError.notFound("Message not found") + if (!anchorRows[0]) throw AppError.notFound(MESSAGE_NOT_FOUND) const anchorTuple = sql`( SELECT a.created_at, a.id FROM ${sql(spec.table)} a diff --git a/services/api/src/services/cleanup-address.ts b/services/api/src/services/cleanup-address.ts new file mode 100644 index 00000000..7ec66a7d --- /dev/null +++ b/services/api/src/services/cleanup-address.ts @@ -0,0 +1,96 @@ +import { AppError, MAX_EVENT_ADDRESS_LENGTH, isLocatedPrecision } from "@civfix/shared" +import type { EventAddressSource } from "@civfix/shared" +import type { AddressResolver } from "./address-resolver.js" + +/** + * Floor for a host-confirmed event address. Long enough to reject the accidental keystroke and the + * lone punctuation mark, short enough to allow a genuinely terse one ("Pier 3"). + */ +const MIN_EVENT_ADDRESS_LENGTH = 3 + +export interface EventAddressWrite { + address: string | null + addressSource: EventAddressSource | null +} + +const NO_EVENT_ADDRESS: EventAddressWrite = { address: null, addressSource: null } + +/** + * The event address, with its provenance, for a create or an update. + * + * `addressSource` is the CLIENT-VERSION discriminator, and it has to be: `address` itself stays + * optional on the wire so the TestFlight build in someone's pocket keeps working. + * + * addressSource PRESENT -> a new client. It resolved the pin, showed the line to the host, and the + * host published with it on screen. That is the confirmation, so the + * server only has to refuse a blank one; a client that sends a source + * without an address has a bug, and storing it would produce an event + * whose address is "verified" and empty. + * addressSource ABSENT -> an old client. Whatever it sent in `address` is the host's own "name the + * spot" text, so it is 'manual' (the same call migration 0179 makes for + * existing rows). If it sent nothing, the shim resolves the pin and stores + * 'resolved': unverified, but an event with a street line beats an event + * with "Meeting point", and only while old clients are still in the wild. + * + * The shim stores NOTHING when the ladder only reached `locality`: "Los Angeles, CA" is not a meeting + * address, and writing it would dress up a non-answer as a host-provided one. + * + * `fromStoredEvent` marks the DUPLICATE path, whose pair did not come off the wire at all: it is this + * server's own stored row, copied verbatim. The new-client length floor is a check on a client payload + * and would reject a backfilled one-or-two-character address that the host has been running for + * months. Slur checks still apply - they run over the whole input before this. + */ +export async function resolveEventAddress( + input: { + address?: string | undefined + addressSource?: EventAddressSource | undefined + lat: number + lng: number + }, + resolveAddress: AddressResolver | undefined, + opts?: { fromStoredEvent?: boolean }, +): Promise { + if (input.addressSource !== undefined) { + if (opts?.fromStoredEvent === true && input.address !== undefined) { + return { address: input.address, addressSource: input.addressSource } + } + return { address: assertConfirmedAddress(input.address), addressSource: input.addressSource } + } + const typed = input.address?.trim() ?? "" + if (typed.length > 0) return { address: typed, addressSource: "manual" } + if (resolveAddress === undefined) return { ...NO_EVENT_ADDRESS } + const resolved = await resolveAddress(input.lat, input.lng) + if (resolved.address === null || !isLocatedPrecision(resolved.precision)) { + return { ...NO_EVENT_ADDRESS } + } + return { + address: resolved.address.slice(0, MAX_EVENT_ADDRESS_LENGTH), + addressSource: "resolved", + } +} + +/** A new client that names a source must carry a real line with it. */ +function assertConfirmedAddress(address: string | undefined): string { + const trimmed = address?.trim() ?? "" + if (trimmed.length < MIN_EVENT_ADDRESS_LENGTH) { + throw AppError.validation({ + address: `must be at least ${MIN_EVENT_ADDRESS_LENGTH} characters`, + }) + } + return trimmed +} + +/** The update-path twin of resolveEventAddress: same rules, but every field stays optional. */ +export function eventAddressPatch(patch: { + address?: string | undefined + addressSource?: EventAddressSource | undefined +}): { address?: string | null; addressSource?: EventAddressSource | null } { + if (patch.addressSource !== undefined) { + return { address: assertConfirmedAddress(patch.address), addressSource: patch.addressSource } + } + if (patch.address === undefined) return {} + const trimmed = patch.address.trim() + return trimmed.length > 0 + ? { address: trimmed, addressSource: "manual" } + : { ...NO_EVENT_ADDRESS } +} diff --git a/services/api/src/services/cleanup-dto.ts b/services/api/src/services/cleanup-dto.ts index acc8c510..af47b4e5 100644 --- a/services/api/src/services/cleanup-dto.ts +++ b/services/api/src/services/cleanup-dto.ts @@ -26,8 +26,6 @@ export const ATTENDEES_DEFAULT_LIMIT = 50 export const THREAD_SIGNAL_MEMBER_CAP = 500 -export { MAX_LINKED_REPORTS } from "@civfix/shared" - export const LINKED_REPORTS_LIST_PREVIEW = 6 export function toAttendeePersonDTO(view: CleanupPersonView, isFollowing: boolean): PersonDTO { @@ -47,7 +45,7 @@ export function toAttendeePersonDTO(view: CleanupPersonView, isFollowing: boolea } } -export function toOrganizerPerson(view: CleanupPersonView): PersonDTO { +function toOrganizerPerson(view: CleanupPersonView): PersonDTO { return toAttendeePersonDTO(view, false) } diff --git a/services/api/src/services/cleanup-enrichment.ts b/services/api/src/services/cleanup-enrichment.ts index 709ddbe8..8ce067ad 100644 --- a/services/api/src/services/cleanup-enrichment.ts +++ b/services/api/src/services/cleanup-enrichment.ts @@ -3,7 +3,7 @@ import type { Container } from "../di.js" import { attachRegistrationFields } from "./host/registration-dto.js" import { withAffiliation } from "./affiliation.js" -export async function attachOrganizerAffiliations( +async function attachOrganizerAffiliations( container: Container, dtos: CleanupDTO[], viewerUserId: string | null, diff --git a/services/api/src/services/cleanup-jobs.ts b/services/api/src/services/cleanup-jobs.ts index f5145ba0..96e3425f 100644 --- a/services/api/src/services/cleanup-jobs.ts +++ b/services/api/src/services/cleanup-jobs.ts @@ -4,13 +4,8 @@ import { makeDrizzleCleanupRepository } from "./cleanup-repository.drizzle.js" import { makeRouteNotificationService } from "./route-notifier.js" import { makeCommsRuntime } from "./host/comms-wiring.js" import { makeContainerGuestRsvpService } from "./guest-rsvp-wiring.js" -import { - CLEANUP_CANCEL_FANOUT_JOB, - makeCleanupService, - type CleanupCancelFanoutJob, -} from "./cleanup-service.js" - -export { CLEANUP_CANCEL_FANOUT_JOB } +import { makeCleanupService } from "./cleanup-service.js" +import { CLEANUP_CANCEL_FANOUT_JOB, type CleanupCancelFanoutJob } from "./cleanup-notifications.js" export async function registerCleanupCancelFanoutJob( container: Container, diff --git a/services/api/src/services/cleanup-notifications.ts b/services/api/src/services/cleanup-notifications.ts new file mode 100644 index 00000000..efeea120 --- /dev/null +++ b/services/api/src/services/cleanup-notifications.ts @@ -0,0 +1,266 @@ +import type { Jobs } from "@civfix/shared/interfaces" +import type { MessageKey } from "../i18n/messages/en.js" +import type { NotificationService } from "./notification-service.js" +import { mapWithLimit } from "./media-presign.js" +import type { CleanupRepository, SlotReconcileResult } from "./cleanup-repository.types.js" +import { CLEANUP_GUEST_UPDATE_FANOUT_JOB, type GuestUpdateFanoutJob } from "./guest-rsvp-service.js" + +const MS_PER_HOUR = 60 * 60 * 1000 + +export const CANCEL_FANOUT_MEMBER_CAP = 2000 + +const CANCEL_FANOUT_CONCURRENCY = 8 + +export const CLEANUP_CANCEL_FANOUT_JOB = "cleanup.cancel.fanout" + +const CANCEL_FANOUT_DEDUPE_WINDOW_MS = MS_PER_HOUR + +export interface CleanupCancelFanoutJob { + cleanupId: string + reason: string | null + actorId: string +} + +interface SlotFanoutBudget { + remaining: number +} + +export interface NotifiedCleanup { + id: string + title: string +} + +export interface CleanupNotificationsDeps { + repo: Pick + notifier?: Pick + attendeeNotifier?: { eventCancelled(cleanupId: string, reason: string | null): Promise } + jobs?: Jobs + logger?: { + warn(obj: unknown, msg?: string): void + error(obj: unknown, msg?: string): void + } +} + +export interface CleanupNotifications { + notifyRoleChange( + targetUserId: string, + event: "promoted" | "demoted" | "removed", + cleanup: NotifiedCleanup, + ): Promise + notifyCancellation( + cleanup: NotifiedCleanup, + reason: string | null, + actorId: string, + ): Promise + dispatchCancelFanout( + cleanup: NotifiedCleanup, + reason: string | null, + actorId: string, + ): Promise + dispatchGuestUpdateFanout(cleanupId: string): Promise + notifySlotChanges(cleanup: NotifiedCleanup, diff: SlotReconcileResult): Promise +} + +export function makeCleanupNotifications(deps: CleanupNotificationsDeps): CleanupNotifications { + async function notifyRoleChange( + targetUserId: string, + event: "promoted" | "demoted" | "removed", + cleanup: NotifiedCleanup, + ): Promise { + if (deps.notifier === undefined) return + try { + await deps.notifier.createNotification(targetUserId, { + type: "cleanup_role", + titleKey: `notification.cleanup_role.${event}.title`, + bodyKey: `notification.cleanup_role.${event}.body`, + vars: { title: cleanup.title }, + link: `/cleanups/${cleanup.id}`, + }) + } catch (err) { + deps.logger?.warn( + { err, targetUserId, cleanupId: cleanup.id, event }, + "cleanup_role notification failed (suppressed)", + ) + } + } + + async function notifyCancellation( + cleanup: NotifiedCleanup, + reason: string | null, + actorId: string, + ): Promise { + await notifyMembersOfCancellation(cleanup, reason, actorId) + await notifyAttendeesOfCancellation(cleanup.id, reason) + } + + async function notifyAttendeesOfCancellation( + cleanupId: string, + reason: string | null, + ): Promise { + if (deps.attendeeNotifier === undefined) return + await deps.attendeeNotifier.eventCancelled(cleanupId, reason) + } + + async function dispatchGuestUpdateFanout(cleanupId: string): Promise { + if (deps.jobs === undefined) { + deps.logger?.warn( + { cleanupId }, + "cleanup.guest.update.fanout: no job queue wired; guests are not notified", + ) + return + } + try { + await deps.jobs.enqueue( + CLEANUP_GUEST_UPDATE_FANOUT_JOB, + { cleanupId } satisfies GuestUpdateFanoutJob, + { singletonKey: cleanupId }, + ) + } catch (err) { + deps.logger?.error( + { err, cleanupId }, + "cleanup.guest.update.fanout enqueue failed; guests are not notified", + ) + } + } + + async function notifyMembersOfCancellation( + cleanup: NotifiedCleanup, + reason: string | null, + actorId: string, + ): Promise { + const notifier = deps.notifier + if (notifier === undefined) return + let memberIds: string[] + try { + memberIds = await deps.repo.listMemberIds(cleanup.id, CANCEL_FANOUT_MEMBER_CAP) + } catch (err) { + deps.logger?.warn( + { err, cleanupId: cleanup.id }, + "cleanup_cancelled roster read failed (suppressed)", + ) + return + } + const recipients = memberIds.filter((userId) => userId !== actorId) + await mapWithLimit(recipients, CANCEL_FANOUT_CONCURRENCY, async (userId) => { + try { + await notifier.createNotification(userId, { + type: "cleanup_cancelled", + titleKey: "notification.cleanup_cancelled.title", + bodyKey: + reason !== null + ? "notification.cleanup_cancelled.body_reason" + : "notification.cleanup_cancelled.body", + ...(reason !== null ? { vars: { reason } } : {}), + link: `/cleanups/${cleanup.id}`, + dedupeWindowMs: CANCEL_FANOUT_DEDUPE_WINDOW_MS, + }) + } catch (err) { + deps.logger?.warn( + { err, cleanupId: cleanup.id, userId }, + "cleanup_cancelled notification failed (suppressed)", + ) + } + }) + } + + async function dispatchCancelFanout( + cleanup: NotifiedCleanup, + reason: string | null, + actorId: string, + ): Promise { + if (deps.jobs !== undefined) { + try { + await deps.jobs.enqueue( + CLEANUP_CANCEL_FANOUT_JOB, + { cleanupId: cleanup.id, reason, actorId } satisfies CleanupCancelFanoutJob, + { singletonKey: cleanup.id }, + ) + return + } catch (err) { + deps.logger?.error( + { err, cleanupId: cleanup.id }, + "cleanup.cancel.fanout enqueue failed; ringing members inline, guests are not notified", + ) + } + } + try { + await notifyMembersOfCancellation(cleanup, reason, actorId) + } catch (err) { + deps.logger?.warn( + { err, cleanupId: cleanup.id }, + "cleanup_cancelled inline member fanout failed (suppressed; the cancellation itself stands)", + ) + } + } + + async function notifySlotClaimants( + cleanup: NotifiedCleanup, + entries: SlotReconcileResult["removed"], + keys: { titleKey: MessageKey; bodyKey: MessageKey }, + budget: SlotFanoutBudget, + ): Promise { + const notifier = deps.notifier + if (notifier === undefined) return + const targets: { userId: string; slot: string }[] = [] + for (const entry of entries) { + for (const userId of entry.claimantUserIds) { + if (budget.remaining <= 0) break + budget.remaining -= 1 + targets.push({ userId, slot: entry.title }) + } + } + await mapWithLimit(targets, CANCEL_FANOUT_CONCURRENCY, async ({ userId, slot }) => { + try { + await notifier.createNotification(userId, { + type: "cleanup_slot", + titleKey: keys.titleKey, + bodyKey: keys.bodyKey, + vars: { slot, title: cleanup.title }, + link: `/cleanups/${cleanup.id}`, + }) + } catch (err) { + deps.logger?.warn( + { err, cleanupId: cleanup.id, userId }, + "cleanup_slot notification failed (suppressed)", + ) + } + }) + } + + async function notifySlotChanges( + cleanup: NotifiedCleanup, + diff: SlotReconcileResult, + ): Promise { + const budget: SlotFanoutBudget = { remaining: CANCEL_FANOUT_MEMBER_CAP } + if (diff.removed.length > 0) { + await notifySlotClaimants( + cleanup, + diff.removed, + { + titleKey: "notification.cleanup_slot.removed.title", + bodyKey: "notification.cleanup_slot.removed.body", + }, + budget, + ) + } + if (diff.rescheduled.length > 0) { + await notifySlotClaimants( + cleanup, + diff.rescheduled, + { + titleKey: "notification.cleanup_slot.moved.title", + bodyKey: "notification.cleanup_slot.moved.body", + }, + budget, + ) + } + } + + return { + notifyRoleChange, + notifyCancellation, + dispatchCancelFanout, + dispatchGuestUpdateFanout, + notifySlotChanges, + } +} diff --git a/services/api/src/services/cleanup-repository.drizzle.ts b/services/api/src/services/cleanup-repository.drizzle.ts index 61852d50..2d1e3711 100644 --- a/services/api/src/services/cleanup-repository.drizzle.ts +++ b/services/api/src/services/cleanup-repository.drizzle.ts @@ -93,6 +93,8 @@ export const MAX_EVENTS_PER_REPORT = 50 const SLOT_TITLE_INDEX = "cleanup_slots_cleanup_title_window_uidx" +const SLOT_TITLE_DETAIL_MARKER = "lower(title)" + export interface SlotIdentity { title: string startsAt: Date | null @@ -113,7 +115,7 @@ function isSlotTitleConflict(err: unknown): boolean { if (e.code !== PG_UNIQUE_VIOLATION) return false const constraint = typeof e.constraint_name === "string" ? e.constraint_name : "" const detail = typeof e.detail === "string" ? e.detail : "" - return constraint === SLOT_TITLE_INDEX || detail.includes("lower(title)") + return constraint === SLOT_TITLE_INDEX || detail.includes(SLOT_TITLE_DETAIL_MARKER) } async function claimEventMediaInTx( @@ -285,6 +287,85 @@ async function privateEventBlocksJoin( return standing.length === 0 } +type JoinRefusal = "not_found" | "closed" | "ended" | "banned" + +// Takes the FOR SHARE lock that holds the event's status and window steady until the caller commits. +async function lockJoinableEvent( + tx: Queryable, + cleanupId: string, + userId: string, +): Promise<{ refusal: JoinRefusal } | { refusal: null; now: Date }> { + const locked = await tx< + { + status: CleanupStatus + visibility: EventVisibility + organization_id: string | null + scheduled_at: Date + ends_at: Date | null + now: Date + }[] + >` + SELECT status, visibility, organization_id, scheduled_at, ends_at, now() AS now + FROM cleanups + WHERE id = ${cleanupId} LIMIT 1 FOR SHARE + ` + const cleanup = locked[0] + if (cleanup === undefined) return { refusal: "not_found" } + if ( + await privateEventBlocksJoin(tx, cleanupId, cleanup.visibility, cleanup.organization_id, userId) + ) { + return { refusal: "not_found" } + } + if (cleanup.status === "cancelled") return { refusal: "closed" } + if (hasEventEnded(eventWindowOfRow(cleanup), cleanup.now.getTime())) return { refusal: "ended" } + const banned = await tx<{ one: number }[]>` + SELECT 1 AS one FROM cleanup_bans + WHERE cleanup_id = ${cleanupId} AND user_id = ${userId} + LIMIT 1 + ` + if (banned.length > 0) return { refusal: "banned" } + return { refusal: null, now: cleanup.now } +} + +async function enrolMember( + tx: Queryable, + cleanupId: string, + userId: string, + seat: SignupSeat, + now: Date, +): Promise { + await tx` + INSERT INTO cleanup_members (cleanup_id, user_id, role) + VALUES (${cleanupId}, ${userId}, 'member') + ON CONFLICT (cleanup_id, user_id) DO NOTHING + ` + await ensureSignupRegistrationIn(tx, { + cleanupId, + userId, + seatId: seat.seatId, + tokenHash: seat.tokenHash, + now, + }) +} + +function timeCursorOrder( + sql: Queryable, + past: boolean, + rawCursor: string | null | undefined, +): { cursorFilter: postgres.Fragment; order: postgres.Fragment } { + const cursor = parseTimeCursor(rawCursor) + const cursorFilter = + cursor !== null + ? past + ? sql`AND (c.scheduled_at, c.id) < (${cursor.at}, ${cursor.id}::uuid)` + : sql`AND (c.scheduled_at, c.id) > (${cursor.at}, ${cursor.id}::uuid)` + : sql`` + const order = past + ? sql`ORDER BY c.scheduled_at DESC, c.id DESC` + : sql`ORDER BY c.scheduled_at ASC, c.id ASC` + return { cursorFilter, order } +} + function hostSetFragments(sql: Queryable, patch: EventHostWrite): postgres.Fragment[] { const sets: postgres.Fragment[] = [] if (patch.endsAt !== undefined) sets.push(sql`ends_at = ${patch.endsAt}`) @@ -958,17 +1039,7 @@ export function makeDrizzleCleanupRepository(sql: Sql): CleanupRepository { ) } - const past = filters.when === "past" - const cursor = parseTimeCursor(filters.cursor) - const cursorFilter = - cursor !== null - ? past - ? sql`AND (c.scheduled_at, c.id) < (${cursor.at}, ${cursor.id}::uuid)` - : sql`AND (c.scheduled_at, c.id) > (${cursor.at}, ${cursor.id}::uuid)` - : sql`` - const order = past - ? sql`ORDER BY c.scheduled_at DESC, c.id DESC` - : sql`ORDER BY c.scheduled_at ASC, c.id ASC` + const { cursorFilter, order } = timeCursorOrder(sql, filters.when === "past", filters.cursor) const rows = await sql` SELECT ${cleanupColumns(sql, null)} FROM cleanups c @@ -992,17 +1063,7 @@ export function makeDrizzleCleanupRepository(sql: Sql): CleanupRepository { async listOrganizationEvents( filters: OrganizationEventsFilters, ): Promise<{ records: CleanupRecord[]; nextCursor: string | null }> { - const past = filters.when === "past" - const cursor = parseTimeCursor(filters.cursor) - const cursorFilter = - cursor !== null - ? past - ? sql`AND (c.scheduled_at, c.id) < (${cursor.at}, ${cursor.id}::uuid)` - : sql`AND (c.scheduled_at, c.id) > (${cursor.at}, ${cursor.id}::uuid)` - : sql`` - const order = past - ? sql`ORDER BY c.scheduled_at DESC, c.id DESC` - : sql`ORDER BY c.scheduled_at ASC, c.id ASC` + const { cursorFilter, order } = timeCursorOrder(sql, filters.when === "past", filters.cursor) const rows = await sql` SELECT ${cleanupColumns(sql, null)} FROM cleanups c @@ -1151,53 +1212,9 @@ export function makeDrizzleCleanupRepository(sql: Sql): CleanupRepository { seat: SignupSeat, ): Promise { return sql.begin(async (tx) => { - const locked = await tx< - { - status: CleanupStatus - visibility: EventVisibility - organization_id: string | null - scheduled_at: Date - ends_at: Date | null - now: Date - }[] - >` - SELECT status, visibility, organization_id, scheduled_at, ends_at, now() AS now - FROM cleanups - WHERE id = ${cleanupId} LIMIT 1 FOR SHARE - ` - const cleanup = locked[0] - if (cleanup === undefined) return "not_found" - if ( - await privateEventBlocksJoin( - tx, - cleanupId, - cleanup.visibility, - cleanup.organization_id, - userId, - ) - ) { - return "not_found" - } - if (cleanup.status === "cancelled") return "closed" - if (hasEventEnded(eventWindowOfRow(cleanup), cleanup.now.getTime())) return "ended" - const banned = await tx<{ one: number }[]>` - SELECT 1 AS one FROM cleanup_bans - WHERE cleanup_id = ${cleanupId} AND user_id = ${userId} - LIMIT 1 - ` - if (banned.length > 0) return "banned" - await tx` - INSERT INTO cleanup_members (cleanup_id, user_id, role) - VALUES (${cleanupId}, ${userId}, 'member') - ON CONFLICT (cleanup_id, user_id) DO NOTHING - ` - await ensureSignupRegistrationIn(tx, { - cleanupId, - userId, - seatId: seat.seatId, - tokenHash: seat.tokenHash, - now: cleanup.now, - }) + const gate = await lockJoinableEvent(tx, cleanupId, userId) + if (gate.refusal !== null) return gate.refusal + await enrolMember(tx, cleanupId, userId, seat, gate.now) return "joined" }) }, @@ -1359,43 +1376,8 @@ export function makeDrizzleCleanupRepository(sql: Sql): CleanupRepository { seat: SignupSeat, ): Promise { return sql.begin(async (tx) => { - const locked = await tx< - { - status: CleanupStatus - visibility: EventVisibility - organization_id: string | null - scheduled_at: Date - ends_at: Date | null - now: Date - }[] - >` - SELECT status, visibility, organization_id, scheduled_at, ends_at, now() AS now - FROM cleanups - WHERE id = ${cleanupId} LIMIT 1 FOR SHARE - ` - const cleanup = locked[0] - if (cleanup === undefined) return { kind: "not_found" } - if ( - await privateEventBlocksJoin( - tx, - cleanupId, - cleanup.visibility, - cleanup.organization_id, - userId, - ) - ) { - return { kind: "not_found" } - } - if (cleanup.status === "cancelled") return { kind: "closed" } - if (hasEventEnded(eventWindowOfRow(cleanup), cleanup.now.getTime())) - return { kind: "ended" } - - const banned = await tx<{ one: number }[]>` - SELECT 1 AS one FROM cleanup_bans - WHERE cleanup_id = ${cleanupId} AND user_id = ${userId} - LIMIT 1 - ` - if (banned.length > 0) return { kind: "banned" } + const gate = await lockJoinableEvent(tx, cleanupId, userId) + if (gate.refusal !== null) return { kind: gate.refusal } const mine = await tx<{ slot_id: string }[]>` SELECT slot_id FROM cleanup_slot_claims @@ -1422,19 +1404,7 @@ export function makeDrizzleCleanupRepository(sql: Sql): CleanupRepository { if ((counted[0]?.n ?? 0) >= slot.capacity) return { kind: "full" } } - await tx` - INSERT INTO cleanup_members (cleanup_id, user_id, role) - VALUES (${cleanupId}, ${userId}, 'member') - ON CONFLICT (cleanup_id, user_id) DO NOTHING - ` - - await ensureSignupRegistrationIn(tx, { - cleanupId, - userId, - seatId: seat.seatId, - tokenHash: seat.tokenHash, - now: cleanup.now, - }) + await enrolMember(tx, cleanupId, userId, seat, gate.now) await tx` INSERT INTO cleanup_slot_claims (cleanup_id, user_id, slot_id) @@ -1619,41 +1589,36 @@ async function linkReportsInTx( return newlyLinked } -async function reconcileSlotsInTx( +type KeptSlot = DesiredSlot & { id: string } + +function claimantsBySlot( + claimants: readonly { slot_id: string; user_id: string }[], + excludeUserId: string | null, +): Map { + const bySlot = new Map() + for (const c of claimants) { + if (c.user_id === excludeUserId) continue + const list = bySlot.get(c.slot_id) + if (list) list.push(c.user_id) + else bySlot.set(c.slot_id, [c.user_id]) + } + return bySlot +} + +async function removeSlotsInTx( tx: Queryable, cleanupId: string, - desired: DesiredSlot[], + have: ReadonlyMap, + toRemove: string[], actorId: string | null, -): Promise { - const existing = await tx< - { id: string; title: string; starts_at: Date | null; ends_at: Date | null }[] - >` - SELECT id, title, starts_at, ends_at FROM cleanup_slots WHERE cleanup_id = ${cleanupId} - ` - const have = new Map( - existing.map((r) => [r.id, { title: r.title, startsAt: r.starts_at, endsAt: r.ends_at }]), - ) - - for (const slot of desired) { - if (slot.id !== undefined && !have.has(slot.id)) { - throw AppError.validation({ slots: `unknown slot: ${slot.id}` }) - } - } - - const keep = new Set(desired.map((s) => s.id).filter((id): id is string => id !== undefined)) - const toRemove = [...have.keys()].filter((id) => !keep.has(id)) +): Promise { const removed: SlotReconcileResult["removed"] = [] if (toRemove.length > 0) { const claimants = await tx<{ slot_id: string; user_id: string }[]>` SELECT slot_id, user_id FROM cleanup_slot_claims WHERE cleanup_id = ${cleanupId} AND slot_id = ANY(${toRemove}::uuid[]) ` - const bySlot = new Map() - for (const c of claimants) { - const list = bySlot.get(c.slot_id) - if (list) list.push(c.user_id) - else bySlot.set(c.slot_id, [c.user_id]) - } + const bySlot = claimantsBySlot(claimants, actorId) await tx` DELETE FROM cleanup_slots WHERE cleanup_id = ${cleanupId} AND id = ANY(${toRemove}::uuid[]) @@ -1662,48 +1627,52 @@ async function reconcileSlotsInTx( removed.push({ slotId, title: have.get(slotId)?.title ?? "", - claimantUserIds: (bySlot.get(slotId) ?? []).filter((u) => u !== actorId), + claimantUserIds: bySlot.get(slotId) ?? [], }) } } + return removed +} - const kept = desired.filter((s): s is DesiredSlot & { id: string } => s.id !== undefined) - const changed = ( - slot: DesiredSlot & { id: string }, - keyOf: (s: SlotIdentity) => string, - ): boolean => { - const before = have.get(slot.id) - return before === undefined || keyOf(before) !== keyOf(slot) - } - const rekeying = kept.filter((s) => changed(s, slotIdentityKey)).map((s) => s.id) +// Parks every retitled or moved slot on a unique placeholder title first, so the per-slot updates +// cannot collide with each other on the title-and-window unique index mid-reconcile. +async function rekeySlotsInTx(tx: Queryable, cleanupId: string, rekeying: string[]): Promise { if (rekeying.length > 0) { await tx` UPDATE cleanup_slots SET title = id::text WHERE cleanup_id = ${cleanupId} AND id = ANY(${rekeying}::uuid[]) ` } +} - const movedIds = kept.filter((s) => changed(s, slotWindowKey)).map((s) => s.id) +async function collectRescheduledInTx( + tx: Queryable, + cleanupId: string, + kept: readonly KeptSlot[], + movedIds: string[], + actorId: string | null, +): Promise { const rescheduled: SlotReconcileResult["rescheduled"] = [] if (movedIds.length > 0) { const claimants = await tx<{ slot_id: string; user_id: string }[]>` SELECT slot_id, user_id FROM cleanup_slot_claims WHERE cleanup_id = ${cleanupId} AND slot_id = ANY(${movedIds}::uuid[]) ` - const bySlot = new Map() - for (const c of claimants) { - if (c.user_id === actorId) continue - const list = bySlot.get(c.slot_id) - if (list) list.push(c.user_id) - else bySlot.set(c.slot_id, [c.user_id]) - } + const bySlot = claimantsBySlot(claimants, actorId) for (const slot of kept) { const userIds = bySlot.get(slot.id) if (userIds === undefined || userIds.length === 0) continue rescheduled.push({ slotId: slot.id, title: slot.title, claimantUserIds: userIds }) } } + return rescheduled +} +async function writeSlotsInTx( + tx: Queryable, + cleanupId: string, + desired: readonly DesiredSlot[], +): Promise<{ added: string[]; updated: string[] }> { const added: string[] = [] const updated: string[] = [] for (const slot of desired) { @@ -1731,6 +1700,49 @@ async function reconcileSlotsInTx( if (row) added.push(row.id) } } + return { added, updated } +} + +async function reconcileSlotsInTx( + tx: Queryable, + cleanupId: string, + desired: DesiredSlot[], + actorId: string | null, +): Promise { + const existing = await tx< + { id: string; title: string; starts_at: Date | null; ends_at: Date | null }[] + >` + SELECT id, title, starts_at, ends_at FROM cleanup_slots WHERE cleanup_id = ${cleanupId} + ` + const have = new Map( + existing.map((r) => [r.id, { title: r.title, startsAt: r.starts_at, endsAt: r.ends_at }]), + ) + + for (const slot of desired) { + if (slot.id !== undefined && !have.has(slot.id)) { + throw AppError.validation({ slots: `unknown slot: ${slot.id}` }) + } + } + + const keep = new Set(desired.map((s) => s.id).filter((id): id is string => id !== undefined)) + const toRemove = [...have.keys()].filter((id) => !keep.has(id)) + const removed = await removeSlotsInTx(tx, cleanupId, have, toRemove, actorId) + + const kept = desired.filter((s): s is KeptSlot => s.id !== undefined) + const changed = (slot: KeptSlot, keyOf: (s: SlotIdentity) => string): boolean => { + const before = have.get(slot.id) + return before === undefined || keyOf(before) !== keyOf(slot) + } + await rekeySlotsInTx( + tx, + cleanupId, + kept.filter((s) => changed(s, slotIdentityKey)).map((s) => s.id), + ) + + const movedIds = kept.filter((s) => changed(s, slotWindowKey)).map((s) => s.id) + const rescheduled = await collectRescheduledInTx(tx, cleanupId, kept, movedIds, actorId) + + const { added, updated } = await writeSlotsInTx(tx, cleanupId, desired) return { added, updated, removed, rescheduled } } diff --git a/services/api/src/services/cleanup-rules.ts b/services/api/src/services/cleanup-rules.ts index e9b2d8a6..30cd2816 100644 --- a/services/api/src/services/cleanup-rules.ts +++ b/services/api/src/services/cleanup-rules.ts @@ -10,20 +10,16 @@ import { deriveCleanupStatus, eventEndsAtMs, hasEventEnded, - hasEventStarted, } from "@civfix/shared/host" import type { EventWindowLike } from "@civfix/shared/host" -export { - DEFAULT_EVENT_DURATION_MS, - deriveCleanupStatus, - eventEndsAtMs, - hasEventEnded, - hasEventStarted, -} -export type { EventWindowLike } +export { DEFAULT_EVENT_DURATION_MS, deriveCleanupStatus, eventEndsAtMs, hasEventEnded } + +const MS_PER_MINUTE = 60 * 1000 + +const MS_PER_DAY = 24 * 60 * MS_PER_MINUTE -export const SCHEDULE_MAX_BACKDATE_MS = 24 * 60 * 60 * 1000 +export const SCHEDULE_MAX_BACKDATE_MS = MS_PER_DAY export const DEFAULT_EVENT_SLOT_TITLE = "General volunteers" @@ -40,17 +36,17 @@ export function defaultEventSlot(capacity: number | null): EventSlotInput { } } -export const MIN_EVENT_DURATION_MS = MIN_EVENT_DURATION_MINUTES * 60 * 1000 +export const MIN_EVENT_DURATION_MS = MIN_EVENT_DURATION_MINUTES * MS_PER_MINUTE -export const MAX_EVENT_DURATION_MS = MAX_EVENT_DURATION_MINUTES * 60 * 1000 +export const MAX_EVENT_DURATION_MS = MAX_EVENT_DURATION_MINUTES * MS_PER_MINUTE -export const SCHEDULE_MAX_AHEAD_MS = 2 * 365 * 24 * 60 * 60 * 1000 +export const SCHEDULE_MAX_AHEAD_MS = 2 * 365 * MS_PER_DAY -export const EVENT_ENDED_FIELD = "event" +const EVENT_ENDED_FIELD = "event" -export const EVENT_ENDED_REASON = "ended" +const EVENT_ENDED_REASON = "ended" -export const EVENT_ENDED_MESSAGE = "This event has already ended." +const EVENT_ENDED_MESSAGE = "This event has already ended." export function eventEndedError(): AppError { return new AppError(ErrorCode.CONFLICT, EVENT_ENDED_MESSAGE, { diff --git a/services/api/src/services/cleanup-service.ts b/services/api/src/services/cleanup-service.ts index 500d8473..3b4a260f 100644 --- a/services/api/src/services/cleanup-service.ts +++ b/services/api/src/services/cleanup-service.ts @@ -1,12 +1,5 @@ import { randomUUID } from "node:crypto" -import { - AppError, - ErrorCode, - MAX_BRING_ITEMS, - MAX_EVENT_ADDRESS_LENGTH, - MAX_EVENT_SLOTS, - MIN_SLOT_DURATION_MINUTES, -} from "@civfix/shared" +import { AppError, ErrorCode, MAX_BRING_ITEMS, MAX_LINKED_REPORTS } from "@civfix/shared" import { can, hostCapabilities, NO_HOST_STANDING, type HostStanding } from "@civfix/shared/host" import { UNKNOWN_JURCODE } from "../db/reference-code.js" import { assertNoSlur } from "../abuse/slur-filter.js" @@ -14,12 +7,11 @@ import { InMemoryCounterStore, type CounterStore } from "../abuse/counter-store. import type { CleanupAttendeesResponse, CleanupDTO, + CleanupMemberRole, CreateCleanupRequest, DuplicateCleanupRequest, - EventAddressSource, EventKind, EventSlotDTO, - EventSlotInput, HostCapability, OrganizationMemberRole, LinkedReportRef, @@ -30,10 +22,8 @@ import type { UpdateCleanupRequest, } from "@civfix/shared" import type { Jobs } from "@civfix/shared/interfaces" -import { isLocatedPrecision } from "@civfix/shared" import type { AddressResolver } from "./address-resolver.js" import type { NotificationService } from "./notification-service.js" -import type { MessageKey } from "../i18n/messages/en.js" import { EVENT_HOURS_MEMBER_CAP } from "./volunteer-hours-service.js" import type { OutboundMailService } from "./admin/outbound-mail-service.js" import { buildEventPacket } from "./admin/mail-format.js" @@ -43,7 +33,6 @@ import { CLEANUPS_DEFAULT_LIMIT, ATTENDEES_DEFAULT_LIMIT, LINKED_REPORTS_LIST_PREVIEW, - MAX_LINKED_REPORTS, toAttendeeDTO, toCleanupDTO, toEventSlotDTO, @@ -90,20 +79,17 @@ import { eventWindowOf, } from "./cleanup-rules.js" import type { EventWindow } from "./cleanup-rules.js" -import { CLEANUP_GUEST_UPDATE_FANOUT_JOB, type GuestUpdateFanoutJob } from "./guest-rsvp-service.js" +import { eventAddressPatch, resolveEventAddress } from "./cleanup-address.js" +import { assertKnownSlotIds, assertTimedSlotsFitWindow, toDesiredSlots } from "./cleanup-slots.js" +import { makeCleanupNotifications, type CleanupCancelFanoutJob } from "./cleanup-notifications.js" export * from "./cleanup-repository.types.js" -export * from "./cleanup-rules.js" +export { CANCEL_FANOUT_MEMBER_CAP, CLEANUP_CANCEL_FANOUT_JOB } from "./cleanup-notifications.js" export { CLEANUPS_DEFAULT_LIMIT, ATTENDEES_DEFAULT_LIMIT, THREAD_SIGNAL_MEMBER_CAP, - MAX_LINKED_REPORTS, - toAttendeeDTO, - toAttendeePersonDTO, - toOrganizerPerson, toCleanupDTO, - toEventSlotDTO, toLinkedReportRef, toLinkedEventRef, } from "./cleanup-dto.js" @@ -114,7 +100,7 @@ export interface CleanupViewer { export type UpdateCleanupPatchRequest = Omit -export type HostEventPatch = Pick< +type HostEventPatch = Pick< UpdateCleanupRequest, | "endsAt" | "timezone" @@ -130,6 +116,10 @@ export type HostEventPatch = Pick< | "hostReplyTo" > & { scheduledAt?: string } +const SECONDS_PER_HOUR = 60 * 60 + +const SECONDS_PER_DAY = 24 * SECONDS_PER_HOUR + // `joined` means an RSVP (a cleanup_members row, which the organizer always has). Org standing // grants host powers and visibility, not attendance: clients key Join/Leave off this flag. function isAttending(standing: HostStanding): boolean { @@ -163,14 +153,18 @@ function isUuid(value: string): boolean { const EVENT_CLOSED_MESSAGE = "This event is closed." -/** - * Floor for a host-confirmed event address. Long enough to reject the accidental keystroke and the - * lone punctuation mark, short enough to allow a genuinely terse one ("Pier 3"). - */ -const MIN_EVENT_ADDRESS_LENGTH = 3 - const CANCELLED_EVENT_EDIT_MESSAGE = "This event has been cancelled and can no longer be edited." +const ONLY_CLEANUPS_LINK_REPORTS_MESSAGE = "only cleanup events can link reports" + +const NOT_ATTENDING_MESSAGE = "That person isn't attending this event." + +const REMOVED_BY_HOST_MESSAGE = "A host removed you from this event, so you can't rejoin it." + +const RESOURCE_NOTE_PREVIEW_CHARS = 140 + +const UNKNOWN_JURISDICTION_BUDGET_KEY = "unknown" + function refusalOnceEnded( patch: UpdateCleanupPatchRequest, current: CleanupRecord, @@ -200,68 +194,154 @@ function refusalOnceEnded( return null } -function assertScheduledAtNotBackdated(next: string | undefined, stored: Date): void { +function assertScheduledAtNotBackdated( + next: string | undefined, + stored: Date, + nowMs: number, +): void { if (next === undefined) return const nextMs = Date.parse(next) if (Number.isNaN(nextMs)) return - if (nextMs >= Date.now() - SCHEDULE_MAX_BACKDATE_MS) return + if (nextMs >= nowMs - SCHEDULE_MAX_BACKDATE_MS) return if (nextMs >= stored.getTime()) return throw AppError.validation({ scheduledAt: "must not be in the past" }) } +function editedWindowOf( + current: CleanupRecord, + patch: UpdateCleanupPatchRequest, +): { window: EventWindow; moved: boolean } { + if (patch.endsAt === null) { + throw AppError.validation({ endsAt: "an event must have an end time" }) + } + const window: EventWindow = { + status: current.status, + scheduledAt: + patch.scheduledAt !== undefined ? new Date(patch.scheduledAt) : current.scheduledAt, + endsAt: patch.endsAt !== undefined ? new Date(patch.endsAt) : current.endsAt, + } + const moved = + window.scheduledAt.getTime() !== current.scheduledAt.getTime() || + (window.endsAt?.getTime() ?? null) !== (current.endsAt?.getTime() ?? null) + return { window, moved } +} + +function plannedEventWindow(input: CreateCleanupRequest): { + scheduledAt: Date + endsAt: Date + slots: DesiredSlot[] +} { + if (input.endsAt === null) throw AppError.validation({ endsAt: "required" }) + if (input.slots !== undefined && input.slots.length === 0) { + throw AppError.validation({ slots: EVENT_NEEDS_A_SLOT_MESSAGE }) + } + const scheduledAt = new Date(input.scheduledAt) + const endsAt = + input.endsAt !== undefined + ? new Date(input.endsAt) + : new Date(scheduledAt.getTime() + DEFAULT_EVENT_DURATION_MS) + const slots = toDesiredSlots( + input.slots ?? [defaultEventSlot(null)], + { keepIds: false }, + { scheduledAt, endsAt }, + ) + return { scheduledAt, endsAt, slots } +} + +function duplicateRequestOf( + source: CleanupRecord, + sourceSlots: EventSlotView[], + input: DuplicateCleanupRequest, + organizationId: string | null, + now: Date, +): CreateCleanupRequest { + const scheduledAt = new Date(input.scheduledAt) + const sourceDurationMs = source.endsAt.getTime() - source.scheduledAt.getTime() + const endsAt = input.endsAt ?? new Date(scheduledAt.getTime() + sourceDurationMs).toISOString() + const shiftMs = scheduledAt.getTime() - source.scheduledAt.getTime() + const shifted = (at: Date | null): string | null => + at === null ? null : new Date(at.getTime() + shiftMs).toISOString() + return { + title: source.title, + type: source.type, + eventKind: source.eventKind, + ...(source.description !== null ? { description: source.description } : {}), + lat: source.lat, + lng: source.lng, + scheduledAt: scheduledAt.toISOString(), + ...(source.bring !== null ? { bring: source.bring } : {}), + ...(source.address !== null + ? { + address: source.address, + ...(source.addressSource !== null ? { addressSource: source.addressSource } : {}), + } + : {}), + slots: + sourceSlots.length > 0 + ? sourceSlots.map((slot) => ({ + title: slot.title, + description: slot.description, + capacity: slot.capacity, + startsAt: shifted(slot.startsAt), + endsAt: shifted(slot.endsAt), + sortOrder: slot.sortOrder, + })) + : [defaultEventSlot(source.capacity)], + endsAt, + timezone: source.timezone, + visibility: source.visibility, + donationUrl: source.donationUrl, + registrationOpensAt: futureOrNull(source.registrationOpensAt, now), + registrationClosesAt: futureOrNull(source.registrationClosesAt, now), + organizationId, + reminderOffsetsMinutes: source.reminderOffsetsMin, + hostReplyTo: source.hostReplyTo, + } +} + export const RESOURCE_REQUEST_PER_HOST_PER_DAY = 10 -const RESOURCE_REQUEST_HOST_WINDOW_SEC = 24 * 60 * 60 +const RESOURCE_REQUEST_HOST_WINDOW_SEC = SECONDS_PER_DAY export const RESOURCE_REQUEST_PER_JURISDICTION_PER_HOUR = 30 -const RESOURCE_REQUEST_JURISDICTION_WINDOW_SEC = 60 * 60 +const RESOURCE_REQUEST_JURISDICTION_WINDOW_SEC = SECONDS_PER_HOUR export const ROLE_CHANGES_PER_TARGET_PER_WINDOW = 6 -const ROLE_CHANGE_WINDOW_SEC = 60 * 60 +const ROLE_CHANGE_WINDOW_SEC = SECONDS_PER_HOUR export { MAX_BRING_ITEMS } -export { MAX_EVENT_SLOTS } - export const SLOT_FLIPS_PER_EVENT_PER_WINDOW = 20 -const SLOT_FLIP_WINDOW_SEC = 60 * 60 +const SLOT_FLIP_WINDOW_SEC = SECONDS_PER_HOUR export const MEMBERSHIP_FLIPS_PER_EVENT_PER_WINDOW = 20 -const MEMBERSHIP_FLIP_WINDOW_SEC = 60 * 60 - -export const CANCEL_FANOUT_MEMBER_CAP = 2000 - -interface SlotFanoutBudget { - remaining: number -} - -const CANCEL_FANOUT_CONCURRENCY = 8 +const MEMBERSHIP_FLIP_WINDOW_SEC = SECONDS_PER_HOUR const fallbackCounters = new InMemoryCounterStore() -export const CLEANUP_CANCEL_FANOUT_JOB = "cleanup.cancel.fanout" - -export const CANCEL_FANOUT_DEDUPE_WINDOW_MS = 60 * 60 * 1000 - -export interface CleanupCancelFanoutJob { - cleanupId: string - reason: string | null - actorId: string -} - export const HOST_EVENTS_PER_DAY = 10 -export const HOST_EVENTS_WINDOW_SEC = 24 * 60 * 60 +export const HOST_EVENTS_WINDOW_SEC = SECONDS_PER_DAY export const HOST_ROSTER_READS_PER_HOUR = 200 -const HOST_ROSTER_READ_WINDOW_SEC = 60 * 60 +const HOST_ROSTER_READ_WINDOW_SEC = SECONDS_PER_HOUR -const ROSTER_AUDIT_DEDUPE_WINDOW_SEC = 60 * 60 +const ROSTER_AUDIT_DEDUPE_WINDOW_SEC = SECONDS_PER_HOUR -export const CLEANUP_CREATE_IDEMPOTENCY_SCOPE = "cleanup.create" +const CLEANUP_CREATE_IDEMPOTENCY_SCOPE = "cleanup.create" export interface EventMediaPresigner { (key: string, opts: { forceSigned: boolean }): Promise } +interface EventMediaUrls { + coverUrl: string | null + galleryUrls: string[] + organizationLogoUrl: string | null +} + +type JurisdictionContact = NonNullable< + Awaited> +> + export interface CleanupServiceDeps { repo: CleanupRepository tickets: TicketTokenSigner @@ -288,6 +368,7 @@ export interface CleanupServiceDeps { error(obj: unknown, msg?: string): void } newId?: () => string + now?: () => number enrichDTOs?: (dtos: CleanupDTO[], viewerUserId: string | null) => Promise affiliations?: AffiliationLoader } @@ -338,9 +419,11 @@ export interface CleanupService { export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { const newId = deps.newId ?? (() => randomUUID()) + const now = deps.now ?? (() => Date.now()) const presignThumb = deps.presignThumb ?? ((thumbKey: string) => Promise.resolve(thumbKey)) const counters = deps.counters ?? fallbackCounters const audit = deps.audit ?? NULL_HOST_AUDIT_SINK + const notifications = makeCleanupNotifications(deps) function newSignupSeat(): SignupSeat { const seatId = randomUUID() return { seatId, tokenHash: deps.tickets.hashFor(seatId) } @@ -405,11 +488,7 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { async function eventMediaUrls( record: CleanupRecord, opts: { gallery: boolean }, - ): Promise<{ - coverUrl: string | null - galleryUrls: string[] - organizationLogoUrl: string | null - }> { + ): Promise { const presign = deps.presignEventMedia if (presign === undefined) { return { coverUrl: null, galleryUrls: [], organizationLogoUrl: null } @@ -431,6 +510,59 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { return { coverUrl, galleryUrls, organizationLogoUrl } } + async function hydrateDetail( + cleanupId: string, + record: CleanupRecord, + standing: HostStanding, + viewerId: string | null, + myRole: CleanupMemberRole | null = standing.eventRole, + ): Promise { + const [linkedReports, slotBoard, media] = await Promise.all([ + hydrateLinkedReports(cleanupId, record.eventKind), + hydrateSlots(cleanupId, viewerId), + eventMediaUrls(record, { gallery: true }), + ]) + return enrichOne( + toCleanupDTO(record, isAttending(standing), linkedReports, myRole, { + slots: slotBoard, + myCapabilities: capabilityList(standing), + ...media, + }), + viewerId, + ) + } + + async function hydrateListItems( + records: CleanupRecord[], + viewerId: string | null, + organizationLogoUrl: () => Promise = () => Promise.resolve(null), + ): Promise { + const ids = records.map((r) => r.id) + const [standingsById, linkedByCleanup, slotCounts, coverUrls, logoUrl] = await Promise.all([ + standingsOf(ids, viewerId), + hydrateLinkedReportsForMany(records), + deps.repo.slotCountsFor(ids), + hydrateCoverUrls(records), + organizationLogoUrl(), + ]) + const items = records.map((record) => { + const standing = standingsById.get(record.id) ?? NO_HOST_STANDING + return toCleanupDTO( + record, + isAttending(standing), + linkedByCleanup.get(record.id) ?? [], + standing.eventRole, + { + slotCount: slotCounts.get(record.id) ?? 0, + myCapabilities: capabilityList(standing), + coverUrl: coverUrls.get(record.id) ?? null, + organizationLogoUrl: logoUrl, + }, + ) + }) + return enrichDTOs(items, viewerId) + } + async function assertRosterReadBudget(userId: string): Promise { const reads = await counters.incr(`host:rosterReads:${userId}`, HOST_ROSTER_READ_WINDOW_SEC) if (reads > HOST_ROSTER_READS_PER_HOUR) { @@ -440,6 +572,24 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { } } + async function recordRosterView( + cleanupId: string, + viewerId: string, + returned: number, + ): Promise { + const seen = await counters.incr( + `host:rosterAudit:${cleanupId}:${viewerId}`, + ROSTER_AUDIT_DEDUPE_WINDOW_SEC, + ) + if (seen !== 1) return + await audit.record({ + actorId: viewerId, + action: "event.roster_viewed", + target: `cleanup:${cleanupId}`, + meta: { returned }, + }) + } + async function assertHostEventBudget(userId: string): Promise { const created = await counters.incr(`host:events:${userId}`, HOST_EVENTS_WINDOW_SEC) if (created > HOST_EVENTS_PER_DAY) { @@ -461,6 +611,53 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { } } + async function assertRoleChangeBudget(cleanupId: string, targetUserId: string): Promise { + const flips = await counters.incr( + `cleanup:role:${cleanupId}:${targetUserId}`, + ROLE_CHANGE_WINDOW_SEC, + ) + if (flips > ROLE_CHANGES_PER_TARGET_PER_WINDOW) { + throw AppError.rateLimited( + "This attendee's role has been changed too many times recently. Please try again later.", + ) + } + } + + async function assertSlotFlipBudget(cleanupId: string, userId: string): Promise { + const flips = await counters.incr(`cleanup:slot:${cleanupId}:${userId}`, SLOT_FLIP_WINDOW_SEC) + if (flips > SLOT_FLIPS_PER_EVENT_PER_WINDOW) { + throw AppError.rateLimited( + "You've changed your slot too many times recently. Please try again later.", + ) + } + } + + async function assertResourceRequestBudget( + actorId: string, + jurisdictionGeoid: string | null, + ): Promise { + const hostSends = await counters.incr( + `cleanup:res-req:host:${actorId}`, + RESOURCE_REQUEST_HOST_WINDOW_SEC, + ) + // The shared jurisdiction budget is charged only after the host's own cap passes, so one + // host hammering past its limit cannot exhaust the area for every other host. + if (hostSends > RESOURCE_REQUEST_PER_HOST_PER_DAY) { + throw AppError.rateLimited( + "You've sent the maximum number of resource requests for today. Please try again tomorrow.", + ) + } + const jurisdictionSends = await counters.incr( + `cleanup:res-req:jur:${jurisdictionGeoid ?? UNKNOWN_JURISDICTION_BUDGET_KEY}`, + RESOURCE_REQUEST_JURISDICTION_WINDOW_SEC, + ) + if (jurisdictionSends > RESOURCE_REQUEST_PER_JURISDICTION_PER_HOUR) { + throw AppError.rateLimited( + "This area has received too many resource requests in the past hour. Please try again later.", + ) + } + } + async function organizationFor( organizationId: string | null, actorId: string, @@ -577,85 +774,6 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { for (const item of input.bring ?? []) assertNoSlur(item, "bring") } - /** - * The event address, with its provenance, for a create or an update. - * - * `addressSource` is the CLIENT-VERSION discriminator, and it has to be: `address` itself stays - * optional on the wire so the TestFlight build in someone's pocket keeps working. - * - * addressSource PRESENT -> a new client. It resolved the pin, showed the line to the host, and the - * host published with it on screen. That is the confirmation, so the - * server only has to refuse a blank one; a client that sends a source - * without an address has a bug, and storing it would produce an event - * whose address is "verified" and empty. - * addressSource ABSENT -> an old client. Whatever it sent in `address` is the host's own "name the - * spot" text, so it is 'manual' (the same call migration 0179 makes for - * existing rows). If it sent nothing, the shim resolves the pin and stores - * 'resolved': unverified, but an event with a street line beats an event - * with "Meeting point", and only while old clients are still in the wild. - * - * The shim stores NOTHING when the ladder only reached `locality`: "Los Angeles, CA" is not a meeting - * address, and writing it would dress up a non-answer as a host-provided one. - * - * `fromStoredEvent` marks the DUPLICATE path, whose pair did not come off the wire at all: it is this - * server's own stored row, copied verbatim. The new-client length floor is a check on a client payload - * and would reject a backfilled one-or-two-character address that the host has been running for - * months. Slur checks still apply - they run over the whole input before this. - */ - async function resolveEventAddress( - input: { - address?: string | undefined - addressSource?: EventAddressSource | undefined - lat: number - lng: number - }, - opts?: { fromStoredEvent?: boolean }, - ): Promise<{ address: string | null; addressSource: EventAddressSource | null }> { - if (input.addressSource !== undefined) { - if (opts?.fromStoredEvent === true && input.address !== undefined) { - return { address: input.address, addressSource: input.addressSource } - } - return { address: assertConfirmedAddress(input.address), addressSource: input.addressSource } - } - const typed = input.address?.trim() ?? "" - if (typed.length > 0) return { address: typed, addressSource: "manual" } - if (deps.resolveAddress === undefined) return { address: null, addressSource: null } - const resolved = await deps.resolveAddress(input.lat, input.lng) - if (resolved.address === null || !isLocatedPrecision(resolved.precision)) { - return { address: null, addressSource: null } - } - return { - address: resolved.address.slice(0, MAX_EVENT_ADDRESS_LENGTH), - addressSource: "resolved", - } - } - - /** A new client that names a source must carry a real line with it. */ - function assertConfirmedAddress(address: string | undefined): string { - const trimmed = address?.trim() ?? "" - if (trimmed.length < MIN_EVENT_ADDRESS_LENGTH) { - throw AppError.validation({ - address: `must be at least ${MIN_EVENT_ADDRESS_LENGTH} characters`, - }) - } - return trimmed - } - - /** The update-path twin of resolveEventAddress: same rules, but every field stays optional. */ - function eventAddressPatch(patch: { - address?: string | undefined - addressSource?: EventAddressSource | undefined - }): { address?: string | null; addressSource?: EventAddressSource | null } { - if (patch.addressSource !== undefined) { - return { address: assertConfirmedAddress(patch.address), addressSource: patch.addressSource } - } - if (patch.address === undefined) return {} - const trimmed = patch.address.trim() - return trimmed.length > 0 - ? { address: trimmed, addressSource: "manual" } - : { address: null, addressSource: null } - } - function clampBring(bring: T): T { if (bring !== null && bring !== undefined && bring.length > MAX_BRING_ITEMS) { throw AppError.validation({ bring: `at most ${MAX_BRING_ITEMS} items may be listed` }) @@ -671,137 +789,6 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { throw AppError.notFound("Organization not found") } - async function notifyRoleChange( - targetUserId: string, - event: "promoted" | "demoted" | "removed", - cleanup: { id: string; title: string }, - ): Promise { - if (deps.notifier === undefined) return - try { - await deps.notifier.createNotification(targetUserId, { - type: "cleanup_role", - titleKey: `notification.cleanup_role.${event}.title`, - bodyKey: `notification.cleanup_role.${event}.body`, - vars: { title: cleanup.title }, - link: `/cleanups/${cleanup.id}`, - }) - } catch (err) { - deps.logger?.warn( - { err, targetUserId, cleanupId: cleanup.id, event }, - "cleanup_role notification failed (suppressed)", - ) - } - } - - async function notifyCancellation( - cleanup: { id: string; title: string }, - reason: string | null, - actorId: string, - ): Promise { - await notifyMembersOfCancellation(cleanup, reason, actorId) - await notifyAttendeesOfCancellation(cleanup.id, reason) - } - - async function notifyAttendeesOfCancellation( - cleanupId: string, - reason: string | null, - ): Promise { - if (deps.attendeeNotifier === undefined) return - await deps.attendeeNotifier.eventCancelled(cleanupId, reason) - } - - async function dispatchGuestUpdateFanout(cleanupId: string): Promise { - if (deps.jobs === undefined) { - deps.logger?.warn( - { cleanupId }, - "cleanup.guest.update.fanout: no job queue wired; guests are not notified", - ) - return - } - try { - await deps.jobs.enqueue( - CLEANUP_GUEST_UPDATE_FANOUT_JOB, - { cleanupId } satisfies GuestUpdateFanoutJob, - { singletonKey: cleanupId }, - ) - } catch (err) { - deps.logger?.error( - { err, cleanupId }, - "cleanup.guest.update.fanout enqueue failed; guests are not notified", - ) - } - } - - async function notifyMembersOfCancellation( - cleanup: { id: string; title: string }, - reason: string | null, - actorId: string, - ): Promise { - const notifier = deps.notifier - if (notifier === undefined) return - let memberIds: string[] - try { - memberIds = await deps.repo.listMemberIds(cleanup.id, CANCEL_FANOUT_MEMBER_CAP) - } catch (err) { - deps.logger?.warn( - { err, cleanupId: cleanup.id }, - "cleanup_cancelled roster read failed (suppressed)", - ) - return - } - const recipients = memberIds.filter((userId) => userId !== actorId) - await mapWithLimit(recipients, CANCEL_FANOUT_CONCURRENCY, async (userId) => { - try { - await notifier.createNotification(userId, { - type: "cleanup_cancelled", - titleKey: "notification.cleanup_cancelled.title", - bodyKey: - reason !== null - ? "notification.cleanup_cancelled.body_reason" - : "notification.cleanup_cancelled.body", - ...(reason !== null ? { vars: { reason } } : {}), - link: `/cleanups/${cleanup.id}`, - dedupeWindowMs: CANCEL_FANOUT_DEDUPE_WINDOW_MS, - }) - } catch (err) { - deps.logger?.warn( - { err, cleanupId: cleanup.id, userId }, - "cleanup_cancelled notification failed (suppressed)", - ) - } - }) - } - - async function dispatchCancelFanout( - cleanup: { id: string; title: string }, - reason: string | null, - actorId: string, - ): Promise { - if (deps.jobs !== undefined) { - try { - await deps.jobs.enqueue( - CLEANUP_CANCEL_FANOUT_JOB, - { cleanupId: cleanup.id, reason, actorId } satisfies CleanupCancelFanoutJob, - { singletonKey: cleanup.id }, - ) - return - } catch (err) { - deps.logger?.error( - { err, cleanupId: cleanup.id }, - "cleanup.cancel.fanout enqueue failed; ringing members inline, guests are not notified", - ) - } - } - try { - await notifyMembersOfCancellation(cleanup, reason, actorId) - } catch (err) { - deps.logger?.warn( - { err, cleanupId: cleanup.id }, - "cleanup_cancelled inline member fanout failed (suppressed; the cancellation itself stands)", - ) - } - } - async function hydrateLinkedReports( cleanupId: string, eventKind: EventKind, @@ -859,64 +846,134 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { return ids } - function slotWindowOf(slot: EventSlotInput): { startsAt: Date | null; endsAt: Date | null } { - const startsAt = slot.startsAt != null ? new Date(slot.startsAt) : null - const endsAt = slot.endsAt != null ? new Date(slot.endsAt) : null - return { startsAt, endsAt } + async function resolveCreateLinks( + requested: string[] | undefined, + eventKind: EventKind, + ): Promise { + const linkedReportIds = clampLinkIds(requested ?? []) + if (eventKind !== "cleanup" && linkedReportIds.length > 0) { + throw AppError.validation({ linkedReportIds: ONLY_CLEANUPS_LINK_REPORTS_MESSAGE }) + } + await assertReportsLinkable(linkedReportIds) + return linkedReportIds } - function assertSlotInsideEvent( - title: string, - window: { startsAt: Date; endsAt: Date }, - event: EventWindow, - ): void { - if (event.endsAt === null) { - throw AppError.validation({ - slots: "set an end time for the event before adding timed slots", - }) + /** null leaves the stored links untouched; a non-cleanup event always clears them. */ + async function resolveEditedLinks( + requested: string[] | undefined, + effectiveKind: EventKind, + ): Promise { + if (requested !== undefined && effectiveKind !== "cleanup") { + throw AppError.validation({ linkedReportIds: ONLY_CLEANUPS_LINK_REPORTS_MESSAGE }) } - if (window.startsAt < event.scheduledAt || window.endsAt > event.endsAt) { - throw AppError.validation({ - slots: `slot "${title}" falls outside the event's start and end`, - }) - } - if (window.endsAt.getTime() - window.startsAt.getTime() < MIN_SLOT_DURATION_MINUTES * 60_000) { - throw AppError.validation({ - slots: `slot "${title}" must last at least ${MIN_SLOT_DURATION_MINUTES} minutes`, - }) + const desiredLinks = + requested !== undefined ? clampLinkIds(requested) : effectiveKind !== "cleanup" ? [] : null + if (desiredLinks !== null && desiredLinks.length > 0) { + await assertReportsLinkable(desiredLinks) } + return desiredLinks } - function toDesiredSlots( - slots: EventSlotInput[], - opts: { keepIds: boolean }, - window: EventWindow, - ): DesiredSlot[] { - if (slots.length > MAX_EVENT_SLOTS) { - throw AppError.validation({ slots: `at most ${MAX_EVENT_SLOTS} slots may be listed` }) + async function resolveEditedSlots( + cleanupId: string, + requested: UpdateCleanupPatchRequest["slots"], + edited: { window: EventWindow; moved: boolean }, + ): Promise { + if (requested !== undefined && requested.length === 0) { + throw AppError.validation({ slots: EVENT_NEEDS_A_SLOT_MESSAGE }) } - const seen = new Set() - for (const slot of slots) { - const { startsAt, endsAt } = slotWindowOf(slot) - if (startsAt !== null && endsAt !== null) { - assertSlotInsideEvent(slot.title, { startsAt, endsAt }, window) - } - const key = `${slot.title.trim().toLowerCase()}|${startsAt?.getTime() ?? ""}|${endsAt?.getTime() ?? ""}` - if (seen.has(key)) { - throw AppError.validation({ slots: `duplicate slot title: ${slot.title}` }) + if (requested === undefined) { + if (edited.moved) { + assertTimedSlotsFitWindow(await deps.repo.listSlots(cleanupId, null), edited.window) } - seen.add(key) - assertNoSlur(slot.title, "slots") - assertNoSlur(slot.description ?? null, "slots") + return null + } + const desiredSlots = toDesiredSlots(requested, { keepIds: true }, edited.window) + assertKnownSlotIds(desiredSlots, await deps.repo.listSlots(cleanupId, null)) + return desiredSlots + } + + /** + * Authz plus the cancelled/ended freezes. Returns the ended-event refusal (or null) because the + * transactional write re-checks it under its own lock: the event can end between here and there. + */ + function assertEditable( + current: CleanupRecord, + standing: HostStanding, + patch: UpdateCleanupPatchRequest, + ): AppError | null { + assertCapability(current, standing, "manage_event") + if (patch.organizationId !== undefined && patch.organizationId !== current.organizationId) { + assertCapability(current, standing, "manage_org_link") + } + if (current.status === "cancelled") { + throw AppError.conflict(CANCELLED_EVENT_EDIT_MESSAGE) + } + const hasEnded = deriveCleanupStatus(eventWindowOf(current), now()) === "done" + const endedRefusal = refusalOnceEnded(patch, current) + if (hasEnded && endedRefusal !== null) throw endedRefusal + assertScheduledAtNotBackdated(patch.scheduledAt, current.scheduledAt, now()) + return endedRefusal + } + + async function resolveJurisdiction( + lat: number, + lng: number, + ): Promise<{ jurisdictionGeoid: string | null; jurCode: number }> { + const jurisdictionGeoid = + deps.resolveJurisdictionGeoid !== undefined + ? await deps.resolveJurisdictionGeoid(lat, lng) + : null + const jurCode = + deps.resolveJurisdictionCode !== undefined + ? await deps.resolveJurisdictionCode(jurisdictionGeoid) + : UNKNOWN_JURCODE + return { jurisdictionGeoid, jurCode } + } + + async function resolveEditedScalars( + cleanupId: string, + patch: UpdateCleanupPatchRequest, + current: CleanupRecord, + actorId: string, + ): Promise { + const addressPatch = eventAddressPatch(patch) + const movedTo = + patch.lat !== undefined && patch.lng !== undefined ? { lat: patch.lat, lng: patch.lng } : null + const reresolvedGeoid = + movedTo !== null && deps.resolveJurisdictionGeoid !== undefined + ? await deps.resolveJurisdictionGeoid(movedTo.lat, movedTo.lng) + : undefined + const host = await resolveHostWrite({ patch, actorId, cleanupId, current }) + return { + ...host, + ...(patch.title !== undefined ? { title: patch.title } : {}), + ...(patch.description !== undefined ? { description: patch.description } : {}), + ...(patch.eventKind !== undefined ? { eventKind: patch.eventKind } : {}), + ...(patch.type !== undefined ? { type: patch.type } : {}), + ...(patch.scheduledAt !== undefined ? { scheduledAt: new Date(patch.scheduledAt) } : {}), + ...(movedTo ?? {}), + ...addressPatch, + ...(patch.bring !== undefined ? { bring: patch.bring } : {}), + ...(reresolvedGeoid !== undefined ? { jurisdictionGeoid: reresolvedGeoid } : {}), + } + } + + async function announceUpdate( + record: CleanupRecord, + slotDiff: SlotReconcileResult | null, + current: CleanupRecord, + patch: UpdateCleanupPatchRequest, + ): Promise { + if (slotDiff !== null) { + await notifications.notifySlotChanges(record, slotDiff) + } + if ( + deriveCleanupStatus(eventWindowOf(record), now()) !== "done" && + guestVisibleChange(current, patch) + ) { + await notifications.dispatchGuestUpdateFanout(record.id) } - return slots.map((slot, index) => ({ - ...(opts.keepIds && slot.id !== undefined ? { id: slot.id } : {}), - title: slot.title, - description: slot.description ?? null, - capacity: slot.capacity ?? null, - ...slotWindowOf(slot), - sortOrder: slot.sortOrder ?? index, - })) } async function hydrateCoverUrls(records: CleanupRecord[]): Promise> { @@ -941,69 +998,6 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { return views.map(toEventSlotDTO) } - async function notifySlotClaimants( - cleanup: { id: string; title: string }, - entries: SlotReconcileResult["removed"], - keys: { titleKey: MessageKey; bodyKey: MessageKey }, - budget: SlotFanoutBudget, - ): Promise { - const notifier = deps.notifier - if (notifier === undefined) return - const targets: { userId: string; slot: string }[] = [] - for (const entry of entries) { - for (const userId of entry.claimantUserIds) { - if (budget.remaining <= 0) break - budget.remaining -= 1 - targets.push({ userId, slot: entry.title }) - } - } - await mapWithLimit(targets, CANCEL_FANOUT_CONCURRENCY, async ({ userId, slot }) => { - try { - await notifier.createNotification(userId, { - type: "cleanup_slot", - titleKey: keys.titleKey, - bodyKey: keys.bodyKey, - vars: { slot, title: cleanup.title }, - link: `/cleanups/${cleanup.id}`, - }) - } catch (err) { - deps.logger?.warn( - { err, cleanupId: cleanup.id, userId }, - "cleanup_slot notification failed (suppressed)", - ) - } - }) - } - - async function notifySlotChanges( - cleanup: { id: string; title: string }, - diff: SlotReconcileResult, - ): Promise { - const budget: SlotFanoutBudget = { remaining: CANCEL_FANOUT_MEMBER_CAP } - if (diff.removed.length > 0) { - await notifySlotClaimants( - cleanup, - diff.removed, - { - titleKey: "notification.cleanup_slot.removed.title", - bodyKey: "notification.cleanup_slot.removed.body", - }, - budget, - ) - } - if (diff.rescheduled.length > 0) { - await notifySlotClaimants( - cleanup, - diff.rescheduled, - { - titleKey: "notification.cleanup_slot.moved.title", - bodyKey: "notification.cleanup_slot.moved.body", - }, - budget, - ) - } - } - async function createEvent( input: CreateCleanupRequest, organizerUserId: string, @@ -1011,28 +1005,11 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { ): Promise { assertEventTextClean(input) clampBring(input.bring) - const linkedReportIds = clampLinkIds(input.linkedReportIds ?? []) - if (input.eventKind !== "cleanup" && linkedReportIds.length > 0) { - throw AppError.validation({ linkedReportIds: "only cleanup events can link reports" }) - } - await assertReportsLinkable(linkedReportIds) - const addressWrite = await resolveEventAddress(input, { + const linkedReportIds = await resolveCreateLinks(input.linkedReportIds, input.eventKind) + const addressWrite = await resolveEventAddress(input, deps.resolveAddress, { fromStoredEvent: copyFrom !== undefined, }) - if (input.endsAt === null) throw AppError.validation({ endsAt: "required" }) - if (input.slots !== undefined && input.slots.length === 0) { - throw AppError.validation({ slots: EVENT_NEEDS_A_SLOT_MESSAGE }) - } - const scheduledAt = new Date(input.scheduledAt) - const endsAt = - input.endsAt !== undefined - ? new Date(input.endsAt) - : new Date(scheduledAt.getTime() + DEFAULT_EVENT_DURATION_MS) - const slots = toDesiredSlots( - input.slots ?? [defaultEventSlot(null)], - { keepIds: false }, - { scheduledAt, endsAt }, - ) + const { scheduledAt, endsAt, slots } = plannedEventWindow(input) const host = await resolveHostWrite({ patch: { ...input, scheduledAt: input.scheduledAt, endsAt: endsAt.toISOString() }, @@ -1042,14 +1019,7 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { }) await assertHostEventBudget(organizerUserId) - const jurisdictionGeoid = - deps.resolveJurisdictionGeoid !== undefined - ? await deps.resolveJurisdictionGeoid(input.lat, input.lng) - : null - const jurCode = - deps.resolveJurisdictionCode !== undefined - ? await deps.resolveJurisdictionCode(jurisdictionGeoid) - : UNKNOWN_JURCODE + const { jurisdictionGeoid, jurCode } = await resolveJurisdiction(input.lat, input.lng) const cleanupId = newId() const outcome = await deps.repo.createCleanupTx({ @@ -1084,19 +1054,7 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { }) const record = outcome.record const standing = await standingOf(record.id, organizerUserId) - const [linkedReports, slotBoard, media] = await Promise.all([ - hydrateLinkedReports(record.id, record.eventKind), - hydrateSlots(record.id, organizerUserId), - eventMediaUrls(record, { gallery: true }), - ]) - return enrichOne( - toCleanupDTO(record, isAttending(standing), linkedReports, standing.eventRole, { - slots: slotBoard, - myCapabilities: capabilityList(standing), - ...media, - }), - organizerUserId, - ) + return hydrateDetail(record.id, record, standing, organizerUserId) } async function resolveDuplicateOrganization( @@ -1113,6 +1071,35 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { return organizationId } + async function sendResourceRequest( + outboundMail: OutboundMailService, + record: CleanupRecord, + routing: JurisdictionContact, + message: string, + ): Promise { + const packet = buildEventPacket( + { + title: record.title, + host: record.organizer.displayName, + place: routing.name, + address: record.address, + lat: record.lat, + lng: record.lng, + referenceCode: record.referenceCode, + }, + message, + ) + await outboundMail.sendEventToJurisdiction({ + cleanupId: record.id, + geoid: record.jurisdictionGeoid, + org: routing.name, + toAddr: routing.contact, + subject: packet.subject, + text: packet.text, + html: packet.html, + }) + } + return { createCleanup(input: CreateCleanupRequest, organizerUserId: string): Promise { return createEvent(input, organizerUserId) @@ -1122,51 +1109,9 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { if (input.endsAt === null) throw AppError.validation({ endsAt: "required" }) const { record: source } = await requireCapabilityOn(input.id, actorId, "manage_event") const organizationId = await resolveDuplicateOrganization(source.organizationId, actorId) - const now = new Date() - const scheduledAt = new Date(input.scheduledAt) - const sourceDurationMs = source.endsAt.getTime() - source.scheduledAt.getTime() - const endsAt = - input.endsAt ?? new Date(scheduledAt.getTime() + sourceDurationMs).toISOString() + const copiedAt = new Date(now()) const slots = await deps.repo.listSlots(source.id, null) - const shiftMs = scheduledAt.getTime() - source.scheduledAt.getTime() - const shifted = (at: Date | null): string | null => - at === null ? null : new Date(at.getTime() + shiftMs).toISOString() - const copy: CreateCleanupRequest = { - title: source.title, - type: source.type, - eventKind: source.eventKind, - ...(source.description !== null ? { description: source.description } : {}), - lat: source.lat, - lng: source.lng, - scheduledAt: scheduledAt.toISOString(), - ...(source.bring !== null ? { bring: source.bring } : {}), - ...(source.address !== null - ? { - address: source.address, - ...(source.addressSource !== null ? { addressSource: source.addressSource } : {}), - } - : {}), - slots: - slots.length > 0 - ? slots.map((slot) => ({ - title: slot.title, - description: slot.description, - capacity: slot.capacity, - startsAt: shifted(slot.startsAt), - endsAt: shifted(slot.endsAt), - sortOrder: slot.sortOrder, - })) - : [defaultEventSlot(source.capacity)], - endsAt, - timezone: source.timezone, - visibility: source.visibility, - donationUrl: source.donationUrl, - registrationOpensAt: futureOrNull(source.registrationOpensAt, now), - registrationClosesAt: futureOrNull(source.registrationClosesAt, now), - organizationId, - reminderOffsetsMinutes: source.reminderOffsetsMin, - hostReplyTo: source.hostReplyTo, - } + const copy = duplicateRequestOf(source, slots, input, organizationId, copiedAt) return createEvent(copy, actorId, { cleanupId: source.id, ticketTypes: input.includeTicketTypes, @@ -1185,113 +1130,15 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { const standing = await standingOf(id, requesterUserId) const current = await deps.repo.findCleanupById(id, null) if (!current) notFoundCleanup() - assertCapability(current, standing, "manage_event") - if (patch.organizationId !== undefined && patch.organizationId !== current.organizationId) { - assertCapability(current, standing, "manage_org_link") - } - const requesterRole = standing.eventRole - if (current.status === "cancelled") { - throw AppError.conflict(CANCELLED_EVENT_EDIT_MESSAGE) - } - const hasEnded = deriveCleanupStatus(eventWindowOf(current), Date.now()) === "done" - const endedRefusal = refusalOnceEnded(patch, current) - if (hasEnded && endedRefusal !== null) throw endedRefusal - assertScheduledAtNotBackdated(patch.scheduledAt, current.scheduledAt) - const effectiveKind = patch.eventKind ?? current.eventKind - - if (patch.linkedReportIds !== undefined && effectiveKind !== "cleanup") { - throw AppError.validation({ linkedReportIds: "only cleanup events can link reports" }) - } - const desiredLinks = - patch.linkedReportIds !== undefined - ? clampLinkIds(patch.linkedReportIds) - : effectiveKind !== "cleanup" - ? [] - : null - - if (desiredLinks !== null && desiredLinks.length > 0) { - await assertReportsLinkable(desiredLinks) - } - - if (patch.endsAt === null) { - throw AppError.validation({ endsAt: "an event must have an end time" }) - } - const effectiveWindow: EventWindow = { - status: current.status, - scheduledAt: - patch.scheduledAt !== undefined ? new Date(patch.scheduledAt) : current.scheduledAt, - endsAt: patch.endsAt !== undefined ? new Date(patch.endsAt) : current.endsAt, - } - const windowMoved = - effectiveWindow.scheduledAt.getTime() !== current.scheduledAt.getTime() || - (effectiveWindow.endsAt?.getTime() ?? null) !== (current.endsAt?.getTime() ?? null) - - if (patch.slots !== undefined && patch.slots.length === 0) { - throw AppError.validation({ slots: EVENT_NEEDS_A_SLOT_MESSAGE }) - } - const desiredSlots = - patch.slots !== undefined - ? toDesiredSlots(patch.slots, { keepIds: true }, effectiveWindow) - : null - - if (desiredSlots === null && windowMoved) { - const timed = (await deps.repo.listSlots(id, null)).filter( - (slot): slot is EventSlotView & { startsAt: Date; endsAt: Date } => - slot.startsAt !== null && slot.endsAt !== null, - ) - const outside = timed.some( - (slot) => - effectiveWindow.endsAt === null || - slot.startsAt < effectiveWindow.scheduledAt || - slot.endsAt > effectiveWindow.endsAt, - ) - if (outside) { - throw AppError.validation({ - scheduledAt: - "timed slots would fall outside the new start and end; update the slots in the same save", - }) - } - } - - if (desiredSlots !== null) { - const existingSlotIds = new Set((await deps.repo.listSlots(id, null)).map((s) => s.id)) - for (const slot of desiredSlots) { - if (slot.id !== undefined && !existingSlotIds.has(slot.id)) { - throw AppError.validation({ slots: `unknown slot: ${slot.id}` }) - } - } - } - - const addressPatch = eventAddressPatch(patch) - - const movedTo = - patch.lat !== undefined && patch.lng !== undefined - ? { lat: patch.lat, lng: patch.lng } - : null - const reresolvedGeoid = - movedTo !== null && deps.resolveJurisdictionGeoid !== undefined - ? await deps.resolveJurisdictionGeoid(movedTo.lat, movedTo.lng) - : undefined - - const host = await resolveHostWrite({ - patch, - actorId: requesterUserId, - cleanupId: id, - current, - }) + const endedRefusal = assertEditable(current, standing, patch) + const desiredLinks = await resolveEditedLinks( + patch.linkedReportIds, + patch.eventKind ?? current.eventKind, + ) + const edited = editedWindowOf(current, patch) + const desiredSlots = await resolveEditedSlots(id, patch.slots, edited) + const scalarPatch = await resolveEditedScalars(id, patch, current, requesterUserId) - const scalarPatch: UpdateCleanupPatch = { - ...host, - ...(patch.title !== undefined ? { title: patch.title } : {}), - ...(patch.description !== undefined ? { description: patch.description } : {}), - ...(patch.eventKind !== undefined ? { eventKind: patch.eventKind } : {}), - ...(patch.type !== undefined ? { type: patch.type } : {}), - ...(patch.scheduledAt !== undefined ? { scheduledAt: new Date(patch.scheduledAt) } : {}), - ...(movedTo ?? {}), - ...addressPatch, - ...(patch.bring !== undefined ? { bring: patch.bring } : {}), - ...(reresolvedGeoid !== undefined ? { jurisdictionGeoid: reresolvedGeoid } : {}), - } const outcome = await deps.repo.updateCleanupWithEdits(id, scalarPatch, { actorUserId: requesterUserId, links: desiredLinks, @@ -1300,32 +1147,11 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { }) if (outcome.kind === "not_found") notFoundCleanup() if (outcome.kind === "cancelled") throw AppError.conflict(CANCELLED_EVENT_EDIT_MESSAGE) - const { slotDiff } = outcome const record = await deps.repo.findCleanupById(id, null) if (!record) notFoundCleanup() - if (slotDiff !== null) { - await notifySlotChanges(record, slotDiff) - } - if ( - deriveCleanupStatus(eventWindowOf(record), Date.now()) !== "done" && - guestVisibleChange(current, patch) - ) { - await dispatchGuestUpdateFanout(record.id) - } - const [linkedReports, slotBoard, media] = await Promise.all([ - hydrateLinkedReports(id, record.eventKind), - hydrateSlots(id, requesterUserId), - eventMediaUrls(record, { gallery: true }), - ]) - return enrichOne( - toCleanupDTO(record, isAttending(standing), linkedReports, requesterRole, { - slots: slotBoard, - myCapabilities: capabilityList(standing), - ...media, - }), - requesterUserId, - ) + await announceUpdate(record, outcome.slotDiff, current, patch) + return hydrateDetail(id, record, standing, requesterUserId) }, async cancelCleanup( @@ -1356,26 +1182,10 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { const record = await deps.repo.findCleanupById(id, null) if (!record) notFoundCleanup() - if (outcome === "cancelled") await dispatchCancelFanout(record, cleanReason, requesterUserId) - const [linkedReports, slotBoard, media] = await Promise.all([ - hydrateLinkedReports(id, record.eventKind), - hydrateSlots(id, requesterUserId), - eventMediaUrls(record, { gallery: true }), - ]) - return enrichOne( - toCleanupDTO( - record, - isAttending(standing), - linkedReports, - standing.eventRole ?? "organizer", - { - slots: slotBoard, - myCapabilities: capabilityList(standing), - ...media, - }, - ), - requesterUserId, - ) + if (outcome === "cancelled") { + await notifications.dispatchCancelFanout(record, cleanReason, requesterUserId) + } + return hydrateDetail(id, record, standing, requesterUserId, standing.eventRole ?? "organizer") }, async completeCleanup( @@ -1385,26 +1195,13 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { userAgent: string | null = null, ): Promise { const { standing } = await requireCapabilityOn(id, requesterUserId, "manage_event") - const requesterRole = standing.eventRole const trimmed = note?.trim() assertNoSlur(trimmed && trimmed.length > 0 ? trimmed : null, "note") deps.logger?.info({ cleanupId: id, userAgent }, "cleanup.complete.deprecated") const record = await deps.repo.findCleanupById(id, null) if (!record) notFoundCleanup() - const [linkedReports, slotBoard, media] = await Promise.all([ - hydrateLinkedReports(id, record.eventKind), - hydrateSlots(id, requesterUserId), - eventMediaUrls(record, { gallery: true }), - ]) - return enrichOne( - toCleanupDTO(record, isAttending(standing), linkedReports, requesterRole, { - slots: slotBoard, - myCapabilities: capabilityList(standing), - ...media, - }), - requesterUserId, - ) + return hydrateDetail(id, record, standing, requesterUserId) }, async listCleanups( @@ -1423,29 +1220,7 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { viewerId: viewer.userId, } const { records, nextCursor } = await deps.repo.listCleanups(filters) - const ids = records.map((r) => r.id) - - const [standingsById, linkedByCleanup, slotCounts, coverUrls] = await Promise.all([ - standingsOf(ids, viewer.userId), - hydrateLinkedReportsForMany(records), - deps.repo.slotCountsFor(ids), - hydrateCoverUrls(records), - ]) - const items = records.map((record) => { - const standing = standingsById.get(record.id) ?? NO_HOST_STANDING - return toCleanupDTO( - record, - isAttending(standing), - linkedByCleanup.get(record.id) ?? [], - standing.eventRole, - { - slotCount: slotCounts.get(record.id) ?? 0, - myCapabilities: capabilityList(standing), - coverUrl: coverUrls.get(record.id) ?? null, - }, - ) - }) - return { items: await enrichDTOs(items, viewer.userId), nextCursor } + return { items: await hydrateListItems(records, viewer.userId), nextCursor } }, async listOrganizationEvents( @@ -1463,34 +1238,14 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { cursor: query.cursor, limit: query.limit, }) - const ids = records.map((r) => r.id) const presign = deps.presignEventMedia - const [standingsById, linkedByCleanup, slotCounts, coverUrls, organizationLogoUrl] = - await Promise.all([ - standingsOf(ids, viewer.userId), - hydrateLinkedReportsForMany(records), - deps.repo.slotCountsFor(ids), - hydrateCoverUrls(records), - host.organization.logoKey === null || presign === undefined - ? Promise.resolve(null) - : presign(host.organization.logoKey, { forceSigned: false }), - ]) - const items = records.map((record) => { - const standing = standingsById.get(record.id) ?? NO_HOST_STANDING - return toCleanupDTO( - record, - isAttending(standing), - linkedByCleanup.get(record.id) ?? [], - standing.eventRole, - { - slotCount: slotCounts.get(record.id) ?? 0, - myCapabilities: capabilityList(standing), - coverUrl: coverUrls.get(record.id) ?? null, - organizationLogoUrl, - }, - ) - }) - return { items: await enrichDTOs(items, viewer.userId), nextCursor } + const logoKey = host.organization.logoKey + const items = await hydrateListItems(records, viewer.userId, () => + logoKey === null || presign === undefined + ? Promise.resolve(null) + : presign(logoKey, { forceSigned: false }), + ) + return { items, nextCursor } }, async getCleanup(id: string, viewer: CleanupViewer): Promise { @@ -1501,28 +1256,14 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { if (!record) notFoundCleanup() const standing = await standingOf(record.id, viewer.userId) assertVisible(record, standing) - const [linkedReports, slotBoard, media] = await Promise.all([ - hydrateLinkedReports(record.id, record.eventKind), - hydrateSlots(record.id, viewer.userId), - eventMediaUrls(record, { gallery: true }), - ]) - return enrichOne( - toCleanupDTO(record, isAttending(standing), linkedReports, standing.eventRole, { - slots: slotBoard, - myCapabilities: capabilityList(standing), - ...media, - }), - viewer.userId, - ) + return hydrateDetail(record.id, record, standing, viewer.userId) }, async joinCleanup(id: string, userId: string): Promise<{ joined: boolean; going: number }> { await assertMembershipFlipBudget(id, userId) const outcome = await deps.repo.joinCleanupTx(id, userId, newSignupSeat()) if (outcome === "not_found") notFoundCleanup() - if (outcome === "banned") { - throw AppError.forbidden("A host removed you from this event, so you can't rejoin it.") - } + if (outcome === "banned") throw AppError.forbidden(REMOVED_BY_HOST_MESSAGE) if (outcome === "closed") throw AppError.conflict(EVENT_CLOSED_MESSAGE) if (outcome === "ended") throw eventEndedError() await deps.insightsInvalidator?.bumpInsightsGeneration(id) @@ -1563,18 +1304,7 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { limit: actsAsHost ? EVENT_HOURS_MEMBER_CAP : ATTENDEES_DEFAULT_LIMIT, }) if (actsAsHost && viewer.userId !== null) { - const seen = await counters.incr( - `host:rosterAudit:${id}:${viewer.userId}`, - ROSTER_AUDIT_DEDUPE_WINDOW_SEC, - ) - if (seen === 1) { - await audit.record({ - actorId: viewer.userId, - action: "event.roster_viewed", - target: `cleanup:${id}`, - meta: { returned: views.length }, - }) - } + await recordRosterView(id, viewer.userId, views.length) } const attendees = await attachAffiliations( deps.affiliations, @@ -1604,22 +1334,14 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { await deps.repo.unbanMember(id, targetUserId) return { ok: true } } - throw AppError.notFound("That person isn't attending this event.") + throw AppError.notFound(NOT_ATTENDING_MESSAGE) } if (targetRole === role) return { ok: true } - const flips = await counters.incr( - `cleanup:role:${id}:${targetUserId}`, - ROLE_CHANGE_WINDOW_SEC, - ) - if (flips > ROLE_CHANGES_PER_TARGET_PER_WINDOW) { - throw AppError.rateLimited( - "This attendee's role has been changed too many times recently. Please try again later.", - ) - } + await assertRoleChangeBudget(id, targetUserId) const flipped = await deps.repo.setMemberRole(id, targetUserId, role) - if (!flipped) throw AppError.notFound("That person isn't attending this event.") + if (!flipped) throw AppError.notFound(NOT_ATTENDING_MESSAGE) await audit.record({ actorId, @@ -1627,10 +1349,11 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { target: `cleanup:${id}`, meta: { targetUserId, from: targetRole, to: role }, }) - await notifyRoleChange(targetUserId, role === "member" ? "demoted" : "promoted", { - id: record.id, - title: record.title, - }) + await notifications.notifyRoleChange( + targetUserId, + role === "member" ? "demoted" : "promoted", + { id: record.id, title: record.title }, + ) return { ok: true } }, @@ -1648,7 +1371,7 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { } const targetRole = await deps.repo.roleOf(id, targetUserId) if (targetRole === null) { - throw AppError.notFound("That person isn't attending this event.") + throw AppError.notFound(NOT_ATTENDING_MESSAGE) } if (targetRole !== "member" && !can(standing, "manage_team")) { throw AppError.forbidden(hostForbiddenCopy("manage_team")) @@ -1658,7 +1381,7 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { if (outcome.kind === "not_found") notFoundCleanup() if (outcome.kind === "closed") throw AppError.conflict(EVENT_CLOSED_MESSAGE) if (outcome.kind === "not_member") { - throw AppError.notFound("That person isn't attending this event.") + throw AppError.notFound(NOT_ATTENDING_MESSAGE) } await enqueueWaitlistPromotion(deps.jobs, outcome.releasedWaitlistTicketTypeIds, deps.logger) @@ -1670,17 +1393,15 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { target: `cleanup:${id}`, meta: { targetUserId, role: targetRole }, }) - await notifyRoleChange(targetUserId, "removed", { id: record.id, title: record.title }) + await notifications.notifyRoleChange(targetUserId, "removed", { + id: record.id, + title: record.title, + }) return { ok: true, going: outcome.going } }, async claimEventSlot(id: string, userId: string, slotId: string | null): Promise { - const flips = await counters.incr(`cleanup:slot:${id}:${userId}`, SLOT_FLIP_WINDOW_SEC) - if (flips > SLOT_FLIPS_PER_EVENT_PER_WINDOW) { - throw AppError.rateLimited( - "You've changed your slot too many times recently. Please try again later.", - ) - } + await assertSlotFlipBudget(id, userId) const [record, standing] = await Promise.all([ deps.repo.findCleanupById(id, null), @@ -1698,9 +1419,7 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { if (outcome.kind === "slot_not_found") { throw AppError.notFound("That slot no longer exists.") } - if (outcome.kind === "banned") { - throw AppError.forbidden("A host removed you from this event, so you can't rejoin it.") - } + if (outcome.kind === "banned") throw AppError.forbidden(REMOVED_BY_HOST_MESSAGE) if (outcome.kind === "closed") throw AppError.conflict(EVENT_CLOSED_MESSAGE) if (outcome.kind === "ended") throw eventEndedError() if (outcome.kind === "full") throw AppError.conflict("That slot is already full.") @@ -1710,19 +1429,7 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { const updated = await deps.repo.findCleanupById(id, null) if (!updated) notFoundCleanup() const nextStanding = await standingOf(id, userId) - const [linkedReports, slotBoard, media] = await Promise.all([ - hydrateLinkedReports(id, updated.eventKind), - hydrateSlots(id, userId), - eventMediaUrls(updated, { gallery: true }), - ]) - return enrichOne( - toCleanupDTO(updated, isAttending(nextStanding), linkedReports, nextStanding.eventRole, { - slots: slotBoard, - myCapabilities: capabilityList(nextStanding), - ...media, - }), - userId, - ) + return hydrateDetail(id, updated, nextStanding, userId) }, async requestResources(input: { @@ -1730,7 +1437,8 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { message: string actorId: string }): Promise { - if (deps.outboundMail === undefined) { + const outboundMail = deps.outboundMail + if (outboundMail === undefined) { throw AppError.internal("Event resource requests are not available") } const { record, standing } = await requireCapabilityOn( @@ -1754,48 +1462,8 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { ) } - const hostSends = await counters.incr( - `cleanup:res-req:host:${input.actorId}`, - RESOURCE_REQUEST_HOST_WINDOW_SEC, - ) - // The shared jurisdiction budget is charged only after the host's own cap passes, so one - // host hammering past its limit cannot exhaust the area for every other host. - if (hostSends > RESOURCE_REQUEST_PER_HOST_PER_DAY) { - throw AppError.rateLimited( - "You've sent the maximum number of resource requests for today. Please try again tomorrow.", - ) - } - const jurisdictionSends = await counters.incr( - `cleanup:res-req:jur:${record.jurisdictionGeoid ?? "unknown"}`, - RESOURCE_REQUEST_JURISDICTION_WINDOW_SEC, - ) - if (jurisdictionSends > RESOURCE_REQUEST_PER_JURISDICTION_PER_HOUR) { - throw AppError.rateLimited( - "This area has received too many resource requests in the past hour. Please try again later.", - ) - } - - const packet = buildEventPacket( - { - title: record.title, - host: record.organizer.displayName, - place: routing.name, - address: record.address, - lat: record.lat, - lng: record.lng, - referenceCode: record.referenceCode, - }, - input.message, - ) - await deps.outboundMail.sendEventToJurisdiction({ - cleanupId: record.id, - geoid: record.jurisdictionGeoid, - org: routing.name, - toAddr: routing.contact, - subject: packet.subject, - text: packet.text, - html: packet.html, - }) + await assertResourceRequestBudget(input.actorId, record.jurisdictionGeoid) + await sendResourceRequest(outboundMail, record, routing, input.message) await deps.repo.appendCleanupTimeline(record.id, { kind: "resource_request", @@ -1808,7 +1476,11 @@ export function makeCleanupService(deps: CleanupServiceDeps): CleanupService { async runCancelFanout(job: CleanupCancelFanoutJob): Promise { const record = await deps.repo.findCleanupById(job.cleanupId, null) if (record === null) return - await notifyCancellation({ id: record.id, title: record.title }, job.reason, job.actorId) + await notifications.notifyCancellation( + { id: record.id, title: record.title }, + job.reason, + job.actorId, + ) }, } } @@ -1831,6 +1503,9 @@ function guestVisibleChange( function resourceRequestNote(message: string): string { const collapsed = message.replace(/\s+/g, " ").trim() - const preview = collapsed.length > 140 ? `${collapsed.slice(0, 140)}…` : collapsed + const preview = + collapsed.length > RESOURCE_NOTE_PREVIEW_CHARS + ? `${collapsed.slice(0, RESOURCE_NOTE_PREVIEW_CHARS)}…` + : collapsed return preview.length > 0 ? `Resources requested: ${preview}` : "Resources requested" } diff --git a/services/api/src/services/cleanup-slots.ts b/services/api/src/services/cleanup-slots.ts new file mode 100644 index 00000000..d7f3f9fe --- /dev/null +++ b/services/api/src/services/cleanup-slots.ts @@ -0,0 +1,104 @@ +import { AppError, MAX_EVENT_SLOTS, MIN_SLOT_DURATION_MINUTES } from "@civfix/shared" +import type { EventSlotInput } from "@civfix/shared" +import { assertNoSlur } from "../abuse/slur-filter.js" +import type { DesiredSlot, EventSlotView } from "./cleanup-repository.types.js" +import type { EventWindow } from "./cleanup-rules.js" + +const MS_PER_MINUTE = 60_000 + +const MIN_SLOT_DURATION_MS = MIN_SLOT_DURATION_MINUTES * MS_PER_MINUTE + +interface SlotWindow { + startsAt: Date | null + endsAt: Date | null +} + +function slotWindowOf(slot: EventSlotInput): SlotWindow { + const startsAt = slot.startsAt != null ? new Date(slot.startsAt) : null + const endsAt = slot.endsAt != null ? new Date(slot.endsAt) : null + return { startsAt, endsAt } +} + +function assertSlotInsideEvent( + title: string, + window: { startsAt: Date; endsAt: Date }, + event: EventWindow, +): void { + if (event.endsAt === null) { + throw AppError.validation({ + slots: "set an end time for the event before adding timed slots", + }) + } + if (window.startsAt < event.scheduledAt || window.endsAt > event.endsAt) { + throw AppError.validation({ + slots: `slot "${title}" falls outside the event's start and end`, + }) + } + if (window.endsAt.getTime() - window.startsAt.getTime() < MIN_SLOT_DURATION_MS) { + throw AppError.validation({ + slots: `slot "${title}" must last at least ${MIN_SLOT_DURATION_MINUTES} minutes`, + }) + } +} + +function slotIdentityKey(title: string, window: SlotWindow): string { + return `${title.trim().toLowerCase()}|${window.startsAt?.getTime() ?? ""}|${window.endsAt?.getTime() ?? ""}` +} + +export function toDesiredSlots( + slots: EventSlotInput[], + opts: { keepIds: boolean }, + window: EventWindow, +): DesiredSlot[] { + if (slots.length > MAX_EVENT_SLOTS) { + throw AppError.validation({ slots: `at most ${MAX_EVENT_SLOTS} slots may be listed` }) + } + const seen = new Set() + for (const slot of slots) { + const slotWindow = slotWindowOf(slot) + const { startsAt, endsAt } = slotWindow + if (startsAt !== null && endsAt !== null) { + assertSlotInsideEvent(slot.title, { startsAt, endsAt }, window) + } + const key = slotIdentityKey(slot.title, slotWindow) + if (seen.has(key)) { + throw AppError.validation({ slots: `duplicate slot title: ${slot.title}` }) + } + seen.add(key) + assertNoSlur(slot.title, "slots") + assertNoSlur(slot.description ?? null, "slots") + } + return slots.map((slot, index) => ({ + ...(opts.keepIds && slot.id !== undefined ? { id: slot.id } : {}), + title: slot.title, + description: slot.description ?? null, + capacity: slot.capacity ?? null, + ...slotWindowOf(slot), + sortOrder: slot.sortOrder ?? index, + })) +} + +/** Stored timed slots must still fit when an edit moves the event window without resending them. */ +export function assertTimedSlotsFitWindow(stored: EventSlotView[], window: EventWindow): void { + const outside = stored.some( + (slot) => + slot.startsAt !== null && + slot.endsAt !== null && + (window.endsAt === null || slot.startsAt < window.scheduledAt || slot.endsAt > window.endsAt), + ) + if (outside) { + throw AppError.validation({ + scheduledAt: + "timed slots would fall outside the new start and end; update the slots in the same save", + }) + } +} + +export function assertKnownSlotIds(desired: DesiredSlot[], stored: EventSlotView[]): void { + const existingSlotIds = new Set(stored.map((s) => s.id)) + for (const slot of desired) { + if (slot.id !== undefined && !existingSlotIds.has(slot.id)) { + throw AppError.validation({ slots: `unknown slot: ${slot.id}` }) + } + } +} diff --git a/services/api/src/services/content-report-subject.ts b/services/api/src/services/content-report-subject.ts index cf5d70c1..ac538d9d 100644 --- a/services/api/src/services/content-report-subject.ts +++ b/services/api/src/services/content-report-subject.ts @@ -89,14 +89,7 @@ async function isChatMessageReportable( SELECT thread_id FROM dm_messages WHERE id = ${messageId} LIMIT 1 ` const dm = dmRows[0] - if (dm) { - const member = await sql<{ ok: number }[]>` - SELECT 1 AS ok FROM dm_threads - WHERE id = ${dm.thread_id} AND (user_lo = ${reporterUserId} OR user_hi = ${reporterUserId}) - LIMIT 1 - ` - return member.length > 0 - } + if (dm) return isDmParticipant(sql, dm.thread_id, reporterUserId) const chatRows = await sql< { cleanup_id: string | null; report_id: string | null; group_id: string | null }[] @@ -106,46 +99,73 @@ async function isChatMessageReportable( ` const msg = chatRows[0] if (!msg) return false + if (msg.cleanup_id !== null) return isCleanupMember(sql, msg.cleanup_id, reporterUserId) + if (msg.group_id !== null) return isGroupMessageVisible(sql, msg.group_id, reporterUserId) + if (msg.report_id !== null) return isReportChatVisible(sql, msg.report_id, reporterUserId) + return false +} + +async function isDmParticipant( + sql: Sql, + threadId: string, + reporterUserId: string, +): Promise { + const member = await sql<{ ok: number }[]>` + SELECT 1 AS ok FROM dm_threads + WHERE id = ${threadId} AND (user_lo = ${reporterUserId} OR user_hi = ${reporterUserId}) + LIMIT 1 + ` + return member.length > 0 +} - if (msg.cleanup_id !== null) { - const member = await sql<{ ok: number }[]>` +async function isCleanupMember( + sql: Sql, + cleanupId: string, + reporterUserId: string, +): Promise { + const member = await sql<{ ok: number }[]>` SELECT 1 AS ok FROM cleanup_members - WHERE cleanup_id = ${msg.cleanup_id} AND user_id = ${reporterUserId} LIMIT 1 + WHERE cleanup_id = ${cleanupId} AND user_id = ${reporterUserId} LIMIT 1 ` - return member.length > 0 - } + return member.length > 0 +} - if (msg.group_id !== null) { - const rows = await sql<{ visibility: string; is_member: boolean }[]>` +async function isGroupMessageVisible( + sql: Sql, + groupId: string, + reporterUserId: string, +): Promise { + const rows = await sql<{ visibility: string; is_member: boolean }[]>` SELECT g.visibility, EXISTS ( SELECT 1 FROM chat_group_members m WHERE m.group_id = g.id AND m.user_id = ${reporterUserId} ) AS is_member - FROM chat_groups g WHERE g.id = ${msg.group_id} LIMIT 1 + FROM chat_groups g WHERE g.id = ${groupId} LIMIT 1 ` - const group = rows[0] - if (!group) return false - return group.is_member || group.visibility === "public" - } + const group = rows[0] + if (!group) return false + return group.is_member || group.visibility === "public" +} - if (msg.report_id !== null) { - const rows = await sql< - { - reporter_user_id: string | null - status: string - visibility: string - deleted_at: Date | null - }[] - >` +async function isReportChatVisible( + sql: Sql, + reportId: string, + reporterUserId: string, +): Promise { + const rows = await sql< + { + reporter_user_id: string | null + status: string + visibility: string + deleted_at: Date | null + }[] + >` SELECT reporter_user_id, status, visibility, deleted_at - FROM reports WHERE id = ${msg.report_id} LIMIT 1 + FROM reports WHERE id = ${reportId} LIMIT 1 ` - const report = rows[0] - if (!report || report.deleted_at !== null) return false - if (isPubliclyVisibleStatus(report.status) && report.visibility === "public") return true - return report.reporter_user_id === reporterUserId - } - - return false + const report = rows[0] + if (!report || report.deleted_at !== null) return false + if (isPubliclyVisibleStatus(report.status) && report.visibility === "public") return true + return report.reporter_user_id === reporterUserId } diff --git a/services/api/src/services/deterministic-uuid.ts b/services/api/src/services/deterministic-uuid.ts index b2135f0b..b486fa3d 100644 --- a/services/api/src/services/deterministic-uuid.ts +++ b/services/api/src/services/deterministic-uuid.ts @@ -1,12 +1,22 @@ import { createHash } from "node:crypto" +const UUID_BYTES = 16 const UUID_V5_SHAPE = /^(.{8})(.{4})(.{4})(.{4})(.{12})$/u +/** RFC 4122 section 4.1.3: the high nibble of byte 6 carries the version. */ +const VERSION_BYTE = 6 +const VERSION_KEEP_MASK = 0x0f +const VERSION_5_BITS = 0x50 +/** RFC 4122 section 4.1.1: the top two bits of byte 8 carry the variant. */ +const VARIANT_BYTE = 8 +const VARIANT_KEEP_MASK = 0x3f +const RFC4122_VARIANT_BITS = 0x80 + export function deterministicUuid(parts: readonly string[]): string { const digest = createHash("sha256").update(parts.join(" ")).digest() - const bytes = Buffer.from(digest.subarray(0, 16)) - bytes[6] = ((bytes[6] as number) & 0x0f) | 0x50 - bytes[8] = ((bytes[8] as number) & 0x3f) | 0x80 + const bytes = Buffer.from(digest.subarray(0, UUID_BYTES)) + bytes[VERSION_BYTE] = ((bytes[VERSION_BYTE] as number) & VERSION_KEEP_MASK) | VERSION_5_BITS + bytes[VARIANT_BYTE] = ((bytes[VARIANT_BYTE] as number) & VARIANT_KEEP_MASK) | RFC4122_VARIANT_BITS const hex = bytes.toString("hex") const groups = UUID_V5_SHAPE.exec(hex) if (groups === null) throw new Error("deterministic uuid: unexpected digest shape") diff --git a/services/api/src/services/dm-peer.ts b/services/api/src/services/dm-peer.ts index 1ed7ae42..ecf762ab 100644 --- a/services/api/src/services/dm-peer.ts +++ b/services/api/src/services/dm-peer.ts @@ -3,7 +3,7 @@ import type { DmRepository, DmThread } from "./dm-repository.drizzle.js" -export function dmPeerOfThread(thread: DmThread, userId: string): string | null { +function dmPeerOfThread(thread: DmThread, userId: string): string | null { if (thread.userLo === userId) return thread.userHi if (thread.userHi === userId) return thread.userLo return null diff --git a/services/api/src/services/dm-repository.drizzle.ts b/services/api/src/services/dm-repository.drizzle.ts index 23572daa..6ec0c876 100644 --- a/services/api/src/services/dm-repository.drizzle.ts +++ b/services/api/src/services/dm-repository.drizzle.ts @@ -32,7 +32,7 @@ import { liveMessageIds, toTombstoneDTO } from "./chat-tombstone.js" // dm_messages is range-partitioned on created_at and an ack carries only the message id, so the bound // lets the planner prune the lookup to recent partitions instead of probing every month ever created. -export const DM_ACK_LOOKUP_WINDOW_DAYS = 90 +const DM_ACK_LOOKUP_WINDOW_DAYS = 90 export interface DmThread { id: string @@ -144,27 +144,30 @@ interface DmRowSelect { sender_deleted_at: Date | null } -function toMessageDTO( - r: DmRowSelect, - reactions: ReactionSummaryDTO[], - mentions: UserMentionDTO[], - viewerUserId?: string | null, - clientId?: string, - attachments: MediaDTO[] = [], - replyTo?: ReplyToDTO | null, -): ChatMessageDTO { - const dto = buildMessageDTO(r, reactions, mentions, viewerUserId, clientId, attachments, replyTo) +interface MessageExtras { + reactions?: ReactionSummaryDTO[] + mentions?: UserMentionDTO[] + viewerUserId?: string | null + clientId?: string + attachments?: MediaDTO[] + replyTo?: ReplyToDTO | null +} + +function toMessageDTO(r: DmRowSelect, extras: MessageExtras = {}): ChatMessageDTO { + const dto = buildMessageDTO(r, extras) return r.deleted_at !== null ? toTombstoneDTO(dto, r.deleted_at) : dto } function buildMessageDTO( r: DmRowSelect, - reactions: ReactionSummaryDTO[], - mentions: UserMentionDTO[], - viewerUserId?: string | null, - clientId?: string, - attachments: MediaDTO[] = [], - replyTo?: ReplyToDTO | null, + { + reactions = [], + mentions = [], + viewerUserId, + clientId, + attachments = [], + replyTo, + }: MessageExtras, ): ChatMessageDTO { const author = publicAuthorIdentity({ id: r.sender_id, @@ -204,6 +207,18 @@ function buildMessageDTO( } } +function replyFor( + row: Pick, + replyByTarget: Map, +): ReplyToDTO | null { + return row.reply_to_id !== null ? (replyByTarget.get(row.reply_to_id) ?? null) : null +} + +// dm_threads stores each pair once as (user_lo, user_hi), so both lookups and the insert order the ids. +function orderedPair(userA: string, userB: string): [string, string] { + return userA < userB ? [userA, userB] : [userB, userA] +} + function selectDmRowFrom(tag: Queryable, cte: string) { return tag` SELECT @@ -263,15 +278,13 @@ export function makeDrizzleDmRepository(sql: Sql, presign?: PresignMedia): DmRep replyMapForRows(sql, "dm_messages", page), ]) return page.map((r) => - toMessageDTO( - r, - reactionsByMessage.get(r.id) ?? [], - mentionsByMessage.get(r.id) ?? [], + toMessageDTO(r, { + reactions: reactionsByMessage.get(r.id) ?? [], + mentions: mentionsByMessage.get(r.id) ?? [], viewerUserId, - undefined, - attachmentsByMessage.get(r.id) ?? [], - r.reply_to_id !== null ? (replyByTarget.get(r.reply_to_id) ?? null) : null, - ), + attachments: attachmentsByMessage.get(r.id) ?? [], + replyTo: replyFor(r, replyByTarget), + }), ) } @@ -288,15 +301,13 @@ export function makeDrizzleDmRepository(sql: Sql, presign?: PresignMedia): DmRep : Promise.resolve(new Map()), replyMapForRows(sql, "dm_messages", [row]), ]) - return toMessageDTO( - row, + return toMessageDTO(row, { reactions, mentions, viewerUserId, - undefined, - attachmentsByMessage.get(row.id) ?? [], - row.reply_to_id !== null ? (replyByTarget.get(row.reply_to_id) ?? null) : null, - ) + attachments: attachmentsByMessage.get(row.id) ?? [], + replyTo: replyFor(row, replyByTarget), + }) } function roomSql(threadId: string): RoomScopeSql { @@ -317,8 +328,7 @@ export function makeDrizzleDmRepository(sql: Sql, presign?: PresignMedia): DmRep return { async openOrCreateThread(userA: string, userB: string): Promise { - const lo = userA < userB ? userA : userB - const hi = userA < userB ? userB : userA + const [lo, hi] = orderedPair(userA, userB) const inserted = await sql<{ id: string; created_at: Date }[]>` INSERT INTO dm_threads (user_lo, user_hi) @@ -337,8 +347,7 @@ export function makeDrizzleDmRepository(sql: Sql, presign?: PresignMedia): DmRep }, async getThreadForPair(userA: string, userB: string): Promise { - const lo = userA < userB ? userA : userB - const hi = userA < userB ? userB : userA + const [lo, hi] = orderedPair(userA, userB) const rows = await sql<{ id: string; created_at: Date }[]>` SELECT id, created_at FROM dm_threads WHERE user_lo = ${lo} AND user_hi = ${hi} LIMIT 1 ` @@ -409,7 +418,12 @@ export function makeDrizzleDmRepository(sql: Sql, presign?: PresignMedia): DmRep ? ((await loadChatAttachments(sql, [messageId], presign!, input.senderId)).get(messageId) ?? []) : [] - return toMessageDTO(rows[0]!, [], [], input.senderId, input.clientId, attachments, replyTo) + return toMessageDTO(rows[0]!, { + viewerUserId: input.senderId, + clientId: input.clientId, + attachments, + replyTo, + }) }, async editMessage( @@ -483,15 +497,7 @@ export function makeDrizzleDmRepository(sql: Sql, presign?: PresignMedia): DmRep const row = rows[0] if (!row) return null const replyByTarget = await replyMapForRows(sql, "dm_messages", [row]) - return toMessageDTO( - row, - [], - [], - senderId, - undefined, - [], - row.reply_to_id !== null ? (replyByTarget.get(row.reply_to_id) ?? null) : null, - ) + return toMessageDTO(row, { viewerUserId: senderId, replyTo: replyFor(row, replyByTarget) }) }, history( diff --git a/services/api/src/services/dm-repository.memory.ts b/services/api/src/services/dm-repository.memory.ts index 3dbd00aa..dc09d3c9 100644 --- a/services/api/src/services/dm-repository.memory.ts +++ b/services/api/src/services/dm-repository.memory.ts @@ -3,7 +3,7 @@ import { avatarGradient, AppError } from "@civfix/shared" import type { ChatMessageDTO, ReactionEmoji, ReactionSummaryDTO, ReplyToDTO } from "@civfix/shared" import type { ChatHistoryPage } from "@civfix/shared/interfaces" import { REPLY_EXCERPT_MAX, replyDeletedTarget, replyWrongRoom } from "./chat-reply-hydration.js" -import { PIN_LIST_CAP } from "./chat-repository.drizzle.js" +import { PIN_LIST_CAP } from "./chat-room-scope.drizzle.js" import { publicAuthorIdentity } from "./public-author.js" import { aroundLimits } from "./chat-history-window.js" import { toTombstoneDTO } from "./chat-tombstone.js" @@ -42,10 +42,29 @@ interface StoredDmMessage { insertedAtMs: number } +// Writes are stamped at fixed one-millisecond steps so ordering is deterministic. +const SYNTHETIC_EPOCH_MS = Date.UTC(2026, 0, 1) + +const PLACEHOLDER_NAME_ID_CHARS = 4 + function orderPair(a: string, b: string): [string, string] { return a < b ? [a, b] : [b, a] } +function placeholderDisplayName(id: string): string { + return `User ${id.slice(0, PLACEHOLDER_NAME_ID_CHARS)}` +} + +function readKey(threadId: string, userId: string): string { + return `${threadId}:${userId}` +} + +function peerIn(thread: DmThread, userId: string): string | null { + if (thread.userLo === userId) return thread.userHi + if (thread.userHi === userId) return thread.userLo + return null +} + function lastSenderId(message: ChatMessageDTO): string { if (!message.from) throw new Error("DM message unexpectedly has no author") return message.from.id @@ -71,13 +90,13 @@ export class InMemoryDmRepository implements DmRepository { private userOf(id: string): DmUser { return ( - this.users.get(id) ?? { id, displayName: `User ${id.slice(0, 4)}`, handle: null, bio: null } + this.users.get(id) ?? { id, displayName: placeholderDisplayName(id), handle: null, bio: null } ) } private nextDate(): Date { this.tick += 1 - return new Date(Date.UTC(2026, 0, 1, 0, 0, 0, this.tick)) + return new Date(SYNTHETIC_EPOCH_MS + this.tick) } private replyToFor(threadId: string, replyToId: string): ReplyToDTO | null { @@ -130,10 +149,7 @@ export class InMemoryDmRepository implements DmRepository { peerOf(threadId: string, userId: string): string | null { const t = this.threads.get(threadId) - if (!t) return null - if (t.userLo === userId) return t.userHi - if (t.userHi === userId) return t.userLo - return null + return t ? peerIn(t, userId) : null } persist(input: DmPersistInput): Promise { @@ -371,14 +387,14 @@ export class InMemoryDmRepository implements DmRepository { } markRead(threadId: string, userId: string, at: Date): Promise { - const key = `${threadId}:${userId}` + const key = readKey(threadId, userId) const prev = this.reads.get(key) ?? 0 if (at.getTime() > prev) this.reads.set(key, at.getTime()) return Promise.resolve() } lastReadAt(threadId: string, userId: string): Promise { - const ms = this.reads.get(`${threadId}:${userId}`) + const ms = this.reads.get(readKey(threadId, userId)) return Promise.resolve(ms !== undefined ? new Date(ms) : null) } @@ -387,7 +403,7 @@ export class InMemoryDmRepository implements DmRepository { if (!thread) return Promise.resolve(0) const baseline = Math.max( thread.createdAt.getTime(), - this.reads.get(`${threadId}:${userId}`) ?? 0, + this.reads.get(readKey(threadId, userId)) ?? 0, ) const unread = (this.log.get(threadId) ?? []).filter( (m) => @@ -408,7 +424,7 @@ export class InMemoryDmRepository implements DmRepository { ): Promise { const out: DmThreadAggregate[] = [] for (const t of this.threads.values()) { - const peerId = t.userLo === userId ? t.userHi : t.userHi === userId ? t.userLo : null + const peerId = peerIn(t, userId) if (peerId === null) continue if (this.isBlockedEitherWay && (await this.isBlockedEitherWay(userId, peerId))) continue @@ -422,7 +438,7 @@ export class InMemoryDmRepository implements DmRepository { }) const live = (this.log.get(t.id) ?? []).filter((m) => !m.deleted).map((m) => m.dto) const last = live.length > 0 ? live[live.length - 1]! : null - const lastReadMs = this.reads.get(`${t.id}:${userId}`) ?? 0 + const lastReadMs = this.reads.get(readKey(t.id, userId)) ?? 0 const baseline = Math.max(t.createdAt.getTime(), lastReadMs) const unread = live.filter( (m) => m.from?.id === peerId && new Date(m.createdAt).getTime() > baseline, @@ -514,7 +530,7 @@ export class InMemoryBlocksRepository implements BlocksRepository { const limit = args?.limit ?? LIST_BLOCKS_DEFAULT_LIMIT const ids = [...(this.edges.get(blockerId) ?? [])].slice(0, limit) const blocked: PersonDTO[] = ids.map((id) => { - const u = this.users.get(id) ?? { id, displayName: `User ${id.slice(0, 4)}` } + const u = this.users.get(id) ?? { id, displayName: placeholderDisplayName(id) } return { id: u.id, name: u.displayName, diff --git a/services/api/src/services/dm-service.ts b/services/api/src/services/dm-service.ts index 6d1a118f..501e2084 100644 --- a/services/api/src/services/dm-service.ts +++ b/services/api/src/services/dm-service.ts @@ -5,6 +5,8 @@ import type { DmRepository } from "./dm-repository.drizzle.js" export const DM_FORBIDDEN_MESSAGE = "You can't message this account." +const DM_MEMBER_COUNT = 2 + export interface DmTargetUser { id: string displayName: string @@ -50,6 +52,11 @@ function peerOf(target: DmTargetUser): PersonDTO { } } +function dmTitle(target: DmTargetUser): string { + if (target.displayName.trim() !== "") return target.displayName + return target.handle !== null ? `@${target.handle}` : target.displayName +} + export function makeDmService(deps: DmServiceDeps): DmService { const now = deps.now ?? (() => new Date()) @@ -83,12 +90,7 @@ export function makeDmService(deps: DmServiceDeps): DmService { const lastAt = last !== null ? new Date(last.createdAt) : null const peer = peerOf(target) - const title = - target.displayName.trim() !== "" - ? target.displayName - : target.handle !== null - ? `@${target.handle}` - : target.displayName + const title = dmTitle(target) return { id: thread.id, @@ -101,7 +103,7 @@ export function makeDmService(deps: DmServiceDeps): DmService { lastMessageAt: lastAt !== null ? lastAt.toISOString() : null, lastFromMe: last !== null && last.from?.id === viewerId, unread, - members: 2, + members: DM_MEMBER_COUNT, muted, } }, diff --git a/services/api/src/services/group-chat-notifier.ts b/services/api/src/services/group-chat-notifier.ts index ec358c1f..9dc7d8bb 100644 --- a/services/api/src/services/group-chat-notifier.ts +++ b/services/api/src/services/group-chat-notifier.ts @@ -1,40 +1,14 @@ import type { ChatMessageDTO } from "@civfix/shared" -import type { FastifyBaseLogger } from "fastify" -import type { NotificationService } from "./notification-service.js" -import { makeRoomFanoutNotifier, ROOM_FANOUT_SPEC } from "./chat-room-fanout-notifier.js" +import { makeRoomChatNotifier, type RoomChatNotifierDeps } from "./room-chat-notifier-adapter.js" -export interface GroupChatNotifierDeps { - notificationService: Pick +export interface GroupChatNotifierDeps extends RoomChatNotifierDeps<"group"> { groupRepo: { listMemberIds(groupId: string, limit: number): Promise } - isMuted: (userId: string, roomId: string) => Promise - mutedUserIdsFor?: (roomId: string, userIds: string[]) => Promise> - presence?: { online(roomKey: string): Promise } - roomKeyFor: (kind: "group", id: string) => string - isBlockedEitherWay: (a: string, b: string) => Promise - blockedIdsFor?: (actorId: string, candidateIds: string[]) => Promise> - coalesceWindowMs?: number - now?: () => number - claimWindow?: (roomId: string, windowMs: number) => Promise - dispatchToJob?: (roomId: string, messageId: string) => Promise - logger?: Pick | undefined } export function makeGroupChatNotifier( deps: GroupChatNotifierDeps, ): (groupId: string, message: ChatMessageDTO) => Promise { - return makeRoomFanoutNotifier(ROOM_FANOUT_SPEC.group, { - notificationService: deps.notificationService, - listMemberIds: (groupId, limit) => deps.groupRepo.listMemberIds(groupId, limit), - isMuted: deps.isMuted, - ...(deps.mutedUserIdsFor ? { mutedUserIdsFor: deps.mutedUserIdsFor } : {}), - presence: deps.presence, - roomKey: (groupId) => deps.roomKeyFor("group", groupId), - isBlockedEitherWay: deps.isBlockedEitherWay, - ...(deps.blockedIdsFor ? { blockedIdsFor: deps.blockedIdsFor } : {}), - ...(deps.coalesceWindowMs !== undefined ? { coalesceWindowMs: deps.coalesceWindowMs } : {}), - ...(deps.now !== undefined ? { now: deps.now } : {}), - ...(deps.claimWindow !== undefined ? { claimWindow: deps.claimWindow } : {}), - ...(deps.dispatchToJob !== undefined ? { dispatchToJob: deps.dispatchToJob } : {}), - ...(deps.logger !== undefined ? { logger: deps.logger } : {}), - }) + return makeRoomChatNotifier("group", deps, (groupId, limit) => + deps.groupRepo.listMemberIds(groupId, limit), + ) } diff --git a/services/api/src/services/guest-jobs.ts b/services/api/src/services/guest-jobs.ts index e57d037c..fd8cb0a8 100644 --- a/services/api/src/services/guest-jobs.ts +++ b/services/api/src/services/guest-jobs.ts @@ -9,7 +9,9 @@ import { type GuestUpdateFanoutJob, } from "./guest-rsvp-service.js" -export { CLEANUP_GUEST_UPDATE_FANOUT_JOB, GUEST_RETENTION_SWEEP_JOB } +const GUEST_UPDATE_SINGLETON_PREFIX = "guest-update:" + +const GUEST_UPDATE_FANOUT_RETRY_LIMIT = 3 export interface GuestUpdateFanoutDeps { announce: (cleanupId: string) => Promise @@ -63,7 +65,11 @@ export async function registerGuestJobs( await container.jobs.enqueue( CLEANUP_GUEST_UPDATE_FANOUT_JOB, { cleanupId }, - { singletonKey: `guest-update:${cleanupId}`, startAfter: startAfterSec, retryLimit: 3 }, + { + singletonKey: `${GUEST_UPDATE_SINGLETON_PREFIX}${cleanupId}`, + startAfter: startAfterSec, + retryLimit: GUEST_UPDATE_FANOUT_RETRY_LIMIT, + }, ) }, ...(logger !== undefined ? { logger } : {}), @@ -86,7 +92,7 @@ export async function registerGuestJobs( }) } -export function parseUpdateFanoutJob(data: unknown): GuestUpdateFanoutJob | null { +function parseUpdateFanoutJob(data: unknown): GuestUpdateFanoutJob | null { if (typeof data !== "object" || data === null) return null const cleanupId = (data as { cleanupId?: unknown }).cleanupId if (typeof cleanupId !== "string" || cleanupId.length === 0) return null diff --git a/services/api/src/services/guest-rsvp-service.ts b/services/api/src/services/guest-rsvp-service.ts index 89ef3ea4..c4c3acdd 100644 --- a/services/api/src/services/guest-rsvp-service.ts +++ b/services/api/src/services/guest-rsvp-service.ts @@ -57,41 +57,63 @@ import { DEFAULT_EVENT_TIME_ZONE } from "./host/event-fields.js" export const MAX_GUESTS_PER_EVENT = 500 -export const GUEST_OTP_TTL_SECONDS = OTP_TTL_SECONDS +const GUEST_OTP_TTL_SECONDS = OTP_TTL_SECONDS -export const GUEST_CONTACT_COOLDOWN_SECONDS = 60 +const GUEST_CONTACT_COOLDOWN_SECONDS = 60 export const GUEST_CONTACT_MAX_PER_DAY = 5 export const GUEST_IP_MAX_PER_HOUR = 10 -export const GUEST_IP_WINDOW_SECONDS = 60 * 60 +const GUEST_IP_WINDOW_SECONDS = 60 * 60 -export const DAY_SECONDS = 24 * 60 * 60 +const DAY_SECONDS = 24 * 60 * 60 -export const GUESTS_DEFAULT_LIMIT = 25 +const GUESTS_DEFAULT_LIMIT = 25 type SmsPurpose = "otp" | "confirmation" | "notice" -export const GUEST_SMS_NOTICE_CONCURRENCY = 8 +const GUEST_SMS_NOTICE_CONCURRENCY = 8 export const SMS_BUDGET_KEY_PREFIX = "sms:day:" -export const SMS_TITLE_MAX_CHARS = 20 +const SMS_TITLE_MAX_CHARS = 20 -export const GUEST_RETENTION_MAX_PAGES = 20 +const GUEST_RETENTION_MAX_PAGES = 20 -export const GUEST_CONTACT_RETENTION_DAYS = 30 +const GUEST_CONTACT_RETENTION_DAYS = 30 -export const GUEST_OTP_RETENTION_HOURS = 24 +const GUEST_OTP_RETENTION_HOURS = 24 -export const GUEST_RETENTION_BATCH = 500 +const GUEST_RETENTION_BATCH = 500 -export const GUEST_CONTACT_READ_COUNTER_KEY = "host:guestContactReads" +const GUEST_CONTACT_READ_COUNTER_KEY = "host:guestContactReads" -export const GUEST_CONTACT_READS_PER_HOUR = 50 +const GUEST_CONTACT_READS_PER_HOUR = 50 -export const GUEST_CONTACT_READ_WINDOW_SECONDS = 60 * 60 +const GUEST_CONTACT_READ_WINDOW_SECONDS = 60 * 60 + +const MS_PER_SECOND = 1000 + +const SECONDS_PER_MINUTE = 60 + +const MS_PER_HOUR = 60 * 60 * MS_PER_SECOND + +const SMS_LOCALE = "en" + +const SMS_PLACE_COORD_DECIMALS = 5 + +const GUEST_CODE_COOLDOWN_KEY_PREFIX = "guest:rl:code:" + +const GUEST_CONTACT_DAY_KEY_PREFIX = "guest:rl:contact:day:" + +const GUEST_IP_KEY_PREFIX = "guest:rl:ip:" + +const GUEST_CODE_FAIL_KEY_PREFIX = "guest:vf:code:" + +const GUEST_IP_FAIL_KEY_PREFIX = "guest:vf:ip:" + +const GUEST_REGISTRATION_IDEMPOTENCY_PREFIX = "guest:" export interface GuestRegistrationFields { ticketTypeId?: string @@ -287,7 +309,7 @@ export interface GuestRsvpService { notifyGuestsBySms(cleanupId: string, kind: "cancelled" | "updated"): Promise } -export function smsUnavailableError(): AppError { +function smsUnavailableError(): AppError { return new AppError( ErrorCode.CONFLICT, "Text message codes aren't available right now. Use email instead.", @@ -295,7 +317,7 @@ export function smsUnavailableError(): AppError { ) } -export function smsOptedOutError(): AppError { +function smsOptedOutError(): AppError { return new AppError( ErrorCode.CONFLICT, "That number has opted out of text messages. Use email instead.", @@ -313,7 +335,7 @@ const RETRY_REQUEST = "Check your details, then try again." export const GUEST_REGISTRATION_ERROR_FIELD = "registration" -export const GuestRegistrationRefusalReason = { +const GuestRegistrationRefusalReason = { soldOut: "sold_out", registrationClosed: "registration_closed", salesClosed: "sales_closed", @@ -325,7 +347,7 @@ export const GuestRegistrationRefusalReason = { answersInvalid: "answers_invalid", } as const -export type GuestRegistrationRefusalReason = +type GuestRegistrationRefusalReason = (typeof GuestRegistrationRefusalReason)[keyof typeof GuestRegistrationRefusalReason] type GuestSeat = Pick< @@ -431,7 +453,7 @@ function withinWindow(at: Date, opensAt: Date | null, closesAt: Date | null): bo * see. It mirrors the order of the registration transaction and answers null whenever that * transaction could still accept, so verify stays the authority on everything else. */ -export function foreseeableGuestRefusal( +function foreseeableGuestRefusal( gate: GuestRegistrationGate, fields: GuestRegistrationFields, at: Date, @@ -475,7 +497,7 @@ function utcDayKey(nowMs: number): string { return new Date(nowMs).toISOString().slice(0, 10) } -export function guestContactOf(input: { +function guestContactOf(input: { channel: GuestContactChannel email?: string | undefined phone?: string | undefined @@ -490,7 +512,7 @@ export function guestContactOf(input: { return phone } -export function registrationFieldsOf( +function registrationFieldsOf( input: GuestRsvpRequestRequest | GuestRsvpVerifyRequest, ): GuestRegistrationFields { return { @@ -502,7 +524,7 @@ export function registrationFieldsOf( } } -export function toCleanupGuestDTO(row: GuestRosterRow): CleanupGuestDTO { +function toCleanupGuestDTO(row: GuestRosterRow): CleanupGuestDTO { return { id: row.id, name: row.name, @@ -532,23 +554,23 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi } function cooldownKey(cleanupId: string, digest: string): string { - return `guest:rl:code:${cleanupId}:${digest}` + return `${GUEST_CODE_COOLDOWN_KEY_PREFIX}${cleanupId}:${digest}` } function contactDayKey(digest: string): string { - return `guest:rl:contact:day:${digest}` + return `${GUEST_CONTACT_DAY_KEY_PREFIX}${digest}` } function ipKey(bucket: string): string { - return `guest:rl:ip:${bucket}` + return `${GUEST_IP_KEY_PREFIX}${bucket}` } function codeFailKey(otpId: string): string { - return `guest:vf:code:${otpId}` + return `${GUEST_CODE_FAIL_KEY_PREFIX}${otpId}` } function ipFailKey(bucket: string): string { - return `guest:vf:ip:${bucket}` + return `${GUEST_IP_FAIL_KEY_PREFIX}${bucket}` } async function readCounter(key: string): Promise { @@ -617,13 +639,13 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi await deps.mailer.sendTransactional(args.contact, "guest_otp", { title: args.eventTitle, code: args.code, - minutes: String(Math.floor(GUEST_OTP_TTL_SECONDS / 60)), + minutes: String(Math.floor(GUEST_OTP_TTL_SECONDS / SECONDS_PER_MINUTE)), }) return } await deps.smsSender.send( args.contact, - renderMessage("en", "sms.guest_otp.body", { + renderMessage(SMS_LOCALE, "sms.guest_otp.body", { title: smsTitle(args.eventTitle), code: args.code, }), @@ -726,11 +748,13 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi const body = kind === "cancelled" - ? renderMessage("en", "sms.guest_cancelled.body", { title: smsTitle(event.title) }) - : renderMessage("en", "sms.guest_updated.body", { + ? renderMessage(SMS_LOCALE, "sms.guest_cancelled.body", { title: smsTitle(event.title) }) + : renderMessage(SMS_LOCALE, "sms.guest_updated.body", { title: smsTitle(event.title), when: formatEventWhen(event.scheduledAt, event.timezone ?? DEFAULT_EVENT_TIME_ZONE), - place: event.address ?? `${event.lat.toFixed(5)}, ${event.lng.toFixed(5)}`, + place: + event.address ?? + `${event.lat.toFixed(SMS_PLACE_COORD_DECIMALS)}, ${event.lng.toFixed(SMS_PLACE_COORD_DECIMALS)}`, }) let sent = 0 @@ -773,7 +797,7 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi const response = await bridge.register( { id: cleanupId, - idempotencyKey: `guest:${guestId}`, + idempotencyKey: `${GUEST_REGISTRATION_IDEMPOTENCY_PREFIX}${guestId}`, partySize: registration.partySize ?? 1, joinWaitlistIfFull: false, ...(registration.ticketTypeId !== undefined @@ -894,7 +918,7 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi } await deps.smsSender.send( args.contact, - renderMessage("en", "sms.guest_confirmed.body", { + renderMessage(SMS_LOCALE, "sms.guest_confirmed.body", { title: smsTitle(args.event.title), link: manageLink(args.rawToken), }), @@ -943,6 +967,113 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi return total } + function validatedGuestName(raw: string): string { + const name = raw.trim() + if (name.length === 0 || name.length > MAX_GUEST_NAME) { + throw AppError.validation({ name: `must be 1-${MAX_GUEST_NAME} characters` }) + } + assertNoSlur(name, "name") + return name + } + + async function throttleCodeRequest(args: { + cleanupId: string + channel: GuestContactChannel + contact: string + ip: string | null + }): Promise { + const digest = await contactDigest(args.contact) + const bucket = args.ip === null ? null : normalizeIp(args.ip) + if (bucket !== null) { + const hits = await deps.cache.incr(ipKey(bucket), GUEST_IP_WINDOW_SECONDS) + if (hits > GUEST_IP_MAX_PER_HOUR) { + throw AppError.rateLimited("Too many code requests from this network.") + } + } + const daily = await deps.cache.incr(contactDayKey(digest), DAY_SECONDS) + if (daily > GUEST_CONTACT_MAX_PER_DAY) { + throw AppError.rateLimited("Too many code requests for this contact today.") + } + + if (args.channel === "sms" && (await deps.repo.isPhoneOptedOut(args.contact))) { + throw smsOptedOutError() + } + + const cooldown = cooldownKey(args.cleanupId, digest) + const cooldownHits = await deps.cache.incr(cooldown, GUEST_CONTACT_COOLDOWN_SECONDS) + if (cooldownHits > 1) { + throw AppError.rateLimited("Please wait before requesting another code.") + } + + if (args.channel === "sms" && !(await reserveSmsBudget("otp"))) { + await releaseCooldown(cooldown) + throw smsUnavailableError() + } + return cooldown + } + + async function issueCode(args: { + event: GuestEventView + channel: GuestContactChannel + contact: string + name: string + cooldown: string + }): Promise { + try { + const at = new Date(now()) + await deps.repo.invalidateActiveOtps(args.event.id, args.contact, at) + const code = newCode() + const codeHash = await hashOtpCode(code) + await deps.repo.insertOtp({ + cleanupId: args.event.id, + channel: args.channel, + contact: args.contact, + name: args.name, + codeHash, + expiresAt: new Date(now() + GUEST_OTP_TTL_SECONDS * MS_PER_SECOND), + }) + await deliverCode({ + channel: args.channel, + contact: args.contact, + code, + eventTitle: args.event.title, + }) + } catch (err) { + await releaseCooldown(args.cooldown) + throw await mapDeliveryError(err, args.channel === "sms" ? args.contact : null) + } + } + + async function consumeOtp( + record: GuestOtpRecord, + code: string, + at: Date, + bucket: string | null, + ): Promise { + if ((await readCounter(codeFailKey(record.id))) >= OTP_VERIFY_CODE_FAIL_MAX) { + await deps.repo.markOtpConsumed(record.id, at) + throw attemptsExhaustedError() + } + + const attempts = await deps.repo.incrementOtpAttempts(record.id) + if (attempts > OTP_MAX_ATTEMPTS) { + await deps.repo.markOtpConsumed(record.id, at) + await bumpVerifyFailure(record.id, bucket) + throw attemptsExhaustedError() + } + + const ok = await verifyOtpCode(record.codeHash, code) + if (!ok) { + const burned = attempts >= OTP_MAX_ATTEMPTS + if (burned) await deps.repo.markOtpConsumed(record.id, at) + await bumpVerifyFailure(record.id, bucket) + throw burned ? attemptsExhaustedError() : invalidCodeError() + } + + const claimed = await deps.repo.markOtpConsumed(record.id, at) + if (!claimed) throw invalidCodeError() + } + return { async requestCode( input: GuestRsvpRequestRequest, @@ -967,14 +1098,10 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi if (!human) throw AppError.turnstileFailed() const event = await loadOpenEvent(input.id) - await assertRegistrationInputValid(event.id, registrationFieldsOf(input)) - - const name = input.name.trim() - if (name.length === 0 || name.length > MAX_GUEST_NAME) { - throw AppError.validation({ name: `must be 1-${MAX_GUEST_NAME} characters` }) - } - assertNoSlur(name, "name") + const registration = registrationFieldsOf(input) + await assertRegistrationInputValid(event.id, registration) + const name = validatedGuestName(input.name) const contact = guestContactOf(input) if (isReviewerContact(input.channel, contact)) { @@ -982,7 +1109,7 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi } // Refused before any budget is spent or code sent: the guest can fix the form and ask again. - await refuseForeseeableRegistration(event.id, registrationFieldsOf(input)) + await refuseForeseeableRegistration(event.id, registration) const active = await deps.repo.countActiveGuests(event.id) if (active >= MAX_GUESTS_PER_EVENT) { @@ -991,53 +1118,13 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi if (input.channel === "sms" && !deps.smsGuestEnabled) throw smsUnavailableError() - const digest = await contactDigest(contact) - const bucket = ctx.ip === null ? null : normalizeIp(ctx.ip) - if (bucket !== null) { - const hits = await deps.cache.incr(ipKey(bucket), GUEST_IP_WINDOW_SECONDS) - if (hits > GUEST_IP_MAX_PER_HOUR) { - throw AppError.rateLimited("Too many code requests from this network.") - } - } - const daily = await deps.cache.incr(contactDayKey(digest), DAY_SECONDS) - if (daily > GUEST_CONTACT_MAX_PER_DAY) { - throw AppError.rateLimited("Too many code requests for this contact today.") - } - - if (input.channel === "sms" && (await deps.repo.isPhoneOptedOut(contact))) { - throw smsOptedOutError() - } - - const cooldown = cooldownKey(event.id, digest) - const cooldownHits = await deps.cache.incr(cooldown, GUEST_CONTACT_COOLDOWN_SECONDS) - if (cooldownHits > 1) { - throw AppError.rateLimited("Please wait before requesting another code.") - } - - if (input.channel === "sms" && !(await reserveSmsBudget("otp"))) { - await releaseCooldown(cooldown) - throw smsUnavailableError() - } - - try { - const at = new Date(now()) - await deps.repo.invalidateActiveOtps(event.id, contact, at) - const code = newCode() - const codeHash = await hashOtpCode(code) - await deps.repo.insertOtp({ - cleanupId: event.id, - channel: input.channel, - contact, - name, - codeHash, - expiresAt: new Date(now() + GUEST_OTP_TTL_SECONDS * 1000), - }) - await deliverCode({ channel: input.channel, contact, code, eventTitle: event.title }) - } catch (err) { - await releaseCooldown(cooldown) - throw await mapDeliveryError(err, input.channel === "sms" ? contact : null) - } - + const cooldown = await throttleCodeRequest({ + cleanupId: event.id, + channel: input.channel, + contact, + ip: ctx.ip, + }) + await issueCode({ event, channel: input.channel, contact, name, cooldown }) return fakeSuccess }, @@ -1052,7 +1139,8 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi } const event = await loadOpenEvent(input.id) - await assertRegistrationInputValid(event.id, registrationFieldsOf(input)) + const registration = registrationFieldsOf(input) + await assertRegistrationInputValid(event.id, registration) const contact = guestContactOf(input) if (isReviewerContact(input.channel, contact)) { @@ -1063,7 +1151,7 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi channel: input.channel, contact, confirm: false, - registration: registrationFieldsOf(input), + registration, }) } await bumpVerifyFailure(null, bucket) @@ -1075,29 +1163,7 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi await bumpVerifyFailure(null, bucket) throw invalidCodeError() } - - if ((await readCounter(codeFailKey(record.id))) >= OTP_VERIFY_CODE_FAIL_MAX) { - await deps.repo.markOtpConsumed(record.id, at) - throw attemptsExhaustedError() - } - - const attempts = await deps.repo.incrementOtpAttempts(record.id) - if (attempts > OTP_MAX_ATTEMPTS) { - await deps.repo.markOtpConsumed(record.id, at) - await bumpVerifyFailure(record.id, bucket) - throw attemptsExhaustedError() - } - - const ok = await verifyOtpCode(record.codeHash, input.code) - if (!ok) { - const burned = attempts >= OTP_MAX_ATTEMPTS - if (burned) await deps.repo.markOtpConsumed(record.id, at) - await bumpVerifyFailure(record.id, bucket) - throw burned ? attemptsExhaustedError() : invalidCodeError() - } - - const claimed = await deps.repo.markOtpConsumed(record.id, at) - if (!claimed) throw invalidCodeError() + await consumeOtp(record, input.code, at, bucket) const digest = await contactDigest(contact) await deps.cache.del(cooldownKey(event.id, digest)).catch((err: unknown) => { @@ -1113,7 +1179,7 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi channel: record.channel, contact, confirm: true, - registration: registrationFieldsOf(input), + registration, }) }, @@ -1162,8 +1228,10 @@ export function makeGuestRsvpService(deps: GuestRsvpServiceDeps): GuestRsvpServi async runRetentionSweep(): Promise { const at = new Date(now()) - const contactCutoff = new Date(now() - GUEST_CONTACT_RETENTION_DAYS * DAY_SECONDS * 1000) - const otpCutoff = new Date(now() - GUEST_OTP_RETENTION_HOURS * 60 * 60 * 1000) + const contactCutoff = new Date( + now() - GUEST_CONTACT_RETENTION_DAYS * DAY_SECONDS * MS_PER_SECOND, + ) + const otpCutoff = new Date(now() - GUEST_OTP_RETENTION_HOURS * MS_PER_HOUR) const scrubbedGuests = await drainPages("guest contact scrub", (batchSize) => deps.repo.scrubExpiredGuestContacts({ cutoff: contactCutoff, now: at, batchSize }), ) diff --git a/services/api/src/services/guest-rsvp-wiring.ts b/services/api/src/services/guest-rsvp-wiring.ts index 45c9502f..3d17ae99 100644 --- a/services/api/src/services/guest-rsvp-wiring.ts +++ b/services/api/src/services/guest-rsvp-wiring.ts @@ -7,6 +7,7 @@ import { webBaseUrlOf } from "../lib/base-url.js" import { makeDrizzleGuestRsvpRepository } from "./guest-rsvp-repository.drizzle.js" import { makeGuestRsvpService, + type GuestRegistrationBridge, type GuestRsvpService, type GuestRsvpServiceDeps, } from "./guest-rsvp-service.js" @@ -25,7 +26,7 @@ export interface GuestRsvpOverrides { now?: GuestRsvpServiceDeps["now"] } -export function guestReviewerConfig(container: Container): GuestRsvpServiceDeps["reviewer"] { +function guestReviewerConfig(container: Container): GuestRsvpServiceDeps["reviewer"] { const code = container.env.REVIEWER_OTP_CODE if (!container.env.REVIEWER_OTP_BYPASS || code === undefined || code.length === 0) { return undefined @@ -33,19 +34,11 @@ export function guestReviewerConfig(container: Container): GuestRsvpServiceDeps[ return { email: REVIEWER_OTP_EMAIL, code } } -export function makeContainerGuestRsvpService( +function containerRegistrationBridge( container: Container, - overrides?: GuestRsvpOverrides, - logger?: GuestRsvpServiceDeps["logger"], -): GuestRsvpService { - const reviewer = overrides?.reviewer ?? guestReviewerConfig(container) - const repo = overrides?.repo ?? makeDrizzleGuestRsvpRepository(container.getDb().sql) - const requireGuestContact = - overrides?.requireGuestContact ?? - (async (cleanupId: string, userId: string) => { - await requireCapability(container.getDb().sql, cleanupId, userId, "view_guest_contact") - }) - const registrations: GuestRsvpServiceDeps["registrations"] = overrides?.registrations ?? { + logger: GuestRsvpServiceDeps["logger"], +): GuestRegistrationBridge { + return { register: (input, subject) => makeContainerRegistrationServices(container, undefined, logger).registrations.register( input, @@ -85,6 +78,22 @@ export function makeContainerGuestRsvpService( } }, } +} + +export function makeContainerGuestRsvpService( + container: Container, + overrides?: GuestRsvpOverrides, + logger?: GuestRsvpServiceDeps["logger"], +): GuestRsvpService { + const reviewer = overrides?.reviewer ?? guestReviewerConfig(container) + const repo = overrides?.repo ?? makeDrizzleGuestRsvpRepository(container.getDb().sql) + const requireGuestContact = + overrides?.requireGuestContact ?? + (async (cleanupId: string, userId: string) => { + await requireCapability(container.getDb().sql, cleanupId, userId, "view_guest_contact") + }) + const registrations: GuestRsvpServiceDeps["registrations"] = + overrides?.registrations ?? containerRegistrationBridge(container, logger) const audit: GuestRsvpServiceDeps["audit"] = overrides?.audit ?? (async (input) => { diff --git a/services/api/src/services/jurisdiction-service.ts b/services/api/src/services/jurisdiction-service.ts index 46d03f33..ccbba955 100644 --- a/services/api/src/services/jurisdiction-service.ts +++ b/services/api/src/services/jurisdiction-service.ts @@ -11,7 +11,7 @@ import type { export const JURISDICTION_DISCOVERY_JOB = "jurisdiction.discovery" -export const CONTACT_STALE_MONTHS = 18 +const CONTACT_STALE_MONTHS = 18 export interface JurisdictionDiscoveryJob { geoid: string @@ -30,26 +30,22 @@ function hasUsableLegacyContact(row: Pick e.trim() !== "") } +function isValidDate(value: Date | null): value is Date { + return value instanceof Date && !Number.isNaN(value.getTime()) +} + function isStale(updatedAt: Date | null, now: Date): boolean { - if (!(updatedAt instanceof Date) || Number.isNaN(updatedAt.getTime())) return true + if (!isValidDate(updatedAt)) return true const staleBefore = new Date(now) staleBefore.setMonth(staleBefore.getMonth() - CONTACT_STALE_MONTHS) return updatedAt.getTime() < staleBefore.getTime() } export function needsDiscovery(row: JurisdictionHealthRow, now: Date): boolean { - const hasRoutingContact = row.hasRoutingContact === true - - if (!hasRoutingContact && !hasUsableLegacyContact(row)) return true - - if (hasRoutingContact) { - if (!(row.contactUpdatedAt instanceof Date) || Number.isNaN(row.contactUpdatedAt.getTime())) { - return false - } - return isStale(row.contactUpdatedAt, now) + if (row.hasRoutingContact === true) { + return isValidDate(row.contactUpdatedAt) && isStale(row.contactUpdatedAt, now) } - - return isStale(row.contactUpdatedAt, now) + return !hasUsableLegacyContact(row) || isStale(row.contactUpdatedAt, now) } export function isRoutable( diff --git a/services/api/src/services/media-authorization.ts b/services/api/src/services/media-authorization.ts index ada5987f..47d34af9 100644 --- a/services/api/src/services/media-authorization.ts +++ b/services/api/src/services/media-authorization.ts @@ -23,12 +23,7 @@ export function makeUnboundOnlyMediaViewAuthorizer( ): MediaViewAuthorizer { return { authorize(asset: MediaAssetView): Promise { - if (asset.purpose === "verification") return Promise.resolve(DENY) - if ( - asset.purpose === "event_cover" || - asset.purpose === "event_gallery" || - asset.purpose === "org_logo" - ) { + if (asset.purpose === "verification" || isEntityBoundPurpose(asset.purpose)) { return Promise.resolve(DENY) } if (asset.reportId || asset.chatMessageId || asset.postId) return Promise.resolve(DENY) @@ -37,6 +32,10 @@ export function makeUnboundOnlyMediaViewAuthorizer( } } +function isEntityBoundPurpose(purpose: MediaAssetView["purpose"]): boolean { + return purpose === "event_cover" || purpose === "event_gallery" || purpose === "org_logo" +} + function withinGrace(createdAt: Date | null | undefined, now: Date): boolean { if (!createdAt) return false return now.getTime() - createdAt.getTime() <= UNBOUND_GRACE_MS @@ -85,12 +84,7 @@ export async function authorizeChatBound( const dm = dmRows[0] if (dm) { if (dm.deleted_at !== null) return DENY - const member = await sql<{ ok: number }[]>` - SELECT 1 AS ok FROM dm_threads - WHERE id = ${dm.thread_id} AND (user_lo = ${viewerId} OR user_hi = ${viewerId}) - LIMIT 1 - ` - return member.length > 0 ? ALLOW_PRIVATE : DENY + return authorizeDmThread(sql, dm.thread_id, viewerId) } const chatRows = await sql< @@ -106,35 +100,56 @@ export async function authorizeChatBound( ` const msg = chatRows[0] if (!msg || msg.deleted_at !== null) return DENY + if (msg.cleanup_id !== null) return authorizeCleanupRoom(sql, msg.cleanup_id, viewerId) + if (msg.group_id !== null) return authorizeGroupRoom(sql, msg.group_id, viewerId) + if (msg.report_id !== null) { + const { allowed } = await authorizeReportBound(sql, msg.report_id, viewerId) + return allowed ? ALLOW_PRIVATE : DENY + } + return DENY +} - if (msg.cleanup_id !== null) { - const member = await sql<{ ok: number }[]>` +async function authorizeDmThread( + sql: Sql, + threadId: string, + viewerId: string, +): Promise { + const member = await sql<{ ok: number }[]>` + SELECT 1 AS ok FROM dm_threads + WHERE id = ${threadId} AND (user_lo = ${viewerId} OR user_hi = ${viewerId}) + LIMIT 1 + ` + return member.length > 0 ? ALLOW_PRIVATE : DENY +} + +async function authorizeCleanupRoom( + sql: Sql, + cleanupId: string, + viewerId: string, +): Promise { + const member = await sql<{ ok: number }[]>` SELECT 1 AS ok FROM cleanup_members - WHERE cleanup_id = ${msg.cleanup_id} AND user_id = ${viewerId} LIMIT 1 + WHERE cleanup_id = ${cleanupId} AND user_id = ${viewerId} LIMIT 1 ` - return member.length > 0 ? ALLOW_PRIVATE : DENY - } + return member.length > 0 ? ALLOW_PRIVATE : DENY +} - if (msg.group_id !== null) { - const rows = await sql<{ visibility: string; is_member: boolean }[]>` +async function authorizeGroupRoom( + sql: Sql, + groupId: string, + viewerId: string, +): Promise { + const rows = await sql<{ visibility: string; is_member: boolean }[]>` SELECT g.visibility, EXISTS ( SELECT 1 FROM chat_group_members m WHERE m.group_id = g.id AND m.user_id = ${viewerId} ) AS is_member - FROM chat_groups g WHERE g.id = ${msg.group_id} LIMIT 1 + FROM chat_groups g WHERE g.id = ${groupId} LIMIT 1 ` - const group = rows[0] - if (!group) return DENY - return group.is_member || group.visibility === "public" ? ALLOW_PRIVATE : DENY - } - - if (msg.report_id !== null) { - const visible = await reportVisible(sql, msg.report_id, viewerId) - return visible ? ALLOW_PRIVATE : DENY - } - - return DENY + const group = rows[0] + if (!group) return DENY + return group.is_member || group.visibility === "public" ? ALLOW_PRIVATE : DENY } export async function authorizeEventBound( @@ -215,15 +230,6 @@ export async function authorizeReportBound( return DENY } -async function reportVisible( - sql: Sql, - reportId: string, - viewerId: string | null, -): Promise { - const decision = await authorizeReportBound(sql, reportId, viewerId) - return decision.allowed -} - async function authorizeUnbound( sql: Sql, asset: MediaAssetView, diff --git a/services/api/src/services/media-bindings.ts b/services/api/src/services/media-bindings.ts index f86d232d..af1ee9d7 100644 --- a/services/api/src/services/media-bindings.ts +++ b/services/api/src/services/media-bindings.ts @@ -10,8 +10,6 @@ export const MEDIA_BINDING_RELATIONS = [ "cleanup_page_media.media_id", ] as const -export const MEDIA_BINDING_COLUMNS = ["report_id", "chat_message_id", "post_id"] as const - export function mediaBoundElsewhere(tag: Queryable, exceptCleanupId: string | null) { return tag` EXISTS (SELECT 1 FROM users u WHERE u.avatar_media_id = media_assets.id) diff --git a/services/api/src/services/media-byte-quota.ts b/services/api/src/services/media-byte-quota.ts index 345e576b..5e3602af 100644 --- a/services/api/src/services/media-byte-quota.ts +++ b/services/api/src/services/media-byte-quota.ts @@ -23,6 +23,8 @@ export const MEDIA_UPLOAD_BYTE_WINDOW_SECONDS = 24 * 60 * 60 /** Distinct bucket from every other abuse counter (see abuse/counter-store.ts). */ export const MEDIA_UPLOAD_BYTE_PREFIX = "abuse:media:bytes:" +const MS_PER_SECOND = 1000 + export interface ByteMeter { /** Returns the running total after the add. */ add(subject: string, bytes: number): Promise @@ -55,7 +57,7 @@ export class InMemoryByteMeter implements ByteMeter { if (!existing || existing.expiresAtMs <= now) { this.store.set(subject, { total: bytes, - expiresAtMs: now + MEDIA_UPLOAD_BYTE_WINDOW_SECONDS * 1000, + expiresAtMs: now + MEDIA_UPLOAD_BYTE_WINDOW_SECONDS * MS_PER_SECOND, }) return Promise.resolve(bytes) } diff --git a/services/api/src/services/media-intake-service.ts b/services/api/src/services/media-intake-service.ts index 59ff84a5..32dda3d5 100644 --- a/services/api/src/services/media-intake-service.ts +++ b/services/api/src/services/media-intake-service.ts @@ -11,8 +11,6 @@ import type { } from "@civfix/shared" import { MAX_IMAGE_BYTES, MAX_VIDEO_BYTES } from "@civfix/shared" import type { MEDIA_PURPOSE_VALUES } from "../db/schema/types-host.js" - -type MediaPurpose = (typeof MEDIA_PURPOSE_VALUES)[number] import type { Jobs, Storage } from "@civfix/shared/interfaces" import { readEtag } from "./media-etag.js" import { uploaderOf, uploadersOf } from "./media-uploader.js" @@ -23,6 +21,8 @@ import { type MediaViewAuthorizer, } from "./media-authorization.js" +type MediaPurpose = (typeof MEDIA_PURPOSE_VALUES)[number] + interface IntakeLogger { warn(obj: unknown, msg?: string): void } @@ -33,17 +33,21 @@ export const MEDIA_GET_URL_TTL_SEC = 15 * 60 export const MEDIA_PRIVATE_GET_URL_TTL_SEC = 5 * 60 -export const ALLOWED_IMAGE_CONTENT_TYPES: ReadonlySet = new Set([ +const ALLOWED_IMAGE_CONTENT_TYPES: ReadonlySet = new Set([ "image/jpeg", "image/png", "image/webp", ]) -export const ALLOWED_VIDEO_CONTENT_TYPES: ReadonlySet = new Set([ - "video/mp4", - "video/quicktime", -]) +const ALLOWED_VIDEO_CONTENT_TYPES: ReadonlySet = new Set(["video/mp4", "video/quicktime"]) const SHA256_HEX = /^[0-9a-f]{64}$/ +const UPLOAD_KEY_PREFIX = "uploads/" +const MS_PER_SECOND = 1000 +const USER_QUOTA_PREFIX = "u:" +const ANON_QUOTA_PREFIX = "a:" +const IP_QUOTA_PREFIX = "ip:" +const UNKNOWN_IP_KEY = "unknown" +const MEDIA_NOT_FOUND = "Media not found" export interface MediaChecksJob { mediaId: string @@ -149,15 +153,36 @@ export function precheckUpload(input: CreateMediaUploadRequest): void { export function buildR2Key(uploadId: string, now: Date): string { const yyyy = String(now.getUTCFullYear()).padStart(4, "0") const mm = String(now.getUTCMonth() + 1).padStart(2, "0") - return `uploads/${yyyy}/${mm}/${uploadId}` + return `${UPLOAD_KEY_PREFIX}${yyyy}/${mm}/${uploadId}` } const SIZE_MISMATCH_TOLERANCE = 1024 export function quotaSubjects(owner: MediaOwner): string[] { - if (owner.userId) return [`u:${owner.userId}`] - const ipSubject = `ip:${owner.ipKey ?? "unknown"}` - return owner.anonSessionId ? [`a:${owner.anonSessionId}`, ipSubject] : [ipSubject] + if (owner.userId) return [`${USER_QUOTA_PREFIX}${owner.userId}`] + const ipSubject = `${IP_QUOTA_PREFIX}${owner.ipKey ?? UNKNOWN_IP_KEY}` + return owner.anonSessionId + ? [`${ANON_QUOTA_PREFIX}${owner.anonSessionId}`, ipSubject] + : [ipSubject] +} + +async function enforceByteQuota( + quota: MediaByteQuota, + owner: MediaOwner, + byteSize: number, + logger: IntakeLogger | undefined, +): Promise { + let over: { subject: string; total: number } | null = null + for (const subject of quotaSubjects(owner)) { + const total = await quota.charge(subject, byteSize) + if (total > quota.limitBytes && over === null) over = { subject, total } + } + if (!over) return + logger?.warn( + { subject: over.subject, totalBytes: over.total, limitBytes: quota.limitBytes }, + "media upload byte quota exceeded", + ) + throw AppError.rateLimited("Upload quota exceeded. Try again later.") } export function makeMediaIntakeService(deps: MediaIntakeDeps): MediaIntakeService { @@ -173,7 +198,10 @@ export function makeMediaIntakeService(deps: MediaIntakeDeps): MediaIntakeServic function finalizableBy(asset: MediaAssetView, owner: MediaOwner): boolean { if (asset.uploader == null) { const createdAt = asset.createdAt?.getTime() - return createdAt !== undefined && now().getTime() - createdAt < MEDIA_CLAIM_WINDOW_SEC * 1000 + return ( + createdAt !== undefined && + now().getTime() - createdAt < MEDIA_CLAIM_WINDOW_SEC * MS_PER_SECOND + ) } return uploadersOf(owner).includes(asset.uploader) } @@ -185,20 +213,8 @@ export function makeMediaIntakeService(deps: MediaIntakeDeps): MediaIntakeServic ): Promise { precheckUpload(input) - const quota = deps.byteQuota - if (quota) { - let over: { subject: string; total: number } | null = null - for (const subject of quotaSubjects(owner)) { - const total = await quota.charge(subject, input.byteSize) - if (total > quota.limitBytes && over === null) over = { subject, total } - } - if (over) { - deps.logger?.warn( - { subject: over.subject, totalBytes: over.total, limitBytes: quota.limitBytes }, - "media upload byte quota exceeded", - ) - throw AppError.rateLimited("Upload quota exceeded. Try again later.") - } + if (deps.byteQuota) { + await enforceByteQuota(deps.byteQuota, owner, input.byteSize, deps.logger) } const uploadId = newId() @@ -282,16 +298,16 @@ export function makeMediaIntakeService(deps: MediaIntakeDeps): MediaIntakeServic async getMedia(id: string, viewer: MediaOwner): Promise { const asset = await deps.repo.findById(id) if (!asset || asset.status !== "ready") { - throw AppError.notFound("Media not found") + throw AppError.notFound(MEDIA_NOT_FOUND) } const decision = await authorizer.authorize(asset, viewer) if (!decision.allowed) { - throw AppError.notFound("Media not found") + throw AppError.notFound(MEDIA_NOT_FOUND) } if (asset.servedKey === null) { - throw AppError.notFound("Media not found") + throw AppError.notFound(MEDIA_NOT_FOUND) } const issue = decision.private ? presignPrivate : presign const { url, thumbUrl } = await issue(asset.servedKey, asset.thumbKey) diff --git a/services/api/src/services/media-presign.ts b/services/api/src/services/media-presign.ts index 10a7c329..73cf9e21 100644 --- a/services/api/src/services/media-presign.ts +++ b/services/api/src/services/media-presign.ts @@ -14,15 +14,19 @@ interface PresignStorage { presignGet(key: string, ttlSec: number, opts?: { forceSigned?: boolean }): Promise } -export function makeMediaPresigner(storage: PresignStorage): PresignMedia { +function makePairPresigner(sign: (key: string) => Promise): PresignMedia { return async (r2Key, thumbKey) => { - const url = await storage.presignGet(r2Key, MEDIA_GET_URL_TTL_SEC) + const url = await sign(r2Key) if (thumbKey === null) return { url } - const thumbUrl = await storage.presignGet(thumbKey, MEDIA_GET_URL_TTL_SEC) + const thumbUrl = await sign(thumbKey) return { url, thumbUrl } } } +export function makeMediaPresigner(storage: PresignStorage): PresignMedia { + return makePairPresigner((key) => storage.presignGet(key, MEDIA_GET_URL_TTL_SEC)) +} + /** * Presigner for PRIVATE media (chat/DM attachments, an owner's own held/unlisted report media, * not-yet-committed uploads). Two differences from the public presigner, both load-bearing: @@ -33,16 +37,9 @@ export function makeMediaPresigner(storage: PresignStorage): PresignMedia { * - a much shorter TTL, so a leaked URL expires in minutes rather than an hour. */ export function makePrivateMediaPresigner(storage: PresignStorage): PresignMedia { - return async (r2Key, thumbKey) => { - const url = await storage.presignGet(r2Key, MEDIA_PRIVATE_GET_URL_TTL_SEC, { - forceSigned: true, - }) - if (thumbKey === null) return { url } - const thumbUrl = await storage.presignGet(thumbKey, MEDIA_PRIVATE_GET_URL_TTL_SEC, { - forceSigned: true, - }) - return { url, thumbUrl } - } + return makePairPresigner((key) => + storage.presignGet(key, MEDIA_PRIVATE_GET_URL_TTL_SEC, { forceSigned: true }), + ) } export const PACKET_MEDIA_URL_TTL_SEC = 7 * 24 * 60 * 60 diff --git a/services/api/src/services/media-uploader.ts b/services/api/src/services/media-uploader.ts index da67b1d8..f9df7e6c 100644 --- a/services/api/src/services/media-uploader.ts +++ b/services/api/src/services/media-uploader.ts @@ -2,12 +2,16 @@ // ever means a row written before uploads were attributed. Never an IP: the column outlives the request. export const UNSESSIONED_UPLOADER = "anon" +const USER_UPLOADER_PREFIX = "u:" + +const ANON_UPLOADER_PREFIX = "a:" + export function userUploader(userId: string): string { - return `u:${userId}` + return `${USER_UPLOADER_PREFIX}${userId}` } export function anonUploader(anonSessionId: string): string { - return `a:${anonSessionId}` + return `${ANON_UPLOADER_PREFIX}${anonSessionId}` } export function uploaderOf(owner: { diff --git a/services/api/src/services/media-worker-repo.ts b/services/api/src/services/media-worker-repo.ts index 7adfbb8d..b46b7452 100644 --- a/services/api/src/services/media-worker-repo.ts +++ b/services/api/src/services/media-worker-repo.ts @@ -16,6 +16,11 @@ export type { StorageHeadWithEtag } from "./media-etag.js" export type WorkerAbuseReason = "nsfw" | "phash_dup" | "gps" const ANONYMOUS_REPORTER = "Anonymous" +const DUPLICATE_MODERATION_FLAG = "Near-duplicate media" +const HELD_MODERATION_FLAG = "Held media (NSFW)" +const HELD_MODERATION_AUTO_ACTION = "Hidden pending review" + +export const PARTITION_MONTHS_AHEAD = 2 export interface MediaWorkerAsset { id: string @@ -307,38 +312,9 @@ export function makeDrizzleMediaWorkerRepo(db: Db, tag: Sql): MediaWorkerRepo { kind?: "image" | "duplicate" note?: string | null }): Promise { - // A held source folding into an open item means the owner's takedown is no longer its only - // origin, so removing it must strike the author again. - const foldIntoOpenItem = async (): Promise => { - const folded = await db - .update(moderationItems) - .set({ meta: sql`${moderationItems.meta} - 'ownerTakedown'` }) - .where( - and( - eq(moderationItems.subjectType, "report"), - eq(moderationItems.subjectId, input.reportId), - eq(moderationItems.status, "open"), - ), - ) - .returning({ id: moderationItems.id }) - return folded.length > 0 - } - if (await foldIntoOpenItem()) return + if (await foldIntoOpenModerationItem(db, input.reportId)) return - const ctx = await db - .select({ - category: reports.category, - description: reports.description, - place: jurisdictions.name, - reporterName: users.displayName, - reporterUserId: users.id, - }) - .from(reports) - .leftJoin(jurisdictions, eq(jurisdictions.geoid, reports.jurisdictionGeoid)) - .leftJoin(users, eq(users.id, reports.reporterUserId)) - .where(eq(reports.id, input.reportId)) - .limit(1) - const row = ctx[0] + const row = await loadModerationContext(db, input.reportId) if (!row) return const kind = input.kind ?? "image" @@ -348,12 +324,12 @@ export function makeDrizzleMediaWorkerRepo(db: Db, tag: Sql): MediaWorkerRepo { kind, subjectType: "report", subjectId: input.reportId, - flag: kind === "duplicate" ? "Near-duplicate media" : "Held media (NSFW)", + flag: kind === "duplicate" ? DUPLICATE_MODERATION_FLAG : HELD_MODERATION_FLAG, reason: input.reason, category: row.category, place: row.place, priority: "high", - autoAction: "Hidden pending review", + autoAction: HELD_MODERATION_AUTO_ACTION, status: "open", meta: { reporter: row.reporterName ?? ANONYMOUS_REPORTER, @@ -367,7 +343,7 @@ export function makeDrizzleMediaWorkerRepo(db: Db, tag: Sql): MediaWorkerRepo { where: sql`status = 'open'`, }) .returning({ id: moderationItems.id }) - if (inserted.length === 0) await foldIntoOpenItem() + if (inserted.length === 0) await foldIntoOpenModerationItem(db, input.reportId) }, async recordLeakedObjects(input: { @@ -415,6 +391,40 @@ export function makeDrizzleMediaWorkerRepo(db: Db, tag: Sql): MediaWorkerRepo { } } +// A held source folding into an open item means the owner's takedown is no longer its only origin, so +// removing it must strike the author again. +async function foldIntoOpenModerationItem(db: Db, reportId: string): Promise { + const folded = await db + .update(moderationItems) + .set({ meta: sql`${moderationItems.meta} - 'ownerTakedown'` }) + .where( + and( + eq(moderationItems.subjectType, "report"), + eq(moderationItems.subjectId, reportId), + eq(moderationItems.status, "open"), + ), + ) + .returning({ id: moderationItems.id }) + return folded.length > 0 +} + +async function loadModerationContext(db: Db, reportId: string) { + const ctx = await db + .select({ + category: reports.category, + description: reports.description, + place: jurisdictions.name, + reporterName: users.displayName, + reporterUserId: users.id, + }) + .from(reports) + .leftJoin(jurisdictions, eq(jurisdictions.geoid, reports.jurisdictionGeoid)) + .leftJoin(users, eq(users.id, reports.reporterUserId)) + .where(eq(reports.id, reportId)) + .limit(1) + return ctx[0] +} + const ORPHAN_COLUMNS_SQL = (tag: Queryable) => tag` media_assets.id AS "id", media_assets.r2_key AS "r2Key", @@ -436,7 +446,7 @@ export function orphanPredicate(tag: Queryable, olderThan: Date) { type MessageParent = "chat_messages" | "dm_messages" async function ensureMonthPartition( - sqlTag: import("../db/client.js").Sql, + sqlTag: Sql, parent: MessageParent, year: number, monthIndex0: number, @@ -460,7 +470,7 @@ async function ensureMonthPartition( } async function ensureNextMonthPartition( - sqlTag: import("../db/client.js").Sql, + sqlTag: Sql, parent: MessageParent, now: Date, ): Promise { @@ -468,7 +478,7 @@ async function ensureNextMonthPartition( } async function ensurePartitionWindow( - sqlTag: import("../db/client.js").Sql, + sqlTag: Sql, parent: MessageParent, now: Date, monthsAhead: number, @@ -482,37 +492,29 @@ async function ensurePartitionWindow( return tables } -export function ensureNextMonthChatPartition( - sqlTag: import("../db/client.js").Sql, - now: Date = new Date(), -): Promise { +export function ensureNextMonthChatPartition(sqlTag: Sql, now: Date = new Date()): Promise { return ensureNextMonthPartition(sqlTag, "chat_messages", now) } -export function ensureNextMonthDmPartition( - sqlTag: import("../db/client.js").Sql, - now: Date = new Date(), -): Promise { +export function ensureNextMonthDmPartition(sqlTag: Sql, now: Date = new Date()): Promise { return ensureNextMonthPartition(sqlTag, "dm_messages", now) } export function ensureChatPartitionWindow( - sqlTag: import("../db/client.js").Sql, + sqlTag: Sql, now: Date = new Date(), - monthsAhead = 2, + monthsAhead = PARTITION_MONTHS_AHEAD, ): Promise { return ensurePartitionWindow(sqlTag, "chat_messages", now, monthsAhead) } export function ensureDmPartitionWindow( - sqlTag: import("../db/client.js").Sql, + sqlTag: Sql, now: Date = new Date(), - monthsAhead = 2, + monthsAhead = PARTITION_MONTHS_AHEAD, ): Promise { return ensurePartitionWindow(sqlTag, "dm_messages", now, monthsAhead) } export { MEDIA_CHECKS_JOB } from "./media-intake-service.js" export type { MediaChecksJob } from "./media-intake-service.js" - -export { and, eq, isNull, lt, ne, sql } diff --git a/services/api/src/services/message-attachments.drizzle.ts b/services/api/src/services/message-attachments.drizzle.ts index 126f860b..ea7b69cd 100644 --- a/services/api/src/services/message-attachments.drizzle.ts +++ b/services/api/src/services/message-attachments.drizzle.ts @@ -8,8 +8,6 @@ import { userUploader } from "./media-uploader.js" export type MessageMediaColumn = "chat_message_id" -const ALL_COLUMNS: readonly MessageMediaColumn[] = ["chat_message_id"] - const CLAIM_GUARD_COLUMNS: readonly string[] = ["report_id", "post_id"] export interface MessageAttachmentRepo { @@ -35,11 +33,10 @@ interface MediaRow { } export function makeAttachmentRepo(column: MessageMediaColumn): MessageAttachmentRepo { - const otherCols = ALL_COLUMNS.filter((c) => c !== column) return { async attach(tx, messageId, uploadIds, messageCreatedAt, senderId) { if (uploadIds.length === 0) return - const nullGuards = [...otherCols, ...CLAIM_GUARD_COLUMNS].reduce( + const nullGuards = CLAIM_GUARD_COLUMNS.reduce( (acc, c) => tx`${acc} AND ${tx(c)} IS NULL`, tx``, ) diff --git a/services/api/src/services/report-chat-emitter.ts b/services/api/src/services/report-chat-emitter.ts index d8713821..07bfc818 100644 --- a/services/api/src/services/report-chat-emitter.ts +++ b/services/api/src/services/report-chat-emitter.ts @@ -20,7 +20,7 @@ import { type ReportChatSystemEmitter, } from "./report-timeline-event.js" -export const NOOP_REPORT_CHAT_EMITTER: ReportChatSystemEmitter = { +const NOOP_REPORT_CHAT_EMITTER: ReportChatSystemEmitter = { emit: () => Promise.resolve(), } diff --git a/services/api/src/services/report-chat-notifier.ts b/services/api/src/services/report-chat-notifier.ts index eefc501b..1bd278ab 100644 --- a/services/api/src/services/report-chat-notifier.ts +++ b/services/api/src/services/report-chat-notifier.ts @@ -1,42 +1,16 @@ import type { ChatMessageDTO } from "@civfix/shared" -import type { FastifyBaseLogger } from "fastify" -import type { NotificationService } from "./notification-service.js" -import { makeRoomFanoutNotifier, ROOM_FANOUT_SPEC } from "./chat-room-fanout-notifier.js" +import { makeRoomChatNotifier, type RoomChatNotifierDeps } from "./room-chat-notifier-adapter.js" export { REPORT_CHAT_FANOUT_MEMBER_CAP } from "./chat-room-fanout-notifier.js" -export interface ReportChatNotifierDeps { - notificationService: Pick +export interface ReportChatNotifierDeps extends RoomChatNotifierDeps<"report"> { reportChatRepo: { listMemberIds(reportId: string, limit: number): Promise } - isMuted: (userId: string, roomId: string) => Promise - mutedUserIdsFor?: (roomId: string, userIds: string[]) => Promise> - presence?: { online(roomKey: string): Promise } - roomKeyFor: (kind: "report", id: string) => string - isBlockedEitherWay: (a: string, b: string) => Promise - blockedIdsFor?: (actorId: string, candidateIds: string[]) => Promise> - coalesceWindowMs?: number - now?: () => number - claimWindow?: (roomId: string, windowMs: number) => Promise - dispatchToJob?: (roomId: string, messageId: string) => Promise - logger?: Pick | undefined } export function makeReportChatNotifier( deps: ReportChatNotifierDeps, ): (reportId: string, message: ChatMessageDTO) => Promise { - return makeRoomFanoutNotifier(ROOM_FANOUT_SPEC.report, { - notificationService: deps.notificationService, - listMemberIds: (reportId, limit) => deps.reportChatRepo.listMemberIds(reportId, limit), - isMuted: deps.isMuted, - ...(deps.mutedUserIdsFor ? { mutedUserIdsFor: deps.mutedUserIdsFor } : {}), - presence: deps.presence, - roomKey: (reportId) => deps.roomKeyFor("report", reportId), - isBlockedEitherWay: deps.isBlockedEitherWay, - ...(deps.blockedIdsFor ? { blockedIdsFor: deps.blockedIdsFor } : {}), - ...(deps.coalesceWindowMs !== undefined ? { coalesceWindowMs: deps.coalesceWindowMs } : {}), - ...(deps.now !== undefined ? { now: deps.now } : {}), - ...(deps.claimWindow !== undefined ? { claimWindow: deps.claimWindow } : {}), - ...(deps.dispatchToJob !== undefined ? { dispatchToJob: deps.dispatchToJob } : {}), - ...(deps.logger !== undefined ? { logger: deps.logger } : {}), - }) + return makeRoomChatNotifier("report", deps, (reportId, limit) => + deps.reportChatRepo.listMemberIds(reportId, limit), + ) } diff --git a/services/api/src/services/report-chat-repository.drizzle.ts b/services/api/src/services/report-chat-repository.drizzle.ts index 44a66824..e06edaf9 100644 --- a/services/api/src/services/report-chat-repository.drizzle.ts +++ b/services/api/src/services/report-chat-repository.drizzle.ts @@ -2,16 +2,15 @@ import type { Sql } from "../db/client.js" import { ReportStatusSchema, type ChatMessageDTO, - type PersonDTO, type ReportChatParticipantDTO, } from "@civfix/shared" import type { PresignMedia } from "./media-presign.js" import { monotonicReadWatermarkUpdate } from "./chat-read-state.drizzle.js" -import { publicAuthorIdentity } from "./public-author.js" -import { blockedPairExpr, hiddenIdentity } from "./hidden-identity.js" +import { blockedPairExpr } from "./hidden-identity.js" +import { toRoomMemberPerson, type RoomMemberIdentityRow } from "./room-member-person.js" -export type ChatSystemPayload = NonNullable> -export type ReportSystemStatus = ChatSystemPayload["status"] +type ChatSystemPayload = NonNullable> +type ReportSystemStatus = ChatSystemPayload["status"] export interface SystemChatRow { id: string @@ -23,41 +22,15 @@ export interface SystemChatRow { system_body: string | null } -export interface ReportMemberRowSelect { - user_id: string - role: "owner" | "member" +export type ReportChatRole = "owner" | "member" + +export interface ReportMemberRowSelect extends RoomMemberIdentityRow { + role: ReportChatRole joined_at: Date - display_name: string | null - handle: string | null - bio: string | null - avatar_url: string | null - user_deleted_at: Date | null - is_following: boolean - blocked_pair: boolean } export function toReportParticipantDTO(r: ReportMemberRowSelect): ReportChatParticipantDTO { - const author = publicAuthorIdentity({ - id: r.user_id, - displayName: r.display_name ?? "", - handle: r.handle, - avatarUrl: r.avatar_url, - deletedAt: r.user_deleted_at, - }) - const hidden = r.blocked_pair && !author.deleted ? hiddenIdentity(r.user_id) : null - const user: PersonDTO = { - id: r.user_id, - name: hidden?.name ?? author.name, - handle: hidden !== null ? null : author.handle, - bio: author.deleted || hidden !== null ? null : r.bio, - avatar: author.avatar, - ...(hidden === null && author.avatarUrl !== undefined ? { avatarUrl: author.avatarUrl } : {}), - followers: 0, - following: 0, - isFollowing: r.is_following, - ...(author.deleted ? { deleted: true } : {}), - } - return { user, role: r.role, joinedAt: r.joined_at.toISOString() } + return { user: toRoomMemberPerson(r), role: r.role, joinedAt: r.joined_at.toISOString() } } export const REPORT_CHAT_ROSTER_CAP = 200 @@ -86,8 +59,8 @@ export function mapSystemRow(row: SystemChatRow): ChatMessageDTO { export interface ReportChatRepository { isMember(reportId: string, userId: string): Promise - roleOf(reportId: string, userId: string): Promise<"owner" | "member" | null> - join(reportId: string, userId: string, role?: "owner" | "member"): Promise + roleOf(reportId: string, userId: string): Promise + join(reportId: string, userId: string, role?: ReportChatRole): Promise leave(reportId: string, userId: string): Promise advanceReadWatermark(reportId: string, userId: string, upToMessageId: string): Promise markRead(reportId: string, userId: string, at: Date): Promise @@ -118,8 +91,8 @@ export function makeReportChatRepository( return rows[0]?.exists ?? false }, - async roleOf(reportId: string, userId: string): Promise<"owner" | "member" | null> { - const rows = await sql<{ role: "owner" | "member" }[]>` + async roleOf(reportId: string, userId: string): Promise { + const rows = await sql<{ role: ReportChatRole }[]>` SELECT role FROM report_chat_members WHERE report_id = ${reportId} AND user_id = ${userId} LIMIT 1 @@ -127,11 +100,7 @@ export function makeReportChatRepository( return rows[0]?.role ?? null }, - async join( - reportId: string, - userId: string, - role: "owner" | "member" = "member", - ): Promise { + async join(reportId: string, userId: string, role: ReportChatRole = "member"): Promise { await sql` INSERT INTO report_chat_members (report_id, user_id, role) VALUES (${reportId}, ${userId}, ${role}) diff --git a/services/api/src/services/report-chat-send-wiring.ts b/services/api/src/services/report-chat-send-wiring.ts index 14be2a03..8ba40a64 100644 --- a/services/api/src/services/report-chat-send-wiring.ts +++ b/services/api/src/services/report-chat-send-wiring.ts @@ -19,6 +19,8 @@ import type { ReportChatSendDeps } from "./report-chat-send.js" import type { ChatRepository } from "./chat-repository.drizzle.js" import { writeAudit } from "./admin/audit.js" +const REPORT_MESSAGE_POSTED_AUDIT_ACTION = "report.message_posted" + export interface ContainerReportChatSendOptions { chatRepo: () => ChatRepository mentions?: ChatMentionRecordSeam | undefined @@ -33,7 +35,7 @@ export function makeAuditedReportChatPersist( inTx: async (tx, row) => { await writeAudit(tx, { actorId: actingUserId, - action: "report.message_posted", + action: REPORT_MESSAGE_POSTED_AUDIT_ACTION, target: `report:${input.cleanupId}`, meta: { messageId: row.id }, }) diff --git a/services/api/src/services/report-chat-send.ts b/services/api/src/services/report-chat-send.ts index 6d99dee5..54aeeccc 100644 --- a/services/api/src/services/report-chat-send.ts +++ b/services/api/src/services/report-chat-send.ts @@ -9,7 +9,7 @@ import { mapWithLimit } from "./media-presign.js" import { roomKeyFor } from "../ws/gateway.js" import type { GatewayChatMentions } from "../ws/types.js" -export const REPORT_MENTION_BELL_CONCURRENCY = 4 +const REPORT_MENTION_BELL_CONCURRENCY = 4 export type ReportChatMentionSeam = ChatMentionRecordSeam & Partial> diff --git a/services/api/src/services/report-city-forward-wiring.ts b/services/api/src/services/report-city-forward-wiring.ts index 47f5638a..1366384b 100644 --- a/services/api/src/services/report-city-forward-wiring.ts +++ b/services/api/src/services/report-city-forward-wiring.ts @@ -24,7 +24,16 @@ export interface ReportCityForwardWiringOverrides { logger?: CityForwardLogger } -export const NOOP_REPORT_CITY_FORWARD: ReportCityForwardEffect = () => Promise.resolve() +function makeContainerOutboundMail(container: Container): OutboundMailService { + return makeOutboundMailService({ + repo: makeDrizzleMailRepository(container.getDb().sql), + mailer: container.mailer, + env: { + MAIL_FROM_OUTREACH: container.env.MAIL_FROM_OUTREACH, + MAIL_REPLY_DOMAIN: container.env.MAIL_REPLY_DOMAIN, + }, + }) +} export function makeContainerReportCityForward( container: Container, @@ -44,14 +53,7 @@ export function makeContainerReportCityForward( (reportRepo ??= makeDrizzleDiscussionRepository(container.getDb().sql)) return async (reportId, message, actorUserId) => { - outboundMail ??= makeOutboundMailService({ - repo: makeDrizzleMailRepository(container.getDb().sql), - mailer: container.mailer, - env: { - MAIL_FROM_OUTREACH: container.env.MAIL_FROM_OUTREACH, - MAIL_REPLY_DOMAIN: container.env.MAIL_REPLY_DOMAIN, - }, - }) + outboundMail ??= makeContainerOutboundMail(container) audit ??= makeReportForwardAudit(container.getDb().sql) const report = await getReportRepo().findReportForDiscussion(reportId) if (report === null) return diff --git a/services/api/src/services/report-city-forward.ts b/services/api/src/services/report-city-forward.ts index 5d10ac08..1d7e176f 100644 --- a/services/api/src/services/report-city-forward.ts +++ b/services/api/src/services/report-city-forward.ts @@ -15,11 +15,17 @@ export interface CityForwardContext { actorDisplayName?: string | null } -export const CITY_FORWARD_DEDUP_TTL_SECONDS = 10 * 60 -export const CITY_FORWARD_WINDOW_SECONDS = 60 * 60 +const CITY_FORWARD_DEDUP_TTL_SECONDS = 10 * 60 +const CITY_FORWARD_WINDOW_SECONDS = 60 * 60 export const CITY_FORWARD_PER_SENDER_PER_HOUR = 3 export const CITY_FORWARD_PER_GEOID_PER_HOUR = 20 +const CITY_FORWARD_KEY_PREFIX = "citfwd:" +const dedupKey = (actorUserId: string, reportId: string, geoid: string): string => + `${CITY_FORWARD_KEY_PREFIX}dedup:${actorUserId}:${reportId}:${geoid}` +const senderKey = (actorUserId: string): string => `${CITY_FORWARD_KEY_PREFIX}user:${actorUserId}` +const geoidKey = (geoid: string): string => `${CITY_FORWARD_KEY_PREFIX}geoid:${geoid}` + export interface CityForwardResult { mentioned: boolean geoid: string | null @@ -51,18 +57,15 @@ export function makeCityForwardThrottle( return async (reportId, geoid, actorUserId) => { try { const dedup = await counters.incr( - `citfwd:dedup:${actorUserId}:${reportId}:${geoid}`, + dedupKey(actorUserId, reportId, geoid), CITY_FORWARD_DEDUP_TTL_SECONDS, ) if (dedup > 1) return false - const perSender = await counters.incr( - `citfwd:user:${actorUserId}`, - CITY_FORWARD_WINDOW_SECONDS, - ) + const perSender = await counters.incr(senderKey(actorUserId), CITY_FORWARD_WINDOW_SECONDS) if (perSender > CITY_FORWARD_PER_SENDER_PER_HOUR) return false - const perGeoid = await counters.incr(`citfwd:geoid:${geoid}`, CITY_FORWARD_WINDOW_SECONDS) + const perGeoid = await counters.incr(geoidKey(geoid), CITY_FORWARD_WINDOW_SECONDS) if (perGeoid > CITY_FORWARD_PER_GEOID_PER_HOUR) return false return true @@ -92,17 +95,17 @@ export async function forwardReportCityMention( await recordMention(opts, geoid) const contact = jurisdiction.contactEmail if (contact === null || contact === "") { - return { mentioned: true, geoid, forwarded: false, forwardedAt: null } + return mentionedNotForwarded(geoid) } const thread = await existingReportThread(outboundMail, ctx.reportId, opts.logger) if (thread === null) { - return { mentioned: true, geoid, forwarded: false, forwardedAt: null } + return mentionedNotForwarded(geoid) } if ( opts.canForward !== undefined && !(await opts.canForward(ctx.reportId, geoid, ctx.actorUserId)) ) { - return { mentioned: true, geoid, forwarded: false, forwardedAt: null } + return mentionedNotForwarded(geoid) } const packet = buildDiscussionForwardPacket( { @@ -127,11 +130,15 @@ export async function forwardReportCityMention( return { mentioned: true, geoid, forwarded: true, forwardedAt: createdAt } } catch (err) { opts.logger?.warn({ err, reportId: ctx.reportId, geoid }, "city forward send failed") - return { mentioned: true, geoid, forwarded: false, forwardedAt: null } + return mentionedNotForwarded(geoid) } } -export function discussionForwardSubject(threadSubject: string | null, fallback: string): string { +function mentionedNotForwarded(geoid: string): CityForwardResult { + return { mentioned: true, geoid, forwarded: false, forwardedAt: null } +} + +function discussionForwardSubject(threadSubject: string | null, fallback: string): string { if (threadSubject === null || threadSubject.trim() === "") return fallback return replySubject(threadSubject) } @@ -149,22 +156,34 @@ async function existingReportThread( } } -async function recordMention(opts: CityForwardOptions, geoid: string): Promise { +// The audit trail is best-effort: a failed write is logged and never blocks the forward itself. +async function writeAudit( + opts: CityForwardOptions, + geoid: string, + write: (audit: ReportForwardAudit, messageId: string) => Promise, + failureMessage: string, +): Promise { if (opts.audit === undefined || opts.messageId === undefined) return const messageId = opts.messageId - await opts.audit - .recordMention(messageId, geoid) - .catch((err: unknown) => - opts.logger?.warn({ err, messageId, geoid }, "city forward mention audit write failed"), - ) + await write(opts.audit, messageId).catch((err: unknown) => + opts.logger?.warn({ err, messageId, geoid }, failureMessage), + ) } -async function markForwarded(opts: CityForwardOptions, geoid: string): Promise { - if (opts.audit === undefined || opts.messageId === undefined) return - const messageId = opts.messageId - await opts.audit - .markForwarded(messageId, geoid) - .catch((err: unknown) => - opts.logger?.warn({ err, messageId, geoid }, "city forward delivery audit write failed"), - ) +function recordMention(opts: CityForwardOptions, geoid: string): Promise { + return writeAudit( + opts, + geoid, + (audit, messageId) => audit.recordMention(messageId, geoid), + "city forward mention audit write failed", + ) +} + +function markForwarded(opts: CityForwardOptions, geoid: string): Promise { + return writeAudit( + opts, + geoid, + (audit, messageId) => audit.markForwarded(messageId, geoid), + "city forward delivery audit write failed", + ) } diff --git a/services/api/src/services/report-clustering.ts b/services/api/src/services/report-clustering.ts index 61207784..ac4e2f99 100644 --- a/services/api/src/services/report-clustering.ts +++ b/services/api/src/services/report-clustering.ts @@ -30,7 +30,10 @@ export const CLUSTER_ZOOM_THRESHOLD = 10 * an implied zoom of 3 and can therefore NEVER reach the per-pin branch. MAP_REPORTS_CANDIDATE_CAP * still bounds the pin payload at 2000 rows, and the route's 60s Cache-Control is unchanged. */ -export const MAP_VIEWPORT_REFERENCE_TILES = 8 +const MAP_VIEWPORT_REFERENCE_TILES = 8 + +const DEGREES_OF_LONGITUDE = 360 +const MAX_MAP_ZOOM = 22 export interface MapBBox { west: number @@ -47,9 +50,9 @@ export function impliedZoomForBBox(bbox: MapBBox): number { // The route already rejects west >= east, but a zero span would send log2 to +Infinity, i.e. no clamp. const span = Math.max(lngSpan, latSpan) if (!Number.isFinite(span) || span <= 0) return 0 - const z = Math.log2((360 * MAP_VIEWPORT_REFERENCE_TILES) / span) + const z = Math.log2((DEGREES_OF_LONGITUDE * MAP_VIEWPORT_REFERENCE_TILES) / span) if (!Number.isFinite(z)) return 0 - return Math.max(0, Math.min(22, Math.floor(z))) + return Math.max(0, Math.min(MAX_MAP_ZOOM, Math.floor(z))) } export function effectiveMapZoom(bbox: MapBBox, requestedZoom: number): number { @@ -68,7 +71,7 @@ export function clusterCellSizeDeg(zoom: number): number { // at NaN coords, which serializes to null (a broken pin). const safeZoom = Number.isFinite(zoom) ? zoom : 0 const z = Math.max(0, Math.floor(safeZoom)) - return 360 / Math.pow(2, z + 1) + return DEGREES_OF_LONGITUDE / Math.pow(2, z + 1) } // An individual pin BEFORE its thumbnail is presigned. clusterByZoom is pure/sync and cannot reach the diff --git a/services/api/src/services/report-repository.drizzle.ts b/services/api/src/services/report-repository.drizzle.ts index 34355456..d445c9c6 100644 --- a/services/api/src/services/report-repository.drizzle.ts +++ b/services/api/src/services/report-repository.drizzle.ts @@ -29,7 +29,6 @@ import type { ReportTimelineView, ReportVisibilityTimelineKind, } from "./report-service.types.js" -import { REPORT_CREATE_SCOPE } from "./report-service.types.js" import { servedKeyExpr, servableMediaFilter } from "./media-served-key.js" import { claimableAsReportMedia, lockUploadsForClaim } from "./media-bindings.js" import { uploadersOf } from "./media-uploader.js" @@ -68,6 +67,62 @@ function notOwnerOutcome(row: { return publiclyVisible ? "forbidden" : "not_found" } +interface OwnerLockRow { + reporter_user_id: string | null + deleted_at: Date | null + status: ReportStatus + visibility: ReportVisibility +} + +// Locks the row for the rest of the transaction, so the ownership verdict cannot go stale before the write. +async function lockOwnedReport( + tx: Queryable, + reportId: string, + userId: string, +): Promise<{ row: OwnerLockRow } | { outcome: "not_found" | "forbidden" }> { + const rows = await tx` + SELECT reporter_user_id, deleted_at, status, visibility + FROM reports + WHERE id = ${reportId} + LIMIT 1 + FOR UPDATE + ` + const row = rows[0] + if (!row || row.deleted_at !== null) return { outcome: "not_found" } + if (row.reporter_user_id !== userId) return { outcome: notOwnerOutcome(row) } + return { row } +} + +// An asset bound to a post, a chat/DM message or any other owner is never re-bindable to a report, or +// the holder of an uploadId could cross-publish private media into a public report gallery. +async function claimReportMedia( + tx: postgres.TransactionSql, + args: CreateReportTxArgs, +): Promise { + await lockUploadsForClaim(tx, args.mediaUploadIds) + const claimed = await tx<{ upload_id: string }[]>` + UPDATE media_assets + SET report_id = ${args.reportId} + WHERE upload_id IN ${tx(args.mediaUploadIds)} + AND (report_id IS NULL OR report_id = ${args.reportId}) + AND post_id IS NULL AND chat_message_id IS NULL + AND ${claimableAsReportMedia( + tx, + uploadersOf({ + userId: args.reporterUserId, + guestAnonSessionId: args.guestAnonSessionId, + }), + )} + AND (status = 'ready' OR (status = 'validating' AND finalized_at IS NOT NULL)) + RETURNING upload_id + ` + if (claimed.length !== new Set(args.mediaUploadIds).size) { + throw AppError.validation({ + mediaUploadIds: "One or more media uploads are unavailable.", + }) + } +} + export function makeDrizzleReportRepository(sql: Sql): ReportRepository { async function readSnapshot( key: string, @@ -190,31 +245,7 @@ export function makeDrizzleReportRepository(sql: Sql): ReportRepository { ` if (args.mediaUploadIds.length > 0) { - // An asset bound to a post, a chat/DM message or any other owner is never re-bindable to a - // report, or the holder of an uploadId could cross-publish private media into a public - // report gallery. - await lockUploadsForClaim(tx, args.mediaUploadIds) - const claimed = await tx<{ upload_id: string }[]>` - UPDATE media_assets - SET report_id = ${args.reportId} - WHERE upload_id IN ${tx(args.mediaUploadIds)} - AND (report_id IS NULL OR report_id = ${args.reportId}) - AND post_id IS NULL AND chat_message_id IS NULL - AND ${claimableAsReportMedia( - tx, - uploadersOf({ - userId: args.reporterUserId, - guestAnonSessionId: args.guestAnonSessionId, - }), - )} - AND (status = 'ready' OR (status = 'validating' AND finalized_at IS NOT NULL)) - RETURNING upload_id - ` - if (claimed.length !== new Set(args.mediaUploadIds).size) { - throw AppError.validation({ - mediaUploadIds: "One or more media uploads are unavailable.", - }) - } + await claimReportMedia(tx, args) } await tx` @@ -411,23 +442,9 @@ export function makeDrizzleReportRepository(sql: Sql): ReportRepository { input: { status: OwnerToggleStatus; note: string }, ): Promise<"updated" | "unchanged" | "not_found" | "forbidden" | "invalid_state"> { return sql.begin(async (tx) => { - const rows = await tx< - { - reporter_user_id: string | null - deleted_at: Date | null - status: ReportStatus - visibility: ReportVisibility - }[] - >` - SELECT reporter_user_id, deleted_at, status, visibility - FROM reports - WHERE id = ${reportId} - LIMIT 1 - FOR UPDATE - ` - const row = rows[0] - if (!row || row.deleted_at !== null) return "not_found" - if (row.reporter_user_id !== userId) return notOwnerOutcome(row) + const locked = await lockOwnedReport(tx, reportId, userId) + if ("outcome" in locked) return locked.outcome + const { row } = locked const transition = ownerStatusTransition(row.status, input.status) if (transition !== "apply") return transition @@ -446,23 +463,9 @@ export function makeDrizzleReportRepository(sql: Sql): ReportRepository { input: { visibility: ReportVisibility; note: string; kind: ReportVisibilityTimelineKind }, ): Promise<"updated" | "unchanged" | "not_found" | "forbidden"> { return sql.begin(async (tx) => { - const rows = await tx< - { - reporter_user_id: string | null - deleted_at: Date | null - status: ReportStatus - visibility: ReportVisibility - }[] - >` - SELECT reporter_user_id, deleted_at, status, visibility - FROM reports - WHERE id = ${reportId} - LIMIT 1 - FOR UPDATE - ` - const row = rows[0] - if (!row || row.deleted_at !== null) return "not_found" - if (row.reporter_user_id !== userId) return notOwnerOutcome(row) + const locked = await lockOwnedReport(tx, reportId, userId) + if ("outcome" in locked) return locked.outcome + const { row } = locked if (row.visibility === input.visibility) return "unchanged" await tx`UPDATE reports SET visibility = ${input.visibility} WHERE id = ${reportId}` @@ -475,5 +478,3 @@ export function makeDrizzleReportRepository(sql: Sql): ReportRepository { }, } } - -export { REPORT_CREATE_SCOPE } diff --git a/services/api/src/services/report-service.ts b/services/api/src/services/report-service.ts index f68b7fb4..8e7ba8c6 100644 --- a/services/api/src/services/report-service.ts +++ b/services/api/src/services/report-service.ts @@ -29,7 +29,11 @@ import { type UnsignedReportPin, } from "./report-clustering.js" import { isPubliclyVisibleStatus } from "./report-visibility.js" -import { addressProvenance, resolveAddressOrNull } from "./address-resolver.js" +import { + addressProvenance, + resolveAddressOrNull, + type ResolvedAddress, +} from "./address-resolver.js" import { REPORT_AUTOFORWARD_JOB, REPORT_CREATE_SCOPE, @@ -55,85 +59,109 @@ export * from "./report-service.types.js" export * from "./report-clustering.js" const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i +const REPORT_NOT_FOUND = "Report not found" +const RESOLVED_BY_REPORTER_NOTE = "Marked resolved by the reporter" +const REOPENED_BY_REPORTER_NOTE = "Reopened by the reporter" +const HIDDEN_BY_REPORTER_NOTE = "Hidden from the public map by the reporter" +const RELISTED_BY_REPORTER_NOTE = "Re-listed by the reporter" function isUuid(value: string): boolean { return UUID_RE.test(value) } -export function makeReportService(deps: ReportServiceDeps): ReportService { - const newId = deps.newId ?? (() => randomUUID()) - const now = deps.now ?? (() => new Date()) +type PresignMedia = ReportServiceDeps["presignMedia"] - const publicPresign = deps.presignMedia - const privatePresign = deps.presignPrivateMedia ?? deps.presignMedia +interface ReportViewFlags { + mine: boolean + mediaPending?: number + linkedEvents?: LinkedEventRef[] + discussionMeta?: ReportDiscussionMeta | null + chatMeta?: ReportChatMeta | null +} - async function toMediaDTO( - view: ReportMediaView, - presign: ReportServiceDeps["presignMedia"], - ): Promise { - const { url, thumbUrl } = await presign(view.r2Key, view.thumbKey) - return { - id: view.id, - kind: view.kind, - codec: view.codec, - url, - ...(thumbUrl !== undefined ? { thumbUrl } : {}), - width: view.width, - height: view.height, - status: view.status, - } +async function toMediaDTO(view: ReportMediaView, presign: PresignMedia): Promise { + const { url, thumbUrl } = await presign(view.r2Key, view.thumbKey) + return { + id: view.id, + kind: view.kind, + codec: view.codec, + url, + ...(thumbUrl !== undefined ? { thumbUrl } : {}), + width: view.width, + height: view.height, + status: view.status, } +} - async function toMapPinDTO(pin: UnsignedReportPin): Promise { - let thumbUrl: string | null = null - if (pin.r2Key !== null) { - const signed = await deps.presignMedia(pin.r2Key, pin.thumbKey) - thumbUrl = signed.thumbUrl ?? signed.url - } - return { - id: pin.id, - category: pin.category, - type: pin.type, - lat: pin.lat, - lng: pin.lng, - status: pin.status, - ...(pin.title !== null ? { title: pin.title } : {}), - description: pin.description, - thumbUrl, - addr: pin.addr, - ...(pin.referenceCode !== null ? { referenceCode: pin.referenceCode } : {}), - } +async function toMapPinDTO(pin: UnsignedReportPin, presign: PresignMedia): Promise { + let thumbUrl: string | null = null + if (pin.r2Key !== null) { + const signed = await presign(pin.r2Key, pin.thumbKey) + thumbUrl = signed.thumbUrl ?? signed.url } + return { + id: pin.id, + category: pin.category, + type: pin.type, + lat: pin.lat, + lng: pin.lng, + status: pin.status, + ...(pin.title !== null ? { title: pin.title } : {}), + description: pin.description, + thumbUrl, + addr: pin.addr, + ...(pin.referenceCode !== null ? { referenceCode: pin.referenceCode } : {}), + } +} - function toTimelineDTO(view: ReportTimelineView): ReportTimelineEntryDTO { - return { - status: view.status, - at: view.createdAt.toISOString(), - ...(view.note !== null ? { note: view.note } : {}), - ...(view.kind !== null ? { kind: view.kind } : {}), - ...(view.body !== null ? { body: view.body } : {}), - } +function toTimelineDTO(view: ReportTimelineView): ReportTimelineEntryDTO { + return { + status: view.status, + at: view.createdAt.toISOString(), + ...(view.note !== null ? { note: view.note } : {}), + ...(view.kind !== null ? { kind: view.kind } : {}), + ...(view.body !== null ? { body: view.body } : {}), } +} + +function discussionFields(meta: ReportDiscussionMeta | null): Partial { + if (meta === null) return {} + return { + discussionCount: meta.discussionCount, + cityHandle: meta.cityHandle, + cityName: meta.cityName, + canForwardToCity: meta.canForwardToCity, + } +} + +function chatFields(chat: ReportChatMeta | null): Partial { + if (chat === null) return {} + return { + chatJoined: chat.joined, + chatMemberCount: chat.memberCount, + chatMessageCount: chat.messageCount, + chatUnread: chat.unread, + } +} + +export function makeReportService(deps: ReportServiceDeps): ReportService { + const newId = deps.newId ?? (() => randomUUID()) + const now = deps.now ?? (() => new Date()) + + const publicPresign = deps.presignMedia + const privatePresign = deps.presignPrivateMedia ?? deps.presignMedia async function toReportDTO( record: ReportRecord, media: ReportMediaView[], timeline: ReportTimelineView[], - flags: { - mine: boolean - mediaPending?: number - linkedEvents?: LinkedEventRef[] - discussionMeta?: ReportDiscussionMeta | null - chatMeta?: ReportChatMeta | null - }, + flags: ReportViewFlags, ): Promise { const reportIsPublic = isPubliclyVisibleStatus(record.status) && record.visibility === "public" const mediaDTOs = await mapWithLimit(media, PRESIGN_CONCURRENCY, (view) => { const usePrivate = !reportIsPublic || view.status === "validating" return toMediaDTO(view, usePrivate ? privatePresign : publicPresign) }) - const meta = flags.discussionMeta ?? null - const chat = flags.chatMeta ?? null return { id: record.id, category: record.category, @@ -159,22 +187,8 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { mediaPending: flags.mediaPending ?? 0, timeline: timeline.map(toTimelineDTO), linkedEvents: flags.linkedEvents ?? [], - ...(meta !== null - ? { - discussionCount: meta.discussionCount, - cityHandle: meta.cityHandle, - cityName: meta.cityName, - canForwardToCity: meta.canForwardToCity, - } - : {}), - ...(chat !== null - ? { - chatJoined: chat.joined, - chatMemberCount: chat.memberCount, - chatMessageCount: chat.messageCount, - chatUnread: chat.unread, - } - : {}), + ...discussionFields(flags.discussionMeta ?? null), + ...chatFields(flags.chatMeta ?? null), } } @@ -265,16 +279,11 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { const category = REPORT_TYPE_TO_CATEGORY[input.type] const suppliedAddr = input.addr?.trim() ?? "" - const [jurisdictionGeoid, reversed] = await Promise.all([ - deps.resolveJurisdictionGeoid(input.lat, input.lng), - suppliedAddr.length > 0 - ? Promise.resolve(null) - : resolveAddressOrNull(deps.resolveAddress, input.lat, input.lng), - ]) - const jurCode = - deps.resolveJurisdictionCode !== undefined - ? await deps.resolveJurisdictionCode(jurisdictionGeoid) - : UNKNOWN_JURCODE + const { jurisdictionGeoid, reversed, jurCode } = await resolvePlacement( + deps, + input, + suppliedAddr, + ) const h3Cell = reportH3Cell(input.lat, input.lng) const publishedAt = now() const reportId = newId() @@ -326,7 +335,7 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { ? await deps.repo.findReportById(id) : await deps.repo.findReportByReferenceCode(id) const dto = record ? await viewReport(record, viewer.userId ?? null) : null - if (dto === null) throw AppError.notFound("Report not found") + if (dto === null) throw AppError.notFound(REPORT_NOT_FOUND) return dto }, @@ -372,10 +381,8 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { const { clusters, pins: unsignedPins } = clusterByZoom(points, effectiveMapZoom(bbox, zoom)) const counts = countByCategory(points) - const pins: ReportPinDTO[] = await mapWithLimit( - unsignedPins, - PRESIGN_CONCURRENCY, - toMapPinDTO, + const pins: ReportPinDTO[] = await mapWithLimit(unsignedPins, PRESIGN_CONCURRENCY, (pin) => + toMapPinDTO(pin, deps.presignMedia), ) return { @@ -404,7 +411,7 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { }) const items: ReportPinDTO[] = await mapWithLimit(points, PRESIGN_CONCURRENCY, (p) => - toMapPinDTO(mapPointToUnsignedPin(p)), + toMapPinDTO(mapPointToUnsignedPin(p), deps.presignMedia), ) return { items, nextCursor } @@ -412,9 +419,9 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { async resolveReport(userId: string, reportId: string, resolved: boolean): Promise { const status: OwnerToggleStatus = resolved ? "resolved" : "published" - const note = resolved ? "Marked resolved by the reporter" : "Reopened by the reporter" + const note = resolved ? RESOLVED_BY_REPORTER_NOTE : REOPENED_BY_REPORTER_NOTE const outcome = await deps.repo.resolveByOwner(reportId, userId, { status, note }) - if (outcome === "not_found") throw AppError.notFound("Report not found") + if (outcome === "not_found") throw AppError.notFound(REPORT_NOT_FOUND) if (outcome === "forbidden") { throw AppError.forbidden("You can only change the status of your own report") } @@ -429,15 +436,13 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { async unlistReport(userId: string, reportId: string, unlisted: boolean): Promise { const visibility: ReportVisibility = unlisted ? "hidden" : "public" const kind = REPORT_VISIBILITY_TIMELINE_KIND[visibility] - const note = unlisted - ? "Hidden from the public map by the reporter" - : "Re-listed by the reporter" + const note = unlisted ? HIDDEN_BY_REPORTER_NOTE : RELISTED_BY_REPORTER_NOTE const outcome = await deps.repo.setVisibilityByOwner(reportId, userId, { visibility, note, kind, }) - if (outcome === "not_found") throw AppError.notFound("Report not found") + if (outcome === "not_found") throw AppError.notFound(REPORT_NOT_FOUND) if (outcome === "forbidden") { throw AppError.forbidden("You can only hide your own report") } @@ -450,6 +455,28 @@ export function makeReportService(deps: ReportServiceDeps): ReportService { return service } +async function resolvePlacement( + deps: ReportServiceDeps, + input: Pick, + suppliedAddr: string, +): Promise<{ + jurisdictionGeoid: string | null + reversed: ResolvedAddress | null + jurCode: number +}> { + const [jurisdictionGeoid, reversed] = await Promise.all([ + deps.resolveJurisdictionGeoid(input.lat, input.lng), + suppliedAddr.length > 0 + ? Promise.resolve(null) + : resolveAddressOrNull(deps.resolveAddress, input.lat, input.lng), + ]) + const jurCode = + deps.resolveJurisdictionCode !== undefined + ? await deps.resolveJurisdictionCode(jurisdictionGeoid) + : UNKNOWN_JURCODE + return { jurisdictionGeoid, reversed, jurCode } +} + async function maybeEnqueueAutoForward( deps: ReportServiceDeps, reportId: string, diff --git a/services/api/src/services/report-service.types.ts b/services/api/src/services/report-service.types.ts index 21db896d..7cafeb0d 100644 --- a/services/api/src/services/report-service.types.ts +++ b/services/api/src/services/report-service.types.ts @@ -40,7 +40,6 @@ export const REPORTS_SEARCH_DEFAULT_LIMIT = 20 export interface ReportOwner { userId?: string | undefined - anonSessionId?: string | undefined } export interface SignedInReportOwner { diff --git a/services/api/src/services/report-timeline-event.ts b/services/api/src/services/report-timeline-event.ts index e625028f..ebccacd5 100644 --- a/services/api/src/services/report-timeline-event.ts +++ b/services/api/src/services/report-timeline-event.ts @@ -13,6 +13,8 @@ import type { ChatMessageDTO } from "@civfix/shared" +const EMIT_FAILED = "report-chat: system-message emit failed (suppressed)" + export interface ReportTimelineEvent { reportId: string /** The report status AT this event (a valid ReportStatus; validated by insertSystemMessage). */ @@ -56,20 +58,14 @@ export function makeReportChatSystemEmitter( msg = await deps.reportChat.insertSystemMessage(event) } catch (err) { if (deps.propagateInsertFailure === true) throw err - warn( - { err, reportId: event.reportId }, - "report-chat: system-message emit failed (suppressed)", - ) + warn({ err, reportId: event.reportId }, EMIT_FAILED) return } try { await deps.broadcast(deps.roomKeyFor("report", event.reportId), msg) await deps.notify(event.reportId, msg) } catch (err) { - warn( - { err, reportId: event.reportId }, - "report-chat: system-message emit failed (suppressed)", - ) + warn({ err, reportId: event.reportId }, EMIT_FAILED) } }, } diff --git a/services/api/src/services/room-chat-notifier-adapter.ts b/services/api/src/services/room-chat-notifier-adapter.ts new file mode 100644 index 00000000..73763106 --- /dev/null +++ b/services/api/src/services/room-chat-notifier-adapter.ts @@ -0,0 +1,47 @@ +import type { ChatMessageDTO } from "@civfix/shared" +import type { FastifyBaseLogger } from "fastify" +import { + makeRoomFanoutNotifier, + ROOM_FANOUT_SPEC, + type RoomFanoutKind, + type RoomFanoutNotifierDeps, +} from "./chat-room-fanout-notifier.js" + +export type RoomChatNotifierDeps = Pick< + RoomFanoutNotifierDeps, + | "notificationService" + | "isMuted" + | "mutedUserIdsFor" + | "isBlockedEitherWay" + | "blockedIdsFor" + | "coalesceWindowMs" + | "now" + | "claimWindow" + | "dispatchToJob" +> & { + presence?: { online(roomKey: string): Promise } + roomKeyFor: (kind: K, id: string) => string + logger?: Pick | undefined +} + +export function makeRoomChatNotifier( + kind: K, + deps: RoomChatNotifierDeps, + listMemberIds: RoomFanoutNotifierDeps["listMemberIds"], +): (roomId: string, message: ChatMessageDTO) => Promise { + return makeRoomFanoutNotifier(ROOM_FANOUT_SPEC[kind], { + notificationService: deps.notificationService, + listMemberIds, + isMuted: deps.isMuted, + ...(deps.mutedUserIdsFor ? { mutedUserIdsFor: deps.mutedUserIdsFor } : {}), + presence: deps.presence, + roomKey: (roomId) => deps.roomKeyFor(kind, roomId), + isBlockedEitherWay: deps.isBlockedEitherWay, + ...(deps.blockedIdsFor ? { blockedIdsFor: deps.blockedIdsFor } : {}), + ...(deps.coalesceWindowMs !== undefined ? { coalesceWindowMs: deps.coalesceWindowMs } : {}), + ...(deps.now !== undefined ? { now: deps.now } : {}), + ...(deps.claimWindow !== undefined ? { claimWindow: deps.claimWindow } : {}), + ...(deps.dispatchToJob !== undefined ? { dispatchToJob: deps.dispatchToJob } : {}), + ...(deps.logger !== undefined ? { logger: deps.logger } : {}), + }) +} diff --git a/services/api/src/services/room-member-person.ts b/services/api/src/services/room-member-person.ts new file mode 100644 index 00000000..f334080d --- /dev/null +++ b/services/api/src/services/room-member-person.ts @@ -0,0 +1,38 @@ +import type { PersonDTO } from "@civfix/shared" +import { publicAuthorIdentity } from "./public-author.js" +import { hiddenIdentity } from "./hidden-identity.js" + +export interface RoomMemberIdentityRow { + user_id: string + display_name: string | null + handle: string | null + bio: string | null + avatar_url: string | null + user_deleted_at: Date | null + is_following: boolean + blocked_pair: boolean +} + +// Report and group rosters share this mapper so a deleted or blocked member redacts identically in both. +export function toRoomMemberPerson(r: RoomMemberIdentityRow): PersonDTO { + const author = publicAuthorIdentity({ + id: r.user_id, + displayName: r.display_name ?? "", + handle: r.handle, + avatarUrl: r.avatar_url, + deletedAt: r.user_deleted_at, + }) + const hidden = r.blocked_pair && !author.deleted ? hiddenIdentity(r.user_id) : null + return { + id: r.user_id, + name: hidden?.name ?? author.name, + handle: hidden !== null ? null : author.handle, + bio: author.deleted || hidden !== null ? null : r.bio, + avatar: author.avatar, + ...(hidden === null && author.avatarUrl !== undefined ? { avatarUrl: author.avatarUrl } : {}), + followers: 0, + following: 0, + isFollowing: r.is_following, + ...(author.deleted ? { deleted: true } : {}), + } +} diff --git a/services/api/src/services/room-read-service.ts b/services/api/src/services/room-read-service.ts index 73fc3e87..dcf091aa 100644 --- a/services/api/src/services/room-read-service.ts +++ b/services/api/src/services/room-read-service.ts @@ -16,14 +16,12 @@ export interface RoomReadDeps { export function makeMarkRoomRead(deps: RoomReadDeps): MarkRoomRead { const now = deps.now ?? (() => new Date()) - const advanceFor = (kind: RoomKind): AdvanceReadAt | undefined => - kind === "cleanup" - ? deps.cleanup - : kind === "dm" - ? deps.dm - : kind === "report" - ? deps.report - : deps.group + const advanceFor = (kind: RoomKind): AdvanceReadAt | undefined => { + if (kind === "cleanup") return deps.cleanup + if (kind === "dm") return deps.dm + if (kind === "report") return deps.report + return deps.group + } return async (kind, roomId, userId) => { const advance = advanceFor(kind) diff --git a/services/api/src/services/threads-repository.drizzle.ts b/services/api/src/services/threads-repository.drizzle.ts index a0365565..16d495e2 100644 --- a/services/api/src/services/threads-repository.drizzle.ts +++ b/services/api/src/services/threads-repository.drizzle.ts @@ -5,13 +5,14 @@ import type { Sql } from "../db/client.js" import type { ConversationHideRoomKind } from "../db/schema/conversation_hides.js" import { publicReportFilter } from "./report-sql.js" import type { TimeCursor } from "../db/cursor-helpers.js" -import type { - GroupThreadAggregateView, - GroupThreadsSource, - ReportThreadAggregateView, - ReportThreadsSource, - ThreadAggregate, - ThreadsRepository, +import { + THREADS_DEFAULT_LIMIT, + type GroupThreadAggregateView, + type GroupThreadsSource, + type ReportThreadAggregateView, + type ReportThreadsSource, + type ThreadAggregate, + type ThreadsRepository, } from "./threads-service.js" type SqlFragment = postgres.Fragment @@ -201,7 +202,7 @@ export function makeDrizzleReportThreadsSource(sql: Sql): ReportThreadsSource { return { async listReportThreadsFor( userId: string, - limit = 30, + limit = THREADS_DEFAULT_LIMIT, cursor?: TimeCursor | null, ): Promise { const rows = await listThreadFamily< @@ -223,7 +224,7 @@ export function makeDrizzleGroupThreadsSource(sql: Sql): GroupThreadsSource { return { async listGroupThreadsFor( userId: string, - limit = 30, + limit = THREADS_DEFAULT_LIMIT, cursor?: TimeCursor | null, ): Promise { const rows = await listThreadFamily< diff --git a/services/api/src/services/threads-service.ts b/services/api/src/services/threads-service.ts index ba5a2f9f..41b1a1c4 100644 --- a/services/api/src/services/threads-service.ts +++ b/services/api/src/services/threads-service.ts @@ -1,5 +1,5 @@ import { relativeAgo, avatarGradient } from "@civfix/shared" -import type { MessageThreadDTO, PersonDTO } from "@civfix/shared" +import type { MessageThreadDTO, PersonDTO, RoomKind } from "@civfix/shared" import { encodeTimeCursor, pageWith, @@ -121,15 +121,13 @@ export interface GroupThreadsSource { } export interface ThreadsMutesSource { - mutedRoomIdsFor( - userId: string, - roomKind: "cleanup" | "dm" | "report" | "group", - roomIds: string[], - ): Promise> + mutedRoomIdsFor(userId: string, roomKind: RoomKind, roomIds: string[]): Promise> } export const THREADS_DEFAULT_LIMIT = 30 +const DM_MEMBER_COUNT = 2 + export interface ThreadsServiceDeps { repo: ThreadsRepository readState: ChatReadState @@ -160,15 +158,65 @@ export interface ListThreadsOptions { cursor?: string | null } +export interface ThreadsPage { + items: MessageThreadDTO[] + nextCursor: string | null +} + export interface ThreadsService { - list( - userId: string, - opts?: ListThreadsOptions, - ): Promise<{ items: MessageThreadDTO[]; nextCursor: string | null }> - listThreads( - userId: string, - limit?: number, - ): Promise<{ items: MessageThreadDTO[]; nextCursor: string | null }> + list(userId: string, opts?: ListThreadsOptions): Promise + listThreads(userId: string, limit?: number): Promise +} + +interface ThreadLastMessage { + body: string | null + createdAt: Date + senderId: string | null +} + +interface ThreadEntryInput { + id: string + kind: MessageThreadDTO["kind"] + title: string + peer?: PersonDTO + last: ThreadLastMessage | null + unread: number + members: number + muted: boolean + since: Date + channel?: boolean +} + +interface ThreadEntry { + dto: MessageThreadDTO + activity: number +} + +function toThreadEntry(input: ThreadEntryInput, userId: string, now: () => Date): ThreadEntry { + const { last } = input + return { + dto: { + id: input.id, + kind: input.kind, + refId: input.id, + title: input.title, + ...(input.peer !== undefined ? { peer: input.peer } : {}), + last: last !== null ? (last.body ?? "") : null, + ago: last !== null ? relativeAgo(last.createdAt, now()) : null, + lastMessageAt: last !== null ? last.createdAt.toISOString() : null, + lastFromMe: last !== null && last.senderId === userId, + unread: input.unread, + members: input.members, + muted: input.muted, + ...(input.channel === true ? { channel: true as const } : {}), + }, + activity: (last?.createdAt ?? input.since).getTime(), + } +} + +function dmThreadTitle(peer: DmThreadAggregateView["peer"]): string { + if (peer.displayName.trim() !== "") return peer.displayName + return peer.handle !== null ? `@${peer.handle}` : peer.displayName } function beforeCursor(cursor: TimeCursor | null, activity: number, id: string): boolean { @@ -180,10 +228,14 @@ function beforeCursor(cursor: TimeCursor | null, activity: number, id: string): export function makeThreadsService(deps: ThreadsServiceDeps): ThreadsService { const now = deps.now ?? (() => new Date()) - async function list( - userId: string, - opts?: ListThreadsOptions, - ): Promise<{ items: MessageThreadDTO[]; nextCursor: string | null }> { + async function countCleanupUnread(agg: ThreadAggregate, userId: string): Promise { + const lastRead = await deps.readState.lastReadAt(agg.cleanupId, userId) + const watermark = + lastRead !== null && lastRead.getTime() > agg.joinedAt.getTime() ? lastRead : agg.joinedAt + return deps.repo.countUnread(agg.cleanupId, userId, watermark) + } + + async function list(userId: string, opts?: ListThreadsOptions): Promise { const limit = Math.max(1, opts?.limit ?? THREADS_DEFAULT_LIMIT) const cursor = parseTimeCursor(opts?.cursor ?? null) const fetchLimit = limit + 1 @@ -209,10 +261,7 @@ export function makeThreadsService(deps: ThreadsServiceDeps): ThreadsService { groupFetch, ]) - const mutedIdsFor = async ( - roomKind: "cleanup" | "dm" | "report" | "group", - roomIds: string[], - ): Promise> => + const mutedIdsFor = async (roomKind: RoomKind, roomIds: string[]): Promise> => deps.mutes && roomIds.length > 0 ? await deps.mutes.mutedRoomIdsFor(userId, roomKind, roomIds) : new Set() @@ -236,108 +285,76 @@ export function makeThreadsService(deps: ThreadsServiceDeps): ThreadsService { ]) const cleanupEntries = await Promise.all( - aggregates.map(async (agg): Promise<{ dto: MessageThreadDTO; activity: number }> => { - let unread = agg.unread - if (unread === undefined) { - const lastRead = await deps.readState.lastReadAt(agg.cleanupId, userId) - const watermark = - lastRead !== null && lastRead.getTime() > agg.joinedAt.getTime() - ? lastRead - : agg.joinedAt - unread = await deps.repo.countUnread(agg.cleanupId, userId, watermark) - } - - const lastFromMe = agg.last !== null && agg.last.senderId === userId - - return { - dto: { + aggregates.map(async (agg) => + toThreadEntry( + { id: agg.cleanupId, kind: "cleanup", - refId: agg.cleanupId, title: agg.title, - last: agg.last !== null ? (agg.last.body ?? "") : null, - ago: agg.last !== null ? relativeAgo(agg.last.createdAt, now()) : null, - lastMessageAt: agg.last !== null ? agg.last.createdAt.toISOString() : null, - lastFromMe, - unread, + last: agg.last, + unread: agg.unread ?? (await countCleanupUnread(agg, userId)), members: agg.members, muted: mutedCleanup.has(agg.cleanupId), + since: agg.joinedAt, }, - activity: (agg.last?.createdAt ?? agg.joinedAt).getTime(), - } - }), + userId, + now, + ), + ), ) - const dmEntries = dmAggregates.map((agg): { dto: MessageThreadDTO; activity: number } => { - const lastFromMe = agg.last !== null && agg.last.senderId === userId - const peer = peerOf(agg.peer) - const title = - agg.peer.displayName.trim() !== "" - ? agg.peer.displayName - : agg.peer.handle !== null - ? `@${agg.peer.handle}` - : agg.peer.displayName - return { - dto: { + const dmEntries = dmAggregates.map((agg) => + toThreadEntry( + { id: agg.threadId, kind: "dm", - refId: agg.threadId, - title, - peer, - last: agg.last !== null ? (agg.last.body ?? "") : null, - ago: agg.last !== null ? relativeAgo(agg.last.createdAt, now()) : null, - lastMessageAt: agg.last !== null ? agg.last.createdAt.toISOString() : null, - lastFromMe, + title: dmThreadTitle(agg.peer), + peer: peerOf(agg.peer), + last: agg.last, unread: agg.unread, - members: 2, + members: DM_MEMBER_COUNT, muted: mutedDm.has(agg.threadId), + since: agg.createdAt, }, - activity: (agg.last?.createdAt ?? agg.createdAt).getTime(), - } - }) - - const reportEntries = reportAggregates.map( - (agg): { dto: MessageThreadDTO; activity: number } => { - const lastFromMe = agg.last !== null && agg.last.senderId === userId - return { - dto: { - id: agg.reportId, - kind: "report", - refId: agg.reportId, - title: agg.title, - last: agg.last !== null ? (agg.last.body ?? "") : null, - ago: agg.last !== null ? relativeAgo(agg.last.createdAt, now()) : null, - lastMessageAt: agg.last !== null ? agg.last.createdAt.toISOString() : null, - lastFromMe, - unread: agg.unread, - members: agg.members, - muted: mutedReport.has(agg.reportId), - }, - activity: (agg.last?.createdAt ?? agg.joinedAt).getTime(), - } - }, + userId, + now, + ), + ) + + const reportEntries = reportAggregates.map((agg) => + toThreadEntry( + { + id: agg.reportId, + kind: "report", + title: agg.title, + last: agg.last, + unread: agg.unread, + members: agg.members, + muted: mutedReport.has(agg.reportId), + since: agg.joinedAt, + }, + userId, + now, + ), ) - const groupEntries = groupAggregates.map((agg): { dto: MessageThreadDTO; activity: number } => { - const lastFromMe = agg.last !== null && agg.last.senderId === userId - return { - dto: { + const groupEntries = groupAggregates.map((agg) => + toThreadEntry( + { id: agg.groupId, kind: "group", - refId: agg.groupId, title: agg.title, - last: agg.last !== null ? (agg.last.body ?? "") : null, - ago: agg.last !== null ? relativeAgo(agg.last.createdAt, now()) : null, - lastMessageAt: agg.last !== null ? agg.last.createdAt.toISOString() : null, - lastFromMe, + last: agg.last, unread: agg.unread, members: agg.members, muted: mutedGroup.has(agg.groupId), - ...(agg.kind === "channel" ? { channel: true as const } : {}), + since: agg.joinedAt, + channel: agg.kind === "channel", }, - activity: (agg.last?.createdAt ?? agg.joinedAt).getTime(), - } - }) + userId, + now, + ), + ) const merged = [...cleanupEntries, ...dmEntries, ...reportEntries, ...groupEntries] .filter((e) => beforeCursor(cursor, e.activity, e.dto.id)) diff --git a/services/api/src/services/volunteer-hours-anomaly.ts b/services/api/src/services/volunteer-hours-anomaly.ts index 94515c84..7d843d3a 100644 --- a/services/api/src/services/volunteer-hours-anomaly.ts +++ b/services/api/src/services/volunteer-hours-anomaly.ts @@ -1,8 +1,10 @@ import type { VolunteerHoursAnomalyKind } from "./volunteer-hours-service.js" -export const HOURS_ANOMALY_FLAG = "Volunteer hours anomaly" +const HOURS_ANOMALY_FLAG = "Volunteer hours anomaly" -export const HOURS_ANOMALY_REASONS: Record = { +const HOURS_ROUNDING_FACTOR = 100 + +const HOURS_ANOMALY_REASONS: Record = { weekly_hours: "volunteer_hours.weekly_threshold", reciprocal_credit: "volunteer_hours.reciprocal_credit", } @@ -44,5 +46,5 @@ export function toHoursAnomalyModerationItem(input: HoursAnomalyInput): HoursAno } function round2(n: number): number { - return Math.round(n * 100) / 100 + return Math.round(n * HOURS_ROUNDING_FACTOR) / HOURS_ROUNDING_FACTOR } diff --git a/services/api/src/services/volunteer-hours-repository.drizzle.ts b/services/api/src/services/volunteer-hours-repository.drizzle.ts index 53b0e45f..ce59975f 100644 --- a/services/api/src/services/volunteer-hours-repository.drizzle.ts +++ b/services/api/src/services/volunteer-hours-repository.drizzle.ts @@ -36,7 +36,11 @@ import type { const MORE_PAGES = "more" -const RECIPROCAL_LOOKBACK_INTERVAL = `${RECIPROCAL_LOOKBACK_MS / 1000} seconds` +const MS_PER_SECOND = 1000 + +const HOURS_ROUNDING_FACTOR = 100 + +const RECIPROCAL_LOOKBACK_INTERVAL = `${RECIPROCAL_LOOKBACK_MS / MS_PER_SECOND} seconds` const WEEKLY_WINDOW_INTERVAL = "7 days" interface LedgerRow { @@ -66,6 +70,10 @@ interface OrgHoursRow { hours: number } +function round2(n: number): number { + return Math.round(n * HOURS_ROUNDING_FACTOR) / HOURS_ROUNDING_FACTOR +} + function toOrgHoursView(r: OrgHoursRow): OrgHoursView { return { organizationId: r.id, @@ -115,7 +123,7 @@ async function computeTotalHours(sql: Sql, userId: string): Promise { AND source <> 'report' AND jurisdiction_geoid IS NULL) )::float8 AS total ` - return Math.round((rows[0]?.total ?? 0) * 100) / 100 + return round2(rows[0]?.total ?? 0) } async function detectHoursAnomalies( @@ -165,24 +173,29 @@ async function detectHoursAnomalies( return anomalies } -export function makeDrizzleVolunteerHoursRepository(sql: Sql): VolunteerHoursRepository { - return { - async logEventHours(args: LogEventHoursArgs): Promise { - if (args.entries.length === 0) return { credited: 0, changed: [], anomalies: [] } - const userIds = args.entries.map((e) => e.userId) - const hoursByRow = args.entries.map((e) => e.hours) - return sql.begin(async (tx) => { - await tx`SELECT pg_advisory_xact_lock(hashtext('volunteer_event:' || ${args.cleanupId}))` - const lockIds = [...new Set(userIds)].sort() - if (lockIds.length > 0) { - await tx` +interface EventHoursWrite { + args: LogEventHoursArgs + userIds: string[] + hoursByRow: number[] +} + +async function lockEventAndUsers(tx: Queryable, { args, userIds }: EventHoursWrite): Promise { + await tx`SELECT pg_advisory_xact_lock(hashtext('volunteer_event:' || ${args.cleanupId}))` + const lockIds = [...new Set(userIds)].sort() + if (lockIds.length > 0) { + await tx` SELECT pg_advisory_xact_lock(hashtext('volunteer_user:' || u)) FROM unnest(${lockIds}::uuid[]) AS t(u) ORDER BY u ` - } + } +} - const reciprocal = await tx<{ logged_by_user_id: string }[]>` +async function assertNoReciprocalCredit( + tx: Queryable, + { args, userIds }: EventHoursWrite, +): Promise { + const reciprocal = await tx<{ logged_by_user_id: string }[]>` SELECT DISTINCT logged_by_user_id FROM volunteer_hours WHERE cleanup_id = ${args.cleanupId} @@ -192,13 +205,18 @@ export function makeDrizzleVolunteerHoursRepository(sql: Sql): VolunteerHoursRep AND logged_by_user_id <> ${args.actorId} AND logged_by_user_id = ANY(${userIds}::uuid[]) ` - if (reciprocal.length > 0) { - throw AppError.conflict( - "You can't credit hours to someone who has already credited you for this event.", - ) - } + if (reciprocal.length > 0) { + throw AppError.conflict( + "You can't credit hours to someone who has already credited you for this event.", + ) + } +} - const sameDay = await tx<{ user_id: string; hours: number }[]>` +async function assertWithinDailyCap( + tx: Queryable, + { args, userIds }: EventHoursWrite, +): Promise { + const sameDay = await tx<{ user_id: string; hours: number }[]>` SELECT vh.user_id, COALESCE(SUM(vh.hours), 0)::float8 AS hours FROM volunteer_hours vh JOIN cleanups c ON c.id = vh.cleanup_id @@ -212,20 +230,23 @@ export function makeDrizzleVolunteerHoursRepository(sql: Sql): VolunteerHoursRep ) GROUP BY vh.user_id ` - const dailyCapHours = args.dailyCapHours ?? DAILY_HOURS_CAP - const heldByUser = new Map(sameDay.map((r) => [r.user_id, r.hours])) - for (const entry of args.entries) { - const held = heldByUser.get(entry.userId) ?? 0 - if (held + entry.hours > dailyCapHours) { - throw AppError.conflict( - `That attendee already holds ${Math.round(held * 100) / 100} h for events on this date; the daily limit is ${dailyCapHours} h.`, - ) - } - } + const dailyCapHours = args.dailyCapHours ?? DAILY_HOURS_CAP + const heldByUser = new Map(sameDay.map((r) => [r.user_id, r.hours])) + for (const entry of args.entries) { + const held = heldByUser.get(entry.userId) ?? 0 + if (held + entry.hours > dailyCapHours) { + throw AppError.conflict( + `That attendee already holds ${round2(held)} h for events on this date; the daily limit is ${dailyCapHours} h.`, + ) + } + } +} - const audit = await tx< - { user_id: string; previous_hours: number | null; new_hours: number }[] - >` +async function writeHoursAudit( + tx: Queryable, + { args, userIds, hoursByRow }: EventHoursWrite, +): Promise { + const audit = await tx<{ user_id: string; previous_hours: number | null; new_hours: number }[]>` INSERT INTO volunteer_hours_audit (cleanup_id, user_id, actor_user_id, previous_hours, new_hours) SELECT @@ -240,14 +261,21 @@ export function makeDrizzleVolunteerHoursRepository(sql: Sql): VolunteerHoursRep previous_hours::float8 AS previous_hours, new_hours::float8 AS new_hours ` - const changed = audit.map((r) => ({ - userId: r.user_id, - hours: r.new_hours, - previousHours: r.previous_hours, - })) - - if (args.geoid === null) { - const upserted = await tx<{ user_id: string }[]>` + return audit.map((r) => ({ + userId: r.user_id, + hours: r.new_hours, + previousHours: r.previous_hours, + })) +} + +async function upsertEventHours( + tx: Queryable, + { args, userIds, hoursByRow }: EventHoursWrite, +): Promise { + if (args.geoid === null) { + // With no jurisdiction there is no rollup to credit, so only a stale one from an earlier geoid is + // reversed. + const upserted = await tx<{ user_id: string }[]>` WITH prev AS ( SELECT user_id, hours AS old_hours, jurisdiction_geoid AS old_geoid FROM volunteer_hours @@ -278,15 +306,9 @@ export function makeDrizzleVolunteerHoursRepository(sql: Sql): VolunteerHoursRep ) SELECT user_id FROM upsert ` - const anomalies = await detectHoursAnomalies(tx, { - actorId: args.actorId, - cleanupId: args.cleanupId, - userIds, - weeklyFlagHours: args.weeklyFlagHours ?? WEEKLY_HOURS_FLAG_DEFAULT, - }) - return { credited: upserted.length, changed, anomalies } - } - const upserted = await tx<{ user_id: string }[]>` + return upserted.length + } + const upserted = await tx<{ user_id: string }[]>` WITH prev AS ( SELECT user_id, hours AS old_hours, jurisdiction_geoid AS old_geoid FROM volunteer_hours @@ -329,13 +351,31 @@ export function makeDrizzleVolunteerHoursRepository(sql: Sql): VolunteerHoursRep DO UPDATE SET total_hours = user_jurisdiction_hours.total_hours + EXCLUDED.total_hours RETURNING user_id ` + return new Set(upserted.map((r) => r.user_id)).size +} + +export function makeDrizzleVolunteerHoursRepository(sql: Sql): VolunteerHoursRepository { + return { + async logEventHours(args: LogEventHoursArgs): Promise { + if (args.entries.length === 0) return { credited: 0, changed: [], anomalies: [] } + const write: EventHoursWrite = { + args, + userIds: args.entries.map((e) => e.userId), + hoursByRow: args.entries.map((e) => e.hours), + } + return sql.begin(async (tx) => { + await lockEventAndUsers(tx, write) + await assertNoReciprocalCredit(tx, write) + await assertWithinDailyCap(tx, write) + const changed = await writeHoursAudit(tx, write) + const credited = await upsertEventHours(tx, write) const anomalies = await detectHoursAnomalies(tx, { actorId: args.actorId, cleanupId: args.cleanupId, - userIds, + userIds: write.userIds, weeklyFlagHours: args.weeklyFlagHours ?? WEEKLY_HOURS_FLAG_DEFAULT, }) - return { credited: new Set(upserted.map((r) => r.user_id)).size, changed, anomalies } + return { credited, changed, anomalies } }) }, @@ -624,7 +664,7 @@ export function makeDrizzleVolunteerHoursRepository(sql: Sql): VolunteerHoursRep const items = rows.reverse().map(toEntryView) return { items, - totalHours: Math.round(items.reduce((sum, r) => sum + r.hours, 0) * 100) / 100, + totalHours: round2(items.reduce((sum, r) => sum + r.hours, 0)), entryCount: countRows[0]?.count ?? items.length, } }, diff --git a/services/api/src/services/volunteer-hours-repository.memory.ts b/services/api/src/services/volunteer-hours-repository.memory.ts index 16b35ee9..4985bd15 100644 --- a/services/api/src/services/volunteer-hours-repository.memory.ts +++ b/services/api/src/services/volunteer-hours-repository.memory.ts @@ -73,8 +73,12 @@ interface LedgerEntry { voidedAt?: Date } +const HOURS_ROUNDING_FACTOR = 100 + +const DEFAULT_LEGACY_REPORT_HOURS = 0.1 + function round2(n: number): number { - return Math.round(n * 100) / 100 + return Math.round(n * HOURS_ROUNDING_FACTOR) / HOURS_ROUNDING_FACTOR } const WEEK_MS = 7 * 24 * 60 * 60 * 1000 @@ -125,7 +129,7 @@ export class InMemoryVolunteerHoursRepository implements VolunteerHoursRepositor userId: string, reportId: string, geoid: string | null, - hours = 0.1, + hours = DEFAULT_LEGACY_REPORT_HOURS, ): string { const id = this.newId() this.entries.push({ diff --git a/services/api/src/services/volunteer-hours-service.ts b/services/api/src/services/volunteer-hours-service.ts index 964b0ce7..a4d3bc41 100644 --- a/services/api/src/services/volunteer-hours-service.ts +++ b/services/api/src/services/volunteer-hours-service.ts @@ -35,23 +35,27 @@ import type { TopVolunteerRow } from "./host/analytics-repository.drizzle.js" import type { InsightsInvalidator } from "./host/host-analytics-cache.js" import type { NotificationService } from "./notification-service.js" -export const LEADERBOARD_DEFAULT_LIMIT = 20 +const LEADERBOARD_DEFAULT_LIMIT = 20 export const LEADERBOARD_MAX_LIMIT = 50 export const LEADERBOARD_MAX_OFFSET = 500 export const EVENT_HOURS_MEMBER_CAP = 2000 -export { MAX_EVENT_HOURS_ENTRIES } from "@civfix/shared" +const HOURS_ENTRIES_DEFAULT_LIMIT = 20 +const HOURS_ENTRIES_MAX_LIMIT = 50 -export const HOURS_ENTRIES_DEFAULT_LIMIT = 20 -export const HOURS_ENTRIES_MAX_LIMIT = 50 - -export const LEADERBOARD_EXTRAS_MIN_LIMIT = 25 +const LEADERBOARD_EXTRAS_MIN_LIMIT = 25 export const MAX_ORG_CHIPS_FETCH = 20 -export const HOURS_NOTIFY_CONCURRENCY = 8 +const HOURS_NOTIFY_CONCURRENCY = 8 + +const MS_PER_MINUTE = 60_000 + +const MS_PER_HOUR = 60 * MS_PER_MINUTE -export const EVENT_WINDOW_GRACE_MS = 60 * 60 * 1000 +const HOURS_ROUNDING_FACTOR = 100 + +const EVENT_WINDOW_GRACE_MS = MS_PER_HOUR export const DAILY_HOURS_CAP = 24 @@ -87,11 +91,11 @@ export interface EventHoursWindow { export function creditableHoursForEvent(cleanup: EventHoursWindow): number { const windowMs = eventDurationMs(cleanup) if (windowMs <= 0) return 0 - const hours = (windowMs + EVENT_WINDOW_GRACE_MS) / (60 * 60 * 1000) - return Math.min(MAX_EVENT_HOURS, Math.round(hours * 100) / 100) + const hours = (windowMs + EVENT_WINDOW_GRACE_MS) / MS_PER_HOUR + return Math.min(MAX_EVENT_HOURS, round2(hours)) } -export function eventDurationMs(cleanup: EventHoursWindow): number { +function eventDurationMs(cleanup: EventHoursWindow): number { const end = cleanup.completedAt ?? cleanup.endsAt return end.getTime() - cleanup.scheduledAt.getTime() } @@ -317,10 +321,10 @@ function neutralPublicHours(): PublicVolunteerHoursResponse { } function round2(n: number): number { - return Math.round(n * 100) / 100 + return Math.round(n * HOURS_ROUNDING_FACTOR) / HOURS_ROUNDING_FACTOR } -export async function entriesWithCreditorAffiliation( +async function entriesWithCreditorAffiliation( load: AffiliationLoader | undefined, views: readonly VolunteerHoursEntryView[], viewerId: string | null, @@ -346,7 +350,7 @@ export async function entriesWithCreditorAffiliation( ) } -export function toVolunteerHoursEntryDTO(view: VolunteerHoursEntryView): VolunteerHoursEntryDTO { +function toVolunteerHoursEntryDTO(view: VolunteerHoursEntryView): VolunteerHoursEntryDTO { return { id: view.id, source: view.source, @@ -386,6 +390,64 @@ function toEventHoursRow(entry: EventHoursLedgerEntry): { } } +function assertHoursLoggable( + cleanup: CleanupHoursView, + actorStanding: HostStanding, +): { durationMs: number; windowCap: number } { + if (!can(actorStanding, "manage_event")) { + throw AppError.forbidden("Only the event hosts can log volunteer hours.") + } + if (cleanup.status === "cancelled") { + throw AppError.conflict("Volunteer hours can't be logged for a cancelled event.") + } + if (!hasEventEnded(eventWindowOf(cleanup), Date.now())) { + throw AppError.conflict("Volunteer hours can be logged once the event has ended.") + } + const durationMs = eventDurationMs(cleanup) + if (durationMs < MIN_EVENT_DURATION_MS) { + throw AppError.conflict( + `This event ran for less than ${MIN_EVENT_DURATION_MS / MS_PER_MINUTE} minutes, so no volunteer hours can be logged against it.`, + ) + } + return { durationMs, windowCap: creditableHoursForEvent(cleanup) } +} + +function assertCreditableEntries( + entries: readonly EventHoursEntry[], + actorId: string, + durationMs: number, + windowCap: number, +): void { + if (entries.length > MAX_EVENT_HOURS_ENTRIES) { + throw AppError.validation({ + entries: `at most ${MAX_EVENT_HOURS_ENTRIES} attendees may be credited in one request`, + }) + } + + const seen = new Set() + for (const entry of entries) { + if (entry.userId === actorId) { + throw AppError.forbidden( + "You can't log volunteer hours for yourself. Another host must credit you.", + ) + } + if (!(entry.hours >= MIN_EVENT_HOURS) || entry.hours > MAX_EVENT_HOURS) { + throw AppError.validation({ + entries: `hours must be at least ${MIN_EVENT_HOURS} and at most ${MAX_EVENT_HOURS}`, + }) + } + if (entry.hours > windowCap) { + throw AppError.validation({ + entries: `this event ran for ${round2(durationMs / MS_PER_HOUR)} h, so at most ${windowCap} h may be credited per attendee`, + }) + } + if (seen.has(entry.userId)) { + throw AppError.validation({ entries: `duplicate userId: ${entry.userId}` }) + } + seen.add(entry.userId) + } +} + export function makeVolunteerHoursService(deps: VolunteerHoursServiceDeps): VolunteerHoursService { async function organizationChips(views: readonly OrgHoursView[]): Promise { const presign = deps.presignOrgLogo @@ -488,6 +550,19 @@ export function makeVolunteerHoursService(deps: VolunteerHoursServiceDeps): Volu } } + async function assertAllAttending( + cleanupId: string, + entries: readonly EventHoursEntry[], + ): Promise { + const memberIds = new Set(await deps.cleanups.listMemberIds(cleanupId, EVENT_HOURS_MEMBER_CAP)) + const nonMembers = entries.filter((e) => !memberIds.has(e.userId)) + if (nonMembers.length > 0) { + throw AppError.validation({ + entries: `not attending this event: ${nonMembers.map((e) => e.userId).join(", ")}`, + }) + } + } + return { async getMyHours(userId: string): Promise { const totals = await deps.repo.totalsFor(userId) @@ -590,60 +665,9 @@ export function makeVolunteerHoursService(deps: VolunteerHoursServiceDeps): Volu input.cleanupId, input.actorId, ) - if (!can(actorStanding, "manage_event")) { - throw AppError.forbidden("Only the event hosts can log volunteer hours.") - } - if (cleanup.status === "cancelled") { - throw AppError.conflict("Volunteer hours can't be logged for a cancelled event.") - } - if (!hasEventEnded(eventWindowOf(cleanup), Date.now())) { - throw AppError.conflict("Volunteer hours can be logged once the event has ended.") - } - const durationMs = eventDurationMs(cleanup) - if (durationMs < MIN_EVENT_DURATION_MS) { - throw AppError.conflict( - `This event ran for less than ${MIN_EVENT_DURATION_MS / 60_000} minutes, so no volunteer hours can be logged against it.`, - ) - } - const windowCap = creditableHoursForEvent(cleanup) - - if (input.entries.length > MAX_EVENT_HOURS_ENTRIES) { - throw AppError.validation({ - entries: `at most ${MAX_EVENT_HOURS_ENTRIES} attendees may be credited in one request`, - }) - } - - const seen = new Set() - for (const entry of input.entries) { - if (entry.userId === input.actorId) { - throw AppError.forbidden( - "You can't log volunteer hours for yourself. Another host must credit you.", - ) - } - if (!(entry.hours >= MIN_EVENT_HOURS) || entry.hours > MAX_EVENT_HOURS) { - throw AppError.validation({ - entries: `hours must be at least ${MIN_EVENT_HOURS} and at most ${MAX_EVENT_HOURS}`, - }) - } - if (entry.hours > windowCap) { - throw AppError.validation({ - entries: `this event ran for ${round2(durationMs / 3_600_000)} h, so at most ${windowCap} h may be credited per attendee`, - }) - } - if (seen.has(entry.userId)) { - throw AppError.validation({ entries: `duplicate userId: ${entry.userId}` }) - } - seen.add(entry.userId) - } - const memberIds = new Set( - await deps.cleanups.listMemberIds(input.cleanupId, EVENT_HOURS_MEMBER_CAP), - ) - const nonMembers = input.entries.filter((e) => !memberIds.has(e.userId)) - if (nonMembers.length > 0) { - throw AppError.validation({ - entries: `not attending this event: ${nonMembers.map((e) => e.userId).join(", ")}`, - }) - } + const { durationMs, windowCap } = assertHoursLoggable(cleanup, actorStanding) + assertCreditableEntries(input.entries, input.actorId, durationMs, windowCap) + await assertAllAttending(input.cleanupId, input.entries) const result = await deps.repo.logEventHours({ actorId: input.actorId, diff --git a/services/api/src/ws/frame-handler.ts b/services/api/src/ws/frame-handler.ts index 4b5edda7..8633c121 100644 --- a/services/api/src/ws/frame-handler.ts +++ b/services/api/src/ws/frame-handler.ts @@ -1,5 +1,6 @@ import { AppError, + ErrorCode, WsClientMessageSchema, type RoomKind, type WsClientMessage, @@ -33,19 +34,35 @@ import { const PASSTHROUGH_SEND_RESILIENCE: SendResilience = makeSendResilience() +const DEFAULT_ROOM_KIND: RoomKind = "cleanup" + +const BAD_FRAME_CODE = "BAD_FRAME" + +const BLOCKED_CODE = "BLOCKED" + +const CHANNEL_READ_ONLY_CODE = "channel_read_only" + +const NOT_GROUP_MEMBER_MESSAGE = "You are not a member of this group." + +const DM_NOT_ALLOWED_MESSAGE = "You can't message in this conversation." + type ClientFrame = WsClientMessage type ExtractFrame = Extract +interface FrameRoom { + kind: RoomKind + id: string +} + +function frameRoomKind(frame: { roomKind?: RoomKind | undefined }): RoomKind { + return frame.roomKind ?? DEFAULT_ROOM_KIND +} + function serverFrame(frame: WsServerMessage): string { return JSON.stringify(frame) } -function sendError( - conn: ChatConnection, - code: string, - message: string, - room?: { kind: RoomKind; id: string }, -): void { +function sendError(conn: ChatConnection, code: string, message: string, room?: FrameRoom): void { conn.send( serverFrame({ type: "error", @@ -85,7 +102,7 @@ export function broadcastMessageUpdate( void Promise.resolve(chat.broadcastEvent?.(roomKeyFor(roomKind, roomId), frame)).catch(() => {}) } -function decodeRoomKey(roomKey: string): { kind: RoomKind; id: string } { +function decodeRoomKey(roomKey: string): FrameRoom { if (roomKey.startsWith(DM_ROOM_PREFIX)) { return { kind: "dm", id: roomKey.slice(DM_ROOM_PREFIX.length) } } @@ -95,7 +112,7 @@ function decodeRoomKey(roomKey: string): { kind: RoomKind; id: string } { if (roomKey.startsWith(GROUP_ROOM_PREFIX)) { return { kind: "group", id: roomKey.slice(GROUP_ROOM_PREFIX.length) } } - return { kind: "cleanup", id: roomKey } + return { kind: DEFAULT_ROOM_KIND, id: roomKey } } export async function leaveRoomAndAnnounce( @@ -134,55 +151,59 @@ async function authorizeRoom( const ok = await deps.isMember(id, userId) return ok ? { ok: true } - : { ok: false, code: "FORBIDDEN", message: "You are not a member of this cleanup." } + : { ok: false, code: ErrorCode.FORBIDDEN, message: "You are not a member of this cleanup." } } if (kind === "report") { if (deps.reportVisible && !(await deps.reportVisible(id, userId))) { - return { ok: false, code: "NOT_FOUND", message: "Report not found." } + return { ok: false, code: ErrorCode.NOT_FOUND, message: "Report not found." } } if (requireMember) { if (!deps.reportChat) { - return { ok: false, code: "FORBIDDEN", message: "Report chat is not available." } + return { ok: false, code: ErrorCode.FORBIDDEN, message: "Report chat is not available." } } if (!(await deps.reportChat.isMember(id, userId))) { - return { ok: false, code: "FORBIDDEN", message: "Join this report chat to send messages." } + return { + ok: false, + code: ErrorCode.FORBIDDEN, + message: "Join this report chat to send messages.", + } } } return { ok: true } } if (kind === "group") { if (!deps.groupChat) { - return { ok: false, code: "FORBIDDEN", message: "Group chat is not available." } + return { ok: false, code: ErrorCode.FORBIDDEN, message: "Group chat is not available." } } const access = await deps.groupChat.access(id, userId) if (access === null) { - return { ok: false, code: "FORBIDDEN", message: "You are not a member of this group." } + return { ok: false, code: ErrorCode.FORBIDDEN, message: NOT_GROUP_MEMBER_MESSAGE } } if (!requireMember) { if (access.isMember || access.visibility === "public") return { ok: true } - return { ok: false, code: "FORBIDDEN", message: "You are not a member of this group." } + return { ok: false, code: ErrorCode.FORBIDDEN, message: NOT_GROUP_MEMBER_MESSAGE } } if (!access.isMember) { - return { ok: false, code: "FORBIDDEN", message: "You are not a member of this group." } + return { ok: false, code: ErrorCode.FORBIDDEN, message: NOT_GROUP_MEMBER_MESSAGE } } if (!access.canPost) { return { ok: false, - code: "channel_read_only", + code: CHANNEL_READ_ONLY_CODE, message: "Only owners and admins can post in this channel.", } } return { ok: true } } if (!deps.dm) { - return { ok: false, code: "FORBIDDEN", message: "Direct messages are not available." } + return { ok: false, code: ErrorCode.FORBIDDEN, message: "Direct messages are not available." } } const peer = await deps.dm.peerOf(id, userId) if (peer === null) { - return { ok: false, code: "FORBIDDEN", message: "You can't message in this conversation." } + return { ok: false, code: ErrorCode.FORBIDDEN, message: DM_NOT_ALLOWED_MESSAGE } } if (deps.isBlockedEitherWay && (await deps.isBlockedEitherWay(userId, peer))) { - return { ok: false, code: "FORBIDDEN", message: "You can't message in this conversation." } + return { ok: false, code: ErrorCode.FORBIDDEN, message: DM_NOT_ALLOWED_MESSAGE } } return { ok: true, peer } } @@ -204,11 +225,13 @@ export async function reauthorizeJoinedRooms(session: GatewaySession): Promise {}) } } @@ -219,7 +242,7 @@ function selfSignalThreads(channel: UserChannel | undefined, userId: string, id: async function handleJoin(session: GatewaySession, frame: ExtractFrame<"join">): Promise { const { conn, deps, userId } = session - const kind: RoomKind = frame.roomKind ?? "cleanup" + const kind = frameRoomKind(frame) const id = frame.cleanupId const roomKey = roomKeyFor(kind, id) if (session.joined.has(roomKey)) { @@ -238,7 +261,10 @@ async function handleJoin(session: GatewaySession, frame: ExtractFrame<"join">): return } if (session.joined.size >= WS_MAX_JOINED_ROOMS) { - sendError(conn, "RATE_LIMITED", "You've joined too many rooms. Leave one first.", { kind, id }) + sendError(conn, ErrorCode.RATE_LIMITED, "You've joined too many rooms. Leave one first.", { + kind, + id, + }) return } const auth = await authorizeRoom(deps, kind, id, userId) @@ -286,7 +312,7 @@ async function handleJoin(session: GatewaySession, frame: ExtractFrame<"join">): } async function handleLeave(session: GatewaySession, frame: ExtractFrame<"leave">): Promise { - const kind: RoomKind = frame.roomKind ?? "cleanup" + const kind = frameRoomKind(frame) const roomKey = roomKeyFor(kind, frame.cleanupId) if (!session.joined.has(roomKey)) return await leaveRoomAndAnnounce(session, roomKey) @@ -299,54 +325,121 @@ const CLIENT_AUTHORABLE_KINDS: ReadonlySet = new Set([ "rsvp_change", ]) -const CLIENT_ID_MAX = 64 - export const MENTION_BELL_CONCURRENCY = 8 -async function handleSend(session: GatewaySession, frame: ExtractFrame<"send">): Promise { - const { conn, deps, userId } = session - const kind: RoomKind = frame.roomKind ?? "cleanup" - const id = frame.cleanupId +type SendFrame = ExtractFrame<"send"> + +async function rejectSendFrame( + session: GatewaySession, + frame: SendFrame, + body: string, + room: FrameRoom, +): Promise { + const { conn } = session const verdict = await socketWriteVerdict(session) if (verdict === "revoked") { if (session.closeForAuth) session.closeForAuth() - else sendError(conn, "UNAUTHORIZED", WS_SESSION_ENDED_MESSAGE, { kind, id }) - return + else sendError(conn, ErrorCode.UNAUTHORIZED, WS_SESSION_ENDED_MESSAGE, room) + return true } if (verdict === "suspended") { - sendError(conn, "FORBIDDEN", SUSPENDED_MESSAGE, { kind, id }) - return + sendError(conn, ErrorCode.FORBIDDEN, SUSPENDED_MESSAGE, room) + return true } if (frame.kind !== undefined && !CLIENT_AUTHORABLE_KINDS.has(frame.kind)) { - sendError(conn, "BAD_FRAME", "That message kind can't be sent by a client.", { kind, id }) - return - } - if (frame.clientId.length > CLIENT_ID_MAX) { - sendError(conn, "BAD_FRAME", "clientId is too long.", { kind, id }) - return + sendError(conn, BAD_FRAME_CODE, "That message kind can't be sent by a client.", room) + return true } if (containsSlur(frame.body)) { - sendError(conn, "BLOCKED", "This contains language that isn't allowed.", { kind, id }) - return + sendError(conn, BLOCKED_CODE, "This contains language that isn't allowed.", room) + return true } - const body = frame.body.trim() const carriesMedia = (frame.mediaUploadIds?.length ?? 0) > 0 const isTextFrame = frame.kind === undefined || frame.kind === "text" if (body.length === 0 && !carriesMedia && isTextFrame) { - sendError(conn, "BAD_FRAME", "A message needs text or an attachment.", { kind, id }) - return + sendError(conn, BAD_FRAME_CODE, "A message needs text or an attachment.", room) + return true + } + return false +} + +function optionalSendFields(frame: SendFrame): { + kind?: NonNullable + clientId: string + mediaUploadIds?: string[] + replyToId?: string +} { + const { mediaUploadIds } = frame + return { + ...(frame.kind !== undefined ? { kind: frame.kind } : {}), + clientId: frame.clientId, + ...(mediaUploadIds && mediaUploadIds.length > 0 ? { mediaUploadIds } : {}), + ...(frame.replyToId !== undefined ? { replyToId: frame.replyToId } : {}), + } +} + +function persistSendFrame( + deps: GatewayDeps, + room: FrameRoom, + userId: string, + frame: SendFrame, + body: string, +): Promise { + if (room.kind === "dm") { + return deps.dm!.persist({ + threadId: room.id, + senderId: userId, + body, + ...optionalSendFields(frame), + }) } + return deps.chat.persist({ + cleanupId: room.id, + roomKind: room.kind, + userId, + body, + ...optionalSendFields(frame), + }) +} + +function fireSendSideEffects( + deps: GatewayDeps, + room: FrameRoom, + userId: string, + peer: string | null, + roomKey: string, + message: ChatMessageDTO, + mentions: UserMentionDTO[], +): void { + const { kind, id } = room + const replyTargetUserId = replyBellTarget(message, userId) + fireMentionBells(deps, kind, id, userId, mentions, message, replyTargetUserId) + fireThreadSignal(deps, kind, id, userId) + fireDmBell(deps, kind, id, peer, roomKey, message) + fireReplyBell(deps, kind, id, userId, replyTargetUserId, message) + fireReportCityForward(deps, kind, id, userId, message) + fireGroupFanOut(deps, kind, id, message) +} + +async function handleSend(session: GatewaySession, frame: SendFrame): Promise { + const { conn, deps, userId } = session + const room: FrameRoom = { kind: frameRoomKind(frame), id: frame.cleanupId } + const { kind, id } = room + const body = frame.body.trim() + if (await rejectSendFrame(session, frame, body, room)) return const roomKey = roomKeyFor(kind, id) const auth = await authorizeRoom(deps, kind, id, userId, true) if (!auth.ok) { - sendError(conn, auth.code, auth.message, { kind, id }) + sendError(conn, auth.code, auth.message, room) return } if (deps.reportSendLimiter && !deps.reportSendLimiter.tryConsume(`${userId}:${roomKey}`)) { - sendError(conn, "RATE_LIMITED", "You're sending messages too fast. Please slow down.", { - kind, - id, - }) + sendError( + conn, + ErrorCode.RATE_LIMITED, + "You're sending messages too fast. Please slow down.", + room, + ) return } const resilience = deps.chat.sendResilience ?? PASSTHROUGH_SEND_RESILIENCE @@ -359,39 +452,18 @@ async function handleSend(session: GatewaySession, frame: ExtractFrame<"send">): return } } - const mediaUploadIds = frame.mediaUploadIds let message: ChatMessageDTO try { - if (kind === "dm") { - message = await deps.dm!.persist({ - threadId: id, - senderId: userId, - body, - ...(frame.kind !== undefined ? { kind: frame.kind } : {}), - clientId: frame.clientId, - ...(mediaUploadIds && mediaUploadIds.length > 0 ? { mediaUploadIds } : {}), - ...(frame.replyToId !== undefined ? { replyToId: frame.replyToId } : {}), - }) - } else { - message = await deps.chat.persist({ - cleanupId: id, - roomKind: kind, - userId, - body, - ...(frame.kind !== undefined ? { kind: frame.kind } : {}), - clientId: frame.clientId, - ...(mediaUploadIds && mediaUploadIds.length > 0 ? { mediaUploadIds } : {}), - ...(frame.replyToId !== undefined ? { replyToId: frame.replyToId } : {}), - }) - } + message = await persistSendFrame(deps, room, userId, frame, body) } catch (err) { if (reservation.state === "reserved") void resilience.release(dedupeKey) if (err instanceof AppError) { - sendError(conn, err.fields?.code ?? err.code, err.message, { kind, id }) + sendError(conn, err.fields?.code ?? err.code, err.message, room) return } throw err } + // Committed before the mention lookup so a client retrying this clientId re-acks instead of inserting. void resilience.commit(dedupeKey, message.id) const mentions: UserMentionDTO[] = await resolveAndRecordChatMentions(deps.chatMentions, { body, @@ -412,13 +484,7 @@ async function handleSend(session: GatewaySession, frame: ExtractFrame<"send">): conn.id, ) - const replyTargetUserId = replyBellTarget(message, userId) - fireMentionBells(deps, kind, id, userId, mentions, message, replyTargetUserId) - fireThreadSignal(deps, kind, id, userId) - fireDmBell(deps, kind, id, auth.peer ?? null, roomKey, message) - fireReplyBell(deps, kind, id, userId, replyTargetUserId, message) - fireReportCityForward(deps, kind, id, userId, message) - fireGroupFanOut(deps, kind, id, message) + fireSendSideEffects(deps, room, userId, auth.peer ?? null, roomKey, message, mentions) } function replyBellTarget(message: ChatMessageDTO, authorUserId: string): string | null { @@ -512,7 +578,7 @@ function fireDmBell( async function handleTyping(session: GatewaySession, frame: ExtractFrame<"typing">): Promise { const { conn, deps, userId } = session - const kind: RoomKind = frame.roomKind ?? "cleanup" + const kind = frameRoomKind(frame) const id = frame.cleanupId const roomKey = roomKeyFor(kind, id) const now = Date.now() @@ -541,7 +607,7 @@ async function handleAck(session: GatewaySession, frame: ExtractFrame<"ack">): P let kind: RoomKind let id: string if (frame.cleanupId !== undefined) { - kind = frame.roomKind ?? "cleanup" + kind = frameRoomKind(frame) id = frame.cleanupId } else { if (session.joined.size !== 1) return @@ -588,19 +654,19 @@ export async function handleClientFrame(session: GatewaySession, raw: string): P if (session.closed) return const limiter = (session.frameLimiter ??= makeTokenBucketLimiter(WS_FRAME_LIMIT)) if (!limiter.tryConsume(session.conn.id)) { - sendError(session.conn, "RATE_LIMITED", WS_FRAME_RATE_LIMITED_MESSAGE) + sendError(session.conn, ErrorCode.RATE_LIMITED, WS_FRAME_RATE_LIMITED_MESSAGE) return } let parsedJson: unknown try { parsedJson = JSON.parse(raw) } catch { - sendError(session.conn, "BAD_FRAME", "Malformed frame: not JSON.") + sendError(session.conn, BAD_FRAME_CODE, "Malformed frame: not JSON.") return } const result = WsClientMessageSchema.safeParse(parsedJson) if (!result.success) { - sendError(session.conn, "BAD_FRAME", "Frame failed schema validation.") + sendError(session.conn, BAD_FRAME_CODE, "Frame failed schema validation.") return } const frame = result.data diff --git a/services/api/src/ws/handshake.ts b/services/api/src/ws/handshake.ts index 32281604..dbe4ae47 100644 --- a/services/api/src/ws/handshake.ts +++ b/services/api/src/ws/handshake.ts @@ -121,4 +121,4 @@ export async function checkWsHandshake( } } -export { originHeader, wsHasSessionCookie } +export { originHeader } diff --git a/services/api/src/ws/report-rate-limit.ts b/services/api/src/ws/report-rate-limit.ts index 4976450d..958516e0 100644 --- a/services/api/src/ws/report-rate-limit.ts +++ b/services/api/src/ws/report-rate-limit.ts @@ -16,9 +16,20 @@ interface Bucket { const EVICT_PREFER_FULL_WINDOW = 32 +const DEFAULT_MAX_KEYS = 50_000 + +const MS_PER_SECOND = 1000 + +function refilledTokens(bucket: Bucket, at: number, opts: TokenBucketOptions): number { + return Math.min( + opts.capacity, + bucket.tokens + ((at - bucket.last) / MS_PER_SECOND) * opts.refillPerSec, + ) +} + export function makeTokenBucketLimiter(opts: TokenBucketOptions): RateLimiter { const now = opts.now ?? (() => Date.now()) - const maxKeys = Math.max(1, opts.maxKeys ?? 50_000) + const maxKeys = Math.max(1, opts.maxKeys ?? DEFAULT_MAX_KEYS) const buckets = new Map() const evictToCap = (): void => { @@ -29,11 +40,7 @@ export function makeTokenBucketLimiter(opts: TokenBucketOptions): RateLimiter { for (const [k, b] of buckets) { if (scanned >= EVICT_PREFER_FULL_WINDOW) break scanned += 1 - const refilled = Math.min( - opts.capacity, - b.tokens + ((t - b.last) / 1000) * opts.refillPerSec, - ) - if (refilled >= opts.capacity) { + if (refilledTokens(b, t, opts) >= opts.capacity) { victim = k break } @@ -50,7 +57,7 @@ export function makeTokenBucketLimiter(opts: TokenBucketOptions): RateLimiter { const existing = buckets.get(key) const b = existing ?? { tokens: opts.capacity, last: t } if (existing) buckets.delete(key) - b.tokens = Math.min(opts.capacity, b.tokens + ((t - b.last) / 1000) * opts.refillPerSec) + b.tokens = refilledTokens(b, t, opts) b.last = t const allowed = b.tokens >= 1 if (allowed) b.tokens -= 1 diff --git a/services/api/src/ws/send-resilience.ts b/services/api/src/ws/send-resilience.ts index 13d74e84..8bf6ba73 100644 --- a/services/api/src/ws/send-resilience.ts +++ b/services/api/src/ws/send-resilience.ts @@ -13,13 +13,17 @@ export const SEND_DEDUPE_INFLIGHT_DELAY_MS = 100 export const BROADCAST_ATTEMPTS = 3 -export const BROADCAST_BACKOFF_MS: readonly number[] = [100, 250] +const BROADCAST_BACKOFF_MS: readonly number[] = [100, 250] -export const BROADCAST_ATTEMPT_TIMEOUT_MS = 2000 +const BROADCAST_ATTEMPT_TIMEOUT_MS = 2000 -export const RESERVE_TIMEOUT_MS = 1000 +const RESERVE_TIMEOUT_MS = 1000 -export const DEDUPE_WARN_INTERVAL_MS = 60_000 +const DEDUPE_WARN_INTERVAL_MS = 60_000 + +const SEND_DEDUPE_KEY_PREFIX = "chat:send:" + +const DEDUPE_LOG_COMPONENT = "chat-send-dedupe" export type SendReservation = | { state: "reserved" } @@ -33,11 +37,13 @@ export interface SendDedupeStore { } export function sendDedupeKey(userId: string, roomKey: string, clientId: string): string { - return `chat:send:${userId}:${roomKey}:${clientId}` + return `${SEND_DEDUPE_KEY_PREFIX}${userId}:${roomKey}:${clientId}` } const IN_MEMORY_SWEEP_THRESHOLD = 5000 +const MS_PER_SECOND = 1000 + export class InMemorySendDedupeStore implements SendDedupeStore { private readonly store = new Map() private readonly now: () => number @@ -63,7 +69,7 @@ export class InMemorySendDedupeStore implements SendDedupeStore { if (entry.expiresAtMs <= at) this.store.delete(k) } } - this.store.set(key, { value, expiresAtMs: at + ttlSeconds * 1000 }) + this.store.set(key, { value, expiresAtMs: at + ttlSeconds * MS_PER_SECOND }) } reserve(key: string): Promise { @@ -144,7 +150,6 @@ export interface SendResilience { message: ChatMessageDTO, excludeConnId: string, ): Promise - broadcastFailureCount(): number dedupeFailureCount(): number } @@ -213,7 +218,7 @@ export function makeSendResilience(deps: SendResilienceDeps = {}): SendResilienc } catch { dedupeFailures += 1 warn( - { component: "chat-send-dedupe", op: "reserve", dedupeFailures }, + { component: DEDUPE_LOG_COMPONENT, op: "reserve", dedupeFailures }, "chat: send dedupe unavailable; the send will insert without an idempotency reservation", ) return OPEN @@ -224,7 +229,7 @@ export function makeSendResilience(deps: SendResilienceDeps = {}): SendResilienc ? deps.dedupe.commit(key, messageId).catch(() => { dedupeFailures += 1 warn( - { component: "chat-send-dedupe", op: "commit", dedupeFailures }, + { component: DEDUPE_LOG_COMPONENT, op: "commit", dedupeFailures }, "chat: send dedupe commit failed; a retry of this clientId may duplicate", ) }) @@ -234,7 +239,7 @@ export function makeSendResilience(deps: SendResilienceDeps = {}): SendResilienc ? deps.dedupe.release(key).catch(() => { dedupeFailures += 1 warn( - { component: "chat-send-dedupe", op: "release", dedupeFailures }, + { component: DEDUPE_LOG_COMPONENT, op: "release", dedupeFailures }, "chat: send dedupe release failed; the reservation will expire on its own", ) }) @@ -243,7 +248,7 @@ export function makeSendResilience(deps: SendResilienceDeps = {}): SendResilienc deps.findRoomMessage ? deps.findRoomMessage(kind, roomId, messageId, viewerUserId).catch((err: unknown) => { deps.logger?.warn( - { err, kind, roomId, messageId, component: "chat-send-dedupe" }, + { err, kind, roomId, messageId, component: DEDUPE_LOG_COMPONENT }, "chat: idempotent re-ack lookup failed; the resend will insert a new message", ) return null @@ -285,7 +290,6 @@ export function makeSendResilience(deps: SendResilienceDeps = {}): SendResilienc }) }, - broadcastFailureCount: () => failures, dedupeFailureCount: () => dedupeFailures, } } diff --git a/services/api/src/ws/socket-lifecycle.ts b/services/api/src/ws/socket-lifecycle.ts index 0c95849c..44f30aa3 100644 --- a/services/api/src/ws/socket-lifecycle.ts +++ b/services/api/src/ws/socket-lifecycle.ts @@ -1,5 +1,6 @@ import type { FastifyBaseLogger, FastifyInstance, FastifyRequest } from "fastify" import type { WebSocket } from "@fastify/websocket" +import { ErrorCode } from "@civfix/shared" import type { ChatConnection, UserChannel } from "@civfix/shared/interfaces" import { randomUUID } from "node:crypto" import { checkWsHandshake, originHeader } from "./handshake.js" @@ -16,9 +17,11 @@ import { normalizeIp } from "../abuse/ip-rate-limit.js" import { type GatewaySession, type RegisterGatewayOptions, + type WsHandshakeResult, WS_BUFFER_DROP_THRESHOLD, WS_BUFFER_TERMINATE_TICKS, WS_CLOSE_POLICY_VIOLATION, + WS_CONNECTION_CAP_REASON, WS_FRAME_RATE_LIMITED_MESSAGE, WS_FRAME_BACKLOG_REASON, WS_HANDSHAKE_BUFFER_BYTES, @@ -28,6 +31,8 @@ import { WS_MAX_QUEUED_FRAMES, WS_REAUTH_INTERVAL_MS, WS_REAUTH_JITTER_MS, + WS_ROUTE, + WS_SEND_LIMITS, WS_SESSION_ENDED_MESSAGE, WS_SESSION_ENDED_REASON, } from "./types.js" @@ -37,17 +42,21 @@ import type { SocketStatusCheck } from "../auth/account-status.js" const READY_STATE_OPEN = 1 -const CLEANUP_SEND_LIMIT = { capacity: 30, refillPerSec: 0.5 } as const - -const DM_SEND_LIMIT = { capacity: 20, refillPerSec: 0.5 } as const - export const MAX_CONNECTIONS_PER_USER = 10 export const MAX_CONNECTIONS_PER_IP = 30 +const DROPPABLE_FRAME_TYPES = new Set(["presence", "presence_snapshot", "typing", "discussion"]) + +const FRAME_TYPE_PATTERN = /"type"\s*:\s*"([^"]+)"/ + +type SocketLog = Pick + +type AcceptedHandshake = Extract + function makeSendLimiter(reportLimiter: RateLimiter | undefined): RateLimiter { - const cleanup = makeTokenBucketLimiter(CLEANUP_SEND_LIMIT) - const dm = makeTokenBucketLimiter(DM_SEND_LIMIT) + const cleanup = makeTokenBucketLimiter(WS_SEND_LIMITS.cleanup) + const dm = makeTokenBucketLimiter(WS_SEND_LIMITS.dm) return { tryConsume(key: string): boolean { const { kind } = decodeRoomKey(key.slice(key.indexOf(":") + 1)) @@ -64,10 +73,36 @@ function bumpCount(counts: Map, key: string, delta: number): voi else counts.set(key, next) } -const DROPPABLE_FRAME_TYPES = new Set(["presence", "presence_snapshot", "typing", "discussion"]) +interface ConnectionSlots { + tryAcquire(userId: string, ipKey: string): (() => void) | null +} + +function makeConnectionSlots(): ConnectionSlots { + const connectionsPerUser = new Map() + const connectionsPerIp = new Map() + return { + tryAcquire(userId, ipKey) { + if ( + (connectionsPerUser.get(userId) ?? 0) >= MAX_CONNECTIONS_PER_USER || + (connectionsPerIp.get(ipKey) ?? 0) >= MAX_CONNECTIONS_PER_IP + ) { + return null + } + bumpCount(connectionsPerUser, userId, 1) + bumpCount(connectionsPerIp, ipKey, 1) + let released = false + return () => { + if (released) return + released = true + bumpCount(connectionsPerUser, userId, -1) + bumpCount(connectionsPerIp, ipKey, -1) + } + }, + } +} function isDroppableFrame(data: string): boolean { - const m = /"type"\s*:\s*"([^"]+)"/.exec(data) + const m = FRAME_TYPE_PATTERN.exec(data) return m?.[1] !== undefined && DROPPABLE_FRAME_TYPES.has(m[1]) } @@ -91,6 +126,22 @@ function wrapSocket(socket: WebSocket): ChatConnection { } } +function sendErrorThenClose( + socket: WebSocket, + send: (data: string) => void, + log: SocketLog, + error: { code: string; message: string }, + closeReason: string, + sendFailureLog: string, +): void { + try { + send(serverFrame({ type: "error", code: error.code, message: error.message })) + } catch (err) { + log.debug({ err }, sendFailureLog) + } + socket.close(WS_CLOSE_POLICY_VIOLATION, closeReason) +} + export async function subscribeUserChannel( userChannel: UserChannel | undefined, userId: string, @@ -132,7 +183,7 @@ export async function checkSocketAuthorization( } } -export function makeStatusRevalidator( +function makeStatusRevalidator( sessions: SessionService, userId: string, sessionHash: string, @@ -158,301 +209,342 @@ export async function isSocketStillAuthorized( return (await checkSocketAuthorization(sessions, userId, sessionHash, fullCheck)).authorized } -export function registerChatGateway(app: FastifyInstance, opts: RegisterGatewayOptions): void { - const sendLimiter = makeSendLimiter(opts.reportSendLimiter) - const connectionsPerUser = new Map() - const connectionsPerIp = new Map() +interface HandshakeFrameBuffer { + drop(): void + drain(run: (raw: string) => Promise): Promise + route(handler: (raw: string) => void): void +} - app.get("/ws", { websocket: true }, (socket: WebSocket, request: FastifyRequest) => { - let pending: string[] | undefined = [] - let pendingBytes = 0 - let onFrame: ((raw: string) => void) | undefined - const dropPending = (): void => { - pending = undefined - pendingBytes = 0 +function bufferFramesUntilAccepted(socket: WebSocket): HandshakeFrameBuffer { + let pending: string[] | undefined = [] + let pendingBytes = 0 + let onFrame: ((raw: string) => void) | undefined + const drop = (): void => { + pending = undefined + pendingBytes = 0 + } + socket.on("message", (data: unknown) => { + const raw = decodeFrame(data) + if (onFrame !== undefined) { + onFrame(raw) + return } - socket.on("message", (data: unknown) => { - const raw = decodeFrame(data) - if (onFrame !== undefined) { - onFrame(raw) - return - } - if (pending === undefined || pending.length >= WS_HANDSHAKE_FRAME_BUFFER) return - const bytes = Buffer.byteLength(raw, "utf8") - if (pendingBytes + bytes > WS_HANDSHAKE_BUFFER_BYTES) return - pendingBytes += bytes - pending.push(raw) - }) - - void (async () => { - const handshake = await checkWsHandshake(request, { - sessions: opts.sessions, - webOrigins: opts.webOrigins, - redeemTicket: opts.redeemTicket, - }) - if (!handshake.ok) { - dropPending() - if (handshake.code === "FORBIDDEN") { - request.log.warn({ origin: originHeader(request) }, "ws: rejected cross-site Origin") - } - try { - socket.send( - serverFrame({ type: "error", code: handshake.code, message: handshake.message }), - ) - } catch (err) { - request.log.debug({ err }, "ws: handshake-reject send failed (socket already closing)") - } - socket.close(WS_CLOSE_POLICY_VIOLATION, handshake.reason) - return + if (pending === undefined || pending.length >= WS_HANDSHAKE_FRAME_BUFFER) return + const bytes = Buffer.byteLength(raw, "utf8") + if (pendingBytes + bytes > WS_HANDSHAKE_BUFFER_BYTES) return + pendingBytes += bytes + pending.push(raw) + }) + return { + drop, + async drain(run) { + const buffered = pending ?? [] + while (buffered.length > 0) { + const raw = buffered.shift() + if (raw === undefined) break + pendingBytes = Math.max(0, pendingBytes - Buffer.byteLength(raw, "utf8")) + await run(raw) } - const userId = handshake.userId - const ipKey = normalizeIp(request.ip) + drop() + }, + route(handler) { + onFrame = handler + }, + } +} - if ( - (connectionsPerUser.get(userId) ?? 0) >= MAX_CONNECTIONS_PER_USER || - (connectionsPerIp.get(ipKey) ?? 0) >= MAX_CONNECTIONS_PER_IP - ) { - dropPending() - try { - socket.send( - serverFrame({ - type: "error", - code: "RATE_LIMITED", - message: "Too many open connections.", - }), - ) - } catch (err) { - request.log.debug( - { err }, - "ws: connection-cap reject send failed (socket already closing)", - ) +function makeGatewaySession( + socket: WebSocket, + opts: RegisterGatewayOptions, + handshake: AcceptedHandshake, + sendLimiter: RateLimiter, +): GatewaySession { + const { userId } = handshake + return { + userId, + ...(handshake.accountStatus !== undefined ? { accountStatus: handshake.accountStatus } : {}), + ...(handshake.sessionHash !== undefined && opts.sessions !== undefined + ? { + revalidateStatus: makeStatusRevalidator(opts.sessions, userId, handshake.sessionHash), } - socket.close(WS_CLOSE_POLICY_VIOLATION, "too many connections") - return - } + : {}), + conn: wrapSocket(socket), + joined: new Set(), + typingThrottle: new Map(), + deps: { + chat: opts.chat, + isMember: opts.isMember, + markRead: opts.markRead, + markReadOnOpen: opts.markReadOnOpen, + presence: opts.presence, + dm: opts.dm, + isBlockedEitherWay: opts.isBlockedEitherWay, + userChannel: opts.userChannel, + threadRecipientsOf: opts.threadRecipientsOf, + onDmDelivered: opts.onDmDelivered, + onReportMessage: opts.onReportMessage, + onGroupMessage: opts.onGroupMessage, + onChatReply: opts.onChatReply, + reportVisible: opts.reportVisible, + reportSendLimiter: sendLimiter, + chatMentions: opts.chatMentions, + reportChat: opts.reportChat, + groupChat: opts.groupChat, + }, + } +} - bumpCount(connectionsPerUser, userId, 1) - bumpCount(connectionsPerIp, ipKey, 1) - let released = false - const releaseConnectionSlot = (): void => { - if (released) return - released = true - bumpCount(connectionsPerUser, userId, -1) - bumpCount(connectionsPerIp, ipKey, -1) - } - socket.on("close", releaseConnectionSlot) - - const session: GatewaySession = { - userId, - ...(handshake.accountStatus !== undefined - ? { accountStatus: handshake.accountStatus } - : {}), - ...(handshake.sessionHash !== undefined && opts.sessions !== undefined - ? { - revalidateStatus: makeStatusRevalidator(opts.sessions, userId, handshake.sessionHash), - } - : {}), - conn: wrapSocket(socket), - joined: new Set(), - typingThrottle: new Map(), - deps: { - chat: opts.chat, - isMember: opts.isMember, - markRead: opts.markRead, - markReadOnOpen: opts.markReadOnOpen, - presence: opts.presence, - dm: opts.dm, - isBlockedEitherWay: opts.isBlockedEitherWay, - userChannel: opts.userChannel, - threadRecipientsOf: opts.threadRecipientsOf, - onDmDelivered: opts.onDmDelivered, - onReportMessage: opts.onReportMessage, - onGroupMessage: opts.onGroupMessage, - onChatReply: opts.onChatReply, - reportVisible: opts.reportVisible, - reportSendLimiter: sendLimiter, - chatMentions: opts.chatMentions, - reportChat: opts.reportChat, - groupChat: opts.groupChat, - }, - } +interface AuthCloser { + close(): void + isClosing(): boolean +} - let unsubscribeUser = await subscribeUserChannel( - opts.userChannel, - userId, - session.conn, - request.log, +function makeAuthCloser(socket: WebSocket, session: GatewaySession, log: SocketLog): AuthCloser { + let closingForAuth = false + return { + close() { + if (closingForAuth) return + closingForAuth = true + log.info({ userId: session.userId }, "ws: closing socket, session no longer valid") + sendErrorThenClose( + socket, + (data) => session.conn.send(data), + log, + { code: ErrorCode.UNAUTHORIZED, message: WS_SESSION_ENDED_MESSAGE }, + WS_SESSION_ENDED_REASON, + "ws: reauth-reject send failed (socket already closing)", ) + }, + isClosing: () => closingForAuth, + } +} - if (socket.readyState !== READY_STATE_OPEN) { - dropPending() - if (unsubscribeUser) { - void unsubscribeUser().catch(() => {}) - unsubscribeUser = undefined - } - releaseConnectionSlot() +function nextReauthInterval(): number { + return WS_REAUTH_INTERVAL_MS + Math.floor(Math.random() * WS_REAUTH_JITTER_MS) +} + +function startHeartbeat( + socket: WebSocket, + session: GatewaySession, + opts: RegisterGatewayOptions, + sessionHash: string | undefined, + auth: AuthCloser, + log: SocketLog, +): () => void { + let alive = true + let overBufferTicks = 0 + let reauthIntervalMs = nextReauthInterval() + let lastFullReauthAt = Date.now() - Math.floor(Math.random() * WS_REAUTH_INTERVAL_MS) + + const reauthorize = async (): Promise => { + const now = Date.now() + const fullCheck = now - lastFullReauthAt >= reauthIntervalMs + if (fullCheck) { + lastFullReauthAt = now + reauthIntervalMs = nextReauthInterval() + } + const check = await checkSocketAuthorization( + opts.sessions, + session.userId, + sessionHash, + fullCheck, + ) + if (!check.authorized) { + auth.close() + return + } + if (check.accountStatus !== undefined) session.accountStatus = check.accountStatus + if (fullCheck && !auth.isClosing() && !session.closed) { + await reauthorizeJoinedRooms(session) + } + } + + socket.on("pong", () => { + alive = true + }) + const heartbeat = setInterval(() => { + if (!alive) { + socket.terminate() + return + } + void reauthorize().catch((err: unknown) => { + log.warn({ err }, "ws: heartbeat reauthorization failed") + }) + if (socket.bufferedAmount > WS_BUFFER_DROP_THRESHOLD) { + if (++overBufferTicks >= WS_BUFFER_TERMINATE_TICKS) { + socket.terminate() return } - - let alive = true - let overBufferTicks = 0 - const sessionHash = handshake.sessionHash - const nextReauthInterval = (): number => - WS_REAUTH_INTERVAL_MS + Math.floor(Math.random() * WS_REAUTH_JITTER_MS) - let reauthIntervalMs = nextReauthInterval() - let lastFullReauthAt = Date.now() - Math.floor(Math.random() * WS_REAUTH_INTERVAL_MS) - let closingForAuth = false - const closeForAuth = (): void => { - if (closingForAuth) return - closingForAuth = true - request.log.info({ userId }, "ws: closing socket, session no longer valid") - try { - session.conn.send( - serverFrame({ - type: "error", - code: "UNAUTHORIZED", - message: WS_SESSION_ENDED_MESSAGE, - }), - ) - } catch (err) { - request.log.debug({ err }, "ws: reauth-reject send failed (socket already closing)") - } - socket.close(WS_CLOSE_POLICY_VIOLATION, WS_SESSION_ENDED_REASON) + } else { + overBufferTicks = 0 + } + alive = false + if (opts.presence) { + for (const roomKey of session.joined) { + void opts.presence.refresh(roomKey, session.conn.id, session.userId).catch(() => {}) } - session.closeForAuth = closeForAuth - socket.on("pong", () => { - alive = true - }) - const heartbeat = setInterval(() => { - if (!alive) { - socket.terminate() - return - } - void (async () => { - const now = Date.now() - const fullCheck = now - lastFullReauthAt >= reauthIntervalMs - if (fullCheck) { - lastFullReauthAt = now - reauthIntervalMs = nextReauthInterval() - } - const check = await checkSocketAuthorization( - opts.sessions, - userId, - sessionHash, - fullCheck, - ) - if (check.authorized) { - if (check.accountStatus !== undefined) session.accountStatus = check.accountStatus - if (fullCheck && !closingForAuth && !session.closed) { - await reauthorizeJoinedRooms(session) - } - return - } - closeForAuth() - })().catch((err: unknown) => { - request.log.warn({ err }, "ws: heartbeat reauthorization failed") - }) - if (socket.bufferedAmount > WS_BUFFER_DROP_THRESHOLD) { - if (++overBufferTicks >= WS_BUFFER_TERMINATE_TICKS) { - socket.terminate() - return - } - } else { - overBufferTicks = 0 - } - alive = false - if (opts.presence) { - for (const roomKey of session.joined) { - void opts.presence.refresh(roomKey, session.conn.id, session.userId).catch(() => {}) - } - } - try { - socket.ping() - } catch { - socket.terminate() - } - }, WS_HEARTBEAT_MS) - if (typeof heartbeat.unref === "function") heartbeat.unref() - - const runFrame = async (raw: string): Promise => { - try { - await handleClientFrame(session, raw) - } catch (err) { - request.log.error({ err }, "ws frame handler failed") - sendError(session.conn, "INTERNAL", "Failed to handle frame.") - } + } + try { + socket.ping() + } catch { + socket.terminate() + } + }, WS_HEARTBEAT_MS) + if (typeof heartbeat.unref === "function") heartbeat.unref() + return () => clearInterval(heartbeat) +} + +// The per-frame token bucket only runs when a frame is dequeued, so while one handler awaits a slow +// store every later frame would otherwise sit in memory unbounded. +function makeFrameQueue( + socket: WebSocket, + session: GatewaySession, + runFrame: (raw: string) => Promise, + log: SocketLog, +): (raw: string) => void { + let frameChain: Promise = Promise.resolve() + let queuedFrames = 0 + let queuedBytes = 0 + let backlogClosed = false + const closeForBacklog = (): void => { + backlogClosed = true + log.warn( + { userId: session.userId, queuedFrames, queuedBytes }, + "ws: closing socket, inbound frame backlog over cap", + ) + sendErrorThenClose( + socket, + (data) => session.conn.send(data), + log, + { code: ErrorCode.RATE_LIMITED, message: WS_FRAME_RATE_LIMITED_MESSAGE }, + WS_FRAME_BACKLOG_REASON, + "ws: backlog-reject send failed (socket already closing)", + ) + } + return (raw: string): void => { + if (backlogClosed || session.closed) return + const bytes = Buffer.byteLength(raw, "utf8") + if (queuedFrames >= WS_MAX_QUEUED_FRAMES || queuedBytes + bytes > WS_MAX_QUEUED_BYTES) { + closeForBacklog() + return + } + queuedFrames += 1 + queuedBytes += bytes + frameChain = frameChain.then(async () => { + try { + await runFrame(raw) + } finally { + queuedFrames -= 1 + queuedBytes -= bytes } + }) + } +} - socket.on("close", () => { - session.closed = true - dropPending() - clearInterval(heartbeat) - for (const roomKey of [...session.joined]) { - void leaveRoomAndAnnounce(session, roomKey).catch(() => {}) - } - session.joined.clear() - if (unsubscribeUser) { - void unsubscribeUser().catch(() => {}) - unsubscribeUser = undefined - } - }) +interface SocketContext { + socket: WebSocket + request: FastifyRequest + opts: RegisterGatewayOptions + sendLimiter: RateLimiter + slots: ConnectionSlots + frames: HandshakeFrameBuffer +} - socket.on("error", (err: unknown) => { - request.log.warn({ err }, "ws socket error") - }) +async function acceptSocket(ctx: SocketContext): Promise { + const { socket, request, opts, slots, frames } = ctx + const log = request.log + const handshake = await checkWsHandshake(request, { + sessions: opts.sessions, + webOrigins: opts.webOrigins, + redeemTicket: opts.redeemTicket, + }) + if (!handshake.ok) { + frames.drop() + if (handshake.code === "FORBIDDEN") { + log.warn({ origin: originHeader(request) }, "ws: rejected cross-site Origin") + } + sendErrorThenClose( + socket, + (data) => socket.send(data), + log, + handshake, + handshake.reason, + "ws: handshake-reject send failed (socket already closing)", + ) + return + } + const userId = handshake.userId - const buffered = pending ?? [] - while (buffered.length > 0) { - const raw = buffered.shift() - if (raw === undefined) break - pendingBytes = Math.max(0, pendingBytes - Buffer.byteLength(raw, "utf8")) - await runFrame(raw) - } - dropPending() - let frameChain: Promise = Promise.resolve() - let queuedFrames = 0 - let queuedBytes = 0 - let backlogClosed = false - const closeForBacklog = (): void => { - backlogClosed = true - request.log.warn( - { userId, queuedFrames, queuedBytes }, - "ws: closing socket, inbound frame backlog over cap", - ) - try { - session.conn.send( - serverFrame({ - type: "error", - code: "RATE_LIMITED", - message: WS_FRAME_RATE_LIMITED_MESSAGE, - }), - ) - } catch (err) { - request.log.debug({ err }, "ws: backlog-reject send failed (socket already closing)") - } - socket.close(WS_CLOSE_POLICY_VIOLATION, WS_FRAME_BACKLOG_REASON) - } - // The per-frame token bucket only runs when a frame is dequeued, so while one handler awaits - // a slow store every later frame would otherwise sit in memory unbounded. - onFrame = (raw: string): void => { - if (backlogClosed || session.closed) return - const bytes = Buffer.byteLength(raw, "utf8") - if (queuedFrames >= WS_MAX_QUEUED_FRAMES || queuedBytes + bytes > WS_MAX_QUEUED_BYTES) { - closeForBacklog() - return - } - queuedFrames += 1 - queuedBytes += bytes - frameChain = frameChain.then(async () => { - try { - await runFrame(raw) - } finally { - queuedFrames -= 1 - queuedBytes -= bytes - } - }) - } - })() + const releaseConnectionSlot = slots.tryAcquire(userId, normalizeIp(request.ip)) + if (releaseConnectionSlot === null) { + frames.drop() + sendErrorThenClose( + socket, + (data) => socket.send(data), + log, + { code: ErrorCode.RATE_LIMITED, message: "Too many open connections." }, + WS_CONNECTION_CAP_REASON, + "ws: connection-cap reject send failed (socket already closing)", + ) + return + } + socket.on("close", releaseConnectionSlot) + + const session = makeGatewaySession(socket, opts, handshake, ctx.sendLimiter) + + let unsubscribeUser = await subscribeUserChannel(opts.userChannel, userId, session.conn, log) + const unsubscribe = (): void => { + if (!unsubscribeUser) return + void unsubscribeUser().catch(() => {}) + unsubscribeUser = undefined + } + + if (socket.readyState !== READY_STATE_OPEN) { + frames.drop() + unsubscribe() + releaseConnectionSlot() + return + } + + const auth = makeAuthCloser(socket, session, log) + session.closeForAuth = () => auth.close() + const stopHeartbeat = startHeartbeat(socket, session, opts, handshake.sessionHash, auth, log) + + const runFrame = async (raw: string): Promise => { + try { + await handleClientFrame(session, raw) + } catch (err) { + log.error({ err }, "ws frame handler failed") + sendError(session.conn, ErrorCode.INTERNAL, "Failed to handle frame.") + } + } + + socket.on("close", () => { + session.closed = true + frames.drop() + stopHeartbeat() + for (const roomKey of [...session.joined]) { + void leaveRoomAndAnnounce(session, roomKey).catch(() => {}) + } + session.joined.clear() + unsubscribe() + }) + + socket.on("error", (err: unknown) => { + log.warn({ err }, "ws socket error") + }) + + await frames.drain(runFrame) + frames.route(makeFrameQueue(socket, session, runFrame, log)) +} + +export function registerChatGateway(app: FastifyInstance, opts: RegisterGatewayOptions): void { + const sendLimiter = makeSendLimiter(opts.reportSendLimiter) + const slots = makeConnectionSlots() + + app.get(WS_ROUTE, { websocket: true }, (socket: WebSocket, request: FastifyRequest) => { + const frames = bufferFramesUntilAccepted(socket) + void acceptSocket({ socket, request, opts, sendLimiter, slots, frames }) }) } diff --git a/services/api/src/ws/types.ts b/services/api/src/ws/types.ts index 3e614a90..3de367d0 100644 --- a/services/api/src/ws/types.ts +++ b/services/api/src/ws/types.ts @@ -1,8 +1,15 @@ -import type { RoomKind } from "@civfix/shared" +import type { ChatMessageDTO, ChatMessageKind, RoomKind, UserMentionDTO } from "@civfix/shared" import type { FastifyBaseLogger } from "fastify" import type { ChatService, ChatConnection, UserChannel } from "@civfix/shared/interfaces" import type { ChatPresence } from "../adapters/chat-presence.js" +import type { SocketStatusCheck } from "../auth/account-status.js" +import type { SessionService } from "../auth/session-service.js" +import type { AccountStatus } from "../auth/stores.js" +import type { WsTicketPayload } from "../auth/ws-ticket.js" import type { RateLimiter } from "./report-rate-limit.js" +import type { SendResilience } from "./send-resilience.js" + +export const WS_ROUTE = "/ws" export const WS_HEARTBEAT_MS = 30_000 @@ -47,6 +54,15 @@ export const WS_FRAME_RATE_LIMITED_MESSAGE = "You're sending frames too fast. Pl export const WS_FRAME_BACKLOG_REASON = "too many queued frames" +export const WS_CONNECTION_CAP_REASON = "too many connections" + +// Group rooms have no bucket of their own: their sends spend the cleanup bucket, keyed per room. +export const WS_SEND_LIMITS = { + cleanup: { capacity: 30, refillPerSec: 0.5 }, + dm: { capacity: 20, refillPerSec: 0.5 }, + report: { capacity: 30, refillPerSec: 0.5 }, +} as const + export type IsMemberFn = (cleanupId: string, userId: string) => Promise export type MarkReadFn = (cleanupId: string, userId: string, upToId: string) => Promise @@ -59,11 +75,11 @@ export interface GatewayDmDeps { threadId: string senderId: string body: string - kind?: import("@civfix/shared").ChatMessageKind + kind?: ChatMessageKind clientId?: string mediaUploadIds?: string[] replyToId?: string - }): Promise + }): Promise markRead(threadId: string, userId: string, upToId: string): Promise } @@ -74,26 +90,23 @@ export type ThreadRecipientsOf = (kind: RoomKind, id: string, senderId: string) export type OnDmDelivered = ( threadId: string, recipientId: string, - message: import("@civfix/shared").ChatMessageDTO, + message: ChatMessageDTO, ) => Promise export type OnReportMessage = ( reportId: string, - message: import("@civfix/shared").ChatMessageDTO, + message: ChatMessageDTO, actorUserId: string, ) => Promise -export type OnGroupMessage = ( - groupId: string, - message: import("@civfix/shared").ChatMessageDTO, -) => Promise +export type OnGroupMessage = (groupId: string, message: ChatMessageDTO) => Promise export type OnChatReply = (input: { kind: RoomKind roomId: string actorUserId: string targetUserId: string - message: import("@civfix/shared").ChatMessageDTO + message: ChatMessageDTO }) => Promise export interface GatewayReportChat { @@ -120,7 +133,7 @@ export interface GatewayChatMentions { authorUserId: string kind: RoomKind roomId: string - }): Promise + }): Promise recordChatMentions(messageId: string, mentionedUserIds: string[]): Promise logger?: Pick | undefined notifyChatMention(input: { @@ -128,7 +141,7 @@ export interface GatewayChatMentions { roomId: string actorUserId: string mentionedUserId: string - message: import("@civfix/shared").ChatMessageDTO + message: ChatMessageDTO }): Promise } @@ -138,7 +151,7 @@ export type GatewayChatService = Omit & { msg: Parameters[1], opts?: { excludeConnId?: string }, ): Promise - sendResilience?: import("./send-resilience.js").SendResilience | undefined + sendResilience?: SendResilience | undefined } export interface GatewayDeps { @@ -170,8 +183,8 @@ export interface GatewaySession { readonly typingThrottle: Map frameLimiter?: RateLimiter closed?: boolean - accountStatus?: import("../auth/stores.js").AccountStatus - revalidateStatus?: () => Promise + accountStatus?: AccountStatus + revalidateStatus?: () => Promise closeForAuth?: () => void } @@ -180,17 +193,15 @@ export type WsHandshakeResult = ok: true userId: string sessionHash?: string - accountStatus?: import("../auth/stores.js").AccountStatus + accountStatus?: AccountStatus } | { ok: false; code: "FORBIDDEN" | "UNAUTHORIZED"; message: string; reason: string } export interface RegisterGatewayOptions { chat: ChatService isMember: IsMemberFn - sessions: import("../auth/session-service.js").SessionService | undefined - redeemTicket?: - | ((ticket: string) => Promise) - | undefined + sessions: SessionService | undefined + redeemTicket?: ((ticket: string) => Promise) | undefined markRead?: MarkReadFn | undefined markReadOnOpen?: MarkReadOnOpenFn | undefined presence?: ChatPresence | undefined diff --git a/services/media-worker/src/config.ts b/services/media-worker/src/config.ts index c52e6abf..85e7bf06 100644 --- a/services/media-worker/src/config.ts +++ b/services/media-worker/src/config.ts @@ -1,9 +1,11 @@ import { dirname, join } from "node:path" import { MAX_VIDEO_BYTES } from "@civfix/shared" +const TRUTHY_FLAG_VALUES = ["1", "true", "yes", "on"] + export function parseBool(raw: string | undefined, fallback: boolean): boolean { if (raw === undefined || raw === "") return fallback - return ["1", "true", "yes", "on"].includes(raw.trim().toLowerCase()) + return TRUTHY_FLAG_VALUES.includes(raw.trim().toLowerCase()) } export function assertRealSeamInProd( @@ -86,12 +88,14 @@ export function loadSandboxIdentity( return null } +const IMAGE_LANE_ENTRY_FILE = "image-lane.js" + export function loadImageLaneEntry(source: NodeJS.ProcessEnv = process.env): string { const configured = (source.MEDIA_IMAGE_LANE_ENTRY ?? "").trim() if (configured) return configured const entry = process.argv[1] const dir = entry === undefined ? process.cwd() : dirname(entry) - return join(dir, "image-lane.js") + return join(dir, IMAGE_LANE_ENTRY_FILE) } export function loadHttpsProxy(source: NodeJS.ProcessEnv = process.env): string | null { @@ -102,6 +106,34 @@ export function loadHttpsProxy(source: NodeJS.ProcessEnv = process.env): string export const ALLOWED_VIDEO_CODECS: ReadonlySet = new Set(["h264", "hevc"]) const ONE_MB = 1024 * 1024 +const HOUR_MS = 60 * 60 * 1000 + +const DEFAULT_LIMITS = { + maxImagePixels: 24_000_000, + sharpPixelLimit: 32_000_000, + maxChildOutputBytes: MAX_VIDEO_BYTES + ONE_MB, + maxToolStdoutBytes: ONE_MB, + ffprobeTimeoutMs: 10_000, + ffmpegTimeoutMs: 30_000, + imageTimeoutMs: 15_000, + jobTimeoutMs: 90_000, + maxVideoDurationSec: 30, + maxVideoPixels: 3840 * 2160, + maxVideoFps: 120, + maxVideoBitrateBps: 50_000_000, + thumbnailMaxEdge: 400, + nsfwHoldThreshold: 0.8, + mediaChecksConcurrency: 2, + orphanTtlMs: 6 * HOUR_MS, + orphanSweepBatch: 1000, + orphanSweepMaxPages: 50, + holdReleaseSweepBatch: 200, + retentionSweepBatch: 5000, + retentionSweepMaxPages: 20, + stuckMediaTtlMs: HOUR_MS, + stuckSweepBatch: 500, + stuckSweepMaxAttempts: 5, +} as const const IMAGE_LANE_SPAWN_OVERHEAD_MS = 5_000 @@ -130,32 +162,65 @@ function assertImageLaneFitsJobBudget(limits: WorkerLimits): void { export function loadLimits(source: NodeJS.ProcessEnv = process.env): WorkerLimits { const limits: WorkerLimits = { maxDownloadBytes: parsePosInt(source.MEDIA_MAX_DOWNLOAD_BYTES, MAX_VIDEO_BYTES), - maxImagePixels: parsePosInt(source.MEDIA_MAX_IMAGE_PIXELS, 24_000_000), - sharpPixelLimit: parsePosInt(source.MEDIA_SHARP_PIXEL_LIMIT, 32_000_000), - maxChildOutputBytes: parsePosInt(source.MEDIA_MAX_CHILD_OUTPUT_BYTES, MAX_VIDEO_BYTES + ONE_MB), - maxToolStdoutBytes: parsePosInt(source.MEDIA_MAX_TOOL_STDOUT_BYTES, ONE_MB), - ffprobeTimeoutMs: parsePosInt(source.MEDIA_FFPROBE_TIMEOUT_MS, 10_000), - ffmpegTimeoutMs: parsePosInt(source.MEDIA_FFMPEG_TIMEOUT_MS, 30_000), - imageTimeoutMs: parsePosInt(source.MEDIA_IMAGE_TIMEOUT_MS, 15_000), - jobTimeoutMs: parsePosInt(source.MEDIA_JOB_TIMEOUT_MS, 90_000), - maxVideoDurationSec: parsePosInt(source.MEDIA_MAX_VIDEO_DURATION_SEC, 30), - maxVideoPixels: parsePosInt(source.MEDIA_VIDEO_MAX_PIXELS, 3840 * 2160), - maxVideoFps: parsePosInt(source.MEDIA_VIDEO_MAX_FPS, 120), - maxVideoBitrateBps: parsePosInt(source.MEDIA_VIDEO_MAX_BITRATE, 50_000_000), - thumbnailMaxEdge: parsePosInt(source.MEDIA_THUMBNAIL_MAX_EDGE, 400), - nsfwHoldThreshold: clampUnit(source.MEDIA_NSFW_HOLD_THRESHOLD, 0.8), + maxImagePixels: parsePosInt(source.MEDIA_MAX_IMAGE_PIXELS, DEFAULT_LIMITS.maxImagePixels), + sharpPixelLimit: parsePosInt(source.MEDIA_SHARP_PIXEL_LIMIT, DEFAULT_LIMITS.sharpPixelLimit), + maxChildOutputBytes: parsePosInt( + source.MEDIA_MAX_CHILD_OUTPUT_BYTES, + DEFAULT_LIMITS.maxChildOutputBytes, + ), + maxToolStdoutBytes: parsePosInt( + source.MEDIA_MAX_TOOL_STDOUT_BYTES, + DEFAULT_LIMITS.maxToolStdoutBytes, + ), + ffprobeTimeoutMs: parsePosInt(source.MEDIA_FFPROBE_TIMEOUT_MS, DEFAULT_LIMITS.ffprobeTimeoutMs), + ffmpegTimeoutMs: parsePosInt(source.MEDIA_FFMPEG_TIMEOUT_MS, DEFAULT_LIMITS.ffmpegTimeoutMs), + imageTimeoutMs: parsePosInt(source.MEDIA_IMAGE_TIMEOUT_MS, DEFAULT_LIMITS.imageTimeoutMs), + jobTimeoutMs: parsePosInt(source.MEDIA_JOB_TIMEOUT_MS, DEFAULT_LIMITS.jobTimeoutMs), + maxVideoDurationSec: parsePosInt( + source.MEDIA_MAX_VIDEO_DURATION_SEC, + DEFAULT_LIMITS.maxVideoDurationSec, + ), + maxVideoPixels: parsePosInt(source.MEDIA_VIDEO_MAX_PIXELS, DEFAULT_LIMITS.maxVideoPixels), + maxVideoFps: parsePosInt(source.MEDIA_VIDEO_MAX_FPS, DEFAULT_LIMITS.maxVideoFps), + maxVideoBitrateBps: parsePosInt( + source.MEDIA_VIDEO_MAX_BITRATE, + DEFAULT_LIMITS.maxVideoBitrateBps, + ), + thumbnailMaxEdge: parsePosInt(source.MEDIA_THUMBNAIL_MAX_EDGE, DEFAULT_LIMITS.thumbnailMaxEdge), + nsfwHoldThreshold: clampUnit( + source.MEDIA_NSFW_HOLD_THRESHOLD, + DEFAULT_LIMITS.nsfwHoldThreshold, + ), nsfwUnscoredPolicy: source.MEDIA_UNSCORED_POLICY?.trim().toLowerCase() === "hold" ? "hold" : "flag", - mediaChecksConcurrency: parsePosInt(source.MEDIA_CHECKS_CONCURRENCY, 2), - orphanTtlMs: parsePosInt(source.MEDIA_ORPHAN_TTL_MS, 6 * 60 * 60 * 1000), - orphanSweepBatch: parsePosInt(source.MEDIA_ORPHAN_SWEEP_BATCH, 1000), - orphanSweepMaxPages: parsePosInt(source.MEDIA_ORPHAN_SWEEP_MAX_PAGES, 50), - holdReleaseSweepBatch: parsePosInt(source.MEDIA_HOLD_RELEASE_SWEEP_BATCH, 200), - retentionSweepBatch: parsePosInt(source.RETENTION_SWEEP_BATCH, 5000), - retentionSweepMaxPages: parsePosInt(source.RETENTION_SWEEP_MAX_PAGES, 20), - stuckMediaTtlMs: parsePosInt(source.MEDIA_STUCK_TTL_MS, 60 * 60 * 1000), - stuckSweepBatch: parsePosInt(source.MEDIA_STUCK_SWEEP_BATCH, 500), - stuckSweepMaxAttempts: parsePosInt(source.MEDIA_STUCK_SWEEP_MAX_ATTEMPTS, 5), + mediaChecksConcurrency: parsePosInt( + source.MEDIA_CHECKS_CONCURRENCY, + DEFAULT_LIMITS.mediaChecksConcurrency, + ), + orphanTtlMs: parsePosInt(source.MEDIA_ORPHAN_TTL_MS, DEFAULT_LIMITS.orphanTtlMs), + orphanSweepBatch: parsePosInt(source.MEDIA_ORPHAN_SWEEP_BATCH, DEFAULT_LIMITS.orphanSweepBatch), + orphanSweepMaxPages: parsePosInt( + source.MEDIA_ORPHAN_SWEEP_MAX_PAGES, + DEFAULT_LIMITS.orphanSweepMaxPages, + ), + holdReleaseSweepBatch: parsePosInt( + source.MEDIA_HOLD_RELEASE_SWEEP_BATCH, + DEFAULT_LIMITS.holdReleaseSweepBatch, + ), + retentionSweepBatch: parsePosInt( + source.RETENTION_SWEEP_BATCH, + DEFAULT_LIMITS.retentionSweepBatch, + ), + retentionSweepMaxPages: parsePosInt( + source.RETENTION_SWEEP_MAX_PAGES, + DEFAULT_LIMITS.retentionSweepMaxPages, + ), + stuckMediaTtlMs: parsePosInt(source.MEDIA_STUCK_TTL_MS, DEFAULT_LIMITS.stuckMediaTtlMs), + stuckSweepBatch: parsePosInt(source.MEDIA_STUCK_SWEEP_BATCH, DEFAULT_LIMITS.stuckSweepBatch), + stuckSweepMaxAttempts: parsePosInt( + source.MEDIA_STUCK_SWEEP_MAX_ATTEMPTS, + DEFAULT_LIMITS.stuckSweepMaxAttempts, + ), } assertImageLaneFitsJobBudget(limits) return limits diff --git a/services/media-worker/src/download.ts b/services/media-worker/src/download.ts index 5f967b0c..0b1f8013 100644 --- a/services/media-worker/src/download.ts +++ b/services/media-worker/src/download.ts @@ -39,11 +39,13 @@ function isInMemoryReadable(s: unknown): s is InMemoryReadable { } const DOWNLOAD_GET_TTL_SEC = 120 +const MAX_R2_KEY_LENGTH = 512 +const SAFE_R2_KEY_PATTERN = /^[A-Za-z0-9._\-/]+$/ function isSafeR2Key(key: string): boolean { - if (key.length === 0 || key.length > 512) return false + if (key.length === 0 || key.length > MAX_R2_KEY_LENGTH) return false if (key.startsWith("/") || key.includes("..") || key.includes("\\")) return false - return /^[A-Za-z0-9._\-/]+$/.test(key) + return SAFE_R2_KEY_PATTERN.test(key) } export function makeDownloader(storage: Storage): DownloadFn { @@ -53,14 +55,7 @@ export function makeDownloader(storage: Storage): DownloadFn { signal?: AbortSignal, ): Promise { if (isInMemoryReadable(storage)) { - const bytes = storage.get(r2Key) - if (bytes === null) { - throw new StorageUnavailableError(r2Key) - } - if (bytes.byteLength > maxBytes) { - throw new DownloadTooLargeError(maxBytes) - } - return { bytes, etag: await headEtag(storage, r2Key) } + return downloadInMemory(storage, r2Key, maxBytes) } if (!isSafeR2Key(r2Key)) { @@ -72,11 +67,7 @@ export function makeDownloader(storage: Storage): DownloadFn { } catch (err) { throw new StorageUnavailableError(r2Key, err) } - const controller = new AbortController() - if (signal) { - if (signal.aborted) controller.abort() - else signal.addEventListener("abort", () => controller.abort(), { once: true }) - } + const controller = linkedAbortController(signal) let res: Response try { res = await proxyAwareFetch(url, controller.signal) @@ -96,50 +87,90 @@ export function makeDownloader(storage: Storage): DownloadFn { } const etag = normalizeEtag(res.headers.get("etag")) - const body = res.body if (!body) { - let buf: Uint8Array - try { - buf = new Uint8Array(await res.arrayBuffer()) - } catch (err) { - throw new StorageUnavailableError(r2Key, err) - } - if (buf.byteLength > maxBytes) throw new DownloadTooLargeError(maxBytes) - return { bytes: buf, etag } + return { bytes: await readWholeBody(res, r2Key, maxBytes), etag } } + return { bytes: await readCappedStream(body, r2Key, maxBytes, controller), etag } + } +} - const reader = body.getReader() - const chunks: Uint8Array[] = [] - let total = 0 - try { - for (;;) { - const { done, value } = await reader.read() - if (done) break - if (value) { - total += value.byteLength - if (total > maxBytes) { - controller.abort() - throw new DownloadTooLargeError(maxBytes) - } - chunks.push(value) +async function downloadInMemory( + storage: Storage & InMemoryReadable, + r2Key: string, + maxBytes: number, +): Promise { + const bytes = storage.get(r2Key) + if (bytes === null) { + throw new StorageUnavailableError(r2Key) + } + if (bytes.byteLength > maxBytes) { + throw new DownloadTooLargeError(maxBytes) + } + return { bytes, etag: await headEtag(storage, r2Key) } +} + +function linkedAbortController(signal: AbortSignal | undefined): AbortController { + const controller = new AbortController() + if (signal) { + if (signal.aborted) controller.abort() + else signal.addEventListener("abort", () => controller.abort(), { once: true }) + } + return controller +} + +async function readWholeBody(res: Response, r2Key: string, maxBytes: number): Promise { + let buf: Uint8Array + try { + buf = new Uint8Array(await res.arrayBuffer()) + } catch (err) { + throw new StorageUnavailableError(r2Key, err) + } + if (buf.byteLength > maxBytes) throw new DownloadTooLargeError(maxBytes) + return buf +} + +// The cap is enforced while streaming, so a body that lies about (or omits) its content-length is cut +// off at maxBytes instead of being buffered whole. +async function readCappedStream( + body: ReadableStream, + r2Key: string, + maxBytes: number, + controller: AbortController, +): Promise { + const reader = body.getReader() + const chunks: Uint8Array[] = [] + let total = 0 + try { + for (;;) { + const { done, value } = await reader.read() + if (done) break + if (value) { + total += value.byteLength + if (total > maxBytes) { + controller.abort() + throw new DownloadTooLargeError(maxBytes) } + chunks.push(value) } - } catch (err) { - if (err instanceof DownloadTooLargeError) throw err - throw new StorageUnavailableError(r2Key, err) - } finally { - reader.releaseLock() } + } catch (err) { + if (err instanceof DownloadTooLargeError) throw err + throw new StorageUnavailableError(r2Key, err) + } finally { + reader.releaseLock() + } + return concatChunks(chunks, total) +} - const out = new Uint8Array(total) - let offset = 0 - for (const c of chunks) { - out.set(c, offset) - offset += c.byteLength - } - return { bytes: out, etag } +function concatChunks(chunks: Uint8Array[], total: number): Uint8Array { + const out = new Uint8Array(total) + let offset = 0 + for (const c of chunks) { + out.set(c, offset) + offset += c.byteLength } + return out } async function headEtag(storage: Storage, r2Key: string): Promise { diff --git a/services/media-worker/src/jobs.ts b/services/media-worker/src/jobs.ts index 2db283f2..3a9279a1 100644 --- a/services/media-worker/src/jobs.ts +++ b/services/media-worker/src/jobs.ts @@ -119,6 +119,47 @@ export function stopGraceMsFor(jobTimeoutMs: number): number { return STOP_GRACE_PHASES * jobTimeoutMs + STOP_GRACE_MARGIN_MS } +/** + * pg-boss v10 delivers a BATCH array and treats the callback's outcome as a verdict on the WHOLE batch: it + * completes every id when the callback resolves and FAILS every id when it throws (manager.js onFetch). + * Both are wrong for us. The media.checks handler deliberately throws MediaInfraError to request a retry, + * so throwing out of the callback would re-deliver every clean sibling (burning its retryLimit and + * re-downloading/re-decoding/re-encoding bytes that already succeeded, which for a JPEG also means a + * second lossy pass; the abuse_flags rows themselves are safe, since + * drizzle/0056_abuse_flags_worker_open_unique.sql plus insertAbuseFlag's ON CONFLICT DO NOTHING keep one + * OPEN worker flag per subject+reason); resolving instead would mark the failed job COMPLETE and lose it. + * So each job is completed/failed INDIVIDUALLY by id and the callback resolves, leaving pg-boss's + * batch-level complete a no-op (completeJobs only matches state 'active', and a failed job has already + * left it). + * + * Any throw - not just MediaInfraError - fails that one job and gets the queue's bounded retry: an + * unexpected error is a bug, and silently completing the job would hide it AND wedge the asset. + */ +async function settleEachJob( + boss: PgBoss, + name: string, + jobs: PgBoss.Job[], + handler: JobHandler, +): Promise { + const settled = await Promise.allSettled( + jobs.map(async (j) => { + try { + await handler({ id: j.id, data: j.data }) + } catch (err) { + await boss.fail(name, j.id, toFailureOutput(err)) + return + } + await boss.complete(name, j.id) + }), + ) + // A rejection here is the complete/fail WRITE failing (DB blip), not the handler: rethrow so pg-boss's + // batch-level fail retries the batch rather than losing the outcome silently. Jobs already marked + // complete are unaffected (failJobsById only matches state < 'completed'). + const broken = settled.find((r): r is PromiseRejectedResult => r.status === "rejected") + if (broken) + throw broken.reason instanceof Error ? broken.reason : new Error(String(broken.reason)) +} + export class PgBossWorkerJobs implements WorkerJobs { private readonly connectionString: string private readonly stopGraceMs: number @@ -207,39 +248,7 @@ export class PgBossWorkerJobs implements WorkerJobs { options.pollingIntervalSeconds = settings.pollingIntervalSeconds } const boss = this.requireBoss() - await boss.work(name, options, async (jobs: PgBoss.Job[]) => { - // pg-boss v10 delivers a BATCH array and treats the callback's outcome as a verdict on the WHOLE - // batch: it completes every id when the callback resolves and FAILS every id when it throws - // (manager.js onFetch). Both are wrong for us. The media.checks handler deliberately throws - // MediaInfraError to request a retry, so throwing out of here would re-deliver every clean sibling - // (burning its retryLimit and re-downloading/re-decoding/re-encoding bytes that already succeeded, - // which for a JPEG also means a second lossy pass; the abuse_flags rows themselves are safe, since - // drizzle/0056_abuse_flags_worker_open_unique.sql plus insertAbuseFlag's ON CONFLICT DO NOTHING keep - // one OPEN worker flag per subject+reason); resolving instead would mark the failed - // job COMPLETE and lose it. So each job is completed/failed INDIVIDUALLY by id and the callback - // resolves, leaving pg-boss's batch-level complete a no-op (completeJobs only matches state - // 'active', and a failed job has already left it). - // - // Any throw - not just MediaInfraError - fails that one job and gets the queue's bounded retry: an - // unexpected error is a bug, and silently completing the job would hide it AND wedge the asset. - const settled = await Promise.allSettled( - jobs.map(async (j) => { - try { - await handler({ id: j.id, data: j.data }) - } catch (err) { - await boss.fail(name, j.id, toFailureOutput(err)) - return - } - await boss.complete(name, j.id) - }), - ) - // A rejection here is the complete/fail WRITE failing (DB blip), not the handler: rethrow so - // pg-boss's batch-level fail retries the batch rather than losing the outcome silently. Jobs - // already marked complete are unaffected (failJobsById only matches state < 'completed'). - const broken = settled.find((r): r is PromiseRejectedResult => r.status === "rejected") - if (broken) - throw broken.reason instanceof Error ? broken.reason : new Error(String(broken.reason)) - }) + await boss.work(name, options, (jobs: PgBoss.Job[]) => settleEachJob(boss, name, jobs, handler)) } async complete(jobId: string): Promise { diff --git a/services/media-worker/src/jobs/hold-release-sweep.ts b/services/media-worker/src/jobs/hold-release-sweep.ts index 92874ec8..7d98471f 100644 --- a/services/media-worker/src/jobs/hold-release-sweep.ts +++ b/services/media-worker/src/jobs/hold-release-sweep.ts @@ -24,6 +24,8 @@ import type { AbuseChecks } from "@civfix/shared/interfaces" import { releaseAnonHoldIfReady, type AnonHoldReleaseRepo } from "@civfix/api/anon-hold-release" import { resolveJobObs, type JobObsDeps } from "./obs.js" +const HOLD_RELEASE_SWEEP = "anon.hold.release.sweep" + export interface HoldReleaseSweepDeps extends JobObsDeps { repo: AnonHoldReleaseRepo abuseChecks: AbuseChecks @@ -45,7 +47,7 @@ export async function runHoldReleaseSweep( try { ids = await deps.repo.findHeldAnonReportIds(deps.batchSize) } catch (err) { - report(err, { job: "anon.hold.release.sweep", phase: "find" }) + report(err, { job: HOLD_RELEASE_SWEEP, phase: "find" }) log("anon.hold.release.sweep: find failed", { err: String(err) }) return { scanned: 0, published: 0, errors: 1 } } @@ -63,7 +65,7 @@ export async function runHoldReleaseSweep( if (result.published) published++ } catch (err) { errors++ - report(err, { job: "anon.hold.release.sweep", phase: "release", reportId }) + report(err, { job: HOLD_RELEASE_SWEEP, phase: "release", reportId }) log("anon.hold.release.sweep: report failed", { reportId, err: String(err) }) } } diff --git a/services/media-worker/src/jobs/media-checks.ts b/services/media-worker/src/jobs/media-checks.ts index 6cb98564..e0a4bea6 100644 --- a/services/media-worker/src/jobs/media-checks.ts +++ b/services/media-worker/src/jobs/media-checks.ts @@ -5,8 +5,14 @@ import type { FindPhashDuplicateFn } from "@civfix/api/adapters/abuse-checks" import type { WorkerLimits } from "../config.js" import { DownloadTooLargeError, type DownloadedObject, type DownloadFn } from "../download.js" import { settleWithin } from "../timeout.js" -import { resolveJobObs, type JobObsDeps, type JobLogFn, type JobReportFn } from "./obs.js" -import { readEtag } from "@civfix/api/media-repo" +import { + resolveJobObs, + type JobObs, + type JobObsDeps, + type JobLogFn, + type JobReportFn, +} from "./obs.js" +import { MEDIA_CHECKS_JOB, readEtag } from "@civfix/api/media-repo" import { SandboxSpawnError } from "../sandbox/exec.js" import { ScratchSetupError } from "../sandbox/tmp.js" import { servedKey, thumbnailKey } from "./media-keys.js" @@ -17,7 +23,7 @@ export * from "./media-pipeline.js" export type { DownloadFn } -export class JobTimeoutError extends Error { +class JobTimeoutError extends Error { constructor(ms: number) { super(`media.checks exceeded the per-job wall-clock budget of ${ms}ms`) this.name = "JobTimeoutError" @@ -34,7 +40,7 @@ export class MediaInfraError extends Error { } } -export function withJobTimeout(p: Promise, ms: number, onTimeout?: () => void): Promise { +function withJobTimeout(p: Promise, ms: number, onTimeout?: () => void): Promise { return settleWithin(p, ms, { timeoutError: () => new JobTimeoutError(ms), ...(onTimeout ? { onElapsed: onTimeout } : {}), @@ -52,7 +58,7 @@ export interface MediaChecksDeps extends JobObsDeps { publicMediaBase?: string } -export function publicMediaUrl(base: string, key: string): string { +function publicMediaUrl(base: string, key: string): string { return `${base.replace(/\/+$/, "")}/${key.replace(/^\/+/, "")}` } @@ -89,26 +95,52 @@ export interface MediaChecksOutcome { reportId: string | null } +interface RetryableFailure { + phase: string + reportPhase?: string + line: string + ids: Record + logExtra?: Record +} + +// The one shape every retryable failure takes: report it, log it, and hand back the MediaInfraError the +// caller throws. MediaInfraError is the only error allowed to leave a media.checks job, so every infra +// branch below funnels through here rather than wrapping the whole job in a catch that would misclassify +// a rejection as infra. +function retryableFailure( + { log, report }: JobObs, + err: unknown, + failure: RetryableFailure, +): MediaInfraError { + report(err, { + job: MEDIA_CHECKS_JOB, + phase: failure.reportPhase ?? failure.phase, + ...failure.ids, + }) + log(failure.line, { ...failure.ids, ...failure.logExtra, err: String(err) }) + return new MediaInfraError(failure.phase, err) +} + export async function runMediaChecksJobDetailed( payload: MediaChecksPayload, deps: MediaChecksDeps, ): Promise { - const { log, report } = resolveJobObs(deps) + const obs = resolveJobObs(deps) let asset: MediaWorkerAsset | null = null try { asset = await deps.repo.findById(payload.mediaId) if (!asset) asset = await deps.repo.findByUploadId(payload.uploadId) } catch (err) { - report(err, { job: "media.checks", phase: "load-infra", uploadId: payload.uploadId }) - log("media.checks: failed to load asset, will retry", { - uploadId: payload.uploadId, - err: String(err), + throw retryableFailure(obs, err, { + phase: "load", + reportPhase: "load-infra", + line: "media.checks: failed to load asset, will retry", + ids: { uploadId: payload.uploadId }, }) - throw new MediaInfraError("load", err) } if (!asset) { - log("media.checks: asset not found (already swept?)", { uploadId: payload.uploadId }) + obs.log("media.checks: asset not found (already swept?)", { uploadId: payload.uploadId }) return { status: "missing", reportId: null } } const reportId = asset.reportId ?? null @@ -123,15 +155,15 @@ export async function runMediaChecksJob( return outcome.status === "missing" ? "rejected" : outcome.status } +type PhaseOutcome = { settled: true; status: MediaStatus } | { settled: false; value: T } + async function processAsset( asset: MediaWorkerAsset, payload: MediaChecksPayload, deps: MediaChecksDeps, ): Promise { - const { log, report } = resolveJobObs(deps) - if (asset.status !== "validating") { - log("media.checks: asset already terminal, skipping", { + resolveJobObs(deps).log("media.checks: asset already terminal, skipping", { mediaId: asset.id, uploadId: asset.uploadId, status: asset.status, @@ -139,6 +171,33 @@ async function processAsset( return asset.status } + const download = await downloadUpload(asset, payload, deps) + if (download.settled) return download.status + const downloaded = download.value + + const processing = await runPipeline(asset, downloaded.bytes, deps) + if (processing.settled) return processing.status + const result = processing.value + + const publishes = result.status !== "rejected" && result.processedBytes !== null + if (publishes) { + const drift = await uploadDriftBeforePublish(asset, downloaded.etag, deps) + if (drift !== null) return persistRejection(asset, deps, drift) + } + + const { applied, uploaded } = await persistResult(asset, result, publishes, deps) + if (applied === null) { + return settleTerminalRace(asset, deps, { attempted: result.status, uploaded }) + } + await afterApply(asset, result, applied, uploaded, deps) + return result.status +} + +async function downloadUpload( + asset: MediaWorkerAsset, + payload: MediaChecksPayload, + deps: MediaChecksDeps, +): Promise> { let downloaded: DownloadedObject const downloadAbort = new AbortController() try { @@ -149,25 +208,57 @@ async function processAsset( ) } catch (err) { if (err instanceof DownloadTooLargeError) { - return persistRejection(asset, deps, errNote("download too large", err)) + return rejectedOutcome(asset, deps, errNote("download too large", err)) } - report(err, { job: "media.checks", phase: "download-infra", mediaId: asset.id }) - log("media.checks: download infra failure, will retry", { - mediaId: asset.id, - r2Key: asset.r2Key, - err: String(err), + throw retryableFailure(resolveJobObs(deps), err, { + phase: "download", + reportPhase: "download-infra", + line: "media.checks: download infra failure, will retry", + ids: { mediaId: asset.id }, + logExtra: { r2Key: asset.r2Key }, }) - throw new MediaInfraError("download", err) } - const bytes = downloaded.bytes if (payload.uploadEtag && downloaded.etag && payload.uploadEtag !== downloaded.etag) { - return persistRejection(asset, deps, "upload object changed between finalize and processing") + return rejectedOutcome(asset, deps, "upload object changed between finalize and processing") + } + return { settled: false, value: downloaded } +} + +async function rejectedOutcome( + asset: MediaWorkerAsset, + deps: MediaChecksDeps, + note: string, +): Promise> { + return { settled: true, status: await persistRejection(asset, deps, note) } +} + +function retryableProcessFailure(err: unknown): { phase: string; line: string } | null { + if (err instanceof JobTimeoutError) { + return { phase: "process-timeout", line: "media.checks: processing timed out, will retry" } + } + if (err instanceof SandboxSpawnError) { + return { + phase: "sandbox-spawn", + line: "media.checks: a decoder could not be started, will retry", + } + } + if (err instanceof ScratchSetupError) { + return { + phase: "scratch", + line: "media.checks: the sandbox scratch dir could not be prepared, will retry", + } } + return null +} - let result: MediaProcessResult +async function runPipeline( + asset: MediaWorkerAsset, + bytes: Uint8Array, + deps: MediaChecksDeps, +): Promise> { try { - result = await withJobTimeout( + const result = await withJobTimeout( processMedia( { bytes, kind: asset.kind, selfAssetId: asset.id, selfReportId: asset.reportId }, { @@ -178,31 +269,61 @@ async function processAsset( ), deps.limits.jobTimeoutMs, ) + return { settled: false, value: result } } catch (err) { - if (err instanceof JobTimeoutError) { - report(err, { job: "media.checks", phase: "process-timeout", mediaId: asset.id }) - log("media.checks: processing timed out, will retry", { mediaId: asset.id, err: String(err) }) - throw new MediaInfraError("process-timeout", err) - } - if (err instanceof SandboxSpawnError) { - report(err, { job: "media.checks", phase: "sandbox-spawn", mediaId: asset.id }) - log("media.checks: a decoder could not be started, will retry", { - mediaId: asset.id, - err: String(err), - }) - throw new MediaInfraError("sandbox-spawn", err) + const retryable = retryableProcessFailure(err) + if (retryable !== null) { + throw retryableFailure(resolveJobObs(deps), err, { ...retryable, ids: { mediaId: asset.id } }) } - if (err instanceof ScratchSetupError) { - report(err, { job: "media.checks", phase: "scratch", mediaId: asset.id }) - log("media.checks: the sandbox scratch dir could not be prepared, will retry", { - mediaId: asset.id, - err: String(err), - }) - throw new MediaInfraError("scratch", err) - } - return persistRejection(asset, deps, errNote("process failed", err)) + return rejectedOutcome(asset, deps, errNote("process failed", err)) + } +} + +async function uploadDriftBeforePublish( + asset: MediaWorkerAsset, + downloadedEtag: string | null, + deps: MediaChecksDeps, +): Promise { + let current: StorageHead | null + try { + current = await deps.storage.head(asset.r2Key) + } catch (err) { + throw retryableFailure(resolveJobObs(deps), err, { + phase: "publish-precheck", + line: "media.checks: pre-publish head failed, will retry", + ids: { mediaId: asset.id }, + }) } + return uploadDriftNote(current, downloadedEtag) +} + +function contentTypeOption(contentType: string | null): { contentType?: string } { + return contentType !== null ? { contentType } : {} +} +async function putProcessedObjects( + asset: MediaWorkerAsset, + processedBytes: Buffer, + result: MediaProcessResult, + storage: Storage, +): Promise<{ servedKey: string; thumbKey: string | null }> { + const sKey = servedKey(asset.r2Key) + const tKey = result.thumbnailBytes ? thumbnailKey(asset.r2Key) : null + await Promise.all([ + storage.put(sKey, processedBytes, contentTypeOption(result.processedContentType)), + tKey && result.thumbnailBytes + ? storage.put(tKey, result.thumbnailBytes, contentTypeOption(result.thumbnailContentType)) + : Promise.resolve(), + ]) + return { servedKey: sKey, thumbKey: tKey } +} + +async function persistResult( + asset: MediaWorkerAsset, + result: MediaProcessResult, + publishes: boolean, + deps: MediaChecksDeps, +): Promise<{ applied: MediaWorkerAsset | null; uploaded: boolean }> { const patch: MediaResultPatch = { status: result.status, codec: result.codec, @@ -210,50 +331,13 @@ async function processAsset( height: result.height, phash: result.phash, } - - const publishes = result.status !== "rejected" && result.processedBytes !== null - - if (publishes) { - let current: StorageHead | null - try { - current = await deps.storage.head(asset.r2Key) - } catch (err) { - report(err, { job: "media.checks", phase: "publish-precheck", mediaId: asset.id }) - log("media.checks: pre-publish head failed, will retry", { - mediaId: asset.id, - err: String(err), - }) - throw new MediaInfraError("publish-precheck", err) - } - const note = uploadDriftNote(current, downloaded.etag) - if (note !== null) { - return persistRejection(asset, deps, note) - } - } - let uploaded = false - let applied: MediaWorkerAsset | null try { if (publishes && result.processedBytes) { - const sKey = servedKey(asset.r2Key) - const tKey = result.thumbnailBytes ? thumbnailKey(asset.r2Key) : null - await Promise.all([ - deps.storage.put(sKey, result.processedBytes, { - ...(result.processedContentType !== null - ? { contentType: result.processedContentType } - : {}), - }), - tKey && result.thumbnailBytes - ? deps.storage.put(tKey, result.thumbnailBytes, { - ...(result.thumbnailContentType !== null - ? { contentType: result.thumbnailContentType } - : {}), - }) - : Promise.resolve(), - ]) + const keys = await putProcessedObjects(asset, result.processedBytes, result, deps.storage) patch.byteSize = result.processedBytes.byteLength - patch.servedKey = sKey - if (tKey) patch.thumbKey = tKey + patch.servedKey = keys.servedKey + if (keys.thumbKey) patch.thumbKey = keys.thumbKey uploaded = true } @@ -261,60 +345,84 @@ async function processAsset( await deps.repo.insertAbuseFlag({ subjectId: asset.id, reason: flag.reason }) } - applied = await deps.repo.applyResult(asset.id, patch) + return { applied: await deps.repo.applyResult(asset.id, patch), uploaded } } catch (err) { - report(err, { job: "media.checks", phase: "persist", mediaId: asset.id }) - log("media.checks: persist infra failure, will retry", { mediaId: asset.id, err: String(err) }) - throw new MediaInfraError("persist", err) - } - - if (applied === null) { - return settleTerminalRace(asset, deps, { attempted: result.status, uploaded }) + throw retryableFailure(resolveJobObs(deps), err, { + phase: "persist", + line: "media.checks: persist infra failure, will retry", + ids: { mediaId: asset.id }, + }) } +} +async function afterApply( + asset: MediaWorkerAsset, + result: MediaProcessResult, + applied: MediaWorkerAsset, + uploaded: boolean, + deps: MediaChecksDeps, +): Promise { + const { log, report } = resolveJobObs(deps) if (uploaded) { await deleteSupersededUpload(asset, deps, log, report) } - if (result.status === "rejected") { logRejection(asset, log, report, result.note) await deleteRejectedObjects(asset, deps, log, report) - } else if (result.status === "held") { - log("media.checks: held", { - mediaId: asset.id, - note: result.note, - flags: result.flags.map((f) => f.reason), - }) - if (asset.reportId && deps.repo.enqueueHeldModerationItem) { - await deps.repo - .enqueueHeldModerationItem({ - reportId: asset.reportId, - reason: `NSFW model over threshold (${asset.kind})`, - kind: "image", - note: result.note ?? null, - }) - .catch((err: unknown) => - log("media.checks: moderation enqueue failed (non-fatal)", { err: String(err) }), - ) - } - } else { - log("media.checks: ready", { - mediaId: asset.id, - kind: asset.kind, - width: result.width, - height: result.height, - codec: result.codec, - exifGpsPresent: result.exifGps !== null, - }) - if (applied.servedKey && deps.publicMediaBase && deps.repo.refreshAvatarUrls) { - await deps.repo - .refreshAvatarUrls(asset.id, publicMediaUrl(deps.publicMediaBase, applied.servedKey)) - .catch((err: unknown) => - log("media.checks: avatar url refresh failed (non-fatal)", { err: String(err) }), - ) - } + return } - return result.status + if (result.status === "held") { + await onHeld(asset, result, deps, log) + return + } + await onReady(asset, result, applied, deps, log) +} + +async function onHeld( + asset: MediaWorkerAsset, + result: MediaProcessResult, + deps: MediaChecksDeps, + log: JobLogFn, +): Promise { + log("media.checks: held", { + mediaId: asset.id, + note: result.note, + flags: result.flags.map((f) => f.reason), + }) + if (!asset.reportId || !deps.repo.enqueueHeldModerationItem) return + await deps.repo + .enqueueHeldModerationItem({ + reportId: asset.reportId, + reason: `NSFW model over threshold (${asset.kind})`, + kind: "image", + note: result.note ?? null, + }) + .catch((err: unknown) => + log("media.checks: moderation enqueue failed (non-fatal)", { err: String(err) }), + ) +} + +async function onReady( + asset: MediaWorkerAsset, + result: MediaProcessResult, + applied: MediaWorkerAsset, + deps: MediaChecksDeps, + log: JobLogFn, +): Promise { + log("media.checks: ready", { + mediaId: asset.id, + kind: asset.kind, + width: result.width, + height: result.height, + codec: result.codec, + exifGpsPresent: result.exifGps !== null, + }) + if (!applied.servedKey || !deps.publicMediaBase || !deps.repo.refreshAvatarUrls) return + await deps.repo + .refreshAvatarUrls(asset.id, publicMediaUrl(deps.publicMediaBase, applied.servedKey)) + .catch((err: unknown) => + log("media.checks: avatar url refresh failed (non-fatal)", { err: String(err) }), + ) } async function settleTerminalRace( @@ -330,7 +438,7 @@ async function settleTerminalRace( current = await deps.repo.findById(asset.id) } catch (err) { reread = false - report(err, { job: "media.checks", phase: "terminal-race-reread", mediaId: asset.id }) + report(err, { job: MEDIA_CHECKS_JOB, phase: "terminal-race-reread", mediaId: asset.id }) } const winner = current?.status ?? null @@ -348,7 +456,7 @@ async function settleTerminalRace( new Error( "media.checks re-uploaded bytes for an already-terminal asset it could not re-read", ), - { job: "media.checks", phase: "terminal-race", mediaId: asset.id, r2Key: asset.r2Key }, + { job: MEDIA_CHECKS_JOB, phase: "terminal-race", mediaId: asset.id, r2Key: asset.r2Key }, ) } else if (winner === null || winner === "rejected") { await deleteRejectedObjects( @@ -383,30 +491,30 @@ function logRejection( ): void { log("media.checks: rejected", { mediaId: asset.id, kind: asset.kind, note }) report(new Error(note ?? "media rejected"), { - job: "media.checks", + job: MEDIA_CHECKS_JOB, mediaId: asset.id, kind: asset.kind, note, }) } -export async function persistRejection( +async function persistRejection( asset: MediaWorkerAsset, deps: MediaChecksDeps, note: string, ): Promise { - const { log, report } = resolveJobObs(deps) + const obs = resolveJobObs(deps) + const { log, report } = obs let applied: MediaWorkerAsset | null try { applied = await deps.repo.applyResult(asset.id, { status: "rejected" }) } catch (err) { - report(err, { job: "media.checks", phase: "reject", mediaId: asset.id }) - log("media.checks: failed to persist rejection, will retry", { - mediaId: asset.id, - note, - err: String(err), + throw retryableFailure(obs, err, { + phase: "reject", + line: "media.checks: failed to persist rejection, will retry", + ids: { mediaId: asset.id }, + logExtra: { note }, }) - throw new MediaInfraError("reject", err) } if (applied === null) { return settleTerminalRace(asset, deps, { attempted: "rejected", uploaded: false }) diff --git a/services/media-worker/src/jobs/media-keys.ts b/services/media-worker/src/jobs/media-keys.ts index 760db49d..9fabdb14 100644 --- a/services/media-worker/src/jobs/media-keys.ts +++ b/services/media-worker/src/jobs/media-keys.ts @@ -1,7 +1,11 @@ +const THUMBNAIL_PREFIX = "thumbs/" +const THUMBNAIL_SUFFIX = ".jpg" +const SERVED_PREFIX = "processed/" + export function thumbnailKey(r2Key: string): string { - return `thumbs/${r2Key}.jpg` + return `${THUMBNAIL_PREFIX}${r2Key}${THUMBNAIL_SUFFIX}` } export function servedKey(r2Key: string): string { - return `processed/${r2Key}` + return `${SERVED_PREFIX}${r2Key}` } diff --git a/services/media-worker/src/jobs/media-pipeline.ts b/services/media-worker/src/jobs/media-pipeline.ts index 30a3531e..142cda26 100644 --- a/services/media-worker/src/jobs/media-pipeline.ts +++ b/services/media-worker/src/jobs/media-pipeline.ts @@ -11,6 +11,11 @@ import { ScratchSetupError } from "../sandbox/tmp.js" import { probeBytes } from "../sandbox/ffprobe.js" import { grabFrameJpeg, remuxStripMetadata } from "../sandbox/ffmpeg-remux.js" +const MAX_NOTE_CHARS = 300 +const REMUXED_VIDEO_CONTENT_TYPE = "video/mp4" +const FRAME_GRAB_MAX_OFFSET_SEC = 1 +const NSFW_SCORE_DIGITS = 3 + export interface PipelineFlag { reason: WorkerAbuseReason } @@ -45,11 +50,11 @@ export interface ProcessDeps { // A decoder that could not start or a scratch dir the worker could not build says nothing about the // bytes: these escape so media.checks retries them as infrastructure instead of rejecting the upload. -export function isSandboxInfraFailure(err: unknown): err is SandboxSpawnError | ScratchSetupError { +function isSandboxInfraFailure(err: unknown): err is SandboxSpawnError | ScratchSetupError { return err instanceof SandboxSpawnError || err instanceof ScratchSetupError } -export function rejected(note: string): MediaProcessResult { +function rejected(note: string): MediaProcessResult { return { status: "rejected", width: null, @@ -68,7 +73,7 @@ export function rejected(note: string): MediaProcessResult { export function errNote(prefix: string, err: unknown): string { const msg = err instanceof Error ? err.message : String(err) - return `${prefix}: ${msg}`.slice(0, 300) + return `${prefix}: ${msg}`.slice(0, MAX_NOTE_CHARS) } function hasNsfwScorer(checks: unknown): boolean { @@ -99,7 +104,11 @@ async function applyAbuseSeams( } } if (scorerAvailable && nsfw >= deps.limits.nsfwHoldThreshold) { - return { status: "held", flags: [{ reason: "nsfw" }], note: `nsfw score ${nsfw.toFixed(3)}` } + return { + status: "held", + flags: [{ reason: "nsfw" }], + note: `nsfw score ${nsfw.toFixed(NSFW_SCORE_DIGITS)}`, + } } if (!scorerAvailable) { const unscored = "nsfw scorer not configured (no verdict)" @@ -161,8 +170,9 @@ async function processImageBytes( } } -function videoGeometryNote( +function videoLimitNote( probe: { + durationSec: number width: number | null height: number | null fps: number | null @@ -170,6 +180,9 @@ function videoGeometryNote( }, limits: WorkerLimits, ): string | null { + if (probe.durationSec <= 0 || probe.durationSec > limits.maxVideoDurationSec) { + return `duration ${probe.durationSec}s outside (0, ${limits.maxVideoDurationSec}]` + } const { width, height } = probe if (width === null || height === null || width <= 0 || height <= 0) { return "video reports no usable resolution" @@ -187,11 +200,45 @@ function videoGeometryNote( return null } +type VideoProbe = Awaited> + +// A frame that cannot be grabbed is no verdict on the video: without one it is held for review, not rejected. +async function grabPosterFrame( + bytes: Uint8Array, + durationSec: number, + limits: WorkerLimits, +): Promise { + try { + return await grabFrameJpeg(bytes, Math.min(FRAME_GRAB_MAX_OFFSET_SEC, durationSec / 2), limits) + } catch (err) { + if (isSandboxInfraFailure(err)) throw err + return null + } +} + +async function posterThumbnail( + frameJpeg: Buffer | null, + limits: WorkerLimits, +): Promise<{ thumbnailBytes: Buffer | null; thumbnailContentType: string | null }> { + const none = { thumbnailBytes: null, thumbnailContentType: null } + if (!frameJpeg) return none + try { + const thumb = await processImageLane(frameJpeg, limits) + return { + thumbnailBytes: thumb.thumbnailBytes, + thumbnailContentType: thumb.thumbnailContentType, + } + } catch (err) { + if (isSandboxInfraFailure(err)) throw err + return none + } +} + async function processVideoBytes( bytes: Uint8Array, deps: ProcessDeps, ): Promise { - let probe: Awaited> + let probe: VideoProbe try { probe = await probeBytes(bytes, deps.limits) } catch (err) { @@ -204,14 +251,9 @@ async function processVideoBytes( if (probe.codec === null || !ALLOWED_VIDEO_CODECS.has(probe.codec.toLowerCase())) { return rejected(`unsupported codec: ${probe.codec ?? "unknown"}`) } - if (probe.durationSec <= 0 || probe.durationSec > deps.limits.maxVideoDurationSec) { - return rejected( - `duration ${probe.durationSec}s outside (0, ${deps.limits.maxVideoDurationSec}]`, - ) - } - const geometryNote = videoGeometryNote(probe, deps.limits) - if (geometryNote !== null) { - return rejected(geometryNote) + const limitNote = videoLimitNote(probe, deps.limits) + if (limitNote !== null) { + return rejected(limitNote) } let remuxed: Buffer @@ -222,28 +264,8 @@ async function processVideoBytes( return rejected(errNote("remux failed", err)) } - let frameJpeg: Buffer | null = null - try { - const at = Math.min(1, probe.durationSec / 2) - frameJpeg = await grabFrameJpeg(bytes, at, deps.limits) - } catch (err) { - if (isSandboxInfraFailure(err)) throw err - frameJpeg = null - } - - let thumbnailBytes: Buffer | null = null - let thumbnailContentType: string | null = null - if (frameJpeg) { - try { - const thumb = await processImageLane(frameJpeg, deps.limits) - thumbnailBytes = thumb.thumbnailBytes - thumbnailContentType = thumb.thumbnailContentType - } catch (err) { - if (isSandboxInfraFailure(err)) throw err - thumbnailBytes = null - thumbnailContentType = null - } - } + const frameJpeg = await grabPosterFrame(bytes, probe.durationSec, deps.limits) + const thumbnail = await posterThumbnail(frameJpeg, deps.limits) const seam = frameJpeg ? await applyAbuseSeams(frameJpeg, null, deps) @@ -260,9 +282,9 @@ async function processVideoBytes( codec: probe.codec.toLowerCase(), phash: null, processedBytes: remuxed, - processedContentType: "video/mp4", - thumbnailBytes, - thumbnailContentType, + processedContentType: REMUXED_VIDEO_CONTENT_TYPE, + thumbnailBytes: thumbnail.thumbnailBytes, + thumbnailContentType: thumbnail.thumbnailContentType, exifGps: null, flags: seam.flags, note: seam.note, diff --git a/services/media-worker/src/jobs/orphan-sweep.ts b/services/media-worker/src/jobs/orphan-sweep.ts index cdf10f18..be290855 100644 --- a/services/media-worker/src/jobs/orphan-sweep.ts +++ b/services/media-worker/src/jobs/orphan-sweep.ts @@ -21,7 +21,10 @@ async function mapWithLimit( await Promise.all(runners) } +const ORPHAN_SWEEP = "orphan.sweep" const ORPHAN_CONCURRENCY = 8 +const MS_PER_SECOND = 1000 +const LEGACY_PROCESSED_SUFFIXES = [".img", ".mp4"] export const LEAK_RETRY_MAX_ATTEMPTS = 5 @@ -45,11 +48,11 @@ export interface OrphanSweepResult { } function derivedKeys(o: OrphanRow): string[] { + const served = servedKey(o.r2Key) const keys = [ o.r2Key, - servedKey(o.r2Key), - `processed/${o.r2Key}.img`, - `processed/${o.r2Key}.mp4`, + served, + ...LEGACY_PROCESSED_SUFFIXES.map((suffix) => `${served}${suffix}`), thumbnailKey(o.r2Key), ] if (o.servedKey && !keys.includes(o.servedKey)) keys.push(o.servedKey) @@ -82,7 +85,7 @@ export async function runOrphanSweep(deps: OrphanSweepDeps): Promise boundMeanwhile++, onError: (err, id) => { errors++ - report(err, { job: "orphan.sweep", phase: "delete", mediaId: id }) + report(err, { job: ORPHAN_SWEEP, phase: "delete", mediaId: id }) log("orphan.sweep: row failed", { mediaId: id, err: String(err) }) }, onLeak: (keys, id) => { @@ -92,7 +95,7 @@ export async function runOrphanSweep(deps: OrphanSweepDeps): Promise - log("orphan.sweep: tombstone bump failed", { key: row.r2Key, err: String(bumpErr) }), - ) - log("orphan.sweep: tombstoned object delete failed again", { - key: row.r2Key, - attempts, - err: String(err), - }) - if (attempts >= LEAK_RETRY_MAX_ATTEMPTS) { - report(new Error(`orphan.sweep gave up on a leaked R2 object after ${attempts} attempts`), { - job: "orphan.sweep", - phase: "leak-retry", - key: row.r2Key, - mediaId: row.mediaId, - }) - } + await recordLeakRetryFailure(row, err, repo, log, report) } }) return { retried, reclaimed, errors } } +async function recordLeakRetryFailure( + row: LeakedObjectRow, + err: unknown, + repo: MediaWorkerRepo, + log: JobLogFn, + report: JobReportFn, +): Promise { + const attempts = row.attempts + 1 + await repo + .recordLeakedObjects?.({ mediaId: row.mediaId, keys: [row.r2Key], error: String(err) }) + .catch((bumpErr: unknown) => + log("orphan.sweep: tombstone bump failed", { key: row.r2Key, err: String(bumpErr) }), + ) + log("orphan.sweep: tombstoned object delete failed again", { + key: row.r2Key, + attempts, + err: String(err), + }) + if (attempts >= LEAK_RETRY_MAX_ATTEMPTS) { + report(new Error(`orphan.sweep gave up on a leaked R2 object after ${attempts} attempts`), { + job: ORPHAN_SWEEP, + phase: "leak-retry", + key: row.r2Key, + mediaId: row.mediaId, + }) + } +} + let legacyServedKeyAdoptionDrained = false export function resetLegacyServedKeyAdoption(): void { @@ -195,7 +208,7 @@ async function adoptLegacyServedKeys( ): Promise { if (legacyServedKeyAdoptionDrained) return 0 const { now: clock } = resolveJobObs(deps) - const cutoff = new Date(clock().getTime() - R2_PUT_TTL_SEC * 1000) + const cutoff = new Date(clock().getTime() - R2_PUT_TTL_SEC * MS_PER_SECOND) try { const { adopted, remaining } = await deps.repo.adoptLegacyServedKeys( cutoff, @@ -210,7 +223,7 @@ async function adoptLegacyServedKeys( } return adopted } catch (err) { - report(err, { job: "orphan.sweep", phase: "adopt-legacy-served-keys" }) + report(err, { job: ORPHAN_SWEEP, phase: "adopt-legacy-served-keys" }) log("orphan.sweep: legacy served-key adoption failed", { err: String(err) }) return 0 } diff --git a/services/media-worker/src/jobs/partition-maintenance.ts b/services/media-worker/src/jobs/partition-maintenance.ts index 50a8c35b..f0afaae8 100644 --- a/services/media-worker/src/jobs/partition-maintenance.ts +++ b/services/media-worker/src/jobs/partition-maintenance.ts @@ -1,8 +1,12 @@ -import { ensureChatPartitionWindow, ensureDmPartitionWindow } from "@civfix/api/media-repo" +import { + ensureChatPartitionWindow, + ensureDmPartitionWindow, + PARTITION_MONTHS_AHEAD, +} from "@civfix/api/media-repo" import type { Sql } from "@civfix/api/db" import { resolveJobObs, type JobObsDeps } from "./obs.js" -export const PARTITION_MONTHS_AHEAD = 2 +const CHAT_PARTITION_MAINTENANCE = "chat.partition.maintenance" export interface PartitionMaintenanceDeps extends JobObsDeps { sql: Sql @@ -26,7 +30,7 @@ export async function runPartitionMaintenance( log("chat.partition.maintenance: ensured", { chat, dm }) return { chat, dm } } catch (err) { - report(err, { job: "chat.partition.maintenance" }) + report(err, { job: CHAT_PARTITION_MAINTENANCE }) log("chat.partition.maintenance: failed", { err: String(err) }) throw err } diff --git a/services/media-worker/src/jobs/reject-cleanup.ts b/services/media-worker/src/jobs/reject-cleanup.ts index 38de2cef..40ecf883 100644 --- a/services/media-worker/src/jobs/reject-cleanup.ts +++ b/services/media-worker/src/jobs/reject-cleanup.ts @@ -1,5 +1,5 @@ import type { Storage } from "@civfix/shared/interfaces" -import type { MediaWorkerRepo } from "@civfix/api/media-repo" +import { MEDIA_CHECKS_JOB, type MediaWorkerRepo } from "@civfix/api/media-repo" import type { JobLogFn, JobReportFn } from "./obs.js" import { servedKey, thumbnailKey } from "./media-keys.js" @@ -15,24 +15,35 @@ export interface RejectCleanupDeps { storage: Pick } -export async function deleteSupersededUpload( +// A failed reference check counts as "still referenced": deleting bytes another row may serve is the +// unrecoverable mistake, a leaked object is not. +async function keepSharedBytes( asset: { id: string; r2Key: string }, deps: RejectCleanupDeps, log: JobLogFn, report: JobReportFn, -): Promise { - let stillShared: boolean + failure: { phase: string; line: string }, +): Promise { try { - stillShared = await deps.repo.r2KeyReferencedByOthers(asset.id, asset.r2Key) + return await deps.repo.r2KeyReferencedByOthers(asset.id, asset.r2Key) } catch (err) { - report(err, { job: "media.checks", phase: "upload-cleanup", mediaId: asset.id }) - log("media.checks: superseded-upload reference check failed (bytes left in place)", { - mediaId: asset.id, - err: String(err), - }) - return + report(err, { job: MEDIA_CHECKS_JOB, phase: failure.phase, mediaId: asset.id }) + log(failure.line, { mediaId: asset.id, err: String(err) }) + return true } - if (stillShared) return +} + +export async function deleteSupersededUpload( + asset: { id: string; r2Key: string }, + deps: RejectCleanupDeps, + log: JobLogFn, + report: JobReportFn, +): Promise { + const keep = await keepSharedBytes(asset, deps, log, report, { + phase: "upload-cleanup", + line: "media.checks: superseded-upload reference check failed (bytes left in place)", + }) + if (keep) return try { await deps.storage.delete(asset.r2Key) @@ -59,7 +70,7 @@ export async function deleteSupersededUpload( }), ) report(new Error("media.checks leaked the superseded upload object (tombstoned for retry)"), { - job: "media.checks", + job: MEDIA_CHECKS_JOB, phase: "upload-cleanup", mediaId: asset.id, key: asset.r2Key, @@ -72,18 +83,11 @@ export async function deleteRejectedObjects( log: JobLogFn, report: JobReportFn, ): Promise { - let stillShared: boolean - try { - stillShared = await deps.repo.r2KeyReferencedByOthers(asset.id, asset.r2Key) - } catch (err) { - report(err, { job: "media.checks", phase: "reject-cleanup", mediaId: asset.id }) - log("media.checks: rejected-media reference check failed (bytes left in place)", { - mediaId: asset.id, - err: String(err), - }) - return - } - if (stillShared) return + const keep = await keepSharedBytes(asset, deps, log, report, { + phase: "reject-cleanup", + line: "media.checks: rejected-media reference check failed (bytes left in place)", + }) + if (keep) return const keys = [asset.r2Key, servedKey(asset.r2Key), thumbnailKey(asset.r2Key)] if (asset.servedKey && !keys.includes(asset.servedKey)) keys.push(asset.servedKey) @@ -121,6 +125,6 @@ export async function deleteRejectedObjects( new Error( `media.checks leaked ${leaked.length} rejected-media R2 object(s) (tombstoned for retry)`, ), - { job: "media.checks", phase: "reject-cleanup", mediaId: asset.id, keys: leaked }, + { job: MEDIA_CHECKS_JOB, phase: "reject-cleanup", mediaId: asset.id, keys: leaked }, ) } diff --git a/services/media-worker/src/jobs/retention-sweep.ts b/services/media-worker/src/jobs/retention-sweep.ts index e49b9491..8017b9de 100644 --- a/services/media-worker/src/jobs/retention-sweep.ts +++ b/services/media-worker/src/jobs/retention-sweep.ts @@ -34,13 +34,16 @@ export interface RetentionSweepResult { errors: number } -export const RETENTION_GRACE_MS = 60 * 60 * 1000 -export const RETENTION_BATCH = 5000 -export const RETENTION_IDEMPOTENCY_MS = 48 * 60 * 60 * 1000 -export const RETENTION_NOTIFICATIONS_MS = 90 * 24 * 60 * 60 * 1000 +const RETENTION_SWEEP = "retention.sweep" +const HOUR_MS = 60 * 60 * 1000 +const DAY_MS = 24 * HOUR_MS + +export const RETENTION_GRACE_MS = HOUR_MS +const RETENTION_BATCH = 5000 +const RETENTION_IDEMPOTENCY_MS = 48 * HOUR_MS +const RETENTION_NOTIFICATIONS_MS = 90 * DAY_MS export const RETENTION_GEOCODE_CACHE_MS = GEOCODE_CACHE_TTL_MS export const RETENTION_INBOUND_EMAILS_MS = INBOUND_EMAIL_RETENTION_MS -export const RETENTION_INBOUND_EMAILS_BATCH = INBOUND_EMAIL_RETENTION_BATCH export const RETENTION_MAX_PAGES = 20 export async function runRetentionSweep(deps: RetentionSweepDeps): Promise { @@ -84,7 +87,7 @@ export async function runRetentionSweep(deps: RetentionSweepDeps): Promise onDeleted(rows.length), { pageSize, maxPages }) } catch (err) { result.errors++ - report(err, { job: "retention.sweep", table }) + report(err, { job: RETENTION_SWEEP, table }) log(`retention.sweep: ${table} failed`, { err: String(err) }) } } @@ -222,20 +225,7 @@ export async function runInboundEmailRetentionLane( async (rows) => { const reaped: string[] = [] for (const row of rows) { - let objectsGone = true - for (const key of row.attachmentKeys) { - try { - await storage.delete(key) - } catch (err) { - objectsGone = false - result.inboundEmailObjectsLeaked += 1 - opts.report(err, { - job: "retention.sweep", - table: "inbound_emails", - phase: "attachment", - }) - } - } + const objectsGone = await deleteAttachments(row.attachmentKeys, storage, result, opts) if (objectsGone) reaped.push(row.id) } if (reaped.length === 0) { @@ -248,7 +238,30 @@ export async function runInboundEmailRetentionLane( ) } catch (err) { result.errors++ - opts.report(err, { job: "retention.sweep", table: "inbound_emails" }) + opts.report(err, { job: RETENTION_SWEEP, table: "inbound_emails" }) opts.log("retention.sweep: inbound_emails failed", { err: String(err) }) } } + +async function deleteAttachments( + keys: readonly string[], + storage: Pick, + result: Pick, + opts: Pick, +): Promise { + let objectsGone = true + for (const key of keys) { + try { + await storage.delete(key) + } catch (err) { + objectsGone = false + result.inboundEmailObjectsLeaked += 1 + opts.report(err, { + job: RETENTION_SWEEP, + table: "inbound_emails", + phase: "attachment", + }) + } + } + return objectsGone +} diff --git a/services/media-worker/src/jobs/stuck-sweep.ts b/services/media-worker/src/jobs/stuck-sweep.ts index 07be9049..676345e1 100644 --- a/services/media-worker/src/jobs/stuck-sweep.ts +++ b/services/media-worker/src/jobs/stuck-sweep.ts @@ -10,6 +10,8 @@ import { resolveJobObs, type JobObsDeps } from "./obs.js" import { deleteRejectedObjects } from "./reject-cleanup.js" import { MEDIA_UPLOAD_REAP_JOB, uploadReapDelaySec } from "./upload-reap.js" +const STUCK_SWEEP = "media.stuck.sweep" + export interface StuckSweepDeps extends JobObsDeps { repo: MediaWorkerRepo jobs: Pick @@ -33,7 +35,7 @@ export async function runStuckSweep(deps: StuckSweepDeps): Promise maxAttempts) { - try { - const rejected = await deps.repo.terminalizeStuck(row.id) - if (rejected === null) { - log("media.stuck.sweep: give-up skipped, media already terminal", { - mediaId: row.id, - uploadId: row.uploadId, - }) - continue - } - terminalized++ - log("media.stuck.sweep: gave up, media rejected", { - mediaId: row.id, - uploadId: row.uploadId, - checkCount: row.checkCount, - maxAttempts, - }) - await deleteRejectedObjects(rejected, deps, log, report) - await deps.jobs - .enqueue( - MEDIA_UPLOAD_REAP_JOB, - { mediaId: row.id, uploadId: row.uploadId, r2Key: row.r2Key }, - { singletonKey: row.uploadId, startAfter: uploadReapDelaySec() }, - ) - .catch((err: unknown) => - log("media.stuck.sweep: failed to schedule media.upload.reap (non-fatal)", { - mediaId: row.id, - err: String(err), - }), - ) - } catch (err) { - errors++ - report(err, { job: "media.stuck.sweep", phase: "terminalize", mediaId: row.id }) - log("media.stuck.sweep: terminalize failed", { mediaId: row.id, err: String(err) }) - } - continue - } - - try { - await deps.jobs.enqueue( - MEDIA_CHECKS_JOB, - { - mediaId: row.id, - uploadId: row.uploadId, - r2Key: row.r2Key, - kind: row.kind, - uploadEtag: row.uploadEtag, - } satisfies MediaChecksJob, - { singletonKey: row.uploadId }, - ) - requeued++ - } catch (err) { - errors++ - report(err, { job: "media.stuck.sweep", phase: "requeue", mediaId: row.id }) - log("media.stuck.sweep: re-enqueue failed", { mediaId: row.id, err: String(err) }) - } + const failed = + row.checkCount > maxAttempts + ? await giveUpOnStuck(row, maxAttempts, deps, () => terminalized++) + : await requeueStuck(row, deps, () => requeued++) + if (failed) errors++ } log("media.stuck.sweep: done", { @@ -109,3 +60,74 @@ export async function runStuckSweep(deps: StuckSweepDeps): Promise void, +): Promise { + const { log, report } = resolveJobObs(deps) + try { + const rejected = await deps.repo.terminalizeStuck(row.id) + if (rejected === null) { + log("media.stuck.sweep: give-up skipped, media already terminal", { + mediaId: row.id, + uploadId: row.uploadId, + }) + return false + } + onTerminalized() + log("media.stuck.sweep: gave up, media rejected", { + mediaId: row.id, + uploadId: row.uploadId, + checkCount: row.checkCount, + maxAttempts, + }) + await deleteRejectedObjects(rejected, deps, log, report) + await deps.jobs + .enqueue( + MEDIA_UPLOAD_REAP_JOB, + { mediaId: row.id, uploadId: row.uploadId, r2Key: row.r2Key }, + { singletonKey: row.uploadId, startAfter: uploadReapDelaySec() }, + ) + .catch((err: unknown) => + log("media.stuck.sweep: failed to schedule media.upload.reap (non-fatal)", { + mediaId: row.id, + err: String(err), + }), + ) + return false + } catch (err) { + report(err, { job: STUCK_SWEEP, phase: "terminalize", mediaId: row.id }) + log("media.stuck.sweep: terminalize failed", { mediaId: row.id, err: String(err) }) + return true + } +} + +async function requeueStuck( + row: StuckMediaRow, + deps: StuckSweepDeps, + onRequeued: () => void, +): Promise { + try { + await deps.jobs.enqueue( + MEDIA_CHECKS_JOB, + { + mediaId: row.id, + uploadId: row.uploadId, + r2Key: row.r2Key, + kind: row.kind, + uploadEtag: row.uploadEtag, + } satisfies MediaChecksJob, + { singletonKey: row.uploadId }, + ) + onRequeued() + return false + } catch (err) { + const { log, report } = resolveJobObs(deps) + report(err, { job: STUCK_SWEEP, phase: "requeue", mediaId: row.id }) + log("media.stuck.sweep: re-enqueue failed", { mediaId: row.id, err: String(err) }) + return true + } +} diff --git a/services/media-worker/src/jobs/upload-reap.ts b/services/media-worker/src/jobs/upload-reap.ts index f474229f..7fefffe3 100644 --- a/services/media-worker/src/jobs/upload-reap.ts +++ b/services/media-worker/src/jobs/upload-reap.ts @@ -5,7 +5,7 @@ import { resolveJobObs, type JobObsDeps } from "./obs.js" export const MEDIA_UPLOAD_REAP_JOB = "media.upload.reap" -export const UPLOAD_REAP_SLACK_SEC = 5 * 60 +const UPLOAD_REAP_SLACK_SEC = 5 * 60 export function uploadReapDelaySec(): number { return R2_PUT_TTL_SEC + UPLOAD_REAP_SLACK_SEC diff --git a/services/media-worker/src/sandbox/exec.ts b/services/media-worker/src/sandbox/exec.ts index 2bab12e2..edf7b409 100644 --- a/services/media-worker/src/sandbox/exec.ts +++ b/services/media-worker/src/sandbox/exec.ts @@ -1,7 +1,15 @@ import { dirname } from "node:path" import { tmpdir } from "node:os" import { execa, type Options as ExecaOptions } from "execa" -import { CHILD_KILL_SIGNAL, loadSandboxIdentity, type SandboxIdentity } from "../config.js" +import { + CHILD_KILL_SIGNAL, + loadSandboxIdentity, + parseBool, + type SandboxIdentity, +} from "../config.js" + +const STDERR_TAIL_CHARS = 800 +const SANDBOX_LOCALE = "C" export interface RunResult { stdout: string @@ -121,13 +129,8 @@ export function dropBoundingSet(source?: NodeJS.ProcessEnv): boolean { return cachedDropBounding } -export function resetDropBoundingSet(): void { - cachedDropBounding = undefined -} - -export function parseDropBounding(source: NodeJS.ProcessEnv): boolean { - const raw = (source.MEDIA_SANDBOX_DROP_BOUNDING ?? "").trim().toLowerCase() - return raw === "1" || raw === "true" || raw === "yes" || raw === "on" +function parseDropBounding(source: NodeJS.ProcessEnv): boolean { + return parseBool(source.MEDIA_SANDBOX_DROP_BOUNDING, false) } export function sandboxEnv(binaryPath: string, cwd: string): Record { @@ -135,7 +138,7 @@ export function sandboxEnv(binaryPath: string, cwd: string): Record> + let result: ToolResult const subprocess = execa(spawned.command, spawned.argv, execaOpts) const leaderPid = typeof subprocess.pid === "number" ? subprocess.pid : undefined const cleanup = trackSandboxChild(leaderPid) @@ -212,37 +215,9 @@ export async function runTool( cleanup() } - const stderrText = - typeof result.stderr === "string" - ? result.stderr - : Buffer.isBuffer(result.stderr) - ? result.stderr.toString("utf8") - : "" - + const stderrText = outputText(result.stderr) if (result.failed || result.timedOut || (result.exitCode ?? 1) !== 0) { - const failure = result as { - signal?: string - isTerminated?: boolean - isMaxBuffer?: boolean - cause?: unknown - } - const signal = typeof failure.signal === "string" ? failure.signal : null - const ranAndDied = - signal !== null || failure.isTerminated === true || failure.isMaxBuffer === true - const neverRan = - !result.timedOut && - !ranAndDied && - typeof result.exitCode !== "number" && - (leaderPid === undefined || isSpawnSyscallFailure(failure.cause)) - if (neverRan) { - throw new SandboxSpawnError(name, failure.cause ?? tail(stderrText)) - } - throw new SandboxToolError(name, { - timedOut: Boolean(result.timedOut), - exitCode: typeof result.exitCode === "number" ? result.exitCode : null, - signal, - stderrTail: tail(stderrText), - }) + throw toolFailure(name, result, leaderPid, stderrText) } const stdoutBuffer = Buffer.isBuffer(result.stdout) @@ -258,3 +233,43 @@ export async function runTool( exitCode: result.exitCode ?? 0, } } + +type ToolResult = Awaited> + +function outputText(output: ToolResult["stderr"]): string { + if (typeof output === "string") return output + return Buffer.isBuffer(output) ? output.toString("utf8") : "" +} + +// A tool that never started says nothing about the bytes (infra, retried); one that ran and died is a +// verdict on them. With no exit code, a missing pid or a spawn-syscall failure counts as "never started". +function toolFailure( + name: string, + result: ToolResult, + leaderPid: number | undefined, + stderrText: string, +): SandboxSpawnError | SandboxToolError { + const failure = result as { + signal?: string + isTerminated?: boolean + isMaxBuffer?: boolean + cause?: unknown + } + const signal = typeof failure.signal === "string" ? failure.signal : null + const ranAndDied = + signal !== null || failure.isTerminated === true || failure.isMaxBuffer === true + const neverRan = + !result.timedOut && + !ranAndDied && + typeof result.exitCode !== "number" && + (leaderPid === undefined || isSpawnSyscallFailure(failure.cause)) + if (neverRan) { + return new SandboxSpawnError(name, failure.cause ?? tail(stderrText)) + } + return new SandboxToolError(name, { + timedOut: Boolean(result.timedOut), + exitCode: typeof result.exitCode === "number" ? result.exitCode : null, + signal, + stderrTail: tail(stderrText), + }) +} diff --git a/services/media-worker/src/sandbox/ffmpeg-remux.ts b/services/media-worker/src/sandbox/ffmpeg-remux.ts index c6a8caee..61681aac 100644 --- a/services/media-worker/src/sandbox/ffmpeg-remux.ts +++ b/services/media-worker/src/sandbox/ffmpeg-remux.ts @@ -3,47 +3,23 @@ import { mediaToolPath } from "./binaries.js" import { makeScratch, readScratchOutput } from "./tmp.js" import type { WorkerLimits } from "../config.js" +const QUIET_ARGS = ["-hide_banner", "-loglevel", "error", "-nostdin"] as const const SAFE_INPUT_ARGS = ["-protocol_whitelist", "file", "-f", "mov"] as const const REMUX_OUTPUT = "out.mp4" const FRAME_OUTPUT = "frame.jpg" +const SEEK_DECIMALS = 3 -export async function remuxStripMetadata(bytes: Uint8Array, limits: WorkerLimits): Promise { +async function runFfmpegToScratchOutput( + bytes: Uint8Array, + outputName: string, + limits: WorkerLimits, + buildArgs: (inputPath: string, outPath: string) => string[], +): Promise { const scratch = await makeScratch(bytes, "bin") - const out = scratch.outPath(REMUX_OUTPUT) + const out = scratch.outPath(outputName) try { - const args = [ - "-hide_banner", - "-loglevel", - "error", - "-nostdin", - ...SAFE_INPUT_ARGS, - "-i", - scratch.inputPath, - "-map_metadata", - "-1", - "-map_metadata:s", - "-1", - "-map_chapters", - "-1", - "-map", - "0:v:0", - "-map", - "0:a?", - "-c", - "copy", - "-metadata", - "location=", - "-metadata", - "location-eng=", - "-movflags", - "+faststart", - "-f", - "mp4", - "-y", - out, - ] - await runTool("ffmpeg", await mediaToolPath("ffmpeg"), args, { + await runTool("ffmpeg", await mediaToolPath("ffmpeg"), buildArgs(scratch.inputPath, out), { timeoutMs: limits.ffmpegTimeoutMs, maxStdoutBytes: limits.maxToolStdoutBytes, cwd: scratch.dir, @@ -51,7 +27,7 @@ export async function remuxStripMetadata(bytes: Uint8Array, limits: WorkerLimits await scratch.seal() return await readScratchOutput( scratch.dir, - REMUX_OUTPUT, + outputName, sandboxIdentity()?.uid ?? null, limits.maxChildOutputBytes, ) @@ -60,52 +36,62 @@ export async function remuxStripMetadata(bytes: Uint8Array, limits: WorkerLimits } } +export async function remuxStripMetadata(bytes: Uint8Array, limits: WorkerLimits): Promise { + return runFfmpegToScratchOutput(bytes, REMUX_OUTPUT, limits, (inputPath, out) => [ + ...QUIET_ARGS, + ...SAFE_INPUT_ARGS, + "-i", + inputPath, + "-map_metadata", + "-1", + "-map_metadata:s", + "-1", + "-map_chapters", + "-1", + "-map", + "0:v:0", + "-map", + "0:a?", + "-c", + "copy", + "-metadata", + "location=", + "-metadata", + "location-eng=", + "-movflags", + "+faststart", + "-f", + "mp4", + "-y", + out, + ]) +} + export async function grabFrameJpeg( bytes: Uint8Array, atSec: number, limits: WorkerLimits, ): Promise { - const scratch = await makeScratch(bytes, "bin") - const out = scratch.outPath(FRAME_OUTPUT) - try { - const seek = Number.isFinite(atSec) && atSec > 0 ? atSec.toFixed(3) : "0" - const args = [ - "-hide_banner", - "-loglevel", - "error", - "-nostdin", - ...SAFE_INPUT_ARGS, - "-max_pixels", - String(limits.maxVideoPixels), - "-threads", - "1", - "-ss", - seek, - "-i", - scratch.inputPath, - "-frames:v", - "1", - "-an", - "-map_metadata", - "-1", - "-f", - "image2", - "-y", - out, - ] - await runTool("ffmpeg", await mediaToolPath("ffmpeg"), args, { - timeoutMs: limits.ffmpegTimeoutMs, - maxStdoutBytes: limits.maxToolStdoutBytes, - cwd: scratch.dir, - }) - await scratch.seal() - return await readScratchOutput( - scratch.dir, - FRAME_OUTPUT, - sandboxIdentity()?.uid ?? null, - limits.maxChildOutputBytes, - ) - } finally { - await scratch.cleanup() - } + const seek = Number.isFinite(atSec) && atSec > 0 ? atSec.toFixed(SEEK_DECIMALS) : "0" + return runFfmpegToScratchOutput(bytes, FRAME_OUTPUT, limits, (inputPath, out) => [ + ...QUIET_ARGS, + ...SAFE_INPUT_ARGS, + "-max_pixels", + String(limits.maxVideoPixels), + "-threads", + "1", + "-ss", + seek, + "-i", + inputPath, + "-frames:v", + "1", + "-an", + "-map_metadata", + "-1", + "-f", + "image2", + "-y", + out, + ]) } diff --git a/services/media-worker/src/sandbox/ffprobe.ts b/services/media-worker/src/sandbox/ffprobe.ts index 04e292dc..ff489ca9 100644 --- a/services/media-worker/src/sandbox/ffprobe.ts +++ b/services/media-worker/src/sandbox/ffprobe.ts @@ -92,34 +92,35 @@ export async function probeBytes(bytes: Uint8Array, limits: WorkerLimits): Promi cwd: scratch.dir, }) - let parsed: FfprobeJson - try { - parsed = JSON.parse(res.stdout) as FfprobeJson - } catch (err) { - throw new SandboxToolError("ffprobe", { - timedOut: false, - exitCode: 0, - stderrTail: "unparseable ffprobe json", - cause: err, - }) - } - - const streams = parsed.streams ?? [] - const video = streams.find((s) => s.codec_type === "video") - const isVideo = video !== undefined - - const durationSec = num(parsed.format?.duration) || num(video?.duration) - - return { - durationSec, - codec: video?.codec_name ?? null, - width: typeof video?.width === "number" ? video.width : null, - height: typeof video?.height === "number" ? video.height : null, - fps: frameRate(video?.avg_frame_rate, video?.r_frame_rate), - bitrateBps: bitrate(video?.bit_rate, parsed.format?.bit_rate), - isVideo, - } + return toProbeResult(parseProbeJson(res.stdout)) } finally { await scratch.cleanup() } } + +function parseProbeJson(stdout: string): FfprobeJson { + try { + return JSON.parse(stdout) as FfprobeJson + } catch (err) { + throw new SandboxToolError("ffprobe", { + timedOut: false, + exitCode: 0, + stderrTail: "unparseable ffprobe json", + cause: err, + }) + } +} + +function toProbeResult(parsed: FfprobeJson): ProbeResult { + const streams = parsed.streams ?? [] + const video = streams.find((s) => s.codec_type === "video") + return { + durationSec: num(parsed.format?.duration) || num(video?.duration), + codec: video?.codec_name ?? null, + width: typeof video?.width === "number" ? video.width : null, + height: typeof video?.height === "number" ? video.height : null, + fps: frameRate(video?.avg_frame_rate, video?.r_frame_rate), + bitrateBps: bitrate(video?.bit_rate, parsed.format?.bit_rate), + isVideo: video !== undefined, + } +} diff --git a/services/media-worker/src/sandbox/image-lane-main.ts b/services/media-worker/src/sandbox/image-lane-main.ts index f430204d..76296b02 100644 --- a/services/media-worker/src/sandbox/image-lane-main.ts +++ b/services/media-worker/src/sandbox/image-lane-main.ts @@ -2,7 +2,7 @@ import { readFile, writeFile } from "node:fs/promises" import { join } from "node:path" import type { WorkerLimits } from "../config.js" import { processImage } from "./image.js" -import { perceptualHash } from "./phash.js" +import { bestEffortPerceptualHash } from "./phash.js" export const RUN_FLAG = "--civfix-image-lane" @@ -31,7 +31,7 @@ export interface ImageLaneFailure { export type ImageLaneResponse = ImageLaneSuccess | ImageLaneFailure -export function requestArg(argv: string[]): string | undefined { +function requestArg(argv: string[]): string | undefined { const at = argv.indexOf(RUN_FLAG) return at === -1 ? argv[2] : argv[at + 1] } @@ -64,13 +64,7 @@ export async function runImageLane(req: ImageLaneRequest): Promise m.width * m.height <= limits.maxImagePixels, "pixel budget exceeded"), @@ -40,7 +47,7 @@ function envelopeSchema(limits: WorkerLimits) { phash: z.string().regex(PHASH_RE).nullable(), }) .strict(), - z.object({ ok: z.literal(false), error: z.string().max(500) }).strict(), + z.object({ ok: z.literal(false), error: z.string().max(MAX_LANE_ERROR_CHARS) }).strict(), ]) } @@ -65,30 +72,18 @@ export async function processImageLane( const entry = imageLaneEntry() if (!existsSync(entry)) { throw new SandboxSpawnError( - "image-lane", + IMAGE_LANE_TOOL, new Error(`the image lane entry ${entry} does not exist (set MEDIA_IMAGE_LANE_ENTRY)`), ) } - const res = await runTool("image-lane", process.execPath, [entry, RUN_FLAG, request], { + const res = await runTool(IMAGE_LANE_TOOL, process.execPath, [entry, RUN_FLAG, request], { timeoutMs: imageLaneTimeoutMs(limits), maxStdoutBytes: limits.maxToolStdoutBytes, cwd: scratch.dir, }) await scratch.seal() - let raw: unknown - try { - raw = JSON.parse(res.stdout) - } catch (err) { - throw new ImageProcessingError("image lane produced no parseable result", err) - } - const parsed = envelopeSchema(limits).safeParse(raw) - if (!parsed.success) { - throw new ImageProcessingError( - `image lane returned an invalid result: ${parsed.error.message}`, - ) - } - if (!parsed.data.ok) throw new ImageProcessingError(parsed.data.error) + const envelope = parseLaneEnvelope(res.stdout, limits) const [strippedBytes, thumbnailBytes] = await Promise.all([ readScratchOutput(scratch.dir, STRIPPED_FILE, identity.uid, limits.maxChildOutputBytes), @@ -96,27 +91,37 @@ export async function processImageLane( ]) return { - meta: parsed.data.meta, + meta: envelope.meta, strippedBytes, - strippedContentType: parsed.data.strippedContentType, + strippedContentType: envelope.strippedContentType, thumbnailBytes, - thumbnailContentType: parsed.data.thumbnailContentType, - exifGps: parsed.data.exifGps, - phash: parsed.data.phash, + thumbnailContentType: envelope.thumbnailContentType, + exifGps: envelope.exifGps, + phash: envelope.phash, } } finally { await scratch.cleanup() } } -async function processInProcess(bytes: Uint8Array, limits: WorkerLimits): Promise { - const processed = await processImage(bytes, limits) - let phash: string | null = null +// The child's stdout is untrusted output of a process that decoded hostile bytes: anything that is not a +// schema-valid success envelope becomes an ImageProcessingError (a rejection, never an infra retry). +function parseLaneEnvelope(stdout: string, limits: WorkerLimits) { + let raw: unknown try { - phash = await perceptualHash(bytes, limits) - } catch { - // The hash only feeds the non-blocking near-duplicate note; an image that decoded above still ships. - phash = null + raw = JSON.parse(stdout) + } catch (err) { + throw new ImageProcessingError("image lane produced no parseable result", err) } - return { ...processed, phash } + const parsed = envelopeSchema(limits).safeParse(raw) + if (!parsed.success) { + throw new ImageProcessingError(`image lane returned an invalid result: ${parsed.error.message}`) + } + if (!parsed.data.ok) throw new ImageProcessingError(parsed.data.error) + return parsed.data +} + +async function processInProcess(bytes: Uint8Array, limits: WorkerLimits): Promise { + const processed = await processImage(bytes, limits) + return { ...processed, phash: await bestEffortPerceptualHash(bytes, limits) } } diff --git a/services/media-worker/src/sandbox/image.ts b/services/media-worker/src/sandbox/image.ts index 8bfbd84a..d66c7d20 100644 --- a/services/media-worker/src/sandbox/image.ts +++ b/services/media-worker/src/sandbox/image.ts @@ -30,6 +30,22 @@ import { settleWithin } from "../timeout.js" sharp.cache(false) sharp.concurrency(1) +export const ALLOWED_IMAGE_FORMATS = ["jpeg", "png", "webp"] as const +export type AllowedImageFormat = (typeof ALLOWED_IMAGE_FORMATS)[number] + +const PNG_COMPRESSION_LEVEL = 9 +const STRIPPED_QUALITY = 90 +const THUMB_QUALITY = 80 +const THUMB_CONTENT_TYPE = "image/jpeg" +const MS_PER_SECOND = 1000 +const MIN_SHARP_TIMEOUT_SEC = 1 + +const JPEG_MAGIC = [0xff, 0xd8, 0xff] +const PNG_MAGIC = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a] +const RIFF_MAGIC = [0x52, 0x49, 0x46, 0x46] +const WEBP_FORM_TYPE = [0x57, 0x45, 0x42, 0x50] +const WEBP_FORM_TYPE_OFFSET = 8 + export interface ExifGps { lat: number lng: number @@ -73,7 +89,7 @@ function withTimeout(p: Promise, ms: number, label: string): Promise { /** A spoofed-MIME input that libvips detects as anything else (SVG, TIFF, AVIF, GIF, ...) is rejected * before any full decode rather than re-encoded via an unintended codec. */ -const ALLOWED_DECODED_FORMATS: ReadonlySet = new Set(["jpeg", "png", "webp"]) +const ALLOWED_DECODED_FORMATS: ReadonlySet = new Set(ALLOWED_IMAGE_FORMATS) /** * The ALLOWED_DECODED_FORMATS check runs on `meta.format`, i.e. AFTER `metadata()`, and `metadata()` @@ -92,39 +108,25 @@ const ALLOWED_DECODED_FORMATS: ReadonlySet = new Set(["jpeg", "png", "we * PNG 89 50 4E 47 0D 0A 1A 0A * WebP "RIFF" .... "WEBP" (RIFF container, WEBP form type at offset 8) */ -export function sniffAllowedImageContainer(bytes: Uint8Array): "jpeg" | "png" | "webp" | null { - if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { - return "jpeg" - } - if ( - bytes.length >= 8 && - bytes[0] === 0x89 && - bytes[1] === 0x50 && - bytes[2] === 0x4e && - bytes[3] === 0x47 && - bytes[4] === 0x0d && - bytes[5] === 0x0a && - bytes[6] === 0x1a && - bytes[7] === 0x0a - ) { - return "png" - } +export function sniffAllowedImageContainer(bytes: Uint8Array): AllowedImageFormat | null { + if (hasBytesAt(bytes, 0, JPEG_MAGIC)) return "jpeg" + if (hasBytesAt(bytes, 0, PNG_MAGIC)) return "png" if ( - bytes.length >= 12 && - bytes[0] === 0x52 && - bytes[1] === 0x49 && - bytes[2] === 0x46 && - bytes[3] === 0x46 && - bytes[8] === 0x57 && - bytes[9] === 0x45 && - bytes[10] === 0x42 && - bytes[11] === 0x50 + hasBytesAt(bytes, 0, RIFF_MAGIC) && + hasBytesAt(bytes, WEBP_FORM_TYPE_OFFSET, WEBP_FORM_TYPE) ) { return "webp" } return null } +function hasBytesAt(bytes: Uint8Array, offset: number, signature: readonly number[]): boolean { + return ( + bytes.length >= offset + signature.length && + signature.every((byte, i) => bytes[offset + i] === byte) + ) +} + /** * EVERY untrusted-decode entry point must go through this, not a bare `sharp()`: it is where the container * sniff, pixel ceiling, failOn, single-page and sequential-read guards live as ONE set. phash.ts once kept @@ -147,7 +149,9 @@ export function guardedSharp(bytes: Uint8Array, limits: WorkerLimits): Sharp { // concurrent job under adversarial large-but-legal uploads). sequentialRead: true, // Seconds, rounded up so a sub-second config still yields >= 1s. - }).timeout({ seconds: Math.max(1, Math.ceil(limits.imageTimeoutMs / 1000)) }) + }).timeout({ + seconds: Math.max(MIN_SHARP_TIMEOUT_SEC, Math.ceil(limits.imageTimeoutMs / MS_PER_SECOND)), + }) } function chooseOutput(format: string): { @@ -156,11 +160,17 @@ function chooseOutput(format: string): { } { switch (format) { case "png": - return { apply: (s) => s.png({ compressionLevel: 9 }), contentType: "image/png" } + return { + apply: (s) => s.png({ compressionLevel: PNG_COMPRESSION_LEVEL }), + contentType: "image/png", + } case "webp": - return { apply: (s) => s.webp({ quality: 90 }), contentType: "image/webp" } + return { apply: (s) => s.webp({ quality: STRIPPED_QUALITY }), contentType: "image/webp" } default: - return { apply: (s) => s.jpeg({ quality: 90, mozjpeg: false }), contentType: "image/jpeg" } + return { + apply: (s) => s.jpeg({ quality: STRIPPED_QUALITY, mozjpeg: false }), + contentType: "image/jpeg", + } } } @@ -193,6 +203,20 @@ export async function processImage( bytes: Uint8Array, limits: WorkerLimits, ): Promise { + const meta = await readGuardedMeta(bytes, limits) + const exifGps = await readExifGps(bytes) + const encoded = await encodeStrippedOutputs(bytes, meta.format, limits) + return { + meta, + strippedBytes: encoded.strippedBytes, + strippedContentType: encoded.strippedContentType, + thumbnailBytes: encoded.thumbnailBytes, + thumbnailContentType: THUMB_CONTENT_TYPE, + exifGps, + } +} + +async function readGuardedMeta(bytes: Uint8Array, limits: WorkerLimits): Promise { // metadata() parses the header and enforces limitInputPixels; a bomb or garbage throws here before any // full decode. const meta = await withTimeout( @@ -216,15 +240,20 @@ export async function processImage( `image ${meta.width}x${meta.height}x${meta.pages ?? 1} exceeds pixel budget ${limits.maxImagePixels}`, ) } + return { width: meta.width, height: meta.height, format: meta.format } +} - const exifGps = await readExifGps(bytes) - +async function encodeStrippedOutputs( + bytes: Uint8Array, + format: string, + limits: WorkerLimits, +): Promise<{ strippedBytes: Buffer; strippedContentType: string; thumbnailBytes: Buffer }> { // Stripped full image AND thumbnail from a SINGLE decode: a fresh guardedSharp() per output would run // an independent full libvips decode (roughly doubling per-image CPU + peak surface on this CPU-bound // worker). Instead build one guarded pipeline, bake the EXIF orientation into the pixels once // (`.rotate()` with no args), then `.clone()` per output so sharp shares the one decoded surface. // Neither output calls withMetadata(), so all EXIF/XMP/ICC metadata is dropped on re-encode. - const out = chooseOutput(meta.format) + const out = chooseOutput(format) const base = guardedSharp(bytes, limits).rotate() const [strippedBytes, thumbnailBytes] = await Promise.all([ withTimeout(out.apply(base.clone()).toBuffer(), limits.imageTimeoutMs, "strip"), @@ -237,24 +266,16 @@ export async function processImage( fit: "inside", withoutEnlargement: true, }) - .jpeg({ quality: 80 }) + .jpeg({ quality: THUMB_QUALITY }) .toBuffer(), limits.imageTimeoutMs, "thumbnail", ), ]) - - return { - meta: { width: meta.width, height: meta.height, format: meta.format }, - strippedBytes, - strippedContentType: out.contentType, - thumbnailBytes, - thumbnailContentType: "image/jpeg", - exifGps, - } + return { strippedBytes, strippedContentType: out.contentType, thumbnailBytes } } -/** Proves the strip worked; used by tests and as a cheap post-strip self-check. */ +/** Proves the strip worked: true when no GPS survives in the re-encoded bytes. */ export async function hasNoGps(bytes: Uint8Array): Promise { return (await readExifGps(bytes)) === null } diff --git a/services/media-worker/src/sandbox/phash.ts b/services/media-worker/src/sandbox/phash.ts index 79720091..6a5e2f70 100644 --- a/services/media-worker/src/sandbox/phash.ts +++ b/services/media-worker/src/sandbox/phash.ts @@ -59,3 +59,16 @@ export async function perceptualHash(bytes: Uint8Array, limits: WorkerLimits): P } return hex } + +// The hash only feeds the non-blocking near-duplicate note, so an image that already decoded still ships +// without one. +export async function bestEffortPerceptualHash( + bytes: Uint8Array, + limits: WorkerLimits, +): Promise { + try { + return await perceptualHash(bytes, limits) + } catch { + return null + } +} diff --git a/services/media-worker/src/sandbox/preflight.ts b/services/media-worker/src/sandbox/preflight.ts index 1e70d8b0..993df852 100644 --- a/services/media-worker/src/sandbox/preflight.ts +++ b/services/media-worker/src/sandbox/preflight.ts @@ -16,7 +16,15 @@ const ONE_PIXEL_PNG = Buffer.from( const STATUS_READER = "/bin/cat" const PROC_STATUS = "/proc/self/status" -const ALLOWED_BOUNDING_MASK = (1n << 7n) | (1n << 6n) +const PREFLIGHT_MAX_STDOUT_BYTES = 64 * 1024 +const CAP_SETGID_BIT = 6n +const CAP_SETUID_BIT = 7n +const ALLOWED_BOUNDING_MASK = (1n << CAP_SETUID_BIT) | (1n << CAP_SETGID_BIT) +// Real, effective, saved-set and filesystem ids, in that order. +const PROC_STATUS_ID_COUNT = 4 +const CLEARED_CAP_FIELDS = ["CapInh", "CapPrm", "CapEff", "CapAmb"] as const +const CAP_FIELDS = [...CLEARED_CAP_FIELDS, "CapBnd"] as const +const HEX_DIGITS = /^[0-9a-f]+$/i export type PreflightLog = (msg: string, fields?: Record) => void @@ -27,21 +35,23 @@ export interface SandboxProof { } export function parseProcStatus(text: string): SandboxProof { + const lines = text.split("\n") + const field = (label: string): string | undefined => + lines.find((l) => l.startsWith(`${label}:`))?.slice(label.length + 1) const ids = (label: string): number[] => { - const line = text.split("\n").find((l) => l.startsWith(`${label}:`)) - if (!line) return [] - return line - .slice(label.length + 1) + const value = field(label) + if (value === undefined) return [] + return value .trim() .split(/\s+/) .map((v) => Number.parseInt(v, 10)) } const caps: Record = {} - for (const name of ["CapInh", "CapPrm", "CapEff", "CapAmb", "CapBnd"]) { - const line = text.split("\n").find((l) => l.startsWith(`${name}:`)) - if (line === undefined) continue - const hex = line.slice(name.length + 1).trim() - caps[name] = /^[0-9a-f]+$/i.test(hex) ? BigInt(`0x${hex}`) : -1n + for (const name of CAP_FIELDS) { + const value = field(name) + if (value === undefined) continue + const hex = value.trim() + caps[name] = HEX_DIGITS.test(hex) ? BigInt(`0x${hex}`) : -1n } return { uid: ids("Uid"), gid: ids("Gid"), caps } } @@ -58,33 +68,49 @@ export function sandboxProofFailure( context: SandboxProofContext, ): string | null { const proof = parseProcStatus(text) + return ( + idFailure(proof, identity, context) ?? + clearedCapsFailure(proof) ?? + boundingSetFailure(proof, context.boundingMustBeZero === true) + ) +} - const { parentUid, parentGid } = context +function idFailure( + proof: SandboxProof, + identity: SandboxIdentity, + { parentUid, parentGid }: SandboxProofContext, +): string | null { if (parentUid !== undefined && proof.uid.some((v) => v === parentUid)) { return `child Uid [${proof.uid.join(" ")}] still contains the worker's own uid ${parentUid}` } if (parentGid !== undefined && proof.gid.some((v) => v === parentGid)) { return `child Gid [${proof.gid.join(" ")}] still contains the worker's own gid ${parentGid}` } - - if (proof.uid.length !== 4 || proof.uid.some((v) => v !== identity.uid)) { + if (proof.uid.length !== PROC_STATUS_ID_COUNT || proof.uid.some((v) => v !== identity.uid)) { return `child Uid is [${proof.uid.join(" ")}], expected all four ids to be ${identity.uid}` } - if (proof.gid.length !== 4 || proof.gid.some((v) => v !== identity.gid)) { + if (proof.gid.length !== PROC_STATUS_ID_COUNT || proof.gid.some((v) => v !== identity.gid)) { return `child Gid is [${proof.gid.join(" ")}], expected all four ids to be ${identity.gid}` } + return null +} - for (const name of ["CapInh", "CapPrm", "CapEff", "CapAmb"]) { +function clearedCapsFailure(proof: SandboxProof): string | null { + for (const name of CLEARED_CAP_FIELDS) { const value = proof.caps[name] if (value === undefined) return `child /proc/self/status has no ${name} line` if (value !== 0n) return `child ${name} is ${value.toString(16)}, expected 0` } + return null +} +function boundingSetFailure(proof: SandboxProof, mustBeZero: boolean): string | null { const bounding = proof.caps.CapBnd if (bounding === undefined) return "child /proc/self/status has no CapBnd line" - if (context.boundingMustBeZero === true) { - if (bounding !== 0n) return `child CapBnd is ${bounding.toString(16)}, expected 0` - } else if ((bounding & ~ALLOWED_BOUNDING_MASK) !== 0n) { + if (mustBeZero) { + return bounding !== 0n ? `child CapBnd is ${bounding.toString(16)}, expected 0` : null + } + if ((bounding & ~ALLOWED_BOUNDING_MASK) !== 0n) { return ( `child CapBnd is ${bounding.toString(16)}, which contains capabilities beyond ` + "CAP_SETUID/CAP_SETGID (the container's own grant)" @@ -152,7 +178,7 @@ async function assertVideoLaneRuns(ffprobePath: string): Promise { try { await runTool("preflight-ffprobe", ffprobePath, ["-hide_banner", "-version"], { timeoutMs: PREFLIGHT_TIMEOUT_MS, - maxStdoutBytes: 64 * 1024, + maxStdoutBytes: PREFLIGHT_MAX_STDOUT_BYTES, }) } catch (err) { throw new Error( @@ -205,7 +231,7 @@ async function assertUnprivilegedChild(identity: SandboxIdentity): Promise try { const res = await runTool("sandbox-preflight", STATUS_READER, [PROC_STATUS], { timeoutMs: PREFLIGHT_TIMEOUT_MS, - maxStdoutBytes: 64 * 1024, + maxStdoutBytes: PREFLIGHT_MAX_STDOUT_BYTES, identity, }) stdout = res.stdout diff --git a/services/media-worker/src/sandbox/tmp.ts b/services/media-worker/src/sandbox/tmp.ts index b31b6f52..5ca322b7 100644 --- a/services/media-worker/src/sandbox/tmp.ts +++ b/services/media-worker/src/sandbox/tmp.ts @@ -14,6 +14,15 @@ import { join } from "node:path" import { sandboxIdentity } from "./exec.js" const SCRATCH_PREFIX = "civfix-media-" +const SCRATCH_STALE_MS = 60 * 60 * 1000 +const DEFAULT_INPUT_EXT = "bin" +const SAFE_EXT_PATTERN = /^[a-z0-9]{1,8}$/i +const UNSAFE_NAME_CHARS = /[^a-z0-9._-]/gi +// Group-writable (setgid, so outputs inherit the sandbox group) while the sandboxed child writes; +// owner-only once sealed for the read-back. +const SHARED_DIR_MODE = 0o2770 +const SHARED_INPUT_MODE = 0o660 +const SEALED_DIR_MODE = 0o700 export class ScratchOutputError extends Error { constructor(message: string, cause?: unknown) { @@ -42,27 +51,27 @@ export interface Scratch { cleanup(): Promise } -export async function makeScratch(bytes?: Uint8Array, ext = "bin"): Promise { +export async function makeScratch(bytes?: Uint8Array, ext = DEFAULT_INPUT_EXT): Promise { let dir: string try { dir = await mkdtemp(join(tmpdir(), SCRATCH_PREFIX)) } catch (err) { throw new ScratchSetupError("could not create the sandbox scratch dir", err) } - const safeExt = /^[a-z0-9]{1,8}$/i.test(ext) ? ext : "bin" + const safeExt = SAFE_EXT_PATTERN.test(ext) ? ext : DEFAULT_INPUT_EXT const inputPath = join(dir, `input.${safeExt}`) try { const identity = sandboxIdentity() if (identity !== null) { await chown(dir, process.getuid?.() ?? -1, identity.gid) - await chmod(dir, 0o2770) + await chmod(dir, SHARED_DIR_MODE) } if (bytes !== undefined) { await writeFile(inputPath, bytes) - if (identity !== null) await chmod(inputPath, 0o660) + if (identity !== null) await chmod(inputPath, SHARED_INPUT_MODE) } } catch (err) { - await rm(dir, { recursive: true, force: true }).catch(() => {}) + await removeQuietly(dir) throw new ScratchSetupError("could not prepare the sandbox scratch dir", err) } let cleaned = false @@ -70,12 +79,12 @@ export async function makeScratch(bytes?: Uint8Array, ext = "bin"): Promise { try { - await chmod(dir, 0o700) + await chmod(dir, SEALED_DIR_MODE) } catch (err) { throw new ScratchSetupError("could not seal the sandbox scratch dir", err) } @@ -84,7 +93,7 @@ export async function makeScratch(bytes?: Uint8Array, ext = "bin"): Promise {}) + await removeQuietly(dir) }, } } @@ -124,7 +133,11 @@ export async function readScratchOutput( } } -export async function sweepStaleScratchDirs(maxAgeMs = 60 * 60 * 1000): Promise { +async function removeQuietly(dir: string): Promise { + await rm(dir, { recursive: true, force: true }).catch(() => {}) +} + +export async function sweepStaleScratchDirs(maxAgeMs = SCRATCH_STALE_MS): Promise { const root = tmpdir() const cutoff = Date.now() - maxAgeMs let removed = 0 diff --git a/services/media-worker/src/seams.ts b/services/media-worker/src/seams.ts index 53f4f40d..5d82851f 100644 --- a/services/media-worker/src/seams.ts +++ b/services/media-worker/src/seams.ts @@ -40,6 +40,9 @@ export interface BuildSeamsOptions { // can redirect every seam's output at once instead of each adapter choosing its own console fallback. const defaultSeamLog: JobLogFn = (line, extra) => console.warn(line, extra ?? {}) +const DEFAULT_NODE_ENV = "development" +const DEFAULT_SERVICE_VERSION = "0.0.0" + export async function buildSeams( source: NodeJS.ProcessEnv = process.env, options: BuildSeamsOptions = {}, @@ -61,41 +64,19 @@ export async function buildSeams( await initErrorReporting({ ...(source.GLITCHTIP_DSN ? { dsn: source.GLITCHTIP_DSN } : {}), - environment: source.NODE_ENV ?? "development", - release: `media-worker@${source.SERVICE_VERSION ?? "0.0.0"}`, + environment: source.NODE_ENV ?? DEFAULT_NODE_ENV, + release: `media-worker@${source.SERVICE_VERSION ?? DEFAULT_SERVICE_VERSION}`, }) const localStorageDir = (source.LOCAL_STORAGE_DIR ?? "").trim() const storage: Storage = localStorageDir.length > 0 - ? new LocalDiskStorage({ - rootDirectory: localStorageDir, - namespace: "media", - publicApiUrl: req(source, "PUBLIC_API_URL"), - signingKey: - (source.LOCAL_STORAGE_SIGNING_KEY ?? "").trim() || LOCAL_STORAGE_DEV_SIGNING_KEY, - nodeEnv: source.NODE_ENV ?? "development", - }) + ? localDiskStorage(source, localStorageDir, "media") : fakeStorage ? new FakeStorage() - : new R2Storage({ - accountId: req(source, "R2_ACCOUNT_ID"), - accessKeyId: req(source, "R2_ACCESS_KEY_ID"), - secretAccessKey: req(source, "R2_SECRET_ACCESS_KEY"), - bucket: req(source, "R2_BUCKET"), - }) - - let dbHandle: DbHandle | undefined - let repo: MediaWorkerRepo | undefined - let anonHoldRepo: AnonHoldReleaseRepo | undefined - const databaseUrl = (source.DATABASE_URL ?? "").trim() - if (databaseUrl) { - dbHandle = makeDb(databaseUrl) - repo = makeDrizzleMediaWorkerRepo(dbHandle.db, dbHandle.sql) - anonHoldRepo = makeDrizzleAnonHoldReleaseRepo(dbHandle.sql) - } else if (source.NODE_ENV === "production") { - throw new Error("media-worker: DATABASE_URL is required in production to persist media results") - } + : r2Storage(source, () => req(source, "R2_BUCKET")) + + const { dbHandle, repo, anonHoldRepo } = buildDbSeams(source) const findPhashDuplicate: FindPhashDuplicateFn | undefined = dbHandle ? makePhashDuplicateLookup(dbHandle) @@ -107,36 +88,11 @@ export async function buildSeams( const download = makeDownloader(storage) - const inboundBucket = - (source.R2_INBOUND_BUCKET ?? "").trim() || - ((source.R2_PUBLIC_BASE ?? "").trim().length === 0 ? (source.R2_BUCKET ?? "").trim() : "") - const usesR2 = localStorageDir.length === 0 && !fakeStorage - if (usesR2 && inboundBucket.length === 0) { - console.warn( - "media-worker: no R2_INBOUND_BUCKET (and R2_PUBLIC_BASE is set), so the inbound-email retention " + - "lane is DISABLED and archived inbound_emails rows are kept. Set R2_INBOUND_BUCKET to the same " + - "bucket the API writes inbound mail to.", - ) - } - const inboundStorage: Storage | undefined = usesR2 - ? inboundBucket.length === 0 - ? undefined - : new R2Storage({ - accountId: req(source, "R2_ACCOUNT_ID"), - accessKeyId: req(source, "R2_ACCESS_KEY_ID"), - secretAccessKey: req(source, "R2_SECRET_ACCESS_KEY"), - bucket: inboundBucket, - }) - : localStorageDir.length > 0 - ? new LocalDiskStorage({ - rootDirectory: localStorageDir, - namespace: "inbound", - publicApiUrl: req(source, "PUBLIC_API_URL"), - signingKey: - (source.LOCAL_STORAGE_SIGNING_KEY ?? "").trim() || LOCAL_STORAGE_DEV_SIGNING_KEY, - nodeEnv: source.NODE_ENV ?? "development", - }) - : storage + const inboundStorage = buildInboundStorage(source, { + storage, + localStorageDir, + usesR2: localStorageDir.length === 0 && !fakeStorage, + }) const publicMediaBaseRaw = (source.R2_PUBLIC_BASE ?? "").trim() @@ -159,6 +115,71 @@ export async function buildSeams( } } +function localDiskStorage( + source: NodeJS.ProcessEnv, + rootDirectory: string, + namespace: "media" | "inbound", +): LocalDiskStorage { + return new LocalDiskStorage({ + rootDirectory, + namespace, + publicApiUrl: req(source, "PUBLIC_API_URL"), + signingKey: (source.LOCAL_STORAGE_SIGNING_KEY ?? "").trim() || LOCAL_STORAGE_DEV_SIGNING_KEY, + nodeEnv: source.NODE_ENV ?? DEFAULT_NODE_ENV, + }) +} + +// The bucket is resolved after the credentials so a missing credential is the error reported first. +function r2Storage(source: NodeJS.ProcessEnv, bucket: () => string): R2Storage { + return new R2Storage({ + accountId: req(source, "R2_ACCOUNT_ID"), + accessKeyId: req(source, "R2_ACCESS_KEY_ID"), + secretAccessKey: req(source, "R2_SECRET_ACCESS_KEY"), + bucket: bucket(), + }) +} + +function buildDbSeams( + source: NodeJS.ProcessEnv, +): Pick { + const databaseUrl = (source.DATABASE_URL ?? "").trim() + if (databaseUrl) { + const dbHandle = makeDb(databaseUrl) + return { + dbHandle, + repo: makeDrizzleMediaWorkerRepo(dbHandle.db, dbHandle.sql), + anonHoldRepo: makeDrizzleAnonHoldReleaseRepo(dbHandle.sql), + } + } + if (source.NODE_ENV === "production") { + throw new Error("media-worker: DATABASE_URL is required in production to persist media results") + } + return { dbHandle: undefined, repo: undefined, anonHoldRepo: undefined } +} + +function buildInboundStorage( + source: NodeJS.ProcessEnv, + media: { storage: Storage; localStorageDir: string; usesR2: boolean }, +): Storage | undefined { + const inboundBucket = + (source.R2_INBOUND_BUCKET ?? "").trim() || + ((source.R2_PUBLIC_BASE ?? "").trim().length === 0 ? (source.R2_BUCKET ?? "").trim() : "") + if (media.usesR2 && inboundBucket.length === 0) { + console.warn( + "media-worker: no R2_INBOUND_BUCKET (and R2_PUBLIC_BASE is set), so the inbound-email retention " + + "lane is DISABLED and archived inbound_emails rows are kept. Set R2_INBOUND_BUCKET to the same " + + "bucket the API writes inbound mail to.", + ) + } + if (media.usesR2) { + return inboundBucket.length === 0 ? undefined : r2Storage(source, () => inboundBucket) + } + if (media.localStorageDir.length > 0) { + return localDiskStorage(source, media.localStorageDir, "inbound") + } + return media.storage +} + function req(source: NodeJS.ProcessEnv, key: string): string { const v = (source[key] ?? "").trim() if (!v) throw new Error(`media-worker: ${key} is required when its real seam is enabled`) diff --git a/services/media-worker/src/worker.ts b/services/media-worker/src/worker.ts index 8cc2392d..c7d7212f 100644 --- a/services/media-worker/src/worker.ts +++ b/services/media-worker/src/worker.ts @@ -1,7 +1,13 @@ import type { JobHandler } from "@civfix/shared/interfaces" import { MEDIA_CHECKS_JOB } from "@civfix/api/media-repo" import { releaseAnonHoldIfReady, type HeldReportView } from "@civfix/api/anon-hold-release" -import { buildJobs, stopGraceMsFor, type JobsHandle, type WorkerJobs } from "./jobs.js" +import { + buildJobs, + stopGraceMsFor, + type JobsHandle, + type QueueOptions, + type WorkerJobs, +} from "./jobs.js" import { buildSeams, type WorkerSeams } from "./seams.js" import { CHAT_PARTITION_CRON, @@ -11,7 +17,12 @@ import { RETENTION_SWEEP_CRON, type WorkerLimits, } from "./config.js" -import { runMediaChecksJobDetailed, parsePayload } from "./jobs/media-checks.js" +import { + runMediaChecksJobDetailed, + parsePayload, + type MediaChecksOutcome, + type MediaChecksPayload, +} from "./jobs/media-checks.js" import { runOrphanSweep } from "./jobs/orphan-sweep.js" import { runHoldReleaseSweep } from "./jobs/hold-release-sweep.js" import { runPartitionMaintenance } from "./jobs/partition-maintenance.js" @@ -35,6 +46,32 @@ export const RETENTION_SWEEP_JOB = "retention.sweep" export const MEDIA_STUCK_SWEEP_JOB = "media.stuck.sweep" const CRON_EXPIRE_SECONDS = 25 * 60 +const MEDIA_CHECKS_RETRY_LIMIT = 5 +const UPLOAD_REAP_RETRY_LIMIT = 3 +const MS_PER_SECOND = 1000 +const COMPOSE_STOP_GRACE_HEADROOM_SEC = 15 + +const QUEUES: readonly [name: string, options: QueueOptions][] = [ + [MEDIA_CHECKS_JOB, { policy: "short", retryLimit: MEDIA_CHECKS_RETRY_LIMIT, retryBackoff: true }], + [ORPHAN_SWEEP_JOB, { policy: "singleton" }], + [CHAT_PARTITION_JOB, { policy: "singleton" }], + [ANON_HOLD_RELEASE_JOB, { policy: "short" }], + [ANON_HOLD_RELEASE_SWEEP_JOB, { policy: "singleton" }], + [RETENTION_SWEEP_JOB, { policy: "singleton" }], + [MEDIA_STUCK_SWEEP_JOB, { policy: "singleton" }], + [ + MEDIA_UPLOAD_REAP_JOB, + { policy: "short", retryLimit: UPLOAD_REAP_RETRY_LIMIT, retryBackoff: true }, + ], +] + +const CRON_SCHEDULES: readonly [name: string, cron: string][] = [ + [ORPHAN_SWEEP_JOB, ORPHAN_SWEEP_CRON], + [CHAT_PARTITION_JOB, CHAT_PARTITION_CRON], + [ANON_HOLD_RELEASE_SWEEP_JOB, HOLD_RELEASE_SWEEP_CRON], + [RETENTION_SWEEP_JOB, RETENTION_SWEEP_CRON], + [MEDIA_STUCK_SWEEP_JOB, MEDIA_STUCK_SWEEP_CRON], +] export interface Worker { jobs: WorkerJobs @@ -78,38 +115,50 @@ function makeMediaChecksHandler(jobs: WorkerJobs, seams: WorkerSeams): JobHandle }) if (outcome.status !== "missing") { - try { - await jobs.enqueue( - MEDIA_UPLOAD_REAP_JOB, - { mediaId: payload.mediaId, uploadId: payload.uploadId, r2Key: payload.r2Key }, - { singletonKey: payload.uploadId, startAfter: uploadReapDelaySec() }, - ) - } catch (err) { - console.warn("media.checks: failed to schedule media.upload.reap (non-fatal)", { - uploadId: payload.uploadId, - err: String(err), - }) - } + await scheduleUploadReap(jobs, payload) } + await enqueueHoldReleaseIfAnonHeld(jobs, seams, repo, payload, outcome) + } +} - try { - const reportId = - outcome.reportId ?? - (outcome.status === "missing" ? null : await findReportId(repo, payload)) - if (reportId && (await shouldEnqueueHoldRelease(seams, reportId))) { - await jobs.enqueue(ANON_HOLD_RELEASE_JOB, { reportId }, { singletonKey: reportId }) - } else { - console.debug("media.checks: hold-release skipped (no row/reportId, or not anon-held)", { - uploadId: payload.uploadId, - status: outcome.status, - }) - } - } catch (err) { - console.warn("media.checks: failed to enqueue anon.hold.release (non-fatal)", { +async function scheduleUploadReap(jobs: WorkerJobs, payload: MediaChecksPayload): Promise { + try { + await jobs.enqueue( + MEDIA_UPLOAD_REAP_JOB, + { mediaId: payload.mediaId, uploadId: payload.uploadId, r2Key: payload.r2Key }, + { singletonKey: payload.uploadId, startAfter: uploadReapDelaySec() }, + ) + } catch (err) { + console.warn("media.checks: failed to schedule media.upload.reap (non-fatal)", { + uploadId: payload.uploadId, + err: String(err), + }) + } +} + +async function enqueueHoldReleaseIfAnonHeld( + jobs: WorkerJobs, + seams: WorkerSeams, + repo: NonNullable, + payload: MediaChecksPayload, + outcome: MediaChecksOutcome, +): Promise { + try { + const reportId = + outcome.reportId ?? (outcome.status === "missing" ? null : await findReportId(repo, payload)) + if (reportId && (await shouldEnqueueHoldRelease(seams, reportId))) { + await jobs.enqueue(ANON_HOLD_RELEASE_JOB, { reportId }, { singletonKey: reportId }) + } else { + console.debug("media.checks: hold-release skipped (no row/reportId, or not anon-held)", { uploadId: payload.uploadId, - err: String(err), + status: outcome.status, }) } + } catch (err) { + console.warn("media.checks: failed to enqueue anon.hold.release (non-fatal)", { + uploadId: payload.uploadId, + err: String(err), + }) } } @@ -250,18 +299,9 @@ async function registerHandlers( seams: WorkerSeams, limits: WorkerLimits, ): Promise { - await jobs.createQueue(MEDIA_CHECKS_JOB, { policy: "short", retryLimit: 5, retryBackoff: true }) - await jobs.createQueue(ORPHAN_SWEEP_JOB, { policy: "singleton" }) - await jobs.createQueue(CHAT_PARTITION_JOB, { policy: "singleton" }) - await jobs.createQueue(ANON_HOLD_RELEASE_JOB, { policy: "short" }) - await jobs.createQueue(ANON_HOLD_RELEASE_SWEEP_JOB, { policy: "singleton" }) - await jobs.createQueue(RETENTION_SWEEP_JOB, { policy: "singleton" }) - await jobs.createQueue(MEDIA_STUCK_SWEEP_JOB, { policy: "singleton" }) - await jobs.createQueue(MEDIA_UPLOAD_REAP_JOB, { - policy: "short", - retryLimit: 3, - retryBackoff: true, - }) + for (const [name, options] of QUEUES) { + await jobs.createQueue(name, options) + } await jobs.workWithSettings(MEDIA_CHECKS_JOB, makeMediaChecksHandler(jobs, seams), { batchSize: limits.mediaChecksConcurrency, @@ -276,36 +316,9 @@ async function registerHandlers( await jobs.work(MEDIA_STUCK_SWEEP_JOB, makeStuckSweepHandler(jobs, seams)) await jobs.work(MEDIA_UPLOAD_REAP_JOB, makeUploadReapHandler(seams)) - await jobs.schedule( - ORPHAN_SWEEP_JOB, - ORPHAN_SWEEP_CRON, - undefined, - cronSchedule(ORPHAN_SWEEP_JOB), - ) - await jobs.schedule( - CHAT_PARTITION_JOB, - CHAT_PARTITION_CRON, - undefined, - cronSchedule(CHAT_PARTITION_JOB), - ) - await jobs.schedule( - ANON_HOLD_RELEASE_SWEEP_JOB, - HOLD_RELEASE_SWEEP_CRON, - undefined, - cronSchedule(ANON_HOLD_RELEASE_SWEEP_JOB), - ) - await jobs.schedule( - RETENTION_SWEEP_JOB, - RETENTION_SWEEP_CRON, - undefined, - cronSchedule(RETENTION_SWEEP_JOB), - ) - await jobs.schedule( - MEDIA_STUCK_SWEEP_JOB, - MEDIA_STUCK_SWEEP_CRON, - undefined, - cronSchedule(MEDIA_STUCK_SWEEP_JOB), - ) + for (const [name, cron] of CRON_SCHEDULES) { + await jobs.schedule(name, cron, undefined, cronSchedule(name)) + } } function cronSchedule(name: string): { expireInSeconds: number; singletonKey: string } { @@ -317,7 +330,8 @@ function logStopGraceRequirement(limits: WorkerLimits): void { console.log("media-worker: graceful-stop budget", { jobTimeoutMs: limits.jobTimeoutMs, stopGraceMs: graceMs, - requiredComposeStopGracePeriodSec: Math.ceil(graceMs / 1000) + 15, + requiredComposeStopGracePeriodSec: + Math.ceil(graceMs / MS_PER_SECOND) + COMPOSE_STOP_GRACE_HEADROOM_SEC, }) }