diff --git a/services/api/src/abuse/anon-token.ts b/services/api/src/abuse/anon-token.ts index 7d95f1bc..77792352 100644 --- a/services/api/src/abuse/anon-token.ts +++ b/services/api/src/abuse/anon-token.ts @@ -19,6 +19,9 @@ export const ANON_TOKEN_TTL_SECONDS = 24 * 60 * 60 export const ANON_TOKEN_REPORT_CAP = 5 +export const ANON_REPORT_CAP_MESSAGE = + "This anonymous session has reached its report limit. Sign in to continue." + /** base64url never contains ".", so the separator cannot collide with the id or the HMAC. */ const TOKEN_SEP = "." @@ -108,9 +111,7 @@ export function assertUnderReportCap( cap: number = ANON_TOKEN_REPORT_CAP, ): { remaining: number } { if (row.reportCount >= cap) { - throw AppError.rateLimited( - "This anonymous session has reached its report limit. Sign in to continue.", - ) + throw AppError.rateLimited(ANON_REPORT_CAP_MESSAGE) } return { remaining: cap - row.reportCount } } diff --git a/services/api/src/abuse/ip-rate-limit.ts b/services/api/src/abuse/ip-rate-limit.ts index 0c667b6a..b118e4fd 100644 --- a/services/api/src/abuse/ip-rate-limit.ts +++ b/services/api/src/abuse/ip-rate-limit.ts @@ -12,24 +12,29 @@ import type { CounterStore } from "./counter-store.js" export const IP_HARD_LIMIT_PER_HOUR = 10 -/** Higher soft cap reserved for a future known-clean-ASN classifier; unused until `classifyIpAllowance` consults it. */ -export const IP_SOFT_LIMIT_PER_HOUR = 50 - -export const IP_WINDOW_SECONDS = 60 * 60 +const IP_WINDOW_SECONDS = 60 * 60 const IP_COUNTER_PREFIX = "abuse:ip:" const IPV6_PREFIX_HEXTETS = 4 +const UNKNOWN_IP_BUCKET = "unknown" + +const IPV4_MAPPED_IPV6_RE = /^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/ + +const IPV4_RE = /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ + +const LEADING_ZEROS_RE = /^0+(?=.)/ + /** A missing IP normalizes to a stable "unknown" bucket so it is still limited instead of bypassing. */ export function normalizeIp(ip: string | undefined | null): string { const raw = (ip ?? "").trim().toLowerCase() - if (raw === "") return "unknown" + if (raw === "") return UNKNOWN_IP_BUCKET - const mapped = /^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/.exec(raw) + const mapped = IPV4_MAPPED_IPV6_RE.exec(raw) if (mapped) return mapped[1]! - if (/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(raw)) return raw + if (IPV4_RE.test(raw)) return raw if (raw.includes(":")) { return ipv6Prefix64(raw) @@ -46,14 +51,14 @@ function ipv6Prefix64(addr: string): string { const prefix: string[] = [] for (let i = 0; i < IPV6_PREFIX_HEXTETS; i++) { const h = hextets[i] ?? "0" - const trimmed = h.replace(/^0+(?=.)/, "") + const trimmed = h.replace(LEADING_ZEROS_RE, "") prefix.push(trimmed === "" ? "0" : trimmed) } return `${prefix.join(":")}::/64` } // The hard cap for every IP today: no GeoIP/ASN database ships. This is the one place to wire an ASN -// lookup that grants IP_SOFT_LIMIT_PER_HOUR to known-clean residential or shared-NAT ranges. +// lookup that grants a higher cap to known-clean residential or shared-NAT ranges. export function classifyIpAllowance(_normalizedIp: string): number { return IP_HARD_LIMIT_PER_HOUR } diff --git a/services/api/src/abuse/slur-filter.ts b/services/api/src/abuse/slur-filter.ts index b4d941ef..71493fbd 100644 --- a/services/api/src/abuse/slur-filter.ts +++ b/services/api/src/abuse/slur-filter.ts @@ -43,6 +43,16 @@ const DIGIT_LEET: Readonly> = { } const SYMBOL_LEET: Readonly> = { "!": "i", "|": "i", "@": "a" } +const LETTER_RE = /[a-z]/i + +const SEPARATOR_BETWEEN_CHARS_RE = /([a-z0-9])[._\-*]+([a-z0-9])/gi + +const SPACED_OUT_RUN_RE = /\b[a-z0-9](?: [a-z0-9])+\b/gi + +const SPACE_RE = / /g + +const SLUR_MESSAGE = "This contains language that isn't allowed." + function deLeet(s: string): string { let out = "" for (let i = 0; i < s.length; i++) { @@ -54,7 +64,7 @@ function deLeet(s: string): string { } const digit = DIGIT_LEET[ch] if (digit !== undefined) { - const adjacentToLetter = /[a-z]/i.test(s.charAt(i - 1)) || /[a-z]/i.test(s.charAt(i + 1)) + const adjacentToLetter = LETTER_RE.test(s.charAt(i - 1)) || LETTER_RE.test(s.charAt(i + 1)) out += adjacentToLetter ? digit : ch continue } @@ -69,9 +79,9 @@ function deobfuscate(text: string): string { let prev: string do { prev = collapsed - collapsed = collapsed.replace(/([a-z0-9])[._\-*]+([a-z0-9])/gi, "$1$2") + collapsed = collapsed.replace(SEPARATOR_BETWEEN_CHARS_RE, "$1$2") } while (collapsed !== prev) - return collapsed.replace(/\b[a-z0-9](?: [a-z0-9])+\b/gi, (run) => run.replace(/ /g, "")) + return collapsed.replace(SPACED_OUT_RUN_RE, (run) => run.replace(SPACE_RE, "")) } // Combining marks survive NFKD as separate code points and format characters (zero-width joiners, soft @@ -96,6 +106,6 @@ export function containsSlur(text: string | null | undefined): boolean { export function assertNoSlur(text: string | null | undefined, field = "body"): void { if (containsSlur(text)) { - throw AppError.validation({ [field]: "This contains language that isn't allowed." }) + throw AppError.validation({ [field]: SLUR_MESSAGE }) } } diff --git a/services/api/src/adapters/abuse-checks.ts b/services/api/src/adapters/abuse-checks.ts index d7d2d016..b7c2da98 100644 --- a/services/api/src/adapters/abuse-checks.ts +++ b/services/api/src/adapters/abuse-checks.ts @@ -2,7 +2,7 @@ import { AppError } from "@civfix/shared" import type { AbuseChecks, NearDuplicateResult } from "@civfix/shared/interfaces" import type { LatLng } from "@civfix/shared" import { haversineKm } from "@civfix/shared" -import { fetchJsonWithTimeout } from "./http-fetch.js" +import { fetchJsonWithTimeout, type FetchJsonResult } from "./http-fetch.js" export type PerceptualHashFn = (buffer: Uint8Array) => Promise @@ -34,6 +34,9 @@ const TURNSTILE_TIMEOUT_MS = 4000 const GPS_MAX_KM = 50 +const FNV1A_32_OFFSET_BASIS = 0x811c9dc5 +const FNV1A_32_PRIME = 0x01000193 + interface TurnstileVerifyResponse { success?: boolean hostname?: string @@ -79,17 +82,7 @@ export class RealAbuseChecks implements AbuseChecks { body, }, }) - if (!result.ok) { - const wrapped = AppError.internal( - result.kind === "http" - ? `Turnstile verification returned HTTP ${result.status}` - : result.kind === "body" - ? "Turnstile verification returned a non-JSON body" - : "Turnstile verification request failed", - ) - if (result.kind !== "http") (wrapped as { cause?: unknown }).cause = result.error - throw wrapped - } + if (!result.ok) throw turnstileFailure(result) const json = result.json if (json.success !== true) return false @@ -98,23 +91,22 @@ export class RealAbuseChecks implements AbuseChecks { this.log("Turnstile token rejected: unexpected hostname", { hostname: json.hostname }) return false } - if (expect?.action !== undefined) { - if (json.action === undefined || json.action === "") { - if (!this.turnstileActionNoticeLogged) { - this.turnstileActionNoticeLogged = true - this.log("Turnstile token carried no action; binding not enforced (soft-enforce)", { - expected: expect.action, - }) - } - } else if (json.action !== expect.action) { - this.log("Turnstile token rejected: action mismatch", { - expected: expect.action, - actual: json.action, + return expect?.action === undefined || this.actionAccepted(json.action, expect.action) + } + + private actionAccepted(actual: string | undefined, expected: string): boolean { + if (actual === undefined || actual === "") { + if (!this.turnstileActionNoticeLogged) { + this.turnstileActionNoticeLogged = true + this.log("Turnstile token carried no action; binding not enforced (soft-enforce)", { + expected, }) - return false } + return true } - return true + if (actual === expected) return true + this.log("Turnstile token rejected: action mismatch", { expected, actual }) + return false } private hostnameAccepted(hostname: string | undefined): boolean { @@ -179,13 +171,25 @@ export class RealAbuseChecks implements AbuseChecks { } } +function turnstileFailure(result: Exclude, { ok: true }>): AppError { + const wrapped = AppError.internal( + result.kind === "http" + ? `Turnstile verification returned HTTP ${result.status}` + : result.kind === "body" + ? "Turnstile verification returned a non-JSON body" + : "Turnstile verification request failed", + ) + if (result.kind !== "http") (wrapped as { cause?: unknown }).cause = result.error + return wrapped +} + function fnv1a64Hex(buffer: Uint8Array): string { - let lo = 0x811c9dc5 - let hi = 0x811c9dc5 + let lo = FNV1A_32_OFFSET_BASIS + let hi = FNV1A_32_OFFSET_BASIS for (let i = 0; i < buffer.length; i++) { const b = buffer[i] ?? 0 - lo = Math.imul(lo ^ b, 0x01000193) >>> 0 - hi = Math.imul(hi ^ (b ^ (i & 0xff)), 0x01000193) >>> 0 + lo = Math.imul(lo ^ b, FNV1A_32_PRIME) >>> 0 + hi = Math.imul(hi ^ (b ^ (i & 0xff)), FNV1A_32_PRIME) >>> 0 } const toHex8 = (n: number): string => (n >>> 0).toString(16).padStart(8, "0") return toHex8(hi) + toHex8(lo) diff --git a/services/api/src/adapters/chat-presence.ts b/services/api/src/adapters/chat-presence.ts index 8e87cdc9..75f07b8c 100644 --- a/services/api/src/adapters/chat-presence.ts +++ b/services/api/src/adapters/chat-presence.ts @@ -18,6 +18,8 @@ export const PRESENCE_TTL_MS = 90_000 const PRESENCE_KEY_TTL_SECONDS = 7200 +const PRESENCE_KEY_PREFIX = "presence:" + /** UUIDs never contain "::", so splitting on the first occurrence is unambiguous. */ const SEP = "::" @@ -59,6 +61,12 @@ function userOf(memberId: string): string { return idx === -1 ? memberId : memberId.slice(0, idx) } +// ZREMRANGEBYSCORE max bound: the "(" makes it exclusive, so an entry seen exactly at the cutoff survives +// as it does in the in-memory prune. +function staleScoreBound(now: number): string { + return `(${now - PRESENCE_TTL_MS}` +} + function distinctUsers(members: string[]): string[] { return [...new Set(members.map(userOf))].sort() } @@ -97,7 +105,7 @@ export class RedisChatPresence implements ChatPresence { constructor(private readonly redis: RedisClient) {} private key(cleanupId: string): string { - return `presence:${cleanupId}` + return `${PRESENCE_KEY_PREFIX}${cleanupId}` } async join(cleanupId: string, connId: string, userId: string): Promise { @@ -107,7 +115,7 @@ export class RedisChatPresence implements ChatPresence { // cannot be perturbed by an interleaving command. const replies = await this.redis .multi() - .zremrangebyscore(key, "-inf", `(${now - PRESENCE_TTL_MS}`) + .zremrangebyscore(key, "-inf", staleScoreBound(now)) .zadd(key, now, member(userId, connId)) .expire(key, PRESENCE_KEY_TTL_SECONDS) .zrange(key, 0, -1) @@ -125,7 +133,7 @@ export class RedisChatPresence implements ChatPresence { const replies = await this.redis .multi() .zrem(key, member(userId, connId)) - .zremrangebyscore(key, "-inf", `(${now - PRESENCE_TTL_MS}`) + .zremrangebyscore(key, "-inf", staleScoreBound(now)) .zrange(key, 0, -1) .exec() const members = presenceMembers(replies) @@ -151,7 +159,7 @@ export class RedisChatPresence implements ChatPresence { const now = Date.now() const replies = await this.redis .multi() - .zremrangebyscore(key, "-inf", `(${now - PRESENCE_TTL_MS}`) + .zremrangebyscore(key, "-inf", staleScoreBound(now)) .zrange(key, 0, -1) .exec() return distinctUsers(presenceMembers(replies)) diff --git a/services/api/src/adapters/chat-pubsub.ts b/services/api/src/adapters/chat-pubsub.ts index 47f8bbfc..0dd6e1e9 100644 --- a/services/api/src/adapters/chat-pubsub.ts +++ b/services/api/src/adapters/chat-pubsub.ts @@ -1,6 +1,8 @@ import { attachRedisErrorHandler, type RedisClient } from "./redis.js" import { RefCountedSubscriptions } from "./ref-counted-subscriptions.js" +const CHAT_CHANNEL_PREFIX = "chat:" + export type ChatPubSubHandler = (payload: string) => void export interface ChatPubSub { @@ -10,7 +12,7 @@ export interface ChatPubSub { } export function chatChannel(cleanupId: string): string { - return `chat:${cleanupId}` + return `${CHAT_CHANNEL_PREFIX}${cleanupId}` } export class RedisChatPubSub implements ChatPubSub { diff --git a/services/api/src/adapters/chat-send-dedupe.redis.ts b/services/api/src/adapters/chat-send-dedupe.redis.ts index 9875a3f5..3c963496 100644 --- a/services/api/src/adapters/chat-send-dedupe.redis.ts +++ b/services/api/src/adapters/chat-send-dedupe.redis.ts @@ -16,6 +16,12 @@ export interface RedisSendDedupeOptions { sleep?: (ms: number) => Promise } +const MS_PER_SECOND = 1000 + +function wholeSecondsToMs(seconds: number): number { + return Math.max(1, Math.ceil(seconds)) * MS_PER_SECOND +} + const realSleep = (ms: number): Promise => new Promise((resolve) => { const timer = setTimeout(resolve, ms) @@ -35,9 +41,8 @@ export class RedisSendDedupeStore implements SendDedupeStore { constructor(redis: RedisClient, opts: RedisSendDedupeOptions = {}) { this.redis = redis - this.ttlMs = Math.max(1, Math.ceil(opts.ttlSeconds ?? SEND_DEDUPE_TTL_SECONDS)) * 1000 - this.pendingTtlMs = - Math.max(1, Math.ceil(opts.pendingTtlSeconds ?? SEND_DEDUPE_PENDING_TTL_SECONDS)) * 1000 + this.ttlMs = wholeSecondsToMs(opts.ttlSeconds ?? SEND_DEDUPE_TTL_SECONDS) + this.pendingTtlMs = wholeSecondsToMs(opts.pendingTtlSeconds ?? SEND_DEDUPE_PENDING_TTL_SECONDS) this.inFlightAttempts = opts.inFlightAttempts ?? SEND_DEDUPE_INFLIGHT_ATTEMPTS this.sleep = opts.sleep ?? realSleep } diff --git a/services/api/src/adapters/chat-service.ws.ts b/services/api/src/adapters/chat-service.ws.ts index 48104a3f..9d7db67a 100644 --- a/services/api/src/adapters/chat-service.ws.ts +++ b/services/api/src/adapters/chat-service.ws.ts @@ -14,27 +14,44 @@ export interface WsChatServiceDeps { repo: ChatRepository pubsub: ChatPubSub newId?: () => string + logger?: { warn(obj: unknown, msg?: string): void } +} + +interface DecodedEnvelope { + frame: string + excludeConnId: string | undefined } export class WsChatService implements ChatService { private readonly repo: ChatRepository private readonly pubsub: ChatPubSub private readonly newId: () => string + private readonly logger: WsChatServiceDeps["logger"] private readonly rooms: RefCountedSubscriptions constructor(deps: WsChatServiceDeps) { this.repo = deps.repo this.pubsub = deps.pubsub this.newId = deps.newId ?? (() => randomUUID()) - this.rooms = new RefCountedSubscriptions((cleanupId, connections) => - this.pubsub.subscribe(chatChannel(cleanupId), (payload) => { - const { frame, excludeConnId } = decodeEnvelope(payload) + this.logger = deps.logger + this.rooms = new RefCountedSubscriptions((cleanupId, connections) => { + const channel = chatChannel(cleanupId) + return this.pubsub.subscribe(channel, (payload) => { + const envelope = decodeEnvelope(payload) + if (envelope === null) { + // The payload is never logged: it can carry message bodies from any room member. + this.logger?.warn( + { channel }, + "chat: dropped a pub/sub payload that is not a frame envelope", + ) + return + } for (const c of [...connections()]) { - if (excludeConnId !== undefined && c.id === excludeConnId) continue - c.send(frame) + if (envelope.excludeConnId !== undefined && c.id === envelope.excludeConnId) continue + c.send(envelope.frame) } - }), - ) + }) + }) } async joinRoom(cleanupId: string, conn: ChatConnection, _userId: string): Promise { @@ -111,24 +128,19 @@ export class WsChatService implements ChatService { } } -function decodeEnvelope(payload: string): { frame: string; excludeConnId: string | undefined } { +// publishFrame is the only publisher on chat:, so anything else on the channel is foreign and is +// dropped rather than forwarded unparsed to every socket in the room. +function decodeEnvelope(payload: string): DecodedEnvelope | null { + let parsed: { frame?: unknown; excludeConnId?: unknown } try { - const parsed = JSON.parse(payload) as { - frame?: unknown - type?: unknown - message?: unknown - excludeConnId?: unknown - } - const excludeConnId = - typeof parsed.excludeConnId === "string" ? parsed.excludeConnId : undefined - if (parsed.frame !== undefined && parsed.frame !== null && typeof parsed.frame === "object") { - return { frame: JSON.stringify(parsed.frame), excludeConnId } - } - if (parsed.type === "message" && parsed.message !== undefined) { - return { frame: JSON.stringify({ type: "message", message: parsed.message }), excludeConnId } - } + parsed = JSON.parse(payload) as { frame?: unknown; excludeConnId?: unknown } } catch { - // Not a JSON envelope: the payload is already a bare frame and is forwarded as-is. + return null + } + if (typeof parsed !== "object" || parsed === null) return null + if (parsed.frame === null || typeof parsed.frame !== "object") return null + return { + frame: JSON.stringify(parsed.frame), + excludeConnId: typeof parsed.excludeConnId === "string" ? parsed.excludeConnId : undefined, } - return { frame: payload, excludeConnId: undefined } } diff --git a/services/api/src/adapters/geocoder.tiger.ts b/services/api/src/adapters/geocoder.tiger.ts index e95926c3..83029aad 100644 --- a/services/api/src/adapters/geocoder.tiger.ts +++ b/services/api/src/adapters/geocoder.tiger.ts @@ -76,8 +76,12 @@ const STATE_FIPS_TO_USPS: Readonly> = { "78": "VI", } +const STATE_FIPS_LENGTH = 2 + +const FIPS_HIERARCHICAL_LAYERS: ReadonlySet = new Set(["place", "county", "state"]) + export function uspsFromGeoid(geoid: string): string | null { - const fips = geoid.slice(0, 2) + const fips = geoid.slice(0, STATE_FIPS_LENGTH) return STATE_FIPS_TO_USPS[fips] ?? null } @@ -110,8 +114,7 @@ export class TigerGeocoder implements Geocoder { // The geoid prefix is trusted only on FIPS-hierarchical layers; a federal/tribal numeric id (e.g. a // BIA/ArcGIS OBJECTID) would yield a valid-but-WRONG state (see file header). - const fipsLayer = - resolved.layer === "place" || resolved.layer === "county" || resolved.layer === "state" + const fipsLayer = FIPS_HIERARCHICAL_LAYERS.has(resolved.layer) const usps = (fipsLayer ? uspsFromGeoid(resolved.geoid) : null) ?? (await this.stateAbbrFor(sql, lng, lat)) return formatCityStateLabel(resolved.name, usps) diff --git a/services/api/src/adapters/inbound-mail.cf.ts b/services/api/src/adapters/inbound-mail.cf.ts index 3269a68f..69d31ad8 100644 --- a/services/api/src/adapters/inbound-mail.cf.ts +++ b/services/api/src/adapters/inbound-mail.cf.ts @@ -11,7 +11,6 @@ import { domainOfOrNull } from "./mail-text.js" const THREAD_TOKEN_RE = /^[a-z0-9]{8,40}$/ export interface CfInboundMailConfig { - webhookSecret?: string replyDomain?: string } @@ -19,6 +18,8 @@ export const DEFAULT_REPLY_DOMAIN = "civfix.org" const REPLY_ADDRESS_RE = /^(?:reply|report|event)[-+]([^@\s]+)@([^@\s]+)$/ +const MAILPARSER_OPTIONS = { skipImageLinks: true, skipHtmlToText: true } as const + const MAX_MIME_PARTS = 200 const MAX_DISTINCT_BOUNDARIES = 32 const BOUNDARY_DECL_RE = /boundary\s*=\s*(?:"([^"\r\n]{1,200})"|([^;"\s\r\n]{1,200}))/gi @@ -59,10 +60,7 @@ export class CfInboundMail implements InboundMail { throw new Error("inbound mail: too many MIME parts") } const { simpleParser } = await import("mailparser") - const parsed = await simpleParser(Buffer.from(raw), { - skipImageLinks: true, - skipHtmlToText: true, - }) + const parsed = await simpleParser(Buffer.from(raw), MAILPARSER_OPTIONS) const fromValue = singleFromMailbox(parsed.headerLines, parsed.from) const headers = flattenHeaders(parsed.headers) @@ -116,6 +114,18 @@ const QUOTED_REMOTE_IP_RE = /smtp\.remote-ip\s*=\s*"[0-9a-f:.]+"/gi const FLAT_COMMENT_RE = /\([^()]*\)/g +const QUOTE_OR_ESCAPE_RE = /["\\]/ + +const PARENTHESIS_RE = /[()]/ + +const SPACED_EQUALS_RE = /\s*=\s*/g + +const WHITESPACE_RUN_RE = /\s+/ + +const AUTHSERV_ID_END_RE = /[\s;]/ + +const SMTP_MAILFROM_RE = /smtp\.mailfrom/gi + const ENVELOPE_ADDRESS_RE = /^[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@([^@]+)$/ @@ -134,7 +144,9 @@ interface AuthResult { export function readMailAuthVerdict(mail: ParsedMail): MailAuthVerdict { const stamp = mail.headers[AUTHENTICATION_RESULTS_HEADER] ?? "" - if (stamp.trim().split(/[\s;]/, 1)[0]?.toLowerCase() !== CLOUDFLARE_AUTHSERV_ID) return "unknown" + if (stamp.trim().split(AUTHSERV_ID_END_RE, 1)[0]?.toLowerCase() !== CLOUDFLARE_AUTHSERV_ID) { + return "unknown" + } const results = parseStamp(stamp) if (results === null) return "fail" if (results.length === 0) return "unknown" @@ -164,16 +176,16 @@ export function readMailAuthVerdict(mail: ParsedMail): MailAuthVerdict { function parseStamp(stamp: string): AuthResult[] | null { const unquoted = stamp.replace(QUOTED_REMOTE_IP_RE, "") - if (/["\\]/.test(unquoted)) return null + if (QUOTE_OR_ESCAPE_RE.test(unquoted)) return null const uncommented = unquoted.replace(FLAT_COMMENT_RE, " ") - if (/[()]/.test(uncommented)) return null + if (PARENTHESIS_RE.test(uncommented)) return null const results: AuthResult[] = [] for (const resinfo of uncommented.split(";").slice(1)) { const [methodSpec = "", ...propSpecs] = resinfo - .replace(/\s*=\s*/g, "=") + .replace(SPACED_EQUALS_RE, "=") .trim() .toLowerCase() - .split(/\s+/) + .split(WHITESPACE_RUN_RE) if (propSpecs.length === 0 && (methodSpec === "" || methodSpec === "none")) continue const [, method, result] = METHOD_SPEC_RE.exec(methodSpec) ?? [] if (method === undefined || result === undefined) return null @@ -198,7 +210,7 @@ function leadingDkimResults(results: readonly AuthResult[]): readonly AuthResult } function soleMailFromResult(stamp: string, results: readonly AuthResult[]): AuthResult | undefined { - if (stamp.match(/smtp\.mailfrom/gi)?.length !== 1) return undefined + if (stamp.match(SMTP_MAILFROM_RE)?.length !== 1) return undefined const index = results.findIndex((r) => r.props.has("smtp.mailfrom")) const carrier = results[index] if (carrier?.method !== "spf" || carrier.props.size !== 1) return undefined diff --git a/services/api/src/adapters/jobs.pgboss.ts b/services/api/src/adapters/jobs.pgboss.ts index 6096d6dd..375edb9c 100644 --- a/services/api/src/adapters/jobs.pgboss.ts +++ b/services/api/src/adapters/jobs.pgboss.ts @@ -3,6 +3,18 @@ import type { Jobs, EnqueueOptions, JobHandler } from "@civfix/shared/interfaces import { REGISTRATION_QUEUE_NAMES } from "../services/host/registration-queues.js" import { COMMS_QUEUE_NAMES } from "../services/host/broadcast-queues.js" import type { JobHandlerArgWithAttempt } from "../services/job-attempt.js" +import { MEDIA_CHECKS_JOB } from "../services/media-intake-service.js" +import { JURISDICTION_DISCOVERY_JOB } from "../services/jurisdiction-service.js" +import { OUTREACH_DIGEST_JOB } from "../services/admin/jurisdiction-contacts-types.js" +import { INBOUND_SWEEP_JOB } from "../services/admin/inbound-jobs.js" +import { REPORT_AUTOFORWARD_JOB } from "../services/report-service.types.js" +import { DATA_EXPORT_JOB } from "../services/data-export-jobs.js" +import { CLEANUP_CANCEL_FANOUT_JOB } from "../services/cleanup-service.js" +import { + CLEANUP_GUEST_UPDATE_FANOUT_JOB, + GUEST_RETENTION_SWEEP_JOB, +} from "../services/guest-rsvp-service.js" +import { CHAT_ROOM_FANOUT_JOB } from "../services/chat-fanout-jobs.js" export interface PgBossJobsLogger { error(obj: unknown, msg?: string): void @@ -20,18 +32,25 @@ export interface PgBossJobsConfig { logger?: PgBossJobsLogger } +// Enqueued by the claim route and worked by the media worker, which owns the exported constant. +const ANON_HOLD_RELEASE_JOB = "anon.hold.release" + +// The media worker creates media.checks and anon.hold.release with this same policy. pg-boss keeps the +// last createQueue/updateQueue policy, so a mismatch silently breaks singletonKey dedup on those queues. +const SHARED_QUEUE_POLICY = "short" + export const API_QUEUE_NAMES = [ - "media.checks", - "jurisdiction.discovery", - "outreach.digest", - "inbound.sweep", - "report.autoforward", - "data.export", - "anon.hold.release", - "cleanup.cancel.fanout", - "cleanup.guest.update.fanout", - "guest.retention.sweep", - "chat.room.fanout", + MEDIA_CHECKS_JOB, + JURISDICTION_DISCOVERY_JOB, + OUTREACH_DIGEST_JOB, + INBOUND_SWEEP_JOB, + REPORT_AUTOFORWARD_JOB, + DATA_EXPORT_JOB, + ANON_HOLD_RELEASE_JOB, + CLEANUP_CANCEL_FANOUT_JOB, + CLEANUP_GUEST_UPDATE_FANOUT_JOB, + GUEST_RETENTION_SWEEP_JOB, + CHAT_ROOM_FANOUT_JOB, ...REGISTRATION_QUEUE_NAMES, ...COMMS_QUEUE_NAMES, ] as const @@ -41,17 +60,22 @@ type ApiQueueName = (typeof API_QUEUE_NAMES)[number] type QueueRetryPolicy = Required> const SECONDS_PER_MINUTE = 60 +const DATA_EXPORT_RETRY_LIMIT = 10 // A data export that fails on a mail credential or approved-sender fault has to wait for an operator to // fix the config. pg-boss's default (2 immediate retries) would rebuild and resend the whole export three // times within seconds and then drop the request, so it backs off from a minute to hours instead; the // handler records the request for an operator on the last attempt. const QUEUE_RETRY_POLICIES: Partial> = { - "data.export": { retryLimit: 10, retryDelay: SECONDS_PER_MINUTE, retryBackoff: true }, + [DATA_EXPORT_JOB]: { + retryLimit: DATA_EXPORT_RETRY_LIMIT, + retryDelay: SECONDS_PER_MINUTE, + retryBackoff: true, + }, } function queueOptions(name: ApiQueueName): PgBoss.Queue { - return { name, policy: "short", ...QUEUE_RETRY_POLICIES[name] } + return { name, policy: SHARED_QUEUE_POLICY, ...QUEUE_RETRY_POLICIES[name] } } function toSendOptions(opts?: EnqueueOptions): PgBoss.SendOptions { @@ -127,13 +151,12 @@ export class PgBossJobs implements Jobs { ) } - async complete(jobId: string): Promise { - void jobId + // The work loop completes or fails each job from its handler's outcome; nothing calls these. + async complete(_jobId: string): Promise { return Promise.resolve() } - async fail(jobId: string, _err?: unknown): Promise { - void jobId + async fail(_jobId: string, _err?: unknown): Promise { return Promise.resolve() } } diff --git a/services/api/src/adapters/jurisdiction-lookup.census.ts b/services/api/src/adapters/jurisdiction-lookup.census.ts index 1d240052..1a0fec0c 100644 --- a/services/api/src/adapters/jurisdiction-lookup.census.ts +++ b/services/api/src/adapters/jurisdiction-lookup.census.ts @@ -58,7 +58,13 @@ export interface CensusJurisdictionLookupOptions { fetchImpl?: typeof fetch } -const DEFAULT_TIMEOUT_MS = 2500 +export const CENSUS_DEFAULT_TIMEOUT_MS = 2500 + +const CENSUS_QUERY_PARAMS = { + benchmark: "Public_AR_Current", + vintage: "Current_Current", + format: "json", +} as const export class CensusJurisdictionLookup implements JurisdictionLookup { private readonly baseUrl: string @@ -67,7 +73,7 @@ export class CensusJurisdictionLookup implements JurisdictionLookup { constructor(options: CensusJurisdictionLookupOptions) { this.baseUrl = options.baseUrl - this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS + this.timeoutMs = options.timeoutMs ?? CENSUS_DEFAULT_TIMEOUT_MS this.fetchImpl = options.fetchImpl } @@ -76,9 +82,7 @@ export class CensusJurisdictionLookup implements JurisdictionLookup { const params = new URLSearchParams({ x: String(lng), y: String(lat), - benchmark: "Public_AR_Current", - vintage: "Current_Current", - format: "json", + ...CENSUS_QUERY_PARAMS, }) const result = await fetchJsonWithTimeout(`${this.baseUrl}?${params.toString()}`, { timeoutMs: this.timeoutMs, @@ -92,8 +96,8 @@ export class CensusJurisdictionLookup implements JurisdictionLookup { } export const JURISDICTION_LOOKUP_CACHE_TTL_MS = 5 * 60_000 -export const JURISDICTION_LOOKUP_CACHE_MAX_ENTRIES = 512 -export const JURISDICTION_LOOKUP_CACHE_DECIMALS = 4 +const JURISDICTION_LOOKUP_CACHE_MAX_ENTRIES = 512 +const JURISDICTION_LOOKUP_CACHE_DECIMALS = 4 export interface CachedJurisdictionLookupOptions { ttlMs?: number diff --git a/services/api/src/adapters/mail-failure.ts b/services/api/src/adapters/mail-failure.ts index 8f77a4c9..5c4f50f2 100644 --- a/services/api/src/adapters/mail-failure.ts +++ b/services/api/src/adapters/mail-failure.ts @@ -30,6 +30,18 @@ const APP_ERROR_CODES: ReadonlySet = new Set(Object.values(Error const MAX_CAUSE_DEPTH = 4 +export const SMTP_AUTH_FAILURE_CODE = "EAUTH" + +const SMTP_OVERSIZE_REPLY_CODES: ReadonlySet = new Set([552, 523]) + +const SMTP_PERMANENT_REPLY_CLASS = 5 + +const SMTP_TRANSIENT_REPLY_CLASS = 4 + +function smtpReplyClass(responseCode: number | undefined): number | undefined { + return responseCode === undefined ? undefined : Math.floor(responseCode / 100) +} + function readString(source: Record, key: string): string | undefined { const value = source[key] return typeof value === "string" ? value : undefined @@ -66,25 +78,25 @@ export function mailFailure(err: unknown): MailFailure { const command = readString(source, "command") const base = { responseCode, code, response, command } - if (code === "EAUTH") { + if (code === SMTP_AUTH_FAILURE_CODE) { return { ...base, kind: "auth", senderRejected: true } } if ( - responseCode === 552 || - responseCode === 523 || + (responseCode !== undefined && SMTP_OVERSIZE_REPLY_CODES.has(responseCode)) || (response !== undefined && OVERSIZE_RESPONSE_RE.test(response)) ) { return { ...base, kind: "oversize", senderRejected: false } } - if (responseCode !== undefined && responseCode >= 500 && responseCode < 600) { + const replyClass = smtpReplyClass(responseCode) + if (replyClass === SMTP_PERMANENT_REPLY_CLASS) { return isRecipientRejection(response) ? { ...base, kind: "permanent", senderRejected: false } : { ...base, kind: "auth", senderRejected: true } } - if (responseCode !== undefined && responseCode >= 400 && responseCode < 500) { + if (replyClass === SMTP_TRANSIENT_REPLY_CLASS) { return { ...base, kind: "transient", senderRejected: false } } diff --git a/services/api/src/adapters/mail-text.ts b/services/api/src/adapters/mail-text.ts index 18b2470f..98a080f8 100644 --- a/services/api/src/adapters/mail-text.ts +++ b/services/api/src/adapters/mail-text.ts @@ -3,6 +3,10 @@ const HEADER_VALUE_MAX = 998 // RFC 5322 line-length ceiling +const FALLBACK_MAIL_DOMAIN = "civfix.org" + +const HEADER_BREAKING_CHARS_RE = /[\r\n\0]/g + // `&` MUST be escaped first or the later entity ampersands get double-escaped. export function escapeHtml(s: string): string { return s @@ -29,7 +33,7 @@ export function domainOfOrNull(addr: string | null | undefined): string | null { // For Message-ID hosts and operator copy only. Never use it for a comparison decision: the fallback would // silently align an unparseable address with civfix.org. -export function domainOf(addr: string, fallback = "civfix.org"): string { +export function domainOf(addr: string, fallback = FALLBACK_MAIL_DOMAIN): string { return domainOfOrNull(addr) ?? fallback } @@ -37,6 +41,6 @@ export function domainOf(addr: string, fallback = "civfix.org"): string { // (SMTP header injection); this has bitten the OCI From/Reply-To path in prod. Not for envelope addresses, // which must be REJECTED on a bad char rather than silently sanitized. export function sanitizeHeaderValue(value: string, maxLength = HEADER_VALUE_MAX): string { - const stripped = value.replace(/[\r\n\0]/g, "") + const stripped = value.replace(HEADER_BREAKING_CHARS_RE, "") return stripped.length > maxLength ? stripped.slice(0, maxLength) : stripped } diff --git a/services/api/src/adapters/mailer.oci.ts b/services/api/src/adapters/mailer.oci.ts index 2ce10429..a5520ca6 100644 --- a/services/api/src/adapters/mailer.oci.ts +++ b/services/api/src/adapters/mailer.oci.ts @@ -3,7 +3,7 @@ import { AppError, ErrorCode, MailSendError } from "@civfix/shared" import type { Mailer, OutboundEmail, SentMail } from "@civfix/shared/interfaces" import type { Transporter } from "nodemailer" import { domainOf, escapeHtml, sanitizeHeaderValue } from "./mail-text.js" -import { mailFailure } from "./mail-failure.js" +import { mailFailure, SMTP_AUTH_FAILURE_CODE } from "./mail-failure.js" import { button, code, @@ -62,6 +62,16 @@ const consoleLogger: OciMailerLogger = { export const OCI_MAILER_DEFAULT_TIMEOUT_MS = 15_000 +const SMTPS_IMPLICIT_TLS_PORT = 465 + +const SECONDS_PER_MINUTE = 60 + +const DEFAULT_EVENT_TITLE = "the event" + +const GUEST_OTP_DEFAULT_MINUTES = "5" + +const DEFAULT_CTA_LABEL = "Open" + interface Rendered { subject: string text: string @@ -80,7 +90,7 @@ function classifyMailError(err: unknown, from: string): MailSendError { switch (failure.kind) { case "auth": - return failure.code === "EAUTH" + return failure.code === SMTP_AUTH_FAILURE_CODE ? new MailSendError( ErrorCode.INTERNAL, `Email not sent: the SMTP server rejected our credentials. Check ` + @@ -146,7 +156,7 @@ export class OciMailer implements Mailer { const transporter = nodemailer.createTransport({ host: this.config.host, port: this.config.port, - secure: this.config.port === 465, + secure: this.config.port === SMTPS_IMPLICIT_TLS_PORT, requireTLS: true, auth: { user: this.config.user, pass: this.config.pass }, connectionTimeout: timeout, @@ -217,7 +227,7 @@ export function renderOtp(passcode: string, locale: Locale): Rendered { code(passcode), paragraph( renderMessage(locale, "email.otp.body_expiry", { - minutes: String(Math.floor(OTP_TTL_SECONDS / 60)), + minutes: String(Math.floor(OTP_TTL_SECONDS / SECONDS_PER_MINUTE)), }), { muted: true }, ), @@ -226,110 +236,128 @@ export function renderOtp(passcode: string, locale: Locale): Rendered { return { subject, text, html } } -export function renderTemplate(template: string, vars: Record): Rendered { +export type TemplateVars = Record + +interface TemplateContent { + subject: string + blocks: EmailBlock[] +} + +type TemplateRenderer = (vars: TemplateVars, locale: Locale) => TemplateContent + +const TEMPLATES: ReadonlyMap = new Map([ + ["report_update", renderReportUpdate], + ["guest_otp", renderGuestOtp], + ["guest_confirmed", renderGuestConfirmed], + ["guest_promoted", renderGuestPromoted], + ["action", renderAction], +]) + +export function renderTemplate(template: string, vars: TemplateVars): Rendered { const locale = resolveLocale(typeof vars.locale === "string" ? vars.locale : undefined) - switch (template) { - case "report_update": { - const status = stringVar(vars, "status", "updated") - const subject = renderMessage(locale, "email.report_update.subject", { status }) - const message = renderMessage(locale, "email.report_update.body", { status }) - const { text, html } = renderEmailBody({ preheader: subject, blocks: [paragraph(message)] }) - return { subject, text, html } - } - case "guest_otp": { - const title = stringVar(vars, "title", "the event") - const passcode = stringVar(vars, "code", "") - const minutes = stringVar(vars, "minutes", "5") - const subject = renderMessage(locale, "email.guest_otp.subject", { title }) - const { text, html } = renderEmailBody({ - preheader: subject, - blocks: [ - paragraph(renderMessage(locale, "email.guest_otp.html_intro", { title })), - code(passcode), - paragraph(renderMessage(locale, "email.guest_otp.body_expiry", { minutes }), { - muted: true, - }), - ], - }) - return { subject, text, html } - } - case "guest_confirmed": { - const title = stringVar(vars, "title", "the event") - const when = optionalVar(vars, "when") - const place = optionalVar(vars, "place") - const cancelUrl = optionalVar(vars, "cancelUrl") - const subject = renderMessage(locale, "email.guest_confirmed.subject", { title }) - const details: Array<[string, string]> = [] - if (when !== undefined) details.push([renderMessage(locale, "email.event.when"), when]) - if (place !== undefined) details.push([renderMessage(locale, "email.event.where"), place]) - const blocks: EmailBlock[] = [heading(title)] - if (details.length > 0) blocks.push(kvTable(details)) - blocks.push(paragraph(renderMessage(locale, "email.guest_confirmed.checkin"))) - if (cancelUrl !== undefined) { - blocks.push( - paragraph(renderMessage(locale, "email.guest_confirmed.cancel_hint"), { muted: true }), - button(cancelUrl, renderMessage(locale, "email.guest_confirmed.cancel_cta")), - ) - } - const { text, html } = renderEmailBody({ preheader: subject, blocks }) - return { subject, text, html } - } - case "guest_promoted": { - const title = stringVar(vars, "title", "the event") - const when = stringVar(vars, "when", "") - const eventUrl = optionalVar(vars, "eventUrl") - const subject = renderMessage(locale, "email.guest_promoted.subject", { title }) - const blocks: EmailBlock[] = [ - paragraph(renderMessage(locale, "email.guest_promoted.intro", { title, when })), - ] - if (eventUrl !== undefined) { - blocks.push(button(eventUrl, renderMessage(locale, "email.guest_promoted.cta"))) - } - blocks.push(paragraph(renderMessage(locale, "email.guest_promoted.ignore"), { muted: true })) - const { text, html } = renderEmailBody({ preheader: subject, blocks }) - return { subject, text, html } - } - case "action": { - const subject = stringVar(vars, "subject", renderMessage(locale, "email.generic.subject")) - const blocks: EmailBlock[] = paragraphsVar(vars).map((p) => paragraph(p)) - const quoteHeading = optionalVar(vars, "quoteHeading") - const quoted = optionalVar(vars, "quote") - if (quoted !== undefined) { - if (quoteHeading !== undefined) blocks.push(heading(quoteHeading)) - blocks.push(quote(quoted)) - } - const ctaUrl = optionalVar(vars, "ctaUrl") - if (ctaUrl !== undefined) { - blocks.push(button(ctaUrl, stringVar(vars, "ctaLabel", "Open"))) - } - const note = optionalVar(vars, "note") - if (note !== undefined) blocks.push(paragraph(note, { muted: true })) - if (blocks.length === 0) { - blocks.push(paragraph(renderMessage(locale, "email.generic.body"))) - } - const { text, html } = renderEmailBody({ preheader: subject, blocks }) - return { subject, text, html } - } - default: { - const subject = stringVar(vars, "subject", renderMessage(locale, "email.generic.subject")) - const message = stringVar(vars, "message", renderMessage(locale, "email.generic.body")) - const { text, html } = renderEmailBody({ preheader: subject, blocks: [paragraph(message)] }) - return { subject, text, html } - } + const render = TEMPLATES.get(template) ?? renderGeneric + const { subject, blocks } = render(vars, locale) + const { text, html } = renderEmailBody({ preheader: subject, blocks }) + return { subject, text, html } +} + +function renderReportUpdate(vars: TemplateVars, locale: Locale): TemplateContent { + const status = stringVar(vars, "status", "updated") + const subject = renderMessage(locale, "email.report_update.subject", { status }) + const message = renderMessage(locale, "email.report_update.body", { status }) + return { subject, blocks: [paragraph(message)] } +} + +function renderGuestOtp(vars: TemplateVars, locale: Locale): TemplateContent { + const title = stringVar(vars, "title", DEFAULT_EVENT_TITLE) + const passcode = stringVar(vars, "code", "") + const minutes = stringVar(vars, "minutes", GUEST_OTP_DEFAULT_MINUTES) + return { + subject: renderMessage(locale, "email.guest_otp.subject", { title }), + blocks: [ + paragraph(renderMessage(locale, "email.guest_otp.html_intro", { title })), + code(passcode), + paragraph(renderMessage(locale, "email.guest_otp.body_expiry", { minutes }), { + muted: true, + }), + ], + } +} + +function renderGuestConfirmed(vars: TemplateVars, locale: Locale): TemplateContent { + const title = stringVar(vars, "title", DEFAULT_EVENT_TITLE) + const when = optionalVar(vars, "when") + const place = optionalVar(vars, "place") + const cancelUrl = optionalVar(vars, "cancelUrl") + const subject = renderMessage(locale, "email.guest_confirmed.subject", { title }) + const details: Array<[string, string]> = [] + if (when !== undefined) details.push([renderMessage(locale, "email.event.when"), when]) + if (place !== undefined) details.push([renderMessage(locale, "email.event.where"), place]) + const blocks: EmailBlock[] = [heading(title)] + if (details.length > 0) blocks.push(kvTable(details)) + blocks.push(paragraph(renderMessage(locale, "email.guest_confirmed.checkin"))) + if (cancelUrl !== undefined) { + blocks.push( + paragraph(renderMessage(locale, "email.guest_confirmed.cancel_hint"), { muted: true }), + button(cancelUrl, renderMessage(locale, "email.guest_confirmed.cancel_cta")), + ) } + return { subject, blocks } +} + +function renderGuestPromoted(vars: TemplateVars, locale: Locale): TemplateContent { + const title = stringVar(vars, "title", DEFAULT_EVENT_TITLE) + const when = stringVar(vars, "when", "") + const eventUrl = optionalVar(vars, "eventUrl") + const subject = renderMessage(locale, "email.guest_promoted.subject", { title }) + const blocks: EmailBlock[] = [ + paragraph(renderMessage(locale, "email.guest_promoted.intro", { title, when })), + ] + if (eventUrl !== undefined) { + blocks.push(button(eventUrl, renderMessage(locale, "email.guest_promoted.cta"))) + } + blocks.push(paragraph(renderMessage(locale, "email.guest_promoted.ignore"), { muted: true })) + return { subject, blocks } +} + +function renderAction(vars: TemplateVars, locale: Locale): TemplateContent { + const subject = stringVar(vars, "subject", renderMessage(locale, "email.generic.subject")) + const blocks: EmailBlock[] = paragraphsVar(vars).map((p) => paragraph(p)) + const quoteHeading = optionalVar(vars, "quoteHeading") + const quoted = optionalVar(vars, "quote") + if (quoted !== undefined) { + if (quoteHeading !== undefined) blocks.push(heading(quoteHeading)) + blocks.push(quote(quoted)) + } + const ctaUrl = optionalVar(vars, "ctaUrl") + if (ctaUrl !== undefined) { + blocks.push(button(ctaUrl, stringVar(vars, "ctaLabel", DEFAULT_CTA_LABEL))) + } + const note = optionalVar(vars, "note") + if (note !== undefined) blocks.push(paragraph(note, { muted: true })) + if (blocks.length === 0) { + blocks.push(paragraph(renderMessage(locale, "email.generic.body"))) + } + return { subject, blocks } +} + +function renderGeneric(vars: TemplateVars, locale: Locale): TemplateContent { + const subject = stringVar(vars, "subject", renderMessage(locale, "email.generic.subject")) + const message = stringVar(vars, "message", renderMessage(locale, "email.generic.body")) + return { subject, blocks: [paragraph(message)] } } -function stringVar(vars: Record, key: string, fallback: string): string { +function stringVar(vars: TemplateVars, key: string, fallback: string): string { const v = vars[key] return typeof v === "string" && v.length > 0 ? v : fallback } -function optionalVar(vars: Record, key: string): string | undefined { +function optionalVar(vars: TemplateVars, key: string): string | undefined { const v = vars[key] return typeof v === "string" && v.length > 0 ? v : undefined } -function paragraphsVar(vars: Record): string[] { +function paragraphsVar(vars: TemplateVars): string[] { const v = vars.paragraphs if (!Array.isArray(v)) return [] return v.filter((p): p is string => typeof p === "string" && p.length > 0) diff --git a/services/api/src/adapters/net-ipv6.ts b/services/api/src/adapters/net-ipv6.ts index 6074d734..bc385a6c 100644 --- a/services/api/src/adapters/net-ipv6.ts +++ b/services/api/src/adapters/net-ipv6.ts @@ -5,6 +5,8 @@ * compressed address, exactly 8 hextets, and its own per-hextet check; a lenient one just reads `hextets`. */ +const IPV6_HEXTET_COUNT = 8 + export interface Ipv6Expansion { hextets: string[] runs: number @@ -25,7 +27,7 @@ export function expandIpv6Hextets(addr: string): Ipv6Expansion { if (tail === null) { return { hextets: head, runs, fill: 0 } } - const fill = 8 - head.length - tail.length + const fill = IPV6_HEXTET_COUNT - head.length - tail.length return { hextets: [...head, ...Array(Math.max(0, fill)).fill("0"), ...tail], runs, diff --git a/services/api/src/adapters/push-apns.ts b/services/api/src/adapters/push-apns.ts index 6f813a0e..cbc7139b 100644 --- a/services/api/src/adapters/push-apns.ts +++ b/services/api/src/adapters/push-apns.ts @@ -1,8 +1,14 @@ import type { PushSenderConfig, PushLogger, PlatformDispatcher } from "./push-sender.js" import { hashForLog } from "./push-sender.js" +const APNS_GONE_STATUS = "410" + +const APNS_PRUNE_REASONS: ReadonlySet = new Set(["Unregistered", "BadDeviceToken"]) + +const APNS_SOUND = "default" + export function isApnsPruneFailure(status: string, reason: string): boolean { - return status === "410" || reason === "Unregistered" || reason === "BadDeviceToken" + return status === APNS_GONE_STATUS || APNS_PRUNE_REASONS.has(reason) } export function makeApnsDispatcher( @@ -33,7 +39,7 @@ export function makeApnsDispatcher( title: payload.title, ...(payload.body !== undefined ? { body: payload.body } : {}), } - note.sound = "default" + note.sound = APNS_SOUND note.payload = { ...(payload.data ?? {}), ...(payload.link !== undefined ? { link: payload.link } : {}), diff --git a/services/api/src/adapters/push-expo.ts b/services/api/src/adapters/push-expo.ts index b1c7cfe3..355cba94 100644 --- a/services/api/src/adapters/push-expo.ts +++ b/services/api/src/adapters/push-expo.ts @@ -15,9 +15,15 @@ const EXPO_CHUNK = 100 const EXPO_TIMEOUT_MS = 4000 /** Expo asks senders to back off and retry a 429 / 5xx rather than dropping the batch. */ const EXPO_RETRY_DELAY_MS = 250 +const EXPO_SOUND = "default" +const EXPO_TOKEN_PREFIXES = ["ExponentPushToken[", "ExpoPushToken["] as const +const EXPO_UNREGISTERED_ERROR = "DeviceNotRegistered" + +const HTTP_TOO_MANY_REQUESTS = 429 +const HTTP_SERVER_ERROR_MIN = 500 function isRetryableStatus(status: number): boolean { - return status === 429 || status >= 500 + return status === HTTP_TOO_MANY_REQUESTS || status >= HTTP_SERVER_ERROR_MIN } function sleep(ms: number): Promise { @@ -25,7 +31,7 @@ function sleep(ms: number): Promise { } export function isExpoPushToken(token: string): boolean { - return token.startsWith("ExponentPushToken[") || token.startsWith("ExpoPushToken[") + return EXPO_TOKEN_PREFIXES.some((prefix) => token.startsWith(prefix)) } export interface ExpoTicket { @@ -47,7 +53,7 @@ export function collectExpoInvalidTokens( tickets.forEach((ticket, idx) => { if (ticket?.status !== "error") return const token = chunk[idx] - if (ticket.details?.error === "DeviceNotRegistered" && typeof token === "string") { + if (ticket.details?.error === EXPO_UNREGISTERED_ERROR && typeof token === "string") { invalid.push(token) } else { logger.warn( @@ -80,7 +86,7 @@ export function makeExpoDispatcher(config: ExpoPushConfig, logger: PushLogger): to, title: payload.title, ...(payload.body !== undefined ? { body: payload.body } : {}), - sound: "default", + sound: EXPO_SOUND, ...(hasData ? { data } : {}), })) const send = (): Promise> => diff --git a/services/api/src/adapters/push-fcm.ts b/services/api/src/adapters/push-fcm.ts index 071795a6..f2c5d7b1 100644 --- a/services/api/src/adapters/push-fcm.ts +++ b/services/api/src/adapters/push-fcm.ts @@ -4,6 +4,8 @@ import { hashForLog } from "./push-sender.js" const FCM_MULTICAST_MAX = 500 +const FCM_APP_NAME = "civfix-push" + const PRUNE_CODES = new Set([ "messaging/registration-token-not-registered", "messaging/invalid-registration-token", @@ -19,13 +21,13 @@ export function isFcmPruneCode(code: string): boolean { type FcmSendResponse = { success: boolean; error?: { code?: string; message?: string } } -export interface FcmSliceVerdict { +interface FcmSliceVerdict { invalidTokens: string[] payloadRejected: { message: string | undefined } | null failures: { code: string; token: string | undefined }[] } -export function classifyFcmResponses( +function classifyFcmResponses( tokens: readonly string[], responses: readonly FcmSendResponse[], ): FcmSliceVerdict { @@ -50,7 +52,6 @@ export function makeFcmDispatcher( fcm: NonNullable, logger: PushLogger, ): PlatformDispatcher { - const appName = "civfix-push" let initPromise: Promise<{ messaging: any; deleteApp: () => Promise }> | null = null async function getMessaging() { @@ -59,7 +60,7 @@ export function makeFcmDispatcher( const admin = await import("firebase-admin/app") const messaging = await import("firebase-admin/messaging") const serviceAccount = JSON.parse(fcm.serviceAccountJson) as Record - const existing = admin.getApps().find((a) => a.name === appName) + const existing = admin.getApps().find((a) => a.name === FCM_APP_NAME) const app = existing ?? admin.initializeApp( @@ -67,7 +68,7 @@ export function makeFcmDispatcher( credential: admin.cert(serviceAccount as never), ...(fcm.projectId !== undefined ? { projectId: fcm.projectId } : {}), }, - appName, + FCM_APP_NAME, ) return { messaging: messaging.getMessaging(app), deleteApp: () => admin.deleteApp(app) } })() diff --git a/services/api/src/adapters/push-sender.ts b/services/api/src/adapters/push-sender.ts index 0f3d08d4..3235603b 100644 --- a/services/api/src/adapters/push-sender.ts +++ b/services/api/src/adapters/push-sender.ts @@ -40,12 +40,16 @@ export interface PushLogger { error(obj: unknown, msg?: string): void } +const LOG_HASH_HEX_CHARS = 12 + export function hashForLog(value: string): string { - return createHash("sha256").update(value).digest("hex").slice(0, 12) + return createHash("sha256").update(value).digest("hex").slice(0, LOG_HASH_HEX_CHARS) } const ACTIVE_TOKEN_SCAN_CAP_PER_USER = 20 +const NATIVE_PLATFORMS: readonly PushPlatform[] = ["ios", "android", "web"] + // Only for callers that construct the sender without a logger; the API container always injects // its pino logger so redaction and request context apply. const consoleLogger: PushLogger = { @@ -83,6 +87,7 @@ export const PUSH_MAX_PER_USER_PER_MINUTE = 60 const PUSH_RATE_WINDOW_SECONDS = 60 const PUSH_RATE_CHECK_CONCURRENCY = 16 +const PUSH_RATE_KEY_PREFIX = "push:rate:" export async function allowedByPushRate( userIds: string[], @@ -95,7 +100,10 @@ export async function allowedByPushRate( PUSH_RATE_CHECK_CONCURRENCY, async (userId): Promise => { try { - const used = await counters.incr(`push:rate:${userId}`, PUSH_RATE_WINDOW_SECONDS) + const used = await counters.incr( + `${PUSH_RATE_KEY_PREFIX}${userId}`, + PUSH_RATE_WINDOW_SECONDS, + ) return used <= PUSH_MAX_PER_USER_PER_MINUTE } catch (err) { logger.warn({ err }, "push: per-user rate counter unavailable; allowing") @@ -158,33 +166,46 @@ export class MultiPushSender implements PushSender { if (tokens.length === 0) return const dispatchers = this.getDispatchers() - const invalidAll: string[] = [] + const expoInvalid = await this.dispatchExpo(dispatchers.expo, tokens, payload) + const nativeInvalid = await this.dispatchNative(dispatchers, tokens, payload) + const invalidAll = [...expoInvalid, ...nativeInvalid] + if (invalidAll.length > 0) await this.pruneTokens(invalidAll) + } + private async dispatchExpo( + expo: PlatformDispatcher | undefined, + tokens: ActiveToken[], + payload: PushPayload, + ): Promise { const expoTokens = [ ...new Set(tokens.filter((t) => isExpoPushToken(t.token)).map((t) => t.token)), ] - if (expoTokens.length > 0) { - const expo = dispatchers.expo - if (expo) { - try { - const { invalidTokens } = await expo(expoTokens, payload) - for (const t of invalidTokens) invalidAll.push(t) - } catch (err) { - this.logger.error({ err }, "push: expo dispatch failed") - } - } else { - this.logger.warn( - { count: expoTokens.length }, - "push: expo tokens present but no expo dispatcher; skipping", - ) - } + if (expoTokens.length === 0) return [] + if (!expo) { + this.logger.warn( + { count: expoTokens.length }, + "push: expo tokens present but no expo dispatcher; skipping", + ) + return [] } + try { + const { invalidTokens } = await expo(expoTokens, payload) + return invalidTokens + } catch (err) { + this.logger.error({ err }, "push: expo dispatch failed") + return [] + } + } - const rawTokens = tokens.filter((t) => !isExpoPushToken(t.token)) - const byPlatform = groupByPlatform(rawTokens) - const platforms: PushPlatform[] = ["ios", "android", "web"] + private async dispatchNative( + dispatchers: PushDispatchers, + tokens: ActiveToken[], + payload: PushPayload, + ): Promise { + const byPlatform = groupByPlatform(tokens.filter((t) => !isExpoPushToken(t.token))) + const invalid: string[] = [] await Promise.all( - platforms.map(async (platform) => { + NATIVE_PLATFORMS.map(async (platform) => { const platformTokens = byPlatform[platform] if (!platformTokens || platformTokens.length === 0) return const dispatcher = dispatchers[platform] @@ -197,16 +218,14 @@ export class MultiPushSender implements PushSender { } try { const { invalidTokens } = await dispatcher(platformTokens, payload) - for (const t of invalidTokens) invalidAll.push(t) + invalid.push(...invalidTokens) } catch (err) { this.logger.error({ err, platform }, "push: platform dispatch failed") } }), ) - - if (invalidAll.length > 0) await this.pruneTokens(invalidAll) + return invalid } - private async loadActiveTokens(userIds: string[]): Promise { const rows = await this.db .select({ @@ -261,16 +280,4 @@ export function groupByPlatform(tokens: ActiveToken[]): Record = new Set([404, 410]) + +function agentKey(address: string, family: 4 | 6): string { + return `${family}|${address}` +} + export interface PinnedAgentPool { get(address: string, family: 4 | 6): Agent drop(address: string, family: 4 | 6): void @@ -66,7 +73,7 @@ export function makePinnedAgentPool( } function get(address: string, family: 4 | 6): Agent { - const key = `${family}|${address}` + const key = agentKey(address, family) const cached = cache.get(key) if (cached) { cache.delete(key) @@ -98,7 +105,7 @@ export function makePinnedAgentPool( } function drop(address: string, family: 4 | 6): void { - const key = `${family}|${address}` + const key = agentKey(address, family) const agent = cache.get(key) if (!agent) return cache.delete(key) @@ -128,11 +135,11 @@ const agentPool = makePinnedAgentPool() const WEB_PUSH_CONCURRENCY = 16 -export const WEB_PUSH_REQUEST_TIMEOUT_MS = 8_000 +const WEB_PUSH_REQUEST_TIMEOUT_MS = 8_000 -export const WEB_PUSH_DEADLINE_SLACK_MS = 2_000 +const WEB_PUSH_DEADLINE_SLACK_MS = 2_000 -export const WEB_PUSH_BATCH_BUDGET_MS = 32_000 +const WEB_PUSH_BATCH_BUDGET_MS = 32_000 export class WebPushDeadlineError extends Error { constructor(ms: number) { @@ -182,6 +189,57 @@ export function makeWebPushDispatcher( return webpushPromise } + // Pushes onto invalidTokens directly (rather than returning a verdict) so concurrent sends record + // their prunes in the same order as they settle. + async function deliverToToken( + wp: Pick, + token: string, + body: string, + invalidTokens: string[], + ): Promise { + const subscription = parseSubscription(token) + if (subscription === null) { + invalidTokens.push(token) + return + } + const target = await resolveSafePushTarget(subscription.endpoint, resolveAddresses) + if (target === null) { + logger.warn( + { endpointHash: hashForLog(subscription.endpoint) }, + "push(webpush): refusing unsafe/internal endpoint; pruning", + ) + invalidTokens.push(token) + return + } + try { + await withDeadline( + wp.sendNotification(subscription, body, { + agent: pool.get(target.address, target.family), + timeout: requestTimeoutMs, + }) as Promise, + deadlineMs, + () => pool.drop(target.address, target.family), + ) + } catch (err) { + if (err instanceof WebPushDeadlineError) { + logger.warn( + { deadlineMs, endpointHash: hashForLog(subscription.endpoint) }, + "push(webpush): endpoint exceeded the hard deadline; request torn down", + ) + return + } + const { prune, statusCode } = classifyWebPushError(err) + if (prune) { + invalidTokens.push(token) + } else { + logger.warn( + { statusCode, endpointHash: hashForLog(subscription.endpoint) }, + "push(webpush): delivery failure", + ) + } + } + } + const dispatch: PlatformDispatcher = async (tokens, payload) => { const wp = await getWebPush() const body = JSON.stringify({ @@ -199,47 +257,7 @@ export function makeWebPushDispatcher( overBudget += 1 return } - const subscription = parseSubscription(token) - if (subscription === null) { - invalidTokens.push(token) - return - } - const target = await resolveSafePushTarget(subscription.endpoint, resolveAddresses) - if (target === null) { - logger.warn( - { endpointHash: hashForLog(subscription.endpoint) }, - "push(webpush): refusing unsafe/internal endpoint; pruning", - ) - invalidTokens.push(token) - return - } - try { - await withDeadline( - wp.sendNotification(subscription, body, { - agent: pool.get(target.address, target.family), - timeout: requestTimeoutMs, - }) as Promise, - deadlineMs, - () => pool.drop(target.address, target.family), - ) - } catch (err) { - if (err instanceof WebPushDeadlineError) { - logger.warn( - { deadlineMs, endpointHash: hashForLog(subscription.endpoint) }, - "push(webpush): endpoint exceeded the hard deadline; request torn down", - ) - return - } - const { prune, statusCode } = classifyWebPushError(err) - if (prune) { - invalidTokens.push(token) - } else { - logger.warn( - { statusCode, endpointHash: hashForLog(subscription.endpoint) }, - "push(webpush): delivery failure", - ) - } - } + await deliverToToken(wp, token, body, invalidTokens) }) if (overBudget > 0) { logger.warn( @@ -266,5 +284,8 @@ export function classifyWebPushError(err: unknown): { ? (err as { statusCode?: unknown }).statusCode : undefined const statusCode = typeof raw === "number" ? raw : undefined - return { prune: statusCode === 404 || statusCode === 410, statusCode } + return { + prune: statusCode !== undefined && WEB_PUSH_PRUNE_STATUSES.has(statusCode), + statusCode, + } } diff --git a/services/api/src/adapters/redis-incr.ts b/services/api/src/adapters/redis-incr.ts index 6b43bb75..3642f7ac 100644 --- a/services/api/src/adapters/redis-incr.ts +++ b/services/api/src/adapters/redis-incr.ts @@ -14,6 +14,8 @@ const DECRBY_FLOOR_LUA = const DECRBY_COMMAND_NAME = "civfixDecrByFloor" +const MS_PER_SECOND = 1000 + type IncrExpire = (key: string, ttlSeconds: number) => Promise type IncrByExpire = (key: string, by: number, ttlSeconds: number) => Promise @@ -30,27 +32,35 @@ export function attachAtomicIncr(redis: RedisClient): IncrExpire { return (key: string, ttlSeconds: number): Promise => incrBy(key, 1, ttlSeconds) } -export function attachAtomicIncrBy(redis: RedisClient): IncrByExpire { +function defineOnce( + redis: RedisClient, + name: typeof INCRBY_COMMAND_NAME | typeof DECRBY_COMMAND_NAME, + lua: string, +): WithIncrExpire { const client = redis as WithIncrExpire - if (typeof client[INCRBY_COMMAND_NAME] !== "function") { - redis.defineCommand(INCRBY_COMMAND_NAME, { numberOfKeys: 1, lua: INCRBY_EXPIRE_LUA }) + if (typeof client[name] !== "function") { + redis.defineCommand(name, { numberOfKeys: 1, lua }) } + return client +} + +function wholeAmount(by: number): string { + return String(Math.max(0, Math.floor(by))) +} + +export function attachAtomicIncrBy(redis: RedisClient): IncrByExpire { + const client = defineOnce(redis, INCRBY_COMMAND_NAME, INCRBY_EXPIRE_LUA) return async (key: string, by: number, ttlSeconds: number): Promise => { - const amount = Math.max(0, Math.floor(by)) - const ttlMs = Math.max(1, Math.ceil(ttlSeconds)) * 1000 - const result = await client[INCRBY_COMMAND_NAME]!(key, String(amount), ttlMs) + const ttlMs = Math.max(1, Math.ceil(ttlSeconds)) * MS_PER_SECOND + const result = await client[INCRBY_COMMAND_NAME]!(key, wholeAmount(by), ttlMs) return Number(result) } } export function attachAtomicDecrBy(redis: RedisClient): DecrFloor { - const client = redis as WithIncrExpire - if (typeof client[DECRBY_COMMAND_NAME] !== "function") { - redis.defineCommand(DECRBY_COMMAND_NAME, { numberOfKeys: 1, lua: DECRBY_FLOOR_LUA }) - } + const client = defineOnce(redis, DECRBY_COMMAND_NAME, DECRBY_FLOOR_LUA) return async (key: string, by: number): Promise => { - const amount = Math.max(0, Math.floor(by)) - const result = await client[DECRBY_COMMAND_NAME]!(key, String(amount)) + const result = await client[DECRBY_COMMAND_NAME]!(key, wholeAmount(by)) return Number(result) } } diff --git a/services/api/src/adapters/redis.ts b/services/api/src/adapters/redis.ts index 2b266ad2..efda9790 100644 --- a/services/api/src/adapters/redis.ts +++ b/services/api/src/adapters/redis.ts @@ -4,6 +4,8 @@ export type RedisClient = Redis export const REDIS_COMMAND_TIMEOUT_MS = 5000 +const REDIS_MAX_RETRIES_PER_REQUEST = 2 + export interface MakeRedisOptions { onError?: (err: Error) => void commandTimeout?: number @@ -21,7 +23,7 @@ export function makeRedis(redisUrl: string, opts: MakeRedisOptions = {}): RedisC } const client = new Redis(redisUrl, { lazyConnect: true, - maxRetriesPerRequest: 2, + maxRetriesPerRequest: REDIS_MAX_RETRIES_PER_REQUEST, commandTimeout: opts.commandTimeout ?? REDIS_COMMAND_TIMEOUT_MS, enableReadyCheck: true, enableAutoPipelining: true, diff --git a/services/api/src/adapters/reverse-geocode.mapbox.ts b/services/api/src/adapters/reverse-geocode.mapbox.ts index 73dec65a..9c6b98c3 100644 --- a/services/api/src/adapters/reverse-geocode.mapbox.ts +++ b/services/api/src/adapters/reverse-geocode.mapbox.ts @@ -15,6 +15,15 @@ import { fetchJsonOrNull } from "./http-fetch.js" const MAPBOX_REVERSE_URL = "https://api.mapbox.com/search/geocode/v6/reverse" const DEFAULT_TIMEOUT_MS = 4000 +const HOME_COUNTRY_CODE = "US" +const PROVIDER_NAME = "mapbox" +const LEADING_DIGIT_RE = /^\d/ + +const MAPBOX_QUERY_PARAMS = { + limit: "1", + types: "address", + language: "en", +} as const interface MapboxReverseContext { address?: { name?: string; address_number?: string; street_name?: string } @@ -47,7 +56,7 @@ export function formatMapboxReverse(p: MapboxReverseProps): string | null { const tail = [ ctx.place?.name && ctx.place.name !== primary ? ctx.place.name : null, region, - cc && cc.toUpperCase() !== "US" ? (ctx.country?.name ?? cc.toUpperCase()) : null, + cc && cc.toUpperCase() !== HOME_COUNTRY_CODE ? (ctx.country?.name ?? cc.toUpperCase()) : null, ].filter((v): v is string => !!v) return [primary, ...tail].join(", ") } @@ -57,7 +66,7 @@ export function formatMapboxReverse(p: MapboxReverseProps): string | null { * address. A digit can lead any POI name ("24 Hour Fitness"), and a POI is not a rooftop. */ function startsWithHouseNumber(name: string | undefined): boolean { - return name !== undefined && /^\d/.test(name.trim()) + return name !== undefined && LEADING_DIGIT_RE.test(name.trim()) } export function mapboxPrecision(p: MapboxReverseProps): AddressPrecision | null { @@ -85,9 +94,7 @@ export function makeMapboxReverseGeocode(opts: MapboxReverseOptions): ReverseGeo u.searchParams.set("longitude", String(lng)) u.searchParams.set("latitude", String(lat)) u.searchParams.set("access_token", opts.token) - u.searchParams.set("limit", "1") - u.searchParams.set("types", "address") - u.searchParams.set("language", "en") + for (const [key, value] of Object.entries(MAPBOX_QUERY_PARAMS)) u.searchParams.set(key, value) url = u.toString() } catch { return null @@ -102,6 +109,6 @@ export function makeMapboxReverseGeocode(opts: MapboxReverseOptions): ReverseGeo const precision = mapboxPrecision(props) if (precision === null) return null const line = formatMapboxReverse(props) - return line === null ? null : { line, precision, provider: "mapbox" } + return line === null ? null : { line, precision, provider: PROVIDER_NAME } } } diff --git a/services/api/src/adapters/reverse-geocode.photon.ts b/services/api/src/adapters/reverse-geocode.photon.ts index dcb7284a..5f661c38 100644 --- a/services/api/src/adapters/reverse-geocode.photon.ts +++ b/services/api/src/adapters/reverse-geocode.photon.ts @@ -37,6 +37,16 @@ const LANDMARK_RADIUS_M = 60 const EARTH_RADIUS_M = 6_371_000 +const PROVIDER_NAME = "photon" +const RESPONSE_LANGUAGE = "en" + +const HOME_COUNTRY_NAMES: ReadonlySet = new Set([ + "United States", + "United States of America", +]) + +const RESIDENTIAL_PLACE_VALUES: ReadonlySet = new Set(["house", "farm"]) + const LANDMARK_KEYS = new Set([ "amenity", "leisure", @@ -82,9 +92,7 @@ function addressLine(p: PhotonReverseProps, primary: string): string { const tail = [ p.city && p.city !== primary ? p.city : null, p.state, - p.country && p.country !== "United States" && p.country !== "United States of America" - ? p.country - : null, + p.country && !HOME_COUNTRY_NAMES.has(p.country) ? p.country : null, ].filter((v): v is string => !!v) return [primary, ...tail].join(", ") } @@ -98,8 +106,9 @@ export function formatPhotonReverse(p: PhotonReverseProps): string | null { export function isResidentialName(p: PhotonReverseProps): boolean { if (p.osm_key === "building") return true - if (p.osm_key === "place" && (p.osm_value === "house" || p.osm_value === "farm")) return true - return false + return ( + p.osm_key === "place" && p.osm_value !== undefined && RESIDENTIAL_PLACE_VALUES.has(p.osm_value) + ) } function namedRoad(p: PhotonReverseProps): string | null { @@ -120,10 +129,17 @@ interface Candidate { meters: number } +export type ComposedLine = { line: string; precision: AddressPrecision } + export function composePhotonReverse( features: PhotonFeature[], at: { lat: number; lng: number }, -): { line: string; precision: AddressPrecision } | null { +): ComposedLine | null { + const candidates = nearestFirst(features, at) + return exactAddress(candidates) ?? nearbyRoads(candidates) ?? nearbyLandmark(candidates) +} + +function nearestFirst(features: PhotonFeature[], at: { lat: number; lng: number }): Candidate[] { const candidates: Candidate[] = [] for (const f of features) { const props = f.properties @@ -135,14 +151,17 @@ export function composePhotonReverse( : Number.POSITIVE_INFINITY candidates.push({ props, meters }) } - candidates.sort((a, b) => a.meters - b.meters) + return candidates.sort((a, b) => a.meters - b.meters) +} +function exactAddress(candidates: Candidate[]): ComposedLine | null { const exact = candidates.find((c) => !!c.props.housenumber?.trim() && !!c.props.street?.trim()) - if (exact) { - const line = formatPhotonReverse(exact.props) - if (line) return { line, precision: "street" } - } + if (!exact) return null + const line = formatPhotonReverse(exact.props) + return line ? { line, precision: "street" } : null +} +function nearbyRoads(candidates: Candidate[]): ComposedLine | null { const roads: string[] = [] let roadProps: PhotonReverseProps | null = null for (const c of candidates) { @@ -152,18 +171,18 @@ export function composePhotonReverse( roads.push(road) roadProps ??= c.props } - if (roadProps !== null && roads[0] !== undefined) { - const primary = roads.length >= 2 ? `${roads[0]} & ${roads[1]}` : roads[0] - return { line: addressLine(roadProps, primary), precision: "intersection" } - } + if (roadProps === null || roads[0] === undefined) return null + const primary = roads.length >= 2 ? `${roads[0]} & ${roads[1]}` : roads[0] + return { line: addressLine(roadProps, primary), precision: "intersection" } +} +function nearbyLandmark(candidates: Candidate[]): ComposedLine | null { for (const c of candidates) { if (c.meters > LANDMARK_RADIUS_M) continue const name = landmarkName(c.props) if (name === null) continue return { line: addressLine(c.props, name), precision: "landmark" } } - return null } @@ -186,7 +205,7 @@ export function makePhotonReverseGeocode(opts: PhotonReverseOptions = {}): Rever const u = new URL(baseUrl) u.searchParams.set("lat", String(lat)) u.searchParams.set("lon", String(lng)) - u.searchParams.set("lang", "en") + u.searchParams.set("lang", RESPONSE_LANGUAGE) u.searchParams.set("limit", String(REVERSE_LIMIT)) u.searchParams.set("radius", String(REVERSE_RADIUS_KM)) url = u.toString() @@ -201,6 +220,6 @@ export function makePhotonReverseGeocode(opts: PhotonReverseOptions = {}): Rever init: { headers: { Accept: "application/json" } }, }) const composed = composePhotonReverse(data?.features ?? [], { lat, lng }) - return composed === null ? null : { ...composed, provider: "photon" } + return composed === null ? null : { ...composed, provider: PROVIDER_NAME } } } diff --git a/services/api/src/adapters/routing-provider.ts b/services/api/src/adapters/routing-provider.ts index 9c6dfa2a..fd3061a8 100644 --- a/services/api/src/adapters/routing-provider.ts +++ b/services/api/src/adapters/routing-provider.ts @@ -9,16 +9,9 @@ import { AppError } from "@civfix/shared" import type { RoutingProvider, RouteStop, RouteOpts, Route } from "@civfix/shared/interfaces" import type { LatLng } from "@civfix/shared" -export interface RoutingProviderConfig { - /** Base URL of the routing engine (OSRM/Valhalla-compatible). */ - baseUrl?: string -} - const NOT_IMPL = "adapter not implemented: routing-provider" export class HttpRoutingProvider implements RoutingProvider { - constructor(_config: RoutingProviderConfig = {}) {} - matrix(_points: LatLng[]): Promise { return Promise.reject(AppError.internal(NOT_IMPL)) } diff --git a/services/api/src/adapters/sms-twilio.ts b/services/api/src/adapters/sms-twilio.ts index bc758255..b5f859c9 100644 --- a/services/api/src/adapters/sms-twilio.ts +++ b/services/api/src/adapters/sms-twilio.ts @@ -6,12 +6,15 @@ const TWILIO_API_ROOT = "https://api.twilio.com/2010-04-01/Accounts" export const SMS_SEND_TIMEOUT_MS = 10_000 -export const TWILIO_OPTED_OUT_CODE = 21610 +const TWILIO_OPTED_OUT_CODE = 21610 -export const TWILIO_INVALID_NUMBER_CODES: ReadonlySet = new Set([21211, 21614]) +const TWILIO_INVALID_NUMBER_CODES: ReadonlySet = new Set([21211, 21614]) const MAX_ERROR_BODY_BYTES = 64 * 1024 +const HTTP_TOO_MANY_REQUESTS = 429 +const HTTP_SERVER_ERROR_MIN = 500 + export interface TwilioSmsSenderConfig { accountSid: string authToken: string @@ -160,7 +163,7 @@ export function classifyTwilioError(status: number, payload: TwilioMessageRespon `Text message not sent: that phone number is not a valid mobile number.${detail}`, ) } - if (status === 429 || status >= 500) { + if (status === HTTP_TOO_MANY_REQUESTS || status >= HTTP_SERVER_ERROR_MIN) { return smsFailure( "temporary", `Text message not sent: the SMS provider is temporarily unavailable.${detail}`, diff --git a/services/api/src/adapters/storage.local.ts b/services/api/src/adapters/storage.local.ts index e9ed177a..94859ec1 100644 --- a/services/api/src/adapters/storage.local.ts +++ b/services/api/src/adapters/storage.local.ts @@ -15,13 +15,12 @@ import type { } from "@civfix/shared/interfaces" export const LOCAL_STORAGE_ROUTE_PREFIX = "/_local-storage" -export const LOCAL_STORAGE_PUT_TTL_SEC = 15 * 60 -export const LOCAL_STORAGE_DEFAULT_GET_TTL_SEC = 15 * 60 +const LOCAL_STORAGE_PUT_TTL_SEC = 15 * 60 +const LOCAL_STORAGE_DEFAULT_GET_TTL_SEC = 15 * 60 export const LOCAL_STORAGE_DEV_SIGNING_KEY = "dev-insecure-local-storage-signing-key-do-not-use-in-prod" -export const LOCAL_STORAGE_NAMESPACES = ["media", "inbound"] as const -export type LocalStorageNamespace = (typeof LOCAL_STORAGE_NAMESPACES)[number] +export type LocalStorageNamespace = "media" | "inbound" const OBJECT_KEY_MAX_LENGTH = 512 const OBJECT_KEY_SEGMENT = "[A-Za-z0-9_][A-Za-z0-9._-]*" @@ -30,6 +29,10 @@ export const SIGNATURE_PATTERN = /^[0-9a-f]{64}$/ const DEFAULT_CONTENT_TYPE = "application/octet-stream" const DEFAULT_LIST_LIMIT = 1000 const LOOPBACK_HOSTS = new Set(["localhost", "127.0.0.1", "::1", "[::1]"]) +const LOOPBACK_IPV4_PATTERN = /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ +const TRAILING_SLASHES_RE = /\/+$/ +const STAGED_FILE_RANDOM_BYTES = 16 +const MS_PER_SECOND = 1000 export type SignatureVerdict = "valid" | "expired" | "invalid" @@ -75,29 +78,7 @@ export class LocalDiskStorage implements Storage { private readonly signingKey: string constructor(config: LocalDiskStorageConfig) { - if (config.nodeEnv === "production") { - throw new Error( - "LOCAL_STORAGE_DIR selects the local-disk storage driver, which serves objects from the API " + - "process and mounts development-only PUT/GET routes. It is refused in production; " + - "configure R2 (R2_ACCOUNT_ID / R2_ACCESS_KEY_ID / R2_SECRET_ACCESS_KEY / R2_BUCKET) instead.", - ) - } - const rootDirectory = config.rootDirectory.trim() - if (rootDirectory.length === 0) { - throw new Error("LOCAL_STORAGE_DIR must name a directory for the local-disk storage driver") - } - if (!isAbsolute(rootDirectory)) { - throw new Error( - `LOCAL_STORAGE_DIR must be an ABSOLUTE path, got "${rootDirectory}". The API service ` + - "(services/api) and the media-worker service (services/media-worker) run from different " + - "working directories, so a relative path resolves against each process's own cwd and gives " + - "them two divergent storage trees, so the worker would write thumbnails the API then serves as " + - "404s. Set the SAME absolute path in LOCAL_STORAGE_DIR for both services.", - ) - } - if (config.signingKey.length === 0) { - throw new Error("LOCAL_STORAGE_SIGNING_KEY must not be empty") - } + const rootDirectory = assertUsableRoot(config) this.baseUrl = normalizeBaseUrl(config.publicApiUrl) if (config.signingKey === LOCAL_STORAGE_DEV_SIGNING_KEY && !isLoopbackUrl(this.baseUrl)) { throw new Error( @@ -321,7 +302,10 @@ export class LocalDiskStorage implements Storage { private async writeAtomic(target: string, bytes: Buffer): Promise { await mkdir(dirname(target), { recursive: true }) await mkdir(this.tempRoot, { recursive: true }) - const staged = join(this.tempRoot, `${randomBytes(16).toString("hex")}.part`) + const staged = join( + this.tempRoot, + `${randomBytes(STAGED_FILE_RANDOM_BYTES).toString("hex")}.part`, + ) try { await writeFile(staged, bytes) await rename(staged, target) @@ -332,6 +316,33 @@ export class LocalDiskStorage implements Storage { } } +function assertUsableRoot(config: LocalDiskStorageConfig): string { + if (config.nodeEnv === "production") { + throw new Error( + "LOCAL_STORAGE_DIR selects the local-disk storage driver, which serves objects from the API " + + "process and mounts development-only PUT/GET routes. It is refused in production; " + + "configure R2 (R2_ACCOUNT_ID / R2_ACCESS_KEY_ID / R2_SECRET_ACCESS_KEY / R2_BUCKET) instead.", + ) + } + const rootDirectory = config.rootDirectory.trim() + if (rootDirectory.length === 0) { + throw new Error("LOCAL_STORAGE_DIR must name a directory for the local-disk storage driver") + } + if (!isAbsolute(rootDirectory)) { + throw new Error( + `LOCAL_STORAGE_DIR must be an ABSOLUTE path, got "${rootDirectory}". The API service ` + + "(services/api) and the media-worker service (services/media-worker) run from different " + + "working directories, so a relative path resolves against each process's own cwd and gives " + + "them two divergent storage trees, so the worker would write thumbnails the API then serves as " + + "404s. Set the SAME absolute path in LOCAL_STORAGE_DIR for both services.", + ) + } + if (config.signingKey.length === 0) { + throw new Error("LOCAL_STORAGE_SIGNING_KEY must not be empty") + } + return rootDirectory +} + function contentEtag(body: Uint8Array | Buffer): string { return createHash("sha256").update(body).digest("hex") } @@ -371,16 +382,16 @@ function normalizeBaseUrl(publicApiUrl: string): string { if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { throw new Error(`PUBLIC_API_URL must be http(s): ${trimmed}`) } - return trimmed.replace(/\/+$/, "") + return trimmed.replace(TRAILING_SLASHES_RE, "") } function isLoopbackUrl(baseUrl: string): boolean { const host = new URL(baseUrl).hostname.toLowerCase() - return LOOPBACK_HOSTS.has(host) || /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host) + return LOOPBACK_HOSTS.has(host) || LOOPBACK_IPV4_PATTERN.test(host) } export function nowSec(): number { - return Math.floor(Date.now() / 1000) + return Math.floor(Date.now() / MS_PER_SECOND) } function isNotFound(err: unknown): boolean { diff --git a/services/api/src/adapters/storage.r2.ts b/services/api/src/adapters/storage.r2.ts index b38ffd12..e18cc45f 100644 --- a/services/api/src/adapters/storage.r2.ts +++ b/services/api/src/adapters/storage.r2.ts @@ -19,18 +19,28 @@ export interface R2StorageConfig { publicBase?: string } -export const R2_DEFAULT_GET_TTL_SEC = 15 * 60 +const R2_DEFAULT_GET_TTL_SEC = 15 * 60 export const R2_PUT_TTL_SEC = 15 * 60 -export const R2_CONNECT_TIMEOUT_MS = 5_000 -export const R2_SOCKET_IDLE_TIMEOUT_MS = 30_000 +const R2_CONNECT_TIMEOUT_MS = 5_000 +const R2_SOCKET_IDLE_TIMEOUT_MS = 30_000 export const R2_RESPONSE_TIMEOUT_MS = 30_000 -export const R2_MAX_ATTEMPTS = 3 +const R2_MAX_ATTEMPTS = 3 // The handler's timers stop once response headers arrive, so only an abort signal bounds a body // that stalls mid-stream. Metadata calls sit on request paths; transfers move objects up to tens of MB. -export const R2_METADATA_OPERATION_TIMEOUT_MS = 15_000 +const R2_METADATA_OPERATION_TIMEOUT_MS = 15_000 export const R2_TRANSFER_OPERATION_TIMEOUT_MS = 120_000 +const R2_REGION = "auto" +const R2_ENDPOINT_DOMAIN = "r2.cloudflarestorage.com" +const DEFAULT_CONTENT_TYPE = "application/octet-stream" +const HTTP_NOT_FOUND = 404 +const NOT_FOUND_ERROR_NAMES: ReadonlySet = new Set(["NotFound", "NoSuchKey"]) + +const TRAILING_SLASHES_RE = /\/+$/ +const LEADING_SLASHES_RE = /^\/+/ +const HTTP_SCHEME_RE = /^https?:\/\//i + export class R2Storage implements Storage { private readonly config: R2StorageConfig private client: S3Client | undefined @@ -96,7 +106,7 @@ export class R2Storage implements Storage { const etag = normalizeEtag(res.ETag) return { size: typeof res.ContentLength === "number" ? res.ContentLength : 0, - contentType: res.ContentType ?? "application/octet-stream", + contentType: res.ContentType ?? DEFAULT_CONTENT_TYPE, ...(typeof res.ContentDisposition === "string" ? { contentDisposition: res.ContentDisposition } : {}), @@ -186,9 +196,9 @@ export class R2Storage implements Storage { private async getClient(): Promise { if (!this.client) { const { S3Client: S3ClientCtor } = await import("@aws-sdk/client-s3") - const endpointHost = `${this.config.accountId}.r2.cloudflarestorage.com` + const endpointHost = `${this.config.accountId}.${R2_ENDPOINT_DOMAIN}` this.client = new S3ClientCtor({ - region: "auto", + region: R2_REGION, endpoint: `https://${endpointHost}`, requestHandler: await boundedRequestHandler(endpointHost), maxAttempts: R2_MAX_ATTEMPTS, @@ -235,13 +245,13 @@ async function boundedRequestHandler(host: string): Promise = new Set(["GET", "HEAD", "OPTIONS"]) -export function isSuspendedWrite(request: FastifyRequest): boolean { +function isSuspendedWrite(request: FastifyRequest): boolean { if (request.accountStatus !== "suspended") return false if (READ_ONLY_METHODS.has(request.method)) return false return request.routeOptions?.config?.allowSuspended !== true diff --git a/services/api/src/auth/admin-guard.ts b/services/api/src/auth/admin-guard.ts index 7bf8aac8..57a48d46 100644 --- a/services/api/src/auth/admin-guard.ts +++ b/services/api/src/auth/admin-guard.ts @@ -35,6 +35,10 @@ export const OPERATOR_ALLOWLIST_TTL_SECONDS = 60 const OPERATOR_ALLOWLIST_PREFIX = "opallow:" +const ALLOWLIST_VERDICT_ALLOWED = "1" + +const ALLOWLIST_VERDICT_DENIED = "0" + /** Checks only the session's role claim; admin data routes need assertOperatorAuthority. */ export function requireOperator(request: FastifyRequest): string { const userId = requireAuth(request) @@ -46,7 +50,7 @@ export function requireOperator(request: FastifyRequest): string { * The cheap session-claim check runs first so an anonymous or citizen caller is rejected without any * lookup; only a caller already claiming `operator` reaches the allowlist resolution. */ -export async function assertOperatorAuthority(request: FastifyRequest): Promise { +async function assertOperatorAuthority(request: FastifyRequest): Promise { const userId = requireOperator(request) if (!(await isAllowlistedOperator(request, userId))) { // Deliberately the same generic 403 requireRole produces: an off-boarded operator learns only that they @@ -73,11 +77,15 @@ async function isAllowlistedOperator(request: FastifyRequest, userId: string): P const key = OPERATOR_ALLOWLIST_PREFIX + userId const cached = await services.cache.get(key) - if (cached !== null) return cached === "1" + if (cached !== null) return cached === ALLOWLIST_VERDICT_ALLOWED const user = await services.users.findById(userId) const allowed = user?.email != null && isAdminEmail(env, user.email) // A denied caller must not cost a user lookup per request either. - await services.cache.set(key, allowed ? "1" : "0", OPERATOR_ALLOWLIST_TTL_SECONDS) + await services.cache.set( + key, + allowed ? ALLOWLIST_VERDICT_ALLOWED : ALLOWLIST_VERDICT_DENIED, + OPERATOR_ALLOWLIST_TTL_SECONDS, + ) return allowed } diff --git a/services/api/src/auth/auth-services.ts b/services/api/src/auth/auth-services.ts index 69e074d9..2f82ce30 100644 --- a/services/api/src/auth/auth-services.ts +++ b/services/api/src/auth/auth-services.ts @@ -1,5 +1,6 @@ import type { Container } from "../di.js" import type { OAuthProvider, UserDTO } from "@civfix/shared" +import type { Mailer } from "@civfix/shared/interfaces" import { RedisCacheClient, type CacheClient } from "./cache.js" import { SessionService, type SessionLogger } from "./session-service.js" import { @@ -32,7 +33,7 @@ export interface AuthServices { enabledProviders: OAuthProvider[] } -export function enabledProvidersFromConfig(config: OAuthConfig): OAuthProvider[] { +function enabledProvidersFromConfig(config: OAuthConfig): OAuthProvider[] { const providers: OAuthProvider[] = [] if (config.apple) providers.push("apple") if (config.google) providers.push("google") @@ -43,7 +44,7 @@ export function enabledProvidersFromConfig(config: OAuthConfig): OAuthProvider[] export interface BuildAuthServicesOptions { stores: AuthStores cache: CacheClient - mailer: import("@civfix/shared/interfaces").Mailer + mailer: Mailer oauthConfig: OAuthConfig verifier?: JwksVerifier now?: () => number @@ -120,10 +121,10 @@ export const REVIEWER_OTP_MIN_CODE_LENGTH = REVIEWER_OTP_CODE_MIN_LENGTH export function reviewerOtpConfigFromEnv(env: Container["env"]): ReviewerOtpConfig | null { if (env.REVIEWER_OTP_BYPASS !== true) return null - const code = (env as { REVIEWER_OTP_CODE?: string }).REVIEWER_OTP_CODE + const code = env.REVIEWER_OTP_CODE if (typeof code !== "string") return null const trimmed = code.trim() - if (trimmed.length < REVIEWER_OTP_MIN_CODE_LENGTH) return null + if (trimmed.length < REVIEWER_OTP_CODE_MIN_LENGTH) return null return { email: REVIEWER_OTP_EMAIL, code: trimmed } } diff --git a/services/api/src/auth/cf-access.ts b/services/api/src/auth/cf-access.ts index 6cfe6af6..5abb8bbe 100644 --- a/services/api/src/auth/cf-access.ts +++ b/services/api/src/auth/cf-access.ts @@ -13,6 +13,14 @@ import { createRemoteJWKSet, jwtVerify, type JWTPayload, type JWTVerifyGetKey } from "jose" +const ACCESS_JWT_ALG = "RS256" + +const ACCESS_CLOCK_TOLERANCE_SECONDS = 30 + +const ACCESS_CERTS_PATH = "/cdn-cgi/access/certs" + +const TRAILING_SLASHES_RE = /\/+$/ + export interface AccessIdentity { /** Verified email for an interactive (human) login; null for a service-token (machine) login. */ email: string | null @@ -44,15 +52,16 @@ export function createAccessVerifier( config: AccessVerifierConfig, jwks?: JWTVerifyGetKey, ): VerifyAccessJwt { - const issuer = config.teamDomain.replace(/\/+$/, "") - const keySet = jwks ?? createRemoteJWKSet(new URL(`${issuer}/cdn-cgi/access/certs`)) + const issuer = config.teamDomain.replace(TRAILING_SLASHES_RE, "") + const keySet = jwks ?? createRemoteJWKSet(new URL(`${issuer}${ACCESS_CERTS_PATH}`)) return async function verifyAccessJwt(token: string): Promise { const { payload } = await jwtVerify(token, keySet, { issuer, audience: config.aud, - algorithms: ["RS256"], // pin to prevent alg-confusion - clockTolerance: 30, // seconds; tolerate minor clock drift + // Pinned to prevent alg-confusion. + algorithms: [ACCESS_JWT_ALG], + clockTolerance: ACCESS_CLOCK_TOLERANCE_SECONDS, // jose validates iss/aud/exp only when present; without this a token omitting exp would skip the // expiry check. requiredClaims: ["exp", "iss", "aud"], diff --git a/services/api/src/auth/context.ts b/services/api/src/auth/context.ts index 98e31952..f76e147e 100644 --- a/services/api/src/auth/context.ts +++ b/services/api/src/auth/context.ts @@ -24,7 +24,7 @@ declare module "fastify" { } } -export function anonymousAuth(anonSessionId?: string): AuthContext { +function anonymousAuth(anonSessionId?: string): AuthContext { return { userId: null, roles: [], diff --git a/services/api/src/auth/csrf.ts b/services/api/src/auth/csrf.ts index 68ad69f2..e39b31fe 100644 --- a/services/api/src/auth/csrf.ts +++ b/services/api/src/auth/csrf.ts @@ -34,6 +34,8 @@ export const CSRF_HEADER = "x-csrf-token" /** Domain separation: the signing key is shared with cookie signing, so label what is being signed. */ const CSRF_HMAC_LABEL = "civfix-csrf-v1:" +const CSRF_REJECTED_MESSAGE = "CSRF token missing or invalid." + /** Read lazily, so a lazily-loaded env is not forced early. */ export type CsrfEnv = Pick @@ -74,12 +76,12 @@ export function makeCsrf(env: CsrfEnv): Csrf { const headerValue = request.headers[CSRF_HEADER] const headerToken = Array.isArray(headerValue) ? headerValue[0] : headerValue if (!headerToken) { - throw AppError.forbidden("CSRF token missing or invalid.") + throw AppError.forbidden(CSRF_REJECTED_MESSAGE) } const expected = await tokenForSession(sessionCookie) if (!constantTimeStringEqual(expected, headerToken)) { - throw AppError.forbidden("CSRF token missing or invalid.") + throw AppError.forbidden(CSRF_REJECTED_MESSAGE) } } diff --git a/services/api/src/auth/jwks.ts b/services/api/src/auth/jwks.ts index 5eb595a7..7dfd6177 100644 --- a/services/api/src/auth/jwks.ts +++ b/services/api/src/auth/jwks.ts @@ -3,6 +3,22 @@ import { AppError, ErrorCode } from "@civfix/shared" import { exposeMessage } from "../errors/exposed-message.js" import { constantTimeStringEqual } from "./crypto.js" +const JWKS_CACHE_TTL_MS = 60 * 60 * 1000 + +/** + * Floor between two forced refetches for an unknown `kid`, so a stream of tokens naming forged kids cannot + * turn into a fetch storm against the provider. + */ +const JWKS_FORCE_REFRESH_FLOOR_MS = 60 * 1000 + +const JWKS_FETCH_TIMEOUT_MS = 5000 + +const ID_TOKEN_ALG = "RS256" + +const RSA_SIGNATURE_ALGORITHM = "RSASSA-PKCS1-v1_5" + +const JWT_SEGMENT_COUNT = 3 + export interface VerifiedIdToken { sub: string email: string | null @@ -71,23 +87,22 @@ export class RemoteJwksVerifier implements JwksVerifier { private readonly cache = new Map() private readonly lastForceRefreshAtMs = new Map() private readonly inFlight = new Map>() - private static readonly FORCE_REFRESH_FLOOR_MS = 60 * 1000 constructor(opts: RemoteJwksVerifierOptions = {}) { this.fetchImpl = opts.fetchImpl ?? defaultFetch - this.cacheTtlMs = opts.cacheTtlMs ?? 60 * 60 * 1000 + this.cacheTtlMs = opts.cacheTtlMs ?? JWKS_CACHE_TTL_MS this.now = opts.now ?? Date.now } async verify(idToken: string, params: VerifyParams): Promise { const parts = idToken.split(".") - if (parts.length !== 3) { + if (parts.length !== JWT_SEGMENT_COUNT) { throw AppError.unauthorized("Malformed identity token.") } const [headerB64, payloadB64, signatureB64] = parts as [string, string, string] const header = decodeJsonSegment(headerB64) - if (!header || header.alg !== "RS256" || !header.kid) { + if (!header || header.alg !== ID_TOKEN_ALG || !header.kid) { throw AppError.unauthorized("Unsupported identity token algorithm.") } @@ -109,7 +124,7 @@ export class RemoteJwksVerifier implements JwksVerifier { let match = keys.find((k) => k.kid === kid) if (!match) { const last = this.lastForceRefreshAtMs.get(jwksUrl) - if (last === undefined || this.now() - last >= RemoteJwksVerifier.FORCE_REFRESH_FLOOR_MS) { + if (last === undefined || this.now() - last >= JWKS_FORCE_REFRESH_FLOOR_MS) { this.lastForceRefreshAtMs.set(jwksUrl, this.now()) keys = await this.getKeys(jwksUrl, true) match = keys.find((k) => k.kid === kid) @@ -155,8 +170,6 @@ export class RemoteJwksVerifier implements JwksVerifier { } } -const JWKS_FETCH_TIMEOUT_MS = 5000 - const defaultFetch: FetchLike = async (url: string) => { const controller = new AbortController() const timer = setTimeout(() => controller.abort(), JWKS_FETCH_TIMEOUT_MS) @@ -196,14 +209,14 @@ async function verifyRs256(jwk: Jwk, signingInput: string, signatureB64: string) if (jwk.kty !== "RSA" || !jwk.n || !jwk.e) return false const key = await crypto.subtle.importKey( "jwk", - { kty: "RSA", n: jwk.n, e: jwk.e, alg: "RS256", ext: true }, - { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" }, + { kty: "RSA", n: jwk.n, e: jwk.e, alg: ID_TOKEN_ALG, ext: true }, + { name: RSA_SIGNATURE_ALGORITHM, hash: "SHA-256" }, false, ["verify"], ) const signature = Buffer.from(signatureB64, "base64url") const data = new TextEncoder().encode(signingInput) - return crypto.subtle.verify("RSASSA-PKCS1-v1_5", key, signature, data) + return crypto.subtle.verify(RSA_SIGNATURE_ALGORITHM, key, signature, data) } /** diff --git a/services/api/src/auth/oauth.ts b/services/api/src/auth/oauth.ts index 42d6fc1a..77c1c1a7 100644 --- a/services/api/src/auth/oauth.ts +++ b/services/api/src/auth/oauth.ts @@ -10,15 +10,24 @@ import { import { RemoteJwksVerifier, type JwksVerifier, type VerifiedIdToken } from "./jwks.js" import { newAccountDisplayName } from "./official-account.js" -export const GOOGLE_ISSUERS = ["https://accounts.google.com", "accounts.google.com"] -export const GOOGLE_JWKS_URL = "https://www.googleapis.com/oauth2/v3/certs" -export const APPLE_ISSUER = "https://appleid.apple.com" -export const APPLE_JWKS_URL = "https://appleid.apple.com/auth/keys" +const GOOGLE_ISSUERS = ["https://accounts.google.com", "accounts.google.com"] +const GOOGLE_JWKS_URL = "https://www.googleapis.com/oauth2/v3/certs" +const APPLE_ISSUER = "https://appleid.apple.com" +const APPLE_JWKS_URL = "https://appleid.apple.com/auth/keys" + +const GOOGLE_WEB_SCOPES = ["openid", "email", "profile"] +const APPLE_WEB_SCOPES = ["name", "email"] +// Apple requires form_post whenever the name or email scope is requested, so the callback arrives as a form +// body (parsed in routes/auth.routes.ts). +const APPLE_RESPONSE_MODE = "form_post" + +const APPLE_FALLBACK_DISPLAY_NAME = "Apple user" +const GOOGLE_FALLBACK_DISPLAY_NAME = "Google user" export const PROVIDER_GOOGLE = "google" export const PROVIDER_APPLE = "apple" -export const UNVERIFIED_ACCOUNT_EXISTS_MESSAGE = +const UNVERIFIED_ACCOUNT_EXISTS_MESSAGE = "An account already uses this email address. Sign in the way you did before, or contact support." export interface OAuthConfig { @@ -83,7 +92,7 @@ export class OAuthService { const client = this.requireGoogle() const state = generateState() const codeVerifier = generateCodeVerifier() - const url = client.createAuthorizationURL(state, codeVerifier, ["openid", "email", "profile"]) + const url = client.createAuthorizationURL(state, codeVerifier, GOOGLE_WEB_SCOPES) return { url: url.toString(), state, codeVerifier } } @@ -98,8 +107,8 @@ export class OAuthService { createAppleAuthUrl(): { url: string; state: string } { const client = this.requireAppleWeb() const state = generateState() - const url = client.createAuthorizationURL(state, ["name", "email"]) - url.searchParams.set("response_mode", "form_post") + const url = client.createAuthorizationURL(state, APPLE_WEB_SCOPES) + url.searchParams.set("response_mode", APPLE_RESPONSE_MODE) return { url: url.toString(), state } } @@ -292,7 +301,7 @@ function providerDisplayName( } function providerFallbackName(provider: string): string { - return provider === PROVIDER_APPLE ? "Apple user" : "Google user" + return provider === PROVIDER_APPLE ? APPLE_FALLBACK_DISPLAY_NAME : GOOGLE_FALLBACK_DISPLAY_NAME } function sanitizeDisplayName(raw: string | null | undefined): string | null { diff --git a/services/api/src/auth/official-account.ts b/services/api/src/auth/official-account.ts index b74661cf..6b0bc3ac 100644 --- a/services/api/src/auth/official-account.ts +++ b/services/api/src/auth/official-account.ts @@ -4,6 +4,10 @@ export const CIVFIX_OFFICIAL_HANDLE = "civfix" export const CIVFIX_OFFICIAL_DISPLAY_NAME = "CivFix" +const COMBINING_MARK_RE = /\p{M}/gu + +const NAME_KEY_CHAR_RE = /^[a-z0-9]$/ + export function isOfficialAccount(userId: string | null | undefined): boolean { return typeof userId === "string" && userId.toLowerCase() === CIVFIX_OFFICIAL_USER_ID } @@ -36,10 +40,10 @@ function nameKey(name: string): string { let key = "" for (const raw of name.toLowerCase()) { const folded = - NAME_LOOKALIKES[raw] ?? raw.normalize("NFKD").replace(/\p{M}/gu, "").toLowerCase() + NAME_LOOKALIKES[raw] ?? raw.normalize("NFKD").replace(COMBINING_MARK_RE, "").toLowerCase() for (const ch of folded) { const mapped = NAME_LOOKALIKES[ch] ?? ch - if (/^[a-z0-9]$/.test(mapped)) key += mapped + if (NAME_KEY_CHAR_RE.test(mapped)) key += mapped } } return key diff --git a/services/api/src/auth/otp.ts b/services/api/src/auth/otp.ts index a328dbdd..f2ac5476 100644 --- a/services/api/src/auth/otp.ts +++ b/services/api/src/auth/otp.ts @@ -12,7 +12,7 @@ export const OTP_CODE_LENGTH = 6 export const OTP_TTL_SECONDS = 5 * 60 export const OTP_MAX_ATTEMPTS = 3 export const OTP_EMAIL_WINDOW_SECONDS = 60 -export const OTP_IP_WINDOW_SECONDS = 60 * 60 +const OTP_IP_WINDOW_SECONDS = 60 * 60 export const OTP_IP_MAX_PER_WINDOW = 10 export const OTP_VERIFY_FAIL_WINDOW_SECONDS = 15 * 60 @@ -22,6 +22,14 @@ export const OTP_VERIFY_IP_FAIL_MAX = 30 const ARGON2ID = 2 const UNNAMED_CITIZEN_DISPLAY_NAME = "citizen" +const OTP_ISSUE_IP_KEY_PREFIX = "otp:rl:ip:" +const OTP_EMAIL_COOLDOWN_KEY_PREFIX = "otp:rl:email:" +const OTP_VERIFY_CODE_FAIL_KEY_PREFIX = "otp:vf:code:" +const OTP_VERIFY_IP_FAIL_KEY_PREFIX = "otp:vf:ip:" + +const INVALID_CODE_MESSAGE = "Invalid or expired code." +const TOO_MANY_INCORRECT_MESSAGE = "Too many incorrect attempts. Request a new code." + const ARGON_OPTS_FULL = { algorithm: ARGON2ID, memoryCost: 65536, @@ -36,7 +44,7 @@ const ARGON_OPTS_TEST = { parallelism: 1, } as const -export const ARGON_OPTS = process.env.NODE_ENV === "test" ? ARGON_OPTS_TEST : ARGON_OPTS_FULL +const ARGON_OPTS = process.env.NODE_ENV === "test" ? ARGON_OPTS_TEST : ARGON_OPTS_FULL export function hashOtpCode(code: string): Promise { return argonHash(code, ARGON_OPTS) @@ -47,15 +55,15 @@ export function verifyOtpCode(codeHash: string, code: string): Promise } export const OTP_REFUSED_UNVERIFIED_ACCOUNT_ACTION = "auth.otp_refused_unverified_account" -export const OTP_REFUSED_UNVERIFIED_ACCOUNT_REASON = "email_unverified_with_provider_identity" +const OTP_REFUSED_UNVERIFIED_ACCOUNT_REASON = "email_unverified_with_provider_identity" -export function unverifiedAccountNeedsReviewMessage(supportEmail: string | null): string { +function unverifiedAccountNeedsReviewMessage(supportEmail: string | null): string { const contact = supportEmail === null ? "Contact support" : `Contact support at ${supportEmail}` return `This email address is linked to an account that needs a quick check before you can sign in. ${contact} and we'll sort it out.` } export const REVIEWER_OTP_EMAIL = "reviewer@civfix.org" -export const REVIEWER_HANDLE = "reviewer" +const REVIEWER_HANDLE = "reviewer" export const REVIEWER_DISPLAY_NAME = "Reviewer Reviewer" export interface ReviewerOtpConfig { @@ -113,7 +121,7 @@ export class OtpService { this.now = opts.now ?? Date.now this.logger = opts.logger this.reviewer = opts.reviewer - ? { email: opts.reviewer.email.trim().toLowerCase(), code: opts.reviewer.code } + ? { email: normalizeEmail(opts.reviewer.email), code: opts.reviewer.code } : null this.supportEmail = opts.supportEmail ?? null this.audit = opts.audit @@ -124,7 +132,7 @@ export class OtpService { } async issueOtp(email: string, ip: string | null): Promise { - const normalized = email.trim().toLowerCase() + const normalized = normalizeEmail(email) if (this.isReviewerEmail(normalized)) { return { resendAfterSec: OTP_EMAIL_WINDOW_SECONDS } @@ -132,7 +140,7 @@ export class OtpService { const ipBucket = ip === null ? null : normalizeIp(ip) if (ipBucket) { - const ipKey = `otp:rl:ip:${ipBucket}` + const ipKey = OTP_ISSUE_IP_KEY_PREFIX + ipBucket const ipHits = await this.cache.incr(ipKey, OTP_IP_WINDOW_SECONDS) if (ipHits > OTP_IP_MAX_PER_WINDOW) { throw AppError.rateLimited("Too many code requests from this network.") @@ -172,7 +180,7 @@ export class OtpService { } async verifyOtp(email: string, code: string, ip: string | null): Promise { - const normalized = email.trim().toLowerCase() + const normalized = normalizeEmail(email) if ((await this.proveInbox(normalized, code, ip)) === "reviewer") { return this.ensureReviewerUser(normalized) } @@ -203,7 +211,7 @@ export class OtpService { code: string, ip: string | null, ): Promise { - const normalized = email.trim().toLowerCase() + const normalized = normalizeEmail(email) if ((await this.proveInbox(normalized, code, ip)) === "reviewer") { return this.ensureReviewerUser(normalized) } @@ -227,25 +235,35 @@ export class OtpService { return "reviewer" } await this.bumpVerifyFailure(null, ipBucket) - throw AppError.unauthorized("Invalid or expired code.") + throw AppError.unauthorized(INVALID_CODE_MESSAGE) } + await this.consumeCode(normalized, code, ipBucket, now) + return "code" + } + + private async consumeCode( + normalized: string, + code: string, + ipBucket: string | null, + now: Date, + ): Promise { const record = await this.store.findLatestActive(normalized, now) if (!record) { await this.bumpVerifyFailure(null, ipBucket) - throw AppError.unauthorized("Invalid or expired code.") + throw AppError.unauthorized(INVALID_CODE_MESSAGE) } if ((await this.readCounter(codeFailKey(record.id))) >= OTP_VERIFY_CODE_FAIL_MAX) { await this.store.markConsumed(record.id, now) - throw AppError.unauthorized("Too many incorrect attempts. Request a new code.") + throw AppError.unauthorized(TOO_MANY_INCORRECT_MESSAGE) } const attempts = await this.store.incrementAttempts(record.id) if (attempts > OTP_MAX_ATTEMPTS) { await this.store.markConsumed(record.id, now) await this.bumpVerifyFailure(record.id, ipBucket) - throw AppError.unauthorized("Too many incorrect attempts. Request a new code.") + throw AppError.unauthorized(TOO_MANY_INCORRECT_MESSAGE) } const ok = await verifyOtpCode(record.codeHash, code) @@ -254,12 +272,12 @@ export class OtpService { await this.store.markConsumed(record.id, now) } await this.bumpVerifyFailure(record.id, ipBucket) - throw AppError.unauthorized("Invalid or expired code.") + throw AppError.unauthorized(INVALID_CODE_MESSAGE) } const claimed = await this.store.markConsumed(record.id, now) if (!claimed) { - throw AppError.unauthorized("Invalid or expired code.") + throw AppError.unauthorized(INVALID_CODE_MESSAGE) } await this.cache.del(emailCooldownKey(normalized)).catch((err: unknown) => { this.logger?.warn( @@ -267,7 +285,6 @@ export class OtpService { "otp: failed to release per-email cooldown after successful verify", ) }) - return "code" } // The code proves who owns the inbox, not who created the row. A row that never verified its address @@ -325,16 +342,20 @@ export class OtpService { } } +function normalizeEmail(email: string): string { + return email.trim().toLowerCase() +} + function emailCooldownKey(normalizedEmail: string): string { - return `otp:rl:email:${normalizedEmail}` + return OTP_EMAIL_COOLDOWN_KEY_PREFIX + normalizedEmail } function codeFailKey(codeId: string): string { - return `otp:vf:code:${codeId}` + return OTP_VERIFY_CODE_FAIL_KEY_PREFIX + codeId } function ipFailKey(ip: string): string { - return `otp:vf:ip:${ip}` + return OTP_VERIFY_IP_FAIL_KEY_PREFIX + ip } function defaultDisplayName(email: string): string { diff --git a/services/api/src/auth/pg-stores.ts b/services/api/src/auth/pg-stores.ts index b6863024..8f7f2b3a 100644 --- a/services/api/src/auth/pg-stores.ts +++ b/services/api/src/auth/pg-stores.ts @@ -55,6 +55,14 @@ import { type DbTransaction = Parameters[0]>[0] +const ERASURE_HANDLE_RETRIES = 5 + +const PG_UNIQUE_VIOLATION = "23505" + +const HOST_TRANSFER_TITLE_KEY = "notification.cleanup_role.promoted.title" + +const HOST_TRANSFER_BODY_KEY = "notification.cleanup_role.promoted.body" + interface TransferredEvent extends Record { cleanup_id: string new_organizer: string @@ -371,7 +379,20 @@ export class PgUserStore implements UserStore { } private async transferHostedEvents(tx: DbTransaction, id: string): Promise { - const toOrgOwner = await tx.execute(sql` + const moved = [ + ...(await this.transferToOrganizationOwners(tx, id)), + ...(await this.transferToCohosts(tx, id)), + ] + await this.auditHostTransfers(tx, id, moved) + await this.demoteRemainingTeamRoles(tx, id) + return moved + } + + private async transferToOrganizationOwners( + tx: DbTransaction, + id: string, + ): Promise { + return tx.execute(sql` WITH candidate AS ( SELECT c.id AS cleanup_id, om.user_id AS new_organizer FROM cleanups c @@ -395,8 +416,10 @@ export class PgUserStore implements UserStore { SELECT m.cleanup_id, m.new_organizer, c.title FROM moved m JOIN cleanups c ON c.id = m.cleanup_id `) + } - const toCohost = await tx.execute(sql` + private async transferToCohosts(tx: DbTransaction, id: string): Promise { + return tx.execute(sql` WITH candidate AS ( SELECT DISTINCT ON (c.id) c.id AS cleanup_id, m.user_id AS new_organizer FROM cleanups c @@ -417,8 +440,13 @@ export class PgUserStore implements UserStore { SELECT m.cleanup_id, m.new_organizer, c.title FROM moved m JOIN cleanups c ON c.id = m.cleanup_id `) + } - const moved = [...toOrgOwner, ...toCohost] + private async auditHostTransfers( + tx: DbTransaction, + id: string, + moved: readonly TransferredEvent[], + ): Promise { for (const row of moved) { await tx.execute(sql` INSERT INTO audit_log (actor_id, action, target, meta) @@ -430,7 +458,9 @@ export class PgUserStore implements UserStore { ) `) } + } + private async demoteRemainingTeamRoles(tx: DbTransaction, id: string): Promise { await tx.execute(sql` UPDATE cleanup_members SET role = 'member' WHERE user_id = ${id} AND role = 'organizer' @@ -454,19 +484,10 @@ export class PgUserStore implements UserStore { jsonb_build_object('targetUserId', ${id}::text, 'from', h.role, 'to', 'member') FROM held h `) - return moved } private async releaseOrganizations(tx: DbTransaction, id: string): Promise { - await tx.execute(sql` - SELECT o.id FROM organizations o - WHERE EXISTS ( - SELECT 1 FROM organization_members om - WHERE om.organization_id = o.id AND om.user_id = ${id} AND om.role = 'owner' - ) - ORDER BY o.id - FOR UPDATE - `) + await this.lockOwnedOrganizations(tx, id) const owned = await tx.execute<{ organization_id: string }>(sql` UPDATE organization_members SET role = 'admin' WHERE user_id = ${id} AND role = 'owner' @@ -527,6 +548,21 @@ export class PgUserStore implements UserStore { `) } + // Every membership mutation locks its organization row first, so taking those same row locks (in id + // order, so two erasures cannot deadlock) before the owner step-down keeps a concurrent member or role + // change from racing the successor pick. + private async lockOwnedOrganizations(tx: DbTransaction, id: string): Promise { + await tx.execute(sql` + SELECT o.id FROM organizations o + WHERE EXISTS ( + SELECT 1 FROM organization_members om + WHERE om.organization_id = o.id AND om.user_id = ${id} AND om.role = 'owner' + ) + ORDER BY o.id + FOR UPDATE + `) + } + private async scrubAttendeeContributions(tx: DbTransaction, id: string): Promise { // Waitlist and ticket-type rows before registrations, the order applyBanIn and the waitlist sweep // take, so an erasure racing a ban on one of the user's events cannot deadlock with it. @@ -577,131 +613,158 @@ export class PgUserStore implements UserStore { private async runErasure(id: string): Promise { const erasure = await this.db.transaction(async (tx) => { - const updated = await tx - .update(users) - .set({ - deletedAt: sql`COALESCE(${users.deletedAt}, now())`, - allowDirectMessages: false, - email: null, - emailVerified: false, - displayName: DELETED_USER_LABEL, - handle: generateTombstoneHandle(), - bio: null, - avatarUrl: null, - avatarMediaId: null, - socialLinks: null, - donationUrl: null, - lastActivityGeom: null, - lastActivityAt: null, - primaryOrganizationId: null, - }) - .where(eq(users.id, id)) - .returning() - const r = updated[0] - if (!r) throw new Error("PgUserStore.softDeleteAndAnonymize: user not found") + const tombstoned = await this.tombstoneUser(tx, id) // Revocation commits with the tombstone: if these ran after commit, a failure between the two would // leave a deleted account whose tokens still authenticate and whose devices still get pushes. - await tx.delete(sessions).where(eq(sessions.userId, id)) - await tx.delete(pushTokens).where(eq(pushTokens.userId, id)) - await tx.delete(notifications).where(eq(notifications.userId, id)) + await this.revokeSessionsAndDevices(tx, id) await tx .update(reports) .set({ visibility: "hidden" }) .where(and(eq(reports.reporterUserId, id), eq(reports.visibility, "public"))) await this.releaseOrganizations(tx, id) const moved = await this.transferHostedEvents(tx, id) - await tx - .update(cleanups) - .set({ status: "cancelled" }) - .where( - and( - eq(cleanups.organizerUserId, id), - ne(cleanups.status, "cancelled"), - gt(cleanups.endsAt, new Date()), - ), - ) + await this.cancelRemainingHostedEvents(tx, id) const releasedTicketTypeIds = await this.scrubAttendeeContributions(tx, id) await tx .update(posts) .set({ visibility: "hidden" }) .where(and(eq(posts.authorId, id), eq(posts.visibility, "public"))) + const certificateKeys = await this.revokeCertificates(tx, id) + await this.scrubModerationItems(tx, id) + const verificationKeys = await this.purgeVerificationDocuments(tx, id) + return { + record: toUserRecord(tombstoned), + objectKeys: [...certificateKeys, ...verificationKeys], + moved, + releasedTicketTypeIds, + } + }) - const certificates = await tx - .update(serviceHoursCertificates) - .set({ - revokedAt: sql`COALESCE(${serviceHoursCertificates.revokedAt}, now())`, - revokedReason: sql`COALESCE(${serviceHoursCertificates.revokedReason}, 'account_closed')`, - holderName: DELETED_USER_LABEL, - holderHandle: null, - snapshot: {}, - }) - .where(eq(serviceHoursCertificates.userId, id)) - .returning({ r2Key: serviceHoursCertificates.r2Key }) + await this.notifyNewOrganizers(erasure.moved) + await enqueueWaitlistPromotion(this.jobs, erasure.releasedTicketTypeIds, this.logger) + await this.deleteErasedObjects(id, erasure.objectKeys) + return erasure.record + } - await tx.execute(sql` - UPDATE moderation_items - SET meta = jsonb_set( - jsonb_set( - jsonb_set( - jsonb_set(meta, '{user,name}', to_jsonb(${DELETED_USER_LABEL}::text), false), - '{user,handle}', to_jsonb(''::text), false), - '{user,device}', to_jsonb(''::text), false), - '{user,joined}', to_jsonb(''::text), false) - WHERE meta->'user'->>'id' = ${id} - `) - await tx.execute(sql` - UPDATE moderation_items - SET meta = jsonb_set( - jsonb_set(meta, '{reporter}', to_jsonb(${DELETED_USER_LABEL}::text), false), - '{desc}', to_jsonb(''::text), false) - WHERE meta->>'reporterUserId' = ${id} - `) + private async tombstoneUser(tx: DbTransaction, id: string): Promise { + const updated = await tx + .update(users) + .set({ + deletedAt: sql`COALESCE(${users.deletedAt}, now())`, + allowDirectMessages: false, + email: null, + emailVerified: false, + displayName: DELETED_USER_LABEL, + handle: generateTombstoneHandle(), + bio: null, + avatarUrl: null, + avatarMediaId: null, + socialLinks: null, + donationUrl: null, + lastActivityGeom: null, + lastActivityAt: null, + primaryOrganizationId: null, + }) + .where(eq(users.id, id)) + .returning() + const r = updated[0] + if (!r) throw new Error("PgUserStore.softDeleteAndAnonymize: user not found") + return r + } - const verificationMedia = await tx.execute<{ - r2_key: string - served_key: string | null - thumb_key: string | null - }>(sql` - DELETE FROM media_assets - WHERE purpose = 'verification' - AND id IN ( - SELECT (doc->>'mediaId')::uuid - FROM user_verification uv, - jsonb_array_elements(uv.documents) AS doc - WHERE uv.user_id = ${id} AND doc->>'mediaId' IS NOT NULL - ) - RETURNING r2_key, served_key, thumb_key - `) - await tx.execute(sql` - UPDATE user_verification - SET note = NULL, rejection_reason = NULL, documents = '[]'::jsonb, updated_at = now() - WHERE user_id = ${id} - `) + private async revokeSessionsAndDevices(tx: DbTransaction, id: string): Promise { + await tx.delete(sessions).where(eq(sessions.userId, id)) + await tx.delete(pushTokens).where(eq(pushTokens.userId, id)) + await tx.delete(notifications).where(eq(notifications.userId, id)) + } - const objectKeys = [ - ...certificates.map((c) => c.r2Key), - ...verificationMedia.flatMap((m) => - [m.r2_key, m.served_key, m.thumb_key].filter((k): k is string => k !== null), + private async cancelRemainingHostedEvents(tx: DbTransaction, id: string): Promise { + await tx + .update(cleanups) + .set({ status: "cancelled" }) + .where( + and( + eq(cleanups.organizerUserId, id), + ne(cleanups.status, "cancelled"), + gt(cleanups.endsAt, new Date()), ), - ] - return { record: toUserRecord(r), objectKeys, moved, releasedTicketTypeIds } - }) + ) + } - await this.notifyNewOrganizers(erasure.moved) - await enqueueWaitlistPromotion(this.jobs, erasure.releasedTicketTypeIds, this.logger) + private async revokeCertificates(tx: DbTransaction, id: string): Promise { + const certificates = await tx + .update(serviceHoursCertificates) + .set({ + revokedAt: sql`COALESCE(${serviceHoursCertificates.revokedAt}, now())`, + revokedReason: sql`COALESCE(${serviceHoursCertificates.revokedReason}, 'account_closed')`, + holderName: DELETED_USER_LABEL, + holderHandle: null, + snapshot: {}, + }) + .where(eq(serviceHoursCertificates.userId, id)) + .returning({ r2Key: serviceHoursCertificates.r2Key }) + return certificates.map((c) => c.r2Key) + } - for (const key of erasure.objectKeys) { + private async scrubModerationItems(tx: DbTransaction, id: string): Promise { + await tx.execute(sql` + UPDATE moderation_items + SET meta = jsonb_set( + jsonb_set( + jsonb_set( + jsonb_set(meta, '{user,name}', to_jsonb(${DELETED_USER_LABEL}::text), false), + '{user,handle}', to_jsonb(''::text), false), + '{user,device}', to_jsonb(''::text), false), + '{user,joined}', to_jsonb(''::text), false) + WHERE meta->'user'->>'id' = ${id} + `) + await tx.execute(sql` + UPDATE moderation_items + SET meta = jsonb_set( + jsonb_set(meta, '{reporter}', to_jsonb(${DELETED_USER_LABEL}::text), false), + '{desc}', to_jsonb(''::text), false) + WHERE meta->>'reporterUserId' = ${id} + `) + } + + private async purgeVerificationDocuments(tx: DbTransaction, id: string): Promise { + const verificationMedia = await tx.execute<{ + r2_key: string + served_key: string | null + thumb_key: string | null + }>(sql` + DELETE FROM media_assets + WHERE purpose = 'verification' + AND id IN ( + SELECT (doc->>'mediaId')::uuid + FROM user_verification uv, + jsonb_array_elements(uv.documents) AS doc + WHERE uv.user_id = ${id} AND doc->>'mediaId' IS NOT NULL + ) + RETURNING r2_key, served_key, thumb_key + `) + await tx.execute(sql` + UPDATE user_verification + SET note = NULL, rejection_reason = NULL, documents = '[]'::jsonb, updated_at = now() + WHERE user_id = ${id} + `) + return verificationMedia.flatMap((m) => + [m.r2_key, m.served_key, m.thumb_key].filter((k): k is string => k !== null), + ) + } + + private async deleteErasedObjects(userId: string, keys: readonly string[]): Promise { + for (const key of keys) { if (this.certificateObjects === undefined) { - this.logger?.warn({ userId: id, key }, "erasure object not deleted: no object store wired") + this.logger?.warn({ userId, key }, "erasure object not deleted: no object store wired") continue } try { await this.certificateObjects.delete(key) } catch (err) { - this.logger?.warn({ err, userId: id, key }, "erasure object delete failed") + this.logger?.warn({ err, userId, key }, "erasure object delete failed") } } - return erasure.record } private async notifyNewOrganizers(moved: readonly TransferredEvent[]): Promise { @@ -710,8 +773,8 @@ export class PgUserStore implements UserStore { try { await this.notifier.createNotification(row.new_organizer, { type: "cleanup_role", - titleKey: "notification.cleanup_role.promoted.title", - bodyKey: "notification.cleanup_role.promoted.body", + titleKey: HOST_TRANSFER_TITLE_KEY, + bodyKey: HOST_TRANSFER_BODY_KEY, vars: { title: row.title }, link: `/cleanups/${row.cleanup_id}`, }) @@ -910,10 +973,6 @@ function rolesFor(role: Role): Role[] { return [role] } -const ERASURE_HANDLE_RETRIES = 5 - -const PG_UNIQUE_VIOLATION = "23505" - function isUniqueViolation(err: unknown): boolean { return ( typeof err === "object" && diff --git a/services/api/src/auth/reserved-handles.ts b/services/api/src/auth/reserved-handles.ts index b07f7c76..ade019f6 100644 --- a/services/api/src/auth/reserved-handles.ts +++ b/services/api/src/auth/reserved-handles.ts @@ -10,7 +10,7 @@ import type { Sql } from "../db/client.js" import { TOMBSTONE_HANDLE_RE } from "./stores.js" -export const RESERVED_HANDLES: readonly string[] = [ +const RESERVED_HANDLES: readonly string[] = [ "admin", "administrator", "civfix", diff --git a/services/api/src/auth/session-service.ts b/services/api/src/auth/session-service.ts index b6ed1e9f..be699a07 100644 --- a/services/api/src/auth/session-service.ts +++ b/services/api/src/auth/session-service.ts @@ -9,9 +9,9 @@ export const DEFAULT_SESSION_TTL_SECONDS = 30 * 24 * 60 * 60 export const ABSOLUTE_SESSION_MAX_SECONDS = 90 * 24 * 60 * 60 -export const BANNED_MARKER_GRACE_SECONDS = 60 +const BANNED_MARKER_GRACE_SECONDS = 60 -export const DEFAULT_SLIDE_GRANULARITY_MS = 60 * 60 * 1000 +const DEFAULT_SLIDE_GRANULARITY_MS = 60 * 60 * 1000 const SESSION_KEY_PREFIX = "sess:" @@ -19,6 +19,10 @@ const BANNED_KEY_PREFIX = "banned:" const EPOCH_KEY_PREFIX = "sessepoch:" +const BANNED_MARKER_VALUE = "1" + +const MINT_REFUSED_MESSAGE = "This account cannot start a new session." + export interface ResolvedSession { userId: string roles: Role[] @@ -58,6 +62,9 @@ interface UserGate { const REVOKED_STATUSES: ReadonlySet = new Set(["banned"]) +// Distinct from null, which is a decided rejection: a miss sends the lookup on to the durable row. +const CACHE_MISS = Symbol("session-cache-miss") + function isMintRefused(status: AccountStatus): boolean { return status === "banned" || status === "suspended" } @@ -119,7 +126,7 @@ export class SessionService { async createSession(userId: string, roles: Role[], meta: SessionMeta = {}): Promise { if (isOfficialAccount(userId)) { - throw AppError.forbidden("This account cannot start a new session.") + throw AppError.forbidden(MINT_REFUSED_MESSAGE) } const token = generateToken() const hash = await sha256Hex(token) @@ -134,7 +141,7 @@ export class SessionService { : Promise.resolve(meta.accountStatus ?? "active"), ]) if (isMintRefused(mintStatus)) { - throw AppError.forbidden("This account cannot start a new session.") + throw AppError.forbidden(MINT_REFUSED_MESSAGE) } await this.store.insert({ id: hash, @@ -176,7 +183,7 @@ export class SessionService { const status = await this.users.accountStatus(userId) if (isMintRefused(status)) { await this.store.deleteById(hash) - throw AppError.forbidden("This account cannot start a new session.") + throw AppError.forbidden(MINT_REFUSED_MESSAGE) } return { epoch, status } } @@ -187,31 +194,41 @@ export class SessionService { async resolveSessionByHash(hash: string): Promise { const nowMs = this.now() + const fromCache = await this.resolveFromCache(hash, nowMs) + if (fromCache !== CACHE_MISS) return fromCache + return this.resolveFromStore(hash, nowMs) + } + private async resolveFromCache( + hash: string, + nowMs: number, + ): Promise { const cachedRaw = await this.cache.get(sessionKey(hash)) - if (cachedRaw !== null) { - const cached = this.parseCache(cachedRaw) - if (cached && cached.expiresAtMs > nowMs && !REVOKED_STATUSES.has(cached.accountStatus)) { - if (this.absolutelyExpired(cached.createdAtMs, nowMs)) { - await this.expireSession(hash) - return null - } - const gate = await this.userGate(cached.userId) - if (!gate.active) return null - if (cached.epoch === gate.epoch) { - await this.maybeSlide(hash, cached.expiresAtMs, cached.createdAtMs, nowMs) - return { - userId: cached.userId, - roles: cached.roles, - accountStatus: cached.accountStatus, - source: "cache", - expiresAtMs: cached.expiresAtMs, - } + if (cachedRaw === null) return CACHE_MISS + const cached = this.parseCache(cachedRaw) + if (cached && cached.expiresAtMs > nowMs && !REVOKED_STATUSES.has(cached.accountStatus)) { + if (this.absolutelyExpired(cached.createdAtMs, nowMs)) { + await this.expireSession(hash) + return null + } + const gate = await this.userGate(cached.userId) + if (!gate.active) return null + if (cached.epoch === gate.epoch) { + await this.maybeSlide(hash, cached.expiresAtMs, cached.createdAtMs, nowMs) + return { + userId: cached.userId, + roles: cached.roles, + accountStatus: cached.accountStatus, + source: "cache", + expiresAtMs: cached.expiresAtMs, } } - await this.evictSessionCache(hash, "rejected session cache eviction failed") } + await this.evictSessionCache(hash, "rejected session cache eviction failed") + return CACHE_MISS + } + private async resolveFromStore(hash: string, nowMs: number): Promise { const row = await this.store.findById(hash) if (!row) return null if ( @@ -302,7 +319,11 @@ export class SessionService { // Outlives every cached projection (their TTL never exceeds ttlSeconds), so the marker alone is enough // to reject a session whose durable row is already gone. async markBanned(userId: string): Promise { - await this.cache.set(bannedKey(userId), "1", this.ttlSeconds + BANNED_MARKER_GRACE_SECONDS) + await this.cache.set( + bannedKey(userId), + BANNED_MARKER_VALUE, + this.ttlSeconds + BANNED_MARKER_GRACE_SECONDS, + ) } async clearBan(userId: string): Promise { diff --git a/services/api/src/auth/single-use-secret.ts b/services/api/src/auth/single-use-secret.ts index bd3aa95e..6e85089e 100644 --- a/services/api/src/auth/single-use-secret.ts +++ b/services/api/src/auth/single-use-secret.ts @@ -35,13 +35,15 @@ export interface SingleUseSecretOptions { const PRESENT = "1" +const CLAIM_SEGMENT = "claim:" + export function makeSingleUseSecretStore( cache: CacheClient, opts: SingleUseSecretOptions, ): SingleUseSecretStore { const { prefix, ttlSeconds } = opts const newSecret = opts.newSecret ?? (() => generateToken()) - const claimPrefix = `${prefix}claim:` + const claimPrefix = prefix + CLAIM_SEGMENT return { async mint(value: string = PRESENT): Promise<{ secret: string; expiresInSeconds: number }> { diff --git a/services/api/src/auth/stores.ts b/services/api/src/auth/stores.ts index af4759ec..bb0e6886 100644 --- a/services/api/src/auth/stores.ts +++ b/services/api/src/auth/stores.ts @@ -5,22 +5,30 @@ import { decideHandleWrite, handleChanged } from "./handle-policy.js" export const HANDLE_RENAME_COOLDOWN_MS = 30 * 24 * 60 * 60 * 1000 +const PLACEHOLDER_HANDLE_PREFIX = "user" + +const TOMBSTONE_HANDLE_PREFIX = "deleted_" + +const GENERATED_HANDLE_HEX_LENGTH = 12 + export function handleChangeableAtFrom(handleChangedAt: Date | null, now: Date): string | null { if (handleChangedAt === null) return null const next = new Date(handleChangedAt.getTime() + HANDLE_RENAME_COOLDOWN_MS) return next.getTime() > now.getTime() ? next.toISOString() : null } +function uuidHexPrefix(uuid: string): string { + return uuid.replace(/-/g, "").slice(0, GENERATED_HANDLE_HEX_LENGTH).toLowerCase() +} + export function generatePlaceholderHandle(id?: string): string { - const hex = (id ?? randomUUID()).replace(/-/g, "").slice(0, 12).toLowerCase() - return `user${hex}` + return PLACEHOLDER_HANDLE_PREFIX + uuidHexPrefix(id ?? randomUUID()) } export const TOMBSTONE_HANDLE_RE = /^deleted_[0-9a-f]{12}$/ export function generateTombstoneHandle(): string { - const hex = randomUUID().replace(/-/g, "").slice(0, 12).toLowerCase() - return `deleted_${hex}` + return TOMBSTONE_HANDLE_PREFIX + uuidHexPrefix(randomUUID()) } export type AccountStatus = "active" | "suspended" | "review" | "banned" @@ -215,25 +223,24 @@ export class InMemoryUserStore implements UserStore { return Promise.resolve(row ? { ...row } : null) } - findByEmail(email: string): Promise { + private rowByEmail(email: string): UserRecord | undefined { const normalized = email.toLowerCase() for (const row of this.byId.values()) { - if (row.email !== null && row.email.toLowerCase() === normalized) { - return Promise.resolve({ ...row }) - } + if (row.email !== null && row.email.toLowerCase() === normalized) return row } - return Promise.resolve(null) + return undefined + } + + findByEmail(email: string): Promise { + const row = this.rowByEmail(email) + return Promise.resolve(row ? { ...row } : null) } create(email: string | null, input: CreateUserInput): Promise { - if (email !== null) { - const normalized = email.toLowerCase() - for (const existing of this.byId.values()) { - if (existing.email !== null && existing.email.toLowerCase() === normalized) { - if (input.onEmailConflict === "reject") return Promise.reject(new EmailTakenError()) - return Promise.resolve({ ...existing }) - } - } + const existing = email === null ? undefined : this.rowByEmail(email) + if (existing) { + if (input.onEmailConflict === "reject") return Promise.reject(new EmailTakenError()) + return Promise.resolve({ ...existing }) } const id = randomUUID() const row: UserRecord = { diff --git a/services/api/src/auth/transport.ts b/services/api/src/auth/transport.ts index 8772789a..ed51b314 100644 --- a/services/api/src/auth/transport.ts +++ b/services/api/src/auth/transport.ts @@ -39,7 +39,13 @@ export const CSRF_COOKIE = "civfix_csrf" export const CSRF_COOKIE_HOST = "__Host-civfix_csrf" export const ANON_COOKIE = "civfix_anon" -export function sessionCookieName(): string { +const CLIENT_HEADER = "x-client" + +const MOBILE_CLIENT = "mobile" + +const BEARER_RE = /^Bearer\s+(.+)$/i + +function sessionCookieName(): string { return isProd() ? SESSION_COOKIE_HOST : SESSION_COOKIE } @@ -63,21 +69,19 @@ function firstNonEmptyCookie(request: FastifyRequest, ...names: string[]): strin return null } -export const CLIENT_HEADER = "x-client" - export type ClientKind = "web" | "mobile" export function clientKind(request: FastifyRequest): ClientKind { const raw = request.headers[CLIENT_HEADER] const value = (Array.isArray(raw) ? raw[0] : raw)?.trim().toLowerCase() - return value === "mobile" ? "mobile" : "web" + return value === MOBILE_CLIENT ? "mobile" : "web" } export function bearerToken(request: FastifyRequest): string | null { const raw = request.headers.authorization const header = Array.isArray(raw) ? raw[0] : raw if (!header) return null - const match = /^Bearer\s+(.+)$/i.exec(header.trim()) + const match = BEARER_RE.exec(header.trim()) return match ? match[1]!.trim() : null } diff --git a/services/api/src/db/backfill-jurisdiction-handles.ts b/services/api/src/db/backfill-jurisdiction-handles.ts index 70f6b9ed..27d007a9 100644 --- a/services/api/src/db/backfill-jurisdiction-handles.ts +++ b/services/api/src/db/backfill-jurisdiction-handles.ts @@ -20,8 +20,13 @@ type SqlFragment = postgres.Fragment const BATCH_SIZE = 1000 -/** Exported so the integration harness can drive the same loop main() runs. */ -export async function backfillHandles(sql: Sql): Promise<{ assigned: number; skipped: number }> { +const PG_UNIQUE_VIOLATION = "23505" + +const GEOID_TAIL_LENGTH = 4 + +const FIRST_NUMBERED_SUFFIX = 2 + +async function backfillHandles(sql: Sql): Promise<{ assigned: number; skipped: number }> { let assigned = 0 let skipped = 0 // Catches two rows of the same run deriving the same slug before the DB sees them; the UNIQUE index is @@ -93,8 +98,6 @@ async function assignHandle( } } -const PG_UNIQUE_VIOLATION = "23505" - function isUniqueViolation(err: unknown): boolean { return ( typeof err === "object" && @@ -122,10 +125,10 @@ async function claimHandle( return rows.length > 0 } if (!(await taken(base))) return base - const tail = geoid.slice(-4) + const tail = geoid.slice(-GEOID_TAIL_LENGTH) const withTail = `${base}_${tail}` if (!(await taken(withTail))) return withTail - for (let n = 2; ; n++) { + for (let n = FIRST_NUMBERED_SUFFIX; ; n++) { const candidate = `${withTail}_${n}` if (!(await taken(candidate))) return candidate } diff --git a/services/api/src/db/backfill-jurisdictions.ts b/services/api/src/db/backfill-jurisdictions.ts index 963c1a71..78bb11f0 100644 --- a/services/api/src/db/backfill-jurisdictions.ts +++ b/services/api/src/db/backfill-jurisdictions.ts @@ -8,8 +8,6 @@ import { runDbCli, runIfMain } from "./cli.js" import { backfillReports } from "./backfill-jurisdictions-core.js" -export { backfillReports } from "./backfill-jurisdictions-core.js" - async function main(): Promise { await runDbCli(async (_db, sql) => { const { resolved, stayedNull } = await backfillReports(sql) diff --git a/services/api/src/db/backfill-population-core.ts b/services/api/src/db/backfill-population-core.ts index 23cdc8e4..ffab7c1b 100644 --- a/services/api/src/db/backfill-population-core.ts +++ b/services/api/src/db/backfill-population-core.ts @@ -2,12 +2,18 @@ import type { Sql } from "./client.js" export const ACS_POP_VAR = "B01003_001E" -export const DEFAULT_ACS_YEAR = 2023 +const DEFAULT_ACS_YEAR = 2023 export type CensusJsonFetch = (url: string) => Promise const CENSUS_FETCH_TIMEOUT_MS = 20_000 +const CENSUS_API_BASE = "https://api.census.gov/data" +const CENSUS_KEY_SIGNUP_URL = "https://api.census.gov/data/key_signup.html" + +// One UPDATE per chunk keeps each unnest() parameter array bounded. +const POPULATION_UPDATE_CHUNK = 1000 + const defaultFetchJson: CensusJsonFetch = async (url) => { const controller = new AbortController() const timer = setTimeout(() => controller.abort(), CENSUS_FETCH_TIMEOUT_MS) @@ -16,7 +22,7 @@ const defaultFetchJson: CensusJsonFetch = async (url) => { const contentType = res.headers.get("content-type") ?? "" if (res.url.includes("missing_key") || (!contentType.includes("json") && res.redirected)) { throw new Error( - "Census API requires an API key: set CENSUS_API_KEY (free, instant: https://api.census.gov/data/key_signup.html)", + `Census API requires an API key: set CENSUS_API_KEY (free, instant: ${CENSUS_KEY_SIGNUP_URL})`, ) } if (!res.ok) throw new Error(`Census API ${res.status} ${res.statusText}`) @@ -42,7 +48,7 @@ function acsUrl( params.set("for", forClause) if (inClause) params.set("in", inClause) if (key) params.set("key", key) - return `https://api.census.gov/data/${year}/acs/acs5?${params.toString()}` + return `${CENSUS_API_BASE}/${year}/acs/acs5?${params.toString()}` } /** @@ -77,9 +83,8 @@ async function applyPopulations( rows: { geoid: string; population: number }[], ): Promise { let updated = 0 - const CHUNK = 1000 - for (let i = 0; i < rows.length; i += CHUNK) { - const chunk = rows.slice(i, i + CHUNK) + for (let i = 0; i < rows.length; i += POPULATION_UPDATE_CHUNK) { + const chunk = rows.slice(i, i + POPULATION_UPDATE_CHUNK) const geoids = chunk.map((c) => c.geoid) const pops = chunk.map((c) => c.population) const res = await sql` @@ -134,7 +139,7 @@ export async function backfillPopulation( if (fetched === 0) { log( "fetched 0 ACS rows: every Census call failed. The Census API requires CENSUS_API_KEY " + - "(free, instant: https://api.census.gov/data/key_signup.html); set it and re-run.", + `(free, instant: ${CENSUS_KEY_SIGNUP_URL}); set it and re-run.`, ) return { fetched: 0, updated: 0, states: states.length } } diff --git a/services/api/src/db/backfill-population.ts b/services/api/src/db/backfill-population.ts index 9bd2a6dc..4a62cb01 100644 --- a/services/api/src/db/backfill-population.ts +++ b/services/api/src/db/backfill-population.ts @@ -9,7 +9,7 @@ * Census API rejects unkeyed requests. A free key: https://api.census.gov/data/key_signup.html */ -import { runDbCli, runIfMain } from "./cli.js" +import { EXIT_USAGE, runDbCli, runIfMain } from "./cli.js" import { backfillPopulation } from "./backfill-population-core.js" async function main(): Promise { @@ -17,7 +17,7 @@ async function main(): Promise { const year = yearArg ? Number(yearArg) : undefined if (yearArg && !Number.isInteger(year)) { console.error(`backfill-population: vintage must be a 4-digit year (got "${yearArg}")`) - process.exit(2) + process.exit(EXIT_USAGE) } await runDbCli(async (_db, sql) => { const { fetched, updated, states } = await backfillPopulation(sql, { diff --git a/services/api/src/db/backfill-post-geom-core.ts b/services/api/src/db/backfill-post-geom-core.ts index 05e9c8ee..739c1dfb 100644 --- a/services/api/src/db/backfill-post-geom-core.ts +++ b/services/api/src/db/backfill-post-geom-core.ts @@ -3,7 +3,7 @@ import type { Sql } from "./client.js" type SqlFragment = postgres.Fragment -export const POST_GEOM_BACKFILL_BATCH = 1000 +const POST_GEOM_BACKFILL_BATCH = 1000 export async function backfillPostGeom( sql: Sql, diff --git a/services/api/src/db/backfill-post-geom.ts b/services/api/src/db/backfill-post-geom.ts index d2846d6f..d52d4146 100644 --- a/services/api/src/db/backfill-post-geom.ts +++ b/services/api/src/db/backfill-post-geom.ts @@ -1,8 +1,6 @@ import { runDbCli, runIfMain } from "./cli.js" import { backfillPostGeom } from "./backfill-post-geom-core.js" -export { backfillPostGeom } from "./backfill-post-geom-core.js" - async function main(): Promise { await runDbCli(async (_db, sql) => { const { scanned, filled } = await backfillPostGeom(sql) diff --git a/services/api/src/db/backfill-reference-codes-core.ts b/services/api/src/db/backfill-reference-codes-core.ts index f8608d63..a754379c 100644 --- a/services/api/src/db/backfill-reference-codes-core.ts +++ b/services/api/src/db/backfill-reference-codes-core.ts @@ -19,7 +19,7 @@ const BATCH_SIZE = 500 const LABEL = "reference-codes" -export async function backfillReportReferenceCodes( +async function backfillReportReferenceCodes( sql: Sql, ): Promise<{ stamped: number; failed: number }> { return stampReferenceCodes(sql, { @@ -43,7 +43,7 @@ export async function backfillCleanupJurisdictions( } /** Run after backfillCleanupJurisdictions so each event's JURCODE is resolved. */ -export async function backfillCleanupReferenceCodes( +async function backfillCleanupReferenceCodes( sql: Sql, ): Promise<{ stamped: number; failed: number }> { return stampReferenceCodes(sql, { diff --git a/services/api/src/db/backfill-reference-codes.ts b/services/api/src/db/backfill-reference-codes.ts index 2d89dc67..c0b3fa3a 100644 --- a/services/api/src/db/backfill-reference-codes.ts +++ b/services/api/src/db/backfill-reference-codes.ts @@ -9,8 +9,6 @@ import { runDbCli, runIfMain } from "./cli.js" import { backfillReferenceCodes } from "./backfill-reference-codes-core.js" -export { backfillReferenceCodes } from "./backfill-reference-codes-core.js" - async function main(): Promise { await runDbCli(async (_db, sql) => { const { reports, cleanupJurisdictions, cleanups } = await backfillReferenceCodes(sql) diff --git a/services/api/src/db/backfill-signup-seats.ts b/services/api/src/db/backfill-signup-seats.ts index b49f860e..3dafc8d4 100644 --- a/services/api/src/db/backfill-signup-seats.ts +++ b/services/api/src/db/backfill-signup-seats.ts @@ -17,24 +17,25 @@ import { randomUUID } from "node:crypto" import type postgres from "postgres" import type { Sql } from "./client.js" -import { runDbCli, runIfMain } from "./cli.js" +import { EXIT_USAGE, runDbCli, runIfMain } from "./cli.js" import { loadEnv } from "../env.js" import { makeTicketTokenSigner } from "../services/host/ticket-token.js" type SqlFragment = postgres.Fragment -export const SIGNUP_SEAT_BACKFILL_BATCH = 500 +const SIGNUP_SEAT_BACKFILL_BATCH = 500 export const SIGNUP_SEAT_BACKFILL_MAX_BATCH = 5000 const BATCH_FLAG = "--batch" +const DIGITS_ONLY_RE = /^\d+$/ // undefined = flag absent (use the default); null = a value that is not a usable LIMIT. export function parseSignupSeatBatchArg(argv: readonly string[]): number | undefined | null { const at = argv.indexOf(BATCH_FLAG) if (at < 0) return undefined const raw = argv[at + 1] - if (raw === undefined || !/^\d+$/.test(raw)) return null + if (raw === undefined || !DIGITS_ONLY_RE.test(raw)) return null const size = Number(raw) return size >= 1 && size <= SIGNUP_SEAT_BACKFILL_MAX_BATCH ? size : null } @@ -74,13 +75,48 @@ interface SeatInsertRow { created_at: Date } -export interface SignupSeatBackfillResult { +interface SignupSeatBackfillResult { scanned: number created: number batches: number } -export async function backfillSignupSeats( +/** One free registration plus its single seat per candidate, mirroring the live sign-up write. */ +function signupRowsFor( + page: readonly CandidateRow[], + hashFor: (seatId: string) => string, +): { registrations: RegistrationInsertRow[]; seatOf: Map } { + const registrations: RegistrationInsertRow[] = page.map((row) => ({ + id: randomUUID(), + cleanup_id: row.cleanup_id, + ticket_type_id: null, + user_id: row.user_id, + guest_id: null, + party_size: 1, + status: "registered", + source: "self", + registered_at: row.joined_at, + })) + const seatOf = new Map( + registrations.map((registration) => { + const seatId = randomUUID() + const seat: SeatInsertRow = { + id: seatId, + cleanup_id: registration.cleanup_id, + registration_id: registration.id, + seat_index: 0, + attendee_name: null, + ticket_token_hash: hashFor(seatId), + status: "active", + created_at: registration.registered_at, + } + return [registration.id, seat] + }), + ) + return { registrations, seatOf } +} + +async function backfillSignupSeats( sql: Sql, opts: { hashFor: (seatId: string) => string @@ -122,33 +158,7 @@ export async function backfillSignupSeats( ` if (page.length === 0) break - const registrations: RegistrationInsertRow[] = page.map((row) => ({ - id: randomUUID(), - cleanup_id: row.cleanup_id, - ticket_type_id: null, - user_id: row.user_id, - guest_id: null, - party_size: 1, - status: "registered", - source: "self", - registered_at: row.joined_at, - })) - const seatOf = new Map( - registrations.map((registration) => { - const seatId = randomUUID() - const seat: SeatInsertRow = { - id: seatId, - cleanup_id: registration.cleanup_id, - registration_id: registration.id, - seat_index: 0, - attendee_name: null, - ticket_token_hash: opts.hashFor(seatId), - status: "active", - created_at: registration.registered_at, - } - return [registration.id, seat] - }), - ) + const { registrations, seatOf } = signupRowsFor(page, opts.hashFor) const written = await sql .begin(async (tx) => { @@ -208,14 +218,14 @@ export async function backfillSignupSeats( return { scanned, created, batches } } -export async function main(): Promise { +async function main(): Promise { const commit = process.argv.includes("--yes") const batchSize = parseSignupSeatBatchArg(process.argv) if (batchSize === null) { console.error( `backfill-signup-seats: ${BATCH_FLAG} takes an integer from 1 to ${SIGNUP_SEAT_BACKFILL_MAX_BATCH}`, ) - process.exit(2) + process.exit(EXIT_USAGE) } const signer = makeTicketTokenSigner(loadEnv().TICKET_TOKEN_SECRET.trim()) diff --git a/services/api/src/db/backfill-user-activity-core.ts b/services/api/src/db/backfill-user-activity-core.ts index 848813a0..2cae6c81 100644 --- a/services/api/src/db/backfill-user-activity-core.ts +++ b/services/api/src/db/backfill-user-activity-core.ts @@ -3,7 +3,7 @@ import type { Sql } from "./client.js" type SqlFragment = postgres.Fragment -export const USER_ACTIVITY_BACKFILL_BATCH = 500 +const USER_ACTIVITY_BACKFILL_BATCH = 500 export async function backfillUserActivity( sql: Sql, diff --git a/services/api/src/db/backfill-user-activity.ts b/services/api/src/db/backfill-user-activity.ts index 9757e97e..afd1f82d 100644 --- a/services/api/src/db/backfill-user-activity.ts +++ b/services/api/src/db/backfill-user-activity.ts @@ -1,8 +1,6 @@ import { runDbCli, runIfMain } from "./cli.js" import { backfillUserActivity } from "./backfill-user-activity-core.js" -export { backfillUserActivity } from "./backfill-user-activity-core.js" - async function main(): Promise { await runDbCli(async (_db, sql) => { const { scanned, filled } = await backfillUserActivity(sql) diff --git a/services/api/src/db/boundaries/manifest.ts b/services/api/src/db/boundaries/manifest.ts index c358ae3c..04bf7984 100644 --- a/services/api/src/db/boundaries/manifest.ts +++ b/services/api/src/db/boundaries/manifest.ts @@ -54,21 +54,32 @@ export function vintageTag(tigerVintage: number, padusVersion: string = PADUS_VE return `tiger${tigerVintage}-padus${padusVersion}` } -/** For the runbook and citation (DOI 10.5066/P96WBCHS); the machine fetch uses PADUS_GDB_URL. */ -export const PADUS_DOWNLOAD_URL = - "https://www.usgs.gov/programs/gap-analysis-project/science/pad-us-data-download" - /** + * The human download page, for the runbook and citation (DOI 10.5066/P96WBCHS), is + * https://www.usgs.gov/programs/gap-analysis-project/science/pad-us-data-download. + * * The ScienceBase item id is specific to the 4.1 release and not derivable from the version, so bumping * PADUS_VERSION means finding the new "Full Inventory Database" item. Only the catalog/file/get pattern * serves the zip (the `manager/` hosts serve an HTML shell), and it ignores HTTP Range, so the full * ~1.5 GB is pulled every run. */ -export const PADUS_GDB_URL = +const PADUS_GDB_URL = "https://www.sciencebase.gov/catalog/file/get/652d4fc5d34e44db0e2ee45e?name=PADUS4_1Geodatabase.zip" const PADUS_VERSION_NODOT = PADUS_VERSION.replace(/\./g, "_") +const FIRST_TIGER_VINTAGE = 2007 +const MAX_TIGER_VINTAGE = 2100 + +/** The CRS the geom column and the ST_GeomFromGeoJSON ingest path assume (invariant 1 above). */ +const TARGET_SRS = "EPSG:4326" + +/** MTFCC of an incorporated place with governmental authority (see the header). */ +const INCORPORATED_PLACE_MTFCC = "G4110" + +const AIANNH_GEOID_PREFIX = "AIANNH-" +const PADUS_GEOID_PREFIX = "PADUS-" + /** * Territories and DC (11, 60, 66, 69, 72, 78) are deliberately not covered yet. Order matches the * geocoder's STATE_FIPS_TO_USPS key order. @@ -134,8 +145,8 @@ export function boundaryManifest( vintage: number | "latest" = DEFAULT_TIGER_VINTAGE, ): BoundaryJob[] { const year = vintage === "latest" ? DEFAULT_TIGER_VINTAGE : vintage - // Fail here rather than 404 later on a URL like .../TIGERNaN. TIGER began in 2007. - if (!Number.isInteger(year) || year < 2007 || year > 2100) { + // Fail here rather than 404 later on a URL like .../TIGERNaN. + if (!Number.isInteger(year) || year < FIRST_TIGER_VINTAGE || year > MAX_TIGER_VINTAGE) { throw new Error(`boundaryManifest: invalid TIGER vintage ${String(vintage)}`) } const root = tigerRoot(year) @@ -145,7 +156,7 @@ export function boundaryManifest( jobs.push({ sourceUrl: `${root}/STATE/tl_${year}_us_state.zip`, layer: "state", - ogr2ogrArgs: ["-f", "GeoJSON", "-t_srs", "EPSG:4326", "-makevalid"], + ogr2ogrArgs: ["-f", "GeoJSON", "-t_srs", TARGET_SRS, "-makevalid"], outFile: "states.geojson", sourcePath: `tl_${year}_us_state.shp`, ingestGeoidPrefix: null, @@ -154,7 +165,7 @@ export function boundaryManifest( jobs.push({ sourceUrl: `${root}/COUNTY/tl_${year}_us_county.zip`, layer: "county", - ogr2ogrArgs: ["-f", "GeoJSON", "-t_srs", "EPSG:4326", "-makevalid"], + ogr2ogrArgs: ["-f", "GeoJSON", "-t_srs", TARGET_SRS, "-makevalid"], outFile: "counties.geojson", sourcePath: `tl_${year}_us_county.shp`, ingestGeoidPrefix: null, @@ -168,10 +179,10 @@ export function boundaryManifest( "-f", "GeoJSON", "-t_srs", - "EPSG:4326", + TARGET_SRS, "-makevalid", "-where", - "MTFCC='G4110'", + `MTFCC='${INCORPORATED_PLACE_MTFCC}'`, ], outFile: `places_${ss}.geojson`, sourcePath: `tl_${year}_${ss}_place.shp`, @@ -183,10 +194,10 @@ export function boundaryManifest( jobs.push({ sourceUrl: `${root}/AIANNH/tl_${year}_us_aiannh.zip`, layer: "tribal", - ogr2ogrArgs: ["-f", "GeoJSON", "-t_srs", "EPSG:4326", "-makevalid"], + ogr2ogrArgs: ["-f", "GeoJSON", "-t_srs", TARGET_SRS, "-makevalid"], outFile: "aiannh.geojson", sourcePath: `tl_${year}_us_aiannh.shp`, - ingestGeoidPrefix: "AIANNH-", + ingestGeoidPrefix: AIANNH_GEOID_PREFIX, }) // The default OGR SQL dialect, not SQLITE: with an explicit column projection SQLITE can drop the @@ -209,7 +220,7 @@ export function boundaryManifest( "-f", "GeoJSONSeq", "-t_srs", - "EPSG:4326", + TARGET_SRS, "-makevalid", "-sql", `SELECT OBJECTID AS GEOID, Unit_Nm AS NAME FROM PADUS${PADUS_VERSION_NODOT}Fee WHERE Mang_Type='FED'`, @@ -218,7 +229,7 @@ export function boundaryManifest( ], outFile: "federal.geojsonl", sourcePath: `PADUS${PADUS_VERSION_NODOT}Geodatabase.gdb`, - ingestGeoidPrefix: "PADUS-", + ingestGeoidPrefix: PADUS_GEOID_PREFIX, }) return jobs diff --git a/services/api/src/db/cli.ts b/services/api/src/db/cli.ts index 8abdab59..988f5786 100644 --- a/services/api/src/db/cli.ts +++ b/services/api/src/db/cli.ts @@ -3,13 +3,22 @@ import type { Db, Sql } from "./client.js" import { makeDb } from "./client.js" import { loadEnv } from "../env.js" +const DEFAULT_CLI_POOL_MAX = 1 +const EXIT_FAILURE = 1 + +export const EXIT_USAGE = 2 + +/** + * Timeouts off: these are exactly the long statements the request-path timeouts exist to kill. A + * `databaseUrl` skips loadEnv, so the demo CLIs run from a minimal shell without the API's full env. + */ export async function runDbCli( body: (db: Db, sql: Sql) => Promise, - opts: { max?: number } = {}, + opts: { max?: number; databaseUrl?: string } = {}, ): Promise { - const env = loadEnv() - const handle = makeDb(env.DATABASE_URL, { - max: opts.max ?? 1, + const databaseUrl = opts.databaseUrl ?? loadEnv().DATABASE_URL + const handle = makeDb(databaseUrl, { + max: opts.max ?? DEFAULT_CLI_POOL_MAX, statementTimeoutMs: 0, idleInTxTimeoutMs: 0, }) @@ -20,7 +29,18 @@ export async function runDbCli( } } -export function isMainModule(importMetaUrl: string): boolean { +export function requireDatabaseUrl(): string { + const databaseUrl = process.env.DATABASE_URL + if (!databaseUrl) throw new Error("DATABASE_URL is required") + return databaseUrl +} + +export function argValue(name: string): string | undefined { + const idx = process.argv.indexOf(name) + return idx >= 0 ? process.argv[idx + 1] : undefined +} + +function isMainModule(importMetaUrl: string): boolean { return process.argv[1] !== undefined && fileURLToPath(importMetaUrl) === process.argv[1] } @@ -29,6 +49,6 @@ export function runIfMain(importMetaUrl: string, label: string, main: () => Prom main().catch((err: unknown) => { console.error(`${label}: failed`) console.error(err) - process.exit(1) + process.exit(EXIT_FAILURE) }) } diff --git a/services/api/src/db/client.ts b/services/api/src/db/client.ts index 88eec8ab..57300d30 100644 --- a/services/api/src/db/client.ts +++ b/services/api/src/db/client.ts @@ -13,6 +13,15 @@ export interface DbHandle { close(): Promise } +const DEFAULT_STATEMENT_TIMEOUT_MS = 15_000 +const DEFAULT_IDLE_IN_TX_TIMEOUT_MS = 30_000 +const CONNECT_TIMEOUT_S = 10 +const IDLE_TIMEOUT_S = 60 +const MAX_LIFETIME_S = 30 * 60 +const CLOSE_TIMEOUT_S = 5 +const DEFAULT_POOL_MAX = 10 +const DRIZZLE_POOL_MAX = 4 + /** `false` = plaintext, for dev and testcontainers only. */ export type DbSslOption = false | "require" | "verify-full" | { rejectUnauthorized: true } @@ -56,8 +65,8 @@ export function makeDb( if (!databaseUrl) { throw new Error("makeDb: databaseUrl is required") } - const statementTimeoutMs = opts.statementTimeoutMs ?? 15_000 - const idleInTxTimeoutMs = opts.idleInTxTimeoutMs ?? 30_000 + const statementTimeoutMs = opts.statementTimeoutMs ?? DEFAULT_STATEMENT_TIMEOUT_MS + const idleInTxTimeoutMs = opts.idleInTxTimeoutMs ?? DEFAULT_IDLE_IN_TX_TIMEOUT_MS const connection: Record = { TimeZone: "UTC" } if (statementTimeoutMs > 0) connection.statement_timeout = String(statementTimeoutMs) if (idleInTxTimeoutMs > 0) { @@ -67,21 +76,24 @@ export function makeDb( // Explicit on both clients: postgres.js's default is plaintext, so this is the single place TLS is // decided for every query the API and the raw repositories make. ssl: opts.ssl ?? sslOptionForUrl(databaseUrl), - connect_timeout: 10, - idle_timeout: 60, - max_lifetime: 60 * 30, + connect_timeout: CONNECT_TIMEOUT_S, + idle_timeout: IDLE_TIMEOUT_S, + max_lifetime: MAX_LIFETIME_S, onnotice: () => {}, connection, } - const sql = postgres(databaseUrl, { ...common, max: opts.max ?? 10 }) - const drizzleSql = postgres(databaseUrl, { ...common, max: 4 }) + const sql = postgres(databaseUrl, { ...common, max: opts.max ?? DEFAULT_POOL_MAX }) + const drizzleSql = postgres(databaseUrl, { ...common, max: DRIZZLE_POOL_MAX }) const db = drizzle(drizzleSql, { schema }) let closed = false async function close(): Promise { if (closed) return closed = true - await Promise.all([sql.end({ timeout: 5 }), drizzleSql.end({ timeout: 5 })]) + await Promise.all([ + sql.end({ timeout: CLOSE_TIMEOUT_S }), + drizzleSql.end({ timeout: CLOSE_TIMEOUT_S }), + ]) } return { db, sql, close } diff --git a/services/api/src/db/cursor-helpers.ts b/services/api/src/db/cursor-helpers.ts index 1db551c3..d6f75b59 100644 --- a/services/api/src/db/cursor-helpers.ts +++ b/services/api/src/db/cursor-helpers.ts @@ -15,15 +15,14 @@ export interface TimeCursor { export const MIN_UUID = "00000000-0000-0000-0000-000000000000" export const MAX_UUID = "ffffffff-ffff-ffff-ffff-ffffffffffff" -export const CURSOR_ISO_RE = - /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})$/ +const CURSOR_ISO_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})$/ // Postgres to_char pattern for a timestamptz rendered in UTC at full microsecond precision. A keyset // cursor built from the millisecond Date that postgres.js returns sits below the row it came from. export const TIME_CURSOR_SQL_FORMAT = 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' -export const CURSOR_MIN_MS = Date.UTC(1970, 0, 1) -export const CURSOR_MAX_MS = Date.UTC(2100, 0, 1) +const CURSOR_MIN_MS = Date.UTC(1970, 0, 1) +const CURSOR_MAX_MS = Date.UTC(2100, 0, 1) function parseCursorInstant(iso: string): Date | null { if (!CURSOR_ISO_RE.test(iso)) return null diff --git a/services/api/src/db/demo-join-event.ts b/services/api/src/db/demo-join-event.ts index 8630d095..7fc7ba16 100644 --- a/services/api/src/db/demo-join-event.ts +++ b/services/api/src/db/demo-join-event.ts @@ -17,56 +17,50 @@ * uuid. Optional: --count N (default 15), --seed N (PRNG seed, default 20260902). */ -import { makeDb, type TransactionSql } from "./client.js" -import { runIfMain } from "./cli.js" -import { DEMO_EMAIL_DOMAIN } from "./seed-demo-domain.js" +import type { TransactionSql } from "./client.js" +import { argValue, requireDatabaseUrl, runDbCli, runIfMain } from "./cli.js" +import { isUuid } from "./cursor-helpers.js" +import { EVENT_PREFIX } from "./reference-code.js" +import { DEMO_EMAIL_DOMAIN, DEMO_EMAIL_PATTERN } from "./seed-demo-domain.js" import { demoTicketTokenHasher, mintDemoSignupSeats } from "./demo-signup-seats.js" +import { DEMO_PRNG_SEED, chance, rand, rint, seedDemoRandom, shuffle } from "./demo-random.js" import { deriveCleanupStatus, eventWindowOf } from "../services/cleanup-rules.js" -let rand = (): number => Math.random() +const MINUTE_MS = 60_000 +const HOUR_MS = 60 * MINUTE_MS +const DAY_MS = 24 * HOUR_MS -function mulberry32(seed: number): () => number { - let a = seed >>> 0 - return () => { - a |= 0 - a = (a + 0x6d2b79f5) | 0 - let t = Math.imul(a ^ (a >>> 15), 1 | a) - t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t - return ((t ^ (t >>> 14)) >>> 0) / 4294967296 - } -} -function rint(min: number, max: number): number { - return min + Math.floor(rand() * (max - min + 1)) -} -function chance(p: number): boolean { - return rand() < p -} -function shuffle(arr: T[]): T[] { - for (let i = arr.length - 1; i > 0; i--) { - const j = Math.floor(rand() * (i + 1)) - ;[arr[i], arr[j]] = [arr[j]!, arr[i]!] - } - return arr -} +const DEFAULT_JOIN_COUNT = 15 +const MAX_JOIN_COUNT = 200 +const SLOT_CLAIM_PROBABILITY = 0.45 + +// RSVPs start after the event existed; the spread is capped at the last 5 days so it reads like a burst +// of signups rather than rows backdated before the event was announced. The start is clamped to at +// least 12h ago so an event with a bogus/near-future created_at still yields a spread of past join +// times instead of a one-minute cluster. +const RSVP_SPREAD_MS = 5 * DAY_MS +const MIN_RSVP_LOOKBACK_MS = 12 * HOUR_MS /** Biased toward the recent end, because RSVPs cluster after a promo post. */ function joinTimestamp(start: Date, now: Date): Date { - const span = Math.max(now.getTime() - start.getTime(), 60_000) + const span = Math.max(now.getTime() - start.getTime(), MINUTE_MS) const t = new Date(now.getTime() - Math.pow(rand(), 2) * span) t.setUTCMinutes(rint(0, 59), rint(0, 59), 0) // The minute/second jitter can overshoot either bound; clamp inside (start, now). - if (t.getTime() >= now.getTime()) return new Date(now.getTime() - rint(2, 45) * 60_000) - if (t.getTime() <= start.getTime()) return new Date(start.getTime() + 60_000) + if (t.getTime() >= now.getTime()) return new Date(now.getTime() - rint(2, 45) * MINUTE_MS) + if (t.getTime() <= start.getTime()) return new Date(start.getTime() + MINUTE_MS) return t } -function argValue(name: string): string | undefined { - const idx = process.argv.indexOf(name) - return idx >= 0 ? process.argv[idx + 1] : undefined +function rsvpWindowStart(createdAt: Date, now: Date): Date { + return new Date( + Math.min( + Math.max(createdAt.getTime(), now.getTime() - RSVP_SPREAD_MS), + now.getTime() - MIN_RSVP_LOOKBACK_MS, + ), + ) } -const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i - interface EventRow { id: string title: string @@ -78,15 +72,23 @@ interface EventRow { organizer_user_id: string } +interface MemberRow { + cleanup_id: string + user_id: string + role: string + joined_at: Date +} + async function loadEvent(tx: TransactionSql, ref: string): Promise { - if (UUID_RE.test(ref)) { + if (isUuid(ref)) { const rows = await tx` SELECT id, title, status, scheduled_at, ends_at, created_at, capacity, organizer_user_id FROM cleanups WHERE id = ${ref} LIMIT 1 ` return rows[0] ?? null } - const code = ref.toUpperCase().startsWith("EVENT-") ? ref.toUpperCase() : `EVENT-${ref}` + const upper = ref.toUpperCase() + const code = upper.startsWith(`${EVENT_PREFIX}-`) ? upper : `${EVENT_PREFIX}-${ref}` const rows = await tx` SELECT id, title, status, scheduled_at, ends_at, created_at, capacity, organizer_user_id FROM cleanups WHERE reference_code = ${code} LIMIT 1 @@ -94,176 +96,186 @@ async function loadEvent(tx: TransactionSql, ref: string): Promise { - const commit = process.argv.includes("--yes") - const eventRef = argValue("--event") - const count = Number(argValue("--count") ?? 15) - const prngSeed = Number(argValue("--seed") ?? 20260902) - rand = mulberry32(prngSeed) - - if (!eventRef) - throw new Error("--event is required (e.g. --event 1695-000006)") - if (!Number.isInteger(count) || count < 1 || count > 200) - throw new Error("--count must be 1..200") - const databaseUrl = process.env.DATABASE_URL - if (!databaseUrl) throw new Error("DATABASE_URL is required") - const hashFor = demoTicketTokenHasher() - - console.log(`target database: ${new URL(databaseUrl).host}`) - console.log( - commit - ? "mode: COMMIT" - : "mode: rehearsal (runs everything, then ROLLBACK; pass --yes to commit)", +/** Takes the same lock the live join takes, so a concurrent cancel/complete cannot race the join. */ +async function loadJoinableEvent(tx: TransactionSql, eventRef: string): Promise { + const event = await loadEvent(tx, eventRef) + if (!event) throw new Error(`event not found for "${eventRef}"`) + await tx`SELECT status FROM cleanups WHERE id = ${event.id} FOR SHARE` + const derived = deriveCleanupStatus( + eventWindowOf({ + status: event.status === "cancelled" ? "cancelled" : "upcoming", + scheduledAt: event.scheduled_at, + endsAt: event.ends_at, + }), + Date.now(), ) + if (derived === "done" || derived === "cancelled") { + throw new Error( + `event "${event.title}" is ${derived}; the live join path refuses closed events`, + ) + } + console.log(`event: ${event.title} (${derived}, scheduled ${event.scheduled_at.toISOString()})`) + return event +} - const handle = makeDb(databaseUrl, { max: 1, statementTimeoutMs: 0, idleInTxTimeoutMs: 0 }) - const ROLLBACK = Symbol("rollback") - try { - const outcome = await handle.sql - .begin(async (tx) => { - // Same lock the live join takes, so a concurrent cancel/complete cannot race us. - const event = await loadEvent(tx, eventRef) - if (!event) throw new Error(`event not found for "${eventRef}"`) - await tx`SELECT status FROM cleanups WHERE id = ${event.id} FOR SHARE` - const derived = deriveCleanupStatus( - eventWindowOf({ - status: event.status === "cancelled" ? "cancelled" : "upcoming", - scheduledAt: event.scheduled_at, - endsAt: event.ends_at, - }), - Date.now(), - ) - if (derived === "done" || derived === "cancelled") { - throw new Error( - `event "${event.title}" is ${derived}; the live join path refuses closed events`, - ) - } - console.log( - `event: ${event.title} (${derived}, scheduled ${event.scheduled_at.toISOString()})`, - ) - - const candidates = await tx<{ id: string; handle: string; created_at: Date }[]>` +async function eligibleDemoUsers( + tx: TransactionSql, + event: EventRow, +): Promise<{ id: string; handle: string; created_at: Date }[]> { + const candidates = await tx<{ id: string; handle: string; created_at: Date }[]>` SELECT u.id, u.handle, u.created_at FROM users u - WHERE u.email LIKE ${"%@" + DEMO_EMAIL_DOMAIN} + WHERE u.email LIKE ${DEMO_EMAIL_PATTERN} AND u.deleted_at IS NULL AND u.id <> ${event.organizer_user_id} AND NOT EXISTS (SELECT 1 FROM cleanup_members m WHERE m.cleanup_id = ${event.id} AND m.user_id = u.id) AND NOT EXISTS (SELECT 1 FROM cleanup_bans b WHERE b.cleanup_id = ${event.id} AND b.user_id = u.id) ` - if (candidates.length === 0) { - throw new Error( - `no eligible demo users found (@${DEMO_EMAIL_DOMAIN}); run seed-demo-la first`, - ) - } + if (candidates.length === 0) { + throw new Error(`no eligible demo users found (@${DEMO_EMAIL_DOMAIN}); run seed-demo-la first`) + } + return candidates +} - // The same sum goingCount measures: members plus non-cancelled guests. - let room = Number.POSITIVE_INFINITY - if (event.capacity !== null) { - const [going] = await tx<{ n: number }[]>` - SELECT (SELECT count(*)::int FROM cleanup_members m WHERE m.cleanup_id = ${event.id}) +/** The same sum the cleanup repository's goingCount measures: members plus non-cancelled guests. */ +async function goingCount(tx: TransactionSql, eventId: string): Promise { + const [going] = await tx<{ n: number }[]>` + SELECT (SELECT count(*)::int FROM cleanup_members m WHERE m.cleanup_id = ${eventId}) + (SELECT count(*)::int FROM cleanup_guests g - WHERE g.cleanup_id = ${event.id} AND g.cancelled_at IS NULL) AS n + WHERE g.cleanup_id = ${eventId} AND g.cancelled_at IS NULL) AS n ` - room = Math.max(0, event.capacity - Number(going?.n ?? 0)) - if (room === 0) throw new Error(`event is at capacity (${event.capacity}); nothing to do`) - } - - const joiners = shuffle([...candidates]).slice(0, Math.min(count, candidates.length, room)) - const now = new Date() - // RSVPs start after the event existed; cap the spread at the last 5 days so it reads like a - // burst of signups rather than rows backdated before the event was announced. Clamp the start - // to at least 12h ago so an event with a bogus/near-future created_at still yields a spread of - // past join times instead of a one-minute cluster. - const windowStart = new Date( - Math.min( - Math.max(event.created_at.getTime(), now.getTime() - 5 * 24 * 3600_000), - now.getTime() - 12 * 3600_000, - ), - ) + return Number(going?.n ?? 0) +} - const memberRows = joiners - .map((u) => ({ - cleanup_id: event.id, - user_id: u.id, - role: "member", - joined_at: joinTimestamp(windowStart, now), - })) - .sort((a, b) => a.joined_at.getTime() - b.joined_at.getTime()) - await tx`INSERT INTO cleanup_members ${tx(memberRows)}` - const seats = await mintDemoSignupSeats(tx, { - cleanupId: event.id, - members: memberRows, - hashFor, - }) +async function remainingRoom(tx: TransactionSql, event: EventRow): Promise { + if (event.capacity === null) return Number.POSITIVE_INFINITY + const room = Math.max(0, event.capacity - (await goingCount(tx, event.id))) + if (room === 0) throw new Error(`event is at capacity (${event.capacity}); nothing to do`) + return room +} - const slots = await tx< - { id: string; title: string; capacity: number | null; claims: number }[] - >` +async function claimDemoSlots( + tx: TransactionSql, + eventId: string, + memberRows: readonly MemberRow[], +): Promise { + const slots = await tx<{ id: string; title: string; capacity: number | null; claims: number }[]>` SELECT s.id, s.title, s.capacity, (SELECT count(*)::int FROM cleanup_slot_claims c WHERE c.slot_id = s.id) AS claims - FROM cleanup_slots s WHERE s.cleanup_id = ${event.id} + FROM cleanup_slots s WHERE s.cleanup_id = ${eventId} ORDER BY s.sort_order, s.id ` - let claimed = 0 - if (slots.length > 0) { - const open = slots.map((s) => ({ ...s, claims: Number(s.claims) })) - for (const m of memberRows) { - if (!chance(0.45)) continue - const slot = shuffle( - open.filter((s) => s.capacity === null || s.claims < s.capacity), - )[0] - if (!slot) break - slot.claims++ - claimed++ - await tx` + if (slots.length === 0) return 0 + let claimed = 0 + const open = slots.map((s) => ({ ...s, claims: Number(s.claims) })) + for (const m of memberRows) { + if (!chance(SLOT_CLAIM_PROBABILITY)) continue + const slot = shuffle(open.filter((s) => s.capacity === null || s.claims < s.capacity))[0] + if (!slot) break + slot.claims++ + claimed++ + await tx` INSERT INTO cleanup_slot_claims (cleanup_id, user_id, slot_id, claimed_at) - VALUES (${event.id}, ${m.user_id}, ${slot.id}, ${new Date(m.joined_at.getTime() + rint(1, 30) * 60_000)}) + VALUES (${eventId}, ${m.user_id}, ${slot.id}, ${new Date(m.joined_at.getTime() + rint(1, 30) * MINUTE_MS)}) ON CONFLICT (cleanup_id, user_id) DO NOTHING ` - } - } - - const handles = joiners.map((u) => `@${u.handle}`).join(", ") - console.log( - `joining ${memberRows.length} demo users (${seats} registrations)` + - `${claimed > 0 ? ` (${claimed} slot claims)` : ""}:`, - ) - console.log(` ${handles}`) + } + return claimed +} - const [overCap] = await tx<{ n: number }[]>` +async function verifyJoin(tx: TransactionSql, event: EventRow): Promise { + const [overCap] = await tx<{ n: number }[]>` SELECT count(*)::int AS n FROM cleanup_slots s WHERE s.cleanup_id = ${event.id} AND s.capacity IS NOT NULL AND (SELECT count(*) FROM cleanup_slot_claims c WHERE c.slot_id = s.id) > s.capacity ` - if (Number(overCap?.n ?? 0) > 0) - throw new Error("verification failed: a slot is over capacity") - if (event.capacity !== null) { - const [going] = await tx<{ n: number }[]>` - SELECT (SELECT count(*)::int FROM cleanup_members m WHERE m.cleanup_id = ${event.id}) - + (SELECT count(*)::int FROM cleanup_guests g - WHERE g.cleanup_id = ${event.id} AND g.cancelled_at IS NULL) AS n - ` - if (Number(going?.n ?? 0) > event.capacity) - throw new Error("verification failed: event over capacity") - } + if (Number(overCap?.n ?? 0) > 0) throw new Error("verification failed: a slot is over capacity") + if (event.capacity !== null && (await goingCount(tx, event.id)) > event.capacity) { + throw new Error("verification failed: event over capacity") + } +} - if (!commit) throw ROLLBACK - return memberRows.length - }) - .catch((e: unknown) => { - if (e === ROLLBACK) return "rolledback" as const - throw e - }) +async function joinDemoUsers( + tx: TransactionSql, + opts: { eventRef: string; count: number; hashFor: (seatId: string) => string }, +): Promise { + const event = await loadJoinableEvent(tx, opts.eventRef) + const candidates = await eligibleDemoUsers(tx, event) + const room = await remainingRoom(tx, event) - if (outcome === "rolledback") { - console.log("rehearsal complete, rolled back. Re-run with --yes to commit.") - } else { - console.log(`committed: ${outcome} joins.`) - } - } finally { - await handle.close() - } + const joiners = shuffle([...candidates]).slice(0, Math.min(opts.count, candidates.length, room)) + const now = new Date() + const windowStart = rsvpWindowStart(event.created_at, now) + const memberRows: MemberRow[] = joiners + .map((u) => ({ + cleanup_id: event.id, + user_id: u.id, + role: "member", + joined_at: joinTimestamp(windowStart, now), + })) + .sort((a, b) => a.joined_at.getTime() - b.joined_at.getTime()) + await tx`INSERT INTO cleanup_members ${tx(memberRows)}` + const seats = await mintDemoSignupSeats(tx, { + cleanupId: event.id, + members: memberRows, + hashFor: opts.hashFor, + }) + const claimed = await claimDemoSlots(tx, event.id, memberRows) + + const handles = joiners.map((u) => `@${u.handle}`).join(", ") + console.log( + `joining ${memberRows.length} demo users (${seats} registrations)` + + `${claimed > 0 ? ` (${claimed} slot claims)` : ""}:`, + ) + console.log(` ${handles}`) + + await verifyJoin(tx, event) + return memberRows.length +} + +async function main(): Promise { + const commit = process.argv.includes("--yes") + const eventRef = argValue("--event") + const count = Number(argValue("--count") ?? DEFAULT_JOIN_COUNT) + const prngSeed = Number(argValue("--seed") ?? DEMO_PRNG_SEED) + seedDemoRandom(prngSeed) + + if (!eventRef) + throw new Error("--event is required (e.g. --event 1695-000006)") + if (!Number.isInteger(count) || count < 1 || count > MAX_JOIN_COUNT) + throw new Error(`--count must be 1..${MAX_JOIN_COUNT}`) + const databaseUrl = requireDatabaseUrl() + const hashFor = demoTicketTokenHasher() + + console.log(`target database: ${new URL(databaseUrl).host}`) + console.log( + commit + ? "mode: COMMIT" + : "mode: rehearsal (runs everything, then ROLLBACK; pass --yes to commit)", + ) + + const ROLLBACK = Symbol("rollback") + await runDbCli( + async (_db, sql) => { + const outcome = await sql + .begin(async (tx) => { + const joined = await joinDemoUsers(tx, { eventRef, count, hashFor }) + if (!commit) throw ROLLBACK + return joined + }) + .catch((e: unknown) => { + if (e === ROLLBACK) return "rolledback" as const + throw e + }) + + if (outcome === "rolledback") { + console.log("rehearsal complete, rolled back. Re-run with --yes to commit.") + } else { + console.log(`committed: ${outcome} joins.`) + } + }, + { databaseUrl }, + ) } runIfMain(import.meta.url, "demo-join-event", main) diff --git a/services/api/src/db/demo-random.ts b/services/api/src/db/demo-random.ts new file mode 100644 index 00000000..b2530241 --- /dev/null +++ b/services/api/src/db/demo-random.ts @@ -0,0 +1,92 @@ +/** + * The demo CLIs' shared PRNG. Guard-free (no runIfMain) so both can import it: tsup (splitting: false) + * inlines imports into each bundled entry, and an imported runIfMain guard would fire inside the + * importing bundle. + * + * Seeded so a rehearsal, the committed run and a re-run after purge generate the same rows. Every helper + * draws from the one stream in call order, so reordering calls changes the generated data. + */ + +export const DEMO_PRNG_SEED = 20260902 + +const UINT32_RANGE = 4294967296 + +function mulberry32(seed: number): () => number { + let a = seed >>> 0 + return () => { + a |= 0 + a = (a + 0x6d2b79f5) | 0 + let t = Math.imul(a ^ (a >>> 15), 1 | a) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / UINT32_RANGE + } +} + +let source = mulberry32(DEMO_PRNG_SEED) + +export function seedDemoRandom(seed: number): void { + source = mulberry32(seed) +} + +export function rand(): number { + return source() +} + +export function rint(min: number, max: number): number { + return min + Math.floor(rand() * (max - min + 1)) +} + +export function chance(p: number): boolean { + return rand() < p +} + +export function pick(arr: readonly T[]): T { + return arr[Math.floor(rand() * arr.length)]! +} + +export function pickWeighted(items: readonly (readonly [T, number])[]): T { + let total = 0 + for (const [, w] of items) total += w + let roll = rand() * total + for (const [v, w] of items) { + roll -= w + if (roll <= 0) return v + } + return items[items.length - 1]![0] +} + +export function shuffle(arr: T[]): T[] { + for (let i = arr.length - 1; i > 0; i--) { + const j = Math.floor(rand() * (i + 1)) + ;[arr[i], arr[j]] = [arr[j]!, arr[i]!] + } + return arr +} + +export function sampleWeighted( + items: readonly T[], + weightOf: (t: T) => number, + n: number, + exclude: Set, +): T[] { + const out: T[] = [] + const taken = new Set(exclude) + const pool = items.filter((i) => !taken.has(i)) + for (let k = 0; k < n && pool.length > 0; k++) { + let total = 0 + for (const i of pool) total += weightOf(i) + if (total <= 0) break + let roll = rand() * total + let idx = pool.length - 1 + for (let j = 0; j < pool.length; j++) { + roll -= weightOf(pool[j]!) + if (roll <= 0) { + idx = j + break + } + } + out.push(pool[idx]!) + pool.splice(idx, 1) + } + return out +} diff --git a/services/api/src/db/ingest-jurisdictions-core.ts b/services/api/src/db/ingest-jurisdictions-core.ts index 25cb79df..a4f6da95 100644 --- a/services/api/src/db/ingest-jurisdictions-core.ts +++ b/services/api/src/db/ingest-jurisdictions-core.ts @@ -20,7 +20,7 @@ export interface IngestRow { geometry: { type: string; coordinates: unknown } } -function pickString(props: Record | null, keys: string[]): string | null { +function pickString(props: Record | null, keys: readonly string[]): string | null { if (!props) return null for (const k of keys) { const v = props[k] @@ -30,7 +30,7 @@ function pickString(props: Record | null, keys: string[]): stri return null } -function pickNumber(props: Record | null, keys: string[]): number | null { +function pickNumber(props: Record | null, keys: readonly string[]): number | null { if (!props) return null for (const k of keys) { const v = props[k] @@ -54,7 +54,16 @@ export function isIngestLayer(value: string): value is IngestRow["layer"] { const UPSERT_BATCH_SIZE = 1000 -export function normalizeFeature( +// Property names tried in order, covering TIGER, PAD-US and hand-written GeoJSON sources. +const GEOID_PROPERTY_KEYS = ["geoid", "GEOID", "UNIT_CODE", "unit_code", "id", "OBJECTID"] +const NAME_PROPERTY_KEYS = ["name", "NAME", "UNIT_NAME", "unit_name", "Unit_Name"] +const LAYER_PROPERTY_KEYS = ["layer", "LAYER", "owner_type", "Own_Type"] +const POPULATION_PROPERTY_KEYS = ["population", "POPULATION", "POP", "pop"] + +/** RFC 8142 GeoJSON text sequences may prefix each record with an ASCII record separator. */ +const RECORD_SEPARATOR = 0x1e + +function normalizeFeature( f: GeoJsonFeature, defaultLayer: IngestRow["layer"], geoidPrefix?: string, @@ -62,17 +71,10 @@ export function normalizeFeature( const geometry = f.geometry const isPolygon = geometry !== null && (geometry.type === "Polygon" || geometry.type === "MultiPolygon") - const geoid = pickString(f.properties, [ - "geoid", - "GEOID", - "UNIT_CODE", - "unit_code", - "id", - "OBJECTID", - ]) + const geoid = pickString(f.properties, GEOID_PROPERTY_KEYS) const prefixedGeoid = geoid !== null && geoidPrefix ? geoidPrefix + geoid : geoid - const name = pickString(f.properties, ["name", "NAME", "UNIT_NAME", "unit_name", "Unit_Name"]) - const rawLayer = pickString(f.properties, ["layer", "LAYER", "owner_type", "Own_Type"]) + const name = pickString(f.properties, NAME_PROPERTY_KEYS) + const rawLayer = pickString(f.properties, LAYER_PROPERTY_KEYS) const loweredLayer = rawLayer?.toLowerCase() ?? null const layer = loweredLayer !== null && isIngestLayer(loweredLayer) ? loweredLayer : defaultLayer if (!isPolygon || prefixedGeoid === null || name === null) return null @@ -80,7 +82,7 @@ export function normalizeFeature( geoid: prefixedGeoid, name, layer, - population: pickNumber(f.properties, ["population", "POPULATION", "POP", "pop"]), + population: pickNumber(f.properties, POPULATION_PROPERTY_KEYS), geometry, } } @@ -104,7 +106,7 @@ export function normalizeFeatures( return { rows, skipped } } -export async function upsertJurisdictionBatch( +async function upsertJurisdictionBatch( sql: Queryable, rows: readonly IngestRow[], ): Promise { @@ -207,7 +209,7 @@ export async function ingestGeoJsonSeqFile( } const lines = createInterface({ input: createReadStream(filePath, "utf8"), crlfDelay: Infinity }) for await (const raw of lines) { - const line = (raw.charCodeAt(0) === 0x1e ? raw.slice(1) : raw).trim() + const line = (raw.charCodeAt(0) === RECORD_SEPARATOR ? raw.slice(1) : raw).trim() if (line === "") continue features += 1 const row = normalizeFeature(JSON.parse(line) as GeoJsonFeature, defaultLayer, geoidPrefix) @@ -221,5 +223,3 @@ export async function ingestGeoJsonSeqFile( await flush() return { upserted, skipped, features } } - -export type { GeoJsonFeature, GeoJsonFeatureCollection } diff --git a/services/api/src/db/ingest-jurisdictions.ts b/services/api/src/db/ingest-jurisdictions.ts index a466b925..fa57799a 100644 --- a/services/api/src/db/ingest-jurisdictions.ts +++ b/services/api/src/db/ingest-jurisdictions.ts @@ -5,7 +5,7 @@ */ import { readFile } from "node:fs/promises" -import { runDbCli, runIfMain } from "./cli.js" +import { EXIT_USAGE, runDbCli, runIfMain } from "./cli.js" import { LAYER_RANK, ingestGeoJsonFile, @@ -13,30 +13,31 @@ import { type IngestRow, } from "./ingest-jurisdictions-core.js" +const DEFAULT_LAYER: IngestRow["layer"] = "federal" + // Re-exported from the historical path so existing importers keep resolving. export { normalizeFeatures, upsertJurisdiction, ingestGeoJsonFile, - LAYER_RANK, } from "./ingest-jurisdictions-core.js" export type { IngestRow } from "./ingest-jurisdictions-core.js" async function main(): Promise { const file = process.argv[2] - const defaultLayer = (process.argv[3] ?? "federal") as IngestRow["layer"] + const defaultLayer = (process.argv[3] ?? DEFAULT_LAYER) as IngestRow["layer"] const geoidPrefix = (process.argv[4] ?? "").trim() if (!file) { console.error( "usage: tsx src/db/ingest-jurisdictions.ts [layer] [geoid-prefix]", ) - process.exit(2) + process.exit(EXIT_USAGE) } if (!isIngestLayer(defaultLayer)) { console.error( `ingest: unknown layer "${defaultLayer}" (expected one of ${Object.keys(LAYER_RANK).join(", ")})`, ) - process.exit(2) + process.exit(EXIT_USAGE) } // Read up front so a missing path is a clear error rather than something that looks like a DB failure. diff --git a/services/api/src/db/reference-code.ts b/services/api/src/db/reference-code.ts index 4805f402..eaa9b3bc 100644 --- a/services/api/src/db/reference-code.ts +++ b/services/api/src/db/reference-code.ts @@ -20,6 +20,8 @@ export const UNKNOWN_JURCODE = 0 export const EVENT_PREFIX = "EVENT" +const SEQ_DIGITS = 6 + /** * Falls back to UNKNOWN_JURCODE rather than failing, so a code is always mintable. It is a plain SELECT * that takes no row lock, so it cannot disturb the allocator's lock order wherever it is called. @@ -44,12 +46,8 @@ export function eventScopeKey(jurCode: number): string { return `${EVENT_PREFIX}:${jurCode}` } -function pad6(seq: number): string { - return String(seq).padStart(6, "0") -} - export function formatReferenceCode(prefix: string, jurCode: number, seq: number): string { - return `${prefix}-${jurCode}-${pad6(seq)}` + return `${prefix}-${jurCode}-${String(seq).padStart(SEQ_DIGITS, "0")}` } /** Falls back to the "other" code for an unrecognized type, so a code is always mintable. */ @@ -58,7 +56,7 @@ export function typeCodeFor(type: ReportType): string { } /** Call this FIRST in the create transaction (see the lock-order contract above). */ -export async function allocateNextSeq(sql: Queryable, scopeKey: string): Promise { +async function allocateNextSeq(sql: Queryable, scopeKey: string): Promise { const rows = await sql<{ next_val: number }[]>` INSERT INTO reference_counters (scope_key, next_val) VALUES (${scopeKey}, 1) diff --git a/services/api/src/db/schema/types.ts b/services/api/src/db/schema/types.ts index f425fbca..49ebc32c 100644 --- a/services/api/src/db/schema/types.ts +++ b/services/api/src/db/schema/types.ts @@ -7,10 +7,12 @@ export interface GeometryConfig { srid?: number } +const WGS84_SRID = 4326 + export const geometry = customType<{ data: unknown; driverData: string; config: GeometryConfig }>({ dataType(config) { const subtype = config?.subtype ?? "Geometry" - const srid = config?.srid ?? 4326 + const srid = config?.srid ?? WGS84_SRID return `geometry(${subtype},${srid})` }, }) diff --git a/services/api/src/db/seed-demo-domain.ts b/services/api/src/db/seed-demo-domain.ts index bc01c4ca..3ba6896c 100644 --- a/services/api/src/db/seed-demo-domain.ts +++ b/services/api/src/db/seed-demo-domain.ts @@ -5,3 +5,5 @@ * A real person can never sign up with this domain, so it is safe to key purges and lookups on it. */ export const DEMO_EMAIL_DOMAIN = "demo-seed.civfix.org" + +export const DEMO_EMAIL_PATTERN = `%@${DEMO_EMAIL_DOMAIN}` diff --git a/services/api/src/db/seed-demo-la-data.ts b/services/api/src/db/seed-demo-la-data.ts new file mode 100644 index 00000000..5f5368a3 --- /dev/null +++ b/services/api/src/db/seed-demo-la-data.ts @@ -0,0 +1,964 @@ +/** + * Static content pools for the LA demo seeder: names, neighborhoods, parks and the casual post, report and + * event copy the generator samples from. Guard-free (no runIfMain) because seed-demo-la bundles it. + */ + +import type { ReportType } from "@civfix/shared" + +// About 70% of the cohort is Hispanic, with the rest reflecting LA's mix. + +export const HISPANIC_FIRST_M = [ + "Jose", + "Juan", + "Carlos", + "Luis", + "Jorge", + "Miguel", + "Pedro", + "Rafael", + "Javier", + "Alejandro", + "Fernando", + "Ricardo", + "Eduardo", + "Sergio", + "Hector", + "Oscar", + "Raul", + "Marco", + "Cesar", + "Diego", + "Emiliano", + "Mateo", + "Santiago", + "Sebastian", + "Andres", + "Cristian", + "Ivan", + "Erick", + "Kevin", + "Brandon", + "Anthony", + "Angel", + "Jesus", + "Ernesto", + "Gerardo", + "Rodrigo", + "Ruben", + "Salvador", + "Armando", + "Alfredo", + "Enrique", +] as const + +export const HISPANIC_FIRST_F = [ + "Maria", + "Guadalupe", + "Rosa", + "Carmen", + "Ana", + "Leticia", + "Veronica", + "Claudia", + "Adriana", + "Gabriela", + "Alejandra", + "Daniela", + "Mariana", + "Valeria", + "Ximena", + "Camila", + "Lucia", + "Elena", + "Isabel", + "Sofia", + "Paola", + "Yesenia", + "Marisol", + "Araceli", + "Esmeralda", + "Karina", + "Brenda", + "Jessica", + "Jasmine", + "Vanessa", + "Lorena", + "Norma", + "Silvia", + "Patricia", + "Sandra", + "Monica", + "Angelica", + "Maribel", + "Rocio", + "Beatriz", + "Josefina", + "Cindy", + "Nayeli", + "Itzel", + "Fatima", + "Alondra", + "Giselle", + "Ashley", + "Destiny", + "Selena", +] as const + +export const HISPANIC_LAST = [ + "Garcia", + "Rodriguez", + "Martinez", + "Hernandez", + "Lopez", + "Gonzalez", + "Perez", + "Sanchez", + "Ramirez", + "Torres", + "Flores", + "Rivera", + "Gomez", + "Diaz", + "Reyes", + "Morales", + "Cruz", + "Ortiz", + "Gutierrez", + "Chavez", + "Ramos", + "Ruiz", + "Alvarez", + "Mendoza", + "Vasquez", + "Castillo", + "Jimenez", + "Moreno", + "Romero", + "Herrera", + "Medina", + "Aguilar", + "Vargas", + "Guzman", + "Castro", + "Fernandez", + "Munoz", + "Rojas", + "Soto", + "Contreras", + "Silva", + "Delgado", + "Pena", + "Rios", + "Salazar", + "Estrada", + "Ortega", + "Nunez", + "Maldonado", + "Vega", + "Dominguez", + "Cabrera", + "Velasquez", + "Ibarra", + "Zavala", + "Cervantes", + "Fuentes", + "Carrillo", + "Trejo", + "Solis", + "Cardenas", + "Villanueva", + "Escobar", + "Quintero", + "Barrera", + "Rosales", + "Camacho", + "Arellano", + "Meza", + "Palacios", + "Navarro", + "Padilla", + "Miranda", + "Bautista", + "Orozco", + "Zuniga", + "Ochoa", + "Duran", + "Macias", + "Renteria", +] as const + +/** Display names only: handles and emails stay ASCII. */ +export const ACCENTED: Record = { + Jose: "José", + Maria: "María", + Jesus: "Jesús", + Andres: "Andrés", + Cesar: "César", + Angel: "Ángel", + Lucia: "Lucía", + Sofia: "Sofía", + Ivan: "Iván", + Fatima: "Fátima", + Munoz: "Muñoz", + Nunez: "Núñez", + Pena: "Peña", + Zuniga: "Zúñiga", +} + +export const OTHER_POOLS: readonly { + firstM: readonly string[] + firstF: readonly string[] + last: readonly string[] + weight: number +}[] = [ + { + // Korean American + firstM: ["Daniel", "Brian", "Eric", "Andrew", "Joon", "David"], + firstF: ["Grace", "Esther", "Hannah", "Julie", "Minji", "Susan"], + last: ["Kim", "Park", "Lee", "Choi", "Kang", "Yoon", "Shin", "Cho"], + weight: 5, + }, + { + // Armenian American + firstM: ["Armen", "Narek", "Tigran", "Vahe"], + firstF: ["Ani", "Lilit", "Mariam", "Sona"], + last: ["Hakobyan", "Grigoryan", "Sarkissian", "Petrosyan", "Avetisyan", "Kasparian"], + weight: 3, + }, + { + // Filipino American + firstM: ["Angelo", "Mark", "JR", "Paolo"], + firstF: ["Kristine", "Joanna", "Camille", "Divine"], + last: ["Santos", "Dela Cruz", "Mercado", "Aquino", "Ocampo", "Villareal", "Manalo"], + weight: 4, + }, + { + // Black and White American + firstM: ["Marcus", "Darnell", "James", "Mike", "Tyler", "Jordan", "Chris", "Devin"], + firstF: ["Keisha", "Tiffany", "Sarah", "Emily", "Aaliyah", "Megan", "Lauren", "Renee"], + last: [ + "Johnson", + "Williams", + "Brown", + "Smith", + "Miller", + "Davis", + "Jackson", + "Harris", + "Thompson", + "Robinson", + "Walker", + "Carter", + "Mitchell", + "Turner", + ], + weight: 10, + }, + { + // Chinese American + firstM: ["Wei", "Kevin", "Jason", "Alan"], + firstF: ["Amy", "Cindy", "Michelle", "Tina"], + last: ["Chen", "Wang", "Liu", "Huang", "Lin", "Wu", "Zhang"], + weight: 4, + }, + { + // Vietnamese American + firstM: ["Minh", "Vincent", "Phong", "Tuan"], + firstF: ["Linh", "Thao", "Kim-Ly", "Vy"], + last: ["Nguyen", "Tran", "Pham", "Le", "Vo", "Dang"], + weight: 3, + }, +] + +// Neighborhood jitter radii are in degrees. Weighted toward the Eastside, Southeast LA and the harbor +// corridor. + +export interface Hood { + name: string + lat: number + lng: number + r: number + weight: number + streets: readonly string[] +} + +export const HOODS: readonly Hood[] = [ + { + name: "Boyle Heights", + lat: 34.0397, + lng: -118.2077, + r: 0.01, + weight: 10, + streets: [ + "Cesar Chavez Ave", + "Soto St", + "1st St", + "4th St", + "Whittier Blvd", + "Lorena St", + "Evergreen Ave", + "St Louis St", + ], + }, + { + name: "East LA", + lat: 34.0239, + lng: -118.1721, + r: 0.012, + weight: 9, + streets: ["Whittier Blvd", "Atlantic Blvd", "3rd St", "Mednik Ave", "Arizona Ave", "Hammel St"], + }, + { + name: "Highland Park", + lat: 34.1115, + lng: -118.187, + r: 0.01, + weight: 7, + streets: ["York Blvd", "Figueroa St", "Avenue 56", "Monte Vista St", "Marmion Way"], + }, + { + name: "El Sereno", + lat: 34.0806, + lng: -118.1763, + r: 0.01, + weight: 6, + streets: ["Huntington Dr", "Eastern Ave", "Alhambra Ave", "Valley Blvd"], + }, + { + name: "Lincoln Heights", + lat: 34.07, + lng: -118.2, + r: 0.008, + weight: 6, + streets: ["N Broadway", "Daly St", "Main St", "Avenue 26", "Workman St"], + }, + { + name: "City Terrace", + lat: 34.057, + lng: -118.183, + r: 0.007, + weight: 4, + streets: ["City Terrace Dr", "Eastern Ave", "Herbert Ave"], + }, + { + name: "Huntington Park", + lat: 33.9817, + lng: -118.2251, + r: 0.009, + weight: 7, + streets: ["Pacific Blvd", "Gage Ave", "Slauson Ave", "Florence Ave", "Santa Fe Ave"], + }, + { + name: "South Gate", + lat: 33.9547, + lng: -118.212, + r: 0.01, + weight: 5, + streets: ["Tweedy Blvd", "Long Beach Blvd", "Firestone Blvd", "Atlantic Ave"], + }, + { + name: "Pacoima", + lat: 34.2728, + lng: -118.4201, + r: 0.012, + weight: 6, + streets: ["Van Nuys Blvd", "Glenoaks Blvd", "Laurel Canyon Blvd", "Foothill Blvd", "Paxton St"], + }, + { + name: "Van Nuys", + lat: 34.1899, + lng: -118.4514, + r: 0.012, + weight: 5, + streets: ["Van Nuys Blvd", "Victory Blvd", "Sherman Way", "Sepulveda Blvd", "Kester Ave"], + }, + { + name: "Sylmar", + lat: 34.3078, + lng: -118.4453, + r: 0.012, + weight: 3, + streets: ["San Fernando Rd", "Maclay Ave", "Glenoaks Blvd", "Hubbard St"], + }, + { + name: "Sun Valley", + lat: 34.217, + lng: -118.37, + r: 0.01, + weight: 3, + streets: ["San Fernando Rd", "Sunland Blvd", "Vineland Ave", "Lankershim Blvd"], + }, + { + name: "Wilmington", + lat: 33.7801, + lng: -118.2646, + r: 0.01, + weight: 5, + streets: ["Avalon Blvd", "Anaheim St", "Pacific Coast Hwy", "Wilmington Blvd", "L St"], + }, + { + name: "San Pedro", + lat: 33.7361, + lng: -118.2922, + r: 0.01, + weight: 4, + streets: ["Gaffey St", "Pacific Ave", "25th St", "Western Ave", "6th St"], + }, + { + name: "Watts", + lat: 33.9425, + lng: -118.2417, + r: 0.008, + weight: 5, + streets: ["103rd St", "Central Ave", "Compton Ave", "Wilmington Ave", "Grandee Ave"], + }, + { + name: "South LA", + lat: 34.0, + lng: -118.292, + r: 0.014, + weight: 7, + streets: [ + "Vermont Ave", + "Western Ave", + "Normandie Ave", + "Slauson Ave", + "Manchester Ave", + "Figueroa St", + ], + }, + { + name: "Koreatown", + lat: 34.0577, + lng: -118.3009, + r: 0.009, + weight: 5, + streets: [ + "Wilshire Blvd", + "Olympic Blvd", + "Western Ave", + "Vermont Ave", + "8th St", + "Normandie Ave", + ], + }, + { + name: "Westlake", + lat: 34.057, + lng: -118.276, + r: 0.007, + weight: 5, + streets: ["Alvarado St", "7th St", "Wilshire Blvd", "Union Ave", "Bonnie Brae St"], + }, + { + name: "Pico-Union", + lat: 34.047, + lng: -118.283, + r: 0.007, + weight: 5, + streets: ["Pico Blvd", "Union Ave", "Hoover St", "Venice Blvd", "Alvarado St"], + }, + { + name: "Cypress Park", + lat: 34.093, + lng: -118.224, + r: 0.006, + weight: 3, + streets: ["Cypress Ave", "Figueroa St", "San Fernando Rd", "Division St"], + }, + { + name: "Glassell Park", + lat: 34.113, + lng: -118.232, + r: 0.007, + weight: 3, + streets: ["Eagle Rock Blvd", "Verdugo Rd", "San Fernando Rd", "Fletcher Dr"], + }, + { + name: "Echo Park", + lat: 34.0782, + lng: -118.2606, + r: 0.007, + weight: 4, + streets: ["Sunset Blvd", "Echo Park Ave", "Glendale Blvd", "Alvarado St"], + }, + { + name: "Hollywood", + lat: 34.0928, + lng: -118.3287, + r: 0.01, + weight: 3, + streets: ["Hollywood Blvd", "Sunset Blvd", "Santa Monica Blvd", "Western Ave", "Gower St"], + }, + { + name: "North Hollywood", + lat: 34.172, + lng: -118.377, + r: 0.01, + weight: 4, + streets: ["Lankershim Blvd", "Magnolia Blvd", "Victory Blvd", "Vineland Ave"], + }, + { + name: "Panorama City", + lat: 34.227, + lng: -118.449, + r: 0.009, + weight: 4, + streets: ["Van Nuys Blvd", "Roscoe Blvd", "Nordhoff St", "Woodman Ave"], + }, + { + name: "Harbor Gateway", + lat: 33.86, + lng: -118.29, + r: 0.01, + weight: 2, + streets: ["Vermont Ave", "Figueroa St", "Gardena Blvd", "190th St"], + }, +] + +export const PARKS: readonly { name: string; hood: string; lat: number; lng: number }[] = [ + { name: "Hollenbeck Park", hood: "Boyle Heights", lat: 34.0367, lng: -118.2135 }, + { name: "Ruben Salazar Park", hood: "East LA", lat: 34.0236, lng: -118.1893 }, + { name: "Salt Lake Park", hood: "Huntington Park", lat: 33.9757, lng: -118.2172 }, + { name: "Hazard Park", hood: "Boyle Heights", lat: 34.0645, lng: -118.2005 }, + { name: "Lincoln Park", hood: "Lincoln Heights", lat: 34.0705, lng: -118.2028 }, + { name: "Sycamore Grove Park", hood: "Highland Park", lat: 34.0996, lng: -118.1998 }, + { name: "Ted Watkins Memorial Park", hood: "Watts", lat: 33.933, lng: -118.2379 }, + { name: "MacArthur Park", hood: "Westlake", lat: 34.059, lng: -118.2785 }, + { name: "Rio de Los Angeles State Park", hood: "Cypress Park", lat: 34.0994, lng: -118.2273 }, + { name: "Ernest E. Debs Regional Park", hood: "El Sereno", lat: 34.0873, lng: -118.1935 }, + { name: "Ken Malloy Harbor Regional Park", hood: "Wilmington", lat: 33.7863, lng: -118.2879 }, + { name: "Hansen Dam Recreation Area", hood: "Pacoima", lat: 34.2612, lng: -118.3898 }, + { name: "Sepulveda Basin", hood: "Van Nuys", lat: 34.1755, lng: -118.4838 }, + { name: "Point Fermin Park", hood: "San Pedro", lat: 33.706, lng: -118.2936 }, + { name: "Normandie Recreation Center", hood: "South LA", lat: 34.0261, lng: -118.3003 }, + { name: "Seoul International Park", hood: "Koreatown", lat: 34.0546, lng: -118.3082 }, + { name: "North Hollywood Park", hood: "North Hollywood", lat: 34.1638, lng: -118.3801 }, + { name: "Echo Park Lake", hood: "Echo Park", lat: 34.0723, lng: -118.2606 }, +] + +export function hoodByName(name: string): Hood { + return HOODS.find((h) => h.name === name) ?? HOODS[0]! +} + +// Deliberately casual: lowercase drift, loose punctuation, Spanish and Spanglish mixed in, occasional +// emoji, no long-form writing and no em dashes anywhere (validate() enforces the last). + +export const BIO_NEUTRAL: readonly string[] = [ + "{hood} born and raised", + "trying to keep {hood} clean one block at a time", + "east side til i die", + "community first. {hood}", + "if you see me picking up trash say hi", + "small business owner on {street}", + "youth soccer coach in {hood}", + "keeping it clean in {hood} 🧹", + "born in {hood}, still here, not leaving", + "retired LAUSD. love my neighborhood", + "just here to report potholes tbh", + "organizing cleanups w my neighbors. dm me if you wanna help", + "{hood} neighborhood watch", +] + +export const BIO_MALE: readonly string[] = [ + "dad of 3. tired of the dumping on our streets", + "girl dad in {hood}", + "dog dad, {hood}", +] +export const BIO_FEMALE: readonly string[] = [ + "mom of 3. tired of the dumping on our streets", + "{hood} resident, dog mom, 311 power user", + "abuela energy. {hood}", +] + +/** Only for Hispanic users; vecino/vecina gendered. */ +export const BIO_HISPANIC_NEUTRAL: readonly string[] = [ + "orgullosamente de {hood} 🇲🇽", + "aqui puro {hood} 💪", + "la comunidad es todo", +] +export const BIO_HISPANIC_M: readonly string[] = ["vecino de {hood}, aqui para ayudar"] +export const BIO_HISPANIC_F: readonly string[] = ["vecina de {hood}, aqui para ayudar"] + +/** {street}/{street2}/{hood} slots are filled per author. */ +export const POST_TEMPLATES_EN: readonly string[] = [ + "third couch this month dumped on {street}. who keeps doing this", + "the graffiti on the handball courts finally got painted over 🙏", + "somebody left a whole entertainment center on the corner of {street} and {street2} lol", + "shoutout to the crew that cleaned the alley behind {street} this weekend. looks brand new", + "when is the city gonna fix the streetlight on {street}, its been out for like 2 months", + "reported a mattress on {street} last tuesday and they actually picked it up friday. not bad", + "ok whoever keeps dumping tires by the wash, we see you 🤨", + "morning walk update: {street} is looking clean for once. small wins", + "the bulky item pickup line had me on hold 40 mins. this app is way faster", + "we need more trash cans on {street} near the bus stop. overflowing every weekend", + "just moved to {hood} and joined this app, love seeing neighbors actually fix stuff", + "anyone else notice the illegal dumping gets worse right after the first of the month", + "picked up 3 bags on my street this morning before work. tired but worth it", + "the mural on {street} got tagged again 😔 gonna organize a repaint", + "city came and cleared the {street} underpass today. hope it stays clean this time", + "psa: bulky item pickup is free, you dont have to dump your sofa on {street} 😤", + "not all heroes wear capes, some just bring their own trash grabbers", + "the sidewalk on {street} is basically an obstacle course. reported like 4 spots today", + "my kids and i picked up trash at the park today. teach em young", + "10/10 morning, coffee from the panaderia and a clean street for once", + "does anyone know who to talk to about the abandoned car on {street}, been there 3 weeks", + "the amount of fast food trash on {street} after friday night is wild", + "shout out to the senora on my block who sweeps the whole sidewalk every morning", + "found a shopping cart in the LA river again. classic", + "neighbors really came through this weekend, {street} is spotless", + "why do people dump paint cans in the alley. thats toxic waste man", + "councilman's office actually called me back about the {street} dumping. progress??", + "the little free library on {street} survived another year 🥹 love this block", + "somebody stole the trash can from the bus stop?? lol only in LA", + "green waste everywhere after the wind last night. be careful driving on {street}", + "im convinced the same truck dumps on {street} every sunday night. gonna get a plate next time", + "starting to see more people use this app in {hood}. keep reporting yall, it works", + "cleaned up the parkway strip in front of my house. do your part people", + "walking to the market and counted 6 illegal dump spots on {street}. all reported", + "our block finally got the speed humps. now if we could get the trash handled", + "sunday morning cleanups hit different. peaceful out here", + "reminder that the storm drains go straight to the ocean. keep em clear", + "big respect to the folks who do this every single week without any credit", + "the empty lot on {street} needs some love, thinking of organizing something", + "trash pickup skipped our street again this week?? anyone else on {street}", + "if every block had 2 people who cared we could keep this whole neighborhood clean", +] + +/** Drawn only by Hispanic authors. */ +export const POST_TEMPLATES_ES: readonly string[] = [ + "mucha basura en la calle otra vez. ya reporte, a ver si hacen algo", + "el alley behind my place is getting bad again, gonna report it manana", + "gracias a todos los que vinieron hoy, quedo bien limpio el parque 💪", + "los fines de semana la gente tira basura como si nada. respeten el barrio", + "vamos a limpiar {hood} este sabado, quien se apunta", + "cada quien su bolsa. sabado 9am. no excuses", + "hoy tocó limpiar la esquina de {street}. entre 4 lo hicimos en una hora", + "esta app si funciona, reporte un colchon y en 3 dias lo recogieron", + "que bonito se ve {hood} cuando todos ayudamos", +] + +export const REPORT_POST_EN: readonly string[] = [ + "reported this dump on {street}, yall check it out so the city sees it", + "this has been here over a week. finally reported it", + "look at this mess. reported. lets see how long it takes", + "reported this one this morning, right by the school 😡", + "adding this to the pile of reports on {street}. its bad out here", + "cant even use the sidewalk. reported", + "this is right in front of the panaderia. reported it, share so it gets fixed", + "week 2 of this couch. reported again lol", + "who does this?? reported", +] +export const REPORT_POST_ES: readonly string[] = [ + "como es posible que dejen esto asi. ya lo reporte", + "miren esto. reportado. compartan para que lo vean", +] + +export const EVENT_POST_EN: readonly string[] = [ + "hosting a cleanup this weekend, bring gloves if you got em. everyone welcome", + "cleanup this saturday 🧹 kids welcome, we got extra grabbers", + "we're doing another one. last time we filled 20 bags, lets beat that", + "first cleanup im organizing, be nice lol. hope to see some of you there", + "join us saturday morning, coffee and pan dulce for volunteers ☕", + "one more cleanup before it gets too hot. roll thru", + "big one this weekend. bring the whole family", +] +export const EVENT_POST_ES: readonly string[] = [ + "vamos a limpiar este sabado, traigan agua y guantes. los espero", + "este sabado nos toca limpiar. lleguenle con la familia", +] + +export const EVENT_RECAP_EN: readonly string[] = [ + "{bags} bags today. arms are dead but the block looks brand new. thank you everyone 🙏", + "we got {bags} bags out of the park today. proud of this neighborhood", + "another one done. {bags} bags, a couch, and somehow a car bumper lol. great turnout", + "small crew today but we still pulled {bags} bags. every bit counts", + "thank you to the {n} people who showed up today. {bags} bags collected", +] +export const EVENT_RECAP_ES: readonly string[] = [ + "{bags} bolsas hoy!! gracias a todos los que vinieron 💪", + "terminamos con {bags} bolsas. gracias a mi gente que llego temprano", +] + +export const REPLY_GENERIC_EN: readonly string[] = [ + "same thing on my street", + "reported one like this last week, took 3 weeks but they picked it up", + "this is why i love this app", + "311 never picks up, this is faster fr", + "ugh not again", + "facts", + "thank you for doing this", + "we appreciate you 🙏", + "same in {hood} honestly", + "its been like this for weeks", + "somebody has to say it", + "100%", + "we need cameras out there", + "the city needs to do better", + "keep us posted", + "this made my day", + "couldnt agree more", + "im telling my landlord about this app lol", + "the real MVP", + "on my way to report the one by my house too", + "hope they fix it soon", + "lmk if you need help", + "this block deserves better", + "seen it, its worse in person", +] +export const REPLY_GENERIC_ES: readonly string[] = [ + "gracias por reportar 🙏", + "el respeto al barrio empieza por uno mismo", + "asi es", + "no manches", + "que bueno que alguien hace algo", + "orale, buen trabajo", +] + +export const REPLY_EVENT_EN: readonly string[] = [ + "i'll be there", + "count me in", + "what time does it start?", + "can i bring my kids?", + "do we need to bring our own gloves", + "just signed up 🙌", + "cant make this one but next time for sure", + "bringing 2 friends", + "is there parking nearby", + "see you saturday", + "my whole family is coming lol", + "first time doing one of these, excited", +] +export const REPLY_EVENT_ES: readonly string[] = [ + "yo tambien voy", + "ahi estare", + "llevare bolsas extra", +] + +export const REPLY_REPORT_EN: readonly string[] = [ + "just liked it so it gets visibility", + "reported the same spot last month, they cleared it but it came back", + "thats right by my kids school 😡", + "share it in the group chat too", + "the city cleared one like this on my street in about a week", + "took a pic of the same pile yesterday, glad you reported", + "this intersection is always bad", + "sad that it takes an app for the city to do their job", +] +export const REPLY_REPORT_ES: readonly string[] = [ + "eso esta a una cuadra de mi casa", + "gracias vecino", +] + +export const QUOTE_EN: readonly string[] = [ + "this right here", + "everyone in {hood} needs to see this", + "and people say nobody cares about this neighborhood", + "sharing for the morning crowd", + "this is the kind of stuff that keeps me on this app", + "proof that reporting works yall", +] +export const QUOTE_ES: readonly string[] = ["lo que siempre digo", "mi gente 💪"] + +/** [titles, descriptions]; slots: {street} {street2}. */ +export const REPORT_CONTENT: Record< + ReportType, + { titles: readonly string[]; descs: readonly string[]; descsEs?: readonly string[] } +> = { + dump: { + titles: [ + "mattress dumped on {street}", + "couch left on the corner of {street} and {street2}", + "pile of construction debris on {street}", + "tv and boxes dumped by the alley", + "furniture dumped on the sidewalk", + "trash bags piling up on {street}", + "tires dumped near {street}", + "someone dumped a washer on {street}", + "big pile of junk on the parkway", + "shopping carts and trash on {street}", + ], + descs: [ + "been here over a week now. right in front of the laundromat. kids have to walk around it into the street", + "keeps growing every day. started as one bag now its a whole pile", + "someone dumped this overnight. blocking half the sidewalk", + "third time this month at this exact spot. we need a camera or something", + "smells terrible and there are flies everywhere. please pick up soon", + "right next to the bus stop where people wait every morning", + "wood with nails sticking out, dangerous for kids walking to school", + "looks like a contractor dumped it, there are paint buckets and drywall", + "elderly neighbors cant get around it with their carts", + "its right by the storm drain, gonna wash into the river when it rains", + ], + descsEs: [ + "esta enfrente de mi casa desde el lunes. ya no podemos ni pasar por la banqueta", + "la gente sigue tirando basura aqui, cada semana es lo mismo", + ], + }, + graffiti: { + titles: [ + "tagging on the wall at {street}", + "graffiti on the bus bench", + "fresh tags on the store shutters", + "graffiti covering the street sign", + "wall on {street} tagged again", + "tags all over the underpass", + ], + descs: [ + "whole wall got hit over the weekend. was just painted 2 months ago", + "the stop sign is barely readable now, thats a safety issue", + "small business owner already dealing with a lot, now this", + "gang tags this time, neighbors are worried. please prioritize", + "they tagged the mural too which is really sad, that mural took months", + "same tags as the ones on {street2}, probably the same people", + ], + descsEs: ["rayaron toda la pared otra vez. apenas la habian pintado"], + }, + encampment: { + titles: [ + "encampment growing under the {street} overpass", + "tents blocking the sidewalk on {street}", + "encampment by the wash near {street}", + ], + descs: [ + "not trying to get anyone in trouble, they need services. but the sidewalk is fully blocked and theres a lot of debris", + "its grown from 2 tents to about 8 in a month. trash is piling up around it", + "requesting outreach services, theres an older man there who needs medical help", + "kids walk this route to school and have to go into the street", + ], + }, + infrastructure: { + titles: [ + "broken sprinkler flooding the sidewalk on {street}", + "fire hydrant leaking on {street}", + "water main leaking into the street", + "broken streetlight on {street}", + "exposed wiring on the light pole", + ], + descs: [ + "water has been running down the gutter for 3 days straight. huge waste", + "the whole corner is flooded every morning. slipping hazard", + "light has been out for weeks, its really dark on this block at night. safety issue", + "you can hear the water hissing, probably losing hundreds of gallons", + ], + descsEs: ["el poste tiene cables colgando, esta peligroso"], + }, + pavement: { + titles: [ + "huge pothole on {street}", + "sidewalk buckled by tree roots on {street}", + "pothole damaging cars near {street} and {street2}", + "cracked curb ramp on the corner", + "street cracking apart on {street}", + ], + descs: [ + "hit it last night, almost lost a tire. its deep", + "my neighbor in a wheelchair literally cannot use this sidewalk", + "gets worse every week, and cars swerve into the other lane to miss it", + "seniors trip here all the time, someone fell last week", + "been reported before and patched but the patch is already gone", + ], + descsEs: ["el bache esta enorme, ya varios carros se han danado"], + }, + vegetation: { + titles: [ + "overgrown weeds blocking the sidewalk on {street}", + "dead palm fronds hanging over {street}", + "tree branch about to fall on {street}", + "brush pile fire hazard by {street}", + ], + descs: [ + "the weeds are shoulder height, you cant see around the corner when driving", + "big dead frond hanging right over the bus stop. someone is gonna get hurt", + "branch cracked in the wind last week and is hanging by a thread", + "dry brush right up against the fence, one spark and its a problem", + ], + descsEs: ["las ramas ya tapan toda la banqueta"], + }, + other: { + titles: [ + "abandoned car on {street}", + "shopping carts collecting on the corner", + "broken glass all over the sidewalk", + "dead animal on {street}", + "leaking dumpster behind the businesses", + ], + descs: [ + "hasnt moved in 3 weeks, flat tires, windows are smashed now", + "at least 6 carts from the ranch market piling up", + "please send someone, its been days and it smells really bad", + "someone smashed bottles all over, dogs and kids walk here", + "grease and trash water running into the gutter", + ], + }, +} + +export const EVENT_TITLE_EN: readonly string[] = [ + "{park} cleanup", + "{hood} community cleanup", + "{street} alley cleanup", + "adopt-a-block {hood}", + "{hood} saturday sweep", +] +export const EVENT_TITLE_ES: readonly string[] = ["limpieza comunitaria en {park}"] + +export const EVENT_DESC_EN: readonly string[] = [ + "meet at the main entrance. we'll split into teams and cover the park and the streets around it. bags and some grabbers provided, bring gloves and water if you can", + "monthly cleanup with the neighbors. all ages welcome, we usually finish by noon and someone always brings tamales", + "the alley has gotten bad again so we're getting a crew together. wear closed toe shoes, there might be glass", + "quick 2 hour cleanup then tacos after for whoever can stay. first timers welcome, we'll show you the ropes", + "bringing the community together to take care of our space. supplies provided by the neighborhood council", +] +export const EVENT_DESC_ES: readonly string[] = [ + "juntandonos para limpiar el parque y las calles de alrededor. traigan guantes si tienen, nosotros ponemos las bolsas", +] + +export const BRING_POOL: readonly string[] = [ + "gloves", + "water", + "sunscreen", + "hat", + "trash grabbers", + "closed toe shoes", + "reusable water bottle", +] + +export const SLOT_SETS: readonly (readonly { + title: string + description: string | null + capacity: number | null +}[])[] = [ + [ + { + title: "Registration table", + description: "check people in and hand out supplies", + capacity: 2, + }, + { title: "Supplies and water", description: "keep the water station stocked", capacity: 2 }, + { title: "Street team", description: "cover the blocks around the park", capacity: null }, + ], + [ + { title: "8-10am shift", description: null, capacity: 12 }, + { title: "10-12 shift", description: null, capacity: 12 }, + ], + [ + { + title: "Heavy lifting crew", + description: "for the big stuff, bring work gloves", + capacity: 6, + }, + { title: "General cleanup", description: null, capacity: null }, + { title: "Kids zone", description: "light duty for families with little ones", capacity: 8 }, + ], +] + +export const TIMELINE_ACK_NOTES: readonly string[] = [ + "Forwarded to LA Sanitation", + "Routed to the responsible department", + "Received by the city, reference logged", +] +export const TIMELINE_RESOLVE_NOTES: readonly string[] = [ + "Crew confirmed pickup complete", + "Marked resolved after site inspection", + "Cleared by sanitation crew", +] diff --git a/services/api/src/db/seed-demo-la.ts b/services/api/src/db/seed-demo-la.ts index 69ea57ef..f3a7f618 100644 --- a/services/api/src/db/seed-demo-la.ts +++ b/services/api/src/db/seed-demo-la.ts @@ -36,8 +36,8 @@ */ import { randomUUID } from "node:crypto" -import { REPORT_TYPE_TO_CATEGORY, type ReportType } from "@civfix/shared" -import { makeDb, type Sql, type TransactionSql } from "./client.js" +import { HANDLE_REGEX, REPORT_TYPE_TO_CATEGORY, type ReportType } from "@civfix/shared" +import type { Sql, TransactionSql } from "./client.js" import { allocateEventReferenceCode, allocateReportReferenceCode, @@ -45,92 +45,135 @@ import { } from "./reference-code.js" import { resolveJurisdiction } from "./sql/jurisdiction.js" import { reportH3Cell } from "../services/report-clustering.js" -import { runIfMain } from "./cli.js" -import { DEMO_EMAIL_DOMAIN } from "./seed-demo-domain.js" +import { argValue, requireDatabaseUrl, runDbCli, runIfMain } from "./cli.js" +import { DEMO_EMAIL_DOMAIN, DEMO_EMAIL_PATTERN } from "./seed-demo-domain.js" import { DEFAULT_EVENT_DURATION_MS, DEFAULT_EVENT_SLOT_TITLE } from "../services/cleanup-rules.js" import { demoTicketTokenHasher, mintDemoSignupSeats } from "./demo-signup-seats.js" import { touchUserActivity } from "./sql/user-activity.js" +import { + DEMO_PRNG_SEED, + chance, + pick, + pickWeighted, + rand, + rint, + sampleWeighted, + seedDemoRandom, + shuffle, +} from "./demo-random.js" +import { + ACCENTED, + BIO_FEMALE, + BIO_HISPANIC_F, + BIO_HISPANIC_M, + BIO_HISPANIC_NEUTRAL, + BIO_MALE, + BIO_NEUTRAL, + BRING_POOL, + EVENT_DESC_EN, + EVENT_DESC_ES, + EVENT_POST_EN, + EVENT_POST_ES, + EVENT_RECAP_EN, + EVENT_RECAP_ES, + EVENT_TITLE_EN, + EVENT_TITLE_ES, + HISPANIC_FIRST_F, + HISPANIC_FIRST_M, + HISPANIC_LAST, + HOODS, + OTHER_POOLS, + PARKS, + POST_TEMPLATES_EN, + POST_TEMPLATES_ES, + QUOTE_EN, + QUOTE_ES, + REPLY_EVENT_EN, + REPLY_EVENT_ES, + REPLY_GENERIC_EN, + REPLY_GENERIC_ES, + REPLY_REPORT_EN, + REPLY_REPORT_ES, + REPORT_CONTENT, + REPORT_POST_EN, + REPORT_POST_ES, + SLOT_SETS, + TIMELINE_ACK_NOTES, + TIMELINE_RESOLVE_NOTES, + hoodByName, + type Hood, +} from "./seed-demo-la-data.js" // Seeded addresses are typed by hand, the provenance the live create paths record for that case. const SEEDED_REPORT_ADDR_SOURCE = "user" const SEEDED_EVENT_ADDRESS_SOURCE = "manual" -// Seeded so a rehearsal, the committed run and a re-run after purge generate the same cohort. - -function mulberry32(seed: number): () => number { - let a = seed >>> 0 - return () => { - a |= 0 - a = (a + 0x6d2b79f5) | 0 - let t = Math.imul(a ^ (a >>> 15), 1 | a) - t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t - return ((t ^ (t >>> 14)) >>> 0) / 4294967296 - } -} - -let rand = mulberry32(20260902) - -function rint(min: number, max: number): number { - return min + Math.floor(rand() * (max - min + 1)) -} -function chance(p: number): boolean { - return rand() < p -} -function pick(arr: readonly T[]): T { - return arr[Math.floor(rand() * arr.length)]! -} -function pickWeighted(items: readonly (readonly [T, number])[]): T { - let total = 0 - for (const [, w] of items) total += w - let roll = rand() * total - for (const [v, w] of items) { - roll -= w - if (roll <= 0) return v - } - return items[items.length - 1]![0] -} -function shuffle(arr: T[]): T[] { - for (let i = arr.length - 1; i > 0; i--) { - const j = Math.floor(rand() * (i + 1)) - ;[arr[i], arr[j]] = [arr[j]!, arr[i]!] - } - return arr -} -function sampleWeighted( - items: readonly T[], - weightOf: (t: T) => number, - n: number, - exclude: Set, -): T[] { - const out: T[] = [] - const taken = new Set(exclude) - const pool = items.filter((i) => !taken.has(i)) - for (let k = 0; k < n && pool.length > 0; k++) { - let total = 0 - for (const i of pool) total += weightOf(i) - if (total <= 0) break - let roll = rand() * total - let idx = pool.length - 1 - for (let j = 0; j < pool.length; j++) { - roll -= weightOf(pool[j]!) - if (roll <= 0) { - idx = j - break - } - } - out.push(pool[idx]!) - pool.splice(idx, 1) - } - return out -} +const DEFAULT_USER_COUNT = 250 +const COHORT_HISTORY_DAYS = 185 +const NEWEST_ACCOUNT_AGE_DAYS = 2 +const REPORTS_PER_USER = 0.62 + +const EVENT_COUNT = 20 +const EVENT_ORGANIZER_COUNT = 14 +// Events before this index are past ("done"), the one at it is cancelled, the rest are upcoming. +const PAST_EVENT_COUNT = 13 +const EVENT_HISTORY_DAYS = 150 +const EVENT_MIN_CREATE_LEAD_DAYS = 2 +const EVENT_MIN_AGE_HOURS = 6 +const EVENT_MAX_CREATE_LEAD_DAYS = 28 +const EVENT_START_HOUR = 9 +const EVENT_SITE_JITTER_DEG = 0.0015 + +const MAX_REPLIES_PER_THREAD = 14 +const MAX_REPLY_DEPTH = 3 +const REPLY_WINDOW_DAYS = 5 +const RESHARE_WINDOW_DAYS = 7 +const MIN_FOLLOWERS_TO_RESHARE = 3 +const LIKE_WINDOW_DAYS = 14 +const MAX_LIKES_PER_POST = 60 +const CREDITED_HOUR_CHOICES: readonly number[] = [1.5, 2, 2, 2.5, 2.5, 3, 3, 3.5, 4] + +// Mirrors HANDLE_REGEX's length bounds, which validate() enforces on every generated handle. +const HANDLE_MIN_LENGTH = 3 +const HANDLE_MAX_LENGTH = 20 + +// A degree of longitude is shorter than a degree of latitude at LA's latitude, so the jitter is stretched +// east-west to keep points spread in a circle on the ground rather than an ellipse. +const LNG_JITTER_STRETCH = 1.2 +const COORDINATE_DECIMALS = 6 + +const EM_DASH = "\u2014" +const ERROR_BODY_PREVIEW_CHARS = 40 +const VALIDATION_ERROR_SAMPLE = 25 + +const INSERT_CHUNK = { + users: 200, + notificationPrefs: 300, + follows: 500, + timeline: 500, + posts: 300, + postGeom: 500, + mentions: 500, + engagement: 800, + hours: 300, + rollups: 500, +} as const // Timestamps get an LA-plausible time-of-day (evenings and weekends heavier), stored as UTC. The // seeded window spans both PST and PDT, so the offset is resolved per date. -const DAY = 24 * 60 * 60 * 1000 -const HOUR = 60 * 60 * 1000 +const MINUTE = 60 * 1000 +const HOUR = 60 * MINUTE +const DAY = 24 * HOUR +const HOUR_MINUTES = 60 +const DAYS_PER_WEEK = 7 const LA_TIME_ZONE = "America/Los_Angeles" const LA_STANDARD_OFFSET_HOURS = -8 +const LA_UTC_OFFSET_RE = /^GMT([+-]\d{1,2})$/ +const SUNDAY = 0 +const SATURDAY = 6 +const TIMESTAMP_ATTEMPTS = 6 +const WEEKDAY_KEEP_PROBABILITY = 0.75 const LA_OFFSET_FORMAT = new Intl.DateTimeFormat("en-US", { timeZone: LA_TIME_ZONE, @@ -139,7 +182,7 @@ const LA_OFFSET_FORMAT = new Intl.DateTimeFormat("en-US", { function laOffsetHoursAt(at: Date): number { const name = LA_OFFSET_FORMAT.formatToParts(at).find((p) => p.type === "timeZoneName")?.value - const m = /^GMT([+-]\d{1,2})$/.exec(name ?? "") + const m = LA_UTC_OFFSET_RE.exec(name ?? "") return m ? Number(m[1]) : LA_STANDARD_OFFSET_HOURS } @@ -191,983 +234,32 @@ function atLocalTime(dayMs: number): Date { } function randTimestamp(start: Date, end: Date): Date { - const span = Math.max(end.getTime() - start.getTime(), 60_000) - for (let attempt = 0; attempt < 6; attempt++) { + const span = Math.max(end.getTime() - start.getTime(), MINUTE) + for (let attempt = 0; attempt < TIMESTAMP_ATTEMPTS; attempt++) { const t = atLocalTime(start.getTime() + rand() * span) if (t.getTime() < start.getTime() || t.getTime() > end.getTime()) continue const dow = t.getUTCDay() - if (dow === 0 || dow === 6 || chance(0.75)) return t + if (dow === SUNDAY || dow === SATURDAY || chance(WEEKDAY_KEEP_PROBABILITY)) return t } return new Date(start.getTime() + rand() * span) } function minutesAfter(d: Date, min: number, max: number): Date { - return new Date(d.getTime() + rint(min, max) * 60_000) + return new Date(d.getTime() + rint(min, max) * MINUTE) } function later(a: Date, b: Date): Date { return a.getTime() >= b.getTime() ? a : b } -// About 70% of the cohort is Hispanic, with the rest reflecting LA's mix. - -const HISPANIC_FIRST_M = [ - "Jose", - "Juan", - "Carlos", - "Luis", - "Jorge", - "Miguel", - "Pedro", - "Rafael", - "Javier", - "Alejandro", - "Fernando", - "Ricardo", - "Eduardo", - "Sergio", - "Hector", - "Oscar", - "Raul", - "Marco", - "Cesar", - "Diego", - "Emiliano", - "Mateo", - "Santiago", - "Sebastian", - "Andres", - "Cristian", - "Ivan", - "Erick", - "Kevin", - "Brandon", - "Anthony", - "Angel", - "Jesus", - "Ernesto", - "Gerardo", - "Rodrigo", - "Ruben", - "Salvador", - "Armando", - "Alfredo", - "Enrique", -] as const - -const HISPANIC_FIRST_F = [ - "Maria", - "Guadalupe", - "Rosa", - "Carmen", - "Ana", - "Leticia", - "Veronica", - "Claudia", - "Adriana", - "Gabriela", - "Alejandra", - "Daniela", - "Mariana", - "Valeria", - "Ximena", - "Camila", - "Lucia", - "Elena", - "Isabel", - "Sofia", - "Paola", - "Yesenia", - "Marisol", - "Araceli", - "Esmeralda", - "Karina", - "Brenda", - "Jessica", - "Jasmine", - "Vanessa", - "Lorena", - "Norma", - "Silvia", - "Patricia", - "Sandra", - "Monica", - "Angelica", - "Maribel", - "Rocio", - "Beatriz", - "Josefina", - "Cindy", - "Nayeli", - "Itzel", - "Fatima", - "Alondra", - "Giselle", - "Ashley", - "Destiny", - "Selena", -] as const - -const HISPANIC_LAST = [ - "Garcia", - "Rodriguez", - "Martinez", - "Hernandez", - "Lopez", - "Gonzalez", - "Perez", - "Sanchez", - "Ramirez", - "Torres", - "Flores", - "Rivera", - "Gomez", - "Diaz", - "Reyes", - "Morales", - "Cruz", - "Ortiz", - "Gutierrez", - "Chavez", - "Ramos", - "Ruiz", - "Alvarez", - "Mendoza", - "Vasquez", - "Castillo", - "Jimenez", - "Moreno", - "Romero", - "Herrera", - "Medina", - "Aguilar", - "Vargas", - "Guzman", - "Castro", - "Fernandez", - "Munoz", - "Rojas", - "Soto", - "Contreras", - "Silva", - "Delgado", - "Pena", - "Rios", - "Salazar", - "Estrada", - "Ortega", - "Nunez", - "Maldonado", - "Vega", - "Dominguez", - "Cabrera", - "Velasquez", - "Ibarra", - "Zavala", - "Cervantes", - "Fuentes", - "Carrillo", - "Trejo", - "Solis", - "Cardenas", - "Villanueva", - "Escobar", - "Quintero", - "Barrera", - "Rosales", - "Camacho", - "Arellano", - "Meza", - "Palacios", - "Navarro", - "Padilla", - "Miranda", - "Bautista", - "Orozco", - "Zuniga", - "Ochoa", - "Duran", - "Macias", - "Renteria", -] as const - -/** Display names only: handles and emails stay ASCII. */ -const ACCENTED: Record = { - Jose: "José", - Maria: "María", - Jesus: "Jesús", - Andres: "Andrés", - Cesar: "César", - Angel: "Ángel", - Lucia: "Lucía", - Sofia: "Sofía", - Ivan: "Iván", - Fatima: "Fátima", - Munoz: "Muñoz", - Nunez: "Núñez", - Pena: "Peña", - Zuniga: "Zúñiga", -} - -const OTHER_POOLS: readonly { - firstM: readonly string[] - firstF: readonly string[] - last: readonly string[] - weight: number -}[] = [ - { - // Korean American - firstM: ["Daniel", "Brian", "Eric", "Andrew", "Joon", "David"], - firstF: ["Grace", "Esther", "Hannah", "Julie", "Minji", "Susan"], - last: ["Kim", "Park", "Lee", "Choi", "Kang", "Yoon", "Shin", "Cho"], - weight: 5, - }, - { - // Armenian American - firstM: ["Armen", "Narek", "Tigran", "Vahe"], - firstF: ["Ani", "Lilit", "Mariam", "Sona"], - last: ["Hakobyan", "Grigoryan", "Sarkissian", "Petrosyan", "Avetisyan", "Kasparian"], - weight: 3, - }, - { - // Filipino American - firstM: ["Angelo", "Mark", "JR", "Paolo"], - firstF: ["Kristine", "Joanna", "Camille", "Divine"], - last: ["Santos", "Dela Cruz", "Mercado", "Aquino", "Ocampo", "Villareal", "Manalo"], - weight: 4, - }, - { - // Black and White American - firstM: ["Marcus", "Darnell", "James", "Mike", "Tyler", "Jordan", "Chris", "Devin"], - firstF: ["Keisha", "Tiffany", "Sarah", "Emily", "Aaliyah", "Megan", "Lauren", "Renee"], - last: [ - "Johnson", - "Williams", - "Brown", - "Smith", - "Miller", - "Davis", - "Jackson", - "Harris", - "Thompson", - "Robinson", - "Walker", - "Carter", - "Mitchell", - "Turner", - ], - weight: 10, - }, - { - // Chinese American - firstM: ["Wei", "Kevin", "Jason", "Alan"], - firstF: ["Amy", "Cindy", "Michelle", "Tina"], - last: ["Chen", "Wang", "Liu", "Huang", "Lin", "Wu", "Zhang"], - weight: 4, - }, - { - // Vietnamese American - firstM: ["Minh", "Vincent", "Phong", "Tuan"], - firstF: ["Linh", "Thao", "Kim-Ly", "Vy"], - last: ["Nguyen", "Tran", "Pham", "Le", "Vo", "Dang"], - weight: 3, - }, -] - -// Neighborhood jitter radii are in degrees. Weighted toward the Eastside, Southeast LA and the harbor -// corridor. - -interface Hood { - name: string - lat: number - lng: number - r: number - weight: number - streets: readonly string[] -} - -const HOODS: readonly Hood[] = [ - { - name: "Boyle Heights", - lat: 34.0397, - lng: -118.2077, - r: 0.01, - weight: 10, - streets: [ - "Cesar Chavez Ave", - "Soto St", - "1st St", - "4th St", - "Whittier Blvd", - "Lorena St", - "Evergreen Ave", - "St Louis St", - ], - }, - { - name: "East LA", - lat: 34.0239, - lng: -118.1721, - r: 0.012, - weight: 9, - streets: ["Whittier Blvd", "Atlantic Blvd", "3rd St", "Mednik Ave", "Arizona Ave", "Hammel St"], - }, - { - name: "Highland Park", - lat: 34.1115, - lng: -118.187, - r: 0.01, - weight: 7, - streets: ["York Blvd", "Figueroa St", "Avenue 56", "Monte Vista St", "Marmion Way"], - }, - { - name: "El Sereno", - lat: 34.0806, - lng: -118.1763, - r: 0.01, - weight: 6, - streets: ["Huntington Dr", "Eastern Ave", "Alhambra Ave", "Valley Blvd"], - }, - { - name: "Lincoln Heights", - lat: 34.07, - lng: -118.2, - r: 0.008, - weight: 6, - streets: ["N Broadway", "Daly St", "Main St", "Avenue 26", "Workman St"], - }, - { - name: "City Terrace", - lat: 34.057, - lng: -118.183, - r: 0.007, - weight: 4, - streets: ["City Terrace Dr", "Eastern Ave", "Herbert Ave"], - }, - { - name: "Huntington Park", - lat: 33.9817, - lng: -118.2251, - r: 0.009, - weight: 7, - streets: ["Pacific Blvd", "Gage Ave", "Slauson Ave", "Florence Ave", "Santa Fe Ave"], - }, - { - name: "South Gate", - lat: 33.9547, - lng: -118.212, - r: 0.01, - weight: 5, - streets: ["Tweedy Blvd", "Long Beach Blvd", "Firestone Blvd", "Atlantic Ave"], - }, - { - name: "Pacoima", - lat: 34.2728, - lng: -118.4201, - r: 0.012, - weight: 6, - streets: ["Van Nuys Blvd", "Glenoaks Blvd", "Laurel Canyon Blvd", "Foothill Blvd", "Paxton St"], - }, - { - name: "Van Nuys", - lat: 34.1899, - lng: -118.4514, - r: 0.012, - weight: 5, - streets: ["Van Nuys Blvd", "Victory Blvd", "Sherman Way", "Sepulveda Blvd", "Kester Ave"], - }, - { - name: "Sylmar", - lat: 34.3078, - lng: -118.4453, - r: 0.012, - weight: 3, - streets: ["San Fernando Rd", "Maclay Ave", "Glenoaks Blvd", "Hubbard St"], - }, - { - name: "Sun Valley", - lat: 34.217, - lng: -118.37, - r: 0.01, - weight: 3, - streets: ["San Fernando Rd", "Sunland Blvd", "Vineland Ave", "Lankershim Blvd"], - }, - { - name: "Wilmington", - lat: 33.7801, - lng: -118.2646, - r: 0.01, - weight: 5, - streets: ["Avalon Blvd", "Anaheim St", "Pacific Coast Hwy", "Wilmington Blvd", "L St"], - }, - { - name: "San Pedro", - lat: 33.7361, - lng: -118.2922, - r: 0.01, - weight: 4, - streets: ["Gaffey St", "Pacific Ave", "25th St", "Western Ave", "6th St"], - }, - { - name: "Watts", - lat: 33.9425, - lng: -118.2417, - r: 0.008, - weight: 5, - streets: ["103rd St", "Central Ave", "Compton Ave", "Wilmington Ave", "Grandee Ave"], - }, - { - name: "South LA", - lat: 34.0, - lng: -118.292, - r: 0.014, - weight: 7, - streets: [ - "Vermont Ave", - "Western Ave", - "Normandie Ave", - "Slauson Ave", - "Manchester Ave", - "Figueroa St", - ], - }, - { - name: "Koreatown", - lat: 34.0577, - lng: -118.3009, - r: 0.009, - weight: 5, - streets: [ - "Wilshire Blvd", - "Olympic Blvd", - "Western Ave", - "Vermont Ave", - "8th St", - "Normandie Ave", - ], - }, - { - name: "Westlake", - lat: 34.057, - lng: -118.276, - r: 0.007, - weight: 5, - streets: ["Alvarado St", "7th St", "Wilshire Blvd", "Union Ave", "Bonnie Brae St"], - }, - { - name: "Pico-Union", - lat: 34.047, - lng: -118.283, - r: 0.007, - weight: 5, - streets: ["Pico Blvd", "Union Ave", "Hoover St", "Venice Blvd", "Alvarado St"], - }, - { - name: "Cypress Park", - lat: 34.093, - lng: -118.224, - r: 0.006, - weight: 3, - streets: ["Cypress Ave", "Figueroa St", "San Fernando Rd", "Division St"], - }, - { - name: "Glassell Park", - lat: 34.113, - lng: -118.232, - r: 0.007, - weight: 3, - streets: ["Eagle Rock Blvd", "Verdugo Rd", "San Fernando Rd", "Fletcher Dr"], - }, - { - name: "Echo Park", - lat: 34.0782, - lng: -118.2606, - r: 0.007, - weight: 4, - streets: ["Sunset Blvd", "Echo Park Ave", "Glendale Blvd", "Alvarado St"], - }, - { - name: "Hollywood", - lat: 34.0928, - lng: -118.3287, - r: 0.01, - weight: 3, - streets: ["Hollywood Blvd", "Sunset Blvd", "Santa Monica Blvd", "Western Ave", "Gower St"], - }, - { - name: "North Hollywood", - lat: 34.172, - lng: -118.377, - r: 0.01, - weight: 4, - streets: ["Lankershim Blvd", "Magnolia Blvd", "Victory Blvd", "Vineland Ave"], - }, - { - name: "Panorama City", - lat: 34.227, - lng: -118.449, - r: 0.009, - weight: 4, - streets: ["Van Nuys Blvd", "Roscoe Blvd", "Nordhoff St", "Woodman Ave"], - }, - { - name: "Harbor Gateway", - lat: 33.86, - lng: -118.29, - r: 0.01, - weight: 2, - streets: ["Vermont Ave", "Figueroa St", "Gardena Blvd", "190th St"], - }, -] - -const PARKS: readonly { name: string; hood: string; lat: number; lng: number }[] = [ - { name: "Hollenbeck Park", hood: "Boyle Heights", lat: 34.0367, lng: -118.2135 }, - { name: "Ruben Salazar Park", hood: "East LA", lat: 34.0236, lng: -118.1893 }, - { name: "Salt Lake Park", hood: "Huntington Park", lat: 33.9757, lng: -118.2172 }, - { name: "Hazard Park", hood: "Boyle Heights", lat: 34.0645, lng: -118.2005 }, - { name: "Lincoln Park", hood: "Lincoln Heights", lat: 34.0705, lng: -118.2028 }, - { name: "Sycamore Grove Park", hood: "Highland Park", lat: 34.0996, lng: -118.1998 }, - { name: "Ted Watkins Memorial Park", hood: "Watts", lat: 33.933, lng: -118.2379 }, - { name: "MacArthur Park", hood: "Westlake", lat: 34.059, lng: -118.2785 }, - { name: "Rio de Los Angeles State Park", hood: "Cypress Park", lat: 34.0994, lng: -118.2273 }, - { name: "Ernest E. Debs Regional Park", hood: "El Sereno", lat: 34.0873, lng: -118.1935 }, - { name: "Ken Malloy Harbor Regional Park", hood: "Wilmington", lat: 33.7863, lng: -118.2879 }, - { name: "Hansen Dam Recreation Area", hood: "Pacoima", lat: 34.2612, lng: -118.3898 }, - { name: "Sepulveda Basin", hood: "Van Nuys", lat: 34.1755, lng: -118.4838 }, - { name: "Point Fermin Park", hood: "San Pedro", lat: 33.706, lng: -118.2936 }, - { name: "Normandie Recreation Center", hood: "South LA", lat: 34.0261, lng: -118.3003 }, - { name: "Seoul International Park", hood: "Koreatown", lat: 34.0546, lng: -118.3082 }, - { name: "North Hollywood Park", hood: "North Hollywood", lat: 34.1638, lng: -118.3801 }, - { name: "Echo Park Lake", hood: "Echo Park", lat: 34.0723, lng: -118.2606 }, -] - -function hoodByName(name: string): Hood { - return HOODS.find((h) => h.name === name) ?? HOODS[0]! -} - function jitterPoint(lat: number, lng: number, r: number): { lat: number; lng: number } { const angle = rand() * Math.PI * 2 const dist = Math.sqrt(rand()) * r return { - lat: +(lat + Math.sin(angle) * dist).toFixed(6), - lng: +(lng + Math.cos(angle) * dist * 1.2).toFixed(6), + lat: +(lat + Math.sin(angle) * dist).toFixed(COORDINATE_DECIMALS), + lng: +(lng + Math.cos(angle) * dist * LNG_JITTER_STRETCH).toFixed(COORDINATE_DECIMALS), } } -// Deliberately casual: lowercase drift, loose punctuation, Spanish and Spanglish mixed in, occasional -// emoji, no long-form writing and no em dashes anywhere (validate() enforces the last). - -const BIO_NEUTRAL: readonly string[] = [ - "{hood} born and raised", - "trying to keep {hood} clean one block at a time", - "east side til i die", - "community first. {hood}", - "if you see me picking up trash say hi", - "small business owner on {street}", - "youth soccer coach in {hood}", - "keeping it clean in {hood} 🧹", - "born in {hood}, still here, not leaving", - "retired LAUSD. love my neighborhood", - "just here to report potholes tbh", - "organizing cleanups w my neighbors. dm me if you wanna help", - "{hood} neighborhood watch", -] - -const BIO_MALE: readonly string[] = [ - "dad of 3. tired of the dumping on our streets", - "girl dad in {hood}", - "dog dad, {hood}", -] -const BIO_FEMALE: readonly string[] = [ - "mom of 3. tired of the dumping on our streets", - "{hood} resident, dog mom, 311 power user", - "abuela energy. {hood}", -] - -/** Only for Hispanic users; vecino/vecina gendered. */ -const BIO_HISPANIC_NEUTRAL: readonly string[] = [ - "orgullosamente de {hood} 🇲🇽", - "aqui puro {hood} 💪", - "la comunidad es todo", -] -const BIO_HISPANIC_M: readonly string[] = ["vecino de {hood}, aqui para ayudar"] -const BIO_HISPANIC_F: readonly string[] = ["vecina de {hood}, aqui para ayudar"] - -/** {street}/{street2}/{hood} slots are filled per author. */ -const POST_TEMPLATES_EN: readonly string[] = [ - "third couch this month dumped on {street}. who keeps doing this", - "the graffiti on the handball courts finally got painted over 🙏", - "somebody left a whole entertainment center on the corner of {street} and {street2} lol", - "shoutout to the crew that cleaned the alley behind {street} this weekend. looks brand new", - "when is the city gonna fix the streetlight on {street}, its been out for like 2 months", - "reported a mattress on {street} last tuesday and they actually picked it up friday. not bad", - "ok whoever keeps dumping tires by the wash, we see you 🤨", - "morning walk update: {street} is looking clean for once. small wins", - "the bulky item pickup line had me on hold 40 mins. this app is way faster", - "we need more trash cans on {street} near the bus stop. overflowing every weekend", - "just moved to {hood} and joined this app, love seeing neighbors actually fix stuff", - "anyone else notice the illegal dumping gets worse right after the first of the month", - "picked up 3 bags on my street this morning before work. tired but worth it", - "the mural on {street} got tagged again 😔 gonna organize a repaint", - "city came and cleared the {street} underpass today. hope it stays clean this time", - "psa: bulky item pickup is free, you dont have to dump your sofa on {street} 😤", - "not all heroes wear capes, some just bring their own trash grabbers", - "the sidewalk on {street} is basically an obstacle course. reported like 4 spots today", - "my kids and i picked up trash at the park today. teach em young", - "10/10 morning, coffee from the panaderia and a clean street for once", - "does anyone know who to talk to about the abandoned car on {street}, been there 3 weeks", - "the amount of fast food trash on {street} after friday night is wild", - "shout out to the senora on my block who sweeps the whole sidewalk every morning", - "found a shopping cart in the LA river again. classic", - "neighbors really came through this weekend, {street} is spotless", - "why do people dump paint cans in the alley. thats toxic waste man", - "councilman's office actually called me back about the {street} dumping. progress??", - "the little free library on {street} survived another year 🥹 love this block", - "somebody stole the trash can from the bus stop?? lol only in LA", - "green waste everywhere after the wind last night. be careful driving on {street}", - "im convinced the same truck dumps on {street} every sunday night. gonna get a plate next time", - "starting to see more people use this app in {hood}. keep reporting yall, it works", - "cleaned up the parkway strip in front of my house. do your part people", - "walking to the market and counted 6 illegal dump spots on {street}. all reported", - "our block finally got the speed humps. now if we could get the trash handled", - "sunday morning cleanups hit different. peaceful out here", - "reminder that the storm drains go straight to the ocean. keep em clear", - "big respect to the folks who do this every single week without any credit", - "the empty lot on {street} needs some love, thinking of organizing something", - "trash pickup skipped our street again this week?? anyone else on {street}", - "if every block had 2 people who cared we could keep this whole neighborhood clean", -] - -/** Drawn only by Hispanic authors. */ -const POST_TEMPLATES_ES: readonly string[] = [ - "mucha basura en la calle otra vez. ya reporte, a ver si hacen algo", - "el alley behind my place is getting bad again, gonna report it manana", - "gracias a todos los que vinieron hoy, quedo bien limpio el parque 💪", - "los fines de semana la gente tira basura como si nada. respeten el barrio", - "vamos a limpiar {hood} este sabado, quien se apunta", - "cada quien su bolsa. sabado 9am. no excuses", - "hoy tocó limpiar la esquina de {street}. entre 4 lo hicimos en una hora", - "esta app si funciona, reporte un colchon y en 3 dias lo recogieron", - "que bonito se ve {hood} cuando todos ayudamos", -] - -const REPORT_POST_EN: readonly string[] = [ - "reported this dump on {street}, yall check it out so the city sees it", - "this has been here over a week. finally reported it", - "look at this mess. reported. lets see how long it takes", - "reported this one this morning, right by the school 😡", - "adding this to the pile of reports on {street}. its bad out here", - "cant even use the sidewalk. reported", - "this is right in front of the panaderia. reported it, share so it gets fixed", - "week 2 of this couch. reported again lol", - "who does this?? reported", -] -const REPORT_POST_ES: readonly string[] = [ - "como es posible que dejen esto asi. ya lo reporte", - "miren esto. reportado. compartan para que lo vean", -] - -const EVENT_POST_EN: readonly string[] = [ - "hosting a cleanup this weekend, bring gloves if you got em. everyone welcome", - "cleanup this saturday 🧹 kids welcome, we got extra grabbers", - "we're doing another one. last time we filled 20 bags, lets beat that", - "first cleanup im organizing, be nice lol. hope to see some of you there", - "join us saturday morning, coffee and pan dulce for volunteers ☕", - "one more cleanup before it gets too hot. roll thru", - "big one this weekend. bring the whole family", -] -const EVENT_POST_ES: readonly string[] = [ - "vamos a limpiar este sabado, traigan agua y guantes. los espero", - "este sabado nos toca limpiar. lleguenle con la familia", -] - -const EVENT_RECAP_EN: readonly string[] = [ - "{bags} bags today. arms are dead but the block looks brand new. thank you everyone 🙏", - "we got {bags} bags out of the park today. proud of this neighborhood", - "another one done. {bags} bags, a couch, and somehow a car bumper lol. great turnout", - "small crew today but we still pulled {bags} bags. every bit counts", - "thank you to the {n} people who showed up today. {bags} bags collected", -] -const EVENT_RECAP_ES: readonly string[] = [ - "{bags} bolsas hoy!! gracias a todos los que vinieron 💪", - "terminamos con {bags} bolsas. gracias a mi gente que llego temprano", -] - -const REPLY_GENERIC_EN: readonly string[] = [ - "same thing on my street", - "reported one like this last week, took 3 weeks but they picked it up", - "this is why i love this app", - "311 never picks up, this is faster fr", - "ugh not again", - "facts", - "thank you for doing this", - "we appreciate you 🙏", - "same in {hood} honestly", - "its been like this for weeks", - "somebody has to say it", - "100%", - "we need cameras out there", - "the city needs to do better", - "keep us posted", - "this made my day", - "couldnt agree more", - "im telling my landlord about this app lol", - "the real MVP", - "on my way to report the one by my house too", - "hope they fix it soon", - "lmk if you need help", - "this block deserves better", - "seen it, its worse in person", -] -const REPLY_GENERIC_ES: readonly string[] = [ - "gracias por reportar 🙏", - "el respeto al barrio empieza por uno mismo", - "asi es", - "no manches", - "que bueno que alguien hace algo", - "orale, buen trabajo", -] - -const REPLY_EVENT_EN: readonly string[] = [ - "i'll be there", - "count me in", - "what time does it start?", - "can i bring my kids?", - "do we need to bring our own gloves", - "just signed up 🙌", - "cant make this one but next time for sure", - "bringing 2 friends", - "is there parking nearby", - "see you saturday", - "my whole family is coming lol", - "first time doing one of these, excited", -] -const REPLY_EVENT_ES: readonly string[] = ["yo tambien voy", "ahi estare", "llevare bolsas extra"] - -const REPLY_REPORT_EN: readonly string[] = [ - "just liked it so it gets visibility", - "reported the same spot last month, they cleared it but it came back", - "thats right by my kids school 😡", - "share it in the group chat too", - "the city cleared one like this on my street in about a week", - "took a pic of the same pile yesterday, glad you reported", - "this intersection is always bad", - "sad that it takes an app for the city to do their job", -] -const REPLY_REPORT_ES: readonly string[] = ["eso esta a una cuadra de mi casa", "gracias vecino"] - -const QUOTE_EN: readonly string[] = [ - "this right here", - "everyone in {hood} needs to see this", - "and people say nobody cares about this neighborhood", - "sharing for the morning crowd", - "this is the kind of stuff that keeps me on this app", - "proof that reporting works yall", -] -const QUOTE_ES: readonly string[] = ["lo que siempre digo", "mi gente 💪"] - -/** [titles, descriptions]; slots: {street} {street2}. */ -const REPORT_CONTENT: Record< - ReportType, - { titles: readonly string[]; descs: readonly string[]; descsEs?: readonly string[] } -> = { - dump: { - titles: [ - "mattress dumped on {street}", - "couch left on the corner of {street} and {street2}", - "pile of construction debris on {street}", - "tv and boxes dumped by the alley", - "furniture dumped on the sidewalk", - "trash bags piling up on {street}", - "tires dumped near {street}", - "someone dumped a washer on {street}", - "big pile of junk on the parkway", - "shopping carts and trash on {street}", - ], - descs: [ - "been here over a week now. right in front of the laundromat. kids have to walk around it into the street", - "keeps growing every day. started as one bag now its a whole pile", - "someone dumped this overnight. blocking half the sidewalk", - "third time this month at this exact spot. we need a camera or something", - "smells terrible and there are flies everywhere. please pick up soon", - "right next to the bus stop where people wait every morning", - "wood with nails sticking out, dangerous for kids walking to school", - "looks like a contractor dumped it, there are paint buckets and drywall", - "elderly neighbors cant get around it with their carts", - "its right by the storm drain, gonna wash into the river when it rains", - ], - descsEs: [ - "esta enfrente de mi casa desde el lunes. ya no podemos ni pasar por la banqueta", - "la gente sigue tirando basura aqui, cada semana es lo mismo", - ], - }, - graffiti: { - titles: [ - "tagging on the wall at {street}", - "graffiti on the bus bench", - "fresh tags on the store shutters", - "graffiti covering the street sign", - "wall on {street} tagged again", - "tags all over the underpass", - ], - descs: [ - "whole wall got hit over the weekend. was just painted 2 months ago", - "the stop sign is barely readable now, thats a safety issue", - "small business owner already dealing with a lot, now this", - "gang tags this time, neighbors are worried. please prioritize", - "they tagged the mural too which is really sad, that mural took months", - "same tags as the ones on {street2}, probably the same people", - ], - descsEs: ["rayaron toda la pared otra vez. apenas la habian pintado"], - }, - encampment: { - titles: [ - "encampment growing under the {street} overpass", - "tents blocking the sidewalk on {street}", - "encampment by the wash near {street}", - ], - descs: [ - "not trying to get anyone in trouble, they need services. but the sidewalk is fully blocked and theres a lot of debris", - "its grown from 2 tents to about 8 in a month. trash is piling up around it", - "requesting outreach services, theres an older man there who needs medical help", - "kids walk this route to school and have to go into the street", - ], - }, - infrastructure: { - titles: [ - "broken sprinkler flooding the sidewalk on {street}", - "fire hydrant leaking on {street}", - "water main leaking into the street", - "broken streetlight on {street}", - "exposed wiring on the light pole", - ], - descs: [ - "water has been running down the gutter for 3 days straight. huge waste", - "the whole corner is flooded every morning. slipping hazard", - "light has been out for weeks, its really dark on this block at night. safety issue", - "you can hear the water hissing, probably losing hundreds of gallons", - ], - descsEs: ["el poste tiene cables colgando, esta peligroso"], - }, - pavement: { - titles: [ - "huge pothole on {street}", - "sidewalk buckled by tree roots on {street}", - "pothole damaging cars near {street} and {street2}", - "cracked curb ramp on the corner", - "street cracking apart on {street}", - ], - descs: [ - "hit it last night, almost lost a tire. its deep", - "my neighbor in a wheelchair literally cannot use this sidewalk", - "gets worse every week, and cars swerve into the other lane to miss it", - "seniors trip here all the time, someone fell last week", - "been reported before and patched but the patch is already gone", - ], - descsEs: ["el bache esta enorme, ya varios carros se han danado"], - }, - vegetation: { - titles: [ - "overgrown weeds blocking the sidewalk on {street}", - "dead palm fronds hanging over {street}", - "tree branch about to fall on {street}", - "brush pile fire hazard by {street}", - ], - descs: [ - "the weeds are shoulder height, you cant see around the corner when driving", - "big dead frond hanging right over the bus stop. someone is gonna get hurt", - "branch cracked in the wind last week and is hanging by a thread", - "dry brush right up against the fence, one spark and its a problem", - ], - descsEs: ["las ramas ya tapan toda la banqueta"], - }, - other: { - titles: [ - "abandoned car on {street}", - "shopping carts collecting on the corner", - "broken glass all over the sidewalk", - "dead animal on {street}", - "leaking dumpster behind the businesses", - ], - descs: [ - "hasnt moved in 3 weeks, flat tires, windows are smashed now", - "at least 6 carts from the ranch market piling up", - "please send someone, its been days and it smells really bad", - "someone smashed bottles all over, dogs and kids walk here", - "grease and trash water running into the gutter", - ], - }, -} - -const EVENT_TITLE_EN: readonly string[] = [ - "{park} cleanup", - "{hood} community cleanup", - "{street} alley cleanup", - "adopt-a-block {hood}", - "{hood} saturday sweep", -] -const EVENT_TITLE_ES: readonly string[] = ["limpieza comunitaria en {park}"] - -const EVENT_DESC_EN: readonly string[] = [ - "meet at the main entrance. we'll split into teams and cover the park and the streets around it. bags and some grabbers provided, bring gloves and water if you can", - "monthly cleanup with the neighbors. all ages welcome, we usually finish by noon and someone always brings tamales", - "the alley has gotten bad again so we're getting a crew together. wear closed toe shoes, there might be glass", - "quick 2 hour cleanup then tacos after for whoever can stay. first timers welcome, we'll show you the ropes", - "bringing the community together to take care of our space. supplies provided by the neighborhood council", -] -const EVENT_DESC_ES: readonly string[] = [ - "juntandonos para limpiar el parque y las calles de alrededor. traigan guantes si tienen, nosotros ponemos las bolsas", -] - -const BRING_POOL: readonly string[] = [ - "gloves", - "water", - "sunscreen", - "hat", - "trash grabbers", - "closed toe shoes", - "reusable water bottle", -] - -const SLOT_SETS: readonly (readonly { - title: string - description: string | null - capacity: number | null -}[])[] = [ - [ - { - title: "Registration table", - description: "check people in and hand out supplies", - capacity: 2, - }, - { title: "Supplies and water", description: "keep the water station stocked", capacity: 2 }, - { title: "Street team", description: "cover the blocks around the park", capacity: null }, - ], - [ - { title: "8-10am shift", description: null, capacity: 12 }, - { title: "10-12 shift", description: null, capacity: 12 }, - ], - [ - { - title: "Heavy lifting crew", - description: "for the big stuff, bring work gloves", - capacity: 6, - }, - { title: "General cleanup", description: null, capacity: null }, - { title: "Kids zone", description: "light duty for families with little ones", capacity: 8 }, - ], -] - -const TIMELINE_ACK_NOTES: readonly string[] = [ - "Forwarded to LA Sanitation", - "Routed to the responsible department", - "Received by the city, reference logged", -] -const TIMELINE_RESOLVE_NOTES: readonly string[] = [ - "Crew confirmed pickup complete", - "Marked resolved after site inspection", - "Cleared by sanitation crew", -] - type Tier = "power" | "casual" | "light" | "lurker" interface SeedUser { @@ -1287,63 +379,78 @@ function bilingual(user: SeedUser, en: readonly string[], es: readonly string[]) return pick(useEs ? es : en) } +function pickName(hispanic: boolean, female: boolean): { first: string; last: string } { + if (hispanic) { + return { + first: female ? pick(HISPANIC_FIRST_F) : pick(HISPANIC_FIRST_M), + last: pick(HISPANIC_LAST), + } + } + const pool = pickWeighted(OTHER_POOLS.map((p) => [p, p.weight] as const)) + const first = female ? pick(pool.firstF) : pick(pool.firstM) + return { first, last: pick(pool.last) } +} + +function displayNameFor(first: string, last: string, hispanic: boolean): string { + const dFirst = hispanic && chance(0.35) ? (ACCENTED[first] ?? first) : first + const dLast = hispanic && chance(0.2) ? (ACCENTED[last] ?? last) : last + const fullName = `${dFirst} ${dLast}` + // The full name is listed twice on purpose: merging the weights would remap which name each roll picks + // and so change every cohort generated from a given seed. + return pickWeighted([ + [fullName, 55], + [`${dFirst} ${dLast[0]}.`, 15], + [dFirst, 10], + [`${dFirst.toLowerCase()} ${dLast.toLowerCase()}`, 10], + [fullName, 10], + ]) +} + +function uniqueHandle(first: string, last: string, usedHandles: Set): string { + const fl = first.toLowerCase().replace(/[^a-z0-9]/g, "") + const ll = last.toLowerCase().replace(/[^a-z0-9]/g, "") + const candidates = [ + `${fl}${ll}`, + `${fl}_${ll}`, + `${fl}${ll[0] ?? ""}${rint(1, 99)}`, + `${fl}_${rint(80, 99)}`, + `${fl}${ll}${rint(1, 9)}`, + `${fl}_la`, + `${fl}${rint(100, 999)}`, + ] + for (const c of shuffle(candidates)) { + const h = c.slice(0, HANDLE_MAX_LENGTH) + if (h.length >= HANDLE_MIN_LENGTH && !usedHandles.has(h)) return h + } + let n = rint(10, 9999) + while (usedHandles.has(`${fl}${n}`.slice(0, HANDLE_MAX_LENGTH))) n++ + return `${fl}${n}`.slice(0, HANDLE_MAX_LENGTH) +} + +function popularityFor(tier: Tier): number { + return ( + Math.exp((rand() + rand() + rand() - 1.5) * 1.6) * + (tier === "power" ? 3 : tier === "casual" ? 1.2 : 0.6) + ) +} + +function bioFor(user: SeedUser, female: boolean): string { + const bioPool: string[] = [...BIO_NEUTRAL, ...(female ? BIO_FEMALE : BIO_MALE)] + if (user.hispanic) { + bioPool.push(...BIO_HISPANIC_NEUTRAL, ...(female ? BIO_HISPANIC_F : BIO_HISPANIC_M)) + } + return fill(pick(bioPool), user, { street: pick(user.hood.streets) }) +} + function makeUsers(count: number, start: Date, end: Date): SeedUser[] { const usedHandles = new Set() const users: SeedUser[] = [] for (let i = 0; i < count; i++) { const hispanic = chance(0.7) const female = chance(0.52) - let first: string - let last: string - if (hispanic) { - first = female ? pick(HISPANIC_FIRST_F) : pick(HISPANIC_FIRST_M) - last = pick(HISPANIC_LAST) - } else { - const pool = pickWeighted(OTHER_POOLS.map((p) => [p, p.weight] as const)) - first = female ? pick(pool.firstF) : pick(pool.firstM) - last = pick(pool.last) - } - - const dFirst = hispanic && chance(0.35) ? (ACCENTED[first] ?? first) : first - const dLast = hispanic && chance(0.2) ? (ACCENTED[last] ?? last) : last - const displayName = pickWeighted([ - [`${dFirst} ${dLast}`, 55], - [`${dFirst} ${dLast[0]}.`, 15], - [dFirst, 10], - [`${dFirst.toLowerCase()} ${dLast.toLowerCase()}`, 10], - [ - `${dFirst} ${dLast}`.toUpperCase() === `${dFirst} ${dLast}` - ? `${dFirst} ${dLast}` - : `${dFirst} ${dLast}`, - 10, - ], - ]) - - // Must match HANDLE_REGEX (3-20 of [A-Za-z0-9_]). - const fl = first.toLowerCase().replace(/[^a-z0-9]/g, "") - const ll = last.toLowerCase().replace(/[^a-z0-9]/g, "") - const candidates = [ - `${fl}${ll}`, - `${fl}_${ll}`, - `${fl}${ll[0] ?? ""}${rint(1, 99)}`, - `${fl}_${rint(80, 99)}`, - `${fl}${ll}${rint(1, 9)}`, - `${fl}_la`, - `${fl}${rint(100, 999)}`, - ] - let handle = "" - for (const c of shuffle(candidates)) { - const h = c.slice(0, 20) - if (h.length >= 3 && !usedHandles.has(h)) { - handle = h - break - } - } - if (!handle) { - let n = rint(10, 9999) - while (usedHandles.has(`${fl}${n}`.slice(0, 20))) n++ - handle = `${fl}${n}`.slice(0, 20) - } + const { first, last } = pickName(hispanic, female) + const displayName = displayNameFor(first, last, hispanic) + const handle = uniqueHandle(first, last, usedHandles) usedHandles.add(handle) const hood = pickWeighted(HOODS.map((h) => [h, h.weight] as const)) @@ -1353,9 +460,7 @@ function makeUsers(count: number, start: Date, end: Date): SeedUser[] { ["light", 28], ["lurker", 12], ]) - const popularity = - Math.exp((rand() + rand() + rand() - 1.5) * 1.6) * - (tier === "power" ? 3 : tier === "casual" ? 1.2 : 0.6) + const popularity = popularityFor(tier) const user: SeedUser = { id: randomUUID(), @@ -1378,13 +483,7 @@ function makeUsers(count: number, start: Date, end: Date): SeedUser[] { followerCount: 0, followingCount: 0, } - if (chance(0.62)) { - const bioPool: string[] = [...BIO_NEUTRAL, ...(female ? BIO_FEMALE : BIO_MALE)] - if (hispanic) { - bioPool.push(...BIO_HISPANIC_NEUTRAL, ...(female ? BIO_HISPANIC_F : BIO_HISPANIC_M)) - } - user.bio = fill(pick(bioPool), user, { street: pick(hood.streets) }) - } + if (chance(0.62)) user.bio = bioFor(user, female) users.push(user) } return users @@ -1430,34 +529,106 @@ function makeFollows(users: SeedUser[], now: Date): SeedFollow[] { return list } +function eventScheduledAt(kind: SeedEvent["status"], organizer: SeedUser, now: Date): Date { + if (kind === "upcoming") return nextSaturdayish(now, rint(3, 21)) + return nextSaturdayish( + new Date( + later(organizer.createdAt, new Date(now.getTime() - EVENT_HISTORY_DAYS * DAY)).getTime(), + ), + rint(7, 120), + now, + ) +} + +/** + * The create must land in the PAST even for upcoming events (scheduled_at minus the lead can exceed now + * when the event is less than that lead away, which would backdate joins into the future). + */ +function eventCreatedAt(organizer: SeedUser, scheduledAt: Date, now: Date): Date { + const createdCeiling = new Date( + Math.min( + scheduledAt.getTime() - EVENT_MIN_CREATE_LEAD_DAYS * DAY, + now.getTime() - EVENT_MIN_AGE_HOURS * HOUR, + ), + ) + return randTimestamp( + later(organizer.createdAt, new Date(scheduledAt.getTime() - EVENT_MAX_CREATE_LEAD_DAYS * DAY)), + createdCeiling, + ) +} + +function addEventMembers(ev: SeedEvent, users: SeedUser[], joinEnd: Date): void { + const memberTarget = ev.status === "cancelled" ? rint(3, 8) : rint(6, 26) + const weight = (c: SeedUser) => + (c.hood.name === ev.hood.name ? 4 : 1) * + (c.tier === "lurker" ? 0.3 : 1) * + Math.sqrt(c.popularity) + const joiners = sampleWeighted(users, weight, memberTarget, new Set([ev.organizer])) + for (const [j, u] of joiners.entries()) { + const joinedAt = randTimestamp(later(u.createdAt, ev.createdAt), joinEnd) + const role = j === 0 && chance(0.4) ? "cohost" : "member" + if (role === "cohost") ev.cohost = u + ev.members.push({ user: u, role, joinedAt }) + } +} + +/** + * A richer board on ~40% of events and 0169's default slot on every other OPEN one (past and cancelled + * keep the empty board it skips). + */ +function eventSlots(ev: SeedEvent): SeedEvent["slots"] { + if (chance(0.4)) { + const set = pick(SLOT_SETS) + return set.map((s, idx) => ({ id: randomUUID(), ...s, sortOrder: idx })) + } + if (ev.status !== "upcoming") return [] + return [ + { + id: randomUUID(), + title: DEFAULT_EVENT_SLOT_TITLE, + description: null, + capacity: ev.capacity, + sortOrder: 0, + }, + ] +} + +/** Claims come from members only, one per member, with slot capacity respected. */ +function addSlotClaims(ev: SeedEvent, joinEnd: Date): void { + if (ev.slots.length === 0) return + const claimed = new Set() + for (const m of ev.members) { + if (m.role === "organizer" || !chance(0.5)) continue + const open = ev.slots.filter( + (s) => s.capacity === null || ev.claims.filter((c) => c.slotId === s.id).length < s.capacity, + ) + if (open.length === 0 || claimed.has(m.user.id)) continue + const slot = pick(open) + claimed.add(m.user.id) + ev.claims.push({ + userId: m.user.id, + slotId: slot.id, + claimedAt: randTimestamp(m.joinedAt, joinEnd), + }) + } +} + function makeEvents(users: SeedUser[], now: Date): SeedEvent[] { - const organizers = shuffle(users.filter((u) => u.tier === "power")).slice(0, 14) + const organizers = shuffle(users.filter((u) => u.tier === "power")).slice( + 0, + EVENT_ORGANIZER_COUNT, + ) const events: SeedEvent[] = [] const parkPool = shuffle([...PARKS]) - const count = 20 - for (let i = 0; i < count; i++) { + for (let i = 0; i < EVENT_COUNT; i++) { const organizer = organizers[i % organizers.length]! const park = parkPool[i % parkPool.length]! const hood = hoodByName(park.hood) - const kind: SeedEvent["status"] = i < 13 ? "done" : i === 13 ? "cancelled" : "upcoming" - const scheduledAt = - kind === "upcoming" - ? nextSaturdayish(now, rint(3, 21)) - : nextSaturdayish( - new Date(later(organizer.createdAt, new Date(now.getTime() - 150 * DAY)).getTime()), - rint(7, 120), - now, - ) - // The create must land in the PAST even for upcoming events (scheduled_at - 2d can exceed now - // when the event is less than 2 days out, which would backdate joins into the future). - const createdCeiling = new Date( - Math.min(scheduledAt.getTime() - 2 * DAY, now.getTime() - 6 * 3600_000), - ) - const createdAt = randTimestamp( - later(organizer.createdAt, new Date(scheduledAt.getTime() - 28 * DAY)), - createdCeiling, - ) - const { lat, lng } = jitterPoint(park.lat, park.lng, 0.0015) + const kind: SeedEvent["status"] = + i < PAST_EVENT_COUNT ? "done" : i === PAST_EVENT_COUNT ? "cancelled" : "upcoming" + const scheduledAt = eventScheduledAt(kind, organizer, now) + const createdAt = eventCreatedAt(organizer, scheduledAt, now) + const { lat, lng } = jitterPoint(park.lat, park.lng, EVENT_SITE_JITTER_DEG) const title = fill(bilingual(organizer, EVENT_TITLE_EN, EVENT_TITLE_ES), organizer, { park: park.name, hood: hood.name, @@ -1485,54 +656,10 @@ function makeEvents(users: SeedUser[], now: Date): SeedEvent[] { hood, } - const memberTarget = kind === "cancelled" ? rint(3, 8) : rint(6, 26) - const weight = (c: SeedUser) => - (c.hood.name === hood.name ? 4 : 1) * - (c.tier === "lurker" ? 0.3 : 1) * - Math.sqrt(c.popularity) - const joiners = sampleWeighted(users, weight, memberTarget, new Set([organizer])) const joinEnd = kind === "upcoming" ? now : scheduledAt - for (const [j, u] of joiners.entries()) { - const joinedAt = randTimestamp(later(u.createdAt, createdAt), joinEnd) - const role = j === 0 && chance(0.4) ? "cohost" : "member" - if (role === "cohost") ev.cohost = u - ev.members.push({ user: u, role, joinedAt }) - } - - // A richer board on ~40% of events and 0169's default slot on every other OPEN one (past and - // cancelled keep the empty board it skips); claims from members only, capacity respected. - if (chance(0.4)) { - const set = pick(SLOT_SETS) - ev.slots = set.map((s, idx) => ({ id: randomUUID(), ...s, sortOrder: idx })) - } else if (kind === "upcoming") { - ev.slots = [ - { - id: randomUUID(), - title: DEFAULT_EVENT_SLOT_TITLE, - description: null, - capacity: ev.capacity, - sortOrder: 0, - }, - ] - } - if (ev.slots.length > 0) { - const claimed = new Set() - for (const m of ev.members) { - if (m.role === "organizer" || !chance(0.5)) continue - const open = ev.slots.filter( - (s) => - s.capacity === null || ev.claims.filter((c) => c.slotId === s.id).length < s.capacity, - ) - if (open.length === 0 || claimed.has(m.user.id)) continue - const slot = pick(open) - claimed.add(m.user.id) - ev.claims.push({ - userId: m.user.id, - slotId: slot.id, - claimedAt: randTimestamp(m.joinedAt, joinEnd), - }) - } - } + addEventMembers(ev, users, joinEnd) + ev.slots = eventSlots(ev) + addSlotClaims(ev, joinEnd) events.push(ev) } return events @@ -1540,29 +667,67 @@ function makeEvents(users: SeedUser[], now: Date): SeedEvent[] { function nextSaturdayish(base: Date, minDays: number, latest?: Date): Date { let d = new Date(base.getTime() + minDays * DAY) - for (let i = 0; i < 7; i++) { + for (let i = 0; i < DAYS_PER_WEEK; i++) { const dow = new Date(d.getTime() + i * DAY).getUTCDay() - if (dow === 6 || (dow === 0 && chance(0.4))) { + if (dow === SATURDAY || (dow === SUNDAY && chance(0.4))) { d = new Date(d.getTime() + i * DAY) break } } if (latest && d.getTime() >= latest.getTime()) d = new Date(latest.getTime() - DAY) - return laLocalToUtc(d.getTime(), 9, pick([0, 0, 30]), 0) + return laLocalToUtc(d.getTime(), EVENT_START_HOUR, pick([0, 0, 30]), 0) +} + +function reportTimeline( + status: SeedReport["status"], + reporter: SeedUser, + createdAt: Date, + now: Date, +): { timeline: SeedReport["timeline"]; publishedAt: Date | null } { + const timeline: SeedReport["timeline"] = [ + { status: "submitted", note: null, createdAt, actorId: reporter.id }, + ] + let publishedAt: Date | null = null + if (status !== "submitted") { + publishedAt = minutesAfter(createdAt, 1, 10) + timeline.push({ status: "published", note: null, createdAt: publishedAt, actorId: null }) + } + let cursor = publishedAt ?? createdAt + const nextStep = (maxDays: number): Date => + randTimestamp(cursor, new Date(Math.min(cursor.getTime() + maxDays * DAY, now.getTime()))) + if (status === "acknowledged" || status === "in_progress" || status === "resolved") { + cursor = nextStep(10) + timeline.push({ + status: "acknowledged", + note: pick(TIMELINE_ACK_NOTES), + createdAt: cursor, + actorId: null, + }) + } + if (status === "in_progress" || (status === "resolved" && chance(0.5))) { + cursor = nextStep(12) + timeline.push({ status: "in_progress", note: null, createdAt: cursor, actorId: null }) + } + if (status === "resolved") { + cursor = nextStep(15) + timeline.push({ + status: "resolved", + note: chance(0.6) ? pick(TIMELINE_RESOLVE_NOTES) : null, + createdAt: cursor, + actorId: null, + }) + } + return { timeline, publishedAt } +} + +function reporterWeight(u: SeedUser): number { + return u.tier === "power" ? 4 : u.tier === "casual" ? 2 : u.tier === "light" ? 1 : 0.2 } function makeReports(users: SeedUser[], count: number, now: Date): SeedReport[] { const reports: SeedReport[] = [] for (let i = 0; i < count; i++) { - const reporter = pickWeighted( - users.map( - (u) => - [ - u, - u.tier === "power" ? 4 : u.tier === "casual" ? 2 : u.tier === "light" ? 1 : 0.2, - ] as const, - ), - ) + const reporter = pickWeighted(users.map((u) => [u, reporterWeight(u)] as const)) const hood = chance(0.85) ? reporter.hood : pickWeighted(HOODS.map((h) => [h, h.weight] as const)) @@ -1587,38 +752,7 @@ function makeReports(users: SeedUser[], count: number, now: Date): SeedReport[] ["resolved", 20], ["submitted", 5], ]) - - const timeline: SeedReport["timeline"] = [ - { status: "submitted", note: null, createdAt, actorId: reporter.id }, - ] - let publishedAt: Date | null = null - if (status !== "submitted") { - publishedAt = minutesAfter(createdAt, 1, 10) - timeline.push({ status: "published", note: null, createdAt: publishedAt, actorId: null }) - } - let cursor = publishedAt ?? createdAt - if (status === "acknowledged" || status === "in_progress" || status === "resolved") { - cursor = randTimestamp(cursor, new Date(Math.min(cursor.getTime() + 10 * DAY, now.getTime()))) - timeline.push({ - status: "acknowledged", - note: pick(TIMELINE_ACK_NOTES), - createdAt: cursor, - actorId: null, - }) - } - if (status === "in_progress" || (status === "resolved" && chance(0.5))) { - cursor = randTimestamp(cursor, new Date(Math.min(cursor.getTime() + 12 * DAY, now.getTime()))) - timeline.push({ status: "in_progress", note: null, createdAt: cursor, actorId: null }) - } - if (status === "resolved") { - cursor = randTimestamp(cursor, new Date(Math.min(cursor.getTime() + 15 * DAY, now.getTime()))) - timeline.push({ - status: "resolved", - note: chance(0.6) ? pick(TIMELINE_RESOLVE_NOTES) : null, - createdAt: cursor, - actorId: null, - }) - } + const { timeline, publishedAt } = reportTimeline(status, reporter, createdAt, now) reports.push({ id: randomUUID(), @@ -1642,14 +776,7 @@ function makeReports(users: SeedUser[], count: number, now: Date): SeedReport[] return reports } -function makePosts( - users: SeedUser[], - events: SeedEvent[], - reports: SeedReport[], - follows: SeedFollow[], - now: Date, -): SeedPost[] { - const posts: SeedPost[] = [] +function followersIndex(users: SeedUser[], follows: SeedFollow[]): Map { const byId = new Map(users.map((u) => [u.id, u])) const followersOf = new Map() for (const f of follows) { @@ -1657,45 +784,53 @@ function makePosts( arr.push(byId.get(f.followerId)!) followersOf.set(f.followeeId, arr) } + return followersOf +} - const addTop = ( - author: SeedUser, - body: string, - createdAt: Date, - eventId: string | null, - reportId: string | null, - ) => { - const p: SeedPost = { - id: randomUUID(), - author, - kind: "post", - body, - replyTo: null, - threadRoot: null, - repostOf: null, - eventId, - reportId, - createdAt, - depth: 0, - likeCount: 0, - replyCount: 0, - repostCount: 0, - saveCount: 0, - mentions: [], - } - posts.push(p) - return p +type PostLinks = Partial< + Pick +> + +function seedPost( + fields: Pick & PostLinks, +): SeedPost { + return { + id: randomUUID(), + replyTo: null, + threadRoot: null, + repostOf: null, + eventId: null, + reportId: null, + mentions: [], + likeCount: 0, + replyCount: 0, + repostCount: 0, + saveCount: 0, + ...fields, } +} + +type AddTopPost = ( + author: SeedUser, + body: string, + createdAt: Date, + eventId: string | null, + reportId: string | null, +) => SeedPost + +function authorPostCount(u: SeedUser): number { + return u.tier === "power" + ? rint(4, 10) + : u.tier === "casual" + ? rint(1, 4) + : u.tier === "light" + ? rint(0, 2) + : 0 +} +function addAuthorPosts(users: SeedUser[], now: Date, addTop: AddTopPost): void { for (const u of users) { - const n = - u.tier === "power" - ? rint(4, 10) - : u.tier === "casual" - ? rint(1, 4) - : u.tier === "light" - ? rint(0, 2) - : 0 + const n = authorPostCount(u) for (let i = 0; i < n; i++) { addTop( u, @@ -1706,7 +841,9 @@ function makePosts( ) } } +} +function addEventPosts(events: SeedEvent[], now: Date, addTop: AddTopPost): void { for (const ev of events) { if (ev.status !== "cancelled") { const promoAt = randTimestamp( @@ -1738,7 +875,7 @@ function makePosts( } } if (ev.status === "done" && chance(0.85)) { - const recapAt = minutesAfter(ev.scheduledAt, 3 * 60, 30 * 60) + const recapAt = minutesAfter(ev.scheduledAt, 3 * HOUR_MINUTES, 30 * HOUR_MINUTES) if (recapAt.getTime() < now.getTime()) { addTop( ev.organizer, @@ -1753,10 +890,12 @@ function makePosts( } } } +} +function addReportPosts(reports: SeedReport[], now: Date, addTop: AddTopPost): void { for (const r of reports) { if (r.status === "submitted" || !chance(0.3)) continue - const at = minutesAfter(r.publishedAt ?? r.createdAt, 5, 36 * 60) + const at = minutesAfter(r.publishedAt ?? r.createdAt, 5, 36 * HOUR_MINUTES) if (at.getTime() >= now.getTime()) continue addTop( r.reporter, @@ -1768,77 +907,101 @@ function makePosts( r.id, ) } +} - const topLevel = posts.filter((p) => p.depth === 0) - for (const p of topLevel) { - const base = p.eventId ? 2.2 : p.reportId ? 1.6 : 1 - const n = Math.min( - 14, - Math.floor(Math.pow(rand(), 1.8) * 7 * base * Math.sqrt(p.author.popularity)), +function replyPools(parent: SeedPost): readonly [readonly string[], readonly string[]] { + if (parent.eventId) return [REPLY_EVENT_EN, REPLY_EVENT_ES] + if (parent.reportId) return [REPLY_REPORT_EN, REPLY_REPORT_ES] + return [REPLY_GENERIC_EN, REPLY_GENERIC_ES] +} + +/** + * One reply per person per thread (the root author may answer their own thread), and no repeated body + * text within a thread; both read as bots otherwise. + */ +function addThreadReplies( + posts: SeedPost[], + root: SeedPost, + users: SeedUser[], + followersOf: Map, + now: Date, +): void { + const base = root.eventId ? 2.2 : root.reportId ? 1.6 : 1 + const n = Math.min( + MAX_REPLIES_PER_THREAD, + Math.floor(Math.pow(rand(), 1.8) * 7 * base * Math.sqrt(root.author.popularity)), + ) + const threadRepliers = new Set() + const threadBodies = new Set() + let parent: SeedPost = root + for (let i = 0; i < n; i++) { + parent = chance(0.75) + ? root + : posts[posts.length - 1]!.depth > 0 && chance(0.5) + ? posts[posts.length - 1]! + : root + if (parent.depth >= MAX_REPLY_DEPTH) parent = root + const followerPool = followersOf.get(parent.author.id) ?? [] + const replier = + parent.depth > 0 && parent.author.id !== root.author.id && chance(0.4) + ? root.author + : followerPool.length > 0 && chance(0.65) + ? pick(followerPool) + : pickWeighted(users.map((u) => [u, u.tier === "lurker" ? 0.2 : 1] as const)) + if (replier.id === parent.author.id) continue + if (replier.id !== root.author.id && threadRepliers.has(replier.id)) continue + const start = later(replier.createdAt, parent.createdAt) + const end = new Date( + Math.min(parent.createdAt.getTime() + REPLY_WINDOW_DAYS * DAY, now.getTime()), ) - // One reply per person per thread (the root author may answer their own thread), and no - // repeated body text within a thread; both read as bots otherwise. - const threadRepliers = new Set() - const threadBodies = new Set() - let parent: SeedPost = p - for (let i = 0; i < n; i++) { - parent = chance(0.75) - ? p - : posts[posts.length - 1]!.depth > 0 && chance(0.5) - ? posts[posts.length - 1]! - : p - if (parent.depth >= 3) parent = p - const followerPool = followersOf.get(parent.author.id) ?? [] - const replier = - parent.depth > 0 && parent.author.id !== p.author.id && chance(0.4) - ? p.author - : followerPool.length > 0 && chance(0.65) - ? pick(followerPool) - : pickWeighted(users.map((u) => [u, u.tier === "lurker" ? 0.2 : 1] as const)) - if (replier.id === parent.author.id) continue - if (replier.id !== p.author.id && threadRepliers.has(replier.id)) continue - const start = later(replier.createdAt, parent.createdAt) - const end = new Date(Math.min(parent.createdAt.getTime() + 5 * DAY, now.getTime())) - if (start.getTime() >= end.getTime()) continue - const [poolEn, poolEs] = parent.eventId - ? [REPLY_EVENT_EN, REPLY_EVENT_ES] - : parent.reportId - ? [REPLY_REPORT_EN, REPLY_REPORT_ES] - : [REPLY_GENERIC_EN, REPLY_GENERIC_ES] - let body = fill(bilingual(replier, poolEn, poolEs), replier) - if (threadBodies.has(body)) continue - threadBodies.add(body) - threadRepliers.add(replier.id) - const mentions: string[] = [] - if (parent.depth > 0 && chance(0.3)) { - body = `@${parent.author.handle} ${body}` - mentions.push(parent.author.id) - } - const reply: SeedPost = { - id: randomUUID(), - author: replier, - kind: "reply", - body, - replyTo: parent, - threadRoot: parent.depth === 0 ? parent : (parent.threadRoot ?? parent), - repostOf: null, - eventId: null, - reportId: null, - createdAt: randTimestamp(start, end), - depth: parent.depth + 1, - likeCount: 0, - replyCount: 0, - repostCount: 0, - saveCount: 0, - mentions, - } - parent.replyCount++ - posts.push(reply) + if (start.getTime() >= end.getTime()) continue + const [poolEn, poolEs] = replyPools(parent) + let body = fill(bilingual(replier, poolEn, poolEs), replier) + if (threadBodies.has(body)) continue + threadBodies.add(body) + threadRepliers.add(replier.id) + const mentions: string[] = [] + if (parent.depth > 0 && chance(0.3)) { + body = `@${parent.author.handle} ${body}` + mentions.push(parent.author.id) } + const reply = seedPost({ + author: replier, + kind: "reply", + body, + replyTo: parent, + threadRoot: parent.depth === 0 ? parent : (parent.threadRoot ?? parent), + createdAt: randTimestamp(start, end), + depth: parent.depth + 1, + mentions, + }) + parent.replyCount++ + posts.push(reply) } +} +function reshareWindow( + sharer: SeedUser, + target: SeedPost, + now: Date, +): { start: Date; end: Date } | null { + const start = later(sharer.createdAt, target.createdAt) + const end = new Date( + Math.min(target.createdAt.getTime() + RESHARE_WINDOW_DAYS * DAY, now.getTime()), + ) + return start.getTime() >= end.getTime() ? null : { start, end } +} + +function addRepostsAndQuotes( + posts: SeedPost[], + topLevel: SeedPost[], + followersOf: Map, + now: Date, +): void { const repostKeys = new Set() - const popularTargets = topLevel.filter((p) => (followersOf.get(p.author.id)?.length ?? 0) >= 3) + const popularTargets = topLevel.filter( + (p) => (followersOf.get(p.author.id)?.length ?? 0) >= MIN_FOLLOWERS_TO_RESHARE, + ) for (const target of popularTargets) { const nReposts = chance(0.16) ? rint(1, 3) : 0 const nQuotes = chance(0.07) ? rint(1, 2) : 0 @@ -1847,58 +1010,63 @@ function makePosts( const reposter = pick(pool) const k = `${reposter.id}:${target.id}` if (reposter.id === target.author.id || repostKeys.has(k)) continue - const start = later(reposter.createdAt, target.createdAt) - const end = new Date(Math.min(target.createdAt.getTime() + 7 * DAY, now.getTime())) - if (start.getTime() >= end.getTime()) continue + const window = reshareWindow(reposter, target, now) + if (window === null) continue repostKeys.add(k) - posts.push({ - id: randomUUID(), - author: reposter, - kind: "repost", - body: null, - replyTo: null, - threadRoot: null, - repostOf: target, - eventId: null, - reportId: null, - createdAt: randTimestamp(start, end), - depth: 1, - likeCount: 0, - replyCount: 0, - repostCount: 0, - saveCount: 0, - mentions: [], - }) + posts.push( + seedPost({ + author: reposter, + kind: "repost", + body: null, + repostOf: target, + createdAt: randTimestamp(window.start, window.end), + depth: 1, + }), + ) target.repostCount++ // pure reposts only, matching the live repost() path } for (let i = 0; i < nQuotes && pool.length > 0; i++) { const quoter = pick(pool) if (quoter.id === target.author.id) continue - const start = later(quoter.createdAt, target.createdAt) - const end = new Date(Math.min(target.createdAt.getTime() + 7 * DAY, now.getTime())) - if (start.getTime() >= end.getTime()) continue - posts.push({ - id: randomUUID(), - author: quoter, - kind: "quote", - body: fill(bilingual(quoter, QUOTE_EN, QUOTE_ES), quoter), - replyTo: null, - threadRoot: null, - repostOf: target, - eventId: null, - reportId: null, - createdAt: randTimestamp(start, end), - depth: 1, - likeCount: 0, - replyCount: 0, - repostCount: 0, - saveCount: 0, - mentions: [], - }) + const window = reshareWindow(quoter, target, now) + if (window === null) continue + posts.push( + seedPost({ + author: quoter, + kind: "quote", + body: fill(bilingual(quoter, QUOTE_EN, QUOTE_ES), quoter), + repostOf: target, + createdAt: randTimestamp(window.start, window.end), + depth: 1, + }), + ) // Quotes do NOT bump repost_count (createPost has no bump for kind='quote'). } } +} +function makePosts( + users: SeedUser[], + events: SeedEvent[], + reports: SeedReport[], + follows: SeedFollow[], + now: Date, +): SeedPost[] { + const posts: SeedPost[] = [] + const followersOf = followersIndex(users, follows) + const addTop: AddTopPost = (author, body, createdAt, eventId, reportId) => { + const p = seedPost({ author, kind: "post", body, createdAt, depth: 0, eventId, reportId }) + posts.push(p) + return p + } + + addAuthorPosts(users, now, addTop) + addEventPosts(events, now, addTop) + addReportPosts(reports, now, addTop) + + const topLevel = posts.filter((p) => p.depth === 0) + for (const root of topLevel) addThreadReplies(posts, root, users, followersOf, now) + addRepostsAndQuotes(posts, topLevel, followersOf, now) return posts } @@ -1914,13 +1082,7 @@ function makeLikesAndSaves( follows: SeedFollow[], now: Date, ): { likes: SeedLike[]; saves: SeedLike[] } { - const byId = new Map(users.map((u) => [u.id, u])) - const followersOf = new Map() - for (const f of follows) { - const arr = followersOf.get(f.followeeId) ?? [] - arr.push(byId.get(f.followerId)!) - followersOf.set(f.followeeId, arr) - } + const followersOf = followersIndex(users, follows) const likes: SeedLike[] = [] const saves: SeedLike[] = [] const likeKeys = new Set() @@ -1935,7 +1097,7 @@ function makeLikesAndSaves( (3 + reach * 0.7) * (p.eventId || p.reportId ? 1.4 : 1) * (p.depth === 0 ? 1 : 0.35) - const n = Math.min(Math.floor(base), 60) + const n = Math.min(Math.floor(base), MAX_LIKES_PER_POST) for (let i = 0; i < n; i++) { const liker = followerPool.length > 0 && chance(0.7) @@ -1945,18 +1107,15 @@ function makeLikesAndSaves( const k = `${p.id}:${liker.id}` if (likeKeys.has(k)) continue const start = later(liker.createdAt, p.createdAt) - const end = new Date(Math.min(p.createdAt.getTime() + 14 * DAY, now.getTime())) + const end = new Date(Math.min(p.createdAt.getTime() + LIKE_WINDOW_DAYS * DAY, now.getTime())) if (start.getTime() >= end.getTime()) continue likeKeys.add(k) likes.push({ postId: p.id, userId: liker.id, createdAt: randTimestamp(start, end) }) p.likeCount++ - if (chance(0.06)) { - const sk = `${p.id}:${liker.id}` - if (!saveKeys.has(sk)) { - saveKeys.add(sk) - saves.push({ postId: p.id, userId: liker.id, createdAt: randTimestamp(start, end) }) - p.saveCount++ - } + if (chance(0.06) && !saveKeys.has(k)) { + saveKeys.add(k) + saves.push({ postId: p.id, userId: liker.id, createdAt: randTimestamp(start, end) }) + p.saveCount++ } } } @@ -1967,11 +1126,11 @@ function makeHours(events: SeedEvent[], now: Date): SeedHours[] { const rows: SeedHours[] = [] for (const ev of events) { if (ev.status !== "done") continue - const loggedAt = minutesAfter(ev.scheduledAt, 4 * 60, 48 * 60) + const loggedAt = minutesAfter(ev.scheduledAt, 4 * HOUR_MINUTES, 48 * HOUR_MINUTES) if (loggedAt.getTime() >= now.getTime()) continue for (const m of ev.members) { if (m.role !== "organizer" && !chance(0.85)) continue // a few no-shows never get credited - const hours = (pick([1.5, 2, 2, 2.5, 2.5, 3, 3, 3.5, 4]) as number).toFixed(2) + const hours = pick(CREDITED_HOUR_CHOICES).toFixed(2) rows.push({ userId: m.user.id, cleanupId: ev.id, @@ -1985,23 +1144,11 @@ function makeHours(events: SeedEvent[], now: Date): SeedHours[] { return rows } -const HANDLE_RE = /^[a-zA-Z0-9_]{3,20}$/ - -function validate( - users: SeedUser[], - follows: SeedFollow[], - events: SeedEvent[], - reports: SeedReport[], - posts: SeedPost[], - likes: SeedLike[], - saves: SeedLike[], -): void { - const errors: string[] = [] - const ids = new Set(users.map((u) => u.id)) +function validateUsers(users: SeedUser[], errors: string[]): void { const handles = new Set() const emails = new Set() for (const u of users) { - if (!HANDLE_RE.test(u.handle)) errors.push(`bad handle: ${u.handle}`) + if (!HANDLE_REGEX.test(u.handle)) errors.push(`bad handle: ${u.handle}`) if (handles.has(u.handle.toLowerCase())) errors.push(`dup handle: ${u.handle}`) handles.add(u.handle.toLowerCase()) if (emails.has(u.email)) errors.push(`dup email: ${u.email}`) @@ -2009,6 +1156,10 @@ function validate( if (!u.email.endsWith(`@${DEMO_EMAIL_DOMAIN}`)) errors.push(`email outside demo domain: ${u.email}`) } +} + +function validateFollows(users: SeedUser[], follows: SeedFollow[], errors: string[]): void { + const ids = new Set(users.map((u) => u.id)) const followerCounts = new Map() const followingCounts = new Map() const edgeKeys = new Set() @@ -2027,16 +1178,28 @@ function validate( if ((followingCounts.get(u.id) ?? 0) !== u.followingCount) errors.push(`followingCount drift for @${u.handle}`) } - const likeAgg = new Map() - for (const l of likes) likeAgg.set(l.postId, (likeAgg.get(l.postId) ?? 0) + 1) - const saveAgg = new Map() - for (const s of saves) saveAgg.set(s.postId, (saveAgg.get(s.postId) ?? 0) + 1) +} + +function countBy(rows: readonly T[], keyOf: (row: T) => string): Map { + const counts = new Map() + for (const row of rows) counts.set(keyOf(row), (counts.get(keyOf(row)) ?? 0) + 1) + return counts +} + +function validatePosts( + posts: SeedPost[], + likes: SeedLike[], + saves: SeedLike[], + errors: string[], +): void { + const likeAgg = countBy(likes, (l) => l.postId) + const saveAgg = countBy(saves, (s) => s.postId) const replyAgg = new Map() const repostAgg = new Map() const repostKeys = new Set() for (const p of posts) { - if (p.body && p.body.includes("\u2014")) - errors.push(`em dash in post body: ${p.body.slice(0, 40)}`) + if (p.body && p.body.includes(EM_DASH)) + errors.push(`em dash in post body: ${p.body.slice(0, ERROR_BODY_PREVIEW_CHARS)}`) if (p.kind === "reply") { if (!p.replyTo || !p.threadRoot) errors.push("reply missing parent/root") else { @@ -2060,6 +1223,9 @@ function validate( if ((replyAgg.get(p.id) ?? 0) !== p.replyCount) errors.push("replyCount drift") if ((repostAgg.get(p.id) ?? 0) !== p.repostCount) errors.push("repostCount drift") } +} + +function validateEvents(events: SeedEvent[], errors: string[]): void { for (const ev of events) { const seen = new Set() for (const m of ev.members) { @@ -2079,17 +1245,37 @@ function validate( errors.push(`slot over capacity: ${s.title}`) } } +} + +function validateReports(reports: SeedReport[], errors: string[]): void { for (const r of reports) { for (let i = 1; i < r.timeline.length; i++) { if (r.timeline[i]!.createdAt.getTime() < r.timeline[i - 1]!.createdAt.getTime()) errors.push("timeline out of order") } - if (r.description.includes("\u2014") || r.title.includes("\u2014")) + if (r.description.includes(EM_DASH) || r.title.includes(EM_DASH)) errors.push("em dash in report") } +} + +function validate( + users: SeedUser[], + follows: SeedFollow[], + events: SeedEvent[], + reports: SeedReport[], + posts: SeedPost[], + likes: SeedLike[], + saves: SeedLike[], +): void { + const errors: string[] = [] + validateUsers(users, errors) + validateFollows(users, follows, errors) + validatePosts(posts, likes, saves, errors) + validateEvents(events, errors) + validateReports(reports, errors) if (errors.length > 0) { throw new Error( - `seed validation failed (${errors.length}):\n ${[...new Set(errors)].slice(0, 25).join("\n ")}`, + `seed validation failed (${errors.length}):\n ${[...new Set(errors)].slice(0, VALIDATION_ERROR_SAMPLE).join("\n ")}`, ) } } @@ -2100,25 +1286,21 @@ function chunk(arr: T[], size: number): T[][] { return out } -export async function writeAll( - tx: TransactionSql, - data: { - now: Date - hashFor: (seatId: string) => string - users: SeedUser[] - follows: SeedFollow[] - events: SeedEvent[] - reports: SeedReport[] - posts: SeedPost[] - likes: SeedLike[] - saves: SeedLike[] - hours: SeedHours[] - }, -): Promise { - const { now, users, follows, events, reports, posts, likes, saves, hours } = data - const resolver = tx as unknown as Sql +export interface SeedData { + now: Date + hashFor: (seatId: string) => string + users: SeedUser[] + follows: SeedFollow[] + events: SeedEvent[] + reports: SeedReport[] + posts: SeedPost[] + likes: SeedLike[] + saves: SeedLike[] + hours: SeedHours[] +} - for (const rows of chunk(users, 200)) { +async function writeUsers(tx: TransactionSql, users: SeedUser[]): Promise { + for (const rows of chunk(users, INSERT_CHUNK.users)) { await tx`INSERT INTO users ${tx( rows.map((u) => ({ id: u.id, @@ -2143,7 +1325,7 @@ export async function writeAll( await tx`UPDATE users SET social_links = ${tx.json({ instagram: u.instagram })} WHERE id = ${u.id}` } } - for (const rows of chunk(users, 300)) { + for (const rows of chunk(users, INSERT_CHUNK.notificationPrefs)) { await tx`INSERT INTO notification_prefs ${tx( rows.map((u) => ({ user_id: u.id, @@ -2156,8 +1338,10 @@ export async function writeAll( })), )}` } +} - for (const rows of chunk(follows, 500)) { +async function writeFollows(tx: TransactionSql, follows: SeedFollow[]): Promise { + for (const rows of chunk(follows, INSERT_CHUNK.follows)) { await tx`INSERT INTO follows_people ${tx( rows.map((f) => ({ follower_id: f.followerId, @@ -2166,10 +1350,17 @@ export async function writeAll( })), )}` } +} +/** Also stamps each event's resolved jurisdiction onto its volunteer-hours rows. */ +async function writeEvents( + tx: TransactionSql, + events: SeedEvent[], + hours: SeedHours[], +): Promise { // Per row, with the reference-code counter allocated first as in createCleanupTx (lock order). for (const ev of events) { - const jur = await resolveJurisdiction(resolver, ev.lng, ev.lat) + const jur = await resolveJurisdiction(tx, ev.lng, ev.lat) const jurCode = await resolveJurisdictionCode(tx, jur?.geoid ?? null) const referenceCode = await allocateEventReferenceCode(tx, jurCode) await tx` @@ -2194,6 +1385,14 @@ export async function writeAll( }) for (const h of hours) if (h.cleanupId === ev.id) h.jurisdictionGeoid = jur?.geoid ?? null } +} + +async function writeEventRosters( + tx: TransactionSql, + events: SeedEvent[], + now: Date, + hashFor: (seatId: string) => string, +): Promise { for (const ev of events) { await tx`INSERT INTO cleanup_members ${tx( ev.members.map((m) => ({ @@ -2222,7 +1421,7 @@ export async function writeAll( await mintDemoSignupSeats(tx, { cleanupId: ev.id, members: ev.members.map((m) => ({ user_id: m.user.id, joined_at: m.joinedAt })), - hashFor: data.hashFor, + hashFor, }) } if (ev.claims.length > 0) { @@ -2236,10 +1435,12 @@ export async function writeAll( )}` } } +} +async function writeReports(tx: TransactionSql, reports: SeedReport[]): Promise { // Per row, with the reference-code counter allocated first (lock order). for (const r of reports) { - const jur = await resolveJurisdiction(resolver, r.lng, r.lat) + const jur = await resolveJurisdiction(tx, r.lng, r.lat) const jurCode = await resolveJurisdictionCode(tx, jur?.geoid ?? null) const referenceCode = await allocateReportReferenceCode(tx, r.type, jurCode) await tx` @@ -2267,10 +1468,16 @@ export async function writeAll( created_at: t.createdAt, })), ) - for (const rows of chunk(timelineRows, 500)) { + for (const rows of chunk(timelineRows, INSERT_CHUNK.timeline)) { await tx`INSERT INTO report_timeline ${tx(rows)}` } +} +async function writeReportLinks( + tx: TransactionSql, + events: SeedEvent[], + reports: SeedReport[], +): Promise { const linkRows: { cleanup_id: string report_id: string @@ -2298,7 +1505,9 @@ export async function writeAll( } } if (linkRows.length > 0) await tx`INSERT INTO cleanup_reports ${tx(linkRows)}` +} +async function writePosts(tx: TransactionSql, posts: SeedPost[]): Promise { // Dependency waves: parents before replies, targets before reposts and quotes. const waves = new Map() for (const p of posts) { @@ -2307,7 +1516,7 @@ export async function writeAll( waves.set(p.depth, arr) } for (const depth of [...waves.keys()].sort((a, b) => a - b)) { - for (const rows of chunk(waves.get(depth)!, 300)) { + for (const rows of chunk(waves.get(depth)!, INSERT_CHUNK.posts)) { await tx`INSERT INTO posts ${tx( rows.map((p) => ({ id: p.id, @@ -2333,7 +1542,7 @@ export async function writeAll( const linkedPostIds = posts .filter((p) => p.reportId !== null || p.eventId !== null) .map((p) => p.id) - for (const ids of chunk(linkedPostIds, 500)) { + for (const ids of chunk(linkedPostIds, INSERT_CHUNK.postGeom)) { await tx` UPDATE posts p SET geom = COALESCE( (SELECT r.geom FROM reports r WHERE r.id = p.report_id), @@ -2345,21 +1554,30 @@ export async function writeAll( const mentionRows = posts.flatMap((p) => p.mentions.map((m) => ({ post_id: p.id, mentioned_user_id: m })), ) - for (const rows of chunk(mentionRows, 500)) { + for (const rows of chunk(mentionRows, INSERT_CHUNK.mentions)) { await tx`INSERT INTO post_mentions ${tx(rows)}` } - for (const rows of chunk(likes, 800)) { +} + +async function writeEngagement( + tx: TransactionSql, + likes: SeedLike[], + saves: SeedLike[], +): Promise { + for (const rows of chunk(likes, INSERT_CHUNK.engagement)) { await tx`INSERT INTO post_likes ${tx( rows.map((l) => ({ post_id: l.postId, user_id: l.userId, created_at: l.createdAt })), )}` } - for (const rows of chunk(saves, 800)) { + for (const rows of chunk(saves, INSERT_CHUNK.engagement)) { await tx`INSERT INTO post_saves ${tx( rows.map((s) => ({ post_id: s.postId, user_id: s.userId, created_at: s.createdAt })), )}` } +} - for (const rows of chunk(hours, 300)) { +async function writeHours(tx: TransactionSql, hours: SeedHours[]): Promise { + for (const rows of chunk(hours, INSERT_CHUNK.hours)) { await tx`INSERT INTO volunteer_hours ${tx( rows.map((h) => ({ user_id: h.userId, @@ -2395,7 +1613,7 @@ export async function writeAll( jurisdiction_geoid: r.geoid, total_hours: r.total.toFixed(2), })) - for (const rows of chunk(rollupRows, 500)) { + for (const rows of chunk(rollupRows, INSERT_CHUNK.rollups)) { await tx` INSERT INTO user_jurisdiction_hours ${tx(rows)} ON CONFLICT (user_id, jurisdiction_geoid) @@ -2404,12 +1622,23 @@ export async function writeAll( } } +export async function writeAll(tx: TransactionSql, data: SeedData): Promise { + await writeUsers(tx, data.users) + await writeFollows(tx, data.follows) + await writeEvents(tx, data.events, data.hours) + await writeEventRosters(tx, data.events, data.now, data.hashFor) + await writeReports(tx, data.reports) + await writeReportLinks(tx, data.events, data.reports) + await writePosts(tx, data.posts) + await writeEngagement(tx, data.likes, data.saves) + await writeHours(tx, data.hours) +} + // Runs inside the seed transaction, so a mismatch rolls the whole seed back. async function verify(tx: TransactionSql, now: Date): Promise { const lines: string[] = [] const fail: string[] = [] - const demoEmail = "%@" + DEMO_EMAIL_DOMAIN const checks: { label: string; rows: Promise<{ n: number | string }[]> }[] = [ { label: "demo members of open events hold a registration", @@ -2417,7 +1646,7 @@ async function verify(tx: TransactionSql, now: Date): Promise { SELECT count(*)::int AS n FROM cleanup_members m JOIN cleanups c ON c.id = m.cleanup_id JOIN users u ON u.id = m.user_id - WHERE u.email LIKE ${demoEmail} AND c.status <> 'cancelled' AND c.ends_at > ${now} + WHERE u.email LIKE ${DEMO_EMAIL_PATTERN} AND c.status <> 'cancelled' AND c.ends_at > ${now} AND NOT EXISTS (SELECT 1 FROM cleanup_ticket_types t WHERE t.cleanup_id = c.id) AND NOT EXISTS ( SELECT 1 FROM cleanup_registrations r @@ -2428,7 +1657,7 @@ async function verify(tx: TransactionSql, now: Date): Promise { label: "demo posts linked to a report or event carry its point", rows: tx` SELECT count(*)::int AS n FROM posts p JOIN users u ON u.id = p.author_id - WHERE u.email LIKE ${demoEmail} AND p.geom IS NULL + WHERE u.email LIKE ${DEMO_EMAIL_PATTERN} AND p.geom IS NULL AND (p.report_id IS NOT NULL OR p.event_id IS NOT NULL)`, }, { @@ -2496,7 +1725,7 @@ const PURGE_BLOCKER_SAMPLE = 20 // at demo events and reports with no ON DELETE: purging would fail on the FK anyway, so name the rows // the operator has to decide about instead. Real content is never rewritten here. async function assertNothingRealDependsOnDemo(tx: TransactionSql): Promise { - const demo = tx`SELECT id FROM users WHERE email LIKE ${"%@" + DEMO_EMAIL_DOMAIN}` + const demo = tx`SELECT id FROM users WHERE email LIKE ${DEMO_EMAIL_PATTERN}` const blockers = await tx<{ kind: string; id: string }[]>` WITH demo_posts AS (SELECT id FROM posts WHERE author_id IN (${demo})) SELECT 'post' AS kind, p.id::text AS id @@ -2555,7 +1784,7 @@ export async function purgeDemo(tx: TransactionSql): Promise) => { counts[label] = (await q).length } - const demo = tx`SELECT id FROM users WHERE email LIKE ${"%@" + DEMO_EMAIL_DOMAIN}` + const demo = tx`SELECT id FROM users WHERE email LIKE ${DEMO_EMAIL_PATTERN}` await assertNothingRealDependsOnDemo(tx) const touchedUsers = await tx<{ id: string }[]>` SELECT DISTINCT x.id @@ -2619,7 +1848,7 @@ export async function purgeDemo(tx: TransactionSql): Promise= 0 ? process.argv[idx + 1] : undefined +async function runPurge(sql: Sql, commit: boolean): Promise { + const ROLLBACK = Symbol("rollback") + const result = await sql + .begin(async (tx) => { + const counts = await purgeDemo(tx) + if (!commit) throw ROLLBACK + return counts + }) + .catch((e: unknown) => { + if (e === ROLLBACK) return null + throw e + }) + if (result) { + console.log("purged:", result) + } else { + console.log("purge rehearsal complete, rolled back. Pass --yes to commit.") + } +} + +function generateCohort( + userCount: number, + now: Date, + hashFor: (seatId: string) => string, +): SeedData { + const start = new Date(now.getTime() - COHORT_HISTORY_DAYS * DAY) + const users = makeUsers(userCount, start, new Date(now.getTime() - NEWEST_ACCOUNT_AGE_DAYS * DAY)) + const follows = makeFollows(users, now) + const events = makeEvents(users, now) + const reports = makeReports(users, Math.round(userCount * REPORTS_PER_USER), now) + const posts = makePosts(users, events, reports, follows, now) + const { likes, saves } = makeLikesAndSaves(users, posts, follows, now) + const hours = makeHours(events, now) + validate(users, follows, events, reports, posts, likes, saves) + return { now, hashFor, users, follows, events, reports, posts, likes, saves, hours } +} + +function logCohort(data: SeedData): void { + const { users, follows, events, reports, posts, likes, saves, hours } = data + const memberRows = events.reduce((n, e) => n + e.members.length, 0) + const claimRows = events.reduce((n, e) => n + e.claims.length, 0) + const postsOfKind = (kind: SeedPost["kind"]) => posts.filter((p) => p.kind === kind).length + console.log( + [ + ` users: ${users.length} (hispanic ~${users.filter((u) => u.hispanic).length})`, + ` follows: ${follows.length}`, + ` events: ${events.length} members: ${memberRows} slot claims: ${claimRows}`, + ` reports: ${reports.length} timeline rows: ${reports.reduce((n, r) => n + r.timeline.length, 0)}`, + ` posts: ${postsOfKind("post")} top-level, ${postsOfKind("reply")} replies, ` + + `${postsOfKind("repost")} reposts, ${postsOfKind("quote")} quotes`, + ` likes: ${likes.length} saves: ${saves.length} volunteer hour rows: ${hours.length}`, + ].join("\n"), + ) +} + +async function runSeed( + sql: Sql, + opts: { commit: boolean; userCount: number; prngSeed: number }, +): Promise { + const hashFor = demoTicketTokenHasher() + const now = new Date() + + console.log(`generating cohort (seed ${opts.prngSeed})...`) + const data = generateCohort(opts.userCount, now, hashFor) + logCohort(data) + + const ROLLBACK = Symbol("rollback") + const verification = await sql + .begin(async (tx) => { + const [existing] = await tx<{ n: number }[]>` + SELECT count(*)::int AS n FROM users WHERE email LIKE ${DEMO_EMAIL_PATTERN} + ` + if (Number(existing?.n ?? 0) > 0) { + throw new Error( + `found ${existing!.n} existing demo users (@${DEMO_EMAIL_DOMAIN}); run --purge --yes first`, + ) + } + console.log("writing...") + await writeAll(tx, data) + console.log("verifying...") + const lines = await verify(tx, now) + if (!opts.commit) throw ROLLBACK + return lines + }) + .catch((e: unknown) => { + if (e === ROLLBACK) return "rolledback" as const + throw e + }) + + if (verification === "rolledback") { + console.log("rehearsal complete: all inserts + verification passed, transaction rolled back.") + console.log("re-run with --yes to commit.") + } else { + for (const l of verification) console.log(l) + console.log("committed.") + } } -export async function main(): Promise { +async function main(): Promise { const commit = process.argv.includes("--yes") const purgeMode = process.argv.includes("--purge") - const userCount = Number(argValue("--users") ?? 250) - const prngSeed = Number(argValue("--seed") ?? 20260902) - rand = mulberry32(prngSeed) + const userCount = Number(argValue("--users") ?? DEFAULT_USER_COUNT) + const prngSeed = Number(argValue("--seed") ?? DEMO_PRNG_SEED) + seedDemoRandom(prngSeed) - const databaseUrl = process.env.DATABASE_URL - if (!databaseUrl) { - throw new Error("DATABASE_URL is required") - } - const host = new URL(databaseUrl).host - console.log(`target database: ${host}`) + const databaseUrl = requireDatabaseUrl() + console.log(`target database: ${new URL(databaseUrl).host}`) console.log( commit ? "mode: COMMIT" : "mode: rehearsal (full run + verification, then ROLLBACK; pass --yes to commit)", ) - const handle = makeDb(databaseUrl, { max: 1, statementTimeoutMs: 0, idleInTxTimeoutMs: 0 }) - const ROLLBACK = Symbol("rollback") - try { - if (purgeMode) { - const result = await handle.sql - .begin(async (tx) => { - const counts = await purgeDemo(tx) - if (!commit) throw ROLLBACK - return counts - }) - .catch((e: unknown) => { - if (e === ROLLBACK) return null - throw e - }) - if (result) { - console.log("purged:", result) - } else { - console.log("purge rehearsal complete, rolled back. Pass --yes to commit.") - } - return - } - - const hashFor = demoTicketTokenHasher() - const now = new Date() - const start = new Date(now.getTime() - 185 * DAY) - - console.log(`generating cohort (seed ${prngSeed})...`) - const users = makeUsers(userCount, start, new Date(now.getTime() - 2 * DAY)) - const follows = makeFollows(users, now) - const events = makeEvents(users, now) - const reports = makeReports(users, Math.round(userCount * 0.62), now) - const posts = makePosts(users, events, reports, follows, now) - const { likes, saves } = makeLikesAndSaves(users, posts, follows, now) - const hours = makeHours(events, now) - validate(users, follows, events, reports, posts, likes, saves) - - const memberRows = events.reduce((n, e) => n + e.members.length, 0) - const claimRows = events.reduce((n, e) => n + e.claims.length, 0) - console.log( - [ - ` users: ${users.length} (hispanic ~${users.filter((u) => u.hispanic).length})`, - ` follows: ${follows.length}`, - ` events: ${events.length} members: ${memberRows} slot claims: ${claimRows}`, - ` reports: ${reports.length} timeline rows: ${reports.reduce((n, r) => n + r.timeline.length, 0)}`, - ` posts: ${posts.filter((p) => p.kind === "post").length} top-level, ${posts.filter((p) => p.kind === "reply").length} replies, ` + - `${posts.filter((p) => p.kind === "repost").length} reposts, ${posts.filter((p) => p.kind === "quote").length} quotes`, - ` likes: ${likes.length} saves: ${saves.length} volunteer hour rows: ${hours.length}`, - ].join("\n"), - ) - - const verification = await handle.sql - .begin(async (tx) => { - const [existing] = await tx<{ n: number }[]>` - SELECT count(*)::int AS n FROM users WHERE email LIKE ${"%@" + DEMO_EMAIL_DOMAIN} - ` - if (Number(existing?.n ?? 0) > 0) { - throw new Error( - `found ${existing!.n} existing demo users (@${DEMO_EMAIL_DOMAIN}); run --purge --yes first`, - ) - } - console.log("writing...") - await writeAll(tx, { - now, - hashFor, - users, - follows, - events, - reports, - posts, - likes, - saves, - hours, - }) - console.log("verifying...") - const lines = await verify(tx, now) - if (!commit) throw ROLLBACK - return lines - }) - .catch((e: unknown) => { - if (e === ROLLBACK) return "rolledback" as const - throw e - }) - - if (verification === "rolledback") { - console.log("rehearsal complete: all inserts + verification passed, transaction rolled back.") - console.log("re-run with --yes to commit.") - } else { - for (const l of verification) console.log(l) - console.log("committed.") - } - } finally { - await handle.close() - } + await runDbCli( + async (_db, sql) => { + if (purgeMode) await runPurge(sql, commit) + else await runSeed(sql, { commit, userCount, prngSeed }) + }, + { databaseUrl }, + ) } runIfMain(import.meta.url, "seed-demo-la", main) diff --git a/services/api/src/db/seed-fixtures.ts b/services/api/src/db/seed-fixtures.ts index 4f569d2f..ceafeabd 100644 --- a/services/api/src/db/seed-fixtures.ts +++ b/services/api/src/db/seed-fixtures.ts @@ -82,9 +82,3 @@ export const PROBE_OUTSIDE_ALL: ProbePoint = { lat: 40.0, expectGeoid: null, } - -export const PROBE_POINTS: readonly ProbePoint[] = [ - PROBE_INSIDE_CITY, - PROBE_COUNTY_NOT_CITY, - PROBE_OUTSIDE_ALL, -] diff --git a/services/api/src/db/seed.ts b/services/api/src/db/seed.ts index a23beee7..41b18445 100644 --- a/services/api/src/db/seed.ts +++ b/services/api/src/db/seed.ts @@ -2,6 +2,7 @@ import type { Sql } from "./client.js" import { runDbCli, runIfMain } from "./cli.js" import { JURISDICTION_SEEDS } from "./seed-fixtures.js" import { FEDERAL_LANDS, federalLandGeoJson } from "./data/federal-lands.js" +import { LAYER_RANK } from "./ingest-jurisdictions-core.js" export async function seedJurisdictions(sql: Sql): Promise { let inserted = 0 @@ -28,11 +29,11 @@ export async function seedJurisdictions(sql: Sql): Promise { return inserted } -export async function seedFederalLands(sql: Sql): Promise { +async function seedFederalLands(sql: Sql): Promise { let inserted = 0 for (const land of FEDERAL_LANDS) { const geojson = federalLandGeoJson(land.bbox) - const priority = land.layer === "tribal" ? -1 : -2 + const priority = LAYER_RANK[land.layer] const rows = await sql` INSERT INTO jurisdictions (geoid, name, layer, priority, geom, population, contact_emails, report_form_url, code) VALUES ( diff --git a/services/api/src/db/sql/jurisdiction.ts b/services/api/src/db/sql/jurisdiction.ts index 2fc1d409..9bb0d662 100644 --- a/services/api/src/db/sql/jurisdiction.ts +++ b/services/api/src/db/sql/jurisdiction.ts @@ -8,7 +8,7 @@ * curating it changes routing. */ -import type { Sql } from "../client.js" +import type { Queryable } from "../client.js" import type { JURISDICTION_LAYER_VALUES } from "../schema/types.js" export interface ResolvedJurisdiction { @@ -42,7 +42,7 @@ ORDER BY ${JURISDICTION_RESOLVE_ORDER_BY} LIMIT 1` as const export async function resolveJurisdiction( - sql: Sql, + sql: Queryable, lng: number, lat: number, ): Promise { diff --git a/services/api/src/di.ts b/services/api/src/di.ts index 3b429aed..ba6b2c20 100644 --- a/services/api/src/di.ts +++ b/services/api/src/di.ts @@ -70,7 +70,7 @@ import { type PostRepository, } from "./services/post-repository.drizzle.js" import { makePostService, type PostService } from "./services/post-service.js" -import { makeFeedPresence, type FeedPresence } from "./services/feed-presence.js" +import { makeFeedPresence } from "./services/feed-presence.js" import { makeNotificationService, type NotificationService, @@ -82,7 +82,7 @@ import { RedisByteMeter, type ByteMeter } from "./services/media-byte-quota.js" import { makeTicketTokenSigner, type TicketTokenSigner } from "./services/host/ticket-token.js" import { RedisCounterStore, type CounterStore } from "./abuse/counter-store.js" import { InMemoryBlocksRepository, InMemoryDmRepository } from "./services/dm-repository.memory.js" -import { MultiPushSender } from "./adapters/push-sender.js" +import { MultiPushSender, type PushSenderConfig } from "./adapters/push-sender.js" import { HttpRoutingProvider } from "./adapters/routing-provider.js" import { RealAbuseChecks } from "./adapters/abuse-checks.js" import { PgBossJobs } from "./adapters/jobs.pgboss.js" @@ -147,406 +147,391 @@ const PRE_SERVER_LOGGER: AdapterLogger = { error: (obj, msg) => console.error(msg ?? "", obj), } -export function buildContainer(env: Env): Container { - let dbHandle: DbHandle | undefined - let redis: RedisClient | undefined - - let serverLogger: NotificationLogger | undefined - - const csrf = makeCsrf(env) +interface Lazy { + get(): T + readonly current: T | undefined + reset(): void +} - // Adapters are built before buildServer hands over its logger, so they get a forwarder that resolves - // the server logger at call time instead of capturing a console fallback at construction. - const adapterLogger: AdapterLogger = { - warn: (obj, msg) => forwardLog("warn", obj, msg), - error: (obj, msg) => forwardLog("error", obj, msg), +function lazy(create: () => T): Lazy { + let value: T | undefined + return { + get: () => (value ??= create()), + get current() { + return value + }, + reset: () => { + value = undefined + }, } - function forwardLog(level: "warn" | "error", obj: unknown, msg: string | undefined): void { - if (serverLogger !== undefined) serverLogger[level](obj, msg) +} + +// Adapters are built before buildServer hands over its logger, so they get a forwarder that resolves +// the server logger at call time instead of capturing a console fallback at construction. +function forwardingLogger(resolve: () => NotificationLogger | undefined): AdapterLogger { + const forward = (level: "warn" | "error", obj: unknown, msg: string | undefined): void => { + const target = resolve() + if (target !== undefined) target[level](obj, msg) else PRE_SERVER_LOGGER[level](obj, msg) } + return { + warn: (obj, msg) => forward("warn", obj, msg), + error: (obj, msg) => forward("error", obj, msg), + } +} + +interface Connections { + getDb(): DbHandle + getRedis(): RedisClient + readonly dbHandle: DbHandle | undefined + readonly redis: RedisClient | undefined + markClosed(): void + closeRedis(): Promise + closeDb(): Promise +} +function makeConnections(env: Env, logger: () => NotificationLogger | undefined): Connections { + let dbHandle: DbHandle | undefined + let redis: RedisClient | undefined let closed = false // A getter reached after shutdown would otherwise open a fresh pool that nothing ever closes. function assertOpen(): void { if (closed) throw new Error("DI container is closed") } - - function getDb(): DbHandle { - assertOpen() - if (!dbHandle) dbHandle = makeDb(env.DATABASE_URL) - return dbHandle - } - function getRedis(): RedisClient { - assertOpen() - if (!redis) { - redis = makeRedis(env.REDIS_URL, { - onError: (err) => serverLogger?.error({ err, component: "redis" }, "redis client error"), + return { + getDb() { + assertOpen() + dbHandle ??= makeDb(env.DATABASE_URL) + return dbHandle + }, + getRedis() { + assertOpen() + redis ??= makeRedis(env.REDIS_URL, { + onError: (err) => logger()?.error({ err, component: "redis" }, "redis client error"), }) - } - return redis - } - - let dmRepo: DmRepository | undefined - let blocksRepo: BlocksRepository | undefined - let volunteerHoursRepo: VolunteerHoursRepository | undefined - function getVolunteerHoursRepo(): VolunteerHoursRepository { - if (!volunteerHoursRepo) { - volunteerHoursRepo = makeDrizzleVolunteerHoursRepository(getDb().sql) - } - return volunteerHoursRepo - } - let certificateRepo: CertificateRepository | undefined - function getCertificateRepo(): CertificateRepository { - if (!certificateRepo) { - certificateRepo = makeDrizzleCertificateRepository(getDb().sql) - } - return certificateRepo - } - const hasDatabase = env.DATABASE_URL.length > 0 - function getBlocksRepo(): BlocksRepository { - if (!blocksRepo) { - blocksRepo = hasDatabase - ? makeDrizzleBlocksRepository(getDb().sql) - : new InMemoryBlocksRepository() - } - return blocksRepo - } - function getDmRepo(): DmRepository { - if (!dmRepo) { - if (hasDatabase) { - dmRepo = makeDrizzleDmRepository(getDb().sql, presignPrivateMedia) - } else { - const blocks = getBlocksRepo() - dmRepo = new InMemoryDmRepository((a, b) => blocks.isBlockedEitherWay(a, b)) - } - } - return dmRepo + return redis + }, + get dbHandle() { + return dbHandle + }, + get redis() { + return redis + }, + markClosed() { + closed = true + }, + async closeRedis() { + if (!redis) return false + await redis.quit().catch(() => redis?.disconnect()) + redis = undefined + return true + }, + async closeDb() { + if (!dbHandle) return + await dbHandle.close() + dbHandle = undefined + }, } +} - let postRepo: PostRepository | undefined - let affiliationLoader: AffiliationLoader | undefined - function getAffiliationLoader(): AffiliationLoader { - if (!affiliationLoader) { - affiliationLoader = makeAffiliationLoader(getDb().sql, (k: string) => - storage.presignGet(k, MEDIA_GET_URL_TTL_SEC), - ) - } - return affiliationLoader - } +interface StorageSeams { + storage: Storage + inboundStorage: Storage + localObjectStores: readonly LocalDiskStorage[] | undefined +} - function getPostRepo(): PostRepository { - if (!postRepo) { - postRepo = makeDrizzlePostRepository(getDb().sql, { - presignMedia, - presignAvatar: (k: string) => storage.presignGet(k, MEDIA_GET_URL_TTL_SEC), - affiliations: getAffiliationLoader(), - }) - } - return postRepo +function makeStorageSeams(env: Env): StorageSeams { + const localStorageDir = env.LOCAL_STORAGE_DIR + if (localStorageDir !== undefined) { + const media = makeLocalDiskStorage(env, localStorageDir, "media") + const inbound = makeLocalDiskStorage(env, localStorageDir, "inbound") + return { storage: media, inboundStorage: inbound, localObjectStores: [media, inbound] } } - - let feedPresence: FeedPresence | undefined - function getFeedPresence(): FeedPresence { - if (!feedPresence) { - feedPresence = makeFeedPresence({ - ...(env.REDIS_URL.length > 0 ? { cache: getCache() } : {}), - config: env.FEED_RANKING, - ...(serverLogger !== undefined ? { logger: serverLogger } : {}), - }) - } - return feedPresence + if (env.USE_FAKE_STORAGE) { + const storage = new FakeStorage() + return { storage, inboundStorage: storage, localObjectStores: undefined } } - let postService: PostService | undefined - function getPostService(): PostService { - if (!postService) { - postService = makePostService({ - repo: getPostRepo(), - sql: getDb().sql, - notifier: getNotificationService(), - isBlockedEitherWay: (a: string, b: string) => getBlocksRepo().isBlockedEitherWay(a, b), - feedRanking: env.FEED_RANKING, - feedPresence: getFeedPresence(), - ...(env.USE_FAKE_USER_CHANNEL ? {} : { userChannel: getUserChannel() }), - }) - } - return postService + const storage = new R2Storage({ + ...r2Credentials(env), + bucket: env.R2_BUCKET, + ...(env.R2_PUBLIC_BASE !== undefined ? { publicBase: env.R2_PUBLIC_BASE } : {}), + }) + const inboundBucket = + env.R2_INBOUND_BUCKET ?? (env.R2_PUBLIC_BASE === undefined ? env.R2_BUCKET : "") + if (inboundBucket.length === 0) { + throw new Error( + "R2_INBOUND_BUCKET is required when R2_PUBLIC_BASE is set: refusing to write raw inbound email " + + "into the public media bucket. Set a dedicated, non-public inbound bucket.", + ) } + const inboundStorage = new R2Storage({ ...r2Credentials(env), bucket: inboundBucket }) + return { storage, inboundStorage, localObjectStores: undefined } +} - let notificationService: NotificationService | undefined - let notificationLoggerWired = false - function getNotificationService(logger?: NotificationLogger): NotificationService { - if (logger !== undefined) serverLogger ??= logger - if (notificationService === undefined || (logger !== undefined && !notificationLoggerWired)) { - notificationService = makeNotificationService({ - repo: makeDrizzleNotificationRepository(getDb().sql), - pushSender: getPushSender(), - userChannel: getUserChannel(), - ...(logger !== undefined ? { logger } : {}), - }) - notificationLoggerWired = logger !== undefined - } - return notificationService +function r2Credentials(env: Env): { + accountId: string + accessKeyId: string + secretAccessKey: string +} { + return { + accountId: env.R2_ACCOUNT_ID, + accessKeyId: env.R2_ACCESS_KEY_ID, + secretAccessKey: env.R2_SECRET_ACCESS_KEY, } +} - let redisCounters: CounterStore | undefined - const lazyCounters: CounterStore = { - incr: (key, ttlSeconds) => - (redisCounters ??= new RedisCounterStore(getRedis())).incr(key, ttlSeconds), - incrBy: (key, by, ttlSeconds) => - (redisCounters ??= new RedisCounterStore(getRedis())).incrBy(key, by, ttlSeconds), - decrBy: (key, by) => (redisCounters ??= new RedisCounterStore(getRedis())).decrBy(key, by), - } - function getCounterStore(): CounterStore { - return lazyCounters - } +function makeLocalDiskStorage( + env: Env, + rootDirectory: string, + namespace: LocalStorageNamespace, +): LocalDiskStorage { + return new LocalDiskStorage({ + rootDirectory, + namespace, + publicApiUrl: env.PUBLIC_API_URL, + signingKey: env.LOCAL_STORAGE_SIGNING_KEY ?? LOCAL_STORAGE_DEV_SIGNING_KEY, + nodeEnv: env.NODE_ENV, + }) +} - let cacheClient: CacheClient | undefined - function getCache(): CacheClient { - if (!cacheClient) cacheClient = new RedisCacheClient(getRedis()) - return cacheClient - } +interface StatelessSeams { + mailer: Mailer + smsSender: SmsSender + geocoder: Geocoder + streetReverseGeocode: ReverseGeocode + jurisdictionLookup: JurisdictionLookup + inboundMail: InboundMail + routingProvider: RoutingProvider + abuseChecks: AbuseChecks +} - let redisByteMeter: ByteMeter | undefined - const lazyByteMeter: ByteMeter = { - add: (subject, bytes) => - (redisByteMeter ??= new RedisByteMeter(getRedis())).add(subject, bytes), - } - function getByteMeter(): ByteMeter { - return lazyByteMeter +function makeStatelessSeams( + env: Env, + logger: AdapterLogger, + getDb: () => DbHandle, +): StatelessSeams { + const isProduction = env.NODE_ENV === "production" + return { + mailer: env.USE_FAKE_MAILER + ? new FakeMailer() + : new OciMailer({ + host: env.OCI_EMAIL_SMTP_HOST, + port: env.OCI_EMAIL_SMTP_PORT, + user: env.OCI_EMAIL_SMTP_USER, + pass: env.OCI_EMAIL_SMTP_PASS, + fromNoReply: env.MAIL_FROM_NOREPLY, + fromOutreach: env.MAIL_FROM_OUTREACH, + timeoutMs: env.OCI_EMAIL_SMTP_TIMEOUT_MS, + logger, + }), + smsSender: env.USE_FAKE_SMS + ? new FakeSmsSender() + : new TwilioSmsSender({ + accountSid: env.TWILIO_ACCOUNT_SID, + authToken: env.TWILIO_AUTH_TOKEN, + from: env.TWILIO_SMS_FROM, + }), + geocoder: env.USE_FAKE_GEOCODER + ? new FakeGeocoder() + : new TigerGeocoder({ getSql: () => getDb().sql }), + streetReverseGeocode: chainReverse( + env.MAPBOX_TOKEN ? makeMapboxReverseGeocode({ token: env.MAPBOX_TOKEN }) : null, + makePhotonReverseGeocode(), + ), + jurisdictionLookup: isProduction + ? new CensusJurisdictionLookup({ + baseUrl: env.CENSUS_GEOCODER_URL, + timeoutMs: env.CENSUS_GEOCODER_TIMEOUT_MS, + }) + : new FakeJurisdictionLookup(), + inboundMail: isProduction + ? new CfInboundMail({ replyDomain: env.MAIL_REPLY_DOMAIN }) + : new FakeInboundMail(env.MAIL_REPLY_DOMAIN), + routingProvider: isProduction ? new HttpRoutingProvider() : new FakeRoutingProvider(), + abuseChecks: env.USE_FAKE_ABUSE_NSFW + ? new FakeAbuseChecks() + : new RealAbuseChecks({ + ...(env.CF_TURNSTILE_SECRET !== undefined + ? { turnstileSecret: env.CF_TURNSTILE_SECRET } + : {}), + ...(env.CF_TURNSTILE_HOSTNAMES.length > 0 + ? { turnstileHostnames: env.CF_TURNSTILE_HOSTNAMES } + : {}), + useRealNsfw: env.USE_REAL_NSFW, + log: (line, extra) => logger.warn(extra ?? {}, line), + }), } +} - let ticketTokenSigner: TicketTokenSigner | undefined - function getTicketTokenSigner(): TicketTokenSigner { - if (!ticketTokenSigner) { - ticketTokenSigner = makeTicketTokenSigner(env.TICKET_TOKEN_SECRET.trim()) - } - return ticketTokenSigner - } +async function closeIfClosable(seam: unknown): Promise { + const closable = seam as { close?: () => Promise } + if (typeof closable.close === "function") await closable.close() +} - const localStorageDir = env.LOCAL_STORAGE_DIR - const usesR2 = localStorageDir === undefined && !env.USE_FAKE_STORAGE - - function makeLocalDiskStorage( - rootDirectory: string, - namespace: LocalStorageNamespace, - ): LocalDiskStorage { - return new LocalDiskStorage({ - rootDirectory, - namespace, - publicApiUrl: env.PUBLIC_API_URL, - signingKey: env.LOCAL_STORAGE_SIGNING_KEY ?? LOCAL_STORAGE_DEV_SIGNING_KEY, - nodeEnv: env.NODE_ENV, - }) - } +export function buildContainer(env: Env): Container { + let serverLogger: NotificationLogger | undefined + const currentLogger = (): NotificationLogger | undefined => serverLogger + const adapterLogger = forwardingLogger(currentLogger) + const connections = makeConnections(env, currentLogger) + const { getDb, getRedis } = connections - const localMediaStore = - localStorageDir !== undefined ? makeLocalDiskStorage(localStorageDir, "media") : undefined - const localInboundStore = - localStorageDir !== undefined ? makeLocalDiskStorage(localStorageDir, "inbound") : undefined - const localObjectStores = - localMediaStore !== undefined && localInboundStore !== undefined - ? ([localMediaStore, localInboundStore] as const) - : undefined - - const storage: Storage = - localMediaStore ?? - (env.USE_FAKE_STORAGE - ? new FakeStorage() - : new R2Storage({ - accountId: env.R2_ACCOUNT_ID, - accessKeyId: env.R2_ACCESS_KEY_ID, - secretAccessKey: env.R2_SECRET_ACCESS_KEY, - bucket: env.R2_BUCKET, - ...(env.R2_PUBLIC_BASE !== undefined ? { publicBase: env.R2_PUBLIC_BASE } : {}), - })) + const csrf = makeCsrf(env) + const hasDatabase = env.DATABASE_URL.length > 0 + const { storage, inboundStorage, localObjectStores } = makeStorageSeams(env) const presignMedia = makeMediaPresigner(storage) - const presignPrivateMedia = makePrivateMediaPresigner(storage) + const presignAvatar = (k: string) => storage.presignGet(k, MEDIA_GET_URL_TTL_SEC) - const inboundBucket = - env.R2_INBOUND_BUCKET ?? (env.R2_PUBLIC_BASE === undefined ? env.R2_BUCKET : "") - if (usesR2 && inboundBucket.length === 0) { - throw new Error( - "R2_INBOUND_BUCKET is required when R2_PUBLIC_BASE is set: refusing to write raw inbound email " + - "into the public media bucket. Set a dedicated, non-public inbound bucket.", - ) - } - const inboundStorage: Storage = - localInboundStore ?? - (usesR2 - ? new R2Storage({ - accountId: env.R2_ACCOUNT_ID, - accessKeyId: env.R2_ACCESS_KEY_ID, - secretAccessKey: env.R2_SECRET_ACCESS_KEY, - bucket: inboundBucket, - }) - : storage) - - const mailer: Mailer = env.USE_FAKE_MAILER - ? new FakeMailer() - : new OciMailer({ - host: env.OCI_EMAIL_SMTP_HOST, - port: env.OCI_EMAIL_SMTP_PORT, - user: env.OCI_EMAIL_SMTP_USER, - pass: env.OCI_EMAIL_SMTP_PASS, - fromNoReply: env.MAIL_FROM_NOREPLY, - fromOutreach: env.MAIL_FROM_OUTREACH, - timeoutMs: env.OCI_EMAIL_SMTP_TIMEOUT_MS, - logger: adapterLogger, - }) + const seams = makeStatelessSeams(env, adapterLogger, getDb) - const smsSender: SmsSender = env.USE_FAKE_SMS - ? new FakeSmsSender() - : new TwilioSmsSender({ - accountSid: env.TWILIO_ACCOUNT_SID, - authToken: env.TWILIO_AUTH_TOKEN, - from: env.TWILIO_SMS_FROM, - }) + const volunteerHoursRepo = lazy(() => makeDrizzleVolunteerHoursRepository(getDb().sql)) + const certificateRepo = lazy(() => makeDrizzleCertificateRepository(getDb().sql)) + const blocksRepo = lazy(() => + hasDatabase ? makeDrizzleBlocksRepository(getDb().sql) : new InMemoryBlocksRepository(), + ) + const dmRepo = lazy(() => { + if (hasDatabase) return makeDrizzleDmRepository(getDb().sql, presignPrivateMedia) + const blocks = blocksRepo.get() + return new InMemoryDmRepository((a, b) => blocks.isBlockedEitherWay(a, b)) + }) + const affiliationLoader = lazy(() => makeAffiliationLoader(getDb().sql, presignAvatar)) + const postRepo = lazy(() => + makeDrizzlePostRepository(getDb().sql, { + presignMedia, + presignAvatar, + affiliations: affiliationLoader.get(), + }), + ) - const geocoder: Geocoder = env.USE_FAKE_GEOCODER - ? new FakeGeocoder() - : new TigerGeocoder({ getSql: () => getDb().sql }) + const cacheClient = lazy(() => new RedisCacheClient(getRedis())) + const redisCounters = lazy(() => new RedisCounterStore(getRedis())) + const lazyCounters: CounterStore = { + incr: (key, ttlSeconds) => redisCounters.get().incr(key, ttlSeconds), + incrBy: (key, by, ttlSeconds) => redisCounters.get().incrBy(key, by, ttlSeconds), + decrBy: (key, by) => redisCounters.get().decrBy(key, by), + } + const redisByteMeter = lazy(() => new RedisByteMeter(getRedis())) + const lazyByteMeter: ByteMeter = { + add: (subject, bytes) => redisByteMeter.get().add(subject, bytes), + } + const ticketTokenSigner = lazy(() => makeTicketTokenSigner(env.TICKET_TOKEN_SECRET.trim())) - const streetReverseGeocode: ReverseGeocode = chainReverse( - env.MAPBOX_TOKEN ? makeMapboxReverseGeocode({ token: env.MAPBOX_TOKEN }) : null, - makePhotonReverseGeocode(), + const feedPresence = lazy(() => + makeFeedPresence({ + ...(env.REDIS_URL.length > 0 ? { cache: cacheClient.get() } : {}), + config: env.FEED_RANKING, + ...(serverLogger !== undefined ? { logger: serverLogger } : {}), + }), ) - const jurisdictionLookup: JurisdictionLookup = - env.NODE_ENV === "production" - ? new CensusJurisdictionLookup({ - baseUrl: env.CENSUS_GEOCODER_URL, - timeoutMs: env.CENSUS_GEOCODER_TIMEOUT_MS, - }) - : new FakeJurisdictionLookup() - - const inboundMail: InboundMail = - env.NODE_ENV === "production" - ? new CfInboundMail({ - replyDomain: env.MAIL_REPLY_DOMAIN, - ...(env.CF_EMAIL_WEBHOOK_SECRET !== undefined - ? { webhookSecret: env.CF_EMAIL_WEBHOOK_SECRET } - : {}), - }) - : new FakeInboundMail(env.MAIL_REPLY_DOMAIN) - - const routingProvider: RoutingProvider = - env.NODE_ENV === "production" ? new HttpRoutingProvider() : new FakeRoutingProvider() - - const abuseChecks: AbuseChecks = env.USE_FAKE_ABUSE_NSFW - ? new FakeAbuseChecks() - : new RealAbuseChecks({ - ...(env.CF_TURNSTILE_SECRET !== undefined - ? { turnstileSecret: env.CF_TURNSTILE_SECRET } - : {}), - ...(env.CF_TURNSTILE_HOSTNAMES.length > 0 - ? { turnstileHostnames: env.CF_TURNSTILE_HOSTNAMES } - : {}), - useRealNsfw: env.USE_REAL_NSFW, - log: (line, extra) => adapterLogger.warn(extra ?? {}, line), - }) - - let sharedPubSub: RedisChatPubSub | undefined - function getSharedPubSub(): RedisChatPubSub { - if (!sharedPubSub) { - sharedPubSub = new RedisChatPubSub(getRedis(), (err) => + const sharedPubSub = lazy( + () => + new RedisChatPubSub(getRedis(), (err) => serverLogger?.error( { err, component: "redis", role: "subscriber" }, "redis subscriber error", ), - ) + ), + ) + const userChannel = lazy(() => + env.USE_FAKE_USER_CHANNEL + ? new FakeUserChannel() + : new RedisUserChannel({ + pubsub: sharedPubSub.get(), + ...(serverLogger !== undefined ? { logger: serverLogger } : {}), + }), + ) + const pushSender = lazy(() => + env.USE_FAKE_PUSH + ? new FakePushSender() + : new MultiPushSender({ + db: getDb().db, + config: buildPushConfig(env), + counters: lazyCounters, + logger: adapterLogger, + }), + ) + + let notificationService: NotificationService | undefined + let notificationLoggerWired = false + // Rebuilt once when the first logger arrives, so a service built earlier without one does not keep + // dropping delivery warnings. + function getNotificationService(logger?: NotificationLogger): NotificationService { + if (logger !== undefined) serverLogger ??= logger + if (notificationService === undefined || (logger !== undefined && !notificationLoggerWired)) { + notificationService = makeNotificationService({ + repo: makeDrizzleNotificationRepository(getDb().sql), + pushSender: pushSender.get(), + userChannel: userChannel.get(), + ...(logger !== undefined ? { logger } : {}), + }) + notificationLoggerWired = logger !== undefined } - return sharedPubSub + return notificationService } + const postService = lazy(() => + makePostService({ + repo: postRepo.get(), + sql: getDb().sql, + notifier: getNotificationService(), + isBlockedEitherWay: (a: string, b: string) => blocksRepo.get().isBlockedEitherWay(a, b), + feedRanking: env.FEED_RANKING, + feedPresence: feedPresence.get(), + ...(env.USE_FAKE_USER_CHANNEL ? {} : { userChannel: userChannel.get() }), + }), + ) + const chatService: ChatService = env.USE_FAKE_CHAT ? new FakeChatService() : new WsChatService({ repo: makeDrizzleChatRepository(getDb().sql, presignPrivateMedia), - pubsub: getSharedPubSub(), + pubsub: sharedPubSub.get(), + logger: adapterLogger, }) - let userChannel: UserChannel | undefined - function getUserChannel(): UserChannel { - if (!userChannel) { - userChannel = env.USE_FAKE_USER_CHANNEL - ? new FakeUserChannel() - : new RedisUserChannel({ - pubsub: getSharedPubSub(), - ...(serverLogger !== undefined ? { logger: serverLogger } : {}), - }) - } - return userChannel - } - - let pushSender: PushSender | undefined - function getPushSender(): PushSender { - if (!pushSender) { - pushSender = env.USE_FAKE_PUSH - ? new FakePushSender() - : new MultiPushSender({ - db: getDb().db, - config: buildPushConfig(env), - counters: getCounterStore(), - logger: adapterLogger, - }) - } - return pushSender - } - const jobs: Jobs = env.USE_FAKE_JOBS ? new FakeJobs() : new PgBossJobs({ connectionString: env.DATABASE_URL, logger: adapterLogger }) + const memoized: ReadonlyArray> = [ + dmRepo, + blocksRepo, + volunteerHoursRepo, + certificateRepo, + postRepo, + affiliationLoader, + postService, + userChannel, + pushSender, + ] + async function close(): Promise { const maybePgBoss = jobs as { stop?: () => Promise } if (typeof maybePgBoss.stop === "function") { await maybePgBoss.stop() } - if (userChannel) { - const maybeUserChannel = userChannel as { close?: () => Promise } - if (typeof maybeUserChannel.close === "function") await maybeUserChannel.close() - } - const maybeChat = chatService as { close?: () => Promise } - if (typeof maybeChat.close === "function") { - await maybeChat.close() - } - if (pushSender) { - const maybePush = pushSender as { close?: () => Promise } - if (typeof maybePush.close === "function") await maybePush.close() - } + if (userChannel.current) await closeIfClosable(userChannel.current) + await closeIfClosable(chatService) + if (pushSender.current) await closeIfClosable(pushSender.current) // Only now: a graceful jobs stop waits for running handlers, which still need the pools, and // whatever they opened is torn down below. - closed = true - if (sharedPubSub) { - await sharedPubSub.close() - sharedPubSub = undefined - } - if (redis) { - await redis.quit().catch(() => redis?.disconnect()) - redis = undefined - redisCounters = undefined - cacheClient = undefined - feedPresence = undefined - redisByteMeter = undefined + connections.markClosed() + if (sharedPubSub.current) { + await sharedPubSub.current.close() + sharedPubSub.reset() } - if (dbHandle) { - await dbHandle.close() - dbHandle = undefined + if (await connections.closeRedis()) { + redisCounters.reset() + cacheClient.reset() + feedPresence.reset() + redisByteMeter.reset() } - dmRepo = undefined - blocksRepo = undefined - volunteerHoursRepo = undefined - certificateRepo = undefined - postRepo = undefined - affiliationLoader = undefined - postService = undefined + await connections.closeDb() + for (const memo of memoized) memo.reset() notificationService = undefined - userChannel = undefined - pushSender = undefined } return { @@ -555,44 +540,37 @@ export function buildContainer(env: Env): Container { storage, inboundStorage, developmentOnlyLocalObjectStores: localObjectStores, - mailer, - smsSender, - inboundMail, - geocoder, - streetReverseGeocode, - jurisdictionLookup, + ...seams, chatService, get userChannel() { - return getUserChannel() + return userChannel.get() }, get pushSender() { - return getPushSender() + return pushSender.get() }, - routingProvider, - abuseChecks, jobs, get dbHandle() { - return dbHandle + return connections.dbHandle }, get redis() { - return redis + return connections.redis }, - usesRealDb: env.DATABASE_URL.length > 0, + usesRealDb: hasDatabase, usesRealRedis: env.REDIS_URL.length > 0, getDb, getRedis, - getDmRepo, - getBlocksRepo, - getVolunteerHoursRepo, - getCertificateRepo, - getAffiliationLoader, - getPostRepo, - getPostService, + getDmRepo: dmRepo.get, + getBlocksRepo: blocksRepo.get, + getVolunteerHoursRepo: volunteerHoursRepo.get, + getCertificateRepo: certificateRepo.get, + getAffiliationLoader: affiliationLoader.get, + getPostRepo: postRepo.get, + getPostService: postService.get, getNotificationService, - getCounterStore, - getCache, - getByteMeter, - getTicketTokenSigner, + getCounterStore: () => lazyCounters, + getCache: cacheClient.get, + getByteMeter: () => lazyByteMeter, + getTicketTokenSigner: ticketTokenSigner.get, close, } } @@ -615,8 +593,8 @@ export async function assertRedisReachable(container: Container): Promise } } -function buildPushConfig(env: Env) { - const config: import("./adapters/push-sender.js").PushSenderConfig = {} +function buildPushConfig(env: Env): PushSenderConfig { + const config: PushSenderConfig = {} if (env.APNS_KEY_ID && env.APNS_TEAM_ID && env.APNS_PRIVATE_KEY && env.APNS_BUNDLE_ID) { config.apns = { keyId: env.APNS_KEY_ID, diff --git a/services/api/src/env.ts b/services/api/src/env.ts index d203b23d..5a2d31aa 100644 --- a/services/api/src/env.ts +++ b/services/api/src/env.ts @@ -1,80 +1,72 @@ import { z } from "zod" -import { DEFAULT_FEED_RANKING, FeedRankingConfigSchema } from "@civfix/shared" -import type { FeedRankingConfig } from "@civfix/shared" import type { Env } from "./env/types.js" -import { assertOutboundSendPolicy } from "./services/admin/outbound-send-policy.js" import { loadCommsEnv } from "./env/comms-env.js" import { loadRegistrationEnv } from "./env/registration-env.js" +import { makeEnvReader, type EnvReader } from "./env/reader.js" import { - isCronish, - parseBool, - parseBounds, - parseCsv, - parseCsvLower, - parseDrainMs, - parseIntOr, - parsePositiveIntOr, - parseStrictBool, - STRICT_BOOL_ACCEPTED_FORMS, - parseTrustProxy, -} from "./env/parsers.js" + checkDatabaseTls, + checkGeocoderDatabase, + loadAccessLists, + loadApnsProduction, + loadCoreEnv, + loadFeedRanking, + loadHomeRegion, + loadLocalStorage, + loadMailEnv, + loadR2Env, + loadScheduleEnv, + loadSigningKeys, + loadSmsEnv, + loadTrustProxy, + loadTuningEnv, + type FakeFlags, +} from "./env/core-env.js" +import { parseBool, parseBounds } from "./env/parsers.js" export type { Env } from "./env/types.js" -export type { TrustProxyValue } from "./env/parsers.js" -export { - parseBool, - parseCsv, - parseCsvLower, - parseDrainMs, - parseIntOr, - parsePositiveIntOr, - parseBounds, - parseTrustProxy, - isCronish, - DEFAULT_TRUSTED_PROXY_CIDRS, - SHUTDOWN_DRAIN_MS_MAX, -} from "./env/parsers.js" - -const DEV_SESSION_SIGNING_KEY = "dev-insecure-session-signing-key-do-not-use-in-prod" -const DEV_ANON_TOKEN_SIGNING_KEY = "dev-insecure-anon-token-signing-key-do-not-use-in-prod" +export { DEFAULT_TRUSTED_PROXY_CIDRS, SHUTDOWN_DRAIN_MS_MAX } from "./env/parsers.js" export const REVIEWER_OTP_CODE_MIN_LENGTH = 20 -export const SIGNING_KEY_MIN_LENGTH = 32 - -const TLS_SSLMODES = new Set(["require", "verify-ca", "verify-full"]) - -const TILES_MIN_ZOOM_DEFAULT = 1 -const TILES_MAX_ZOOM_DEFAULT = 19 const TILES_BOUNDS_DEFAULT: [number, number, number, number] = [-125, 24, -66, 50] -const HOME_REGION_LAT_DEFAULT = 34.0522 -const HOME_REGION_LNG_DEFAULT = -118.2437 -const HOME_REGION_RADIUS_KM_DEFAULT = 40 - -const APNS_CREDENTIAL_KEYS = [ +const NODE_ENVS = ["development", "test", "production"] as const +const NodeEnvSchema = z.enum(NODE_ENVS).default("development") + +const OPTIONAL_STRING_KEYS: ReadonlyArray = [ + "R2_INBOUND_BUCKET", + "R2_PUBLIC_BASE", + "TILES_RASTER_URL", + "CF_ACCESS_TEAM_DOMAIN", + "CF_ACCESS_AUD", + "CF_TURNSTILE_SECRET", + "CF_EMAIL_WEBHOOK_SECRET", + "CF_API_TOKEN", + "MAPBOX_TOKEN", + "APPLE_OAUTH_CLIENT_ID", + "APPLE_OAUTH_TEAM_ID", + "APPLE_OAUTH_KEY_ID", + "APPLE_OAUTH_PRIVATE_KEY", + "APPLE_OAUTH_WEB_CLIENT_ID", + "APPLE_OAUTH_IOS_CLIENT_ID", + "GOOGLE_OAUTH_CLIENT_ID", + "GOOGLE_OAUTH_CLIENT_SECRET", + "GOOGLE_OAUTH_REDIRECT_URI", + "GOOGLE_OAUTH_IOS_CLIENT_ID", + "GOOGLE_OAUTH_ANDROID_CLIENT_ID", "APNS_KEY_ID", "APNS_TEAM_ID", "APNS_PRIVATE_KEY", "APNS_BUNDLE_ID", -] as const - -const NodeEnvSchema = z.enum(["development", "test", "production"]).default("development") -const PortSchema = z.coerce.number().int().positive().max(65535) - -type FakeFlags = Pick< - Env, - | "USE_FAKE_STORAGE" - | "USE_FAKE_MAILER" - | "USE_FAKE_PUSH" - | "USE_FAKE_ABUSE_NSFW" - | "USE_FAKE_CHAT" - | "USE_FAKE_JOBS" - | "USE_FAKE_USER_CHANNEL" - | "USE_FAKE_GEOCODER" - | "USE_FAKE_SMS" - | "USE_REAL_NSFW" -> + "FCM_SERVICE_ACCOUNT_JSON", + "FCM_PROJECT_ID", + "VAPID_PUBLIC_KEY", + "VAPID_PRIVATE_KEY", + "VAPID_SUBJECT", + "EXPO_ACCESS_TOKEN", + "GLITCHTIP_DSN", + "GLITCHTIP_DATABASE_URL", +] export const FAKE_SEAM_FLAGS: ReadonlyArray<{ flag: keyof FakeFlags; consequence: string }> = [ { flag: "USE_FAKE_STORAGE", consequence: "uploaded media is kept in memory and lost on restart" }, @@ -118,13 +110,13 @@ export function loadEnv(source: NodeJS.ProcessEnv = process.env): Env { const nodeEnvParsed = NodeEnvSchema.safeParse(source.NODE_ENV) if (!nodeEnvParsed.success) { - errors.push("NODE_ENV: must be one of development | test | production") + errors.push(`NODE_ENV: must be one of ${NODE_ENVS.join(" | ")}`) } const nodeEnv = nodeEnvParsed.success ? nodeEnvParsed.data : "development" const isProd = nodeEnv === "production" + const r = makeEnvReader(source, errors, isProd) const fakeFlags = deriveFakeFlags(source, isProd) - if (isProd) { for (const { flag, consequence } of FAKE_SEAM_FLAGS) { if (fakeFlags[flag]) { @@ -133,295 +125,21 @@ export function loadEnv(source: NodeJS.ProcessEnv = process.env): Env { } } - function reqStr(key: string, opts: { gatedOff?: boolean } = {}): string { - const raw = source[key] - const value = typeof raw === "string" ? raw.trim() : "" - const required = isProd && !(opts.gatedOff ?? false) - if (value.length === 0 && required) { - errors.push(`${key}: required [BOOT] variable is missing`) - } - return value - } - - function reqPort(key: string, fallback: number): number { - const raw = source[key] - if (raw === undefined || raw === "") return fallback - const parsed = PortSchema.safeParse(raw) - if (!parsed.success) { - errors.push(`${key}: must be an integer between 1 and 65535`) - return fallback - } - return parsed.data - } - - function reqNumber( - key: string, - fallback: number, - range: { min: number; max: number; exclusiveMin?: boolean }, - ): number { - const raw = source[key] - if (raw === undefined || raw.trim() === "") return fallback - - const exclusiveMin = range.exclusiveMin ?? false - const value = Number.parseFloat(raw.trim()) - const aboveMin = exclusiveMin ? value > range.min : value >= range.min - if (Number.isFinite(value) && aboveMin && value <= range.max) return value - - const lowerBound = exclusiveMin ? `greater than ${range.min}` : `at least ${range.min}` - errors.push(`${key}: must be a number ${lowerBound} and at most ${range.max}`) - return fallback - } - - function reqFeedRanking(key: string): FeedRankingConfig { - const raw = (source[key] ?? "").trim() - if (raw.length === 0) return DEFAULT_FEED_RANKING - let decoded: unknown - try { - decoded = JSON.parse(raw) - } catch { - errors.push(`${key}: must be a JSON object of feed-ranking overrides`) - return DEFAULT_FEED_RANKING - } - const parsed = FeedRankingConfigSchema.safeParse(decoded) - if (!parsed.success) { - for (const issue of parsed.error.issues) { - const path = issue.path.length > 0 ? `.${issue.path.join(".")}` : "" - errors.push(`${key}${path}: ${issue.message}`) - } - return DEFAULT_FEED_RANKING - } - return parsed.data - } - - function positiveInt(key: string, fallback: number): number { - return parsePositiveIntOr(source[key], fallback, { key, errors }) - } - - function readApnsProduction(): boolean | undefined { - const raw = (source.APNS_PRODUCTION ?? "").trim() - if (raw.length === 0) { - const apnsConfigured = APNS_CREDENTIAL_KEYS.every( - (key) => (source[key] ?? "").trim().length > 0, - ) - if (isProd && apnsConfigured) { - errors.push( - "APNS_PRODUCTION: required [BOOT] once APNs credentials are set (true for App Store and " + - "TestFlight builds; a gateway mismatch makes APNs reject every token as BadDeviceToken)", - ) - } - return undefined - } - const value = parseStrictBool(raw) - if (value === undefined) { - errors.push(`APNS_PRODUCTION: must be one of ${STRICT_BOOL_ACCEPTED_FORMS}`) - } - return value - } - - function reqCron(key: string, fallback: string): string { - const value = (source[key] ?? "").trim() || fallback - if (!isCronish(value)) { - errors.push(`${key}: must be a 5- or 6-field cron expression`) - } - return value - } - - const PORT = reqPort("PORT", 8080) - const PUBLIC_API_URL = reqStr("PUBLIC_API_URL") - const WEB_ORIGINS = parseCsv(source.WEB_ORIGINS) - if (isProd && WEB_ORIGINS.length === 0) { - errors.push("WEB_ORIGINS: required [BOOT] CORS allowlist (comma list) is missing") - } - const DATABASE_URL = reqStr("DATABASE_URL") - const REDIS_URL = reqStr("REDIS_URL") - - let SESSION_SIGNING_KEY = (source.SESSION_SIGNING_KEY ?? "").trim() - let ANON_TOKEN_SIGNING_KEY = (source.ANON_TOKEN_SIGNING_KEY ?? "").trim() - if (isProd) { - if (SESSION_SIGNING_KEY.length === 0) { - errors.push("SESSION_SIGNING_KEY: required [BOOT] variable is missing") - } else if (SESSION_SIGNING_KEY === DEV_SESSION_SIGNING_KEY) { - errors.push("SESSION_SIGNING_KEY: must not be the insecure dev default in production") - } else if (SESSION_SIGNING_KEY.length < SIGNING_KEY_MIN_LENGTH) { - errors.push( - `SESSION_SIGNING_KEY: must be at least ${SIGNING_KEY_MIN_LENGTH} characters in production ` + - "(it is the cookie-signing and session-bound CSRF HMAC secret)", - ) - } - if (ANON_TOKEN_SIGNING_KEY.length === 0) { - errors.push("ANON_TOKEN_SIGNING_KEY: required [BOOT] variable is missing") - } else if (ANON_TOKEN_SIGNING_KEY === DEV_ANON_TOKEN_SIGNING_KEY) { - errors.push("ANON_TOKEN_SIGNING_KEY: must not be the insecure dev default in production") - } else if (ANON_TOKEN_SIGNING_KEY.length < SIGNING_KEY_MIN_LENGTH) { - errors.push( - `ANON_TOKEN_SIGNING_KEY: must be at least ${SIGNING_KEY_MIN_LENGTH} characters in production ` + - "(it signs the anon-report claim tokens)", - ) - } - if (SESSION_SIGNING_KEY.length > 0 && SESSION_SIGNING_KEY === ANON_TOKEN_SIGNING_KEY) { - errors.push( - "SESSION_SIGNING_KEY / ANON_TOKEN_SIGNING_KEY: must be DIFFERENT values in production " + - "(one shared secret lets a session-cookie oracle and an anon-token oracle attack the same key)", - ) - } - } else { - if (SESSION_SIGNING_KEY.length === 0) SESSION_SIGNING_KEY = DEV_SESSION_SIGNING_KEY - if (ANON_TOKEN_SIGNING_KEY.length === 0) ANON_TOKEN_SIGNING_KEY = DEV_ANON_TOKEN_SIGNING_KEY - } - - if ( - isProd && - DATABASE_URL.length > 0 && - !TLS_SSLMODES.has(sslModeOf(DATABASE_URL) ?? "") && - !isNonRoutableDbHost(DATABASE_URL) - ) { - errors.push( - "DATABASE_URL: production requires TLS; append ?sslmode=require (or verify-ca / verify-full); " + - "postgres.js otherwise connects in cleartext", - ) - } - - const TRUST_PROXY = parseTrustProxy(source.TRUST_PROXY) - if (isProd && TRUST_PROXY === true) { - errors.push( - "TRUST_PROXY: must not be `true` in production (it trusts any client-supplied X-Forwarded-For). " + - "Use an explicit CIDR list; leave unset for the safe internal-ranges default", - ) - } - - const HOME_REGION_LAT = reqNumber("HOME_REGION_LAT", HOME_REGION_LAT_DEFAULT, { - min: -90, - max: 90, - }) - const HOME_REGION_LNG = reqNumber("HOME_REGION_LNG", HOME_REGION_LNG_DEFAULT, { - min: -180, - max: 180, - }) - const HOME_REGION_RADIUS_KM = reqNumber("HOME_REGION_RADIUS_KM", HOME_REGION_RADIUS_KM_DEFAULT, { - min: 0, - max: 20_000, - exclusiveMin: true, - }) - - const FEED_RANKING = reqFeedRanking("FEED_RANKING") - - const LOCAL_STORAGE_DIR = (source.LOCAL_STORAGE_DIR ?? "").trim() - const usesLocalStorage = LOCAL_STORAGE_DIR.length > 0 - if (isProd && usesLocalStorage) { - errors.push( - "LOCAL_STORAGE_DIR: the local-disk storage driver is DEVELOPMENT ONLY and must not be set in " + - "production; configure R2 (R2_ACCOUNT_ID / R2_ACCESS_KEY_ID / R2_SECRET_ACCESS_KEY / R2_BUCKET)", - ) - } - if (usesLocalStorage && PUBLIC_API_URL.length === 0) { - errors.push( - "PUBLIC_API_URL: required whenever LOCAL_STORAGE_DIR is set: the local-disk driver's presigned " + - "URLs must be absolute and reachable from the browser and the media worker", - ) - } - const LOCAL_STORAGE_SIGNING_KEY = (source.LOCAL_STORAGE_SIGNING_KEY ?? "").trim() - - if (!fakeFlags.USE_FAKE_GEOCODER && DATABASE_URL.length === 0) { - errors.push( - "DATABASE_URL: required whenever USE_FAKE_GEOCODER is false: the real geocoder resolves its " + - '"City, ST" label from the jurisdictions PostGIS table, so there is nothing to query without a database', - ) - } - - const R2_ACCOUNT_ID = reqStr("R2_ACCOUNT_ID", { gatedOff: fakeFlags.USE_FAKE_STORAGE }) - const R2_ACCESS_KEY_ID = reqStr("R2_ACCESS_KEY_ID", { gatedOff: fakeFlags.USE_FAKE_STORAGE }) - const R2_SECRET_ACCESS_KEY = reqStr("R2_SECRET_ACCESS_KEY", { - gatedOff: fakeFlags.USE_FAKE_STORAGE, - }) - const R2_BUCKET = reqStr("R2_BUCKET", { gatedOff: fakeFlags.USE_FAKE_STORAGE }) - - const R2_INBOUND_BUCKET = (source.R2_INBOUND_BUCKET ?? "").trim() - const R2_PUBLIC_BASE = (source.R2_PUBLIC_BASE ?? "").trim() - if (!fakeFlags.USE_FAKE_STORAGE && !usesLocalStorage) { - if (R2_PUBLIC_BASE.length > 0 && R2_INBOUND_BUCKET.length === 0) { - errors.push( - "R2_INBOUND_BUCKET: required [BOOT] whenever R2_PUBLIC_BASE is set: a shared bucket would " + - "publish raw inbound email and its attachments on the public CDN", - ) - } - if (R2_INBOUND_BUCKET.length > 0 && R2_INBOUND_BUCKET === R2_BUCKET) { - errors.push( - "R2_INBOUND_BUCKET: must be a DIFFERENT bucket from R2_BUCKET (inbound mail must never live in " + - "the media bucket)", - ) - } - } - - const OCI_EMAIL_SMTP_HOST = reqStr("OCI_EMAIL_SMTP_HOST", { gatedOff: fakeFlags.USE_FAKE_MAILER }) - const OCI_EMAIL_SMTP_PORT = reqPort("OCI_EMAIL_SMTP_PORT", 587) - const OCI_EMAIL_SMTP_USER = reqStr("OCI_EMAIL_SMTP_USER", { gatedOff: fakeFlags.USE_FAKE_MAILER }) - const OCI_EMAIL_SMTP_PASS = reqStr("OCI_EMAIL_SMTP_PASS", { gatedOff: fakeFlags.USE_FAKE_MAILER }) - const OCI_EMAIL_SMTP_TIMEOUT_MS = positiveInt("OCI_EMAIL_SMTP_TIMEOUT_MS", 15_000) - const OUTBOUND_SEND_MIN_THROUGHPUT_BPS = positiveInt( - "OUTBOUND_SEND_MIN_THROUGHPUT_BPS", - 256 * 1024, - ) - for (const problem of assertOutboundSendPolicy({ - smtpTimeoutMs: OCI_EMAIL_SMTP_TIMEOUT_MS, - minThroughputBytesPerSec: OUTBOUND_SEND_MIN_THROUGHPUT_BPS, - })) { - errors.push(problem) - } - - const SMS_GUEST_ENABLED = parseBool(source.SMS_GUEST_ENABLED, false) - const SMS_DAILY_CAP = positiveInt("SMS_DAILY_CAP", 50) - const smsCredentialsUnused = fakeFlags.USE_FAKE_SMS || !SMS_GUEST_ENABLED - const TWILIO_ACCOUNT_SID = reqStr("TWILIO_ACCOUNT_SID", { gatedOff: smsCredentialsUnused }) - const TWILIO_AUTH_TOKEN = reqStr("TWILIO_AUTH_TOKEN", { gatedOff: smsCredentialsUnused }) - const TWILIO_SMS_FROM = reqStr("TWILIO_SMS_FROM", { gatedOff: smsCredentialsUnused }) - - const OUTREACH_DIGEST_CRON = reqCron("OUTREACH_DIGEST_CRON", "0 14 * * *") - const OUTREACH_DIGEST_ENABLED = parseBool(source.OUTREACH_DIGEST_ENABLED, false) - const REPORT_AUTOFORWARD_ENABLED = parseBool(source.REPORT_AUTOFORWARD_ENABLED, false) - const GUEST_RETENTION_CRON = reqCron("GUEST_RETENTION_CRON", "15 4 * * *") - const INBOUND_SWEEP_CRON = reqCron("INBOUND_SWEEP_CRON", "*/5 * * * *") - - const OAUTH_REQUIRE_NONCE = parseBool(source.OAUTH_REQUIRE_NONCE, false) - const WS_ALLOW_QUERY_TOKEN = parseBool(source.WS_ALLOW_QUERY_TOKEN, false) - const REVIEWER_OTP_BYPASS = parseBool(source.REVIEWER_OTP_BYPASS, false) - const REVIEWER_OTP_BYPASS_ACK = parseBool(source.REVIEWER_OTP_BYPASS_ACK, false) - const REVIEWER_OTP_CODE = (source.REVIEWER_OTP_CODE ?? "").trim() - if (REVIEWER_OTP_CODE.length > 0 && REVIEWER_OTP_CODE.length < REVIEWER_OTP_CODE_MIN_LENGTH) { - errors.push( - `REVIEWER_OTP_CODE: must be at least ${REVIEWER_OTP_CODE_MIN_LENGTH} characters ` + - "(it is a login secret, not a 6-digit OTP)", - ) - } - if (isProd && REVIEWER_OTP_BYPASS) { - if (!REVIEWER_OTP_BYPASS_ACK) { - errors.push( - "REVIEWER_OTP_BYPASS: refusing to enable an authentication bypass in production without the " + - "explicit second opt-in REVIEWER_OTP_BYPASS_ACK=true", - ) - } - if (REVIEWER_OTP_CODE.length === 0) { - errors.push( - "REVIEWER_OTP_CODE: required whenever REVIEWER_OTP_BYPASS is on in production (a per-review, " + - `rotated secret of at least ${REVIEWER_OTP_CODE_MIN_LENGTH} characters)`, - ) - } - } - - const SHUTDOWN_DRAIN_MS = parseDrainMs(source.SHUTDOWN_DRAIN_MS) - const TILES_MIN_ZOOM = parseIntOr(source.TILES_MIN_ZOOM, TILES_MIN_ZOOM_DEFAULT, { - key: "TILES_MIN_ZOOM", - errors, - }) - const TILES_MAX_ZOOM = parseIntOr(source.TILES_MAX_ZOOM, TILES_MAX_ZOOM_DEFAULT, { - key: "TILES_MAX_ZOOM", - errors, - }) - const CENSUS_GEOCODER_TIMEOUT_MS = positiveInt("CENSUS_GEOCODER_TIMEOUT_MS", 2500) - const VOLUNTEER_HOURS_WEEKLY_FLAG_HOURS = positiveInt("VOLUNTEER_HOURS_WEEKLY_FLAG_HOURS", 60) - const OUTREACH_THROTTLE_DAYS = positiveInt("OUTREACH_THROTTLE_DAYS", 7) - - const APNS_PRODUCTION = readApnsProduction() - + const core = loadCoreEnv(r) + const signingKeys = loadSigningKeys(r) + checkDatabaseTls(r, core.DATABASE_URL) + const TRUST_PROXY = loadTrustProxy(r) + const homeRegion = loadHomeRegion(r) + const FEED_RANKING = loadFeedRanking(r, "FEED_RANKING") + const localStorage = loadLocalStorage(r, core.PUBLIC_API_URL) + checkGeocoderDatabase(r, fakeFlags, core.DATABASE_URL) + const r2 = loadR2Env(r, fakeFlags.USE_FAKE_STORAGE, localStorage.usesLocalStorage) + const mail = loadMailEnv(r, fakeFlags.USE_FAKE_MAILER) + const sms = loadSmsEnv(r, fakeFlags.USE_FAKE_SMS) + const schedules = loadScheduleEnv(r) + const authFlags = loadAuthFlags(r) + const tuning = loadTuningEnv(r) + const apns = loadApnsProduction(r) const comms = loadCommsEnv(source, errors) const registration = loadRegistrationEnv(source, errors) @@ -432,147 +150,84 @@ export function loadEnv(source: NodeJS.ProcessEnv = process.env): Env { throw new Error([header, ...errors.map((e) => ` - ${e}`)].join("\n")) } - const env: Env = { + return { NODE_ENV: nodeEnv, - PORT, - PUBLIC_API_URL, - WEB_ORIGINS, - DATABASE_URL, - REDIS_URL, - SESSION_SIGNING_KEY, - ANON_TOKEN_SIGNING_KEY, + ...core, + ...signingKeys, TRUST_PROXY, - SHUTDOWN_DRAIN_MS, - - R2_ACCOUNT_ID, - R2_ACCESS_KEY_ID, - R2_SECRET_ACCESS_KEY, - R2_BUCKET, - ...(usesLocalStorage ? { LOCAL_STORAGE_DIR } : {}), - ...(usesLocalStorage && LOCAL_STORAGE_SIGNING_KEY.length > 0 - ? { LOCAL_STORAGE_SIGNING_KEY } - : {}), - TILES_MIN_ZOOM, - TILES_MAX_ZOOM, + ...r2, + ...localStorage.fields, TILES_BOUNDS: parseBounds(source.TILES_BOUNDS, TILES_BOUNDS_DEFAULT), - - HOME_REGION_LAT, - HOME_REGION_LNG, - HOME_REGION_RADIUS_KM, - + ...homeRegion, FEED_RANKING, - - CENSUS_GEOCODER_URL: - (source.CENSUS_GEOCODER_URL ?? "").trim() || - "https://geocoding.geo.census.gov/geocoder/geographies/coordinates", - CENSUS_GEOCODER_TIMEOUT_MS, - - OCI_EMAIL_SMTP_HOST, - OCI_EMAIL_SMTP_PORT, - OCI_EMAIL_SMTP_USER, - OCI_EMAIL_SMTP_PASS, - OCI_EMAIL_SMTP_TIMEOUT_MS, - OUTBOUND_SEND_MIN_THROUGHPUT_BPS, - VOLUNTEER_HOURS_WEEKLY_FLAG_HOURS, - MAIL_FROM_NOREPLY: (source.MAIL_FROM_NOREPLY ?? "").trim() || "no-reply@civfix.org", - MAIL_FROM_OUTREACH: (source.MAIL_FROM_OUTREACH ?? "").trim() || "outreach@civfix.org", - HOME_TURF_MAIL_FROM: (source.HOME_TURF_MAIL_FROM ?? "").trim() || "donotreply@civfix.org", - HOME_TURF_NOTIFY_TO: (source.HOME_TURF_NOTIFY_TO ?? "").trim(), - - ADMIN_EMAILS: parseCsvLower(source.ADMIN_EMAILS), - MAIL_REPLY_DOMAIN: (source.MAIL_REPLY_DOMAIN ?? "").trim() || "civfix.org", - OUTREACH_THROTTLE_DAYS, - OUTREACH_DIGEST_CRON, - OUTREACH_DIGEST_ENABLED, - REPORT_AUTOFORWARD_ENABLED, - INBOUND_SWEEP_CRON, - GUEST_RETENTION_CRON, - - TWILIO_ACCOUNT_SID, - TWILIO_AUTH_TOKEN, - TWILIO_SMS_FROM, - SMS_GUEST_ENABLED, - SMS_DAILY_CAP, - - CF_ACCESS_SERVICE_TOKENS: parseCsv(source.CF_ACCESS_SERVICE_TOKENS), - - CF_TURNSTILE_HOSTNAMES: parseCsvLower(source.CF_TURNSTILE_HOSTNAMES), - - OAUTH_REQUIRE_NONCE, - WS_ALLOW_QUERY_TOKEN, - REVIEWER_OTP_BYPASS, - REVIEWER_OTP_BYPASS_ACK, - ...(REVIEWER_OTP_CODE.length > 0 ? { REVIEWER_OTP_CODE } : {}), - - ...optGroup(source, [ - "R2_INBOUND_BUCKET", - "R2_PUBLIC_BASE", - "TILES_RASTER_URL", - "CF_ACCESS_TEAM_DOMAIN", - "CF_ACCESS_AUD", - "CF_TURNSTILE_SECRET", - "CF_EMAIL_WEBHOOK_SECRET", - "CF_API_TOKEN", - "MAPBOX_TOKEN", - "APPLE_OAUTH_CLIENT_ID", - "APPLE_OAUTH_TEAM_ID", - "APPLE_OAUTH_KEY_ID", - "APPLE_OAUTH_PRIVATE_KEY", - "APPLE_OAUTH_WEB_CLIENT_ID", - "APPLE_OAUTH_IOS_CLIENT_ID", - "GOOGLE_OAUTH_CLIENT_ID", - "GOOGLE_OAUTH_CLIENT_SECRET", - "GOOGLE_OAUTH_REDIRECT_URI", - "GOOGLE_OAUTH_IOS_CLIENT_ID", - "GOOGLE_OAUTH_ANDROID_CLIENT_ID", - "APNS_KEY_ID", - "APNS_TEAM_ID", - "APNS_PRIVATE_KEY", - "APNS_BUNDLE_ID", - "FCM_SERVICE_ACCOUNT_JSON", - "FCM_PROJECT_ID", - "VAPID_PUBLIC_KEY", - "VAPID_PRIVATE_KEY", - "VAPID_SUBJECT", - "EXPO_ACCESS_TOKEN", - "GLITCHTIP_DSN", - "GLITCHTIP_DATABASE_URL", - ]), - ...(APNS_PRODUCTION !== undefined ? { APNS_PRODUCTION } : {}), - + ...mail, + ...sms, + ...schedules, + ...authFlags, + ...tuning, + ...loadAccessLists(r), + ...optionalStrings(source, OPTIONAL_STRING_KEYS), + ...apns, ...fakeFlags, - ...comms, ...registration, } - - return env } -export function sslModeOf(databaseUrl: string): string | undefined { - try { - const value = new URL(databaseUrl).searchParams.get("sslmode") - return value === null ? undefined : value.trim().toLowerCase() - } catch { - return undefined +function loadAuthFlags( + r: EnvReader, +): Pick< + Env, + | "OAUTH_REQUIRE_NONCE" + | "WS_ALLOW_QUERY_TOKEN" + | "REVIEWER_OTP_BYPASS" + | "REVIEWER_OTP_BYPASS_ACK" + | "REVIEWER_OTP_CODE" +> { + const { source } = r + const REVIEWER_OTP_BYPASS = parseBool(source.REVIEWER_OTP_BYPASS, false) + const REVIEWER_OTP_BYPASS_ACK = parseBool(source.REVIEWER_OTP_BYPASS_ACK, false) + const REVIEWER_OTP_CODE = (source.REVIEWER_OTP_CODE ?? "").trim() + checkReviewerBypass(r, { + bypass: REVIEWER_OTP_BYPASS, + acknowledged: REVIEWER_OTP_BYPASS_ACK, + code: REVIEWER_OTP_CODE, + }) + return { + OAUTH_REQUIRE_NONCE: parseBool(source.OAUTH_REQUIRE_NONCE, false), + WS_ALLOW_QUERY_TOKEN: parseBool(source.WS_ALLOW_QUERY_TOKEN, false), + REVIEWER_OTP_BYPASS, + REVIEWER_OTP_BYPASS_ACK, + ...(REVIEWER_OTP_CODE.length > 0 ? { REVIEWER_OTP_CODE } : {}), } } -export function isNonRoutableDbHost(databaseUrl: string): boolean { - let host: string - try { - host = new URL(databaseUrl).hostname.trim().toLowerCase() - } catch { - return false +function checkReviewerBypass( + r: EnvReader, + reviewer: { bypass: boolean; acknowledged: boolean; code: string }, +): void { + if (reviewer.code.length > 0 && reviewer.code.length < REVIEWER_OTP_CODE_MIN_LENGTH) { + r.errors.push( + `REVIEWER_OTP_CODE: must be at least ${REVIEWER_OTP_CODE_MIN_LENGTH} characters ` + + "(it is a login secret, not a 6-digit OTP)", + ) + } + if (!r.isProd || !reviewer.bypass) return + if (!reviewer.acknowledged) { + r.errors.push( + "REVIEWER_OTP_BYPASS: refusing to enable an authentication bypass in production without the " + + "explicit second opt-in REVIEWER_OTP_BYPASS_ACK=true", + ) + } + if (reviewer.code.length === 0) { + r.errors.push( + "REVIEWER_OTP_CODE: required whenever REVIEWER_OTP_BYPASS is on in production (a per-review, " + + `rotated secret of at least ${REVIEWER_OTP_CODE_MIN_LENGTH} characters)`, + ) } - if (host.length === 0) return false - if (host.startsWith("[") && host.endsWith("]")) host = host.slice(1, -1) - if (host === "localhost" || host === "::1") return true - if (/^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host)) return true - return /^[a-z0-9_-]+$/.test(host) } -function optGroup( +function optionalStrings( source: NodeJS.ProcessEnv, keys: ReadonlyArray, ): Partial { @@ -588,30 +243,27 @@ function optGroup( let cached: Env | undefined +function loadedEnv(): Env { + cached ??= loadEnv() + return cached +} + export const env: Env = new Proxy({} as Env, { get(_target, prop: string) { if (prop === "toJSON") return () => "[civfix env: redacted]" - if (cached === undefined) cached = loadEnv() - return cached[prop as keyof Env] + return loadedEnv()[prop as keyof Env] }, has(_target, prop: string) { - if (cached === undefined) cached = loadEnv() - return prop in cached + return prop in loadedEnv() }, ownKeys() { - if (cached === undefined) cached = loadEnv() - return Reflect.ownKeys(cached) + return Reflect.ownKeys(loadedEnv()) }, getOwnPropertyDescriptor(_target, prop: string) { - if (cached === undefined) cached = loadEnv() - return Object.getOwnPropertyDescriptor(cached, prop) + return Object.getOwnPropertyDescriptor(loadedEnv(), prop) }, }) export function isProd(): boolean { return env.NODE_ENV === "production" } - -export function resetEnvCache(): void { - cached = undefined -} diff --git a/services/api/src/env/comms-env.ts b/services/api/src/env/comms-env.ts index 20e19b39..14437ef1 100644 --- a/services/api/src/env/comms-env.ts +++ b/services/api/src/env/comms-env.ts @@ -1,8 +1,41 @@ -import { isCronish, parseBool, parseCsvLower, parseIntOr } from "./parsers.js" +import { parseBool, parseCsvLower, parseIntOr } from "./parsers.js" +import { makeEnvReader, type EnvReader, type EnvSource } from "./reader.js" -export const UNSUBSCRIBE_SIGNING_KEY_MIN_LENGTH = 32 +const UNSUBSCRIBE_SIGNING_KEY_MIN_LENGTH = 32 -export const DEFAULT_MAIL_FROM_EVENTS = "events@civfix.org" +const DEFAULT_MAIL_FROM_EVENTS = "events@civfix.org" + +const PAGE_VIEW_DEDUPE_SEC_MAX = 3600 + +const COMMS_CRON_DEFAULTS = { + BROADCAST_SWEEP_CRON: "*/2 * * * *", + EVENT_REMINDERS_CRON: "*/10 * * * *", + METRICS_ROLLUP_CRON: "7 * * * *", + HOST_RETENTION_CRON: "35 4 * * *", + HOST_EXPORT_REAP_CRON: "40 * * * *", +} as const + +interface IntLimit { + fallback: number + min: number + max: number +} + +const COMMS_INT_LIMITS = { + HOST_BROADCAST_PER_EVENT_PER_DAY: { fallback: 3, min: 1, max: 100 }, + HOST_BROADCAST_RECIPIENTS_PER_DAY: { fallback: 2000, min: 1, max: 1_000_000 }, + HOST_BROADCAST_COOLDOWN_SEC: { fallback: 900, min: 1, max: 86_400 }, + HOST_BROADCAST_MIN_ACCOUNT_AGE_HOURS: { fallback: 24, min: 0, max: 8760 }, + HOST_EVENT_UPDATE_PER_EVENT_PER_HOUR: { fallback: 3, min: 1, max: 100 }, + BROADCAST_MAX_RECIPIENTS: { fallback: 5000, min: 1, max: 100_000 }, + BROADCAST_CHUNK_SIZE: { fallback: 200, min: 1, max: 1000 }, + BROADCAST_EMAIL_CONCURRENCY: { fallback: 4, min: 1, max: 32 }, + BROADCAST_EMAIL_RATE_PER_SEC: { fallback: 10, min: 1, max: 500 }, + HOST_ANALYTICS_CACHE_TTL_SEC: { fallback: 120, min: 1, max: 3600 }, + METRICS_ROLLUP_LOOKBACK_DAYS: { fallback: 3, min: 1, max: 90 }, + HOST_EXPORT_MAX_ROWS: { fallback: 50_000, min: 1, max: 1_000_000 }, + HOST_EXPORT_TTL_HOURS: { fallback: 24, min: 1, max: 168 }, +} as const satisfies Record export interface CommsEnv { MAIL_FROM_EVENTS: string @@ -40,25 +73,15 @@ export interface CommsEnv { const DEV_UNSUBSCRIBE_SIGNING_KEY = "dev-insecure-unsubscribe-signing-key-do-not-use-in-prod" -export function loadCommsEnv(source: NodeJS.ProcessEnv, errors: string[]): CommsEnv { +export function loadCommsEnv(source: EnvSource, errors: string[]): CommsEnv { const isProd = (source.NODE_ENV ?? "").trim() === "production" + const r = makeEnvReader(source, errors, isProd) + const cron = (key: keyof typeof COMMS_CRON_DEFAULTS): string => + r.cron(key, COMMS_CRON_DEFAULTS[key]) + const bounded = (key: keyof typeof COMMS_INT_LIMITS): number => + boundedInt(r, key, COMMS_INT_LIMITS[key]) - function cron(key: string, fallback: string): string { - const value = (source[key] ?? "").trim() || fallback - if (!isCronish(value)) errors.push(`${key}: must be a 5- or 6-field cron expression`) - return value - } - - function bounded(key: string, fallback: number, min: number, max: number): number { - const raw = parseIntOr(source[key], fallback, { key, errors }) - if (raw < min || raw > max) { - errors.push(`${key}: must be an integer between ${min} and ${max}`) - return fallback - } - return raw - } - - if (isProd && (source.PUBLIC_API_URL ?? "").trim().length === 0) { + if (isProd && r.trimmed("PUBLIC_API_URL").length === 0) { errors.push( "PUBLIC_API_URL: required [BOOT] for host communications: the RFC 8058 List-Unsubscribe " + "header in every broadcast email is an API-origin URL, and a wrong origin makes one-click " + @@ -66,91 +89,92 @@ export function loadCommsEnv(source: NodeJS.ProcessEnv, errors: string[]): Comms ) } - const mailFromEvents = (source.MAIL_FROM_EVENTS ?? "").trim() || DEFAULT_MAIL_FROM_EVENTS + const mailFromEvents = r.trimmed("MAIL_FROM_EVENTS") || DEFAULT_MAIL_FROM_EVENTS if (!mailFromEvents.includes("@")) { errors.push( "MAIL_FROM_EVENTS: must be an email address (it is the From of every host broadcast)", ) } - let unsubscribeKey = (source.UNSUBSCRIBE_SIGNING_KEY ?? "").trim() - if (isProd) { - const sessionKey = (source.SESSION_SIGNING_KEY ?? "").trim() - const anonKey = (source.ANON_TOKEN_SIGNING_KEY ?? "").trim() - if (unsubscribeKey.length === 0) { - errors.push("UNSUBSCRIBE_SIGNING_KEY: required [BOOT] variable is missing") - } else if (unsubscribeKey === DEV_UNSUBSCRIBE_SIGNING_KEY) { - errors.push("UNSUBSCRIBE_SIGNING_KEY: must not be the insecure dev default in production") - } else if (unsubscribeKey.length < UNSUBSCRIBE_SIGNING_KEY_MIN_LENGTH) { - errors.push( - `UNSUBSCRIBE_SIGNING_KEY: must be at least ${UNSUBSCRIBE_SIGNING_KEY_MIN_LENGTH} characters in ` + - "production (it signs the one-click unsubscribe capability carried in every broadcast email)", - ) - } else if (unsubscribeKey === sessionKey || unsubscribeKey === anonKey) { - errors.push( - "UNSUBSCRIBE_SIGNING_KEY: must be a DIFFERENT value from SESSION_SIGNING_KEY and " + - "ANON_TOKEN_SIGNING_KEY (one shared secret lets an oracle on either surface attack the other, " + - "and rotating it after a mail incident would silently invalidate every live session)", - ) - } - } else if (unsubscribeKey.length === 0) { - unsubscribeKey = DEV_UNSUBSCRIBE_SIGNING_KEY - } + const unsubscribeKey = loadUnsubscribeSigningKey(r) return { MAIL_FROM_EVENTS: mailFromEvents, UNSUBSCRIBE_SIGNING_KEY: unsubscribeKey, - BROADCAST_SWEEP_CRON: cron("BROADCAST_SWEEP_CRON", "*/2 * * * *"), - EVENT_REMINDERS_CRON: cron("EVENT_REMINDERS_CRON", "*/10 * * * *"), - METRICS_ROLLUP_CRON: cron("METRICS_ROLLUP_CRON", "7 * * * *"), - HOST_RETENTION_CRON: cron("HOST_RETENTION_CRON", "35 4 * * *"), - HOST_EXPORT_REAP_CRON: cron("HOST_EXPORT_REAP_CRON", "40 * * * *"), - - HOST_BROADCAST_PER_EVENT_PER_DAY: bounded("HOST_BROADCAST_PER_EVENT_PER_DAY", 3, 1, 100), - HOST_BROADCAST_RECIPIENTS_PER_DAY: bounded( - "HOST_BROADCAST_RECIPIENTS_PER_DAY", - 2000, - 1, - 1_000_000, - ), - HOST_BROADCAST_COOLDOWN_SEC: bounded("HOST_BROADCAST_COOLDOWN_SEC", 900, 1, 86_400), - HOST_BROADCAST_MIN_ACCOUNT_AGE_HOURS: bounded( - "HOST_BROADCAST_MIN_ACCOUNT_AGE_HOURS", - 24, - 0, - 8760, - ), - HOST_EVENT_UPDATE_PER_EVENT_PER_HOUR: bounded( - "HOST_EVENT_UPDATE_PER_EVENT_PER_HOUR", - 3, - 1, - 100, - ), - - BROADCAST_MAX_RECIPIENTS: bounded("BROADCAST_MAX_RECIPIENTS", 5000, 1, 100_000), - BROADCAST_CHUNK_SIZE: bounded("BROADCAST_CHUNK_SIZE", 200, 1, 1000), - BROADCAST_EMAIL_CONCURRENCY: bounded("BROADCAST_EMAIL_CONCURRENCY", 4, 1, 32), - BROADCAST_EMAIL_RATE_PER_SEC: bounded("BROADCAST_EMAIL_RATE_PER_SEC", 10, 1, 500), + BROADCAST_SWEEP_CRON: cron("BROADCAST_SWEEP_CRON"), + EVENT_REMINDERS_CRON: cron("EVENT_REMINDERS_CRON"), + METRICS_ROLLUP_CRON: cron("METRICS_ROLLUP_CRON"), + HOST_RETENTION_CRON: cron("HOST_RETENTION_CRON"), + HOST_EXPORT_REAP_CRON: cron("HOST_EXPORT_REAP_CRON"), + + HOST_BROADCAST_PER_EVENT_PER_DAY: bounded("HOST_BROADCAST_PER_EVENT_PER_DAY"), + HOST_BROADCAST_RECIPIENTS_PER_DAY: bounded("HOST_BROADCAST_RECIPIENTS_PER_DAY"), + HOST_BROADCAST_COOLDOWN_SEC: bounded("HOST_BROADCAST_COOLDOWN_SEC"), + HOST_BROADCAST_MIN_ACCOUNT_AGE_HOURS: bounded("HOST_BROADCAST_MIN_ACCOUNT_AGE_HOURS"), + HOST_EVENT_UPDATE_PER_EVENT_PER_HOUR: bounded("HOST_EVENT_UPDATE_PER_EVENT_PER_HOUR"), + + BROADCAST_MAX_RECIPIENTS: bounded("BROADCAST_MAX_RECIPIENTS"), + BROADCAST_CHUNK_SIZE: bounded("BROADCAST_CHUNK_SIZE"), + BROADCAST_EMAIL_CONCURRENCY: bounded("BROADCAST_EMAIL_CONCURRENCY"), + BROADCAST_EMAIL_RATE_PER_SEC: bounded("BROADCAST_EMAIL_RATE_PER_SEC"), BROADCAST_LINK_ALLOWED_HOSTS: parseCsvLower(source.BROADCAST_LINK_ALLOWED_HOSTS), HOST_MESSAGING_KILL_SWITCH: parseBool(source.HOST_MESSAGING_KILL_SWITCH, false), - HOST_ANALYTICS_CACHE_TTL_SEC: bounded("HOST_ANALYTICS_CACHE_TTL_SEC", 120, 1, 3600), + HOST_ANALYTICS_CACHE_TTL_SEC: bounded("HOST_ANALYTICS_CACHE_TTL_SEC"), PAGE_VIEW_DEDUPE_SEC: clampDedupeSeconds(source.PAGE_VIEW_DEDUPE_SEC, errors), - METRICS_ROLLUP_LOOKBACK_DAYS: bounded("METRICS_ROLLUP_LOOKBACK_DAYS", 3, 1, 90), + METRICS_ROLLUP_LOOKBACK_DAYS: bounded("METRICS_ROLLUP_LOOKBACK_DAYS"), - HOST_EXPORT_MAX_ROWS: bounded("HOST_EXPORT_MAX_ROWS", 50_000, 1, 1_000_000), - HOST_EXPORT_TTL_HOURS: bounded("HOST_EXPORT_TTL_HOURS", 24, 1, 168), + HOST_EXPORT_MAX_ROWS: bounded("HOST_EXPORT_MAX_ROWS"), + HOST_EXPORT_TTL_HOURS: bounded("HOST_EXPORT_TTL_HOURS"), SMS_HOST_BROADCAST_ENABLED: parseBool(source.SMS_HOST_BROADCAST_ENABLED, false), } } +function loadUnsubscribeSigningKey(r: EnvReader): string { + const unsubscribeKey = r.trimmed("UNSUBSCRIBE_SIGNING_KEY") + if (!r.isProd) { + return unsubscribeKey.length > 0 ? unsubscribeKey : DEV_UNSUBSCRIBE_SIGNING_KEY + } + if (unsubscribeKey.length === 0) { + r.errors.push("UNSUBSCRIBE_SIGNING_KEY: required [BOOT] variable is missing") + } else if (unsubscribeKey === DEV_UNSUBSCRIBE_SIGNING_KEY) { + r.errors.push("UNSUBSCRIBE_SIGNING_KEY: must not be the insecure dev default in production") + } else if (unsubscribeKey.length < UNSUBSCRIBE_SIGNING_KEY_MIN_LENGTH) { + r.errors.push( + `UNSUBSCRIBE_SIGNING_KEY: must be at least ${UNSUBSCRIBE_SIGNING_KEY_MIN_LENGTH} characters in ` + + "production (it signs the one-click unsubscribe capability carried in every broadcast email)", + ) + } else if ( + unsubscribeKey === r.trimmed("SESSION_SIGNING_KEY") || + unsubscribeKey === r.trimmed("ANON_TOKEN_SIGNING_KEY") + ) { + r.errors.push( + "UNSUBSCRIBE_SIGNING_KEY: must be a DIFFERENT value from SESSION_SIGNING_KEY and " + + "ANON_TOKEN_SIGNING_KEY (one shared secret lets an oracle on either surface attack the other, " + + "and rotating it after a mail incident would silently invalidate every live session)", + ) + } + return unsubscribeKey +} + +function boundedInt(r: EnvReader, key: string, limit: IntLimit): number { + const value = parseIntOr(r.source[key], limit.fallback, { key, errors: r.errors }) + if (value < limit.min || value > limit.max) { + r.errors.push(`${key}: must be an integer between ${limit.min} and ${limit.max}`) + return limit.fallback + } + return value +} + function clampDedupeSeconds(raw: string | undefined, errors: string[]): number { const value = parseIntOr(raw, 0, { key: "PAGE_VIEW_DEDUPE_SEC", errors }) - if (value < 0 || value > 3600) { - errors.push("PAGE_VIEW_DEDUPE_SEC: must be an integer between 0 (off) and 3600") + if (value < 0 || value > PAGE_VIEW_DEDUPE_SEC_MAX) { + errors.push( + `PAGE_VIEW_DEDUPE_SEC: must be an integer between 0 (off) and ${PAGE_VIEW_DEDUPE_SEC_MAX}`, + ) return 0 } return value diff --git a/services/api/src/env/core-env.ts b/services/api/src/env/core-env.ts new file mode 100644 index 00000000..7adaf856 --- /dev/null +++ b/services/api/src/env/core-env.ts @@ -0,0 +1,467 @@ +import { DEFAULT_FEED_RANKING, FeedRankingConfigSchema } from "@civfix/shared" +import type { FeedRankingConfig } from "@civfix/shared" +import type { Env } from "./types.js" +import type { EnvReader } from "./reader.js" +import { + assertOutboundSendPolicy, + OUTBOUND_SEND_MIN_THROUGHPUT_BPS, +} from "../services/admin/outbound-send-policy.js" +import { CENSUS_DEFAULT_TIMEOUT_MS } from "../adapters/jurisdiction-lookup.census.js" +import { DEFAULT_API_PORT } from "../lib/base-url.js" +import { + parseBool, + parseCsv, + parseCsvLower, + parseDrainMs, + parseIntOr, + parseStrictBool, + parseTrustProxy, + STRICT_BOOL_ACCEPTED_FORMS, +} from "./parsers.js" + +const DEV_SESSION_SIGNING_KEY = "dev-insecure-session-signing-key-do-not-use-in-prod" +const DEV_ANON_TOKEN_SIGNING_KEY = "dev-insecure-anon-token-signing-key-do-not-use-in-prod" + +const SIGNING_KEY_MIN_LENGTH = 32 + +const SMTP_PORT_DEFAULT = 587 +// Mirrors OCI_MAILER_DEFAULT_TIMEOUT_MS in adapters/mailer.oci.ts, which cannot be imported here: that +// module pulls the native argon2 binding in through auth/otp. +const SMTP_TIMEOUT_MS_DEFAULT = 15_000 +const SMS_DAILY_CAP_DEFAULT = 50 +const VOLUNTEER_HOURS_WEEKLY_FLAG_HOURS_DEFAULT = 60 +const OUTREACH_THROTTLE_DAYS_DEFAULT = 7 + +const OUTREACH_DIGEST_CRON_DEFAULT = "0 14 * * *" +const GUEST_RETENTION_CRON_DEFAULT = "15 4 * * *" +const INBOUND_SWEEP_CRON_DEFAULT = "*/5 * * * *" + +const CENSUS_GEOCODER_URL_DEFAULT = + "https://geocoding.geo.census.gov/geocoder/geographies/coordinates" + +const MAIL_FROM_NOREPLY_DEFAULT = "no-reply@civfix.org" +const MAIL_FROM_OUTREACH_DEFAULT = "outreach@civfix.org" +const HOME_TURF_MAIL_FROM_DEFAULT = "donotreply@civfix.org" +const MAIL_REPLY_DOMAIN_DEFAULT = "civfix.org" + +const TLS_SSLMODES = new Set(["require", "verify-ca", "verify-full"]) +const LOOPBACK_IPV4_PATTERN = /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ +const SINGLE_LABEL_HOST_PATTERN = /^[a-z0-9_-]+$/ + +const TILES_MIN_ZOOM_DEFAULT = 1 +const TILES_MAX_ZOOM_DEFAULT = 19 + +const MAX_LATITUDE = 90 +const MAX_LONGITUDE = 180 +const HOME_REGION_LAT_DEFAULT = 34.0522 +const HOME_REGION_LNG_DEFAULT = -118.2437 +const HOME_REGION_RADIUS_KM_DEFAULT = 40 +const HOME_REGION_RADIUS_KM_MAX = 20_000 + +const APNS_CREDENTIAL_KEYS = [ + "APNS_KEY_ID", + "APNS_TEAM_ID", + "APNS_PRIVATE_KEY", + "APNS_BUNDLE_ID", +] as const + +export type FakeFlags = Pick< + Env, + | "USE_FAKE_STORAGE" + | "USE_FAKE_MAILER" + | "USE_FAKE_PUSH" + | "USE_FAKE_ABUSE_NSFW" + | "USE_FAKE_CHAT" + | "USE_FAKE_JOBS" + | "USE_FAKE_USER_CHANNEL" + | "USE_FAKE_GEOCODER" + | "USE_FAKE_SMS" + | "USE_REAL_NSFW" +> + +export function loadCoreEnv( + r: EnvReader, +): Pick { + const PORT = r.port("PORT", DEFAULT_API_PORT) + const PUBLIC_API_URL = r.requiredString("PUBLIC_API_URL") + const WEB_ORIGINS = parseCsv(r.source.WEB_ORIGINS) + if (r.isProd && WEB_ORIGINS.length === 0) { + r.errors.push("WEB_ORIGINS: required [BOOT] CORS allowlist (comma list) is missing") + } + const DATABASE_URL = r.requiredString("DATABASE_URL") + const REDIS_URL = r.requiredString("REDIS_URL") + return { PORT, PUBLIC_API_URL, WEB_ORIGINS, DATABASE_URL, REDIS_URL } +} + +export function loadSigningKeys( + r: EnvReader, +): Pick { + const session = r.trimmed("SESSION_SIGNING_KEY") + const anon = r.trimmed("ANON_TOKEN_SIGNING_KEY") + if (!r.isProd) { + return { + SESSION_SIGNING_KEY: session.length > 0 ? session : DEV_SESSION_SIGNING_KEY, + ANON_TOKEN_SIGNING_KEY: anon.length > 0 ? anon : DEV_ANON_TOKEN_SIGNING_KEY, + } + } + + checkProductionSigningKey(r, { + key: "SESSION_SIGNING_KEY", + value: session, + devDefault: DEV_SESSION_SIGNING_KEY, + purpose: "it is the cookie-signing and session-bound CSRF HMAC secret", + }) + checkProductionSigningKey(r, { + key: "ANON_TOKEN_SIGNING_KEY", + value: anon, + devDefault: DEV_ANON_TOKEN_SIGNING_KEY, + purpose: "it signs the anon-report claim tokens", + }) + if (session.length > 0 && session === anon) { + r.errors.push( + "SESSION_SIGNING_KEY / ANON_TOKEN_SIGNING_KEY: must be DIFFERENT values in production " + + "(one shared secret lets a session-cookie oracle and an anon-token oracle attack the same key)", + ) + } + return { SESSION_SIGNING_KEY: session, ANON_TOKEN_SIGNING_KEY: anon } +} + +function checkProductionSigningKey( + r: EnvReader, + spec: { key: string; value: string; devDefault: string; purpose: string }, +): void { + if (spec.value.length === 0) { + r.errors.push(`${spec.key}: required [BOOT] variable is missing`) + } else if (spec.value === spec.devDefault) { + r.errors.push(`${spec.key}: must not be the insecure dev default in production`) + } else if (spec.value.length < SIGNING_KEY_MIN_LENGTH) { + r.errors.push( + `${spec.key}: must be at least ${SIGNING_KEY_MIN_LENGTH} characters in production ` + + `(${spec.purpose})`, + ) + } +} + +export function checkDatabaseTls(r: EnvReader, databaseUrl: string): void { + if ( + r.isProd && + databaseUrl.length > 0 && + !TLS_SSLMODES.has(sslModeOf(databaseUrl) ?? "") && + !isNonRoutableDbHost(databaseUrl) + ) { + r.errors.push( + "DATABASE_URL: production requires TLS; append ?sslmode=require (or verify-ca / verify-full); " + + "postgres.js otherwise connects in cleartext", + ) + } +} + +function sslModeOf(databaseUrl: string): string | undefined { + try { + const value = new URL(databaseUrl).searchParams.get("sslmode") + return value === null ? undefined : value.trim().toLowerCase() + } catch { + return undefined + } +} + +function isNonRoutableDbHost(databaseUrl: string): boolean { + let host: string + try { + host = new URL(databaseUrl).hostname.trim().toLowerCase() + } catch { + return false + } + if (host.length === 0) return false + if (host.startsWith("[") && host.endsWith("]")) host = host.slice(1, -1) + if (host === "localhost" || host === "::1") return true + if (LOOPBACK_IPV4_PATTERN.test(host)) return true + return SINGLE_LABEL_HOST_PATTERN.test(host) +} + +export function loadTrustProxy(r: EnvReader): Env["TRUST_PROXY"] { + const TRUST_PROXY = parseTrustProxy(r.source.TRUST_PROXY) + if (r.isProd && TRUST_PROXY === true) { + r.errors.push( + "TRUST_PROXY: must not be `true` in production (it trusts any client-supplied X-Forwarded-For). " + + "Use an explicit CIDR list; leave unset for the safe internal-ranges default", + ) + } + return TRUST_PROXY +} + +export function loadHomeRegion( + r: EnvReader, +): Pick { + return { + HOME_REGION_LAT: r.boundedNumber("HOME_REGION_LAT", HOME_REGION_LAT_DEFAULT, { + min: -MAX_LATITUDE, + max: MAX_LATITUDE, + }), + HOME_REGION_LNG: r.boundedNumber("HOME_REGION_LNG", HOME_REGION_LNG_DEFAULT, { + min: -MAX_LONGITUDE, + max: MAX_LONGITUDE, + }), + HOME_REGION_RADIUS_KM: r.boundedNumber("HOME_REGION_RADIUS_KM", HOME_REGION_RADIUS_KM_DEFAULT, { + min: 0, + max: HOME_REGION_RADIUS_KM_MAX, + exclusiveMin: true, + }), + } +} + +export function loadFeedRanking(r: EnvReader, key: string): FeedRankingConfig { + const raw = r.trimmed(key) + if (raw.length === 0) return DEFAULT_FEED_RANKING + let decoded: unknown + try { + decoded = JSON.parse(raw) + } catch { + r.errors.push(`${key}: must be a JSON object of feed-ranking overrides`) + return DEFAULT_FEED_RANKING + } + const parsed = FeedRankingConfigSchema.safeParse(decoded) + if (parsed.success) return parsed.data + for (const issue of parsed.error.issues) { + const path = issue.path.length > 0 ? `.${issue.path.join(".")}` : "" + r.errors.push(`${key}${path}: ${issue.message}`) + } + return DEFAULT_FEED_RANKING +} + +export interface LocalStorageSection { + usesLocalStorage: boolean + fields: Pick +} + +export function loadLocalStorage(r: EnvReader, publicApiUrl: string): LocalStorageSection { + const LOCAL_STORAGE_DIR = r.trimmed("LOCAL_STORAGE_DIR") + const usesLocalStorage = LOCAL_STORAGE_DIR.length > 0 + if (r.isProd && usesLocalStorage) { + r.errors.push( + "LOCAL_STORAGE_DIR: the local-disk storage driver is DEVELOPMENT ONLY and must not be set in " + + "production; configure R2 (R2_ACCOUNT_ID / R2_ACCESS_KEY_ID / R2_SECRET_ACCESS_KEY / R2_BUCKET)", + ) + } + if (usesLocalStorage && publicApiUrl.length === 0) { + r.errors.push( + "PUBLIC_API_URL: required whenever LOCAL_STORAGE_DIR is set: the local-disk driver's presigned " + + "URLs must be absolute and reachable from the browser and the media worker", + ) + } + const LOCAL_STORAGE_SIGNING_KEY = r.trimmed("LOCAL_STORAGE_SIGNING_KEY") + if (!usesLocalStorage) return { usesLocalStorage, fields: {} } + return { + usesLocalStorage, + fields: { + LOCAL_STORAGE_DIR, + ...(LOCAL_STORAGE_SIGNING_KEY.length > 0 ? { LOCAL_STORAGE_SIGNING_KEY } : {}), + }, + } +} + +export function checkGeocoderDatabase( + r: EnvReader, + fakeFlags: FakeFlags, + databaseUrl: string, +): void { + if (!fakeFlags.USE_FAKE_GEOCODER && databaseUrl.length === 0) { + r.errors.push( + "DATABASE_URL: required whenever USE_FAKE_GEOCODER is false: the real geocoder resolves its " + + '"City, ST" label from the jurisdictions PostGIS table, so there is nothing to query without a database', + ) + } +} + +export function loadR2Env( + r: EnvReader, + useFakeStorage: boolean, + usesLocalStorage: boolean, +): Pick { + const gated = { gatedOff: useFakeStorage } + const R2_ACCOUNT_ID = r.requiredString("R2_ACCOUNT_ID", gated) + const R2_ACCESS_KEY_ID = r.requiredString("R2_ACCESS_KEY_ID", gated) + const R2_SECRET_ACCESS_KEY = r.requiredString("R2_SECRET_ACCESS_KEY", gated) + const R2_BUCKET = r.requiredString("R2_BUCKET", gated) + + if (!useFakeStorage && !usesLocalStorage) { + checkInboundBucket(r, R2_BUCKET) + } + return { R2_ACCOUNT_ID, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, R2_BUCKET } +} + +function checkInboundBucket(r: EnvReader, mediaBucket: string): void { + const inboundBucket = r.trimmed("R2_INBOUND_BUCKET") + const publicBase = r.trimmed("R2_PUBLIC_BASE") + if (publicBase.length > 0 && inboundBucket.length === 0) { + r.errors.push( + "R2_INBOUND_BUCKET: required [BOOT] whenever R2_PUBLIC_BASE is set: a shared bucket would " + + "publish raw inbound email and its attachments on the public CDN", + ) + } + if (inboundBucket.length > 0 && inboundBucket === mediaBucket) { + r.errors.push( + "R2_INBOUND_BUCKET: must be a DIFFERENT bucket from R2_BUCKET (inbound mail must never live in " + + "the media bucket)", + ) + } +} + +export function loadMailEnv( + r: EnvReader, + useFakeMailer: boolean, +): Pick< + Env, + | "OCI_EMAIL_SMTP_HOST" + | "OCI_EMAIL_SMTP_PORT" + | "OCI_EMAIL_SMTP_USER" + | "OCI_EMAIL_SMTP_PASS" + | "OCI_EMAIL_SMTP_TIMEOUT_MS" + | "OUTBOUND_SEND_MIN_THROUGHPUT_BPS" + | "MAIL_FROM_NOREPLY" + | "MAIL_FROM_OUTREACH" + | "HOME_TURF_MAIL_FROM" + | "HOME_TURF_NOTIFY_TO" + | "MAIL_REPLY_DOMAIN" +> { + const gated = { gatedOff: useFakeMailer } + const OCI_EMAIL_SMTP_HOST = r.requiredString("OCI_EMAIL_SMTP_HOST", gated) + const OCI_EMAIL_SMTP_PORT = r.port("OCI_EMAIL_SMTP_PORT", SMTP_PORT_DEFAULT) + const OCI_EMAIL_SMTP_USER = r.requiredString("OCI_EMAIL_SMTP_USER", gated) + const OCI_EMAIL_SMTP_PASS = r.requiredString("OCI_EMAIL_SMTP_PASS", gated) + const OCI_EMAIL_SMTP_TIMEOUT_MS = r.positiveInt( + "OCI_EMAIL_SMTP_TIMEOUT_MS", + SMTP_TIMEOUT_MS_DEFAULT, + ) + const minThroughput = r.positiveInt( + "OUTBOUND_SEND_MIN_THROUGHPUT_BPS", + OUTBOUND_SEND_MIN_THROUGHPUT_BPS, + ) + r.errors.push( + ...assertOutboundSendPolicy({ + smtpTimeoutMs: OCI_EMAIL_SMTP_TIMEOUT_MS, + minThroughputBytesPerSec: minThroughput, + }), + ) + + return { + OCI_EMAIL_SMTP_HOST, + OCI_EMAIL_SMTP_PORT, + OCI_EMAIL_SMTP_USER, + OCI_EMAIL_SMTP_PASS, + OCI_EMAIL_SMTP_TIMEOUT_MS, + OUTBOUND_SEND_MIN_THROUGHPUT_BPS: minThroughput, + MAIL_FROM_NOREPLY: r.trimmed("MAIL_FROM_NOREPLY") || MAIL_FROM_NOREPLY_DEFAULT, + MAIL_FROM_OUTREACH: r.trimmed("MAIL_FROM_OUTREACH") || MAIL_FROM_OUTREACH_DEFAULT, + HOME_TURF_MAIL_FROM: r.trimmed("HOME_TURF_MAIL_FROM") || HOME_TURF_MAIL_FROM_DEFAULT, + HOME_TURF_NOTIFY_TO: r.trimmed("HOME_TURF_NOTIFY_TO"), + MAIL_REPLY_DOMAIN: r.trimmed("MAIL_REPLY_DOMAIN") || MAIL_REPLY_DOMAIN_DEFAULT, + } +} + +export function loadSmsEnv( + r: EnvReader, + useFakeSms: boolean, +): Pick< + Env, + | "SMS_GUEST_ENABLED" + | "SMS_DAILY_CAP" + | "TWILIO_ACCOUNT_SID" + | "TWILIO_AUTH_TOKEN" + | "TWILIO_SMS_FROM" +> { + const SMS_GUEST_ENABLED = parseBool(r.source.SMS_GUEST_ENABLED, false) + const SMS_DAILY_CAP = r.positiveInt("SMS_DAILY_CAP", SMS_DAILY_CAP_DEFAULT) + const gated = { gatedOff: useFakeSms || !SMS_GUEST_ENABLED } + return { + SMS_GUEST_ENABLED, + SMS_DAILY_CAP, + TWILIO_ACCOUNT_SID: r.requiredString("TWILIO_ACCOUNT_SID", gated), + TWILIO_AUTH_TOKEN: r.requiredString("TWILIO_AUTH_TOKEN", gated), + TWILIO_SMS_FROM: r.requiredString("TWILIO_SMS_FROM", gated), + } +} + +export function loadScheduleEnv( + r: EnvReader, +): Pick< + Env, + | "OUTREACH_DIGEST_CRON" + | "OUTREACH_DIGEST_ENABLED" + | "REPORT_AUTOFORWARD_ENABLED" + | "GUEST_RETENTION_CRON" + | "INBOUND_SWEEP_CRON" +> { + return { + OUTREACH_DIGEST_CRON: r.cron("OUTREACH_DIGEST_CRON", OUTREACH_DIGEST_CRON_DEFAULT), + OUTREACH_DIGEST_ENABLED: parseBool(r.source.OUTREACH_DIGEST_ENABLED, false), + REPORT_AUTOFORWARD_ENABLED: parseBool(r.source.REPORT_AUTOFORWARD_ENABLED, false), + GUEST_RETENTION_CRON: r.cron("GUEST_RETENTION_CRON", GUEST_RETENTION_CRON_DEFAULT), + INBOUND_SWEEP_CRON: r.cron("INBOUND_SWEEP_CRON", INBOUND_SWEEP_CRON_DEFAULT), + } +} + +export function loadTuningEnv( + r: EnvReader, +): Pick< + Env, + | "SHUTDOWN_DRAIN_MS" + | "TILES_MIN_ZOOM" + | "TILES_MAX_ZOOM" + | "CENSUS_GEOCODER_URL" + | "CENSUS_GEOCODER_TIMEOUT_MS" + | "VOLUNTEER_HOURS_WEEKLY_FLAG_HOURS" + | "OUTREACH_THROTTLE_DAYS" +> { + return { + SHUTDOWN_DRAIN_MS: parseDrainMs(r.source.SHUTDOWN_DRAIN_MS), + TILES_MIN_ZOOM: parseIntOr(r.source.TILES_MIN_ZOOM, TILES_MIN_ZOOM_DEFAULT, { + key: "TILES_MIN_ZOOM", + errors: r.errors, + }), + TILES_MAX_ZOOM: parseIntOr(r.source.TILES_MAX_ZOOM, TILES_MAX_ZOOM_DEFAULT, { + key: "TILES_MAX_ZOOM", + errors: r.errors, + }), + CENSUS_GEOCODER_URL: r.trimmed("CENSUS_GEOCODER_URL") || CENSUS_GEOCODER_URL_DEFAULT, + CENSUS_GEOCODER_TIMEOUT_MS: r.positiveInt( + "CENSUS_GEOCODER_TIMEOUT_MS", + CENSUS_DEFAULT_TIMEOUT_MS, + ), + VOLUNTEER_HOURS_WEEKLY_FLAG_HOURS: r.positiveInt( + "VOLUNTEER_HOURS_WEEKLY_FLAG_HOURS", + VOLUNTEER_HOURS_WEEKLY_FLAG_HOURS_DEFAULT, + ), + OUTREACH_THROTTLE_DAYS: r.positiveInt("OUTREACH_THROTTLE_DAYS", OUTREACH_THROTTLE_DAYS_DEFAULT), + } +} + +export function loadApnsProduction(r: EnvReader): Pick { + const raw = r.trimmed("APNS_PRODUCTION") + if (raw.length === 0) { + const apnsConfigured = APNS_CREDENTIAL_KEYS.every((key) => r.trimmed(key).length > 0) + if (r.isProd && apnsConfigured) { + r.errors.push( + "APNS_PRODUCTION: required [BOOT] once APNs credentials are set (true for App Store and " + + "TestFlight builds; a gateway mismatch makes APNs reject every token as BadDeviceToken)", + ) + } + return {} + } + const APNS_PRODUCTION = parseStrictBool(raw) + if (APNS_PRODUCTION === undefined) { + r.errors.push(`APNS_PRODUCTION: must be one of ${STRICT_BOOL_ACCEPTED_FORMS}`) + return {} + } + return { APNS_PRODUCTION } +} + +export function loadAccessLists( + r: EnvReader, +): Pick { + return { + ADMIN_EMAILS: parseCsvLower(r.source.ADMIN_EMAILS), + CF_ACCESS_SERVICE_TOKENS: parseCsv(r.source.CF_ACCESS_SERVICE_TOKENS), + CF_TURNSTILE_HOSTNAMES: parseCsvLower(r.source.CF_TURNSTILE_HOSTNAMES), + } +} diff --git a/services/api/src/env/parsers.ts b/services/api/src/env/parsers.ts index 866872d7..f55ac450 100644 --- a/services/api/src/env/parsers.ts +++ b/services/api/src/env/parsers.ts @@ -9,9 +9,11 @@ export const DEFAULT_TRUSTED_PROXY_CIDRS: readonly string[] = [ "fc00::/7", ] +const TRUTHY_FORMS = new Set(["1", "true", "yes", "on"]) + export function parseBool(raw: string | undefined, fallback: boolean): boolean { if (raw === undefined || raw === "") return fallback - return ["1", "true", "yes", "on"].includes(raw.trim().toLowerCase()) + return TRUTHY_FORMS.has(raw.trim().toLowerCase()) } // The single-letter forms are accepted because deployed boxes already hold one-character values for @@ -36,6 +38,10 @@ export interface EnvIssueSink { } const INTEGER_PATTERN = /^-?\d+$/ +const UNSIGNED_INTEGER_PATTERN = /^\d+$/ +const CRON_FIELD_PATTERN = /^[\dA-Za-z*/,\-?#]+$/ +const CRON_FIELD_COUNTS = new Set([5, 6]) +const BOUNDS_PART_COUNT = 4 export function parseCsv(raw: string | undefined): string[] { if (!raw) return [] @@ -90,7 +96,7 @@ export function parseBounds( ): [number, number, number, number] { if (raw === undefined || raw.trim() === "") return fallback const parts = raw.split(",").map((s) => Number.parseFloat(s.trim())) - if (parts.length !== 4 || parts.some((n) => !Number.isFinite(n))) return fallback + if (parts.length !== BOUNDS_PART_COUNT || parts.some((n) => !Number.isFinite(n))) return fallback return [parts[0]!, parts[1]!, parts[2]!, parts[3]!] } @@ -100,8 +106,8 @@ export function isCronish(raw: string | undefined): boolean { .trim() .split(/\s+/) .filter((f) => f.length > 0) - if (fields.length !== 5 && fields.length !== 6) return false - return fields.every((f) => /^[\dA-Za-z*/,\-?#]+$/.test(f)) + if (!CRON_FIELD_COUNTS.has(fields.length)) return false + return fields.every((f) => CRON_FIELD_PATTERN.test(f)) } export function parseTrustProxy(raw: string | undefined): TrustProxyValue { @@ -113,7 +119,7 @@ export function parseTrustProxy(raw: string | undefined): TrustProxyValue { if (lower === "true") return true if (lower === "false") return false - if (/^\d+$/.test(value)) return [...DEFAULT_TRUSTED_PROXY_CIDRS] + if (UNSIGNED_INTEGER_PATTERN.test(value)) return [...DEFAULT_TRUSTED_PROXY_CIDRS] const cidrs = value .split(",") diff --git a/services/api/src/env/reader.ts b/services/api/src/env/reader.ts new file mode 100644 index 00000000..48d24fb6 --- /dev/null +++ b/services/api/src/env/reader.ts @@ -0,0 +1,80 @@ +import { z } from "zod" +import { isCronish, parsePositiveIntOr } from "./parsers.js" + +const PortSchema = z.coerce.number().int().positive().max(65535) + +export interface NumberRange { + min: number + max: number + exclusiveMin?: boolean +} + +export type EnvSource = Readonly> + +// Every reader appends to the shared `errors` list instead of throwing, so one boot reports every +// misconfiguration at once. +export interface EnvReader { + readonly source: EnvSource + readonly errors: string[] + readonly isProd: boolean + trimmed(key: string): string + requiredString(key: string, opts?: { gatedOff?: boolean }): string + port(key: string, fallback: number): number + boundedNumber(key: string, fallback: number, range: NumberRange): number + positiveInt(key: string, fallback: number): number + cron(key: string, fallback: string): string +} + +export function makeEnvReader(source: EnvSource, errors: string[], isProd: boolean): EnvReader { + function trimmed(key: string): string { + return (source[key] ?? "").trim() + } + + function requiredString(key: string, opts: { gatedOff?: boolean } = {}): string { + const value = trimmed(key) + const required = isProd && !(opts.gatedOff ?? false) + if (value.length === 0 && required) { + errors.push(`${key}: required [BOOT] variable is missing`) + } + return value + } + + function port(key: string, fallback: number): number { + const raw = source[key] + if (raw === undefined || raw === "") return fallback + const parsed = PortSchema.safeParse(raw) + if (!parsed.success) { + errors.push(`${key}: must be an integer between 1 and 65535`) + return fallback + } + return parsed.data + } + + function boundedNumber(key: string, fallback: number, range: NumberRange): number { + const raw = source[key] + if (raw === undefined || raw.trim() === "") return fallback + + const exclusiveMin = range.exclusiveMin ?? false + const value = Number.parseFloat(raw.trim()) + const aboveMin = exclusiveMin ? value > range.min : value >= range.min + if (Number.isFinite(value) && aboveMin && value <= range.max) return value + + const lowerBound = exclusiveMin ? `greater than ${range.min}` : `at least ${range.min}` + errors.push(`${key}: must be a number ${lowerBound} and at most ${range.max}`) + return fallback + } + + function positiveInt(key: string, fallback: number): number { + return parsePositiveIntOr(source[key], fallback, { key, errors }) + } + + function cron(key: string, fallback: string): string { + const value = trimmed(key) || fallback + if (!isCronish(value)) { + errors.push(`${key}: must be a 5- or 6-field cron expression`) + } + return value + } + + return { source, errors, isProd, trimmed, requiredString, port, boundedNumber, positiveInt, cron } +} diff --git a/services/api/src/env/registration-env.ts b/services/api/src/env/registration-env.ts index 93371660..4dac111f 100644 --- a/services/api/src/env/registration-env.ts +++ b/services/api/src/env/registration-env.ts @@ -1,4 +1,4 @@ -import { isCronish } from "./parsers.js" +import { makeEnvReader, type EnvSource } from "./reader.js" export interface RegistrationEnv { TICKET_TOKEN_SECRET: string @@ -6,36 +6,19 @@ export interface RegistrationEnv { CHECKIN_NOSHOW_CRON: string } -export const TICKET_TOKEN_SECRET_MIN_LENGTH = 32 - -export const WAITLIST_EXPIRE_CRON_DEFAULT = "*/10 * * * *" +export const DEVELOPMENT_TICKET_TOKEN_SECRET = "dev-insecure-ticket-token-secret-do-not-use-in-prod" -export const CHECKIN_NOSHOW_CRON_DEFAULT = "*/30 * * * *" +const TICKET_TOKEN_SECRET_MIN_LENGTH = 32 -export function loadRegistrationEnv( - source: Record, - errors: string[], -): RegistrationEnv { - const isProd = (source.NODE_ENV ?? "").trim() === "production" +const WAITLIST_EXPIRE_CRON_DEFAULT = "*/10 * * * *" - function reqStr(key: string, opts: { gatedOff?: boolean } = {}): string { - const raw = source[key] - const value = typeof raw === "string" ? raw.trim() : "" - if (value.length === 0 && isProd && !(opts.gatedOff ?? false)) { - errors.push(`${key}: required [BOOT] variable is missing`) - } - return value - } +const CHECKIN_NOSHOW_CRON_DEFAULT = "*/30 * * * *" - function reqCron(key: string, fallback: string): string { - const value = (source[key] ?? "").trim() || fallback - if (!isCronish(value)) { - errors.push(`${key}: must be a 5- or 6-field cron expression`) - } - return value - } +export function loadRegistrationEnv(source: EnvSource, errors: string[]): RegistrationEnv { + const isProd = (source.NODE_ENV ?? "").trim() === "production" + const r = makeEnvReader(source, errors, isProd) - const TICKET_TOKEN_SECRET = reqStr("TICKET_TOKEN_SECRET") + const TICKET_TOKEN_SECRET = r.requiredString("TICKET_TOKEN_SECRET") if (isProd && TICKET_TOKEN_SECRET === DEVELOPMENT_TICKET_TOKEN_SECRET) { errors.push("TICKET_TOKEN_SECRET: must not be the insecure dev default in production") } @@ -55,9 +38,7 @@ export function loadRegistrationEnv( TICKET_TOKEN_SECRET.length > 0 || isProd ? TICKET_TOKEN_SECRET : DEVELOPMENT_TICKET_TOKEN_SECRET, - WAITLIST_EXPIRE_CRON: reqCron("WAITLIST_EXPIRE_CRON", WAITLIST_EXPIRE_CRON_DEFAULT), - CHECKIN_NOSHOW_CRON: reqCron("CHECKIN_NOSHOW_CRON", CHECKIN_NOSHOW_CRON_DEFAULT), + WAITLIST_EXPIRE_CRON: r.cron("WAITLIST_EXPIRE_CRON", WAITLIST_EXPIRE_CRON_DEFAULT), + CHECKIN_NOSHOW_CRON: r.cron("CHECKIN_NOSHOW_CRON", CHECKIN_NOSHOW_CRON_DEFAULT), } } - -export const DEVELOPMENT_TICKET_TOKEN_SECRET = "dev-insecure-ticket-token-secret-do-not-use-in-prod" diff --git a/services/api/src/errors/glitchtip.ts b/services/api/src/errors/glitchtip.ts index ca923c38..886aa91e 100644 --- a/services/api/src/errors/glitchtip.ts +++ b/services/api/src/errors/glitchtip.ts @@ -105,6 +105,15 @@ const SECRET_ASSIGN_RE = /\b(access_token|refresh_token|token|password|passwd|secret|api[_-]?key|authorization|auth|otp)\b(\s*[=:]\s*)([^\s,;&"']+)/gi const MAX_MESSAGE_LEN = 2000 +const GLITCHTIP_FLUSH_TIMEOUT_MS = 2000 + +// Tracing would ship request spans (URLs, timings) to the error tracker; only errors are reported. +const TRACES_SAMPLE_RATE = 0 + +// Network and query breadcrumbs carry URLs and SQL text, so they are dropped whole. +const DROPPED_BREADCRUMB_CATEGORIES: ReadonlySet = new Set(["http", "fetch", "xhr"]) +const QUERY_BREADCRUMB_MARKER = "query" + function scrubMessage(text: string): string { const redacted = redactPans( text @@ -180,12 +189,7 @@ export function scrubEvent(event: T): T { export function scrubBreadcrumb(crumb: T): T | null { const category = typeof crumb.category === "string" ? crumb.category.toLowerCase() : "" - if ( - category === "http" || - category === "fetch" || - category === "xhr" || - category.includes("query") - ) { + if (DROPPED_BREADCRUMB_CATEGORIES.has(category) || category.includes(QUERY_BREADCRUMB_MARKER)) { return null } const out = { ...crumb } as SentryBreadcrumbLike @@ -206,7 +210,7 @@ export async function initErrorReporting(opts: ErrorReportingOptions): Promise scrubEvent(event as unknown as SentryEventLike) as never, beforeBreadcrumb: (crumb) => @@ -222,16 +226,12 @@ export async function initErrorReporting(opts: ErrorReportingOptions): Promise): void { if (!enabled || !sentry) return sentry.captureException(err, context ? { extra: context } : undefined) } -export async function flushErrorReporting(timeoutMs = 2000): Promise { +export async function flushErrorReporting(timeoutMs = GLITCHTIP_FLUSH_TIMEOUT_MS): Promise { if (!enabled || !sentry) return try { await sentry.flush(timeoutMs) diff --git a/services/api/src/errors/http-mapper.ts b/services/api/src/errors/http-mapper.ts index 835340e4..8c3ac4a7 100644 --- a/services/api/src/errors/http-mapper.ts +++ b/services/api/src/errors/http-mapper.ts @@ -22,6 +22,17 @@ import { isMessageExposed } from "./exposed-message.js" import { captureError } from "./glitchtip.js" const INTERNAL_ERROR_MESSAGE = "Internal error" +const VALIDATION_FAILED_MESSAGE = "Validation failed" +const CLIENT_ERROR_FALLBACK_MESSAGE = "Request error" +const FIELD_INVALID_FALLBACK_MESSAGE = "invalid" +const STEALTH_NOT_FOUND_MESSAGE = "Not found" + +// Field key for an issue on the payload itself rather than a named field. +const ROOT_FIELD_KEY = "_" + +const HTTP_NOT_FOUND = 404 +const HTTP_UNPROCESSABLE = 422 +const HTTP_INTERNAL = 500 // Operator routes sit behind Cloudflare Access and the operator guard, and the console shows server-side // diagnostics (an SMTP rejection, a misconfigured provider) that the operator is there to fix. @@ -54,102 +65,125 @@ function fieldsFromValidation(err: FastifyError): Record { for (const v of validation) { // instancePath looks like "/body/email"; reduce to the last path segment. const path = (v.instancePath || "").split("/").filter(Boolean) - const key = path.length > 0 ? path[path.length - 1]! : (v.params?.missingProperty ?? "_") - out[String(key)] = v.message ?? "invalid" + const key = + path.length > 0 ? path[path.length - 1]! : (v.params?.missingProperty ?? ROOT_FIELD_KEY) + out[String(key)] = v.message ?? FIELD_INVALID_FALLBACK_MESSAGE } return out } +interface ZodLikeError { + issues: { path: (string | number)[]; message: string }[] +} + +// Structural ZodError match (NOT instanceof) so it survives the dual-zod-realm boundary between +// @civfix/shared's zod and the API's zod: a ZodError thrown outside a route parse() (service-level +// parse / .transform / nested parse) reaches the handler and must render as 422, never 500. +function isZodLikeError(error: Error): error is Error & ZodLikeError { + return error.name === "ZodError" && Array.isArray((error as { issues?: unknown }).issues) +} + +function zodFields(error: ZodLikeError): Record { + const fields: Record = {} + for (const i of error.issues) + fields[i.path.length ? i.path.join(".") : ROOT_FIELD_KEY] = i.message + return fields +} + +function sendValidationFailure( + reply: FastifyReply, + requestId: string, + fields: Record, +): void { + const body: ErrorBody = { + code: ErrorCode.VALIDATION, + message: VALIDATION_FAILED_MESSAGE, + requestId, + fields, + } + reply.status(HTTP_UNPROCESSABLE).send(body) +} + +function captureContext(request: FastifyRequest, requestId: string): Record { + return { requestId, url: loggedRequestUrl(request.url), method: request.method } +} + +function sendAppError(error: AppError, request: FastifyRequest, reply: FastifyReply): void { + const requestId = request.id + error.requestId = requestId + const body: ErrorBody = { + code: error.code, + message: serverMessageHidden(error, request) ? INTERNAL_ERROR_MESSAGE : error.message, + requestId, + ...(error.fields ? { fields: error.fields } : {}), + } + if (error.httpStatus >= HTTP_INTERNAL) { + request.log.error({ err: error, requestId }, "AppError (server)") + captureError(error, captureContext(request, requestId)) + } else { + request.log.info({ code: error.code, requestId }, "AppError (client)") + } + reply.status(error.httpStatus).send(body) +} + +function sendUnknownError(error: Error, request: FastifyRequest, reply: FastifyReply): void { + const requestId = request.id + const fastifyErr = error as FastifyError + const statusCode = + typeof fastifyErr.statusCode === "number" ? fastifyErr.statusCode : HTTP_INTERNAL + if (statusCode < HTTP_INTERNAL) { + request.log.info({ requestId, statusCode }, "client error") + const body: ErrorBody = { + code: STATUS_TO_CODE[statusCode] ?? ErrorCode.VALIDATION, + message: error.message || CLIENT_ERROR_FALLBACK_MESSAGE, + requestId, + } + reply.status(statusCode).send(body) + return + } + + request.log.error({ err: error, requestId }, "unhandled error") + captureError(error, captureContext(request, requestId)) + const body: ErrorBody = { + code: ErrorCode.INTERNAL, + message: isProd() ? INTERNAL_ERROR_MESSAGE : (error.message ?? INTERNAL_ERROR_MESSAGE), + requestId, + } + reply.status(HTTP_INTERNAL).send(body) +} + export function makeErrorHandler() { return function errorHandler( error: FastifyError | AppError | Error, request: FastifyRequest, reply: FastifyReply, ): void { - const requestId = request.id - if (error instanceof AppError) { - error.requestId = requestId - const body: ErrorBody = { - code: error.code, - message: serverMessageHidden(error, request) ? INTERNAL_ERROR_MESSAGE : error.message, - requestId, - ...(error.fields ? { fields: error.fields } : {}), - } - if (error.httpStatus >= 500) { - request.log.error({ err: error, requestId }, "AppError (server)") - captureError(error, { - requestId, - url: loggedRequestUrl(request.url), - method: request.method, - }) - } else { - request.log.info({ code: error.code, requestId }, "AppError (client)") - } - reply.status(error.httpStatus).send(body) + sendAppError(error, request, reply) return } - // Structural ZodError match (NOT instanceof) so it survives the dual-zod-realm boundary between - // @civfix/shared's zod and the API's zod: a ZodError thrown outside a route parse() (service-level - // parse / .transform / nested parse) reaches the handler and must render as 422, never 500. - if (error.name === "ZodError" && Array.isArray((error as { issues?: unknown }).issues)) { - const issues = ( - error as unknown as { issues: { path: (string | number)[]; message: string }[] } - ).issues - const fields: Record = {} - for (const i of issues) fields[i.path.length ? i.path.join(".") : "_"] = i.message - request.log.info({ requestId, fields }, "zod validation error") - const body: ErrorBody = { - code: ErrorCode.VALIDATION, - message: "Validation failed", - requestId, - fields, - } - reply.status(422).send(body) + if (isZodLikeError(error)) { + const fields = zodFields(error) + request.log.info({ requestId: request.id, fields }, "zod validation error") + sendValidationFailure(reply, request.id, fields) return } const fastifyErr = error as FastifyError if (fastifyErr.validation && fastifyErr.validation.length > 0) { const fields = fieldsFromValidation(fastifyErr) - request.log.info({ requestId, fields }, "validation error") - const body: ErrorBody = { - code: ErrorCode.VALIDATION, - message: "Validation failed", - requestId, - fields, - } - reply.status(422).send(body) - return - } - - const statusCode = typeof fastifyErr.statusCode === "number" ? fastifyErr.statusCode : 500 - if (statusCode < 500) { - request.log.info({ requestId, statusCode }, "client error") - const code = STATUS_TO_CODE[statusCode] ?? ErrorCode.VALIDATION - const body: ErrorBody = { - code, - message: error.message || "Request error", - requestId, - } - reply.status(statusCode).send(body) + request.log.info({ requestId: request.id, fields }, "validation error") + sendValidationFailure(reply, request.id, fields) return } - request.log.error({ err: error, requestId }, "unhandled error") - captureError(error, { requestId, url: loggedRequestUrl(request.url), method: request.method }) - const body: ErrorBody = { - code: ErrorCode.INTERNAL, - message: isProd() ? INTERNAL_ERROR_MESSAGE : (error.message ?? INTERNAL_ERROR_MESSAGE), - requestId, - } - reply.status(500).send(body) + sendUnknownError(error, request, reply) } } function serverMessageHidden(error: AppError, request: FastifyRequest): boolean { - if (error.httpStatus < 500 || !isProd()) return false + if (error.httpStatus < HTTP_INTERNAL || !isProd()) return false if (isMessageExposed(error)) return false const routePattern = request.routeOptions.url ?? "" return !routePattern.startsWith(OPERATOR_ROUTE_PREFIX) @@ -165,9 +199,11 @@ export function makeNotFoundHandler() { code: ErrorCode.NOT_FOUND, // Prod: static (stealth). Dev/test: echo method+url so route-coverage can tell an // unregistered-endpoint 404 apart from a domain (AppError) 404. - message: isProd() ? "Not found" : `Route ${request.method} ${request.url} not found`, + message: isProd() + ? STEALTH_NOT_FOUND_MESSAGE + : `Route ${request.method} ${request.url} not found`, requestId: request.id, } - reply.status(404).send(body) + reply.status(HTTP_NOT_FOUND).send(body) } } diff --git a/services/api/src/errors/pan-redaction.ts b/services/api/src/errors/pan-redaction.ts index 23c0422f..20ad2d56 100644 --- a/services/api/src/errors/pan-redaction.ts +++ b/services/api/src/errors/pan-redaction.ts @@ -1,20 +1,27 @@ -const PAN_RE = /\b(?:\d[ -]*?){13,19}\b/g +// ISO/IEC 7812 card numbers run 13 to 19 digits. +const PAN_MIN_DIGITS = 13 +const PAN_MAX_DIGITS = 19 +const PAN_RE = new RegExp(`\\b(?:\\d[ -]*?){${PAN_MIN_DIGITS},${PAN_MAX_DIGITS}}\\b`, "g") +const NON_DIGIT_RE = /\D/g +const CHAR_CODE_ZERO = 48 +const LUHN_MODULUS = 10 +const LUHN_MAX_DIGIT = 9 export function isLuhnValid(candidate: string): boolean { - const digits = candidate.replace(/\D/g, "") - if (digits.length < 13 || digits.length > 19) return false + const digits = candidate.replace(NON_DIGIT_RE, "") + if (digits.length < PAN_MIN_DIGITS || digits.length > PAN_MAX_DIGITS) return false let sum = 0 let doubled = false for (let position = digits.length - 1; position >= 0; position -= 1) { - let value = digits.charCodeAt(position) - 48 + let value = digits.charCodeAt(position) - CHAR_CODE_ZERO if (doubled) { value *= 2 - if (value > 9) value -= 9 + if (value > LUHN_MAX_DIGIT) value -= LUHN_MAX_DIGIT } sum += value doubled = !doubled } - return sum % 10 === 0 + return sum % LUHN_MODULUS === 0 } export function redactPans(text: string, replacement: string): string { diff --git a/services/api/src/errors/sms-failure.ts b/services/api/src/errors/sms-failure.ts index 5cf0155f..dc938a55 100644 --- a/services/api/src/errors/sms-failure.ts +++ b/services/api/src/errors/sms-failure.ts @@ -2,7 +2,7 @@ import { AppError, ErrorCode } from "@civfix/shared" export const SMS_FAILURE_FIELD = "smsDelivery" -export const SMS_FAILURE_KINDS = ["opted_out", "invalid_number", "permanent", "temporary"] as const +const SMS_FAILURE_KINDS = ["opted_out", "invalid_number", "permanent", "temporary"] as const export type SmsFailureKind = (typeof SMS_FAILURE_KINDS)[number] @@ -28,7 +28,3 @@ export function smsFailureKind(err: unknown): SmsFailureKind | null { if (typeof value !== "string") return null return (SMS_FAILURE_KINDS as readonly string[]).includes(value) ? (value as SmsFailureKind) : null } - -export function isRetryableSmsFailure(err: unknown): boolean { - return smsFailureKind(err) === "temporary" -} diff --git a/services/api/src/i18n/locales.ts b/services/api/src/i18n/locales.ts index 5b0094ba..5a14aff9 100644 --- a/services/api/src/i18n/locales.ts +++ b/services/api/src/i18n/locales.ts @@ -6,10 +6,12 @@ export type Locale = (typeof SUPPORTED_LOCALES)[number] export const DEFAULT_LOCALE: Locale = "en" +const LANGUAGE_SUBTAG_SEPARATOR_RE = /[-_]/ + /** Total: a BCP-47 tag reduces to its base language (`es-419` -> `es`); anything else becomes `en`. */ export function resolveLocale(value: unknown): Locale { if (typeof value !== "string" || value.trim() === "") return DEFAULT_LOCALE - const base = value.trim().toLowerCase().split(/[-_]/)[0] + const base = value.trim().toLowerCase().split(LANGUAGE_SUBTAG_SEPARATOR_RE)[0] return (SUPPORTED_LOCALES as readonly string[]).includes(base ?? "") ? (base as Locale) : DEFAULT_LOCALE diff --git a/services/api/src/i18n/messages/de.ts b/services/api/src/i18n/messages/de.ts index f88042f6..fc57d712 100644 --- a/services/api/src/i18n/messages/de.ts +++ b/services/api/src/i18n/messages/de.ts @@ -71,7 +71,6 @@ export const de: Partial> = { 'Die Schicht "{{slot}}" bei {{title}} hat eine neue Zeit. Öffne das Event, um sie zu prüfen.', "email.otp.subject": "Dein civfix-Anmeldecode", - "email.otp.body_line1": "Dein civfix-Anmeldecode lautet {{code}}.", "email.otp.body_expiry": "Er läuft in {{minutes}} Minuten ab. Falls du ihn nicht angefordert hast, kannst du diese E-Mail ignorieren.", "email.otp.html_intro": "Dein civfix-Anmeldecode lautet:", diff --git a/services/api/src/i18n/messages/en.ts b/services/api/src/i18n/messages/en.ts index 9d5051a4..bda6af67 100644 --- a/services/api/src/i18n/messages/en.ts +++ b/services/api/src/i18n/messages/en.ts @@ -143,7 +143,6 @@ export const en = { "certificate.error.no_hours": "You have no recorded service hours yet.", "email.otp.subject": "Your civfix sign-in code", - "email.otp.body_line1": "Your civfix sign-in code is {{code}}.", "email.otp.body_expiry": "It expires in {{minutes}} minutes. If you did not request it, you can ignore this email.", // The code itself is rendered in a styled block by the template. diff --git a/services/api/src/i18n/messages/es.ts b/services/api/src/i18n/messages/es.ts index 773fea92..8745f2fa 100644 --- a/services/api/src/i18n/messages/es.ts +++ b/services/api/src/i18n/messages/es.ts @@ -70,7 +70,6 @@ export const es: Partial> = { 'El turno "{{slot}}" de {{title}} tiene un horario nuevo. Abre el evento para verlo.', "email.otp.subject": "Tu código de acceso a civfix", - "email.otp.body_line1": "Tu código de acceso a civfix es {{code}}.", "email.otp.body_expiry": "Caduca en {{minutes}} minutos. Si no lo solicitaste, puedes ignorar este correo.", "email.otp.html_intro": "Tu código de acceso a civfix es:", diff --git a/services/api/src/i18n/messages/ko.ts b/services/api/src/i18n/messages/ko.ts index 78c11e81..5bf533cb 100644 --- a/services/api/src/i18n/messages/ko.ts +++ b/services/api/src/i18n/messages/ko.ts @@ -70,7 +70,6 @@ export const ko: Partial> = { '{{title}}의 "{{slot}}" 교대 시간이 바뀌었어요. 이벤트를 열어 확인해 주세요.', "email.otp.subject": "civfix 로그인 코드", - "email.otp.body_line1": "civfix 로그인 코드는 {{code}}입니다.", "email.otp.body_expiry": "코드는 {{minutes}}분 후 만료됩니다. 요청하지 않으셨다면 이 이메일을 무시하세요.", "email.otp.html_intro": "civfix 로그인 코드:", diff --git a/services/api/src/lib/base-url.ts b/services/api/src/lib/base-url.ts index 6cb3213a..a73a4616 100644 --- a/services/api/src/lib/base-url.ts +++ b/services/api/src/lib/base-url.ts @@ -1,6 +1,8 @@ -export const LOCAL_WEB_PORT = 3000 +const LOCAL_WEB_PORT = 3000 export const DEFAULT_API_PORT = 8080 +const TRAILING_SLASHES_RE = /\/+$/ + export interface BaseUrlEnv { NODE_ENV?: string PORT?: number @@ -19,7 +21,7 @@ function baseUrlOr( env: BaseUrlEnv, port: number, ): string { - const trimmed = (configured ?? "").trim().replace(/\/+$/, "") + const trimmed = (configured ?? "").trim().replace(TRAILING_SLASHES_RE, "") if (trimmed.length > 0) return trimmed if (env.NODE_ENV === "production") throw new Error(`${variable} is required in production`) return `http://localhost:${port}` diff --git a/services/api/src/lifecycle.ts b/services/api/src/lifecycle.ts index 9e2f3518..8efa6230 100644 --- a/services/api/src/lifecycle.ts +++ b/services/api/src/lifecycle.ts @@ -11,11 +11,14 @@ declare module "fastify" { export const REQUEST_TIMEOUT_MS = 15_000 export const SHUTDOWN_CLOSE_WAIT_MS = REQUEST_TIMEOUT_MS export const SHUTDOWN_TEARDOWN_WATCHDOG_MS = 15_000 -export const SHUTDOWN_IDLE_SWEEP_MS = 250 +const SHUTDOWN_IDLE_SWEEP_MS = 250 export const SHUTDOWN_FORCE_GRACE_MS = 1_000 export const SHUTDOWN_BUDGET_MARGIN_MS = 5_000 export const COMPOSE_STOP_GRACE_PERIOD_SECONDS = 45 +const EXIT_CLEAN = 0 +const EXIT_FAILURE = 1 + export interface Lifecycle { isDraining: () => boolean beginDrain: () => void @@ -32,7 +35,7 @@ export function makeLifecycle(): Lifecycle { draining = true }, escalateExitCode: (code: number): void => { - if (code !== 0 && escalated === undefined) escalated = code + if (code !== EXIT_CLEAN && escalated === undefined) escalated = code }, finalExitCode: (fallback: number): number => escalated ?? fallback, } @@ -116,7 +119,7 @@ export function makeShutdown( const teardownWatchdogMs = options.teardownWatchdogMs ?? SHUTDOWN_TEARDOWN_WATCHDOG_MS const idleSweepMs = options.idleSweepMs ?? SHUTDOWN_IDLE_SWEEP_MS const exit = options.exit ?? ((code: number): void => process.exit(code)) - const cleanExitCode = options.exitCode ?? 0 + const cleanExitCode = options.exitCode ?? EXIT_CLEAN const drainMs = Math.min(SHUTDOWN_DRAIN_MS_MAX, Math.max(0, options.drainMs)) const hardDeadlineMs = drainMs + closeWaitMs + teardownWatchdogMs let started = false @@ -140,7 +143,7 @@ export function makeShutdown( const deadline = setTimeout(() => { app.log.error({ hardDeadlineMs }, "shutdown: hard deadline exceeded; forcing exit") - exit(1) + exit(EXIT_FAILURE) }, hardDeadlineMs) deadline.unref() @@ -156,7 +159,7 @@ export function makeShutdown( })() if ((await settleWithin(teardown, teardownWatchdogMs)) === "timeout") { app.log.error({ teardownWatchdogMs }, "shutdown: teardown timed out; forcing exit") - exit(1) + exit(EXIT_FAILURE) return } app.log.info("shutdown: complete") @@ -164,7 +167,7 @@ export function makeShutdown( } catch (err) { app.log.error({ err }, "shutdown: error during close") await flushErrorReporting() - exit(1) + exit(EXIT_FAILURE) } finally { clearTimeout(deadline) } diff --git a/services/api/src/plugins/cors.ts b/services/api/src/plugins/cors.ts index 908f3cfe..d1957df2 100644 --- a/services/api/src/plugins/cors.ts +++ b/services/api/src/plugins/cors.ts @@ -2,6 +2,20 @@ import fastifyCors from "@fastify/cors" import type { FastifyInstance } from "fastify" import { isProd } from "../env.js" +const CORS_PREFLIGHT_MAX_AGE_SEC = 600 + +const CORS_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"] + +// Browsers hide all but a few simple response headers from cross-origin JS. Same-origin web reads the +// anon token from the civfix_anon cookie instead, but a cross-origin client needs X-Anon-Token here. +const CORS_EXPOSED_HEADERS = [ + "X-Anon-Token", + "X-RateLimit-Limit", + "X-RateLimit-Remaining", + "X-RateLimit-Reset", + "Retry-After", +] + export async function registerCors(app: FastifyInstance, webOrigins: string[]): Promise { const allowlist = new Set(webOrigins) // Security: reflecting ANY origin with credentials:true is a credential-theft hole, so the empty-allowlist @@ -19,17 +33,9 @@ export async function registerCors(app: FastifyInstance, webOrigins: string[]): await app.register(fastifyCors, { credentials: true, - methods: ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], - maxAge: 600, - // Browsers hide all but a few simple response headers from cross-origin JS. Same-origin web reads the - // anon token from the civfix_anon cookie instead, but a cross-origin client needs X-Anon-Token here. - exposedHeaders: [ - "X-Anon-Token", - "X-RateLimit-Limit", - "X-RateLimit-Remaining", - "X-RateLimit-Reset", - "Retry-After", - ], + methods: CORS_METHODS, + maxAge: CORS_PREFLIGHT_MAX_AGE_SEC, + exposedHeaders: CORS_EXPOSED_HEADERS, origin(origin, cb) { if (!origin || allowAll || allowlist.has(origin)) { cb(null, true) diff --git a/services/api/src/plugins/helmet.ts b/services/api/src/plugins/helmet.ts index 673ef697..c2790403 100644 --- a/services/api/src/plugins/helmet.ts +++ b/services/api/src/plugins/helmet.ts @@ -8,6 +8,9 @@ import fastifyHelmet from "@fastify/helmet" import type { FastifyInstance } from "fastify" import { isProd } from "../env.js" +const SECONDS_PER_YEAR = 365 * 24 * 60 * 60 +const HSTS_MAX_AGE_SEC = 2 * SECONDS_PER_YEAR + export async function registerHelmet(app: FastifyInstance): Promise { await app.register(fastifyHelmet, { contentSecurityPolicy: { @@ -26,7 +29,7 @@ export async function registerHelmet(app: FastifyInstance): Promise { ...(isProd() ? { strictTransportSecurity: { - maxAge: 63072000, + maxAge: HSTS_MAX_AGE_SEC, includeSubDomains: true, preload: true, }, diff --git a/services/api/src/plugins/rate-limit.ts b/services/api/src/plugins/rate-limit.ts index 6830f767..e5beabb1 100644 --- a/services/api/src/plugins/rate-limit.ts +++ b/services/api/src/plugins/rate-limit.ts @@ -11,7 +11,7 @@ import { normalizeIp } from "../abuse/ip-rate-limit.js" const RATE_LIMIT_ALLOWLIST = new Set(["/healthz"]) -export const SENSITIVE_RATE_LIMIT_PREFIXES: readonly string[] = [ +const SENSITIVE_RATE_LIMIT_PREFIXES: readonly string[] = [ "/v1/auth", "/auth", "/v1/admin/auth", @@ -24,8 +24,8 @@ export const SENSITIVE_RATE_LIMIT_PREFIXES: readonly string[] = [ "/forms", ] -export const SENSITIVE_WRITE_EXACT_PATHS: readonly string[] = ["/v1/reports"] -export const SENSITIVE_WRITE_PREFIXES: readonly string[] = ["/v1/admin"] +const SENSITIVE_WRITE_EXACT_PATHS: readonly string[] = ["/v1/reports"] +const SENSITIVE_WRITE_PREFIXES: readonly string[] = ["/v1/admin"] const MUTATING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]) @@ -40,8 +40,14 @@ export function isWriteSensitivePath(path: string): boolean { return SENSITIVE_WRITE_PREFIXES.some((p) => path === p || path.startsWith(`${p}/`)) } +const RATE_LIMIT_WINDOW = "1 minute" +const GLOBAL_RATE_LIMIT_PER_MINUTE = 300 const SENSITIVE_MAX = 60 -const SENSITIVE_WINDOW = "1 minute" +const SENSITIVE_WINDOW = RATE_LIMIT_WINDOW + +const IP_KEY_PREFIX = "ip:" +const USER_KEY_PREFIX = "user:" +const HOST_KEY_PREFIX = "host:" export interface RateLimitOptions { max?: number @@ -59,12 +65,12 @@ export function isSensitivePath(path: string): boolean { } export function rateLimitKey(req: FastifyRequest): string { - return `ip:${normalizeIp(req.ip)}` + return `${IP_KEY_PREFIX}${normalizeIp(req.ip)}` } export function identityRateLimitKey(req: FastifyRequest): string { const userId = req.auth?.userId - return userId ? `user:${userId}` : `ip:${normalizeIp(req.ip)}` + return userId ? `${USER_KEY_PREFIX}${userId}` : rateLimitKey(req) } export interface RouteRateLimitSpec { @@ -92,7 +98,7 @@ export function sensitiveRateLimitKey(req: FastifyRequest): string { } export function wsUpgradeRateLimitKey(req: FastifyRequest): string { - return `${WS_UPGRADE_BUCKET}:ip:${normalizeIp(req.ip)}` + return `${WS_UPGRADE_BUCKET}:${rateLimitKey(req)}` } export function perIdentity(spec: RouteRateLimitSpec): RouteRateLimitPolicy { @@ -126,8 +132,8 @@ export async function registerRateLimit( ): Promise { await app.register(fastifyRateLimit, { global: true, - max: opts.max ?? 300, - timeWindow: opts.timeWindow ?? "1 minute", + max: opts.max ?? GLOBAL_RATE_LIMIT_PER_MINUTE, + timeWindow: opts.timeWindow ?? RATE_LIMIT_WINDOW, keyGenerator: rateLimitKey, allowList: (req) => RATE_LIMIT_ALLOWLIST.has(pathOf(req.url)), skipOnError: true, @@ -229,7 +235,7 @@ function hostCeilingHook( const checkHost = app.createRateLimit({ max: limit.hostMax ?? limit.max * HOST_CEILING_MULTIPLIER, timeWindow: limit.timeWindow, - keyGenerator: (req) => `host:${routeId}:ip:${normalizeIp(req.ip)}`, + keyGenerator: (req) => `${HOST_KEY_PREFIX}${routeId}:${rateLimitKey(req)}`, allowList: () => false, skipOnError: true, }) diff --git a/services/api/src/plugins/request-id.ts b/services/api/src/plugins/request-id.ts index 985d9668..c6e679ee 100644 --- a/services/api/src/plugins/request-id.ts +++ b/services/api/src/plugins/request-id.ts @@ -1,12 +1,15 @@ import { randomUUID } from "node:crypto" import type { FastifyInstance, FastifyRequest } from "fastify" +const REQUEST_ID_HEADER = "x-request-id" +const REQUEST_ID_MAX_LENGTH = 200 + /** Keeps hostile bytes (CRLF, control chars, log-injection payloads) out of logs and the echoed header. */ -const SAFE_REQUEST_ID = /^[A-Za-z0-9._-]{1,200}$/ +const SAFE_REQUEST_ID = new RegExp(`^[A-Za-z0-9._-]{1,${REQUEST_ID_MAX_LENGTH}}$`) /** An inbound `x-request-id` is honored for edge trace stitching only when it matches the safe charset. */ export function genReqId(req: { headers: Record }): string { - const header = req.headers["x-request-id"] + const header = req.headers[REQUEST_ID_HEADER] if (typeof header === "string" && SAFE_REQUEST_ID.test(header)) { return header } @@ -15,7 +18,7 @@ export function genReqId(req: { headers: Record }): string { export async function registerRequestId(app: FastifyInstance): Promise { app.addHook("onSend", async (request: FastifyRequest, reply, payload) => { - reply.header("x-request-id", request.id) + reply.header(REQUEST_ID_HEADER, request.id) return payload }) } diff --git a/services/api/src/plugins/trust-proxy.ts b/services/api/src/plugins/trust-proxy.ts index 7f1a64a9..c96e7d9d 100644 --- a/services/api/src/plugins/trust-proxy.ts +++ b/services/api/src/plugins/trust-proxy.ts @@ -5,8 +5,4 @@ * civfix-infra, which SETS (not appends) X-Forwarded-For so a client cannot pre-seed even the trusted hop. */ -export { - parseTrustProxy, - DEFAULT_TRUSTED_PROXY_CIDRS, - type TrustProxyValue, -} from "../env/parsers.js" +export { parseTrustProxy, DEFAULT_TRUSTED_PROXY_CIDRS } from "../env/parsers.js" diff --git a/services/api/src/routes/_validate.ts b/services/api/src/routes/_validate.ts index f2b5b173..76e6ea77 100644 --- a/services/api/src/routes/_validate.ts +++ b/services/api/src/routes/_validate.ts @@ -1,6 +1,8 @@ import { AppError } from "@civfix/shared" import { z, type ZodTypeAny } from "zod" -import type { FastifyRequest } from "fastify" + +// Field key for an issue on the value itself (a non-object body, a refine on the whole schema). +const ROOT_FIELD_KEY = "_" export function parse(schema: S, data: unknown): z.infer { try { @@ -16,7 +18,7 @@ export function parse(schema: S, data: unknown): z.infer = {} for (const issue of issues) { - const key = issue.path.length > 0 ? issue.path.join(".") : "_" + const key = issue.path.length > 0 ? issue.path.join(".") : ROOT_FIELD_KEY fields[key] = issue.message } throw AppError.validation(fields) @@ -47,16 +49,3 @@ export function trimTextFields( return trimmed }, schema) } - -export const validateBody = ( - schema: S, - request: FastifyRequest, -): z.infer => parse(schema, request.body) -export const validateQuery = ( - schema: S, - request: FastifyRequest, -): z.infer => parse(schema, request.query) -export const validateParams = ( - schema: S, - request: FastifyRequest, -): z.infer => parse(schema, request.params) diff --git a/services/api/src/routes/anon.routes.ts b/services/api/src/routes/anon.routes.ts index 92b7890a..25a1f568 100644 --- a/services/api/src/routes/anon.routes.ts +++ b/services/api/src/routes/anon.routes.ts @@ -1,7 +1,9 @@ import { AnonReportRequestSchema, + AnonReportResponseSchema, AnonReportStatusRequestSchema, IdSchema, + ReportStatusSchema, type AnonReportResponse, type AnonReportStatusResponse, } from "@civfix/shared" @@ -47,7 +49,7 @@ const AnonReportResponseJsonSchema = { type: "object", properties: { reportId: { type: "string" }, - status: { type: "string", enum: ["held", "published"] }, + status: { type: "string", enum: [...AnonReportResponseSchema.shape.status.options] }, claimCode: { type: "string" }, }, required: ["reportId", "status", "claimCode"], @@ -56,18 +58,7 @@ const AnonReportResponseJsonSchema = { const AnonReportStatusResponseJsonSchema = { type: "object", properties: { - status: { - type: "string", - enum: [ - "submitted", - "held", - "published", - "acknowledged", - "in_progress", - "resolved", - "rejected", - ], - }, + status: { type: "string", enum: [...ReportStatusSchema.options] }, publishedAt: { type: "string", nullable: true }, }, required: ["status"], diff --git a/services/api/src/routes/auth.routes.ts b/services/api/src/routes/auth.routes.ts index 0ed181a0..0738a0d8 100644 --- a/services/api/src/routes/auth.routes.ts +++ b/services/api/src/routes/auth.routes.ts @@ -36,7 +36,6 @@ import { route } from "../versioning/route.js" import { parse } from "./_validate.js" import { setCsrfCookie, clearCsrfCookie, type Csrf } from "../auth/csrf.js" import { makeSingleUseSecretStore, type SingleUseSecretStore } from "../auth/single-use-secret.js" -import type { CacheClient } from "../auth/cache.js" import { MEDIA_GET_URL_TTL_SEC } from "../services/media-intake-service.js" import { clientKind, @@ -52,38 +51,32 @@ import type { UserRecord } from "../auth/stores.js" const OAUTH_STATE_COOKIE = "civfix_oauth" const OAUTH_STATE_TTL_SECONDS = 10 * 60 +const OAUTH_NONCE_TTL_SECONDS = 10 * 60 +const OAUTH_NONCE_PREFIX = "oauthnonce:" + +const FORM_URLENCODED = "application/x-www-form-urlencoded" + +const INVALID_OAUTH_STATE_MESSAGE = "Invalid OAuth state." + export const OTP_REQUEST_RATE_LIMIT = perHost({ max: 5, timeWindow: "1 minute" }) export const OTP_VERIFY_RATE_LIMIT = perHost({ max: 10, timeWindow: "1 minute" }) export const OAUTH_RATE_LIMIT = perHost({ max: 20, timeWindow: "1 minute" }) -export function guestSmsEnabledFor(env: Container["env"]): boolean { +function guestSmsEnabledFor(env: Container["env"]): boolean { return env.SMS_GUEST_ENABLED && (env.USE_FAKE_SMS || env.TWILIO_SMS_FROM.length > 0) } -const OAUTH_NONCE_TTL_SECONDS = 10 * 60 -const OAUTH_NONCE_PREFIX = "oauthnonce:" - -function oauthNonces(cache: CacheClient): SingleUseSecretStore { - return makeSingleUseSecretStore(cache, { - prefix: OAUTH_NONCE_PREFIX, - ttlSeconds: OAUTH_NONCE_TTL_SECONDS, - }) -} - -async function mintOAuthNonce( - cache: CacheClient, -): Promise<{ nonce: string; expiresInSeconds: number }> { - const { secret, expiresInSeconds } = await oauthNonces(cache).mint() - return { nonce: secret, expiresInSeconds } -} - -async function redeemOAuthNonce(cache: CacheClient, presented: string): Promise { - const issued = await oauthNonces(cache).redeem(presented) - return issued === null ? null : presented +interface AuthRouteContext { + container: Container + services: AuthServices + csrf: Csrf + guestSmsEnabled: boolean + webOrigins: readonly string[] + oauthNonces: SingleUseSecretStore } async function requireIssuedNonce( - cache: CacheClient, + nonces: SingleUseSecretStore, presented: string | undefined, opts: { required: boolean; log: FastifyBaseLogger }, ): Promise { @@ -97,11 +90,10 @@ async function requireIssuedNonce( ) return undefined } - const redeemed = await redeemOAuthNonce(cache, presented) - if (redeemed === null) { + if ((await nonces.redeem(presented)) === null) { throw AppError.unauthorized("Sign-in nonce is unknown, expired, or already used.") } - return redeemed + return presented } export async function registerAuthRoutes( @@ -109,16 +101,26 @@ export async function registerAuthRoutes( container: Container, ): Promise { const services = app.authServices - const guestSmsEnabled = guestSmsEnabledFor(container.env) - const webOrigins = container.env.WEB_ORIGINS - const csrf = container.csrf - const csrfProtect = csrf.protect - - async function isReservedOrJurisdiction(handle: string): Promise { - if (isReservedHandle(handle)) return true - if (!container.env.DATABASE_URL) return false - return handleCollidesWithJurisdiction(container.getDb().sql, handle) + const ctx: AuthRouteContext = { + container, + services, + csrf: container.csrf, + guestSmsEnabled: guestSmsEnabledFor(container.env), + webOrigins: container.env.WEB_ORIGINS, + oauthNonces: makeSingleUseSecretStore(services.cache, { + prefix: OAUTH_NONCE_PREFIX, + ttlSeconds: OAUTH_NONCE_TTL_SECONDS, + }), } + registerOtpRoutes(app, ctx) + registerNativeOAuthRoutes(app, ctx) + await registerWebOAuthRoutes(app, ctx) + registerSessionRoutes(app, ctx) + registerProfileRoutes(app, ctx) +} + +function registerOtpRoutes(app: FastifyInstance, ctx: AuthRouteContext): void { + const { services, csrf, guestSmsEnabled } = ctx route( app, @@ -142,19 +144,23 @@ export async function registerAuthRoutes( await issueSession(services, csrf, request, reply, userId, { guestSmsEnabled }) }, ) +} + +function registerNativeOAuthRoutes(app: FastifyInstance, ctx: AuthRouteContext): void { + const { container, services, csrf, guestSmsEnabled, oauthNonces } = ctx app.post( "/v1/auth/oauth/nonce", { config: { rateLimit: OAUTH_RATE_LIMIT } }, async (_request, reply) => { - const payload = await mintOAuthNonce(services.cache) - reply.status(200).send(payload) + const { secret, expiresInSeconds } = await oauthNonces.mint() + reply.status(200).send({ nonce: secret, expiresInSeconds }) }, ) route(app, "appleSignIn", { config: { rateLimit: OAUTH_RATE_LIMIT } }, async (request, reply) => { const body = parse(AppleSignInRequestSchema, request.body) - const expectedNonce = await requireIssuedNonce(services.cache, body.nonce, { + const expectedNonce = await requireIssuedNonce(oauthNonces, body.nonce, { required: container.env.OAUTH_REQUIRE_NONCE, log: request.log, }) @@ -172,7 +178,7 @@ export async function registerAuthRoutes( { config: { rateLimit: OAUTH_RATE_LIMIT } }, async (request, reply) => { const body = parse(GoogleSignInRequestSchema, request.body) - const expectedNonce = await requireIssuedNonce(services.cache, body.nonce, { + const expectedNonce = await requireIssuedNonce(oauthNonces, body.nonce, { required: container.env.OAUTH_REQUIRE_NONCE, log: request.log, }) @@ -180,29 +186,23 @@ export async function registerAuthRoutes( await issueSessionForUser(services, csrf, request, reply, user, { guestSmsEnabled }) }, ) +} + +async function registerWebOAuthRoutes(app: FastifyInstance, ctx: AuthRouteContext): Promise { + const { services, csrf, guestSmsEnabled, webOrigins } = ctx route(app, "googleStart", { config: { rateLimit: OAUTH_RATE_LIMIT } }, async (request, reply) => { - const startQuery = parse(OAuthStartQuerySchema, request.query) - if ( - startQuery.redirect !== undefined && - !isAllowedPostLoginRedirect(startQuery.redirect, webOrigins) - ) { - throw AppError.validation({ redirect: "must be an allowed origin or a relative path" }) - } + const redirect = allowedStartRedirect(request, webOrigins) const auth = services.oauth.createGoogleAuthUrl() - const stash: OAuthStash = { - state: auth.state, - codeVerifier: auth.codeVerifier, - ...(startQuery.redirect !== undefined ? { redirect: startQuery.redirect } : {}), - } - reply.setCookie(OAUTH_STATE_COOKIE, JSON.stringify(stash), { - signed: true, - httpOnly: true, - sameSite: "lax", - secure: isProd(), - path: "/", - maxAge: OAUTH_STATE_TTL_SECONDS, - }) + setOAuthStateCookie( + reply, + { + state: auth.state, + codeVerifier: auth.codeVerifier, + ...(redirect !== undefined ? { redirect } : {}), + }, + "lax", + ) reply.redirect(auth.url) }) @@ -215,9 +215,9 @@ export async function registerAuthRoutes( const query = parse(OAuthCallbackQuerySchema, request.query) const stash = readOAuthStash(request) if (!stash || stash.state !== query.state || stash.codeVerifier === undefined) { - throw AppError.unauthorized("Invalid OAuth state.") + throw AppError.unauthorized(INVALID_OAUTH_STATE_MESSAGE) } - reply.clearCookie(OAUTH_STATE_COOKIE, { path: "/" }) + clearOAuthStateCookie(reply) const user = await services.oauth.completeGoogleCallback(query.code, stash.codeVerifier) const target = resolvePostLoginRedirect(stash.redirect, webOrigins) await issueSessionForUser(services, csrf, request, reply, user, { @@ -229,42 +229,22 @@ export async function registerAuthRoutes( ) route(app, "appleStart", { config: { rateLimit: OAUTH_RATE_LIMIT } }, async (request, reply) => { - const startQuery = parse(OAuthStartQuerySchema, request.query) - if ( - startQuery.redirect !== undefined && - !isAllowedPostLoginRedirect(startQuery.redirect, webOrigins) - ) { - throw AppError.validation({ redirect: "must be an allowed origin or a relative path" }) - } + const redirect = allowedStartRedirect(request, webOrigins) const auth = services.oauth.createAppleAuthUrl() - const stash: OAuthStash = { - state: auth.state, - ...(startQuery.redirect !== undefined ? { redirect: startQuery.redirect } : {}), - } - const httpsCrossSite = isProd() - reply.setCookie(OAUTH_STATE_COOKIE, JSON.stringify(stash), { - signed: true, - httpOnly: true, - sameSite: httpsCrossSite ? "none" : "lax", - secure: httpsCrossSite, - path: "/", - maxAge: OAUTH_STATE_TTL_SECONDS, - }) + // Apple returns by a cross-site form POST, which a Lax cookie would not ride along with; SameSite=None + // needs Secure, so plain-http dev keeps Lax. + setOAuthStateCookie( + reply, + { state: auth.state, ...(redirect !== undefined ? { redirect } : {}) }, + isProd() ? "none" : "lax", + ) reply.redirect(auth.url) }) await app.register(async (appleScope) => { - appleScope.addContentTypeParser( - "application/x-www-form-urlencoded", - { parseAs: "string" }, - (_req, body, done) => { - try { - done(null, Object.fromEntries(new URLSearchParams(body as string))) - } catch (err) { - done(err as Error) - } - }, - ) + appleScope.addContentTypeParser(FORM_URLENCODED, { parseAs: "string" }, (_req, body, done) => { + done(null, Object.fromEntries(new URLSearchParams(body as string))) + }) route( appleScope, "appleCallback", @@ -274,9 +254,9 @@ export async function registerAuthRoutes( const body = parse(AppleCallbackBodySchema, request.body) const stash = readOAuthStash(request) if (!stash || stash.state !== body.state) { - throw AppError.unauthorized("Invalid OAuth state.") + throw AppError.unauthorized(INVALID_OAUTH_STATE_MESSAGE) } - reply.clearCookie(OAUTH_STATE_COOKIE, { path: "/" }) + clearOAuthStateCookie(reply) const fullName = appleFullNameFromUserField(body.user) const user = await services.oauth.completeAppleCallback(body.code, fullName) const target = resolvePostLoginRedirect(stash.redirect, webOrigins) @@ -288,16 +268,21 @@ export async function registerAuthRoutes( }, ) }) +} + +function registerSessionRoutes(app: FastifyInstance, ctx: AuthRouteContext): void { + const { services, csrf, guestSmsEnabled } = ctx + const wsTickets = makeWsTicketStore(services.cache) route(app, "session", async (request, reply) => { const payload = await buildSessionCheck(services, csrf, request, reply) - reply.status(200).send({ ...payload, guestSmsEnabled: guestSmsEnabledFor(container.env) }) + reply.status(200).send({ ...payload, guestSmsEnabled }) }) route( app, "logout", - { preHandler: csrfProtect, config: { allowSuspended: true } }, + { preHandler: csrf.protect, config: { allowSuspended: true } }, async (request, reply) => { const token = presentedSessionToken(request) if (token === null) { @@ -311,39 +296,46 @@ export async function registerAuthRoutes( }, ) - route(app, "wsTicket", { preHandler: csrfProtect }, async (request, reply) => { + route(app, "wsTicket", { preHandler: csrf.protect }, async (request, reply) => { const userId = requireAuth(request) const token = presentedSessionToken(request) if (token === null) throw AppError.unauthorized() - const payload = await makeWsTicketStore(services.cache).mint(userId, await sha256Hex(token)) + const payload = await wsTickets.mint(userId, await sha256Hex(token)) reply.status(200).send(payload) }) +} + +function registerProfileRoutes(app: FastifyInstance, ctx: AuthRouteContext): void { + const { container, services, csrf } = ctx + + async function isReservedOrJurisdiction(handle: string): Promise { + if (isReservedHandle(handle)) return true + if (!container.env.DATABASE_URL) return false + return handleCollidesWithJurisdiction(container.getDb().sql, handle) + } + + async function handleAvailability( + userId: string, + handle: string, + ): Promise { + if (!isValidHandle(handle)) return { available: false, reason: "invalid" } + const existing = await services.users.findByHandle(handle.trim()) + if (existing !== null && existing.id === userId) return { available: true, reason: null } + if (await isReservedOrJurisdiction(handle.trim())) + return { available: false, reason: "reserved" } + return existing === null + ? { available: true, reason: null } + : { available: false, reason: "taken" } + } route(app, "checkHandle", async (request, reply) => { const userId = requireAuth(request) const { handle } = parse(HandleAvailableRequestSchema, request.query) - if (!isValidHandle(handle)) { - const payload: HandleAvailableResponse = { available: false, reason: "invalid" } - reply.status(200).send(payload) - return - } - const existing = await services.users.findByHandle(handle.trim()) - if (existing !== null && existing.id === userId) { - const payload: HandleAvailableResponse = { available: true, reason: null } - reply.status(200).send(payload) - return - } - if (await isReservedOrJurisdiction(handle.trim())) { - const payload: HandleAvailableResponse = { available: false, reason: "reserved" } - reply.status(200).send(payload) - return - } - const payload: HandleAvailableResponse = - existing === null ? { available: true, reason: null } : { available: false, reason: "taken" } + const payload = await handleAvailability(userId, handle) reply.status(200).send(payload) }) - route(app, "updateProfile", { preHandler: csrfProtect }, async (request, reply) => { + route(app, "updateProfile", { preHandler: csrf.protect }, async (request, reply) => { const userId = requireAuth(request) const body = parse(UpdateProfileRequestSchema, request.body) assertNoSlur(body.displayName, "displayName") @@ -384,6 +376,36 @@ export async function registerAuthRoutes( }) } +function allowedStartRedirect( + request: FastifyRequest, + webOrigins: readonly string[], +): string | undefined { + const { redirect } = parse(OAuthStartQuerySchema, request.query) + if (redirect !== undefined && !isAllowedPostLoginRedirect(redirect, webOrigins)) { + throw AppError.validation({ redirect: "must be an allowed origin or a relative path" }) + } + return redirect +} + +function setOAuthStateCookie( + reply: FastifyReply, + stash: OAuthStash, + sameSite: "lax" | "none", +): void { + reply.setCookie(OAUTH_STATE_COOKIE, JSON.stringify(stash), { + signed: true, + httpOnly: true, + sameSite, + secure: isProd(), + path: "/", + maxAge: OAUTH_STATE_TTL_SECONDS, + }) +} + +function clearOAuthStateCookie(reply: FastifyReply): void { + reply.clearCookie(OAUTH_STATE_COOKIE, { path: "/" }) +} + interface IssueSessionOptions { forceKind?: ClientKind webRedirectTo?: string @@ -547,7 +569,7 @@ function redirectOnProviderError( if (failure === null) return false const stash = readOAuthStash(request) const own = stash !== null && failure.state !== undefined && stash.state === failure.state - if (own) reply.clearCookie(OAUTH_STATE_COOKIE, { path: "/" }) + if (own) clearOAuthStateCookie(reply) request.log.info({ providerError: failure.error }, "web OAuth sign-in ended at the provider") reply.redirect(resolvePostLoginRedirect(own ? stash.redirect : undefined, webOrigins)) return true diff --git a/services/api/src/routes/health.routes.ts b/services/api/src/routes/health.routes.ts index b3fdd705..6919fc1e 100644 --- a/services/api/src/routes/health.routes.ts +++ b/services/api/src/routes/health.routes.ts @@ -26,6 +26,11 @@ interface ReadyBody { const READY_CACHE_MS = 5_000 +const NO_STORE = "no-store" +const REDIS_PING_REPLY = "PONG" +const HTTP_OK = 200 +const HTTP_SERVICE_UNAVAILABLE = 503 + /** * A header, not a body field, so the public 503 body stays a bare {ok:false} and cross-origin browser JS * cannot read it (no Access-Control-Expose-Headers). It is not a secret: any `curl -I` sees it. The deploy @@ -38,10 +43,10 @@ export async function registerHealthRoutes( container: Container, ): Promise { route(app, "health", async (_request, reply) => { - reply.header("cache-control", "no-store") + reply.header("cache-control", NO_STORE) if (app.lifecycle.isDraining()) { reply.header(DRAINING_HEADER, "1") - reply.status(503) + reply.status(HTTP_SERVICE_UNAVAILABLE) return { ok: false } } return { ok: true, service: SERVICE_NAME } @@ -90,7 +95,7 @@ export async function registerHealthRoutes( try { const redis = container.getRedis() const pong = await redis.ping() - body.checks.redis = pong === "PONG" ? "ok" : "down" + body.checks.redis = pong === REDIS_PING_REPLY ? "ok" : "down" if (body.checks.redis === "down") body.ok = false } catch (err) { app.log.error({ err }, "readyz: redis ping failed") @@ -104,7 +109,7 @@ export async function registerHealthRoutes( app.get("/readyz", async (_request, reply) => { const body = await readiness() - reply.header("cache-control", "no-store") - reply.status(body.ok ? 200 : 503).send(body) + reply.header("cache-control", NO_STORE) + reply.status(body.ok ? HTTP_OK : HTTP_SERVICE_UNAVAILABLE).send(body) }) } diff --git a/services/api/src/routes/query-encoding.ts b/services/api/src/routes/query-encoding.ts index 1d18079d..c148a4b1 100644 --- a/services/api/src/routes/query-encoding.ts +++ b/services/api/src/routes/query-encoding.ts @@ -48,10 +48,9 @@ function csvOrRepeated(element: S) { * build an empty ST_MakeEnvelope and SILENTLY return no rows, so a transposed bbox from a client bug is a * loud 422 here instead. The strict `<` also rejects a zero-area bbox, which can never contain a point. */ -export const BBoxQueryParam = jsonParam(BBoxSchema).refine( - (b) => b.west < b.east && b.south < b.north, - { message: "bbox must satisfy west < east and south < north" }, -) +const BBoxQueryParam = jsonParam(BBoxSchema).refine((b) => b.west < b.east && b.south < b.north, { + message: "bbox must satisfy west < east and south < north", +}) /** * Square degrees. Defense in depth behind each read's own row cap: a world-spanning bbox still makes the DB diff --git a/services/api/src/server.ts b/services/api/src/server.ts index 144d3d70..02a8969f 100644 --- a/services/api/src/server.ts +++ b/services/api/src/server.ts @@ -57,6 +57,11 @@ import { registerCommsJobs } from "./services/host/comms-jobs.js" import { SERVICE_VERSION } from "./version.js" import { makeLifecycle, makeShutdown, REQUEST_TIMEOUT_MS } from "./lifecycle.js" +const API_BODY_LIMIT_BYTES = 256 * 1024 +const SOCKET_CONNECTION_TIMEOUT_MS = 30_000 +const KEEP_ALIVE_TIMEOUT_MS = 5_000 +const LISTEN_HOST = "0.0.0.0" + declare module "fastify" { interface FastifyInstance { container: Container @@ -176,11 +181,11 @@ export async function buildServer(opts: BuildServerOptions = {}): Promise { process.on("SIGTERM", () => void shutdown("SIGTERM")) process.on("SIGINT", () => void shutdown("SIGINT")) - await app.listen({ host: "0.0.0.0", port: env.PORT }) + await app.listen({ host: LISTEN_HOST, port: env.PORT }) app.log.info({ port: env.PORT, env: env.NODE_ENV }, "civfix-api listening") return app } diff --git a/services/api/src/services/anon-repository.drizzle.ts b/services/api/src/services/anon-repository.drizzle.ts index 85a45978..7762dfee 100644 --- a/services/api/src/services/anon-repository.drizzle.ts +++ b/services/api/src/services/anon-repository.drizzle.ts @@ -22,7 +22,7 @@ * after the winner committed, which is a client that lost the first response. */ -import type { Sql } from "../db/client.js" +import type { Sql, TransactionSql } from "../db/client.js" import { generateToken, sha256Hex } from "../auth/crypto.js" import type { AnonReportRepository, @@ -32,7 +32,11 @@ import type { } from "./anon-service.js" import { ANON_REPORT_CREATE_SCOPE } from "./anon-service.js" import { allocateReportReferenceCode } from "../db/reference-code.js" -import type { AnonTokenRecord, AnonTokenStore } from "../abuse/anon-token.js" +import { + ANON_REPORT_CAP_MESSAGE, + type AnonTokenRecord, + type AnonTokenStore, +} from "../abuse/anon-token.js" import type { ClaimRepository, PendingAnonReport } from "./claim-service.js" import { AppError } from "@civfix/shared" import type { AnonReportResponse, ReportStatus } from "@civfix/shared" @@ -41,6 +45,20 @@ import { claimableAsReportMedia, lockUploadsForClaim } from "./media-bindings.js const PG_UNIQUE_VIOLATION = "23505" +const HELD_REVIEW_NOTE = "Awaiting automated review" + +const HELD_REPORT_FLAG = "Held report" + +const HELD_AUTO_ACTION = "Hidden pending review" + +const ANON_REPORTER_LABEL = "Anonymous" + +const MEDIA_UNAVAILABLE_MESSAGE = "One or more media uploads are unavailable." + +const KEY_RACE_MESSAGE = "Report submit is still settling; retry" + +const REPLAY_UNCLAIMABLE_MESSAGE = "This report was already submitted and can no longer be claimed." + function isUniqueViolation(err: unknown): boolean { return ( typeof err === "object" && @@ -89,8 +107,6 @@ function anonTokenStore(sql: Sql): AnonTokenStore { type StoredAnonSnapshot = Omit -const REPLAY_UNCLAIMABLE_MESSAGE = "This report was already submitted and can no longer be claimed." - export interface DrizzleAnonReportRepositoryOptions { newClaimCode?: () => string } @@ -152,26 +168,96 @@ export function makeDrizzleAnonReportRepository( // path takes the counter-row lock before any report or token row lock; that consistent order rules // out an ABBA deadlock. Anon reports still get a code; they simply never auto-forward. const referenceCode = await allocateReportReferenceCode(tx, args.type, args.jurCode) + await consumeTokenQuota(tx, args) + await insertHeldReport(tx, args, referenceCode) + await attachReportMedia(tx, args) + await insertInitialTimeline(tx, args.reportId) + // Every anon report is held pending automated review, so the operator moderation queue surfaces it + // immediately. The same transaction keeps the item and the held report atomic; signals and user + // are enriched later by the media worker or abuse detection. + await insertModerationItem(tx, { + kind: "image", + subjectType: "report", + subjectId: args.reportId, + flag: HELD_REPORT_FLAG, + reason: HELD_REVIEW_NOTE, + category: args.category, + autoAction: HELD_AUTO_ACTION, + reporter: ANON_REPORTER_LABEL, + desc: args.description ?? "", + }) + await persistIdempotencySnapshot(sql, tx, args) + return args.responseSnapshot + }) + return { kind: "created", snapshot } + } catch (err) { + if (isUniqueViolation(err)) { + // The winner of the key race is still in flight to its client with the only live claim code, + // so this request must not rotate it, and the plaintext is not at rest to replay. A retry of + // the same key reaches findIdempotentSnapshot, which rotates for a client that lost that + // response. A key held by a different anon session lands here too (reports.idempotency_key is + // globally unique) and gets the same answer, never that session's report. + throw AppError.conflict(KEY_RACE_MESSAGE) + } + // A cap-reached AppError (or any other) propagates unchanged: the tx already rolled back, so no + // report row, timeline, media-attach, or quota bump persisted. + throw err + } + }, + + async findAnonReportStatus(reportId: string): Promise { + // Read off the report row, not through anon_tokens, which held only the latest submit's code and + // broke older reports. The caller hashes the presented code and compares digests, so no secret is + // read back here. + const rows = await sql< + { + id: string + status: ReportStatus + published_at: Date | null + claim_code_hash: string | null + }[] + >` + SELECT r.id, r.status, r.published_at, r.claim_code_hash + FROM reports r + WHERE r.id = ${reportId} AND r.deleted_at IS NULL + LIMIT 1 + ` + const row = rows[0] + if (!row) return null + return { + reportId: row.id, + status: row.status, + publishedAt: row.published_at, + claimCodeHash: row.claim_code_hash, + } + }, + } +} - // Folding the cap into the WHERE makes check-and-consume a single statement, so concurrent submits - // on one token cannot all pass a stale read and overshoot. Zero rows means the cap is reached, and - // the throw rolls the whole transaction back. - const bumped = await tx<{ report_count: number }[]>` +// Folding the cap into the WHERE makes check-and-consume a single statement, so concurrent submits on one +// token cannot all pass a stale read and overshoot. Zero rows means the cap is reached, and the throw rolls +// the whole transaction back. +async function consumeTokenQuota(tx: TransactionSql, args: CreateAnonReportTxArgs): Promise { + const bumped = await tx<{ report_count: number }[]>` UPDATE anon_tokens SET report_count = report_count + 1 WHERE id = ${args.anonSessionId} AND report_count < ${args.reportCap} RETURNING report_count ` - if (bumped.length === 0) { - throw AppError.rateLimited( - "This anonymous session has reached its report limit. Sign in to continue.", - ) - } + if (bumped.length === 0) { + throw AppError.rateLimited(ANON_REPORT_CAP_MESSAGE) + } +} - // The plaintext claim_code column is written NULL: the code never rests in the database, so a - // dump, backup or replica leak cannot bind anyone else's anonymous report to an account, and the - // column's deferred DROP changes nothing here. - await tx` +// The plaintext claim_code column is written NULL: the code never rests in the database, so a dump, backup +// or replica leak cannot bind anyone else's anonymous report to an account, and the column's deferred DROP +// changes nothing here. +async function insertHeldReport( + tx: TransactionSql, + args: CreateAnonReportTxArgs, + referenceCode: string, +): Promise { + await tx` INSERT INTO reports ( id, reporter_user_id, anon_session_id, idempotency_key, geom, geom_source, jurisdiction_geoid, category, type, title, description, addr, addr_source, addr_precision, @@ -201,15 +287,16 @@ export function makeDrizzleAnonReportRepository( ${null} ) ` +} - // An asset bound to a post, a chat or DM message or any other owner is never re-bindable to a - // report, or the holder of an uploadId could cross-publish private media into a public report - // gallery. One set-based UPDATE rather than a per-id loop, so N photos don't lengthen the - // transaction while it holds the report and token row locks; skipped with no ids because `IN ()` - // is invalid SQL. - if (args.mediaUploadIds.length > 0) { - await lockUploadsForClaim(tx, args.mediaUploadIds) - const claimed = await tx<{ upload_id: string }[]>` +// An asset bound to a post, a chat or DM message or any other owner is never re-bindable to a report, or +// the holder of an uploadId could cross-publish private media into a public report gallery. One set-based +// UPDATE rather than a per-id loop, so N photos don't lengthen the transaction while it holds the report and +// token row locks; skipped with no ids because `IN ()` is invalid SQL. +async function attachReportMedia(tx: TransactionSql, args: CreateAnonReportTxArgs): Promise { + if (args.mediaUploadIds.length === 0) return + await lockUploadsForClaim(tx, args.mediaUploadIds) + const claimed = await tx<{ upload_id: string }[]>` UPDATE media_assets SET report_id = ${args.reportId} WHERE upload_id IN ${tx(args.mediaUploadIds)} @@ -219,48 +306,37 @@ export function makeDrizzleAnonReportRepository( AND (status = 'ready' OR (status = 'validating' AND finalized_at IS NOT NULL)) RETURNING upload_id ` - // Reject the whole submit when any id is unclaimable (unknown, rejected, or already bound - // elsewhere) rather than commit a report with the photo silently missing, the same rule the - // authed create path and post-repository.createPost enforce. The throw also rolls back the - // token quota bump. - if (claimed.length !== new Set(args.mediaUploadIds).size) { - throw AppError.validation({ - mediaUploadIds: "One or more media uploads are unavailable.", - }) - } - } + // Reject the whole submit when any id is unclaimable (unknown, rejected, or already bound elsewhere) + // rather than commit a report with the photo silently missing, the same rule the authed create path and + // post-repository.createPost enforce. The throw also rolls back the token quota bump. + if (claimed.length !== new Set(args.mediaUploadIds).size) { + throw AppError.validation({ mediaUploadIds: MEDIA_UNAVAILABLE_MESSAGE }) + } +} - // Both rows would share now() (constant within a transaction) and report_timeline.id is a random - // uuid, so ORDER BY created_at, id has no stable tiebreaker; held is stamped 1ms later so every - // reader sees submitted first. - await tx` +// Both rows would share now() (constant within a transaction) and report_timeline.id is a random uuid, so +// ORDER BY created_at, id has no stable tiebreaker; held is stamped 1ms later so every reader sees +// submitted first. +async function insertInitialTimeline(tx: TransactionSql, reportId: string): Promise { + await tx` INSERT INTO report_timeline (report_id, status, note, actor_id, created_at) VALUES - (${args.reportId}, ${"submitted"}, ${null}, ${null}, now()), - (${args.reportId}, ${"held"}, ${"Awaiting automated review"}, ${null}, now() + interval '1 millisecond') + (${reportId}, ${"submitted"}, ${null}, ${null}, now()), + (${reportId}, ${"held"}, ${HELD_REVIEW_NOTE}, ${null}, now() + interval '1 millisecond') ` +} - // Every anon report is held pending automated review, so the operator moderation queue surfaces it - // immediately. The same transaction keeps the item and the held report atomic; signals and user - // are enriched later by the media worker or abuse detection. - await insertModerationItem(tx, { - kind: "image", - subjectType: "report", - subjectId: args.reportId, - flag: "Held report", - reason: "Awaiting automated review", - category: args.category, - autoAction: "Hidden pending review", - reporter: "Anonymous", - desc: args.description ?? "", - }) - - // Stored without the plaintext claim code: a replay mints a fresh one instead. - const storedSnapshot: StoredAnonSnapshot = { - reportId: args.responseSnapshot.reportId, - status: args.responseSnapshot.status, - } - await tx` +// Stored without the plaintext claim code: a replay mints a fresh one instead. +async function persistIdempotencySnapshot( + sql: Sql, + tx: TransactionSql, + args: CreateAnonReportTxArgs, +): Promise { + const storedSnapshot: StoredAnonSnapshot = { + reportId: args.responseSnapshot.reportId, + status: args.responseSnapshot.status, + } + await tx` INSERT INTO idempotency_keys (key, scope, user_or_anon, response_snapshot) VALUES ( ${args.idempotencyKey}, @@ -269,51 +345,6 @@ export function makeDrizzleAnonReportRepository( ${sql.json(storedSnapshot as Parameters[0])} ) ` - return args.responseSnapshot - }) - return { kind: "created", snapshot } - } catch (err) { - if (isUniqueViolation(err)) { - // The winner of the key race is still in flight to its client with the only live claim code, - // so this request must not rotate it, and the plaintext is not at rest to replay. A retry of - // the same key reaches findIdempotentSnapshot, which rotates for a client that lost that - // response. A key held by a different anon session lands here too (reports.idempotency_key is - // globally unique) and gets the same answer, never that session's report. - throw AppError.conflict("Report submit is still settling; retry") - } - // A cap-reached AppError (or any other) propagates unchanged: the tx already rolled back, so no - // report row, timeline, media-attach, or quota bump persisted. - throw err - } - }, - - async findAnonReportStatus(reportId: string): Promise { - // Read off the report row, not through anon_tokens, which held only the latest submit's code and - // broke older reports. The caller hashes the presented code and compares digests, so no secret is - // read back here. - const rows = await sql< - { - id: string - status: ReportStatus - published_at: Date | null - claim_code_hash: string | null - }[] - >` - SELECT r.id, r.status, r.published_at, r.claim_code_hash - FROM reports r - WHERE r.id = ${reportId} AND r.deleted_at IS NULL - LIMIT 1 - ` - const row = rows[0] - if (!row) return null - return { - reportId: row.id, - status: row.status, - publishedAt: row.published_at, - claimCodeHash: row.claim_code_hash, - } - }, - } } export function makeDrizzleClaimRepository(sql: Sql): ClaimRepository { diff --git a/services/api/src/services/anon-service.ts b/services/api/src/services/anon-service.ts index 6a999e41..cdf3f05c 100644 --- a/services/api/src/services/anon-service.ts +++ b/services/api/src/services/anon-service.ts @@ -33,6 +33,7 @@ import { addressProvenance, resolveAddressOrNull, type AddressResolver, + type ReportAddressWrite, } from "./address-resolver.js" import type { AddressPrecision, ReportAddressSource } from "@civfix/shared" @@ -42,6 +43,12 @@ export const ANON_TURNSTILE_ACTION = "anon-report" export const ANON_MAX_MEDIA_UPLOADS = 5 +const HONEYPOT_REJECTION_MESSAGE = "Please review your report and try again." + +const REPORT_NOT_FOUND_MESSAGE = "Report not found" + +export type AnonLog = (line: string, extra?: Record) => void + export type AnonAbuseReason = "honeypot" | "gps" export interface CreateAnonReportTxArgs { @@ -110,12 +117,11 @@ export interface AnonServiceDeps { subjectId: string, reason: AnonAbuseReason, ) => Promise - enqueueMediaChecks?: (reportId: string, mediaUploadIds: string[]) => Promise newId?: () => string newClaimCode?: () => string now?: () => Date newAnonTokenId?: () => string - log?: (line: string, extra?: Record) => void + log?: AnonLog } export interface AnonSubmitResult { @@ -134,7 +140,6 @@ export function makeAnonService(deps: AnonServiceDeps): AnonService { const newClaimCode = deps.newClaimCode ?? (() => generateToken()) const log = deps.log ?? (() => {}) const raiseAbuseFlag = deps.raiseAbuseFlag ?? (() => Promise.resolve()) - const enqueueMediaChecks = deps.enqueueMediaChecks ?? (() => Promise.resolve()) const tokenDeps: AnonTokenDeps = { store: deps.repo, @@ -143,30 +148,82 @@ export function makeAnonService(deps: AnonServiceDeps): AnonService { ...(deps.newAnonTokenId !== undefined ? { newId: deps.newAnonTokenId } : {}), } + async function rejectBots(input: AnonReportRequest, ctx: AnonSubmitContext): Promise { + const human = await deps.abuseChecks.verifyTurnstile(input.turnstileToken, ctx.ip ?? "", { + action: ANON_TURNSTILE_ACTION, + }) + if (!human) { + throw AppError.turnstileFailed() + } + + if (!honeypotTripped(input.honeypot)) return + const presentedId = input.anonToken + ? verifyAnonTokenSignature(input.anonToken, deps.anonTokenSigningKey) + : null + if (presentedId) { + await raiseAbuseFlag("anon_token", presentedId, "honeypot").catch((err) => { + log("anon-submit: honeypot raiseAbuseFlag failed (non-fatal)", { err: String(err) }) + }) + } + log("anon-submit: honeypot tripped; rejecting", { ip: ctx.ip }) + throw AppError.validation({ description: HONEYPOT_REJECTION_MESSAGE }) + } + + async function enforceContentAndQuotas( + input: AnonReportRequest, + ctx: AnonSubmitContext, + ): Promise { + if (input.mediaUploadIds.length > ANON_MAX_MEDIA_UPLOADS) { + throw AppError.validation({ mediaUploadIds: "Too many media uploads." }) + } + + assertNoSlur(input.title ?? null, "title") + assertNoSlur(input.description ?? null, "description") + assertNoSlur(input.addr ?? null, "addr") + + await enforceIpRateLimit(ctx.ip, { counters: deps.counters }) + + await enforceH3CellCap(input.lat, input.lng, { counters: deps.counters }) + + const ipGeo = resolveTrustedCfGeo(ctx, log) + const gps = await gpsSanityCheck( + { point: { lat: input.lat, lng: input.lng }, ipGeo }, + { abuseChecks: deps.abuseChecks, log }, + ) + if (!gps.ok) { + throw AppError.gpsImplausible() + } + } + + async function resolveLocation(input: AnonReportRequest): Promise<{ + jurisdictionGeoid: string | null + jurCode: number + addressWrite: ReportAddressWrite + }> { + const suppliedAddr = input.addr?.trim() ?? "" + const [jurisdictionGeoid, resolvedAddr] = await Promise.all([ + deps.resolveJurisdictionGeoid(input.lat, input.lng), + suppliedAddr.length > 0 + ? Promise.resolve(null) + : resolveAddressOrNull(deps.resolveAddress, input.lat, input.lng), + ]) + const jurCode = + deps.resolveJurisdictionCode !== undefined + ? await deps.resolveJurisdictionCode(jurisdictionGeoid) + : UNKNOWN_JURCODE + return { + jurisdictionGeoid, + jurCode, + addressWrite: addressProvenance(suppliedAddr, resolvedAddr), + } + } + return { async submitAnonReport( input: AnonReportRequest, ctx: AnonSubmitContext, ): Promise { - const human = await deps.abuseChecks.verifyTurnstile(input.turnstileToken, ctx.ip ?? "", { - action: ANON_TURNSTILE_ACTION, - }) - if (!human) { - throw AppError.turnstileFailed() - } - - if (honeypotTripped(input.honeypot)) { - if (input.anonToken) { - const presentedId = verifyAnonTokenSignature(input.anonToken, deps.anonTokenSigningKey) - if (presentedId) { - await raiseAbuseFlag("anon_token", presentedId, "honeypot").catch((err) => { - log("anon-submit: honeypot raiseAbuseFlag failed (non-fatal)", { err: String(err) }) - }) - } - } - log("anon-submit: honeypot tripped; rejecting", { ip: ctx.ip }) - throw AppError.validation({ description: "Please review your report and try again." }) - } + await rejectBots(input, ctx) const presentedToken = await resolveAnonToken(input.anonToken, tokenDeps) @@ -179,45 +236,15 @@ export function makeAnonService(deps: AnonServiceDeps): AnonService { return { response: existing } } - if (input.mediaUploadIds.length > ANON_MAX_MEDIA_UPLOADS) { - throw AppError.validation({ mediaUploadIds: "Too many media uploads." }) - } - - assertNoSlur(input.title ?? null, "title") - assertNoSlur(input.description ?? null, "description") - assertNoSlur(input.addr ?? null, "addr") - - await enforceIpRateLimit(ctx.ip, { counters: deps.counters }) - - await enforceH3CellCap(input.lat, input.lng, { counters: deps.counters }) - - const ipGeo = resolveTrustedCfGeo(ctx, log) - const gps = await gpsSanityCheck( - { point: { lat: input.lat, lng: input.lng }, ipGeo }, - { abuseChecks: deps.abuseChecks, log }, - ) - if (!gps.ok) { - throw AppError.gpsImplausible() - } + await enforceContentAndQuotas(input, ctx) const { record: tokenRow, issuedToken } = await ensureAnonToken(presentedToken, tokenDeps) - const suppliedAddr = input.addr?.trim() ?? "" - const [jurisdictionGeoid, resolvedAddr] = await Promise.all([ - deps.resolveJurisdictionGeoid(input.lat, input.lng), - suppliedAddr.length > 0 - ? Promise.resolve(null) - : resolveAddressOrNull(deps.resolveAddress, input.lat, input.lng), - ]) - const jurCode = - deps.resolveJurisdictionCode !== undefined - ? await deps.resolveJurisdictionCode(jurisdictionGeoid) - : UNKNOWN_JURCODE + const { jurisdictionGeoid, jurCode, addressWrite } = await resolveLocation(input) const h3Cell = reportH3Cell(input.lat, input.lng) const reportId = newId() const claimCode = newClaimCode() const claimCodeHash = await sha256Hex(claimCode) - const addressWrite = addressProvenance(suppliedAddr, resolvedAddr) const responseSnapshot: AnonReportResponse = { reportId, @@ -249,15 +276,6 @@ export function makeAnonService(deps: AnonServiceDeps): AnonService { responseSnapshot, }) - if (result.kind === "created" && input.mediaUploadIds.length > 0) { - await enqueueMediaChecks(reportId, input.mediaUploadIds).catch((err) => { - log("anon-submit: media.checks enqueue failed (non-fatal)", { - reportId, - err: String(err), - }) - }) - } - return { response: result.snapshot, ...(result.kind === "created" && issuedToken !== undefined @@ -269,10 +287,10 @@ export function makeAnonService(deps: AnonServiceDeps): AnonService { async anonReportStatus(reportId: string, claimCode: string): Promise { const row = await deps.repo.findAnonReportStatus(reportId) if (!row || row.claimCodeHash === null) { - throw AppError.notFound("Report not found") + throw AppError.notFound(REPORT_NOT_FOUND_MESSAGE) } if (!constantTimeStringEqual(row.claimCodeHash, await sha256Hex(claimCode))) { - throw AppError.notFound("Report not found") + throw AppError.notFound(REPORT_NOT_FOUND_MESSAGE) } return { status: row.status, @@ -296,10 +314,7 @@ function anonMediaUploaders( return [...sessions.map(anonUploader), UNSESSIONED_UPLOADER] } -function resolveTrustedCfGeo( - ctx: AnonSubmitContext, - log: (line: string, extra?: Record) => void, -): LatLng | null { +function resolveTrustedCfGeo(ctx: AnonSubmitContext, log: AnonLog): LatLng | null { if (!ctx.cfGeo) return null if (!ctx.cfGeoTrusted) { const parsed = parseCfGeo(ctx.cfGeo) diff --git a/services/api/src/versioning/policy.ts b/services/api/src/versioning/policy.ts index 53d21a62..f9390306 100644 --- a/services/api/src/versioning/policy.ts +++ b/services/api/src/versioning/policy.ts @@ -15,19 +15,19 @@ export interface VersionStatus { readonly sunset?: string } -export const SERVED_VERSIONS: readonly ApiVersion[] = ["v1"] as const +const SERVED_VERSIONS: readonly ApiVersion[] = ["v1"] as const -export const MIN_SUPPORTED_VERSION: ApiVersion = "v1" +const MIN_SUPPORTED_VERSION: ApiVersion = "v1" /** `Record` forces a compile error here when the contract gains a version with no policy. */ -export const VERSION_STATUS: Record = { +const VERSION_STATUS: Record = { v1: { status: "current" }, } as const -const VERSION_SEGMENT_RE = /^v\d+$/ +const VERSION_SEGMENT_RE = /^v(\d+)$/ function majorOf(seg: string): number | null { - const match = /^v(\d+)$/.exec(seg) + const match = VERSION_SEGMENT_RE.exec(seg) if (!match) return null return Number.parseInt(match[1]!, 10) } diff --git a/services/api/src/versioning/version-gate.ts b/services/api/src/versioning/version-gate.ts index 3d09e847..7b5da245 100644 --- a/services/api/src/versioning/version-gate.ts +++ b/services/api/src/versioning/version-gate.ts @@ -8,6 +8,9 @@ import { versionStatus, } from "./policy.js" +const DEPRECATION_HEADER = "Deprecation" +const SUNSET_HEADER = "Sunset" + function firstPathSegment(url: string): string { const queryStart = url.indexOf("?") const path = queryStart === -1 ? url : url.slice(0, queryStart) @@ -54,11 +57,11 @@ export async function registerVersionGate(app: FastifyInstance): Promise { case "current": return case "deprecated": - reply.header("Deprecation", "true") + reply.header(DEPRECATION_HEADER, "true") if (status.sunset) { const sunsetDate = new Date(status.sunset) if (!Number.isNaN(sunsetDate.getTime())) { - reply.header("Sunset", sunsetDate.toUTCString()) + reply.header(SUNSET_HEADER, sunsetDate.toUTCString()) } } return