From 4c89ad4a28531c21f3463051bcd455b4219bf44c Mon Sep 17 00:00:00 2001 From: Volodymyr Zakhovaiko Date: Tue, 13 Oct 2026 00:56:20 +0200 Subject: [PATCH 01/13] fix(core): redeliver failed events, dead-letter, dedup handlers and claim outbox rows atomically The Redis Streams broker acks an entry only once every local handler succeeded, reclaims idle pending entries with XAUTOCLAIM and dead-letters to oss:dlq: after 5 deliveries. Streams trim by MINID (7 days) instead of MAXLEN. The outbox is always bound; the relay claims rows with a leased UPDATE ... SKIP LOCKED, releases a failing row without blocking the batch and prunes old rows. processed_events dedups each plugin handler by (event_id, consumer). Known event payloads are validated on emit and on receive. --- packages/core/src/server/db/index.ts | 5 +- .../db/outbox/__tests__/relay.int.test.ts | 91 ++++++- .../migrations/0001_event_delivery.sql | 9 + .../migrations/meta/0001_snapshot.json | 176 +++++++++++++ .../drizzle/migrations/meta/_journal.json | 7 + .../src/server/db/outbox/processed-events.ts | 22 ++ packages/core/src/server/db/outbox/relay.ts | 152 +++++++---- packages/core/src/server/db/outbox/schema.ts | 44 +++- .../kernel/__tests__/event-bus.int.test.ts | 244 +++++++++++++----- .../redis-streams-broker.int.test.ts | 78 +++++- packages/core/src/server/kernel/event-bus.ts | 115 ++++++--- packages/core/src/server/kernel/index.ts | 10 +- .../src/server/kernel/redis-streams-broker.ts | 131 ++++++++-- .../__tests__/module-registry.test.ts | 7 +- .../src/server/plugin-host/define-plugin.ts | 8 +- packages/core/src/server/plugin-host/index.ts | 1 + .../src/server/plugin-host/module-registry.ts | 9 +- .../core/src/server/runtime/create-app.ts | 62 +++-- .../wallet-balance-changed.plugin.int.test.ts | 4 +- 19 files changed, 929 insertions(+), 246 deletions(-) create mode 100644 packages/core/src/server/db/outbox/drizzle/migrations/0001_event_delivery.sql create mode 100644 packages/core/src/server/db/outbox/drizzle/migrations/meta/0001_snapshot.json create mode 100644 packages/core/src/server/db/outbox/processed-events.ts diff --git a/packages/core/src/server/db/index.ts b/packages/core/src/server/db/index.ts index 0789de3ee..92a7d0799 100644 --- a/packages/core/src/server/db/index.ts +++ b/packages/core/src/server/db/index.ts @@ -25,9 +25,10 @@ export { sumInPivot, } from './query-helpers.js'; -// Transactional outbox - writer binds atomically with the state change; relay publishes to MESSAGE_BROKER. See ADR-0016. -export { eventOutbox, type EventOutboxRow } from './outbox/schema.js'; +// Transactional outbox - writer binds atomically with the state change; relay publishes to MESSAGE_BROKER. See ADR-0016/0044. +export { eventOutbox, processedEvents, type EventOutboxRow } from './outbox/schema.js'; export { DrizzleOutboxWriter } from './outbox/writer.js'; +export { DrizzleProcessedEventLog } from './outbox/processed-events.js'; export { OutboxRelay, type OutboxRelayOptions } from './outbox/relay.js'; // @openora/core/server/orm is a framework-free leaf so drizzle-kit can bundle schemas without diff --git a/packages/core/src/server/db/outbox/__tests__/relay.int.test.ts b/packages/core/src/server/db/outbox/__tests__/relay.int.test.ts index 34b7081ec..ebdeeb4ab 100644 --- a/packages/core/src/server/db/outbox/__tests__/relay.int.test.ts +++ b/packages/core/src/server/db/outbox/__tests__/relay.int.test.ts @@ -5,7 +5,8 @@ import type { MessageBrokerAdapter, EventEnvelope } from '@openora/core/contract import { createTestDb, type TestDb } from '@openora/core/testing'; import { migrate } from '@openora/core/server/migrate'; import { OutboxRelay } from '../relay.js'; -import { eventOutbox } from '../schema.js'; +import { eventOutbox, processedEvents } from '../schema.js'; +import { EVENT_RETENTION_MS } from '../../../kernel/event-bus.js'; let db: TestDb; @@ -47,7 +48,9 @@ afterAll(async () => { }); beforeEach(async () => { - await db.drizzle.db.execute(sql`TRUNCATE ${eventOutbox} RESTART IDENTITY CASCADE`); + await db.drizzle.db.execute( + sql`TRUNCATE ${eventOutbox}, ${processedEvents} RESTART IDENTITY CASCADE`, + ); }); describe('OutboxRelay.drainOnce (real PG)', () => { @@ -86,7 +89,7 @@ describe('OutboxRelay.drainOnce (real PG)', () => { expect(broker.publish).not.toHaveBeenCalled(); }); - it('leaves the row pending when broker.publish throws', async () => { + it('leaves the row pending and its lease released when broker.publish throws', async () => { const row = await seedRow(); const broker: MessageBrokerAdapter = { publish: vi.fn(async () => { @@ -95,18 +98,22 @@ describe('OutboxRelay.drainOnce (real PG)', () => { subscribe: () => () => {}, close: async () => {}, }; - const relay = new OutboxRelay(db.drizzle.db, broker); + const errors: unknown[] = []; + const relay = new OutboxRelay(db.drizzle.db, broker, { onError: (err) => errors.push(err) }); - await expect(relay.drainOnce()).rejects.toThrow('broker unreachable'); + await expect(relay.drainOnce()).resolves.toBe(0); const [after] = await db.drizzle.db .select() .from(eventOutbox) .where(eq(eventOutbox.eventId, row.eventId)); expect(after?.publishedAt).toBeNull(); + expect(after?.claimedAt).toBeNull(); + expect(after?.attempts).toBe(1); + expect(errors[0]).toMatchObject({ message: 'broker unreachable' }); }); - it('keeps rows published before a mid-batch publish failure, retrying only the failing row', async () => { + it('does not let a failing row block the rest of the batch', async () => { const a = await seedRow({ occurredAt: new Date('2026-01-01T00:00:00.000Z') }); const b = await seedRow({ occurredAt: new Date('2026-01-01T00:00:01.000Z') }); const c = await seedRow({ occurredAt: new Date('2026-01-01T00:00:02.000Z') }); @@ -115,26 +122,86 @@ describe('OutboxRelay.drainOnce (real PG)', () => { throw new Error('broker unreachable'); } }); - const relay = new OutboxRelay(db.drizzle.db, broker); + const relay = new OutboxRelay(db.drizzle.db, broker, { onError: () => undefined }); - await expect(relay.drainOnce()).rejects.toThrow('broker unreachable'); + await expect(relay.drainOnce()).resolves.toBe(2); const afterFailure = await rowsById(); expect(afterFailure.get(a.eventId)?.publishedAt).not.toBeNull(); expect(afterFailure.get(b.eventId)?.publishedAt).toBeNull(); - expect(afterFailure.get(c.eventId)?.publishedAt).toBeNull(); + expect(afterFailure.get(c.eventId)?.publishedAt).not.toBeNull(); const publishSpy = broker.publish as ReturnType; publishSpy.mockClear(); publishSpy.mockImplementation(async () => undefined); const n = await relay.drainOnce(); - expect(n).toBe(2); - expect(publishSpy).toHaveBeenCalledTimes(2); - expect(publishSpy.mock.calls.map((call) => call[0].eventId)).toEqual([b.eventId, c.eventId]); + expect(n).toBe(1); + expect(publishSpy.mock.calls.map((call) => call[0].eventId)).toEqual([b.eventId]); const afterRetry = await rowsById(); expect([...afterRetry.values()].every((r) => r.publishedAt !== null)).toBe(true); }); + + it('skips a row claimed by another relay until its lease expires', async () => { + const fresh = await seedRow({ claimedAt: new Date() }); + const stale = await seedRow({ claimedAt: new Date(Date.now() - 60_000) }); + const broker = brokerThat(); + const relay = new OutboxRelay(db.drizzle.db, broker); + + const n = await relay.drainOnce(); + + expect(n).toBe(1); + expect((broker.publish as ReturnType).mock.calls[0][0].eventId).toBe( + stale.eventId, + ); + expect((await rowsById()).get(fresh.eventId)?.publishedAt).toBeNull(); + }); + + it('publishes each row exactly once across two concurrent relays', async () => { + const seeded = await Promise.all(Array.from({ length: 50 }, () => seedRow())); + const published: string[] = []; + // A slow publish holds each relay's claim open so the two drains overlap. + const slowBroker: MessageBrokerAdapter = { + publish: async (envelope) => { + await new Promise((r) => setTimeout(r, 2)); + published.push(envelope.eventId); + }, + subscribe: () => () => {}, + close: async () => {}, + }; + const relayA = new OutboxRelay(db.drizzle.db, slowBroker, { batchSize: 30 }); + const relayB = new OutboxRelay(db.drizzle.db, slowBroker, { batchSize: 30 }); + + const counts: number[] = []; + for (let i = 0; i < 3; i += 1) { + counts.push(...(await Promise.all([relayA.drainOnce(), relayB.drainOnce()]))); + } + + expect(counts.reduce((sum, n) => sum + n, 0)).toBe(seeded.length); + expect(new Set(published).size).toBe(published.length); + expect(new Set(published)).toEqual(new Set(seeded.map((r) => r.eventId))); + }); + + it('prunes published rows and processed_events past retention, keeping the rest', async () => { + const longAgo = new Date(Date.now() - EVENT_RETENTION_MS - 60_000); + const oldPublished = await seedRow({ publishedAt: longAgo }); + const recentPublished = await seedRow({ publishedAt: new Date() }); + const oldPending = await seedRow({ occurredAt: longAgo }); + await db.drizzle.db.insert(processedEvents).values([ + { eventId: randomUUID(), consumer: 'old', processedAt: longAgo }, + { eventId: randomUUID(), consumer: 'recent' }, + ]); + const relay = new OutboxRelay(db.drizzle.db, brokerThat()); + + await relay.drainOnce(); + + const remaining = await rowsById(); + expect(remaining.has(oldPublished.eventId)).toBe(false); + expect(remaining.has(recentPublished.eventId)).toBe(true); + expect(remaining.get(oldPending.eventId)?.publishedAt).not.toBeNull(); + const consumers = await db.drizzle.db.select().from(processedEvents); + expect(consumers.map((r) => r.consumer)).toEqual(['recent']); + }); }); function deferred() { diff --git a/packages/core/src/server/db/outbox/drizzle/migrations/0001_event_delivery.sql b/packages/core/src/server/db/outbox/drizzle/migrations/0001_event_delivery.sql new file mode 100644 index 000000000..d3a99e57b --- /dev/null +++ b/packages/core/src/server/db/outbox/drizzle/migrations/0001_event_delivery.sql @@ -0,0 +1,9 @@ +CREATE TABLE "processed_events" ( + "event_id" text NOT NULL, + "consumer" text NOT NULL, + "processed_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "processed_events_event_id_consumer_pk" PRIMARY KEY("event_id","consumer") +); +--> statement-breakpoint +ALTER TABLE "event_outbox" ADD COLUMN "claimed_at" timestamp with time zone;--> statement-breakpoint +CREATE INDEX "processed_events_processed_at_idx" ON "processed_events" USING btree ("processed_at"); \ No newline at end of file diff --git a/packages/core/src/server/db/outbox/drizzle/migrations/meta/0001_snapshot.json b/packages/core/src/server/db/outbox/drizzle/migrations/meta/0001_snapshot.json new file mode 100644 index 000000000..7e3fefc96 --- /dev/null +++ b/packages/core/src/server/db/outbox/drizzle/migrations/meta/0001_snapshot.json @@ -0,0 +1,176 @@ +{ + "id": "b492b0f0-12ad-4b89-adfe-7d8e0fdc5b6c", + "prevId": "8e20a713-5cd3-423e-8eea-8d2bd3d9832c", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.event_outbox": { + "name": "event_outbox", + "schema": "", + "columns": { + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "topic": { + "name": "topic", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "schema_version": { + "name": "schema_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "trace_id": { + "name": "trace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ordering_key": { + "name": "ordering_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "event_outbox_pending_idx": { + "name": "event_outbox_pending_idx", + "columns": [ + { + "expression": "published_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.processed_events": { + "name": "processed_events", + "schema": "", + "columns": { + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "consumer": { + "name": "consumer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "processed_events_processed_at_idx": { + "name": "processed_events_processed_at_idx", + "columns": [ + { + "expression": "processed_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "processed_events_event_id_consumer_pk": { + "name": "processed_events_event_id_consumer_pk", + "columns": ["event_id", "consumer"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/core/src/server/db/outbox/drizzle/migrations/meta/_journal.json b/packages/core/src/server/db/outbox/drizzle/migrations/meta/_journal.json index f00968884..d8b045239 100644 --- a/packages/core/src/server/db/outbox/drizzle/migrations/meta/_journal.json +++ b/packages/core/src/server/db/outbox/drizzle/migrations/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1783555226979, "tag": "0000_living_snowbird", "breakpoints": true + }, + { + "idx": 1, + "version": "7", + "when": 1791843721840, + "tag": "0001_event_delivery", + "breakpoints": true } ] } diff --git a/packages/core/src/server/db/outbox/processed-events.ts b/packages/core/src/server/db/outbox/processed-events.ts new file mode 100644 index 000000000..a7fea655c --- /dev/null +++ b/packages/core/src/server/db/outbox/processed-events.ts @@ -0,0 +1,22 @@ +import { and, eq } from 'drizzle-orm'; +import type { ProcessedEventLog } from '../../kernel/event-bus.js'; +import { processedEvents } from './schema.js'; +import type { DrizzleDb } from '../drizzle.js'; + +/** Postgres-backed `ProcessedEventLog`; rows are pruned by the `OutboxRelay`. */ +export class DrizzleProcessedEventLog implements ProcessedEventLog { + constructor(private readonly db: DrizzleDb) {} + + async has(eventId: string, consumer: string): Promise { + const [row] = await this.db + .select({ eventId: processedEvents.eventId }) + .from(processedEvents) + .where(and(eq(processedEvents.eventId, eventId), eq(processedEvents.consumer, consumer))) + .limit(1); + return row !== undefined; + } + + async record(eventId: string, consumer: string): Promise { + await this.db.insert(processedEvents).values({ eventId, consumer }).onConflictDoNothing(); + } +} diff --git a/packages/core/src/server/db/outbox/relay.ts b/packages/core/src/server/db/outbox/relay.ts index 2ebb9794a..8eb754899 100644 --- a/packages/core/src/server/db/outbox/relay.ts +++ b/packages/core/src/server/db/outbox/relay.ts @@ -1,32 +1,55 @@ import type { MessageBrokerAdapter, EventEnvelope } from '@openora/core/contracts'; -import { asc, eq, isNull } from 'drizzle-orm'; -import { eventOutbox } from './schema.js'; +import { and, asc, eq, inArray, isNull, lt, or, sql } from 'drizzle-orm'; +import { EVENT_RETENTION_MS } from '../../kernel/event-bus.js'; +import { eventOutbox, processedEvents, type EventOutboxRow } from './schema.js'; import type { DrizzleDb } from '../drizzle.js'; +// A claimed row whose relay died mid-publish becomes claimable again after this. +const CLAIM_LEASE_MS = 30_000; +const PRUNE_INTERVAL_MS = 60 * 60_000; + export type OutboxRelayOptions = { - intervalMs?: number; // poll cadence; default 1000 + intervalMs?: number; // poll cadence; default 200 batchSize?: number; // rows per drain; default 100 onError?: (err: unknown) => void; }; +function toEnvelope(row: EventOutboxRow): EventEnvelope { + return { + eventId: row.eventId, + topic: row.topic, + payload: row.payload, + occurredAt: row.occurredAt.toISOString(), + schemaVersion: row.schemaVersion, + ...(row.traceId ? { traceId: row.traceId } : {}), + ...(row.orderingKey ? { orderingKey: row.orderingKey } : {}), + }; +} + /** * Publishes pending `event_outbox` rows to the `MESSAGE_BROKER`. Runs as a * background poll loop (`start`/`stop`) and is also drainable on demand - * (`drainOnce`) for tests or a synchronous flush. A row is marked published - * only AFTER `broker.publish` resolves, so a crash mid-publish leaves it - * pending and it is retried - at-least-once delivery; consumers dedup on - * `eventId`. + * (`drainOnce`) for tests or a synchronous flush. + * + * Each drain atomically claims a batch in one `UPDATE ... WHERE event_id IN + * (SELECT ... FOR UPDATE SKIP LOCKED) RETURNING`, stamping `claimed_at`, so + * concurrent relays (one per replica) never claim the same row inside its lease. + * A row is marked published only AFTER `broker.publish` resolves; a failing row + * is reported to `onError`, its lease released for the next tick, and the rest of + * the batch carries on. A relay that dies mid-publish leaves its rows claimed until + * the lease expires, then another relay republishes them - at-least-once delivery, + * deduped downstream by `processed_events`. * - * ponytail: claim-then-publish (no `claimed_at` migration) - the SKIP LOCKED - * select commits in its own short transaction, then each row publishes and is - * marked in its own transaction outside that lock, so a publish failure only - * retries the failing row. The tradeoff: two relays on the same tick can both - * grab a row - fine under at-least-once (consumers dedup on `eventId`); add a - * `claimed_at` claim-update if that ever needs closing. + * The same loop prunes published outbox rows and `processed_events` rows older + * than `EVENT_RETENTION_MS`, at most once an hour per replica. + * + * ponytail: polls every 200ms per replica (an indexed, usually-empty claim). Swap + * the poll for LISTEN/NOTIFY if the idle query load or the publish latency matters. */ export class OutboxRelay { private timer: ReturnType | undefined; private draining = false; + private lastPruneAt = 0; constructor( private readonly db: DrizzleDb, @@ -38,7 +61,7 @@ export class OutboxRelay { if (this.timer) { return; } - this.timer = setInterval(() => void this.drainSafe(), this.opts.intervalMs ?? 1000); + this.timer = setInterval(() => void this.drainOnce(), this.opts.intervalMs ?? 200); this.timer.unref?.(); } @@ -52,53 +75,84 @@ export class OutboxRelay { } } - private async drainSafe(): Promise { - try { - await this.drainOnce(); - } catch (err) { - this.opts.onError?.(err); - } - } - - // Publish one batch of pending rows. Returns how many were published. Reentrant- - // safe: a second call while a drain is in flight is a no-op. + // Publish one claimed batch. Returns how many rows were published. Reentrant-safe: + // a second call while a drain is in flight is a no-op. Never throws - every failure + // goes to `onError`. async drainOnce(): Promise { if (this.draining) { return 0; } this.draining = true; try { - const batch = await this.db.transaction((txn) => - txn - .select() - .from(eventOutbox) - .where(isNull(eventOutbox.publishedAt)) - .orderBy(asc(eventOutbox.occurredAt)) - .limit(this.opts.batchSize ?? 100) - .for('update', { skipLocked: true }), - ); - + await this.pruneIfDue(); + const batch = await this.claimBatch(); let published = 0; for (const row of batch) { - const envelope: EventEnvelope = { - eventId: row.eventId, - topic: row.topic, - payload: row.payload, - occurredAt: row.occurredAt.toISOString(), - schemaVersion: row.schemaVersion, - ...(row.traceId ? { traceId: row.traceId } : {}), - ...(row.orderingKey ? { orderingKey: row.orderingKey } : {}), - }; - await this.broker.publish(envelope); - await this.db - .update(eventOutbox) - .set({ publishedAt: new Date() }) - .where(eq(eventOutbox.eventId, row.eventId)); - published += 1; + if (await this.publishRow(row)) { + published += 1; + } } return published; + } catch (err) { + this.opts.onError?.(err); + return 0; } finally { this.draining = false; } } + + private async claimBatch(): Promise { + const leaseExpiredBefore = new Date(Date.now() - CLAIM_LEASE_MS); + const claimable = this.db + .select({ eventId: eventOutbox.eventId }) + .from(eventOutbox) + .where( + and( + isNull(eventOutbox.publishedAt), + or(isNull(eventOutbox.claimedAt), lt(eventOutbox.claimedAt, leaseExpiredBefore)), + ), + ) + .orderBy(asc(eventOutbox.occurredAt)) + .limit(this.opts.batchSize ?? 100) + .for('update', { skipLocked: true }); + const rows = await this.db + .update(eventOutbox) + .set({ claimedAt: sql`now()`, attempts: sql`${eventOutbox.attempts} + 1` }) + .where(inArray(eventOutbox.eventId, claimable)) + .returning(); + return rows.sort((a, b) => a.occurredAt.getTime() - b.occurredAt.getTime()); + } + + private async publishRow(row: EventOutboxRow): Promise { + try { + await this.broker.publish(toEnvelope(row)); + await this.db + .update(eventOutbox) + .set({ publishedAt: new Date() }) + .where(eq(eventOutbox.eventId, row.eventId)); + return true; + } catch (err) { + this.opts.onError?.(err); + await this.db + .update(eventOutbox) + .set({ claimedAt: null }) + .where(eq(eventOutbox.eventId, row.eventId)) + .catch((releaseErr: unknown) => this.opts.onError?.(releaseErr)); + return false; + } + } + + private async pruneIfDue(): Promise { + if (Date.now() - this.lastPruneAt < PRUNE_INTERVAL_MS) { + return; + } + this.lastPruneAt = Date.now(); + const cutoff = new Date(Date.now() - EVENT_RETENTION_MS); + try { + await this.db.delete(eventOutbox).where(lt(eventOutbox.publishedAt, cutoff)); + await this.db.delete(processedEvents).where(lt(processedEvents.processedAt, cutoff)); + } catch (err) { + this.opts.onError?.(err); + } + } } diff --git a/packages/core/src/server/db/outbox/schema.ts b/packages/core/src/server/db/outbox/schema.ts index 7a4df056f..86cf598cd 100644 --- a/packages/core/src/server/db/outbox/schema.ts +++ b/packages/core/src/server/db/outbox/schema.ts @@ -1,11 +1,20 @@ -import { pgTable, text, jsonb, integer, timestamp, index } from 'drizzle-orm/pg-core'; +import { + pgTable, + text, + jsonb, + integer, + timestamp, + index, + PrimaryKeyBuilder, +} from 'drizzle-orm/pg-core'; -// Transactional outbox table. A money/critical service writes the event envelope -// here INSIDE the same db.transaction as its state change (via the OUTBOX port), -// so the row commits atomically with the state - the event can never be lost in -// the gap between "state changed" and "event published". The OutboxRelay then -// publishes pending rows to the MESSAGE_BROKER and stamps publishedAt. Delivery -// is at-least-once; consumers dedup on eventId. See ADR-0016. +// Transactional outbox table. A service writes the event envelope here INSIDE the +// same db.transaction as its state change (EventBus.emitInTransaction), so the row +// commits atomically with the state - the event can never be lost in the gap +// between "state changed" and "event published". The OutboxRelay claims pending +// rows (claimedAt lease), publishes them to the MESSAGE_BROKER and stamps +// publishedAt. Delivery is at-least-once; consumers dedup via processed_events. +// See ADR-0016 and ADR-0044. export const eventOutbox = pgTable( 'event_outbox', { @@ -17,6 +26,7 @@ export const eventOutbox = pgTable( orderingKey: text(), occurredAt: timestamp({ withTimezone: true }).notNull(), publishedAt: timestamp({ withTimezone: true }), + claimedAt: timestamp({ withTimezone: true }), attempts: integer().notNull().default(0), createdAt: timestamp({ withTimezone: true }).notNull().defaultNow(), }, @@ -24,3 +34,23 @@ export const eventOutbox = pgTable( ); export type EventOutboxRow = typeof eventOutbox.$inferSelect; + +// Consumer-side dedup. One row per (event, handler) that finished successfully, so a +// redelivered event skips the handlers that already ran. `consumer` is the stable +// handler identity built in createApp: `//[#n]`. Pruned by the +// OutboxRelay once older than the stream retention - past it the broker can no longer +// redeliver the event. See ADR-0044. +export const processedEvents = pgTable( + 'processed_events', + { + eventId: text().notNull(), + consumer: text().notNull(), + processedAt: timestamp({ withTimezone: true }).notNull().defaultNow(), + }, + (t) => [ + // Same as primaryKey({ columns }); depretec flags every primaryKey() call because + // of its deprecated variadic overload. + new PrimaryKeyBuilder([t.eventId, t.consumer]), + index('processed_events_processed_at_idx').on(t.processedAt), + ], +); diff --git a/packages/core/src/server/kernel/__tests__/event-bus.int.test.ts b/packages/core/src/server/kernel/__tests__/event-bus.int.test.ts index 2943cd4e4..b87c10559 100644 --- a/packages/core/src/server/kernel/__tests__/event-bus.int.test.ts +++ b/packages/core/src/server/kernel/__tests__/event-bus.int.test.ts @@ -1,54 +1,102 @@ import { describe, it, expect, vi, beforeAll, afterEach, afterAll } from 'vitest'; import { randomUUID } from 'node:crypto'; +import { sql } from 'drizzle-orm'; import type { Logger } from 'pino'; import type { MessageBrokerAdapter, BrokerHandler, EventEnvelope } from '@openora/core/contracts'; +import { migrate } from '@openora/core/server/migrate'; import { mock } from '../../../testing/mock.js'; import { RedisStreamsBroker } from '../redis-streams-broker.js'; -import { createEventBus } from '../event-bus.js'; -import { createTestRedis, waitForConsumerGroup, type TestRedis } from '@openora/core/testing'; +import { createEventBus, type EventBus } from '../event-bus.js'; +import { DrizzleOutboxWriter } from '../../db/outbox/writer.js'; +import { DrizzleProcessedEventLog } from '../../db/outbox/processed-events.js'; +import { OutboxRelay } from '../../db/outbox/relay.js'; +import { eventOutbox, processedEvents } from '../../db/outbox/schema.js'; +import { + createTestDb, + createTestRedis, + waitForConsumerGroup, + type TestDb, + type TestRedis, +} from '@openora/core/testing'; const DELIVERY = { timeout: 5000, interval: 20 }; +const FAST_RECLAIM = { intervalMs: 20, minIdleMs: 10 }; const flush = () => new Promise((r) => setTimeout(r, 0)); function fakeLogger(): Logger { return mock({ error: vi.fn(), warn: vi.fn(), info: vi.fn() }); } +let db: TestDb; let redis: TestRedis; const brokers: RedisStreamsBroker[] = []; +function busOver(broker: MessageBrokerAdapter, logger: Logger = fakeLogger()): EventBus { + return createEventBus({ + broker, + outbox: new DrizzleOutboxWriter(), + processedEvents: new DrizzleProcessedEventLog(db.drizzle.db), + logger, + }); +} + // Each test gets a unique service name (= consumer group) so the fixed domain topics // don't collide across tests sharing the per-worker Redis. -function realBus(logger: Logger): { - bus: ReturnType; +function realBus(logger: Logger = fakeLogger()): { + bus: EventBus; serviceName: string; broker: RedisStreamsBroker; } { const serviceName = `evt-${randomUUID()}`; - const broker = new RedisStreamsBroker(redis.client, { serviceName }); + const broker = new RedisStreamsBroker(redis.client, { serviceName, reclaim: FAST_RECLAIM }); brokers.push(broker); - return { bus: createEventBus(broker, logger), serviceName, broker }; + return { bus: busOver(broker, logger), serviceName, broker }; } const streamOf = (topic: string): string => `oss:evt:${topic}`; +const USER_ID = randomUUID(); + +function deposit() { + return { + userId: USER_ID, + playerId: null, + amount: '50', + currency: 'USD', + transactionId: randomUUID(), + }; +} + +const roundEnded = () => ({ roundId: randomUUID(), userId: USER_ID, playerId: null }); + +function envelopeOf(topic: string, payload: unknown): EventEnvelope { + return { + eventId: randomUUID(), + topic, + payload, + occurredAt: new Date().toISOString(), + schemaVersion: 2, + }; +} + beforeAll(async () => { - redis = await createTestRedis(); + [db, redis] = await Promise.all([createTestDb([migrate]), createTestRedis()]); }); afterEach(async () => { await Promise.allSettled(brokers.map((b) => b.close())); brokers.length = 0; await redis.flush(); + await db.drizzle.db.execute(sql`TRUNCATE ${eventOutbox}, ${processedEvents}`); }); afterAll(async () => { - await redis.quit(); + await Promise.all([redis.quit(), db.drop()]); }); describe('createEventBus over Redis Streams', () => { it('round-trips a known event from emit to a subscriber', async () => { - const { bus, serviceName } = realBus(fakeLogger()); + const { bus, serviceName } = realBus(); const received: unknown[] = []; bus.on('wallet.deposit.completed', (p) => { received.push(p); @@ -58,25 +106,14 @@ describe('createEventBus over Redis Streams', () => { group: serviceName, }); - bus.emit('wallet.deposit.completed', { - userId: 'u1', - playerId: null, - amount: '50', - currency: 'USD', - transactionId: 'tx1', - }); + const payload = deposit(); + bus.emit('wallet.deposit.completed', payload); - await vi.waitFor( - () => - expect(received).toEqual([ - { userId: 'u1', playerId: null, amount: '50', currency: 'USD', transactionId: 'tx1' }, - ]), - DELIVERY, - ); + await vi.waitFor(() => expect(received).toEqual([payload]), DELIVERY); }); it('delivers the envelope as the optional second arg to handlers', async () => { - const { bus, serviceName } = realBus(fakeLogger()); + const { bus, serviceName } = realBus(); const envelopes: Array = []; bus.on('wallet.deposit.completed', (_payload, envelope) => { envelopes.push(envelope); @@ -86,13 +123,7 @@ describe('createEventBus over Redis Streams', () => { group: serviceName, }); - bus.emit('wallet.deposit.completed', { - userId: 'u1', - playerId: null, - amount: '10', - currency: 'USD', - transactionId: 'tx2', - }); + bus.emit('wallet.deposit.completed', deposit()); await vi.waitFor(() => expect(envelopes).toHaveLength(1), DELIVERY); const env = envelopes[0]; @@ -104,7 +135,7 @@ describe('createEventBus over Redis Streams', () => { }); it('envelope eventId is unique per emission', async () => { - const { bus, serviceName } = realBus(fakeLogger()); + const { bus, serviceName } = realBus(); const ids: string[] = []; bus.on('wallet.deposit.completed', (_p, env) => { if (env?.eventId) { @@ -116,20 +147,8 @@ describe('createEventBus over Redis Streams', () => { group: serviceName, }); - bus.emit('wallet.deposit.completed', { - userId: 'u1', - playerId: null, - amount: '1', - currency: 'USD', - transactionId: 'a', - }); - bus.emit('wallet.deposit.completed', { - userId: 'u1', - playerId: null, - amount: '2', - currency: 'USD', - transactionId: 'b', - }); + bus.emit('wallet.deposit.completed', deposit()); + bus.emit('wallet.deposit.completed', deposit()); await vi.waitFor(() => expect(ids).toHaveLength(2), DELIVERY); expect(ids[0]).not.toBe(ids[1]); @@ -151,38 +170,130 @@ describe('createEventBus over Redis Streams', () => { group: serviceName, }); - bus.emit('identity.user.registered', { userId: 'u1', playerId: null }); + bus.emit('identity.user.registered', { userId: USER_ID, playerId: null }); - await vi.waitFor(() => expect(order).toEqual(['first', 'second']), DELIVERY); + await vi.waitFor(() => expect(order.slice(0, 2)).toEqual(['first', 'second']), DELIVERY); expect(logger.error).toHaveBeenCalledWith( expect.objectContaining({ event: 'identity.user.registered' }), 'event subscriber threw', ); }); - it('logs a validation error for a bad payload but still delivers', async () => { - const logger = fakeLogger(); - const { bus, serviceName } = realBus(logger); - const received: unknown[] = []; - bus.on('identity.user.registered', (p) => { - received.push(p); - }); + it('throws on emit and emitInTransaction when a known payload fails its schema', async () => { + const { bus } = realBus(); + + expect(() => bus.emit('identity.user.registered', { userId: 123 } as never)).toThrow( + /identity.user.registered/, + ); + await expect( + db.drizzle.db.transaction((tx) => + bus.emitInTransaction(tx, 'identity.user.registered', { userId: 123 } as never), + ), + ).rejects.toThrow(/identity.user.registered/); + expect(await db.drizzle.db.select().from(eventOutbox)).toEqual([]); + }); + + it('dead-letters an inbound event whose payload fails its schema', async () => { + const { bus, serviceName, broker } = realBus(); + const handler = vi.fn(); + bus.on('identity.user.registered', handler); await waitForConsumerGroup(redis.client, { stream: streamOf('identity.user.registered'), group: serviceName, }); - bus.emit('identity.user.registered', { userId: 123 } as never); + await broker.publish(envelopeOf('identity.user.registered', { userId: 123 })); - await vi.waitFor(() => expect(received).toHaveLength(1), DELIVERY); - expect(logger.error).toHaveBeenCalledWith( - expect.objectContaining({ event: 'identity.user.registered' }), - 'event payload failed validation', + await vi.waitFor( + async () => expect(await redis.client.xLen('oss:dlq:identity.user.registered')).toBe(1), + DELIVERY, ); + expect(handler).not.toHaveBeenCalled(); + }); + + it('runs a consumer-keyed handler once for a duplicated delivery', async () => { + const { bus, serviceName, broker } = realBus(); + const keyed = vi.fn(); + const unkeyed = vi.fn(); + bus.on('wallet.deposit.completed', keyed, { consumer: 'svc/wallet.deposit.completed' }); + bus.on('wallet.deposit.completed', unkeyed); + await waitForConsumerGroup(redis.client, { + stream: streamOf('wallet.deposit.completed'), + group: serviceName, + }); + + const envelope = envelopeOf('wallet.deposit.completed', deposit()); + await broker.publish(envelope); + await broker.publish(envelope); + + await vi.waitFor(() => expect(unkeyed).toHaveBeenCalledTimes(2), DELIVERY); + expect(keyed).toHaveBeenCalledTimes(1); + }); + + it('redelivers to the failed handler only, skipping the sibling that already succeeded', async () => { + const { bus, serviceName } = realBus(); + let flakyCalls = 0; + const steady = vi.fn(); + bus.on('wallet.deposit.completed', steady, { consumer: 'svc/steady' }); + bus.on( + 'wallet.deposit.completed', + () => { + flakyCalls += 1; + if (flakyCalls === 1) { + throw new Error('transient'); + } + }, + { consumer: 'svc/flaky' }, + ); + await waitForConsumerGroup(redis.client, { + stream: streamOf('wallet.deposit.completed'), + group: serviceName, + }); + + bus.emit('wallet.deposit.completed', deposit()); + + await vi.waitFor(() => expect(flakyCalls).toBe(2), DELIVERY); + await vi.waitFor(async () => { + const pending = await redis.client.xPending( + streamOf('wallet.deposit.completed'), + serviceName, + ); + expect(pending.pending).toBe(0); + }, DELIVERY); + expect(steady).toHaveBeenCalledTimes(1); + }); + + it('publishes an emitInTransaction event only when its transaction commits', async () => { + const { bus, serviceName, broker } = realBus(); + const received: unknown[] = []; + bus.on('wallet.deposit.completed', (p) => { + received.push(p); + }); + await waitForConsumerGroup(redis.client, { + stream: streamOf('wallet.deposit.completed'), + group: serviceName, + }); + const relay = new OutboxRelay(db.drizzle.db, broker); + + await expect( + db.drizzle.db.transaction(async (tx) => { + await bus.emitInTransaction(tx, 'wallet.deposit.completed', deposit()); + throw new Error('abort'); + }), + ).rejects.toThrow('abort'); + expect(await relay.drainOnce()).toBe(0); + + const committed = deposit(); + await db.drizzle.db.transaction((tx) => + bus.emitInTransaction(tx, 'wallet.deposit.completed', committed), + ); + expect(await relay.drainOnce()).toBe(1); + + await vi.waitFor(() => expect(received).toEqual([committed]), DELIVERY); }); it('applies schema defaults to a legacy backlog payload before the handler runs', async () => { - const { bus, serviceName, broker } = realBus(fakeLogger()); + const { bus, serviceName, broker } = realBus(); const received: Array> = []; const envelopePayloads: unknown[] = []; bus.on('cms.banner.image.deleted', (payload, envelope) => { @@ -219,9 +330,9 @@ describe('createEventBus over Redis Streams', () => { subscribe: (_topic: string, _handler: BrokerHandler) => () => undefined, close: async () => undefined, }; - const bus = createEventBus(rejectingBroker, logger); + const bus = busOver(rejectingBroker, logger); - bus.emit('gaming.round.ended', { roundId: 'r1', userId: 'u1', playerId: null }); + bus.emit('gaming.round.ended', roundEnded()); await flush(); expect(logger.error).toHaveBeenCalledWith( @@ -239,13 +350,14 @@ describe('createEventBus over Redis Streams', () => { subscribe: (_topic: string, _handler: BrokerHandler) => () => undefined, close: async () => undefined, }; - const bus = createEventBus(fakeBroker, fakeLogger()); + const bus = busOver(fakeBroker); - bus.emit('gaming.round.ended', { roundId: 'r1', userId: 'u1', playerId: null }); + const payload = roundEnded(); + bus.emit('gaming.round.ended', payload); expect(published).toHaveLength(1); expect(published[0]?.topic).toBe('gaming.round.ended'); - expect(published[0]?.payload).toEqual({ roundId: 'r1', userId: 'u1', playerId: null }); + expect(published[0]?.payload).toEqual(payload); expect(typeof published[0]?.eventId).toBe('string'); }); }); diff --git a/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts b/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts index 7c4071684..5cdbc3f12 100644 --- a/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts +++ b/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts @@ -1,14 +1,23 @@ import { describe, it, expect, vi, beforeAll, afterEach, afterAll } from 'vitest'; import { randomUUID } from 'node:crypto'; -import { RedisStreamsBroker } from '../redis-streams-broker.js'; +import { MAX_DELIVERIES, RedisStreamsBroker } from '../redis-streams-broker.js'; import { createTestRedis, waitForConsumerGroup, type TestRedis } from '@openora/core/testing'; import type { EventEnvelope } from '@openora/core/contracts'; let redis: TestRedis; const brokers: RedisStreamsBroker[] = []; -function makeBroker(serviceName: string): RedisStreamsBroker { - const broker = new RedisStreamsBroker(redis.client, { serviceName }); +// Reclaim fast so redelivery tests don't wait out the production idle window. +const FAST_RECLAIM = { intervalMs: 20, minIdleMs: 10 }; + +function makeBroker( + serviceName: string, + reclaim?: { intervalMs: number; minIdleMs: number }, +): RedisStreamsBroker { + const broker = new RedisStreamsBroker(redis.client, { + serviceName, + ...(reclaim ? { reclaim } : {}), + }); brokers.push(broker); return broker; } @@ -228,4 +237,67 @@ describe('RedisStreamsBroker', () => { await vi.waitFor(() => expect(seen).toEqual([envelope]), { timeout: 5000 }); }); + + it('leaves a failed entry pending and redelivers it until the handler succeeds', async () => { + const broker = makeBroker('svc', FAST_RECLAIM); + const topic = uniqueTopic(); + let attempts = 0; + broker.subscribe(topic, () => { + attempts += 1; + if (attempts < 3) { + throw new Error('transient'); + } + }); + await waitForConsumerGroup(redis.client, { stream: streamOf(topic), group: 'svc' }); + + await broker.publish(makeEnvelope(topic)); + + await vi.waitFor(() => expect(attempts).toBe(3), { timeout: 5000 }); + await vi.waitFor(async () => { + const pending = await redis.client.xPending(streamOf(topic), 'svc'); + expect(pending.pending).toBe(0); + }); + expect(await redis.client.exists(`oss:dlq:${topic}`)).toBe(0); + }); + + it(`dead-letters an entry still failing after ${MAX_DELIVERIES} deliveries and ACKs it`, async () => { + const broker = makeBroker('svc', FAST_RECLAIM); + const topic = uniqueTopic(); + let attempts = 0; + broker.subscribe(topic, () => { + attempts += 1; + throw new Error('poison'); + }); + await waitForConsumerGroup(redis.client, { stream: streamOf(topic), group: 'svc' }); + + const envelope = makeEnvelope(topic); + await broker.publish(envelope); + + await vi.waitFor(async () => expect(await redis.client.xLen(`oss:dlq:${topic}`)).toBe(1), { + timeout: 5000, + }); + expect(attempts).toBe(MAX_DELIVERIES); + const [dead] = await redis.client.xRange(`oss:dlq:${topic}`, '-', '+'); + expect(JSON.parse(dead!.message['data']!)).toEqual(envelope); + expect(dead!.message).toMatchObject({ + group: 'svc', + deliveries: String(MAX_DELIVERIES), + error: 'poison', + }); + const pending = await redis.client.xPending(streamOf(topic), 'svc'); + expect(pending.pending).toBe(0); + }); + + it('dead-letters an entry that is not valid JSON without calling handlers', async () => { + const broker = makeBroker('svc'); + const topic = uniqueTopic(); + const handler = vi.fn(); + broker.subscribe(topic, handler); + await waitForConsumerGroup(redis.client, { stream: streamOf(topic), group: 'svc' }); + + await redis.client.xAdd(streamOf(topic), '*', { data: '{not json' }); + + await vi.waitFor(async () => expect(await redis.client.xLen(`oss:dlq:${topic}`)).toBe(1)); + expect(handler).not.toHaveBeenCalled(); + }); }); diff --git a/packages/core/src/server/kernel/event-bus.ts b/packages/core/src/server/kernel/event-bus.ts index de34a3501..09fd69818 100644 --- a/packages/core/src/server/kernel/event-bus.ts +++ b/packages/core/src/server/kernel/event-bus.ts @@ -14,37 +14,67 @@ import type { Logger } from 'pino'; import { createLogger } from './logger.js'; import { getCurrentTraceId } from './request-context.js'; +// How long a published event stays redeliverable: the Redis Streams retention, and the +// age past which outbox and processed_events rows are pruned. See ADR-0044. +export const EVENT_RETENTION_MS = 7 * 24 * 60 * 60_000; + export type EventHandler = ( payload: T, envelope?: EventEnvelope, ) => void | Promise; +export type SubscribeHandlerOptions = { + // Stable identity of this handler across restarts and replicas. When set, a + // redelivered event skips the handler once it has succeeded for that event. + consumer?: string; +}; + +// Records which (eventId, consumer) pairs already ran to completion. A port so the +// kernel stays ORM-free; the Drizzle implementation lives next to the outbox. +export type ProcessedEventLog = { + has(eventId: string, consumer: string): Promise; + record(eventId: string, consumer: string): Promise; +}; + // Typed app-facing event API. Services depend on this, never on the broker directly. // The optional second arg to handlers exposes the full envelope; existing (payload)-only // handlers continue to work unchanged. export type EventBus = { + // Best-effort, post-commit fan-out: the publish is fire-and-forget, so a broker + // outage loses the event. For informational and cache-invalidation events only. + // Throws synchronously when a known event's payload fails its schema. emit(event: K, payload: DomainEventPayload): void; // Escape hatch for plugin-defined dynamic topics not in domainEventSchemas. emitRaw(event: string, payload: unknown): void; // Durable, transaction-atomic emit. Call INSIDE a db.transaction, passing the - // transaction handle, when the event must never be lost relative to its state - // change (money, anything a separate service depends on). The envelope is - // written to the outbox within `tx`; a relay publishes it after commit. Requires - // the outbox to be bound (OUTBOX_ENABLED / a durable broker) - otherwise throws, - // so a money path can't silently degrade to best-effort delivery. Prefer this - // over emit() for cross-service-critical events; emit() stays for best-effort - // post-commit fan-out within a single process. + // transaction handle: the envelope is written to the outbox within `tx` and the + // OutboxRelay publishes it after commit, so the event exists if and only if the + // state change committed. Required for anything that reports a committed change on + // a money or compliance path. Rejects (rolling `tx` back) on an invalid payload. emitInTransaction( tx: unknown, event: K, payload: DomainEventPayload, ): Promise; - on(event: K, handler: EventHandler>): () => void; - on(event: string, handler: EventHandler): () => void; + // A handler that throws is retried by the broker and dead-lettered after repeated + // failures (see RedisStreamsBroker), so it must be idempotent. + on( + event: K, + handler: EventHandler>, + options?: SubscribeHandlerOptions, + ): () => void; + on(event: string, handler: EventHandler, options?: SubscribeHandlerOptions): () => void; }; export const EVENT_BUS: Token = createToken('EVENT_BUS'); +export type EventBusDeps = { + broker: MessageBrokerAdapter; + outbox: OutboxWriter; + processedEvents: ProcessedEventLog; + logger?: Logger; +}; + function isKnownEvent(event: string): event is DomainEventName { return event in domainEventSchemas; } @@ -61,37 +91,34 @@ function buildEnvelope(event: string, payload: unknown): EventEnvelope { }; } -/** Typed facade over a MessageBrokerAdapter. Validates known payloads, isolates subscriber failures, and builds the envelope. */ -export function createEventBus( - broker: MessageBrokerAdapter, - logger: Logger = createLogger('event-bus'), - outbox?: OutboxWriter, -): EventBus { - function parseKnownPayload(event: string, payload: unknown, failureMessage: string): unknown { - if (!isKnownEvent(event)) { - return payload; - } - const schema = domainEventSchemas[event] as ZodType; - const result = schema.safeParse(payload); - if (result.success) { - return result.data; - } - logger.error({ event, issues: result.error.issues }, failureMessage); +// Applies a known event's schema (defaults included); throws when the payload fails it. +function parseKnownPayload(event: string, payload: unknown): unknown { + if (!isKnownEvent(event)) { return payload; } + const schema = domainEventSchemas[event] as ZodType; + const result = schema.safeParse(payload); + if (!result.success) { + throw new Error(`Event '${event}' payload failed validation: ${result.error.message}`); + } + return result.data; +} +/** Typed facade over a MessageBrokerAdapter: validates payloads, builds the envelope, dedups handlers. */ +export function createEventBus({ + broker, + outbox, + processedEvents, + logger = createLogger('event-bus'), +}: EventBusDeps): EventBus { function publishEvent(event: string, payload: unknown): void { - parseKnownPayload(event, payload, 'event payload failed validation'); + parseKnownPayload(event, payload); const envelope = buildEnvelope(event, payload); void Promise.resolve(broker.publish(envelope)).catch((err) => logger.error({ event, err }, 'event publish failed'), ); } - function normalizeInbound(event: string, payload: unknown): unknown { - return parseKnownPayload(event, payload, 'inbound event payload failed validation'); - } - return { emit(event, payload) { publishEvent(event, payload); @@ -102,25 +129,29 @@ export function createEventBus( }, async emitInTransaction(tx: unknown, event: string, payload: unknown): Promise { - if (!outbox) { - throw new Error( - `emitInTransaction('${event}') requires the transactional outbox, which is not bound. ` + - `Enable it (set OUTBOX_ENABLED=1) so the event is published reliably, or use emit() ` + - `for best-effort post-commit delivery.`, - ); - } - parseKnownPayload(event, payload, 'event payload failed validation'); + parseKnownPayload(event, payload); await outbox.write(tx, buildEnvelope(event, payload)); }, - on(event: string, handler: EventHandler): () => void { + on(event: string, handler: EventHandler, options?: SubscribeHandlerOptions): () => void { + const consumer = options?.consumer; return broker.subscribe(event, async (envelope: EventEnvelope) => { - const payload = normalizeInbound(event, envelope.payload); - const delivered = payload === envelope.payload ? envelope : { ...envelope, payload }; + if (consumer && (await processedEvents.has(envelope.eventId, consumer))) { + return; + } try { + const payload = parseKnownPayload(event, envelope.payload); + const delivered = payload === envelope.payload ? envelope : { ...envelope, payload }; await handler(payload, delivered); } catch (err) { - logger.error({ event, err }, 'event subscriber threw'); + logger.error( + { event, consumer, eventId: envelope.eventId, err }, + 'event subscriber threw', + ); + throw err; + } + if (consumer) { + await processedEvents.record(envelope.eventId, consumer); } }); }, diff --git a/packages/core/src/server/kernel/index.ts b/packages/core/src/server/kernel/index.ts index 932314b45..5a9e28a04 100644 --- a/packages/core/src/server/kernel/index.ts +++ b/packages/core/src/server/kernel/index.ts @@ -7,8 +7,14 @@ export { withRequestContext, } from './request-context.js'; -export type { EventBus, EventHandler } from './event-bus.js'; -export { EVENT_BUS, createEventBus } from './event-bus.js'; +export type { + EventBus, + EventBusDeps, + EventHandler, + ProcessedEventLog, + SubscribeHandlerOptions, +} from './event-bus.js'; +export { EVENT_BUS, EVENT_RETENTION_MS, createEventBus } from './event-bus.js'; export { BullMqJobQueue } from './bullmq-job-queue.js'; export { makeRateLimitError, assertRateLimit } from './rate-limiter.js'; diff --git a/packages/core/src/server/kernel/redis-streams-broker.ts b/packages/core/src/server/kernel/redis-streams-broker.ts index 3ea4ff73e..db20a107e 100644 --- a/packages/core/src/server/kernel/redis-streams-broker.ts +++ b/packages/core/src/server/kernel/redis-streams-broker.ts @@ -7,14 +7,17 @@ import type { } from '@openora/core/contracts'; import type { RedisClient } from './redis-client.js'; import { createLogger } from './logger.js'; +import { EVENT_RETENTION_MS } from './event-bus.js'; const STREAM_PREFIX = 'oss:evt:'; -const STREAM_MAXLEN = 10_000; +const DLQ_PREFIX = 'oss:dlq:'; const READ_COUNT = 10; const BLOCK_MS = 5_000; const CLAIM_INTERVAL_MS = 30_000; const CLAIM_MIN_IDLE_MS = 60_000; const RETRY_DELAY_MS = 2_000; +// Deliveries (first read + reclaims) after which a still-failing entry is dead-lettered. +export const MAX_DELIVERIES = 5; function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); @@ -40,8 +43,12 @@ type RedisStreamsBrokerOptions = { // that already exists comes back BUSYGROUP and keeps its own cursor, so this never // rewinds a running deployment). '$' - the default - takes only events published // after creation; '0' replays everything still retained in the stream (up to - // STREAM_MAXLEN). See the class docstring for which to pick. + // EVENT_RETENTION_MS). See the class docstring for which to pick. startId?: '$' | '0'; + + // How often idle-pending entries are reclaimed for retry, and how long an entry must + // sit unacknowledged first. Tests shorten these; production keeps the defaults. + reclaim?: { intervalMs: number; minIdleMs: number }; }; type ConsumerLoop = { @@ -54,6 +61,14 @@ function streamKey(topic: string): string { return STREAM_PREFIX + topic; } +function dlqKey(streamKeyName: string): string { + return DLQ_PREFIX + streamKeyName.slice(STREAM_PREFIX.length); +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + function isBusyGroupError(err: unknown): boolean { return err instanceof Error && err.message.includes('BUSYGROUP'); } @@ -79,10 +94,20 @@ function isBusyGroupError(err: unknown): boolean { * so a blocked read never stalls the cache/rate-limiter commands sharing it. * `publish` awaits the XADD and lets errors propagate (the `OutboxRelay` marks a * row published only after this resolves - it must be reliable, not a silent - * no-op). Handler errors are guarded + logged, then the entry is XACKed regardless - * (matches `EventBus.on` - no per-handler retry; must-not-lose events use the - * outbox). A crash before XACK leaves the entry pending; `XAUTOCLAIM` periodically - * reclaims idle-pending entries so at-least-once delivery survives a replica crash. + * no-op). Every local handler runs even when a sibling throws; the entry is XACKed + * only when all of them succeeded. A failure (or a crash before XACK) leaves it + * pending, and `XAUTOCLAIM` reclaims idle-pending entries every `CLAIM_INTERVAL_MS` + * and redelivers them - to every local handler again, so `EventBus.on` skips the + * ones that already succeeded via `processed_events`. An entry still failing on its + * `MAX_DELIVERIES`th delivery (or one that is not valid JSON) is copied to the + * `oss:dlq:` stream with its group, stream id, delivery count and last + * error, then XACKed. The DLQ is never trimmed; an operator drains it. + * + * ponytail: retention is time-based (`MINID ~ now - EVENT_RETENTION_MS` on every + * XADD), so a group that falls more than 7 days behind loses the entries it never + * read, and stream memory is bounded by traffic x 7 days, not by a count. Alert on + * consumer-group lag; move to a broker overlay with disk-backed retention if either + * ceiling bites. * * At-least-once holds from group creation onward, NOT before it. A group is created * at `startId` (default `$`) on first subscribe and persists in Redis from then on, @@ -90,11 +115,8 @@ function isBusyGroupError(err: unknown): boolean { * entries wait in the stream, and pending ones are reclaimed. The one uncovered * window is a group's FIRST-EVER creation - a service extracted after events were * already flowing starts from `$` and never sees the backlog. `startId: '0'` closes - * that window by replaying everything still retained (up to STREAM_MAXLEN), at the - * cost of re-running every handler for those events: safe on a fresh deploy (empty - * streams), a duplicate-side-effect hazard for a late-added service, since core has - * no dedup layer - `EventBus.on` invokes handlers directly, and `eventId` is a key - * consumers must dedup on themselves. + * that window by replaying everything still retained (up to EVENT_RETENTION_MS), at + * the cost of running every handler of the new service for that backlog. * * `orderingKey` is NOT honoured - Redis Streams have no partitioning, so per-key * ordering across concurrent consumers isn't guaranteed (same footnote as @@ -106,6 +128,8 @@ export class RedisStreamsBroker implements MessageBrokerAdapter { private readonly startId: '$' | '0'; private readonly consumerId = randomUUID(); private readonly loops = new Map(); + private readonly claimIntervalMs: number; + private readonly claimMinIdleMs: number; constructor( private readonly client: RedisClient, @@ -113,6 +137,8 @@ export class RedisStreamsBroker implements MessageBrokerAdapter { ) { this.serviceName = opts.serviceName; this.startId = opts.startId ?? '$'; + this.claimIntervalMs = opts.reclaim?.intervalMs ?? CLAIM_INTERVAL_MS; + this.claimMinIdleMs = opts.reclaim?.minIdleMs ?? CLAIM_MIN_IDLE_MS; } async publish(envelope: EventEnvelope): Promise { @@ -120,7 +146,13 @@ export class RedisStreamsBroker implements MessageBrokerAdapter { streamKey(envelope.topic), '*', { data: JSON.stringify(envelope) }, - { TRIM: { strategy: 'MAXLEN', strategyModifier: '~', threshold: STREAM_MAXLEN } }, + { + TRIM: { + strategy: 'MINID', + strategyModifier: '~', + threshold: Date.now() - EVENT_RETENTION_MS, + }, + }, ); } @@ -166,6 +198,7 @@ export class RedisStreamsBroker implements MessageBrokerAdapter { let stopped = false; let lastClaimAt = 0; + let claimCursor = '0-0'; const stop = (): void => { stopped = true; @@ -194,22 +227,22 @@ export class RedisStreamsBroker implements MessageBrokerAdapter { } ready = true; } - if (Date.now() - lastClaimAt > CLAIM_INTERVAL_MS) { + if (Date.now() - lastClaimAt > this.claimIntervalMs) { lastClaimAt = Date.now(); - await this.reclaimPending(reader, key, group, handlers); + claimCursor = await this.reclaimPending(reader, key, group, handlers, claimCursor); } const reply = (await reader.xReadGroup( group, this.consumerId, { key, id: '>' }, - { COUNT: READ_COUNT, BLOCK: BLOCK_MS }, + { COUNT: READ_COUNT, BLOCK: Math.min(BLOCK_MS, this.claimIntervalMs) }, )) as StreamReadReply; if (!reply) { continue; } for (const stream of reply) { for (const message of stream.messages) { - await this.dispatch(reader, key, group, message, handlers); + await this.dispatch(reader, key, group, message, handlers, 1); } } } catch (err) { @@ -226,37 +259,79 @@ export class RedisStreamsBroker implements MessageBrokerAdapter { return { handlers, stop, done }; } + // Returns the XAUTOCLAIM cursor to resume from, so a backlog of pending entries is + // walked across intervals instead of re-scanning the first page every time. private async reclaimPending( reader: RedisClient, key: string, group: string, handlers: Set, - ): Promise { + cursor: string, + ): Promise { try { const claimed = (await reader.xAutoClaim( key, group, this.consumerId, - CLAIM_MIN_IDLE_MS, - '0-0', + this.claimMinIdleMs, + cursor, { COUNT: READ_COUNT }, )) as StreamClaimReply; + if (claimed.deletedMessages.length > 0) { + this.logger.error( + { key, group, ids: claimed.deletedMessages }, + 'pending stream entries were trimmed before they were processed', + ); + } for (const message of claimed.messages) { if (message) { - await this.dispatch(reader, key, group, message, handlers); + const deliveries = await this.deliveryCount(reader, key, group, message.id); + await this.dispatch(reader, key, group, message, handlers, deliveries); } } + return claimed.nextId; } catch (err) { this.logger.error({ err, key, group }, 'stream reclaim failed'); + return cursor; } } + private async deliveryCount( + reader: RedisClient, + key: string, + group: string, + id: string, + ): Promise { + const [entry] = await reader.xPendingRange(key, group, id, id, 1); + return entry?.deliveriesCounter ?? 1; + } + + private async deadLetter( + reader: RedisClient, + key: string, + group: string, + message: StreamMessage, + deliveries: number, + err: unknown, + ): Promise { + await reader.xAdd(dlqKey(key), '*', { + data: message.message['data'] ?? '', + group, + streamId: message.id, + deliveries: String(deliveries), + error: errorMessage(err), + }); + await reader.xAck(key, group, message.id); + this.logger.error({ err, key, group, id: message.id, deliveries }, 'event dead-lettered'); + } + private async dispatch( reader: RedisClient, key: string, group: string, message: StreamMessage, handlers: Set, + deliveries: number, ): Promise { const raw = message.message['data']; if (raw === undefined) { @@ -267,17 +342,25 @@ export class RedisStreamsBroker implements MessageBrokerAdapter { try { envelope = JSON.parse(raw); } catch (err) { - this.logger.error({ err, key, group, id: message.id }, 'stream envelope parse failed'); - await reader.xAck(key, group, message.id); + await this.deadLetter(reader, key, group, message, deliveries, err); return; } + let failure: { err: unknown } | undefined; for (const fn of handlers) { try { await fn(envelope); } catch (err) { - this.logger.error({ err, topic: envelope.topic }, 'message broker handler threw'); + failure = { err }; + this.logger.error( + { err, topic: envelope.topic, deliveries }, + 'message broker handler threw', + ); } } - await reader.xAck(key, group, message.id); + if (!failure) { + await reader.xAck(key, group, message.id); + } else if (deliveries >= MAX_DELIVERIES) { + await this.deadLetter(reader, key, group, message, deliveries, failure.err); + } } } diff --git a/packages/core/src/server/plugin-host/__tests__/module-registry.test.ts b/packages/core/src/server/plugin-host/__tests__/module-registry.test.ts index fe245523a..6ce26dc76 100644 --- a/packages/core/src/server/plugin-host/__tests__/module-registry.test.ts +++ b/packages/core/src/server/plugin-host/__tests__/module-registry.test.ts @@ -59,13 +59,16 @@ describe('ModuleRegistryImpl', () => { expect(() => reg.routers.add('wallet', () => ({}) as never)).toThrow(/already registered/); }); - it('events.on() accumulates handlers per event', () => { + it('events.on() accumulates handlers per event, each with a distinct stable consumer id', () => { const { reg } = newRegistry(); const h1 = () => {}; const h2 = () => {}; reg.events.on('wallet.deposit.completed', h1); reg.events.on('wallet.deposit.completed', h2); - expect(reg.events.getAll().get('wallet.deposit.completed')).toEqual([h1, h2]); + expect(reg.events.getAll().get('wallet.deposit.completed')).toEqual([ + { handler: h1, consumer: 'unknown/wallet.deposit.completed' }, + { handler: h2, consumer: 'unknown/wallet.deposit.completed#1' }, + ]); }); }); diff --git a/packages/core/src/server/plugin-host/define-plugin.ts b/packages/core/src/server/plugin-host/define-plugin.ts index ca056f826..b531bfdd9 100644 --- a/packages/core/src/server/plugin-host/define-plugin.ts +++ b/packages/core/src/server/plugin-host/define-plugin.ts @@ -36,6 +36,12 @@ export type RouterFactory = (c: TypedContainer) => un export type EventHandler = (payload: unknown, envelope?: EventEnvelope) => void | Promise; +// `consumer` is the handler's dedup identity, `/`, suffixed `#` for the +// n-th (n >= 1) further handler the same plugin registers on the same topic. Plugin ids and +// topics are fixed by code, so it survives restarts, replica changes and plugin reordering; +// it shifts only when a plugin reorders its own handlers for one topic. +export type RegisteredEventHandler = { handler: EventHandler; consumer: string }; + // Tool and action-type factories run once, when MCP_KERNEL is built after every provider is // bound - never at registration. export type McpToolFactory< @@ -82,7 +88,7 @@ export type ModuleRegistry = { }; events: { on(event: string, handler: EventHandler): void; - getAll(): Map; + getAll(): Map; }; // Started at boot against the resolved JOB_QUEUE (after all providers, so an overlay's durable driver is in effect). See ADR-0014. jobs: { diff --git a/packages/core/src/server/plugin-host/index.ts b/packages/core/src/server/plugin-host/index.ts index d85f0486e..4b3e12a4d 100644 --- a/packages/core/src/server/plugin-host/index.ts +++ b/packages/core/src/server/plugin-host/index.ts @@ -9,6 +9,7 @@ export type { RegisteredMcpTool, RegisteredActionType, EventHandler, + RegisteredEventHandler, TypedContainer, } from './define-plugin.js'; export { ModuleRegistryImpl } from './module-registry.js'; diff --git a/packages/core/src/server/plugin-host/module-registry.ts b/packages/core/src/server/plugin-host/module-registry.ts index 4b752c2f5..ffb2d3ab7 100644 --- a/packages/core/src/server/plugin-host/module-registry.ts +++ b/packages/core/src/server/plugin-host/module-registry.ts @@ -18,6 +18,7 @@ import type { RouterFactory, TypedContainer, EventHandler, + RegisteredEventHandler, } from './define-plugin.js'; import { assertValidActionType, assertValidMcpTool } from '../mcp/contract-validation.js'; @@ -25,7 +26,8 @@ const UNKNOWN_OWNER = 'unknown'; export class ModuleRegistryImpl implements ModuleRegistry { private _routers = new Map>(); - private _events = new Map(); + private _events = new Map(); + private _consumerOrdinals = new Map(); private _jobs: WorkerRegistration[] = []; private _mcpTools: McpToolDefinition[] = []; private _tools: RegisteredMcpTool[] = []; @@ -91,7 +93,10 @@ export class ModuleRegistryImpl implements ModuleRegistr events = { on: (event: string, handler: EventHandler) => { const handlers = this._events.get(event) ?? []; - handlers.push(handler); + const consumer = `${this._owner}/${event}`; + const ordinal = this._consumerOrdinals.get(consumer) ?? 0; + this._consumerOrdinals.set(consumer, ordinal + 1); + handlers.push({ handler, consumer: ordinal === 0 ? consumer : `${consumer}#${ordinal}` }); this._events.set(event, handlers); }, getAll: () => this._events, diff --git a/packages/core/src/server/runtime/create-app.ts b/packages/core/src/server/runtime/create-app.ts index 1b83040d9..475a6a49d 100644 --- a/packages/core/src/server/runtime/create-app.ts +++ b/packages/core/src/server/runtime/create-app.ts @@ -49,7 +49,13 @@ import { SESSION_IDLE_POLICY, TWO_FACTOR_SETUP_POLICY, } from '@openora/core/contracts'; -import { DrizzleService, DRIZZLE, DrizzleOutboxWriter, OutboxRelay } from '../db/index.js'; +import { + DrizzleService, + DRIZZLE, + DrizzleOutboxWriter, + DrizzleProcessedEventLog, + OutboxRelay, +} from '../db/index.js'; import { AdminGuard, ADMIN_GUARD, SessionResolver, AUTH_SESSION } from '../auth/index.js'; import { loadPlugins, type PluginEntry } from '../plugin-host/index.js'; import { createMcpKernel } from '../mcp/index.js'; @@ -351,8 +357,8 @@ async function captureRawBody(req: Request): Promise { * or a test's `configure` callback. `assertDurableSeamsBound` runs right after plugins * + `configure` and throws a clear, actionable error listing every always-needed seam * that still has no binding - none of the five has an in-process fallback. - * `OUTBOX_ENABLED` (or `AMQP_URL`/`RABBITMQ_URL`) enables the transactional outbox - * relay. + * The transactional outbox is always on: `OUTBOX` is bound and the `OutboxRelay` + * runs in every process (ADR-0044). * `container.dispose()` (via `close()`) runs disposers in REVERSE registration * order, so DRIZZLE is registered before JOB_QUEUE to guarantee workers drain * before the DB closes. A router namespace registered by more than one plugin @@ -378,30 +384,22 @@ export async function createApp( container.onDispose(() => svc.dispose()); return svc; }); - // A distributed deployment wants events that survive a crash between commit and - // publish. Off by default: emit() keeps its best-effort fan-out and - // emitInTransaction() throws a guiding error until this is enabled. - // AMQP_URL/RABBITMQ_URL only signal the INTENT to run a RabbitMQ broker overlay - - // core binds no AMQP driver, so they enable the outbox and nothing else. The relay - // publishes to whatever MESSAGE_BROKER resolves to (the Redis Streams driver below, - // unless an overlay replaced it); set OUTBOX_ENABLED directly when that is the aim. - const outboxEnabled = - !!process.env['OUTBOX_ENABLED'] || !!process.env['AMQP_URL'] || !!process.env['RABBITMQ_URL']; - if (outboxEnabled) { - container.register(OUTBOX, () => new DrizzleOutboxWriter()); - } + // Postgres is always present, so the outbox is too: emitInTransaction() commits the + // event with its state change and the relay below publishes it (ADR-0044). + container.register(OUTBOX, () => new DrizzleOutboxWriter()); container.register(EVENT_BUS, (c) => - createEventBus( - c.get(MESSAGE_BROKER), - createLogger('event-bus'), - outboxEnabled ? c.get(OUTBOX) : undefined, - ), + createEventBus({ + broker: c.get(MESSAGE_BROKER), + outbox: c.get(OUTBOX), + processedEvents: new DrizzleProcessedEventLog(c.get(DRIZZLE).db), + logger: createLogger('event-bus'), + }), ); + // Resolved eagerly: a misconfigured service name is a boot error, not a surprise on + // whichever code path resolves the broker first. It also namespaces handler dedup. + const serviceName = resolveServiceName(); const redisUrl = process.env['REDIS_URL']; if (redisUrl) { - // Resolved eagerly, before any client is opened: a misconfigured service name is - // a boot error, not a surprise on whichever code path resolves the broker first. - const serviceName = resolveServiceName(); container.register(JOB_QUEUE, () => { const q = new BullMqJobQueue(redisUrl); container.onDispose(() => q.close()); @@ -479,8 +477,10 @@ export async function createApp( const bus = container.get(EVENT_BUS); for (const [event, handlers] of registry.events.getAll()) { - for (const handler of handlers) { - bus.on(event, handler); + for (const { handler, consumer } of handlers) { + // Prefixed with the service: an infra overlay loads in every split service, and + // each service's copy of the handler must run. + bus.on(event, handler, { consumer: `${serviceName}/${consumer}` }); } } @@ -503,13 +503,11 @@ export async function createApp( jobQueue.registerWorker(registration); } - if (outboxEnabled) { - const relay = new OutboxRelay(drizzle.db, container.get(MESSAGE_BROKER), { - onError: (err) => createLogger('outbox-relay').error({ err }, 'outbox drain failed'), - }); - relay.start(); - container.onDispose(() => relay.stop()); - } + const relay = new OutboxRelay(drizzle.db, container.get(MESSAGE_BROKER), { + onError: (err) => createLogger('outbox-relay').error({ err }, 'outbox drain failed'), + }); + relay.start(); + container.onDispose(() => relay.stop()); if (!config.disableHealthModule) { const os = implement(healthContract).$context(); diff --git a/packages/core/src/wallet/__tests__/wallet-balance-changed.plugin.int.test.ts b/packages/core/src/wallet/__tests__/wallet-balance-changed.plugin.int.test.ts index 31e98baff..a0eec11b6 100644 --- a/packages/core/src/wallet/__tests__/wallet-balance-changed.plugin.int.test.ts +++ b/packages/core/src/wallet/__tests__/wallet-balance-changed.plugin.int.test.ts @@ -63,7 +63,7 @@ describe('wallet plugin wallet.balance.changed wiring', () => { const userId = randomUUID(); const eventId = randomUUID(); - await handlers[0]?.( + await handlers[0]?.handler( { userId, playerId: null, @@ -93,7 +93,7 @@ describe('wallet plugin wallet.balance.changed wiring', () => { const { registry, realtime } = boot(); const handlers = registry.events.getAll().get('wallet.balance.changed') ?? []; - await handlers[0]?.( + await handlers[0]?.handler( { userId: 'not-a-uuid' }, { eventId: randomUUID(), From b9cb88dc84d258846bcce9ae43fe9167f6eee3a7 Mon Sep 17 00:00:00 2001 From: Volodymyr Zakhovaiko Date: Tue, 13 Oct 2026 00:56:23 +0200 Subject: [PATCH 02/13] fix(core): emit money and compliance events through the outbox Wallet, swap, reconciliation, gaming, bonus lifecycle, KYC, RG, player status and admin geo-rule events are written with emitInTransaction so they exist if and only if the state change commits. Bonus expiry and forfeit are emitted once, by the grant lifecycle service. --- .../__tests__/gaming.service.int.test.ts | 106 +++++--- .../casino/gaming/service/gaming.service.ts | 191 ++++++------- .../__tests__/compliance.service.int.test.ts | 25 +- .../__tests__/kyc-agent-tools.int.test.ts | 6 +- .../__tests__/kyc.service.int.test.ts | 14 +- .../__tests__/rg-self-service.int.test.ts | 30 ++- .../__tests__/rg.router.int.test.ts | 3 +- .../__tests__/rg.service.int.test.ts | 36 ++- .../compliance/service/compliance.service.ts | 117 ++++---- .../src/compliance/service/kyc.service.ts | 161 ++++++----- .../service/rg-self-service.service.ts | 239 ++++++++--------- .../core/src/compliance/service/rg.service.ts | 235 ++++++++-------- .../__tests__/player.service.int.test.ts | 148 +++++++---- .../service/player.service.ts | 96 +++---- .../__tests__/bonus-lifecycle-port.test.ts | 27 +- .../__tests__/grant-lifecycle.int.test.ts | 4 +- packages/core/src/promo/bonus/plugin.ts | 45 +--- packages/core/src/promo/bonus/router/index.ts | 18 +- .../service/bonus-lifecycle-port.service.ts | 19 +- .../bonus/service/grant-lifecycle.service.ts | 17 +- .../wallet/__tests__/swap.service.int.test.ts | 3 +- .../wallet-auto-withdrawal.int.test.ts | 2 +- .../wallet-withdrawal-hold.int.test.ts | 7 +- .../__tests__/wallet.service.int.test.ts | 26 +- .../wallet/service/reconciliation.service.ts | 3 +- .../core/src/wallet/service/swap.service.ts | 17 +- .../core/src/wallet/service/wallet.service.ts | 251 +++++++++--------- .../__tests__/game-geo-blocking.e2e.test.ts | 8 +- 28 files changed, 955 insertions(+), 899 deletions(-) diff --git a/packages/core/src/casino/gaming/__tests__/gaming.service.int.test.ts b/packages/core/src/casino/gaming/__tests__/gaming.service.int.test.ts index 50b729993..859e83e93 100644 --- a/packages/core/src/casino/gaming/__tests__/gaming.service.int.test.ts +++ b/packages/core/src/casino/gaming/__tests__/gaming.service.int.test.ts @@ -1083,12 +1083,16 @@ describe('GamingService.startRound bonus completion (real PG)', () => { 'provider unavailable', ); - expect(events.emit).toHaveBeenCalledWith('promo.bonus.completed', { - userId, - grantId: '00000000-0000-0000-0000-0000000000c5', - currency: 'USD', - convertedAmount: '25', - }); + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), + 'promo.bonus.completed', + { + userId, + grantId: '00000000-0000-0000-0000-0000000000c5', + currency: 'USD', + convertedAmount: '25', + }, + ); expect(launchGame).toHaveBeenCalledOnce(); }); @@ -1115,7 +1119,11 @@ describe('GamingService.startRound bonus completion (real PG)', () => { await startRound(svc, '00000000-0000-0000-0000-000000000406', created.id, 'USD', '25'); - expect(events.emit).not.toHaveBeenCalledWith('promo.bonus.completed', expect.anything()); + expect(events.emitInTransaction).not.toHaveBeenCalledWith( + expect.anything(), + 'promo.bonus.completed', + expect.anything(), + ); }); }); @@ -1145,15 +1153,19 @@ describe('GamingService.startRound wallet.balance.changed event (real PG)', () = await startRound(svc, userId, created.id, 'USD', '10'); - expect(events.emit).toHaveBeenCalledWith('wallet.balance.changed', { - userId, - playerId: null, - amount: '10', - currency: 'USD', - transactionId: '00000000-0000-0000-0000-0000000000d1', - type: 'bet', - direction: 'debit', - }); + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), + 'wallet.balance.changed', + { + userId, + playerId: null, + amount: '10', + currency: 'USD', + transactionId: '00000000-0000-0000-0000-0000000000d1', + type: 'bet', + direction: 'debit', + }, + ); }); it('never emits it when the debit outcome reports nothing moved', async () => { @@ -1183,7 +1195,11 @@ describe('GamingService.startRound wallet.balance.changed event (real PG)', () = await startRound(svc, userId, created.id, 'USD', '10'); - expect(events.emit).not.toHaveBeenCalledWith('wallet.balance.changed', expect.anything()); + expect(events.emitInTransaction).not.toHaveBeenCalledWith( + expect.anything(), + 'wallet.balance.changed', + expect.anything(), + ); }); it('also emits it for a WAGER_TRACKING credit (rank rakeback) alongside the bet debit', async () => { @@ -1218,15 +1234,19 @@ describe('GamingService.startRound wallet.balance.changed event (real PG)', () = await startRound(svc, userId, created.id, 'USD', '10'); - expect(events.emit).toHaveBeenCalledWith('wallet.balance.changed', { - userId, - playerId: null, - amount: '0.10', - currency: 'USD', - transactionId: '00000000-0000-0000-0000-0000000000d3', - type: 'cashback', - direction: 'credit', - }); + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), + 'wallet.balance.changed', + { + userId, + playerId: null, + amount: '0.10', + currency: 'USD', + transactionId: '00000000-0000-0000-0000-0000000000d3', + type: 'cashback', + direction: 'credit', + }, + ); }); }); @@ -1700,15 +1720,19 @@ describe('GamingService.endRound wallet.balance.changed event (real PG)', () => await svc.endRound(userId, round.id); - expect(events.emit).toHaveBeenCalledWith('wallet.balance.changed', { - userId, - playerId: null, - amount: '42.50', - currency: 'USD', - transactionId: '00000000-0000-0000-0000-0000000000e1', - type: 'win', - direction: 'credit', - }); + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), + 'wallet.balance.changed', + { + userId, + playerId: null, + amount: '42.50', + currency: 'USD', + transactionId: '00000000-0000-0000-0000-0000000000e1', + type: 'win', + direction: 'credit', + }, + ); }); it('never emits it when the provider reports no win', async () => { @@ -1733,7 +1757,11 @@ describe('GamingService.endRound wallet.balance.changed event (real PG)', () => await svc.endRound(userId, round.id); expect(walletCommands.credit).not.toHaveBeenCalled(); - expect(events.emit).not.toHaveBeenCalledWith('wallet.balance.changed', expect.anything()); + expect(events.emitInTransaction).not.toHaveBeenCalledWith( + expect.anything(), + 'wallet.balance.changed', + expect.anything(), + ); }); it('never emits it when the credit outcome reports nothing moved', async () => { @@ -1755,7 +1783,11 @@ describe('GamingService.endRound wallet.balance.changed event (real PG)', () => await svc.endRound(userId, round.id); - expect(events.emit).not.toHaveBeenCalledWith('wallet.balance.changed', expect.anything()); + expect(events.emitInTransaction).not.toHaveBeenCalledWith( + expect.anything(), + 'wallet.balance.changed', + expect.anything(), + ); }); }); diff --git a/packages/core/src/casino/gaming/service/gaming.service.ts b/packages/core/src/casino/gaming/service/gaming.service.ts index 7af6dffee..6ac3ea008 100644 --- a/packages/core/src/casino/gaming/service/gaming.service.ts +++ b/packages/core/src/casino/gaming/service/gaming.service.ts @@ -621,101 +621,86 @@ export class GamingService { // engine attributes a win to the grant that funded the stake by this id, and a stake booked // without one can never be settled back. const roundId = randomUUID(); + const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); - const { round, betTransactionId, completed, wagerTrackingCredits } = - await this.drizzle.db.transaction(async (tx) => { - // The same currency the RG pre-check above weighed. Left off, the debit falls on the - // player's active currency, and the two would then judge different moves. - const outcome = await this.walletCommands.debit(tx, { + // Every event below commits with the debit through the outbox - see the module-level + // comment on WalletCommandsService for why the port itself never emits them. + const round = await this.drizzle.db.transaction(async (tx) => { + // The same currency the RG pre-check above weighed. Left off, the debit falls on the + // player's active currency, and the two would then judge different moves. + const outcome = await this.walletCommands.debit(tx, { + userId, + amount: betAmount, + currency, + type: 'bet', + context: { provider: INTERNAL_ROUND_PROVIDER, product: INTERNAL_ROUND_PRODUCT, gameId }, + providerRef: { + providerName: INTERNAL_ROUND_PROVIDER, + providerRefId: `bet:${roundId}`, + externalRoundId: roundId, + }, + }); + if (!outcome.ok) { + throw new InsufficientBalanceError(outcome.available, betAmount); + } + const insertedRound = findOneOrThrow( + await tx + .insert(gameRound) + .values({ + id: roundId, + gameId, + userId, + currency, + betAmount, + status: 'active', + }) + .returning(), + new GameRoundNotFoundError(gameId), + ); + if (outcome.moved) { + await this.events.emitInTransaction(tx, 'wallet.balance.changed', { userId, + playerId, amount: betAmount, currency, + transactionId: outcome.transactionId, type: 'bet', - context: { provider: INTERNAL_ROUND_PROVIDER, product: INTERNAL_ROUND_PRODUCT, gameId }, - providerRef: { - providerName: INTERNAL_ROUND_PROVIDER, - providerRefId: `bet:${roundId}`, - externalRoundId: roundId, - }, + direction: 'debit', }); - if (!outcome.ok) { - throw new InsufficientBalanceError(outcome.available, betAmount); + // Whatever a WAGER_TRACKING consumer (rank rakeback) credited to the real balance + // inside this same transaction. + for (const credit of outcome.wagerTrackingCredits ?? []) { + await this.events.emitInTransaction(tx, 'wallet.balance.changed', { + userId, + playerId, + amount: credit.amount, + currency: credit.currency, + transactionId: credit.transactionId, + type: 'cashback', + direction: 'credit', + }); } - const insertedRound = findOneOrThrow( - await tx - .insert(gameRound) - .values({ - id: roundId, - gameId, - userId, - currency, - betAmount, - status: 'active', - }) - .returning(), - new GameRoundNotFoundError(gameId), - ); - const moved = outcome.moved ? outcome : undefined; - return { - round: insertedRound, - completed: moved?.completed - ? { ...moved.completed, currency: outcome.currency } - : undefined, - betTransactionId: moved?.transactionId, - wagerTrackingCredits: moved?.wagerTrackingCredits ?? [], - }; - }); - - const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); - - // Emitted only after the transaction above committed - see the module-level comment - // on WalletCommandsService for why the port itself never emits this. - if (betTransactionId) { - this.events.emit('wallet.balance.changed', { + if (outcome.completed) { + await this.events.emitInTransaction(tx, 'promo.bonus.completed', { + userId, + grantId: outcome.completed.grantId, + currency: outcome.currency, + convertedAmount: outcome.completed.convertedAmount, + }); + } + } + await this.events.emitInTransaction(tx, 'gaming.round.started', { + roundId: insertedRound.id, + gameId, userId, playerId, - amount: betAmount, currency, - transactionId: betTransactionId, - type: 'bet', - direction: 'debit', - }); - } - - // Whatever a WAGER_TRACKING consumer (rank rakeback) credited to the real balance inside the - // same transaction as the bet above - same post-commit rule as the bet debit itself. - for (const credit of wagerTrackingCredits) { - this.events.emit('wallet.balance.changed', { - userId, - playerId, - amount: credit.amount, - currency: credit.currency, - transactionId: credit.transactionId, - type: 'cashback', - direction: 'credit', - }); - } - - // Post-commit: the money moved inside the transaction above, this only tells the player. - if (completed) { - this.events.emit('promo.bonus.completed', { - userId, - grantId: completed.grantId, - currency: completed.currency, - convertedAmount: completed.convertedAmount, }); - } + return insertedRound; + }); const { launchUrl, token } = await this.provider.launchGame(gameId, userId, currency); - this.events.emit('gaming.round.started', { - roundId: round.id, - gameId, - userId, - playerId, - currency, - }); - return { roundId: round.id, launchUrl, token }; } @@ -742,7 +727,9 @@ export class GamingService { // The provider's number, never the caller's: the win is credited off this alone. const winAmount = outcome?.winAmount ?? '0'; - const { paid, winTransactionId } = await this.drizzle.db.transaction(async (tx) => { + const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); + + const paid = await this.drizzle.db.transaction(async (tx) => { // `status = 'active'` is the payout guard: two concurrent end-round calls both // reach here, only one updates a row, so the win is credited exactly once. const settled = await tx @@ -757,9 +744,8 @@ export class GamingService { ) .returning({ id: gameRound.id }); if (settled.length === 0) { - return { paid: false, winTransactionId: undefined }; + return false; } - let winTransactionId: string | undefined; if (Number(winAmount) > 0) { // The bet already opened this currency's balance, so `allowNewCurrency` only // covers a player whose active currency moved between start and settlement. @@ -778,31 +764,20 @@ export class GamingService { if (!credited.ok) { throw new WinCreditFailedError(roundId, credited.reason); } - winTransactionId = credited.moved ? credited.transactionId : undefined; + if (credited.moved) { + await this.events.emitInTransaction(tx, 'wallet.balance.changed', { + userId, + playerId, + amount: winAmount, + currency: round.currency, + transactionId: credited.transactionId, + type: 'win', + direction: 'credit', + }); + } } - return { paid: true, winTransactionId }; - }); - - const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); - - // Emitted only after the transaction above committed - see the module-level comment - // on WalletCommandsService for why the port itself never emits this. - if (winTransactionId) { - this.events.emit('wallet.balance.changed', { - userId, - playerId, - amount: winAmount, - currency: round.currency, - transactionId: winTransactionId, - type: 'win', - direction: 'credit', - }); - } - - this.events.emit('gaming.round.ended', { - roundId, - userId, - playerId, + await this.events.emitInTransaction(tx, 'gaming.round.ended', { roundId, userId, playerId }); + return true; }); return { success: true, winAmount: paid ? winAmount : round.winAmount }; diff --git a/packages/core/src/compliance/__tests__/compliance.service.int.test.ts b/packages/core/src/compliance/__tests__/compliance.service.int.test.ts index 8730ed845..448c97c99 100644 --- a/packages/core/src/compliance/__tests__/compliance.service.int.test.ts +++ b/packages/core/src/compliance/__tests__/compliance.service.int.test.ts @@ -1151,7 +1151,8 @@ describe('ComplianceService per-game geo rules (real PG)', () => { { ip: '1.2.3.4', userAgent: 'agent' }, ); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'compliance.game-geo-rule.upserted', expect.objectContaining({ gameId, @@ -1160,7 +1161,8 @@ describe('ComplianceService per-game geo rules (real PG)', () => { after: expect.objectContaining({ reason: 'updated restriction' }), }), ); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'compliance.game-geo-rule.deleted', expect.objectContaining({ gameId, actorId, reason: 'licence restored', after: null }), ); @@ -1239,9 +1241,9 @@ describe('ComplianceService per-game geo rules (real PG)', () => { ), ]); - const upsertPayloads = events.emit.mock.calls - .filter(([event]) => event === 'compliance.game-geo-rule.upserted') - .map(([, payload]) => payload); + const upsertPayloads = events.emitInTransaction.mock.calls + .filter(([, event]) => event === 'compliance.game-geo-rule.upserted') + .map(([, , payload]) => payload); expect(upsertPayloads).toHaveLength(2); expect(upsertPayloads.filter(({ before }) => before === null)).toHaveLength(1); @@ -1334,7 +1336,7 @@ describe('ComplianceService per-provider geo rules (real PG)', () => { { ip: null, userAgent: null }, ), ).rejects.toBeInstanceOf(GeoRuleProviderNotFoundError); - expect(events.emit).not.toHaveBeenCalled(); + expect(events.emitInTransaction).not.toHaveBeenCalled(); }); it('upserts and deletes a provider rule with auditable before and after state', async () => { @@ -1365,7 +1367,8 @@ describe('ComplianceService per-provider geo rules (real PG)', () => { meta, ); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'compliance.provider-geo-rule.upserted', expect.objectContaining({ providerId, @@ -1374,7 +1377,8 @@ describe('ComplianceService per-provider geo rules (real PG)', () => { after: expect.objectContaining({ reason: 'updated restriction' }), }), ); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'compliance.provider-geo-rule.deleted', expect.objectContaining({ providerId, actorId, reason: 'licence restored', after: null }), ); @@ -2038,7 +2042,10 @@ describe('ComplianceService.replaceGameGeoRules (real PG)', () => { }); expect(rule).toMatchObject({ source: 'admin', reason: 'confirmed by compliance' }); - expect(emitted(events, 'compliance.game-geo-rule.upserted').at(-1)).toMatchObject({ + const adminUpserts = events.emitInTransaction.mock.calls + .filter(([, event]) => event === 'compliance.game-geo-rule.upserted') + .map(([, , payload]) => payload); + expect(adminUpserts.at(-1)).toMatchObject({ before: { source: SOURCE }, after: { source: 'admin' }, }); diff --git a/packages/core/src/compliance/__tests__/kyc-agent-tools.int.test.ts b/packages/core/src/compliance/__tests__/kyc-agent-tools.int.test.ts index 416aadd6f..988850f1f 100644 --- a/packages/core/src/compliance/__tests__/kyc-agent-tools.int.test.ts +++ b/packages/core/src/compliance/__tests__/kyc-agent-tools.int.test.ts @@ -284,9 +284,11 @@ describe('request_enhanced_kyc through the MCP kernel (real PG)', () => { triggeredBy: 'manual', decisionReason: REASON, }); - const updates = events.emit.mock.calls.filter(([topic]) => topic === 'compliance.kyc.updated'); + const updates = events.emitInTransaction.mock.calls.filter( + ([, topic]) => topic === 'compliance.kyc.updated', + ); expect(updates).toHaveLength(1); - expect(updates[0]?.[1]).toMatchObject({ + expect(updates[0]?.[2]).toMatchObject({ userId: row.userId, actorId: adminId, tier: 'advanced', diff --git a/packages/core/src/compliance/__tests__/kyc.service.int.test.ts b/packages/core/src/compliance/__tests__/kyc.service.int.test.ts index 305ff7bf6..657468d91 100644 --- a/packages/core/src/compliance/__tests__/kyc.service.int.test.ts +++ b/packages/core/src/compliance/__tests__/kyc.service.int.test.ts @@ -137,7 +137,8 @@ describe('KycVerificationService.submit (real PG)', () => { expect(result.tier).toBe('advanced'); expect(statusWriter.setStatus).not.toHaveBeenCalled(); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'compliance.kyc.updated', expect.objectContaining({ userId, tier: 'advanced', previousStatus: 'not_started' }), ); @@ -159,7 +160,8 @@ describe('KycVerificationService.submit (real PG)', () => { }); expect(rows[0]?.decidedAt).toBeInstanceOf(Date); expect(result.status).toBe('approved'); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'compliance.kyc.submitted', expect.objectContaining({ userId, referenceId: adapterResult.referenceId }), ); @@ -170,7 +172,7 @@ describe('KycVerificationService.submit (real PG)', () => { expect.anything(), ); const statusCallOrder = vi.mocked(statusWriter.setStatus).mock.invocationCallOrder[0]; - const emitCallOrder = vi.mocked(events.emit).mock.invocationCallOrder[0]; + const emitCallOrder = vi.mocked(events.emitInTransaction).mock.invocationCallOrder[0]; expect(statusCallOrder).toEqual(expect.any(Number)); expect(emitCallOrder).toEqual(expect.any(Number)); expect(statusCallOrder ?? Number.POSITIVE_INFINITY).toBeLessThan(emitCallOrder ?? 0); @@ -453,7 +455,8 @@ describe('KycVerificationService.handleDeposit - threshold re-KYC (real PG)', () expect.objectContaining({ source: 'reverify' }), expect.anything(), ); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'compliance.kyc.reverify_required', expect.objectContaining({ userId }), ); @@ -679,7 +682,8 @@ describe('KycVerificationService.applyExemption (real PG)', () => { after: expect.objectContaining({ source: 'exemption' }), }), ); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'compliance.kyc.updated', expect.objectContaining({ source: 'exemption', auditRecorded: true }), ); diff --git a/packages/core/src/compliance/__tests__/rg-self-service.int.test.ts b/packages/core/src/compliance/__tests__/rg-self-service.int.test.ts index c83a64f65..a70f247a1 100644 --- a/packages/core/src/compliance/__tests__/rg-self-service.int.test.ts +++ b/packages/core/src/compliance/__tests__/rg-self-service.int.test.ts @@ -163,7 +163,8 @@ describe('RgSelfServiceService.upsertLimit (real PG)', () => { expect(view.amount).toBe('100.000000000000000000'); expect(view.pendingAmount).toBe('500.000000000000000000'); expect(view.pendingStatus).toBe('waiting'); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'rg.limit.change_requested', expect.objectContaining({ kind: 'increase', @@ -197,7 +198,8 @@ describe('RgSelfServiceService.upsertLimit (real PG)', () => { expect(view.amount).toBe('20.000000000000000000'); expect(view.pendingStatus).toBeNull(); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'rg.limit.change_cancelled', expect.objectContaining({ requestedAmount: '500.000000000000000000' }), ); @@ -452,11 +454,13 @@ describe('RgSelfServiceService.confirmPendingChange (real PG)', () => { expect(view?.amount).toBe('500.000000000000000000'); expect(view?.pendingStatus).toBeNull(); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'rg.limit.change_confirmed', expect.objectContaining({ kind: 'increase', initiatedBy: 'player' }), ); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'rg.limit.set', expect.objectContaining({ amount: '500.000000000000000000', initiatedBy: 'player' }), ); @@ -489,7 +493,8 @@ describe('RgSelfServiceService.confirmPendingChange (real PG)', () => { const after = await limitRow(userId); expect(after.amount).toBe('100.000000000000000000'); expect(after.pendingKind).toBeNull(); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'rg.limit.change_expired', expect.objectContaining({ requestedAmount: '500.000000000000000000' }), ); @@ -567,11 +572,11 @@ describe('RgSelfServiceService.cancelPendingChange (real PG)', () => { const userId = randomUUID(); await svc.upsertLimit(userId, deposit100); const row = await limitRow(userId); - events.emit.mockClear(); + events.emitInTransaction.mockClear(); await svc.cancelPendingChange(row.id, userId); - expect(events.emit).not.toHaveBeenCalled(); + expect(events.emitInTransaction).not.toHaveBeenCalled(); }); }); @@ -624,8 +629,8 @@ describe('RgSelfServiceService concurrency (real PG)', () => { svc.cancelPendingChange(row.id, userId), ]); - const confirmed = events.emit.mock.calls.filter( - (c: unknown[]) => c[0] === 'rg.limit.change_confirmed', + const confirmed = events.emitInTransaction.mock.calls.filter( + (c: unknown[]) => c[1] === 'rg.limit.change_confirmed', ); expect(confirmed.length).toBeLessThanOrEqual(1); expect(results.some((r) => r.status === 'fulfilled')).toBe(true); @@ -646,7 +651,9 @@ describe('RgSelfServiceService concurrency (real PG)', () => { expect((await limitRow(userId)).amount).toBe('500.000000000000000000'); expect( - events.emit.mock.calls.filter((c: unknown[]) => c[0] === 'rg.limit.change_confirmed'), + events.emitInTransaction.mock.calls.filter( + (c: unknown[]) => c[1] === 'rg.limit.change_confirmed', + ), ).toHaveLength(1); }); @@ -717,7 +724,8 @@ describe('RgSelfServiceService exclusions (real PG)', () => { confirm: true, }); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'rg.self_exclusion.activated', expect.objectContaining({ initiatedBy: 'player', durationMonths: 6 }), ); diff --git a/packages/core/src/compliance/__tests__/rg.router.int.test.ts b/packages/core/src/compliance/__tests__/rg.router.int.test.ts index 686bb7880..da316b729 100644 --- a/packages/core/src/compliance/__tests__/rg.router.int.test.ts +++ b/packages/core/src/compliance/__tests__/rg.router.int.test.ts @@ -224,7 +224,8 @@ describe('compliance RG router writes', () => { const stored = await limitsOf(USER); expect(stored).toHaveLength(1); expect(stored[0]?.period).toBe('daily'); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'rg.limit.set', expect.objectContaining({ userId: USER, diff --git a/packages/core/src/compliance/__tests__/rg.service.int.test.ts b/packages/core/src/compliance/__tests__/rg.service.int.test.ts index 63ac72982..beb64f79a 100644 --- a/packages/core/src/compliance/__tests__/rg.service.int.test.ts +++ b/packages/core/src/compliance/__tests__/rg.service.int.test.ts @@ -132,7 +132,8 @@ describe('RgService.setPlayerLimit (real PG)', () => { ); expect(Number(dto.amount)).toBe(100); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'rg.limit.set', expect.objectContaining({ userId, @@ -183,7 +184,8 @@ describe('RgService.setPlayerLimit (real PG)', () => { const rows = await db.drizzle.db.select().from(userLimit).where(eq(userLimit.userId, userId)); expect(rows).toHaveLength(1); expect(Number(rows[0]?.amount)).toBe(50); - expect(events.emit).toHaveBeenLastCalledWith( + expect(events.emitInTransaction).toHaveBeenLastCalledWith( + expect.anything(), 'rg.limit.set', expect.objectContaining({ previousAmount: '100.000000000000000000', @@ -914,7 +916,8 @@ describe('RgService.activateCoolingOff (real PG)', () => { expect(row).toMatchObject({ kind: 'cooling_off', status: 'active', isPermanent: false }); expect(enforcement.block).toHaveBeenCalledWith(userId, { until: expect.any(Date) }); expect(enforcement.unblock).not.toHaveBeenCalled(); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'rg.cooling_off.activated', expect.objectContaining({ userId }), ); @@ -957,7 +960,8 @@ describe('RgService.activateCoolingOff (real PG)', () => { expect(rows).toHaveLength(2); expect(rows.find((r) => r.id === lapsed.id)?.status).toBe('expired'); expect(rows.filter((r) => r.status === 'active')).toHaveLength(1); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'rg.cooling_off.expired', expect.objectContaining({ userId, @@ -998,7 +1002,8 @@ describe('RgService.activateSelfExclusion (real PG)', () => { const [row] = await exclusionsOf(userId); expect(row).toMatchObject({ kind: 'self_exclusion', isPermanent: true, expiresAt: null }); expect(enforcement.block).toHaveBeenCalledWith(userId, { until: null }); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'rg.self_exclusion.activated', expect.objectContaining({ isPermanent: true, durationMonths: null, expiresAt: null }), ); @@ -1019,7 +1024,8 @@ describe('RgService.activateSelfExclusion (real PG)', () => { const [row] = await exclusionsOf(userId); expect(row?.expiresAt).toBeInstanceOf(Date); expect(enforcement.block).toHaveBeenCalledWith(userId, { until: null }); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'rg.self_exclusion.activated', expect.objectContaining({ userId, @@ -1098,7 +1104,8 @@ describe('RgService.liftSelfExclusion (real PG)', () => { }); expect(row?.liftedAt).toBeInstanceOf(Date); expect(enforcement.unblock).toHaveBeenCalledWith(userId); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'rg.self_exclusion.lifted', expect.objectContaining({ userId, actorId }), ); @@ -1161,7 +1168,8 @@ describe('RgService.liftCoolingOff (real PG)', () => { liftedBy: actorId, }); expect(enforcement.unblock).toHaveBeenCalledWith(userId); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'rg.cooling_off.lifted', expect.objectContaining({ userId, actorId, reason: 'support ticket 42' }), ); @@ -1233,7 +1241,8 @@ describe('RgService.expireLapsedCoolingOffs (real PG)', () => { expect(running?.status).toBe('active'); expect(enforcement.unblock).toHaveBeenCalledWith(lapsedUser); expect(enforcement.unblock).not.toHaveBeenCalledWith(runningUser); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'rg.cooling_off.expired', expect.objectContaining({ userId: lapsedUser, @@ -1241,7 +1250,8 @@ describe('RgService.expireLapsedCoolingOffs (real PG)', () => { expiresAt: lapsedExpiresAt.toISOString(), }), ); - expect(events.emit).not.toHaveBeenCalledWith( + expect(events.emitInTransaction).not.toHaveBeenCalledWith( + expect.anything(), 'rg.cooling_off.expired', expect.objectContaining({ userId: runningUser }), ); @@ -1254,7 +1264,11 @@ describe('RgService.expireLapsedCoolingOffs (real PG)', () => { await svc.expireLapsedCoolingOffs(); - expect(events.emit).not.toHaveBeenCalledWith('rg.cooling_off.expired', expect.anything()); + expect(events.emitInTransaction).not.toHaveBeenCalledWith( + expect.anything(), + 'rg.cooling_off.expired', + expect.anything(), + ); }); it('keeps the indefinite block when the player is also self-excluded', async () => { diff --git a/packages/core/src/compliance/service/compliance.service.ts b/packages/core/src/compliance/service/compliance.service.ts index 0b89a9422..c41572952 100644 --- a/packages/core/src/compliance/service/compliance.service.ts +++ b/packages/core/src/compliance/service/compliance.service.ts @@ -920,7 +920,7 @@ export class ComplianceService { new GeoRuleGameNotFoundError(input.gameId), ); - return withGameGeoRuleLocks(tx, input.gameId, countryCodes, async () => { + const paired = await withGameGeoRuleLocks(tx, input.gameId, countryCodes, async () => { const before = await tx .select() .from(gameGeoRule) @@ -946,21 +946,23 @@ export class ComplianceService { .returning(); return pairGeoRuleChanges(before, rows); }); + // The audit trail for this change is the event's subscriber, so it commits with the rows. + for (const { before, after } of paired) { + await this.events.emitInTransaction(tx, 'compliance.game-geo-rule.upserted', { + ruleId: after.id, + gameId: input.gameId, + countryCode: after.countryCode, + reason: input.reason, + before, + after, + actorId, + ip: meta.ip, + userAgent: meta.userAgent, + }); + } + return paired; }); - for (const { before, after } of changes) { - this.events.emit('compliance.game-geo-rule.upserted', { - ruleId: after.id, - gameId: input.gameId, - countryCode: after.countryCode, - reason: input.reason, - before, - after, - actorId, - ip: meta.ip, - userAgent: meta.userAgent, - }); - } return changes.map(({ after }) => after); } @@ -981,25 +983,26 @@ export class ComplianceService { if (missing.length > 0) { throw new GameGeoRuleNotFoundError(`${input.gameId}:${missing.join(',')}`); } - return rows + const removed = rows .map(serializeGeoRule) .sort((a, b) => a.countryCode.localeCompare(b.countryCode)); + for (const before of removed) { + await this.events.emitInTransaction(tx, 'compliance.game-geo-rule.deleted', { + ruleId: before.id, + gameId: before.gameId, + countryCode: before.countryCode, + reason: input.reason, + before, + after: null, + actorId, + ip: meta.ip, + userAgent: meta.userAgent, + }); + } + return removed; }), ); - for (const before of deleted) { - this.events.emit('compliance.game-geo-rule.deleted', { - ruleId: before.id, - gameId: before.gameId, - countryCode: before.countryCode, - reason: input.reason, - before, - after: null, - actorId, - ip: meta.ip, - userAgent: meta.userAgent, - }); - } return deleted; } @@ -1384,7 +1387,7 @@ export class ComplianceService { new GeoRuleProviderNotFoundError(input.providerId), ); - return withAdvisoryXactLocks( + const paired = await withAdvisoryXactLocks( tx, countryCodes.map((countryCode) => providerGeoRuleLockKey(input.providerId, countryCode)), async () => { @@ -1414,21 +1417,22 @@ export class ComplianceService { return pairGeoRuleChanges(before, rows); }, ); + for (const { before, after } of paired) { + await this.events.emitInTransaction(tx, 'compliance.provider-geo-rule.upserted', { + ruleId: after.id, + providerId: input.providerId, + countryCode: after.countryCode, + reason: input.reason, + before, + after, + actorId, + ip: meta.ip, + userAgent: meta.userAgent, + }); + } + return paired; }); - for (const { before, after } of changes) { - this.events.emit('compliance.provider-geo-rule.upserted', { - ruleId: after.id, - providerId: input.providerId, - countryCode: after.countryCode, - reason: input.reason, - before, - after, - actorId, - ip: meta.ip, - userAgent: meta.userAgent, - }); - } return changes.map(({ after }) => after); } @@ -1456,26 +1460,27 @@ export class ComplianceService { if (missing.length > 0) { throw new ProviderGeoRuleNotFoundError(`${input.providerId}:${missing.join(',')}`); } - return rows + const removed = rows .map(serializeGeoRule) .sort((a, b) => a.countryCode.localeCompare(b.countryCode)); + for (const before of removed) { + await this.events.emitInTransaction(tx, 'compliance.provider-geo-rule.deleted', { + ruleId: before.id, + providerId: before.providerId, + countryCode: before.countryCode, + reason: input.reason, + before, + after: null, + actorId, + ip: meta.ip, + userAgent: meta.userAgent, + }); + } + return removed; }, ), ); - for (const before of deleted) { - this.events.emit('compliance.provider-geo-rule.deleted', { - ruleId: before.id, - providerId: before.providerId, - countryCode: before.countryCode, - reason: input.reason, - before, - after: null, - actorId, - ip: meta.ip, - userAgent: meta.userAgent, - }); - } return deleted; } diff --git a/packages/core/src/compliance/service/kyc.service.ts b/packages/core/src/compliance/service/kyc.service.ts index 5b150c845..6643809ba 100644 --- a/packages/core/src/compliance/service/kyc.service.ts +++ b/packages/core/src/compliance/service/kyc.service.ts @@ -243,25 +243,28 @@ export class KycVerificationService { return this.platformConfig?.kyc?.provider ?? DEFAULT_PROVIDER; } - private async emitUpdated(params: { - userId: User['id']; - playerId?: Player['id']; - tier: KycTier; - status: KycStatus; - previousStatus: KycStatus | null; - playerTransition: KycStatusTransition | null; - actorId: User['id'] | null; - reason: string | null; - source: 'vendor' | 'manual' | 'webhook' | 'reverify' | 'exemption'; - auditRecorded?: true; - }) { + private async emitUpdated( + trx: DrizzleTx, + params: { + userId: User['id']; + playerId?: Player['id']; + tier: KycTier; + status: KycStatus; + previousStatus: KycStatus | null; + playerTransition: KycStatusTransition | null; + actorId: User['id'] | null; + reason: string | null; + source: 'vendor' | 'manual' | 'webhook' | 'reverify' | 'exemption'; + auditRecorded?: true; + }, + ) { if (params.tier === 'basic' && !params.playerTransition && !params.playerId) { return; } if (params.tier === 'advanced' && params.previousStatus === params.status) { return; } - this.events.emit('compliance.kyc.updated', { + await this.events.emitInTransaction(trx, 'compliance.kyc.updated', { userId: params.userId, playerId: params.playerTransition?.playerId ?? @@ -366,9 +369,9 @@ export class KycVerificationService { ) : null; // compliance.kyc.updated is the audit-visible status-change event (docs/standards/ - // compliance.md): emitted here, inside the transaction, so it can never be dropped - // by a crash between commit and a post-commit emit. - await this.emitUpdated({ + // compliance.md): written to the outbox inside the transaction, so it can never be + // dropped by a crash between commit and publish. + await this.emitUpdated(trx, { userId, tier: input.tier, status, @@ -378,6 +381,17 @@ export class KycVerificationService { reason: decisionReason, source: 'vendor', }); + await this.events.emitInTransaction(trx, 'compliance.kyc.submitted', { + userId, + playerId: + playerTransition?.playerId ?? + (await this.identityReader.getPlayerIdByUserIdSafe(userId)), + referenceId: result.referenceId, + provider: this.provider, + tier: input.tier, + ip: meta?.ip ?? null, + userAgent: meta?.userAgent ?? null, + }); return { duplicate: false as const, row: inserted, @@ -391,15 +405,6 @@ export class KycVerificationService { return { ...toDto(outcome.row), verificationUrl: undefined }; } - this.events.emit('compliance.kyc.submitted', { - userId, - playerId: await this.identityReader.getPlayerIdByUserIdSafe(userId), - referenceId: outcome.referenceId, - provider: this.provider, - tier: input.tier, - ip: meta?.ip ?? null, - userAgent: meta?.userAgent ?? null, - }); return { ...toDto(outcome.row), verificationUrl: outcome.verificationUrl }; } @@ -528,9 +533,9 @@ export class KycVerificationService { ) : null; // compliance.kyc.updated is the audit-visible status-change event (docs/standards/ - // compliance.md): emitted here, inside the transaction, so it can never be dropped - // by a crash between commit and a post-commit emit. - await this.emitUpdated({ + // compliance.md): written to the outbox inside the transaction, so it can never be + // dropped by a crash between commit and publish. + await this.emitUpdated(trx, { userId: row.userId, tier: row.tier, status, @@ -549,21 +554,18 @@ export class KycVerificationService { }); } - return reconciled; - }); - - if (outcomes.length === 0) { - return null; - } - - // Emitted once per reconcile() call, not once per row: the signal describes the - // vendor SESSION (referenceId), not a tier - a shared-workflow decision touching - // both tiers is still exactly one real signal. - const anyChanged = outcomes.some((outcome) => outcome.changed); - if (anyChanged && opts.riskSignals && warrantsHighRiskTag(opts.riskSignals)) { - const [primaryOutcome] = outcomes; - if (primaryOutcome) { - this.events.emit('compliance.kyc.high_risk_signal_detected', { + // Emitted once per reconcile() call, not once per row: the signal describes the + // vendor SESSION (referenceId), not a tier - a shared-workflow decision touching + // both tiers is still exactly one real signal. + const anyChanged = reconciled.some((outcome) => outcome.changed); + const [primaryOutcome] = reconciled; + if ( + primaryOutcome && + anyChanged && + opts.riskSignals && + warrantsHighRiskTag(opts.riskSignals) + ) { + await this.events.emitInTransaction(trx, 'compliance.kyc.high_risk_signal_detected', { userId: primaryOutcome.row.userId, playerId: await this.identityReader.getPlayerIdByUserIdSafe(primaryOutcome.row.userId), referenceId, @@ -574,6 +576,12 @@ export class KycVerificationService { highRiskCountryDetected: opts.riskSignals.highRiskCountryDetected, }); } + + return reconciled; + }); + + if (outcomes.length === 0) { + return null; } const primary = outcomes.find((outcome) => outcome.changed) ?? outcomes[0]; @@ -784,9 +792,9 @@ export class KycVerificationService { trx, ); // compliance.kyc.updated is the audit-visible status-change event (docs/standards/ - // compliance.md): emitted here, inside the transaction, so it can never be dropped - // by a crash between commit and a post-commit emit. - await this.emitUpdated({ + // compliance.md): written to the outbox inside the transaction, so it can never be + // dropped by a crash between commit and publish. + await this.emitUpdated(trx, { userId, tier: 'basic', status: 'resubmission_requested', @@ -796,12 +804,12 @@ export class KycVerificationService { reason, source: 'reverify', }); - }); - this.events.emit('compliance.kyc.reverify_required', { - userId, - playerId: current.id, - reason, - tier: 'basic', + await this.events.emitInTransaction(trx, 'compliance.kyc.reverify_required', { + userId, + playerId: current.id, + reason, + tier: 'basic', + }); }); } @@ -954,9 +962,9 @@ export class KycVerificationService { decidedAt: null, }); // compliance.kyc.updated is the audit-visible status-change event (docs/standards/ - // compliance.md): emitted here, inside the transaction, so it can never be dropped - // by a crash between commit and a post-commit emit. - await this.emitUpdated({ + // compliance.md): written to the outbox inside the transaction, so it can never be + // dropped by a crash between commit and publish. + await this.emitUpdated(trx, { userId, tier, status: 'resubmission_requested', @@ -1008,9 +1016,9 @@ export class KycVerificationService { decidedAt: isDecided(resolvedStatus) ? new Date() : null, }); // compliance.kyc.updated is the audit-visible status-change event (docs/standards/ - // compliance.md): emitted here, inside the transaction, so it can never be dropped - // by a crash between commit and a post-commit emit. - await this.emitUpdated({ + // compliance.md): written to the outbox inside the transaction, so it can never be + // dropped by a crash between commit and publish. + await this.emitUpdated(trx, { userId, tier, status: resolvedStatus, @@ -1069,12 +1077,12 @@ export class KycVerificationService { userId: User['id'], params: { countryCode: string | null; reason: 'global_disabled' | 'country_exempt' }, ): Promise { - const outcome = await this.drizzle.db.transaction((trx) => + await this.drizzle.db.transaction((trx) => withAdvisoryXactLock(trx, `kyc-submit:${userId}:basic`, async () => { const current = await this.requirePlayerRowForUpdate(userId, trx); const existing = await this.latestVerification(userId, 'basic', trx); if (existing) { - return null; + return; } const decisionReason = @@ -1117,32 +1125,23 @@ export class KycVerificationService { source: 'exemption', }, }); - return { - decisionReason, + // The audit record above already covers this change for the regulator; the event + // still serves realtime/tag consumers, and auditRecorded keeps its subscriber from + // appending a duplicate audit row. + await this.emitUpdated(trx, { + userId, playerId: playerTransition?.playerId ?? current.id, + tier: 'basic', + status: 'approved', previousStatus: current.kycStatus, playerTransition, - }; + actorId: null, + reason: decisionReason, + source: 'exemption', + auditRecorded: true, + }); }), ); - if (!outcome) { - return; - } - // The audit record above committed with the KYC state. This post-commit event remains - // available to realtime/tag consumers; auditRecorded prevents its subscriber from - // appending a duplicate audit row. - await this.emitUpdated({ - userId, - playerId: outcome.playerId, - tier: 'basic', - status: 'approved', - previousStatus: outcome.previousStatus, - playerTransition: outcome.playerTransition, - actorId: null, - reason: outcome.decisionReason, - source: 'exemption', - auditRecorded: true, - }); } private async requirePlayerRowForUpdate(userId: User['id'], tx: DrizzleTx) { diff --git a/packages/core/src/compliance/service/rg-self-service.service.ts b/packages/core/src/compliance/service/rg-self-service.service.ts index 3edeef3fc..1c4ded252 100644 --- a/packages/core/src/compliance/service/rg-self-service.service.ts +++ b/packages/core/src/compliance/service/rg-self-service.service.ts @@ -144,6 +144,7 @@ export class RgSelfServiceService { input: UpsertLimitInput, meta?: ClientMeta, ): Promise { + const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); const outcome = await this.drizzle.db.transaction((tx) => withAdvisoryXactLock(tx, limitSlotKey(userId, input.type, input.period), async () => { const [existing] = await tx @@ -164,46 +165,43 @@ export class RgSelfServiceService { // The pending amount is the effective one being validated - it's what // applies once the player confirms it. await this.assertOrdering(tx, userId, input); - return { - applied: false as const, - existing, - row: await this.park(tx, existing, 'increase', input), - }; + const parked = await this.park(tx, existing, 'increase', input); + await this.emitRequested(tx, userId, playerId, existing, parked, meta); + return { applied: false as const, row: parked }; } } await this.assertOrdering(tx, userId, input); const row = await writeLimitRow(tx, userId, existing, input); - return { applied: true as const, existing: existing ?? null, row }; + if (existing?.pendingKind) { + await this.events.emitInTransaction( + tx, + 'rg.limit.change_cancelled', + this.actedPayload(userId, playerId, existing, meta), + ); + } + await this.events.emitInTransaction(tx, 'rg.limit.set', { + userId, + playerId, + actorId: userId, + limitId: row.id, + type: row.type, + period: row.period, + amount: row.amount, + minutes: row.minutes, + previousAmount: existing?.amount ?? null, + previousMinutes: existing?.minutes ?? null, + initiatedBy: 'player', + reason: null, + ip: meta?.ip ?? null, + userAgent: meta?.userAgent ?? null, + }); + return { applied: true as const, row }; }), ); - const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); if (!outcome.applied) { - this.emitRequested(userId, playerId, outcome.existing, outcome.row, meta); return this.toView(userId, outcome.row); } - if (outcome.existing?.pendingKind) { - this.events.emit( - 'rg.limit.change_cancelled', - this.actedPayload(userId, playerId, outcome.existing, meta), - ); - } - this.events.emit('rg.limit.set', { - userId, - playerId, - actorId: userId, - limitId: outcome.row.id, - type: outcome.row.type, - period: outcome.row.period, - amount: outcome.row.amount, - minutes: outcome.row.minutes, - previousAmount: outcome.existing?.amount ?? null, - previousMinutes: outcome.existing?.minutes ?? null, - initiatedBy: 'player', - reason: null, - ip: meta?.ip ?? null, - userAgent: meta?.userAgent ?? null, - }); await this.rg.notifyLimitUpdated(userId, outcome.row, 'player'); return this.toView(userId, outcome.row); } @@ -215,26 +213,19 @@ export class RgSelfServiceService { meta?: ClientMeta, ): Promise { const target = await this.ownedLimit(limitId, userId); - const { existing, row } = await this.drizzle.db.transaction((tx) => + const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); + const row = await this.drizzle.db.transaction((tx) => withAdvisoryXactLock(tx, limitSlotKey(userId, target.type, target.period), async () => { const current = await this.reread(tx, limitId, userId); - return { - existing: current, - row: await this.park(tx, current, 'removal', { - amount: null, - minutes: null, - currency: null, - }), - }; + const parked = await this.park(tx, current, 'removal', { + amount: null, + minutes: null, + currency: null, + }); + await this.emitRequested(tx, userId, playerId, current, parked, meta); + return parked; }), ); - this.emitRequested( - userId, - await this.identityReader.getPlayerIdByUserIdSafe(userId), - existing, - row, - meta, - ); return this.toView(userId, row); } @@ -248,6 +239,7 @@ export class RgSelfServiceService { meta?: ClientMeta, ): Promise { const target = await this.ownedLimit(limitId, userId); + const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); const outcome = await this.drizzle.db.transaction((tx) => withAdvisoryXactLock(tx, limitSlotKey(userId, target.type, target.period), async () => { @@ -268,9 +260,20 @@ export class RgSelfServiceService { .set(NO_PENDING_CHANGE) .where(this.pinnedTo(current)) .returning({ id: userLimit.id }); + await this.events.emitInTransaction( + tx, + 'rg.limit.change_expired', + this.lapsedPayload(userId, playerId, current), + ); return { kind: 'expired' as const, existing: current }; } + await this.events.emitInTransaction( + tx, + 'rg.limit.change_confirmed', + this.actedPayload(userId, playerId, current, meta), + ); + if (current.pendingKind === 'removal') { const deleted = await tx .delete(userLimit) @@ -295,47 +298,36 @@ export class RgSelfServiceService { .returning(), new NoPendingLimitChangeError(limitId), ); + await this.events.emitInTransaction(tx, 'rg.limit.set', { + userId, + playerId, + actorId: userId, + limitId: row.id, + type: row.type, + period: row.period, + amount: row.amount, + minutes: row.minutes, + previousAmount: current.amount, + previousMinutes: current.minutes, + initiatedBy: 'player', + reason: null, + ip: meta?.ip ?? null, + userAgent: meta?.userAgent ?? null, + }); return { kind: 'raised' as const, existing: current, row }; }), ); - const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); - if (outcome.kind === 'expired') { - this.events.emit( - 'rg.limit.change_expired', - this.lapsedPayload(userId, playerId, outcome.existing), - ); throw new LimitChangeExpiredError(); } - this.events.emit( - 'rg.limit.change_confirmed', - this.actedPayload(userId, playerId, outcome.existing, meta), - ); - if (outcome.kind === 'removed') { await this.monitoring.clearLimitThresholdFlag(userId, outcome.existing.type); await this.monitoring.evaluateUser(userId, 'rg.limit.set'); return null; } - this.events.emit('rg.limit.set', { - userId, - playerId, - actorId: userId, - limitId: outcome.row.id, - type: outcome.row.type, - period: outcome.row.period, - amount: outcome.row.amount, - minutes: outcome.row.minutes, - previousAmount: outcome.existing.amount, - previousMinutes: outcome.existing.minutes, - initiatedBy: 'player', - reason: null, - ip: meta?.ip ?? null, - userAgent: meta?.userAgent ?? null, - }); await this.rg.notifyLimitUpdated(userId, outcome.row, 'player'); return this.toView(userId, outcome.row); } @@ -346,32 +338,30 @@ export class RgSelfServiceService { meta?: ClientMeta, ): Promise { const target = await this.ownedLimit(limitId, userId); - const outcome = await this.drizzle.db.transaction((tx) => + const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); + const row = await this.drizzle.db.transaction((tx) => withAdvisoryXactLock(tx, limitSlotKey(userId, target.type, target.period), async () => { const current = await this.reread(tx, limitId, userId); if (current.pendingKind === null) { - return { cleared: false as const, row: current }; + return current; } - const rows = await tx + const [cleared] = await tx .update(userLimit) .set(NO_PENDING_CHANGE) .where(this.pinnedTo(current)) .returning(); - return { cleared: rows.length > 0, existing: current, row: rows[0] ?? current }; + if (!cleared) { + return current; + } + await this.events.emitInTransaction( + tx, + 'rg.limit.change_cancelled', + this.actedPayload(userId, playerId, current, meta), + ); + return cleared; }), ); - if (outcome.cleared && outcome.existing) { - this.events.emit( - 'rg.limit.change_cancelled', - this.actedPayload( - userId, - await this.identityReader.getPlayerIdByUserIdSafe(userId), - outcome.existing, - meta, - ), - ); - } - return this.toView(userId, outcome.row); + return this.toView(userId, row); } requestCoolingOff( @@ -423,42 +413,44 @@ export class RgSelfServiceService { if (candidates.length === 0) { return; } - const cleared = await this.drizzle.db - .update(userLimit) - .set(NO_PENDING_CHANGE) - .where( - and( - inArray( - userLimit.id, - candidates.map((c) => c.id), - ), - isNotNull(userLimit.pendingKind), - lte(userLimit.pendingExpiresAt, now), - ), - ) - .returning({ id: userLimit.id }); - const clearedIds = new Set(cleared.map((r) => r.id)); const playerIds = await this.identityReader.getPlayerIdsByUserIdsSafe( candidates.map((c) => c.userId), ); - for (const row of candidates) { - if (!clearedIds.has(row.id)) { - continue; + await this.drizzle.db.transaction(async (tx) => { + const cleared = await tx + .update(userLimit) + .set(NO_PENDING_CHANGE) + .where( + and( + inArray( + userLimit.id, + candidates.map((c) => c.id), + ), + isNotNull(userLimit.pendingKind), + lte(userLimit.pendingExpiresAt, now), + ), + ) + .returning({ id: userLimit.id }); + const clearedIds = new Set(cleared.map((r) => r.id)); + for (const row of candidates) { + if (!clearedIds.has(row.id)) { + continue; + } + await this.events.emitInTransaction(tx, 'rg.limit.change_expired', { + userId: row.userId, + playerId: playerIds.get(row.userId) ?? null, + limitId: row.id, + type: row.type, + period: row.period, + kind: row.pendingKind as 'increase' | 'removal', + previousAmount: row.amount, + previousMinutes: row.minutes, + requestedAmount: row.pendingAmount, + requestedMinutes: row.pendingMinutes, + expiresAt: (row.pendingExpiresAt ?? now).toISOString(), + }); } - this.events.emit('rg.limit.change_expired', { - userId: row.userId, - playerId: playerIds.get(row.userId) ?? null, - limitId: row.id, - type: row.type, - period: row.period, - kind: row.pendingKind as 'increase' | 'removal', - previousAmount: row.amount, - previousMinutes: row.minutes, - requestedAmount: row.pendingAmount, - requestedMinutes: row.pendingMinutes, - expiresAt: (row.pendingExpiresAt ?? now).toISOString(), - }); - } + }); } private async assertOrdering( @@ -536,13 +528,14 @@ export class RgSelfServiceService { } private emitRequested( + tx: DrizzleTx, userId: User['id'], playerId: string | null, previous: LimitRow, row: LimitRow, meta?: ClientMeta, - ): void { - this.events.emit('rg.limit.change_requested', { + ): Promise { + return this.events.emitInTransaction(tx, 'rg.limit.change_requested', { userId, playerId, actorId: userId, diff --git a/packages/core/src/compliance/service/rg.service.ts b/packages/core/src/compliance/service/rg.service.ts index 7bb58b64a..e0b6fcaf3 100644 --- a/packages/core/src/compliance/service/rg.service.ts +++ b/packages/core/src/compliance/service/rg.service.ts @@ -415,7 +415,8 @@ export class RgService { initiatedBy: RgInitiator, meta?: ClientMeta, ): Promise { - const { prior, row } = await this.drizzle.db.transaction((tx) => + const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); + const row = await this.drizzle.db.transaction((tx) => withAdvisoryXactLock(tx, limitSlotKey(userId, input.type, input.period), async () => { const [existing] = await tx .select() @@ -446,44 +447,44 @@ export class RgService { this.rates, ); } - return { prior: existing, row: await writeLimitRow(tx, userId, existing, input) }; + const written = await writeLimitRow(tx, userId, existing, input); + if (existing?.pendingKind) { + await this.events.emitInTransaction(tx, 'rg.limit.change_cancelled', { + userId, + playerId, + actorId, + limitId: written.id, + type: input.type, + period: input.period, + kind: existing.pendingKind, + previousAmount: existing.amount, + previousMinutes: existing.minutes, + requestedAmount: existing.pendingAmount, + requestedMinutes: existing.pendingMinutes, + initiatedBy, + ip: meta?.ip ?? null, + userAgent: meta?.userAgent ?? null, + }); + } + await this.events.emitInTransaction(tx, 'rg.limit.set', { + userId, + playerId, + actorId, + limitId: written.id, + type: input.type, + period: input.period, + amount: input.amount, + minutes: input.minutes, + previousAmount: existing?.amount ?? null, + previousMinutes: existing?.minutes ?? null, + initiatedBy, + reason: input.reason, + ip: meta?.ip ?? null, + userAgent: meta?.userAgent ?? null, + }); + return written; }), ); - const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); - if (prior?.pendingKind) { - this.events.emit('rg.limit.change_cancelled', { - userId, - playerId, - actorId, - limitId: row.id, - type: input.type, - period: input.period, - kind: prior.pendingKind, - previousAmount: prior.amount, - previousMinutes: prior.minutes, - requestedAmount: prior.pendingAmount, - requestedMinutes: prior.pendingMinutes, - initiatedBy, - ip: meta?.ip ?? null, - userAgent: meta?.userAgent ?? null, - }); - } - this.events.emit('rg.limit.set', { - userId, - playerId, - actorId, - limitId: row.id, - type: input.type, - period: input.period, - amount: input.amount, - minutes: input.minutes, - previousAmount: prior?.amount ?? null, - previousMinutes: prior?.minutes ?? null, - initiatedBy, - reason: input.reason, - ip: meta?.ip ?? null, - userAgent: meta?.userAgent ?? null, - }); await this.notifyLimitUpdated(userId, row, initiatedBy); return toLimitDto(row); } @@ -520,7 +521,8 @@ export class RgService { await this.assertNoActiveExclusion(userId, 'cooling_off'); const now = new Date(); const expiresAt = new Date(now.getTime() + input.durationHours * HOUR_MS); - const { row, lapsed } = await this.drizzle.db.transaction(async (tx) => { + const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); + const row = await this.drizzle.db.transaction(async (tx) => { const lapsedRows = await this.expireLapsedCoolingOff(userId, tx, now); const r = findOneOrThrow( await tx @@ -539,27 +541,26 @@ export class RgService { new ExclusionNotFoundError(userId), ); await this.syncEnforcement(userId, tx); - return { row: r, lapsed: lapsedRows }; - }); - const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); - for (const lapsedRow of lapsed) { - this.events.emit('rg.cooling_off.expired', { + for (const lapsedRow of lapsedRows) { + await this.events.emitInTransaction(tx, 'rg.cooling_off.expired', { + userId, + playerId, + exclusionId: lapsedRow.id, + expiresAt: (lapsedRow.expiresAt ?? now).toISOString(), + }); + } + await this.events.emitInTransaction(tx, 'rg.cooling_off.activated', { userId, playerId, - exclusionId: lapsedRow.id, - expiresAt: (lapsedRow.expiresAt ?? now).toISOString(), + actorId, + exclusionId: r.id, + expiresAt: expiresAt.toISOString(), + reason: input.reason, + initiatedBy, + ip: meta?.ip ?? null, + userAgent: meta?.userAgent ?? null, }); - } - this.events.emit('rg.cooling_off.activated', { - userId, - playerId, - actorId, - exclusionId: row.id, - expiresAt: expiresAt.toISOString(), - reason: input.reason, - initiatedBy, - ip: meta?.ip ?? null, - userAgent: meta?.userAgent ?? null, + return r; }); await this.notify( userId, @@ -591,6 +592,7 @@ export class RgService { input.isPermanent || input.durationMonths === undefined ? null : addMonths(now, input.durationMonths); + const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); const row = await this.drizzle.db.transaction(async (tx) => { const r = findOneOrThrow( await tx @@ -609,21 +611,21 @@ export class RgService { new ExclusionNotFoundError(userId), ); await this.syncEnforcement(userId, tx); + await this.events.emitInTransaction(tx, 'rg.self_exclusion.activated', { + userId, + playerId, + actorId, + exclusionId: r.id, + isPermanent: input.isPermanent, + durationMonths: input.isPermanent ? null : (input.durationMonths ?? null), + expiresAt: expiresAt ? expiresAt.toISOString() : null, + reason: input.reason, + initiatedBy, + ip: meta?.ip ?? null, + userAgent: meta?.userAgent ?? null, + }); return r; }); - this.events.emit('rg.self_exclusion.activated', { - userId, - playerId: await this.identityReader.getPlayerIdByUserIdSafe(userId), - actorId, - exclusionId: row.id, - isPermanent: input.isPermanent, - durationMonths: input.isPermanent ? null : (input.durationMonths ?? null), - expiresAt: expiresAt ? expiresAt.toISOString() : null, - reason: input.reason, - initiatedBy, - ip: meta?.ip ?? null, - userAgent: meta?.userAgent ?? null, - }); await this.notify( userId, { @@ -674,6 +676,7 @@ export class RgService { } const now = new Date(); + const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); const row = await this.drizzle.db.transaction(async (tx) => { const r = findOneOrThrow( await tx @@ -686,18 +689,18 @@ export class RgService { // Recompute from what remains - a still-active cooling-off keeps its own block // rather than being cleared by this lift. await this.syncEnforcement(userId, tx); + await this.events.emitInTransaction(tx, 'rg.self_exclusion.lifted', { + userId, + playerId, + actorId, + exclusionId: r.id, + kind: 'self_exclusion', + reason: input.reason, + ip: meta?.ip ?? null, + userAgent: meta?.userAgent ?? null, + }); return r; }); - this.events.emit('rg.self_exclusion.lifted', { - userId, - playerId: await this.identityReader.getPlayerIdByUserIdSafe(userId), - actorId, - exclusionId: row.id, - kind: 'self_exclusion', - reason: input.reason, - ip: meta?.ip ?? null, - userAgent: meta?.userAgent ?? null, - }); await this.notify( userId, { key: 'rgSelfExclusionLifted', data: { initiatedBy: 'admin' } }, @@ -734,6 +737,7 @@ export class RgService { } const now = new Date(); + const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); const row = await this.drizzle.db.transaction(async (tx) => { const r = findOneOrThrow( await tx @@ -744,17 +748,17 @@ export class RgService { new ExclusionNotFoundError(existing.id), ); await this.syncEnforcement(userId, tx); + await this.events.emitInTransaction(tx, 'rg.cooling_off.lifted', { + userId, + playerId, + actorId, + exclusionId: r.id, + reason: input.reason, + ip: meta?.ip ?? null, + userAgent: meta?.userAgent ?? null, + }); return r; }); - this.events.emit('rg.cooling_off.lifted', { - userId, - playerId: await this.identityReader.getPlayerIdByUserIdSafe(userId), - actorId, - exclusionId: row.id, - reason: input.reason, - ip: meta?.ip ?? null, - userAgent: meta?.userAgent ?? null, - }); await this.notify( userId, { key: 'rgCoolingOffLifted', data: { initiatedBy: 'admin' } }, @@ -789,34 +793,37 @@ export class RgService { // recompute enforcement for the affected players. Called by the rg-monitor sweep. async expireLapsedCoolingOffs(): Promise { const now = new Date(); - const lapsed = await this.drizzle.db - .update(rgExclusion) - .set({ status: 'expired' }) - .where( - and( - eq(rgExclusion.kind, 'cooling_off'), - eq(rgExclusion.status, 'active'), - lte(rgExclusion.expiresAt, now), - ), - ) - .returning({ - id: rgExclusion.id, - userId: rgExclusion.userId, - expiresAt: rgExclusion.expiresAt, - }); + const lapsed = await this.drizzle.db.transaction(async (tx) => { + const expired = await tx + .update(rgExclusion) + .set({ status: 'expired' }) + .where( + and( + eq(rgExclusion.kind, 'cooling_off'), + eq(rgExclusion.status, 'active'), + lte(rgExclusion.expiresAt, now), + ), + ) + .returning({ + id: rgExclusion.id, + userId: rgExclusion.userId, + expiresAt: rgExclusion.expiresAt, + }); + const playerIds = await this.identityReader.getPlayerIdsByUserIdsSafe( + expired.map((r) => r.userId), + ); + for (const row of expired) { + await this.events.emitInTransaction(tx, 'rg.cooling_off.expired', { + userId: row.userId, + playerId: playerIds.get(row.userId) ?? null, + exclusionId: row.id, + expiresAt: (row.expiresAt ?? now).toISOString(), + }); + } + return expired; + }); const lapsedUserIds = [...new Set(lapsed.map((r) => r.userId))]; await mapConcurrent(lapsedUserIds, SWEEP_CONCURRENCY, (userId) => this.syncEnforcement(userId)); - const playerIds = await this.identityReader.getPlayerIdsByUserIdsSafe( - lapsed.map((r) => r.userId), - ); - for (const row of lapsed) { - this.events.emit('rg.cooling_off.expired', { - userId: row.userId, - playerId: playerIds.get(row.userId) ?? null, - exclusionId: row.id, - expiresAt: (row.expiresAt ?? now).toISOString(), - }); - } await mapConcurrent(lapsed, SWEEP_CONCURRENCY, (row) => this.notify( diff --git a/packages/core/src/pam/player-management/__tests__/player.service.int.test.ts b/packages/core/src/pam/player-management/__tests__/player.service.int.test.ts index 6f87e8d13..24ddff0bf 100644 --- a/packages/core/src/pam/player-management/__tests__/player.service.int.test.ts +++ b/packages/core/src/pam/player-management/__tests__/player.service.int.test.ts @@ -509,13 +509,17 @@ describe('PlayerService.update player.level.changed emission (real PG)', () => { await svc.update(seeded.id, { level: 5 }, account.id); - expect(events.emit).toHaveBeenCalledWith('player.level.changed', { - userId: account.id, - playerId: seeded.id, - previousLevel: 1, - newLevel: 5, - actorId: account.id, - }); + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), + 'player.level.changed', + { + userId: account.id, + playerId: seeded.id, + previousLevel: 1, + newLevel: 5, + actorId: account.id, + }, + ); }); it('does not emit when level is omitted from the patch', async () => { @@ -524,7 +528,11 @@ describe('PlayerService.update player.level.changed emission (real PG)', () => { await svc.update(seeded.id, { username: 'new_name' }, account.id); - expect(events.emit).not.toHaveBeenCalledWith('player.level.changed', expect.anything()); + expect(events.emitInTransaction).not.toHaveBeenCalledWith( + expect.anything(), + 'player.level.changed', + expect.anything(), + ); }); it('does not emit when level is unchanged', async () => { @@ -533,7 +541,11 @@ describe('PlayerService.update player.level.changed emission (real PG)', () => { await svc.update(seeded.id, { level: 3 }, account.id); - expect(events.emit).not.toHaveBeenCalledWith('player.level.changed', expect.anything()); + expect(events.emitInTransaction).not.toHaveBeenCalledWith( + expect.anything(), + 'player.level.changed', + expect.anything(), + ); }); }); @@ -544,11 +556,15 @@ describe('PlayerService player.account.closed emission (real PG)', () => { await svc.update(seeded.id, { status: 'closed' }, ACTOR_ID); - expect(events.emit).toHaveBeenCalledWith('player.account.closed', { - playerId: seeded.id, - userId: account.id, - actorId: ACTOR_ID, - }); + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), + 'player.account.closed', + { + playerId: seeded.id, + userId: account.id, + actorId: ACTOR_ID, + }, + ); }); it('emits the same payload from remove as from update', async () => { @@ -557,11 +573,15 @@ describe('PlayerService player.account.closed emission (real PG)', () => { await svc.remove(seeded.id, ACTOR_ID); - expect(events.emit).toHaveBeenCalledWith('player.account.closed', { - playerId: seeded.id, - userId: account.id, - actorId: ACTOR_ID, - }); + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), + 'player.account.closed', + { + playerId: seeded.id, + userId: account.id, + actorId: ACTOR_ID, + }, + ); }); it('emits one status transition when two updates race to the same status', async () => { @@ -573,7 +593,9 @@ describe('PlayerService player.account.closed emission (real PG)', () => { svc.update(seeded.id, { status: 'suspended' }, ACTOR_ID), ]); - const changes = events.emit.mock.calls.filter(([topic]) => topic === 'player.status.changed'); + const changes = events.emitInTransaction.mock.calls.filter( + ([, topic]) => topic === 'player.status.changed', + ); expect(changes).toHaveLength(1); }); @@ -583,7 +605,11 @@ describe('PlayerService player.account.closed emission (real PG)', () => { await svc.update(seeded.id, { status: 'suspended' }, ACTOR_ID); - expect(events.emit).not.toHaveBeenCalledWith('player.account.closed', expect.anything()); + expect(events.emitInTransaction).not.toHaveBeenCalledWith( + expect.anything(), + 'player.account.closed', + expect.anything(), + ); }); it('does not re-emit when the player was already closed (no transition)', async () => { @@ -592,7 +618,11 @@ describe('PlayerService player.account.closed emission (real PG)', () => { await svc.update(seeded.id, { status: 'closed' }, ACTOR_ID); - expect(events.emit).not.toHaveBeenCalledWith('player.account.closed', expect.anything()); + expect(events.emitInTransaction).not.toHaveBeenCalledWith( + expect.anything(), + 'player.account.closed', + expect.anything(), + ); }); it('does not re-emit from remove when the player was already closed', async () => { @@ -601,7 +631,11 @@ describe('PlayerService player.account.closed emission (real PG)', () => { await svc.remove(seeded.id, ACTOR_ID); - expect(events.emit).not.toHaveBeenCalledWith('player.account.closed', expect.anything()); + expect(events.emitInTransaction).not.toHaveBeenCalledWith( + expect.anything(), + 'player.account.closed', + expect.anything(), + ); }); it('emits player.account.reopened when update moves the status out of closed', async () => { @@ -610,11 +644,15 @@ describe('PlayerService player.account.closed emission (real PG)', () => { await svc.update(seeded.id, { status: 'active' }, ACTOR_ID); - expect(events.emit).toHaveBeenCalledWith('player.account.reopened', { - playerId: seeded.id, - userId: account.id, - actorId: ACTOR_ID, - }); + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), + 'player.account.reopened', + { + playerId: seeded.id, + userId: account.id, + actorId: ACTOR_ID, + }, + ); }); it('does not emit reopened for a status change between two non-closed statuses', async () => { @@ -623,7 +661,11 @@ describe('PlayerService player.account.closed emission (real PG)', () => { await svc.update(seeded.id, { status: 'active' }, ACTOR_ID); - expect(events.emit).not.toHaveBeenCalledWith('player.account.reopened', expect.anything()); + expect(events.emitInTransaction).not.toHaveBeenCalledWith( + expect.anything(), + 'player.account.reopened', + expect.anything(), + ); }); }); @@ -634,13 +676,17 @@ describe('PlayerService player.status.changed emission (real PG)', () => { await svc.update(seeded.id, { status: 'suspended' }, ACTOR_ID); - expect(events.emit).toHaveBeenCalledWith('player.status.changed', { - playerId: seeded.id, - userId: account.id, - actorId: ACTOR_ID, - previousStatus: 'active', - newStatus: 'suspended', - }); + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), + 'player.status.changed', + { + playerId: seeded.id, + userId: account.id, + actorId: ACTOR_ID, + previousStatus: 'active', + newStatus: 'suspended', + }, + ); }); it('does not emit when the status is omitted or unchanged', async () => { @@ -650,7 +696,11 @@ describe('PlayerService player.status.changed emission (real PG)', () => { await svc.update(seeded.id, { level: 3 }, ACTOR_ID); await svc.update(seeded.id, { status: 'suspended' }, ACTOR_ID); - expect(events.emit).not.toHaveBeenCalledWith('player.status.changed', expect.anything()); + expect(events.emitInTransaction).not.toHaveBeenCalledWith( + expect.anything(), + 'player.status.changed', + expect.anything(), + ); }); it('emits the transition to closed from remove', async () => { @@ -659,13 +709,17 @@ describe('PlayerService player.status.changed emission (real PG)', () => { await svc.remove(seeded.id, ACTOR_ID); - expect(events.emit).toHaveBeenCalledWith('player.status.changed', { - playerId: seeded.id, - userId: account.id, - actorId: ACTOR_ID, - previousStatus: 'dormant', - newStatus: 'closed', - }); + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), + 'player.status.changed', + { + playerId: seeded.id, + userId: account.id, + actorId: ACTOR_ID, + previousStatus: 'dormant', + newStatus: 'closed', + }, + ); }); it('does not emit from remove when the player was already closed', async () => { @@ -674,7 +728,11 @@ describe('PlayerService player.status.changed emission (real PG)', () => { await svc.remove(seeded.id, ACTOR_ID); - expect(events.emit).not.toHaveBeenCalledWith('player.status.changed', expect.anything()); + expect(events.emitInTransaction).not.toHaveBeenCalledWith( + expect.anything(), + 'player.status.changed', + expect.anything(), + ); }); }); diff --git a/packages/core/src/pam/player-management/service/player.service.ts b/packages/core/src/pam/player-management/service/player.service.ts index 76b5622f9..47bf5532e 100644 --- a/packages/core/src/pam/player-management/service/player.service.ts +++ b/packages/core/src/pam/player-management/service/player.service.ts @@ -7,7 +7,7 @@ import { withAdvisoryXactLock, createLogger, } from '@openora/core/server'; -import type { EventBus } from '@openora/core/server'; +import type { DrizzleTx, EventBus } from '@openora/core/server'; import { normalizeKycStatus, type PlayerStatus, @@ -291,22 +291,29 @@ export class PlayerService implements PlayerActivityTracker { if (Object.keys(patch).length > 0) { await trx.update(player).set(patch).where(eq(player.id, playerId)); } + // Audited and, for a closure, what forfeits the player's bonuses - so committed with + // the change through the outbox. + if (data.level !== undefined && data.level !== locked.level) { + await this.events.emitInTransaction(trx, 'player.level.changed', { + userId: locked.userId, + playerId, + previousLevel: locked.level, + newLevel: data.level, + actorId, + }); + } + if (data.status !== undefined && data.status !== locked.status) { + await this.emitStatusTransition(trx, { + playerId, + userId: locked.userId, + actorId, + previousStatus: locked.status, + newStatus: data.status, + }); + } return locked; }); - // Emitted AFTER commit (not inside the transaction callback, unlike the - // KYC_STATUS_WRITER emit above) - a level change is a best-effort fan-out, not a - // regulated single-writer seam, so it follows the standard post-commit event idiom. - if (data.level !== undefined && data.level !== before.level) { - this.events.emit('player.level.changed', { - userId: before.userId, - playerId, - previousLevel: before.level, - newLevel: data.level, - actorId, - }); - } - // A transition into a blocking status must kill every active session immediately - // AC: "Blocking a player immediately prevents login and all platform activity." // Session validity is checked on every request (SessionResolver -> better-auth @@ -320,33 +327,35 @@ export class PlayerService implements PlayerActivityTracker { await this.sessionCommands.revokeAll(before.userId, actorId); } - if (data.status === 'closed' && before.status !== 'closed') { - this.events.emit('player.account.closed', { - playerId, - userId: before.userId, - actorId, - }); - } + return this.fetchOneWithTags(playerId); + } - if (data.status !== undefined && data.status !== 'closed' && before.status === 'closed') { - this.events.emit('player.account.reopened', { + private async emitStatusTransition( + trx: DrizzleTx, + transition: { + playerId: Player['id']; + userId: User['id']; + actorId: User['id']; + previousStatus: PlayerStatus; + newStatus: PlayerStatus; + }, + ): Promise { + const { playerId, userId, actorId, previousStatus, newStatus } = transition; + if (newStatus === 'closed') { + await this.events.emitInTransaction(trx, 'player.account.closed', { playerId, - userId: before.userId, + userId, actorId, }); } - - if (data.status !== undefined && data.status !== before.status) { - this.events.emit('player.status.changed', { + if (previousStatus === 'closed') { + await this.events.emitInTransaction(trx, 'player.account.reopened', { playerId, - userId: before.userId, + userId, actorId, - previousStatus: before.status, - newStatus: data.status, }); } - - return this.fetchOneWithTags(playerId); + await this.events.emitInTransaction(trx, 'player.status.changed', transition); } async touchLastSeen(userId: Player['userId']): Promise { @@ -368,23 +377,18 @@ export class PlayerService implements PlayerActivityTracker { new PlayerNotFoundError(playerId), ); await trx.update(player).set({ status: 'closed' }).where(eq(player.id, playerId)); + if (locked.status !== 'closed') { + await this.emitStatusTransition(trx, { + playerId, + userId: locked.userId, + actorId, + previousStatus: locked.status, + newStatus: 'closed', + }); + } return locked; }); await this.sessionCommands.revokeAll(before.userId, actorId); - if (before.status !== 'closed') { - this.events.emit('player.account.closed', { - playerId, - userId: before.userId, - actorId, - }); - this.events.emit('player.status.changed', { - playerId, - userId: before.userId, - actorId, - previousStatus: before.status, - newStatus: 'closed', - }); - } return { success: true }; } diff --git a/packages/core/src/promo/bonus/__tests__/bonus-lifecycle-port.test.ts b/packages/core/src/promo/bonus/__tests__/bonus-lifecycle-port.test.ts index a9e96a73d..400d94190 100644 --- a/packages/core/src/promo/bonus/__tests__/bonus-lifecycle-port.test.ts +++ b/packages/core/src/promo/bonus/__tests__/bonus-lifecycle-port.test.ts @@ -1,6 +1,5 @@ -import { describe, it, expect, vi } from 'vitest'; +import { describe, it, expect } from 'vitest'; import { randomUUID } from 'node:crypto'; -import type { EventBus } from '@openora/core/server'; import { createBonusLifecyclePort } from '../service/bonus-lifecycle-port.service.js'; import { GrantNotFoundError, @@ -8,16 +7,13 @@ import { type GrantLifecycleService, } from '../service/grant-lifecycle.service.js'; -const fakeBus = () => ({ emit: vi.fn() }) as unknown as EventBus; - const fakeService = (forfeit: GrantLifecycleService['forfeit']) => ({ forfeit }) as unknown as GrantLifecycleService; describe('createBonusLifecyclePort', () => { - it('announces promo.bonus.forfeited and returns the closed grant on success', async () => { + it('returns the closed grant on success', async () => { const grantId = randomUUID(); const userId = randomUUID(); - const bus = fakeBus(); const port = createBonusLifecyclePort( fakeService(async () => ({ grantId, @@ -26,59 +22,42 @@ describe('createBonusLifecyclePort', () => { forfeitedAmount: '42', actorId: null, })), - bus, ); const outcome = await port.forfeit(grantId, 'terms_breach', 'missed a required wagering day'); expect(outcome).toEqual({ ok: true, grantId, userId, currency: 'USD', forfeitedAmount: '42' }); - expect(bus.emit).toHaveBeenCalledWith('promo.bonus.forfeited', { - userId, - grantId, - currency: 'USD', - forfeitedAmount: '42', - reason: 'terms_breach', - actorId: null, - }); }); - it('maps a not-found grant to ok:false without emitting anything', async () => { - const bus = fakeBus(); + it('maps a not-found grant to ok:false', async () => { const port = createBonusLifecyclePort( fakeService(async () => { throw new GrantNotFoundError(randomUUID()); }), - bus, ); const outcome = await port.forfeit(randomUUID(), 'terms_breach', 'no such grant'); expect(outcome).toEqual({ ok: false, reason: 'not_found' }); - expect(bus.emit).not.toHaveBeenCalled(); }); it('maps a grant that already closed some other way to ok:false, not-forfeitable', async () => { - const bus = fakeBus(); const port = createBonusLifecyclePort( fakeService(async () => { throw new GrantNotForfeitableError(); }), - bus, ); const outcome = await port.forfeit(randomUUID(), 'terms_breach', 'already closed'); expect(outcome).toEqual({ ok: false, reason: 'not_forfeitable' }); - expect(bus.emit).not.toHaveBeenCalled(); }); it('rethrows an error the service did not define', async () => { - const bus = fakeBus(); const port = createBonusLifecyclePort( fakeService(async () => { throw new Error('db exploded'); }), - bus, ); await expect(port.forfeit(randomUUID(), 'terms_breach', 'note')).rejects.toThrow('db exploded'); diff --git a/packages/core/src/promo/bonus/__tests__/grant-lifecycle.int.test.ts b/packages/core/src/promo/bonus/__tests__/grant-lifecycle.int.test.ts index 9b42b45ae..cd875fe69 100644 --- a/packages/core/src/promo/bonus/__tests__/grant-lifecycle.int.test.ts +++ b/packages/core/src/promo/bonus/__tests__/grant-lifecycle.int.test.ts @@ -4,7 +4,7 @@ import { eq, sql } from 'drizzle-orm'; import { findOneOrThrow } from '@openora/core/server'; import { createTestDb, type TestDb } from '@openora/core/testing'; import type { BonusGrantArgs, Uuid } from '@openora/core/contracts'; -import { makeAuditWriter } from '../../../testing/mock.js'; +import { makeAuditWriter, makeEventBus } from '../../../testing/mock.js'; import { migrate } from '../migrate.js'; import { promoGrant, promoGrantEntry, promoWeight, promoWeightProfile } from '../schema/index.js'; import { GrantLifecycleService } from '../service/grant-lifecycle.service.js'; @@ -71,7 +71,7 @@ beforeAll(async () => { db = await createTestDb([migrate]); audit = makeAuditWriter(); grants = new GrantService(audit); - lifecycle = new GrantLifecycleService(db.drizzle, audit); + lifecycle = new GrantLifecycleService(db.drizzle, audit, makeEventBus()); }); afterAll(async () => { diff --git a/packages/core/src/promo/bonus/plugin.ts b/packages/core/src/promo/bonus/plugin.ts index 1c5da9f4e..59acead21 100644 --- a/packages/core/src/promo/bonus/plugin.ts +++ b/packages/core/src/promo/bonus/plugin.ts @@ -6,7 +6,6 @@ import { BONUS_LIFECYCLE, BONUS_WAGERING, BonusForfeitReasonSchema, - type BonusForfeitReason, type BonusLifecycleCommands, CurrencyTickerSchema, GEO_CHECK_COMMANDS, @@ -112,8 +111,7 @@ export default { BONUS_LIFECYCLE, (c): BonusLifecycleCommands => createBonusLifecyclePort( - new GrantLifecycleService(c.get(DRIZZLE), c.get(AUDIT_WRITER)), - c.get(EVENT_BUS), + new GrantLifecycleService(c.get(DRIZZLE), c.get(AUDIT_WRITER), c.get(EVENT_BUS)), ), ); @@ -157,22 +155,6 @@ export default { }); } - const announce = ( - topic: 'promo.bonus.expired' | 'promo.bonus.forfeited', - closed: Awaited>, - reason?: BonusForfeitReason, - ) => { - for (const grant of closed) { - events?.emit(topic, { - userId: grant.userId, - grantId: grant.grantId, - currency: grant.currency, - forfeitedAmount: grant.forfeitedAmount, - ...(reason === undefined ? {} : { reason, actorId: grant.actorId }), - }); - } - }; - ctx.jobs.worker({ queue: EXPIRY_QUEUE, schema: EmptyJobPayloadSchema, @@ -182,7 +164,6 @@ export default { return; } const closed = await lifecycle.expireDue(); - announce('promo.bonus.expired', closed); if (closed.length > 0) { logger.info({ grants: closed.length }, 'promo bonus grants expired'); } @@ -196,14 +177,13 @@ export default { if (!lifecycle) { throw new Error('promo-bonus-forfeit: service not constructed'); } - const closed = await lifecycle.forfeitAllFor( + await lifecycle.forfeitAllFor( payload.userId, payload.reason, payload.actorId === null ? undefined : { id: payload.actorId, isAdmin: payload.actorIsAdmin }, ); - announce('promo.bonus.forfeited', closed, payload.reason); }, }); @@ -291,13 +271,17 @@ export default { throw new Error('promo-offer-deposit: service not constructed'); } const service = offers; - const granted = await drizzle.db.transaction((tx) => service.applyDeposit(tx, payload)); - // After the commit: the criterion is that a player is told about the credit and what it - // obliges them to wager, and an announcement ahead of the commit could promise a bonus - // the transaction then rolled back. - for (const bonus of granted) { - events?.emit('promo.bonus.granted', { ...bonus, source: 'deposit' }); - } + const bus = events; + // Through the outbox in the grant's own transaction: the player is told about the + // credit and what it obliges them to wager if and only if the grant committed. + await drizzle.db.transaction(async (tx) => { + for (const bonus of await service.applyDeposit(tx, payload)) { + await bus?.emitInTransaction(tx, 'promo.bonus.granted', { + ...bonus, + source: 'deposit', + }); + } + }); }, }); @@ -328,7 +312,7 @@ export default { }); ctx.routers.add('promo-bonus', (c) => { - lifecycle = new GrantLifecycleService(c.get(DRIZZLE), c.get(AUDIT_WRITER)); + lifecycle = new GrantLifecycleService(c.get(DRIZZLE), c.get(AUDIT_WRITER), c.get(EVENT_BUS)); drizzle = c.get(DRIZZLE); offers = new OfferService( c.get(DRIZZLE), @@ -350,7 +334,6 @@ export default { realtime: c.get(REALTIME_TRANSPORT), weights: new WeightService(c.get(DRIZZLE), c.get(AUDIT_WRITER)), lifecycle, - events, adminGuard: c.get(ADMIN_GUARD), limiter: c.get(RATE_LIMITER), geoCheck: c.has(GEO_CHECK_COMMANDS) ? c.get(GEO_CHECK_COMMANDS) : undefined, diff --git a/packages/core/src/promo/bonus/router/index.ts b/packages/core/src/promo/bonus/router/index.ts index 7e41c8231..746ecb306 100644 --- a/packages/core/src/promo/bonus/router/index.ts +++ b/packages/core/src/promo/bonus/router/index.ts @@ -5,7 +5,6 @@ import { getUserId, mapErrors, type AdminGuard, - type EventBus, type OssContext, } from '@openora/core/server'; import { @@ -50,7 +49,6 @@ export function createBonusRouter({ offers, weights, lifecycle, - events, realtime, adminGuard, limiter, @@ -60,7 +58,6 @@ export function createBonusRouter({ offers: OfferService; weights: WeightService; lifecycle: GrantLifecycleService; - events: EventBus; realtime: RealtimeTransport; adminGuard: AdminGuard; limiter: RateLimiterAdapter; @@ -119,20 +116,7 @@ export function createBonusRouter({ NOT_FOUND: [GrantNotForfeitableNotFoundError, GrantNotFoundError], }, async () => { - const closed = await lifecycle.forfeit( - input.id, - 'admin', - { id: userId, isAdmin: true }, - input.note, - ); - events.emit('promo.bonus.forfeited', { - userId: closed.userId, - grantId: closed.grantId, - currency: closed.currency, - forfeitedAmount: closed.forfeitedAmount, - reason: 'admin', - actorId: closed.actorId, - }); + await lifecycle.forfeit(input.id, 'admin', { id: userId, isAdmin: true }, input.note); return grants.getForAdmin(input.id); }, ); diff --git a/packages/core/src/promo/bonus/service/bonus-lifecycle-port.service.ts b/packages/core/src/promo/bonus/service/bonus-lifecycle-port.service.ts index 3c9d21180..3fc479a2c 100644 --- a/packages/core/src/promo/bonus/service/bonus-lifecycle-port.service.ts +++ b/packages/core/src/promo/bonus/service/bonus-lifecycle-port.service.ts @@ -1,5 +1,4 @@ import type { BonusForfeitOutcome, BonusLifecycleCommands } from '@openora/core/contracts'; -import type { EventBus } from '@openora/core/server'; import { GrantLifecycleService, GrantNotFoundError, @@ -8,26 +7,14 @@ import { /** * Adapts `GrantLifecycleService.forfeit` (throws on refusal) to the BONUS_LIFECYCLE command port - * shape (returns an outcome) and announces the same `promo.bonus.forfeited` topic a router- or - * rule-driven forfeit already emits, so a job-driven one gets the same realtime balance signal - * and player notification for free rather than a second announcement path to keep in sync. + * shape (returns an outcome). The service itself emits `promo.bonus.forfeited`, so a job-driven + * forfeit gets the same realtime balance signal and player notification as any other. */ -export function createBonusLifecyclePort( - service: GrantLifecycleService, - bus: EventBus, -): BonusLifecycleCommands { +export function createBonusLifecyclePort(service: GrantLifecycleService): BonusLifecycleCommands { return { async forfeit(grantId, reason, note): Promise { try { const closed = await service.forfeit(grantId, reason, undefined, note); - bus.emit('promo.bonus.forfeited', { - userId: closed.userId, - grantId: closed.grantId, - currency: closed.currency, - forfeitedAmount: closed.forfeitedAmount, - reason, - actorId: null, - }); return { ok: true, grantId: closed.grantId, diff --git a/packages/core/src/promo/bonus/service/grant-lifecycle.service.ts b/packages/core/src/promo/bonus/service/grant-lifecycle.service.ts index 4e02a3538..ac543be13 100644 --- a/packages/core/src/promo/bonus/service/grant-lifecycle.service.ts +++ b/packages/core/src/promo/bonus/service/grant-lifecycle.service.ts @@ -6,6 +6,7 @@ import { moneyCompare, type DrizzleService, type DrizzleTx, + type EventBus, } from '@openora/core/server'; import { promoGrant, promoGrantEntry, type PromoGrant } from '../schema/index.js'; @@ -44,6 +45,7 @@ export class GrantLifecycleService { constructor( private readonly drizzle: DrizzleService, private readonly audit: AuditWritePort, + private readonly events: EventBus, ) {} /** @@ -266,7 +268,7 @@ export class GrantLifecycleService { }, }); - return { + const closed: ClosedGrant = { grantId, userId: claimed.userId, currency: claimed.currency, @@ -275,5 +277,18 @@ export class GrantLifecycleService { // id here would tell downstream consumers an admin forfeited their own self-exclusion. actorId: outcome.actor?.isAdmin ? outcome.actor.id : null, }; + const { actorId, ...announced } = closed; + // The single producer of both topics, committed with the close itself. Every forfeit + // carries a reason; only an expiry has none. + if (outcome.reason === undefined) { + await this.events.emitInTransaction(tx, 'promo.bonus.expired', announced); + } else { + await this.events.emitInTransaction(tx, 'promo.bonus.forfeited', { + ...announced, + reason: outcome.reason, + actorId, + }); + } + return closed; } } diff --git a/packages/core/src/wallet/__tests__/swap.service.int.test.ts b/packages/core/src/wallet/__tests__/swap.service.int.test.ts index ee88ef3b6..d4bbbe2ec 100644 --- a/packages/core/src/wallet/__tests__/swap.service.int.test.ts +++ b/packages/core/src/wallet/__tests__/swap.service.int.test.ts @@ -196,7 +196,8 @@ describe('SwapService (real PG)', () => { }), ]), ); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'wallet.swap.completed', expect.objectContaining({ userId: w.userId, toCurrency: 'BTC', toAmount: '0.00152' }), ); diff --git a/packages/core/src/wallet/__tests__/wallet-auto-withdrawal.int.test.ts b/packages/core/src/wallet/__tests__/wallet-auto-withdrawal.int.test.ts index 4c0e4c5f1..530cede87 100644 --- a/packages/core/src/wallet/__tests__/wallet-auto-withdrawal.int.test.ts +++ b/packages/core/src/wallet/__tests__/wallet-auto-withdrawal.int.test.ts @@ -233,7 +233,7 @@ describe('WalletService.withdraw auto-approval (real PG)', () => { providerName: DEFAULT_PAYMENT_PROVIDER, autoApprovalPivotAmount: '40.000000000000000000', }); - expect(events.emit.mock.calls.map(([topic]) => topic)).toEqual([ + expect(events.emitInTransaction.mock.calls.map(([, topic]) => topic)).toEqual([ 'wallet.withdrawal.requested', 'wallet.withdrawal.completed', ]); diff --git a/packages/core/src/wallet/__tests__/wallet-withdrawal-hold.int.test.ts b/packages/core/src/wallet/__tests__/wallet-withdrawal-hold.int.test.ts index 809d75460..fd425cd98 100644 --- a/packages/core/src/wallet/__tests__/wallet-withdrawal-hold.int.test.ts +++ b/packages/core/src/wallet/__tests__/wallet-withdrawal-hold.int.test.ts @@ -169,7 +169,7 @@ describe('WalletService.holdWithdrawal (real PG)', () => { }); expect((await txById(pending.id)).reviewedAt).toBeInstanceOf(Date); expect(await balanceOf(w.id)).toBe(60); - expect(events.emit).not.toHaveBeenCalled(); + expect(events.emitInTransaction).not.toHaveBeenCalled(); }); it('writes the held audit record before the status change commits', async () => { @@ -349,7 +349,7 @@ describe('WalletService review of a held withdrawal (real PG)', () => { reviewReason: null, }); expect(await balanceOf(w.id)).toBe(20); - expect(events.emit.mock.calls.map(([topic]) => topic)).toEqual([ + expect(events.emitInTransaction.mock.calls.map(([, topic]) => topic)).toEqual([ 'wallet.withdrawal.approved', 'wallet.withdrawal.completed', ]); @@ -371,7 +371,8 @@ describe('WalletService review of a held withdrawal (real PG)', () => { reviewedBy: reviewer, reviewReason: 'source of funds unclear', }); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'wallet.withdrawal.rejected', expect.objectContaining({ transactionId: pending.id, adminId: reviewer }), ); diff --git a/packages/core/src/wallet/__tests__/wallet.service.int.test.ts b/packages/core/src/wallet/__tests__/wallet.service.int.test.ts index 6cf621f46..0d53f4d29 100644 --- a/packages/core/src/wallet/__tests__/wallet.service.int.test.ts +++ b/packages/core/src/wallet/__tests__/wallet.service.int.test.ts @@ -203,7 +203,7 @@ async function txById(id: string) { } const emittedTopics = (events: ReturnType) => - events.emit.mock.calls.map(([topic]) => topic); + events.emitInTransaction.mock.calls.map(([, topic]) => topic); beforeAll(async () => { db = await createTestDb([migrate, migrateProfile]); @@ -245,7 +245,8 @@ describe('WalletService.deposit (real PG)', () => { providerName: 'stripe', providerRefId: externalId, }); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'wallet.deposit.completed', expect.objectContaining({ userId, amount: '40', transactionId: result.transactionId }), ); @@ -463,7 +464,8 @@ describe('WalletService.withdraw (real PG)', () => { status: 'pending', rail: 'fiat', }); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'wallet.withdrawal.requested', expect.objectContaining({ userId: w.userId, amount: '40', ip: null, userAgent: null }), ); @@ -636,7 +638,7 @@ describe('WalletService.withdraw (real PG)', () => { expect(await balanceOf(w.userId)).toBe(30); expect(await txRows(w.id)).toHaveLength(0); - expect(events.emit).not.toHaveBeenCalled(); + expect(events.emitInTransaction).not.toHaveBeenCalled(); }); it('lets two concurrent withdrawals share a balance without overdrawing it', async () => { @@ -702,7 +704,7 @@ describe('WalletService.withdraw - synchronous tag evaluation (TAG_EVALUATION_CO svc.withdraw({ userId: w.userId, amount: '40', currency: 'USD', ...NO_CLIENT_META }), ).rejects.toBe(dbError); // The transaction rolled back before the requested event/auto-approval step ran. - expect(events.emit).not.toHaveBeenCalled(); + expect(events.emitInTransaction).not.toHaveBeenCalled(); expect(psp.processWithdrawal).not.toHaveBeenCalled(); expect(await balanceOf(w.userId)).toBe(100); }); @@ -1453,7 +1455,8 @@ describe('WalletService.rejectWithdrawal (real PG)', () => { reviewedBy: adminId, reviewReason: 'suspicious', }); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'wallet.withdrawal.rejected', expect.objectContaining({ userId: w.userId, reason: 'suspicious', adminId }), ); @@ -1705,7 +1708,8 @@ describe('WalletService.reconcileWithdrawalStatus (real PG)', () => { expect(await txById(tx.id)).toMatchObject({ status: 'failed' }); expect(await balanceOf(w.userId)).toBe(40); - expect(events.emit).toHaveBeenCalledWith( + expect(events.emitInTransaction).toHaveBeenCalledWith( + expect.anything(), 'wallet.withdrawal.failed', expect.objectContaining({ userId: w.userId, playerId, transactionId: tx.id, adminId: null }), ); @@ -1740,7 +1744,7 @@ describe('WalletService.reconcileWithdrawalStatus (real PG)', () => { expect(await txById(tx.id)).toMatchObject({ status: 'completed' }); expect(await balanceOf(w.userId)).toBe(0); - expect(events.emit).not.toHaveBeenCalled(); + expect(events.emitInTransaction).not.toHaveBeenCalled(); }); it('no-ops when no transaction carries the providerRefId', async () => { @@ -1752,7 +1756,7 @@ describe('WalletService.reconcileWithdrawalStatus (real PG)', () => { status: 'completed', }); - expect(events.emit).not.toHaveBeenCalled(); + expect(events.emitInTransaction).not.toHaveBeenCalled(); }); }); @@ -1976,7 +1980,7 @@ describe('WalletService.creditDepositByAddress (real PG)', () => { 'default', ); - expect(events.emit).not.toHaveBeenCalled(); + expect(events.emitInTransaction).not.toHaveBeenCalled(); const findings = await db.drizzle.db .select() .from(walletReconciliationFinding) @@ -2217,7 +2221,7 @@ describe('WalletService.creditDepositByAddress (real PG)', () => { txHash: '0xmismatch', }); - expect(events.emit).not.toHaveBeenCalled(); + expect(events.emitInTransaction).not.toHaveBeenCalled(); expect(await balanceOf(w.userId)).toBe(0); const findings = await db.drizzle.db .select() diff --git a/packages/core/src/wallet/service/reconciliation.service.ts b/packages/core/src/wallet/service/reconciliation.service.ts index 1913ee7cd..503b6f836 100644 --- a/packages/core/src/wallet/service/reconciliation.service.ts +++ b/packages/core/src/wallet/service/reconciliation.service.ts @@ -498,7 +498,8 @@ export class ReconciliationService { { runId, openFindings, threshold: cfg.alertThreshold }, 'wallet reconciliation: open findings exceed threshold', ); - this.events.emit('wallet.reconciliation.alert', { + // No surrounding transaction to join: the outbox write alone makes the alert durable. + await this.events.emitInTransaction(this.drizzle.db, 'wallet.reconciliation.alert', { runId, openFindings, threshold: cfg.alertThreshold, diff --git a/packages/core/src/wallet/service/swap.service.ts b/packages/core/src/wallet/service/swap.service.ts index 346bad8f3..7c7d48ef7 100644 --- a/packages/core/src/wallet/service/swap.service.ts +++ b/packages/core/src/wallet/service/swap.service.ts @@ -296,16 +296,15 @@ export class SwapService { } const toCurrency = this.toCurrencyOf(out); - // The transaction returns the player the row belongs to, or `undefined` when another - // delivery already settled this swap. - const settled = await this.drizzle.db.transaction(async (txn) => { + // A no-op when another delivery already settled this swap. + await this.drizzle.db.transaction(async (txn) => { const flipped = await txn .update(walletTransaction) .set({ status: 'completed', providerRefId: externalId }) .where(and(eq(walletTransaction.id, out.id), eq(walletTransaction.status, 'processing'))) .returning({ id: walletTransaction.id }); if (flipped.length === 0) { - return undefined; + return; } // On `txn`, not the pool: the settlement must not be gated on a read that only @@ -342,19 +341,15 @@ export class SwapService { externalId, }, }); - return userId; - }); - - if (settled) { - this.events.emit('wallet.swap.completed', { - userId: settled, + await this.events.emitInTransaction(txn, 'wallet.swap.completed', { + userId, transactionId: out.id, fromCurrency: out.currency, fromAmount: out.amount, toCurrency: balanceKey(toCurrency), toAmount, }); - } + }); return { transactionId: out.id, status: 'completed', toAmount }; } diff --git a/packages/core/src/wallet/service/wallet.service.ts b/packages/core/src/wallet/service/wallet.service.ts index 4fb9048f3..f5a6a7017 100644 --- a/packages/core/src/wallet/service/wallet.service.ts +++ b/packages/core/src/wallet/service/wallet.service.ts @@ -1255,9 +1255,10 @@ export class WalletService { throw err; } + const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); // The status guard is the settle-once lock: only the update that moves the row out of // `pending` credits the balance, so a retry that races here reads a replay instead. - const { transactionId, replayed } = await this.drizzle.db.transaction(async (txn) => { + const { transactionId } = await this.drizzle.db.transaction(async (txn) => { const [settled] = await txn .update(walletTransaction) .set({ status: 'completed', providerRefId: psp.externalId }) @@ -1269,21 +1270,18 @@ export class WalletService { ) .returning(); if (!settled) { - return { transactionId: reservation.row.id, replayed: true }; + return { transactionId: reservation.row.id }; } await creditWalletBalance(txn, reservation.walletRecord.id, currency, amount); - return { transactionId: settled.id, replayed: false }; - }); - - if (!replayed) { - this.events.emit('wallet.deposit.completed', { + await this.events.emitInTransaction(txn, 'wallet.deposit.completed', { userId, - playerId: await this.identityReader.getPlayerIdByUserIdSafe(userId), + playerId, amount, currency, - transactionId, + transactionId: settled.id, }); - } + return { transactionId: settled.id }; + }); return { transactionId, status: 'completed' }; } @@ -1312,7 +1310,7 @@ export class WalletService { throw new PlayerNotFoundError(userId); } - const { result, emitted } = await this.drizzle.db.transaction(async (txn) => { + return this.drizzle.db.transaction(async (txn) => { let walletRecord = ( await txn.select().from(wallet).where(eq(wallet.userId, userId)).for('update') ).at(0); @@ -1346,7 +1344,7 @@ export class WalletService { currency, reason, }); - return { result: { transactionId: existing.id, status: existing.status }, emitted: false }; + return { transactionId: existing.id, status: existing.status }; } const { row, replayed } = await this.insertIdempotentTransaction(txn, { @@ -1376,7 +1374,7 @@ export class WalletService { currency, reason, }); - return { result: { transactionId: row.id, status: row.status }, emitted: false }; + return { transactionId: row.id, status: row.status }; } const balances = @@ -1417,27 +1415,23 @@ export class WalletService { ip, userAgent, }); - return { result: { transactionId: row.id, status: row.status }, emitted: true }; - }); - - // Every other balance-mutating path on this service emits; without this one an admin - // correction moves a real balance while responsible-gaming monitoring, analytics and - // any operator subscriber see nothing at all. - if (emitted) { - this.events.emit('wallet.manual_adjustment.created', { + // Every other balance-mutating path on this service emits; without this one an admin + // correction moves a real balance while responsible-gaming monitoring, analytics and + // any operator subscriber see nothing at all. + await this.events.emitInTransaction(txn, 'wallet.manual_adjustment.created', { userId, playerId, adminId, amount, currency, - transactionId: result.transactionId, + transactionId: row.id, direction, reason, ip, userAgent, }); - } - return result; + return { transactionId: row.id, status: row.status }; + }); } // Pre-PSP replay check for deposit; also returns the resolved wallet so the transaction @@ -1635,6 +1629,7 @@ export class WalletService { destinationAddress, destinationTag, ); + const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); const { transactionId, status, replayed, walletId, rail } = await this.drizzle.db.transaction( async (txn) => { @@ -1709,6 +1704,16 @@ export class WalletService { await this.tagEvaluationCommands.evaluateWithdrawalRequested(txn, { userId, amount }); } + await this.events.emitInTransaction(txn, 'wallet.withdrawal.requested', { + userId, + playerId, + amount, + currency, + transactionId: row.id, + ip: ip ?? null, + userAgent: userAgent ?? null, + }); + return { transactionId: row.id, status: row.status, @@ -1723,16 +1728,6 @@ export class WalletService { return { transactionId, status }; } - this.events.emit('wallet.withdrawal.requested', { - userId, - playerId: await this.identityReader.getPlayerIdByUserIdSafe(userId), - amount, - currency, - transactionId, - ip: ip ?? null, - userAgent: userAgent ?? null, - }); - // Fail-closed post-step: decides who approves (system vs manual queue), never throws out of withdraw() - failure leaves the row pending. const auto = await this.maybeAutoApprove({ userId, @@ -2008,16 +2003,30 @@ export class WalletService { ): Promise { // Two-phase: commit the `processing` flip first (FOR UPDATE lock), then call the PSP OUTSIDE // the tx (a failure refunds in a second tx). Never inline the PSP call inside the hold transaction. - const tx = await this.drizzle.db.transaction((txn) => - this.flipToProcessing({ + // `approved` is admin-attributed and committed with the flip; the system auto path skips + // it - its trail is the AUDIT_WRITER entry plus the shared `completed` event. + const tx = await this.drizzle.db.transaction(async (txn) => { + const flipped = await this.flipToProcessing({ txn, withdrawalId, adminId, from: QUEUED_WITHDRAWAL_STATUSES, reviewReason: null, - }), - ); - return this.settleApproved(tx, adminId, meta); + }); + const userId = await this.userIdForWallet(flipped.walletId, txn); + await this.events.emitInTransaction(txn, 'wallet.withdrawal.approved', { + userId, + playerId: await this.identityReader.getPlayerIdByUserIdSafe(userId), + amount: flipped.amount, + currency: flipped.currency, + transactionId: flipped.id, + adminId, + ip: meta?.ip ?? null, + userAgent: meta?.userAgent ?? null, + }); + return flipped; + }); + return this.settleApproved(tx, adminId); } // Phase one: the flip to processing under a FOR UPDATE lock. Extracted so the auto path can run it @@ -2070,25 +2079,10 @@ export class WalletService { private async settleApproved( tx: WalletTransaction, adminId: User['id'] | null, - meta?: ClientMeta, ): Promise { const userId = await this.userIdForWallet(tx.walletId); const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); const amount = tx.amount; - // `approved` is admin-attributed; the system auto path skips it - its trail is the - // AUDIT_WRITER entry plus the shared `completed` event below. - if (adminId) { - this.events.emit('wallet.withdrawal.approved', { - userId, - playerId, - amount, - currency: tx.currency, - transactionId: tx.id, - adminId, - ip: meta?.ip ?? null, - userAgent: meta?.userAgent ?? null, - }); - } const { providerName, adapter } = await this.paymentAdapterFor(tx.currency, tx.network); // Stamped before the call: if its response is lost, reconciliation must ask THIS vendor, @@ -2161,29 +2155,35 @@ export class WalletService { // Guarded on `processing`: a vendor webhook can settle the row before this line runs, and // overwriting a webhook-driven `failed` would leave the refund standing next to a payout. - const completed = await this.drizzle.db - .update(walletTransaction) - .set({ - status: 'completed', - providerName, - providerRefId: result.externalId, - }) - .where(and(eq(walletTransaction.id, tx.id), eq(walletTransaction.status, 'processing'))) - .returning({ id: walletTransaction.id }); - if (completed.length === 0) { + const completed = await this.drizzle.db.transaction(async (txn) => { + const updated = await txn + .update(walletTransaction) + .set({ + status: 'completed', + providerName, + providerRefId: result.externalId, + }) + .where(and(eq(walletTransaction.id, tx.id), eq(walletTransaction.status, 'processing'))) + .returning({ id: walletTransaction.id }); + if (updated.length === 0) { + return false; + } + await this.events.emitInTransaction(txn, 'wallet.withdrawal.completed', { + userId, + playerId, + amount, + currency: tx.currency, + transactionId: tx.id, + }); + return true; + }); + if (!completed) { const [current] = await this.drizzle.db .select({ status: walletTransaction.status }) .from(walletTransaction) .where(eq(walletTransaction.id, tx.id)); return { transactionId: tx.id, status: current?.status ?? 'processing' }; } - this.events.emit('wallet.withdrawal.completed', { - userId, - playerId, - amount, - currency: tx.currency, - transactionId: tx.id, - }); return { transactionId: tx.id, status: 'completed' }; } @@ -2201,14 +2201,14 @@ export class WalletService { playerId: Player['id'] | null; amount: string; }): Promise { - const transitioned = await this.drizzle.db.transaction(async (txn) => { + await this.drizzle.db.transaction(async (txn) => { const updated = await txn .update(walletTransaction) .set({ status: 'failed' }) .where(and(eq(walletTransaction.id, tx.id), eq(walletTransaction.status, 'processing'))) .returning({ id: walletTransaction.id }); if (updated.length === 0) { - return false; + return; } await creditWalletBalance(txn, tx.walletId, tx.currency, amount); // In the same transaction as the credit, not left to the `wallet.withdrawal.failed` @@ -2221,13 +2221,8 @@ export class WalletService { ...withdrawalAuditSubject(playerId, tx.id), after: { userId, transactionId: tx.id, amount, currency: tx.currency, reason: null }, }); - return true; - }); - // Emitted on every path that returns held funds, not just the admin-reviewed one. - // Notification-only now - the audit plugin no longer subscribes this topic, see - // the recordInTransaction call above. - if (transitioned) { - this.events.emit('wallet.withdrawal.failed', { + // Emitted on every path that returns held funds, not just the admin-reviewed one. + await this.events.emitInTransaction(txn, 'wallet.withdrawal.failed', { userId, playerId, amount, @@ -2235,7 +2230,7 @@ export class WalletService { transactionId: tx.id, adminId, }); - } + }); } async reconcileWithdrawalStatus( @@ -2267,20 +2262,22 @@ export class WalletService { const playerId = await this.identityReader.getPlayerIdByUserIdSafe(userId); if (status === 'completed') { - const updated = await this.drizzle.db - .update(walletTransaction) - .set({ status: 'completed', txHash: txHash ?? tx.txHash }) - .where(and(eq(walletTransaction.id, tx.id), eq(walletTransaction.status, 'processing'))) - .returning({ id: walletTransaction.id }); - if (updated.length === 0) { - return; - } - this.events.emit('wallet.withdrawal.completed', { - userId, - playerId, - amount: tx.amount, - currency: tx.currency, - transactionId: tx.id, + await this.drizzle.db.transaction(async (txn) => { + const updated = await txn + .update(walletTransaction) + .set({ status: 'completed', txHash: txHash ?? tx.txHash }) + .where(and(eq(walletTransaction.id, tx.id), eq(walletTransaction.status, 'processing'))) + .returning({ id: walletTransaction.id }); + if (updated.length === 0) { + return; + } + await this.events.emitInTransaction(txn, 'wallet.withdrawal.completed', { + userId, + playerId, + amount: tx.amount, + currency: tx.currency, + transactionId: tx.id, + }); }); return; } @@ -3004,20 +3001,20 @@ export class WalletService { await creditWalletBalance(txn, updated.walletId, updated.currency, updated.amount); - return updated; - }); + const userId = await this.userIdForWallet(updated.walletId, txn); + await this.events.emitInTransaction(txn, 'wallet.withdrawal.rejected', { + userId, + playerId: await this.identityReader.getPlayerIdByUserIdSafe(userId), + amount: updated.amount, + currency: updated.currency, + transactionId: updated.id, + adminId, + reason, + ip: meta?.ip ?? null, + userAgent: meta?.userAgent ?? null, + }); - const userId = await this.userIdForWallet(tx.walletId); - this.events.emit('wallet.withdrawal.rejected', { - userId, - playerId: await this.identityReader.getPlayerIdByUserIdSafe(userId), - amount: tx.amount, - currency: tx.currency, - transactionId: tx.id, - adminId, - reason, - ip: meta?.ip ?? null, - userAgent: meta?.userAgent ?? null, + return updated; }); return { transactionId: tx.id, status: 'rejected' }; @@ -3332,7 +3329,8 @@ export class WalletService { return; } - const { transactionId, replayed } = await this.drizzle.db.transaction(async (txn) => { + const playerId = await this.identityReader.getPlayerIdByUserIdSafe(depositAddress.userId); + const { transactionId } = await this.drizzle.db.transaction(async (txn) => { let [walletRecord] = await txn .select() .from(wallet) @@ -3348,7 +3346,7 @@ export class WalletService { event.externalId, ); if (credited) { - return { transactionId: credited.id, replayed: true }; + return { transactionId: credited.id }; } // Throws when no fresh rate exists. The funds are on chain, so the deposit is not // dropped: nothing commits, the webhook answers an error and the vendor redelivers it, @@ -3383,7 +3381,14 @@ export class WalletService { if (inserted) { await creditWalletBalance(txn, walletRecord.id, event.currency, event.amount); - return { transactionId: inserted.id, replayed: false }; + await this.events.emitInTransaction(txn, 'wallet.deposit.completed', { + userId: depositAddress.userId, + playerId, + amount: event.amount, + currency: event.currency, + transactionId: inserted.id, + }); + return { transactionId: inserted.id }; } const [winner] = await this.findDepositByProviderRef( @@ -3396,19 +3401,9 @@ export class WalletService { `payment webhook: idempotency conflict but no row found (externalId=${event.externalId})`, ); } - return { transactionId: winner.id, replayed: true }; + return { transactionId: winner.id }; }); - if (!replayed) { - this.events.emit('wallet.deposit.completed', { - userId: depositAddress.userId, - playerId: await this.identityReader.getPlayerIdByUserIdSafe(depositAddress.userId), - amount: event.amount, - currency: event.currency, - transactionId, - }); - } - // The funds are already on chain, so the player's deposit limit cannot refuse this // credit the way it refuses a PSP charge. It is asked anyway, and a breach becomes a // finding so a human can act on the excess. @@ -3420,7 +3415,7 @@ export class WalletService { // committed deposit row is already inside the window, so asking with `event.amount` // would count this deposit twice. // - // Deliberately NOT guarded by `replayed`: this write failing is what makes the vendor + // Deliberately run on a replay too: this write failing is what makes the vendor // retry the webhook, and the retry sees the credit as a replay. The (kind, providerName, externalId) // unique index is what keeps a replay from filing the finding twice, so the retry // fills a gap instead of duplicating a row. @@ -3468,12 +3463,12 @@ export class WalletService { ); } - private async userIdForWallet(walletId: Wallet['id']) { + private async userIdForWallet( + walletId: Wallet['id'], + txn: DrizzleDb | DrizzleTx = this.drizzle.db, + ) { const record = findOneOrThrow( - await this.drizzle.db - .select({ userId: wallet.userId }) - .from(wallet) - .where(eq(wallet.id, walletId)), + await txn.select({ userId: wallet.userId }).from(wallet).where(eq(wallet.id, walletId)), new WalletNotFoundError(walletId), ); return record.userId; diff --git a/packages/testing/src/__tests__/game-geo-blocking.e2e.test.ts b/packages/testing/src/__tests__/game-geo-blocking.e2e.test.ts index 6b6ddebd9..93876c3cb 100644 --- a/packages/testing/src/__tests__/game-geo-blocking.e2e.test.ts +++ b/packages/testing/src/__tests__/game-geo-blocking.e2e.test.ts @@ -696,9 +696,11 @@ describe('bulk geo restrict / unrestrict', () => { }), ]); expect(await bulkAuditEntries(alreadyRestricted.gameId, 'restrict')).toEqual([]); - expect( - await auditEntries(preExistingRule.id, 'compliance.game-geo-rule.upserted'), - ).toHaveLength(1); + await vi.waitFor(async () => { + expect( + await auditEntries(preExistingRule.id, 'compliance.game-geo-rule.upserted'), + ).toHaveLength(1); + }); }); it('rejects a bulk call with no gameIds/providerIds, more than 500 gameIds, a malformed country code, or a reason over 500 characters', async () => { From 672c072e42f696ae68dddee5cd8441af546f265d Mon Sep 17 00:00:00 2001 From: Volodymyr Zakhovaiko Date: Tue, 13 Oct 2026 00:56:24 +0200 Subject: [PATCH 03/13] docs: record at-least-once event delivery in adr-0044 --- .../rules/messaging-and-microservices.md | 3 +- AGENTS.md | 3 +- ...readiness-manifest-outbox-command-ports.md | 2 + .../0030-distributed-only-production-seams.md | 2 + docs/adr/0044-at-least-once-event-delivery.md | 77 +++++++++++++++++++ docs/platform/architecture.md | 6 +- 6 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 docs/adr/0044-at-least-once-event-delivery.md diff --git a/.rulesync/rules/messaging-and-microservices.md b/.rulesync/rules/messaging-and-microservices.md index 2abc6f2b2..69a150097 100644 --- a/.rulesync/rules/messaging-and-microservices.md +++ b/.rulesync/rules/messaging-and-microservices.md @@ -34,7 +34,8 @@ Use the seams and contracts in code; this rule owns channel selection and the sa - Handlers and jobs are at-least-once: they must be idempotent. Money-adjacent work needs a durable database guard, not only an event ID or idempotency key. - Money and any needed-now answer never travel over events. Use a synchronous, transactional command port. -- `emit()` is best-effort. Use `emitInTransaction()` only when the transactional outbox is explicitly enabled by `OUTBOX_ENABLED`, `AMQP_URL`, or `RABBITMQ_URL`; those AMQP variables enable the outbox but do not bind an AMQP broker. +- An event reporting a committed change on a money, compliance, KYC, RG or account-status path uses `emitInTransaction(tx, ...)` inside the same transaction (the outbox is always on). `emit()` is best-effort and only for informational, catalog and cache-invalidation events. Both throw on an invalid payload. +- A throwing handler is redelivered and dead-lettered to `oss:dlq:` after 5 deliveries (ADR-0044); plugin handlers are deduped per consumer via `processed_events`, a bare `bus.on` without a `consumer` is not. - A Redis Streams deployment uses `SERVICE_NAME` as its durable consumer-group identity. Every independently deployed service sharing Redis needs a distinct name. - The shipped Redis Streams and BullMQ drivers do not honor `orderingKey`; use an overlay when strict ordering is required. - `REALTIME_TRANSPORT` is Redis Pub/Sub (`RedisPubSubRealtimeTransport`, auto-bound on `REDIS_URL`, no in-process fallback - ADR-0031) - fire-and-forget UI push, not a durable event: a message published while nobody is subscribed is simply lost, which is why a realtime channel carries a change _signal_ the client refetches, never the state itself. Channels are prefixed with `SERVICE_NAME` for the same cross-deployment isolation reason as the streams broker's consumer group. `RealtimePresence`/`getOnlineUserIds` are shared across replicas via `RedisPresenceStore`, a per-channel sorted set scored by each member's last heartbeat and read with a TTL cutoff, so a crashed replica's members self-expire instead of lingering forever. diff --git a/AGENTS.md b/AGENTS.md index 371df3dad..561ddc261 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -130,7 +130,8 @@ Use the seams and contracts in code; this rule owns channel selection and the sa - Handlers and jobs are at-least-once: they must be idempotent. Money-adjacent work needs a durable database guard, not only an event ID or idempotency key. - Money and any needed-now answer never travel over events. Use a synchronous, transactional command port. -- `emit()` is best-effort. Use `emitInTransaction()` only when the transactional outbox is explicitly enabled by `OUTBOX_ENABLED`, `AMQP_URL`, or `RABBITMQ_URL`; those AMQP variables enable the outbox but do not bind an AMQP broker. +- An event reporting a committed change on a money, compliance, KYC, RG or account-status path uses `emitInTransaction(tx, ...)` inside the same transaction (the outbox is always on). `emit()` is best-effort and only for informational, catalog and cache-invalidation events. Both throw on an invalid payload. +- A throwing handler is redelivered and dead-lettered to `oss:dlq:` after 5 deliveries (ADR-0044); plugin handlers are deduped per consumer via `processed_events`, a bare `bus.on` without a `consumer` is not. - A Redis Streams deployment uses `SERVICE_NAME` as its durable consumer-group identity. Every independently deployed service sharing Redis needs a distinct name. - The shipped Redis Streams and BullMQ drivers do not honor `orderingKey`; use an overlay when strict ordering is required. - `REALTIME_TRANSPORT` is Redis Pub/Sub (`RedisPubSubRealtimeTransport`, auto-bound on `REDIS_URL`, no in-process fallback - ADR-0031) - fire-and-forget UI push, not a durable event: a message published while nobody is subscribed is simply lost, which is why a realtime channel carries a change _signal_ the client refetches, never the state itself. Channels are prefixed with `SERVICE_NAME` for the same cross-deployment isolation reason as the streams broker's consumer group. `RealtimePresence`/`getOnlineUserIds` are shared across replicas via `RedisPresenceStore`, a per-channel sorted set scored by each member's last heartbeat and read with a TTL cutoff, so a crashed replica's members self-expire instead of lingering forever. diff --git a/docs/adr/0017-extraction-readiness-manifest-outbox-command-ports.md b/docs/adr/0017-extraction-readiness-manifest-outbox-command-ports.md index a301586d3..6bfe00a07 100644 --- a/docs/adr/0017-extraction-readiness-manifest-outbox-command-ports.md +++ b/docs/adr/0017-extraction-readiness-manifest-outbox-command-ports.md @@ -4,6 +4,8 @@ **Status**: Accepted; implemented (manifest boot + outbox seam + `WALLET_COMMANDS` reference; outbox off by default). **Relates to**: ADR-0010 (event-driven broker direction), ADR-0014 (job-queue/realtime seams), ADR-0016 (event envelope). +> **Update (2026-10-13)**: [ADR-0044](./0044-at-least-once-event-delivery.md) made the outbox always on - `OUTBOX_ENABLED` is gone, `createApp` always binds `OUTBOX` and starts the `OutboxRelay`, and money and compliance events are emitted with `emitInTransaction`. The "Opt-in" paragraph below no longer holds. The relay now claims rows atomically with a lease, and consumers dedup per handler through `processed_events` instead of on `eventId` alone. + ## Context ADR-0010/0016 made the inter-module _event_ transport swappable so a module can be diff --git a/docs/adr/0030-distributed-only-production-seams.md b/docs/adr/0030-distributed-only-production-seams.md index c4ba3db40..3f02266a0 100644 --- a/docs/adr/0030-distributed-only-production-seams.md +++ b/docs/adr/0030-distributed-only-production-seams.md @@ -7,6 +7,8 @@ > **Update (2026-07-25)**: [ADR-0039](./0039-tests-run-the-production-seams.md) took the next step and deleted the four in-process seam impls entirely - the "survive as test-only doubles" and "zero infra" clauses below no longer hold, and Postgres + Redis are now prerequisites for the suite. `InProcessRealtimeTransport`/`SseClientAuthorizer` remain as `createApp`'s production default. The distributed-only production decision itself is unchanged and still in force. > > **Update (2026-08-31)**: [ADR-0031](./0031-realtime-transport-sse-and-http-over-web-sockets.md) closed the last exception: `InProcessRealtimeTransport` is deleted and `REALTIME_TRANSPORT` joins `assertDurableSeamsBound`, auto-binding `RedisPubSubRealtimeTransport` on `REDIS_URL` with no in-process fallback. Production is now distributed-only for all five durable seams, not four; `SseClientAuthorizer` (a stateless grant-issuer, not a seam with delivery semantics) is unaffected and stays as-is. +> +> **Update (2026-10-13)**: [ADR-0044](./0044-at-least-once-event-delivery.md) changed the Redis Streams broker's delivery semantics: an entry is XACKed only when every local handler succeeded, failed entries are redelivered via `XAUTOCLAIM` and dead-lettered to `oss:dlq:` after 5 deliveries, and retention is `MINID ~ now - 7d` instead of `MAXLEN ~ 10000`. The distributed-only decision is unchanged. ## Context diff --git a/docs/adr/0044-at-least-once-event-delivery.md b/docs/adr/0044-at-least-once-event-delivery.md new file mode 100644 index 000000000..803aa2622 --- /dev/null +++ b/docs/adr/0044-at-least-once-event-delivery.md @@ -0,0 +1,77 @@ +# ADR-0044: At-least-once event delivery - always-on outbox, redelivery, dead-letter, dedup + +**Date**: 2026-10-13 +**Status**: Accepted +**Amends**: [ADR-0017](./0017-extraction-readiness-manifest-outbox-command-ports.md) (outbox no longer opt-in), [ADR-0030](./0030-distributed-only-production-seams.md) (the Redis Streams broker's ACK and retention semantics). + +## Context + +The event path claimed at-least-once delivery but lost or duplicated events in six places: + +1. **The outbox was opt-in.** Without `OUTBOX_ENABLED` every money and compliance event went + through `emit()`, a fire-and-forget publish after commit. A broker blip or a crash between + commit and publish lost the event; a rolled-back transaction could still publish one. +2. **A failing handler lost its event.** The Redis Streams broker XACKed every entry after + dispatch, whether or not a handler threw. Nothing ever retried it. +3. **Retention was a count.** `MAXLEN ~ 10000` per stream dropped unread entries on a busy topic + within minutes, long before a lagging group caught up. +4. **No consumer-side dedup.** A redelivered event re-ran every handler, including the ones + that had already succeeded. +5. **The relay was single-writer.** Every replica polled the same pending rows with no claim, + so two replicas published the same row twice, and one failing row stopped the batch behind it. +6. **Invalid payloads were logged and delivered anyway**, on both the emit and the receive side. + +## Decision + +**Outbox always on.** `createApp` always binds `DrizzleOutboxWriter` and starts the +`OutboxRelay`; `OUTBOX_ENABLED` is gone. `emitInTransaction(tx, ...)` is the rule for any event +that reports a committed change on a money, compliance, KYC, RG or account-status path: the +event exists if and only if the transaction commits. `emit()` stays for informational, +catalog and cache-invalidation events, where losing one on a broker outage is acceptable. + +**ACK only on success, redeliver, then dead-letter.** The broker XACKs an entry only when every +local handler for the topic succeeded. A failed entry stays pending; `XAUTOCLAIM` reclaims it +after it has been idle for 60s (checked every 30s) and redelivers it. On the 5th delivery +(`MAX_DELIVERIES`) a still-failing entry is copied to `oss:dlq:` with its group, stream +id, delivery count and last error, then XACKed. An entry that is not valid JSON goes straight to +the DLQ. The DLQ is never trimmed; an operator drains it. + +**Time-based retention.** Every `XADD` trims with `MINID ~ now - 7d` (`EVENT_RETENTION_MS`). +A group more than 7 days behind loses entries; stream memory is traffic x 7 days. Both are +accepted ceilings; alert on group lag. + +**Per-handler dedup in `processed_events`.** Table `processed_events (event_id, consumer)`, PK +on both, owned by the engine `outbox` migration set. `createApp` subscribes each plugin handler +with a stable consumer id `//` (`#n` for the nth extra handler a +plugin registers on the same topic). `EventBus.on` skips a handler whose (eventId, consumer) is +already recorded, runs it, and records it only on success - so a redelivery re-runs only the +handlers that failed. + +**Relay with an atomic claim and a lease.** A drain claims a batch in one +`UPDATE ... WHERE event_id IN (SELECT ... FOR UPDATE SKIP LOCKED) RETURNING`, stamping +`claimed_at`. Concurrent relays never claim the same row inside the 30s lease; a relay that +dies mid-publish leaves its rows to be reclaimed once the lease expires. A row that fails to +publish has its lease released and the rest of the batch carries on. The relay polls every +200ms and, at most once an hour, deletes published outbox rows and `processed_events` rows +older than 7 days. + +**Validate at both ends.** `emit`/`emitInTransaction` throw on a known topic whose payload fails +its schema (`emitInTransaction` therefore rolls the caller's transaction back). An inbound +payload that fails validation makes the handler throw, so it follows the redeliver-then-DLQ path. + +## Consequences + +- Money and compliance events are durable and transaction-atomic. They now publish up to + ~200ms after commit (relay poll) instead of immediately. +- Handlers must still be idempotent. Dedup is check, run, record: two concurrent deliveries + of the same event (a reclaim racing a slow first attempt) can both run a handler. Money + handlers keep their database guards. +- A handler registered through `bus.on` without a `consumer` option is not deduped and re-runs + on every redelivery of its topic. +- A consumer id is positional for repeat handlers: reordering two handlers a plugin registers + on the same topic swaps their ids, and in-flight events may re-run them once. +- A poison event retries 5 times over ~5 minutes before it is dead-lettered. +- Every deployment needs the engine outbox migration applied (`0001_event_delivery`). +- Events still on `emit()` (gamification payout notifications, geo access informational events, + catalog changes) remain best-effort. Moving gamification payouts to `emitInTransaction` + needs the event threaded into the payout services' transactions. diff --git a/docs/platform/architecture.md b/docs/platform/architecture.md index ac059e1f0..9e3da7736 100644 --- a/docs/platform/architecture.md +++ b/docs/platform/architecture.md @@ -175,8 +175,10 @@ for the full direction; the shape: process boundary, a service emits it inside its transaction with `events.emitInTransaction(tx, ...)`: the envelope is written to `event_outbox` atomically with the state change, and the `OutboxRelay` publishes it after commit - (at-least-once; consumers dedup on `eventId`). Opt-in via `OUTBOX_ENABLED` / a - durable broker; off in the in-process monolith. + (at-least-once). Always on; required for money and compliance events, `emit()` + stays for informational ones. A failing handler is redelivered and dead-lettered + after 5 attempts; handlers are deduped per consumer via `processed_events` + (ADR-0044). - **Client push is separate.** WebSockets/SSE are client-facing only (chat, balance/bonus toasts), never the transport between modules. - **Modular monolith now, microservices later.** The no-cross-module-imports rule, From ae3b1f21284ecf18411ad316136d25856aa5bd74 Mon Sep 17 00:00:00 2001 From: Volodymyr Zakhovaiko Date: Tue, 13 Oct 2026 00:58:59 +0200 Subject: [PATCH 04/13] test(core): cover time-based stream retention in the redis broker --- .../redis-streams-broker.int.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts b/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts index 5cdbc3f12..8e73d4e4b 100644 --- a/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts +++ b/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts @@ -68,6 +68,38 @@ describe('RedisStreamsBroker', () => { expect(JSON.parse(entries[0]!.message['data']!)).toEqual(envelope); }); + it('trims entries older than the retention window on publish', async () => { + const broker = makeBroker('svc'); + const topic = uniqueTopic(); + const ancientEntryCount = 300; + const seed = redis.client.multi(); + for (let i = 1; i <= ancientEntryCount; i++) { + seed.xAdd(streamOf(topic), `${i}-0`, { data: 'ancient' }); + } + await seed.exec(); + + await broker.publish(makeEnvelope(topic)); + + const remaining = await redis.client.xRange(streamOf(topic), '-', '+'); + expect(remaining.length).toBeLessThan(ancientEntryCount); + expect(remaining.at(-1)!.message['data']).not.toBe('ancient'); + }); + + it('keeps recent entries past the old 10,000-entry cap', async () => { + const broker = makeBroker('svc'); + const topic = uniqueTopic(); + const recentEntryCount = 10_200; + const seed = redis.client.multi(); + for (let i = 0; i < recentEntryCount; i++) { + seed.xAdd(streamOf(topic), '*', { data: 'recent' }); + } + await seed.exec(); + + await broker.publish(makeEnvelope(topic)); + + expect(await redis.client.xLen(streamOf(topic))).toBe(recentEntryCount + 1); + }); + it('coalesces two subscribers on one topic: both receive, the entry is ACKed', async () => { const broker = makeBroker('wallet-svc'); const topic = uniqueTopic(); From 5b6d4c26cf78227d8e43d47cc4059ec5d1be3aa6 Mon Sep 17 00:00:00 2001 From: Volodymyr Zakhovaiko Date: Tue, 13 Oct 2026 12:58:22 +0200 Subject: [PATCH 05/13] fix(core): await event handler side effects so failures retry Handlers that fired promises and returned early when their service was not yet constructed were acked as processed even when the work failed. They now return or await the work and throw when a required dependency is missing, so the broker retries and dead-letters instead of dropping the event. --- packages/core/src/audit/plugin.ts | 18 +- packages/core/src/compliance/plugin.ts | 101 +++++------ packages/core/src/engagement/chat/plugin.ts | 66 +++---- .../src/engagement/notifications/plugin.ts | 167 +++++++++--------- packages/core/src/iam/plugin.ts | 12 +- packages/core/src/pam/identity/plugin.ts | 31 ++-- .../core/src/pam/player-management/plugin.ts | 34 ++-- packages/core/src/pam/tag/plugin.ts | 41 +++-- packages/core/src/promo/bonus/plugin.ts | 57 +++--- packages/core/src/wallet/plugin.ts | 33 +++- 10 files changed, 281 insertions(+), 279 deletions(-) diff --git a/packages/core/src/audit/plugin.ts b/packages/core/src/audit/plugin.ts index 5b120e2e5..d3d27614d 100644 --- a/packages/core/src/audit/plugin.ts +++ b/packages/core/src/audit/plugin.ts @@ -1,4 +1,4 @@ -import { EVENT_BUS, DRIZZLE, ADMIN_GUARD, createLogger } from '@openora/core/server'; +import { EVENT_BUS, DRIZZLE, ADMIN_GUARD } from '@openora/core/server'; import type { CoreTokenCatalog, Plugin } from '@openora/core/server'; import { AUDIT_WRITER, IDENTITY_READER, type DomainEventName } from '@openora/core/contracts'; import { AuditService, type RecordInput } from './service/audit.service.js'; @@ -1490,8 +1490,6 @@ const SUBSCRIBED_TOPICS: DomainEventName[] = [ export default { id: 'audit', register(ctx) { - const logger = createLogger('audit'); - // Subscriptions are wired before router factories run (create-app.ts boot order), // so svcRef is null at registration but set before any real event arrives. let svcRef: AuditService | null = null; @@ -1508,8 +1506,10 @@ export default { }); for (const topic of SUBSCRIBED_TOPICS) { - ctx.events.on(topic, (payload) => { - if (!svcRef || !isRecord(payload)) { + // Awaited and thrown, never swallowed: a failed write leaves the event pending, so the + // broker redelivers it instead of the change going unaudited (ADR-0044). + ctx.events.on(topic, async (payload) => { + if (!isRecord(payload)) { return; } // KYC exemptions and game geo rule syncs append their audit record inside the state @@ -1517,10 +1517,10 @@ export default { if (payload['auditRecorded'] === true) { return; } - const svc = svcRef; - void mapEventToRecord(topic, payload) - .then((record) => svc.record(record)) - .catch((err) => logger.error({ err, topic }, 'audit record failed')); + if (!svcRef) { + throw new Error(`audit record for '${topic}' skipped - service not constructed`); + } + await svcRef.record(await mapEventToRecord(topic, payload)); }); } diff --git a/packages/core/src/compliance/plugin.ts b/packages/core/src/compliance/plugin.ts index d276e13e7..b02ef4a6c 100644 --- a/packages/core/src/compliance/plugin.ts +++ b/packages/core/src/compliance/plugin.ts @@ -58,13 +58,6 @@ import { KycWithdrawalGate } from './adapters/kyc-withdrawal-gate.js'; const logger = createLogger('compliance'); -const KYC_EXEMPTION_ENQUEUE_ATTEMPTS = 3; -const KYC_EXEMPTION_ENQUEUE_RETRY_DELAY_MS = 250; - -function delay(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - const makeComplianceService = (c: TypedContainer) => new ComplianceService( c.get(DRIZZLE), @@ -230,86 +223,72 @@ export default { }); }); - ctx.events.on('wallet.deposit.completed', (payload) => { + // Handlers await their work and throw when a dependency is missing: a failure leaves the + // event pending, so the broker redelivers it rather than marking it handled (ADR-0044). + const requireJobQueue = (): JobQueueAdapter => { + if (!jobQueueRef) { + throw new Error('compliance job queue not bound yet'); + } + return jobQueueRef; + }; + + ctx.events.on('wallet.deposit.completed', async (payload) => { const parsed = domainEventSchemas['wallet.deposit.completed'].safeParse(payload); - if (!parsed.success || !kycRef) { + if (!parsed.success) { return; } - kycRef - .handleDeposit(parsed.data.userId) - .catch((err) => logger.error({ err }, 're-KYC deposit hook failed')); + if (!kycRef) { + throw new Error('re-KYC deposit hook - kyc service not constructed'); + } + await kycRef.handleDeposit(parsed.data.userId); }); - // Country-level KYC exemption is durable work: a registration event can be delivered - // at least once and a resolver/database outage must retry rather than strand a player - // at pending. The job's database guard makes repeated delivery harmless. The event - // dispatcher itself only logs a thrown handler error and still acks the message (no - // redelivery), so a transient enqueue failure needs its own bounded retry here - - // otherwise it is silently dropped after a single attempt. + // Country-level KYC exemption is durable work: a resolver/database outage must retry + // rather than strand a player at pending. A failed enqueue throws, so the broker + // redelivers the registration; the idempotency key and the job's database guard make + // repeated delivery harmless. ctx.events.on('identity.user.registered', async (payload) => { const parsed = domainEventSchemas['identity.user.registered'].safeParse(payload); - if (!parsed.success || !jobQueueRef) { + if (!parsed.success) { return; } - const jobQueue = jobQueueRef; - const countryCode = parsed.data.countryCode ?? null; - for (let attempt = 1; attempt <= KYC_EXEMPTION_ENQUEUE_ATTEMPTS; attempt += 1) { - try { - await jobQueue.enqueue( - KYC_EXEMPTION_QUEUE, - { userId: parsed.data.userId, countryCode }, - { - idempotencyKey: `kyc-exemption:${parsed.data.userId}`, - orderingKey: parsed.data.userId, - attempts: 10, - backoff: { type: 'exponential', delayMs: 1_000 }, - }, - ); - return; - } catch (err) { - if (attempt === KYC_EXEMPTION_ENQUEUE_ATTEMPTS) { - logger.error( - { err, userId: parsed.data.userId, countryCode, attempts: attempt }, - 'kyc-exemption enqueue failed after retries; player left without an exemption evaluation', - ); - return; - } - await delay(KYC_EXEMPTION_ENQUEUE_RETRY_DELAY_MS * attempt); - } - } + await requireJobQueue().enqueue( + KYC_EXEMPTION_QUEUE, + { userId: parsed.data.userId, countryCode: parsed.data.countryCode ?? null }, + { + idempotencyKey: `kyc-exemption:${parsed.data.userId}`, + orderingKey: parsed.data.userId, + attempts: 10, + backoff: { type: 'exponential', delayMs: 1_000 }, + }, + ); }); - const enqueueEval = (userId: string, trigger: RgEvalTrigger) => { - if (!jobQueueRef) { - return; - } - void jobQueueRef - .enqueue(RG_EVAL_QUEUE, { userId, trigger }, { orderingKey: userId }) - .catch((err) => logger.error({ err }, 'rg-eval enqueue failed')); - }; + const enqueueEval = (userId: string, trigger: RgEvalTrigger) => + requireJobQueue().enqueue(RG_EVAL_QUEUE, { userId, trigger }, { orderingKey: userId }); - ctx.events.on('wallet.deposit.completed', (payload) => { + ctx.events.on('wallet.deposit.completed', async (payload) => { const parsed = domainEventSchemas['wallet.deposit.completed'].safeParse(payload); if (parsed.success) { - enqueueEval(parsed.data.userId, 'wallet.deposit.completed'); + await enqueueEval(parsed.data.userId, 'wallet.deposit.completed'); } }); - ctx.events.on('gaming.round.ended', (payload) => { + ctx.events.on('gaming.round.ended', async (payload) => { const parsed = domainEventSchemas['gaming.round.ended'].safeParse(payload); if (parsed.success) { - enqueueEval(parsed.data.userId, 'gaming.round.ended'); + await enqueueEval(parsed.data.userId, 'gaming.round.ended'); } }); - ctx.events.on('rg.exclusion.login_blocked', (payload) => { + ctx.events.on('rg.exclusion.login_blocked', async (payload) => { const parsed = domainEventSchemas['rg.exclusion.login_blocked'].safeParse(payload); if (parsed.success) { - enqueueEval(parsed.data.userId, 'rg.exclusion.login_blocked'); + await enqueueEval(parsed.data.userId, 'rg.exclusion.login_blocked'); } }); - ctx.events.on('rg.limit.set', (payload) => { + ctx.events.on('rg.limit.set', async (payload) => { const parsed = domainEventSchemas['rg.limit.set'].safeParse(payload); if (parsed.success) { - enqueueEval(parsed.data.userId, 'rg.limit.set'); + await enqueueEval(parsed.data.userId, 'rg.limit.set'); } }); diff --git a/packages/core/src/engagement/chat/plugin.ts b/packages/core/src/engagement/chat/plugin.ts index 0618137e3..04de341cf 100644 --- a/packages/core/src/engagement/chat/plugin.ts +++ b/packages/core/src/engagement/chat/plugin.ts @@ -159,67 +159,59 @@ export default { let purgeRef: ChatRoomPurgeService | null = null; let jobQueueRef: JobQueueAdapter | null = null; - const onAccountClosed = (userId: string) => { + // Awaited and thrown: a failed enqueue leaves the event pending for redelivery (ADR-0044). + const requireJobQueue = (): JobQueueAdapter => { if (!jobQueueRef) { - logger.error({ userId }, 'chat account-closed dropped: job queue not bound yet'); - return; + throw new Error('chat job queue not bound yet'); } + return jobQueueRef; + }; + + const onAccountClosed = async (userId: string) => { const closedAt = new Date(); - jobQueueRef - .enqueue( - CHAT_ACCOUNT_CLOSED_QUEUE, - { userId, closedAt: closedAt.toISOString() }, - { idempotencyKey: accountJobKey('chat-account-closed', userId, closedAt), ...RETRY }, - ) - .catch((err: unknown) => - logger.error({ err, userId }, 'chat account-closed enqueue failed'), - ); + await requireJobQueue().enqueue( + CHAT_ACCOUNT_CLOSED_QUEUE, + { userId, closedAt: closedAt.toISOString() }, + { idempotencyKey: accountJobKey('chat-account-closed', userId, closedAt), ...RETRY }, + ); }; - const onAccountReopened = (userId: string) => { - if (!jobQueueRef) { - logger.error({ userId }, 'chat account-reopened dropped: job queue not bound yet'); - return; - } - jobQueueRef - .enqueue( - CHAT_ACCOUNT_REOPENED_QUEUE, - { userId }, - { - idempotencyKey: accountJobKey('chat-account-reopened', userId, new Date()), - ...RETRY, - }, - ) - .catch((err: unknown) => - logger.error({ err, userId }, 'chat account-reopened enqueue failed'), - ); + const onAccountReopened = async (userId: string) => { + await requireJobQueue().enqueue( + CHAT_ACCOUNT_REOPENED_QUEUE, + { userId }, + { + idempotencyKey: accountJobKey('chat-account-reopened', userId, new Date()), + ...RETRY, + }, + ); }; - ctx.events.on('player.account.closed', (payload) => { + ctx.events.on('player.account.closed', async (payload) => { const parsed = domainEventSchemas['player.account.closed'].safeParse(payload); if (parsed.success) { - onAccountClosed(parsed.data.userId); + await onAccountClosed(parsed.data.userId); } }); - ctx.events.on('identity.user.deactivated', (payload) => { + ctx.events.on('identity.user.deactivated', async (payload) => { const parsed = domainEventSchemas['identity.user.deactivated'].safeParse(payload); if (parsed.success) { - onAccountClosed(parsed.data.userId); + await onAccountClosed(parsed.data.userId); } }); - ctx.events.on('player.account.reopened', (payload) => { + ctx.events.on('player.account.reopened', async (payload) => { const parsed = domainEventSchemas['player.account.reopened'].safeParse(payload); if (parsed.success) { - onAccountReopened(parsed.data.userId); + await onAccountReopened(parsed.data.userId); } }); - ctx.events.on('identity.user.reactivated', (payload) => { + ctx.events.on('identity.user.reactivated', async (payload) => { const parsed = domainEventSchemas['identity.user.reactivated'].safeParse(payload); if (parsed.success) { - onAccountReopened(parsed.data.userId); + await onAccountReopened(parsed.data.userId); } }); diff --git a/packages/core/src/engagement/notifications/plugin.ts b/packages/core/src/engagement/notifications/plugin.ts index 53167ff03..941a7324a 100644 --- a/packages/core/src/engagement/notifications/plugin.ts +++ b/packages/core/src/engagement/notifications/plugin.ts @@ -570,55 +570,62 @@ export default { } }; + // Event handlers await their work and throw when the module isn't wired: a failure leaves + // the event pending, so the broker redelivers it instead of marking it handled (ADR-0044). + const requireService = (): NotificationsService => { + if (!svcRef) { + throw new Error('notifications service not constructed'); + } + return svcRef; + }; + const requireJobQueue = (): JobQueueAdapter => { + if (!jobQueueRef) { + throw new Error('notifications job queue not bound yet'); + } + return jobQueueRef; + }; + // The one place every `create()` call ends up on the realtime channel from - this // module's own dispatch jobs below, and an overlay's own `NotificationsService` instance // over a consumer-specific event `domainEventSchemas` has no entry for, since both share // this event bus and `create()` always emits this topic. Re-reads the row rather than // carrying it on the event: the event is `{ notificationId, userId }` only, the same shape // every other consumer of this topic (the audit log) already treats as the full payload. - ctx.events.on('notifications.created', (payload) => { + ctx.events.on('notifications.created', async (payload) => { const parsed = domainEventSchemas['notifications.created'].safeParse(payload); - if (!parsed.success || !svcRef) { + if (!parsed.success) { return; } - void svcRef - .getById(parsed.data.notificationId) - .then((record) => { - if (record) { - publishNotification(record); - } - }) - .catch((err: unknown) => logger.error({ err }, 'notification realtime lookup failed')); + const record = await requireService().getById(parsed.data.notificationId); + if (record) { + publishNotification(record); + } }); for (const entry of notificationEventMap) { - ctx.events.on(entry.event, (payload, envelope) => { - if (!jobQueueRef || !envelope) { + ctx.events.on(entry.event, async (payload, envelope) => { + if (!envelope) { return; } const data = entry.parse(payload); if (data === null) { return; } - jobQueueRef - .enqueue( - NOTIFICATIONS_DISPATCH_QUEUE, - { - event: entry.event, - input: { ...entry.buildNotification(data), eventId: envelope.eventId }, - email: entry.buildEmail(data, envelope.occurredAt), - // An opted-in entry's email is preference-gated; every other entry always mails. - securityAlert: entry.securityAlert, - }, - { - idempotencyKey: `notifications-dispatch:${envelope.eventId}`, - attempts: 5, - backoff: { type: 'exponential', delayMs: 1000 }, - }, - ) - .catch((err) => - logger.error({ err, event: entry.event }, 'notification dispatch enqueue failed'), - ); + await requireJobQueue().enqueue( + NOTIFICATIONS_DISPATCH_QUEUE, + { + event: entry.event, + input: { ...entry.buildNotification(data), eventId: envelope.eventId }, + email: entry.buildEmail(data, envelope.occurredAt), + // An opted-in entry's email is preference-gated; every other entry always mails. + securityAlert: entry.securityAlert, + }, + { + idempotencyKey: `notifications-dispatch:${envelope.eventId}`, + attempts: 5, + backoff: { type: 'exponential', delayMs: 1000 }, + }, + ); }); } @@ -645,6 +652,7 @@ export default { }; ctx.events.on('identity.email.verified', async (payload) => { + requireService(); if (!mailDispatchRef) { return; } @@ -654,18 +662,20 @@ export default { } const { userId, ip } = parsed.data; const promotionsEligible = await isEligibleForPromotions(userId, ip); - await mailDispatchRef - .toUser({ - userId, - template: { key: 'welcome', data: { promotionsEligible } }, - idempotencyKey: `welcome-mail:${userId}`, - }) - .catch((err) => logger.error({ err }, 'welcome mail enqueue failed')); + await mailDispatchRef.toUser({ + userId, + template: { key: 'welcome', data: { promotionsEligible } }, + idempotencyKey: `welcome-mail:${userId}`, + }); }); ctx.events.on('identity.authentication.succeeded', async (payload, envelope) => { const parsed = domainEventSchemas['identity.authentication.succeeded'].safeParse(payload); - if (!parsed.success || !jobQueueRef || !identityReaderRef) { + if (!parsed.success) { + return; + } + const jobQueue = requireJobQueue(); + if (!identityReaderRef) { return; } if (!envelope?.eventId) { @@ -681,7 +691,7 @@ export default { if (!(await identityReaderRef.canReceiveLoginWithdrawalAlerts?.(parsed.data.userId))) { return; } - await jobQueueRef.enqueue( + await jobQueue.enqueue( SECURITY_ALERT_DISPATCH_QUEUE, { userId: parsed.data.userId, eventId: envelope.eventId, occurredAt: envelope.occurredAt }, { @@ -694,9 +704,9 @@ export default { // Non-null `reason` marks an admin override (ADR-0037); player-initiated // limit changes always emit `reason: null`. - ctx.events.on('rg.limit.set', (payload) => { + ctx.events.on('rg.limit.set', async (payload, envelope) => { const parsed = domainEventSchemas['rg.limit.set'].safeParse(payload); - if (!parsed.success || !svcRef || parsed.data.reason === null) { + if (!parsed.success || parsed.data.reason === null) { return; } const p = parsed.data; @@ -708,14 +718,19 @@ export default { const next = describeLimitValue(p.amount, p.minutes); const body = `An administrator changed your ${p.period} ${p.type} limit from ${previous} to ${next}.`; // In-app only: `RgService.setPlayerLimit` already sends the `rgLimitUpdated` mail on this write. - svcRef - .create({ userId: p.userId, type: 'rg.limit.admin_updated', title, body }) - .catch((err) => logger.error({ err }, 'rg.limit.set admin-override notification failed')); + // The event id dedups the row when a redelivery re-runs this handler. + await requireService().create({ + userId: p.userId, + type: 'rg.limit.admin_updated', + title, + body, + ...(envelope ? { eventId: envelope.eventId } : {}), + }); }); - ctx.events.on('chat.room.scheduled_for_deletion', (payload, envelope) => { + ctx.events.on('chat.room.scheduled_for_deletion', async (payload, envelope) => { const parsed = domainEventSchemas['chat.room.scheduled_for_deletion'].safeParse(payload); - if (!parsed.success || !jobQueueRef) { + if (!parsed.success) { return; } const p = parsed.data; @@ -724,36 +739,32 @@ export default { return; } const eventId = envelope.eventId; - const jobQueue = jobQueueRef; + const jobQueue = requireJobQueue(); for (const userId of p.memberIds.filter((id) => id !== p.previousOwnerId)) { - jobQueue - .enqueue( - NOTIFICATIONS_DISPATCH_QUEUE, - { - event: 'chat.room.scheduled_for_deletion', - input: buildChatRoomScheduledForDeletionNotification({ - userId, - roomId: p.roomId, - roomName: p.roomName, - }), - // In-app only: `email` is nullish on NotificationDispatchJobSchema, so - // omitting it is the current equivalent of the old `sendEmail: false`. - }, - { - idempotencyKey: `notifications-dispatch:${eventId}:${userId}`, - attempts: 5, - backoff: { type: 'exponential', delayMs: 1000 }, - }, - ) - .catch((err) => - logger.error({ err }, 'chat.room.scheduled_for_deletion dispatch enqueue failed'), - ); + await jobQueue.enqueue( + NOTIFICATIONS_DISPATCH_QUEUE, + { + event: 'chat.room.scheduled_for_deletion', + input: buildChatRoomScheduledForDeletionNotification({ + userId, + roomId: p.roomId, + roomName: p.roomName, + }), + // In-app only: `email` is nullish on NotificationDispatchJobSchema, so + // omitting it is the current equivalent of the old `sendEmail: false`. + }, + { + idempotencyKey: `notifications-dispatch:${eventId}:${userId}`, + attempts: 5, + backoff: { type: 'exponential', delayMs: 1000 }, + }, + ); } }); - ctx.events.on('compliance.kyc.updated', (payload, envelope) => { + ctx.events.on('compliance.kyc.updated', async (payload, envelope) => { const parsed = domainEventSchemas['compliance.kyc.updated'].safeParse(payload); - if (!parsed.success || !jobQueueRef) { + if (!parsed.success) { return; } const p = parsed.data; @@ -770,13 +781,11 @@ export default { ); return; } - jobQueueRef - .enqueue( - KYC_RESUBMISSION_NOTIFY_QUEUE, - { userId: p.userId, reason: p.reason, eventId: envelope.eventId }, - { idempotencyKey: `kyc-resubmission-notify:${envelope.eventId}` }, - ) - .catch((err) => logger.error({ err }, 'kyc-resubmission-notify enqueue failed')); + await requireJobQueue().enqueue( + KYC_RESUBMISSION_NOTIFY_QUEUE, + { userId: p.userId, reason: p.reason, eventId: envelope.eventId }, + { idempotencyKey: `kyc-resubmission-notify:${envelope.eventId}` }, + ); }); ctx.jobs.worker({ diff --git a/packages/core/src/iam/plugin.ts b/packages/core/src/iam/plugin.ts index 384323c37..d1507ec9c 100644 --- a/packages/core/src/iam/plugin.ts +++ b/packages/core/src/iam/plugin.ts @@ -1,4 +1,4 @@ -import { EVENT_BUS, DRIZZLE, ADMIN_GUARD, createLogger } from '@openora/core/server'; +import { EVENT_BUS, DRIZZLE, ADMIN_GUARD } from '@openora/core/server'; import type { CoreTokenCatalog, Plugin, TypedContainer } from '@openora/core/server'; import { ADMIN_PERMISSION_RESOLVER, @@ -22,8 +22,6 @@ import { DrizzleAdminPlayerActivity } from './adapters/admin-player-activity.js' import { DrizzleAdminRoleAssignmentDirectory } from './adapters/admin-role-assignment-directory.js'; import { DrizzleMcpTokenAuthenticator } from './adapters/mcp-token-authenticator.js'; -const logger = createLogger('iam'); - function makeMcpTokenService(c: TypedContainer) { return new McpTokenService({ drizzle: c.get(DRIZZLE), @@ -63,9 +61,7 @@ export default { if (!parsed.success || !resolverRef) { return; } - resolverRef - .invalidateUser(parsed.data.userId) - .catch((err) => logger.error({ err }, 'grant cache purge failed')); + return resolverRef.invalidateUser(parsed.data.userId); }); } ctx.events.on('iam.role.permissions.changed', (payload) => { @@ -73,9 +69,7 @@ export default { if (!parsed.success || !resolverRef) { return; } - resolverRef - .invalidateRole(parsed.data.roleId) - .catch((err) => logger.error({ err }, 'grant cache purge failed')); + return resolverRef.invalidateRole(parsed.data.roleId); }); ctx.provide(ADMIN_PLAYER_ACTIVITY, (c) => new DrizzleAdminPlayerActivity(c.get(DRIZZLE))); diff --git a/packages/core/src/pam/identity/plugin.ts b/packages/core/src/pam/identity/plugin.ts index 63c65dee6..f15b33147 100644 --- a/packages/core/src/pam/identity/plugin.ts +++ b/packages/core/src/pam/identity/plugin.ts @@ -135,29 +135,38 @@ export default { if (!parsed.success) { return; } - void realtimeTransport?.publish(sessionEventsChannel(parsed.data.userId), { - type: 'revoked-all', - ...(parsed.data.exceptSessionId ? { exceptSessionId: parsed.data.exceptSessionId } : {}), - }); + return realtimeTransport?.publish( + sessionEventsChannel(parsed.data.userId), + { + type: 'revoked-all', + ...(parsed.data.exceptSessionId ? { exceptSessionId: parsed.data.exceptSessionId } : {}), + }, + ); }); ctx.events.on('identity.session.revoked', (raw) => { const parsed = domainEventSchemas['identity.session.revoked'].safeParse(raw); if (!parsed.success) { return; } - void realtimeTransport?.publish(sessionEventsChannel(parsed.data.userId), { - type: 'session-revoked', - sessionId: parsed.data.sessionId, - }); + return realtimeTransport?.publish( + sessionEventsChannel(parsed.data.userId), + { + type: 'session-revoked', + sessionId: parsed.data.sessionId, + }, + ); }); ctx.events.on('identity.user.unlocked', (raw) => { const parsed = domainEventSchemas['identity.user.unlocked'].safeParse(raw); if (!parsed.success) { return; } - void realtimeTransport?.publish(sessionEventsChannel(parsed.data.userId), { - type: 'unlocked', - }); + return realtimeTransport?.publish( + sessionEventsChannel(parsed.data.userId), + { + type: 'unlocked', + }, + ); }); const withdrawalPinHmacSecret = process.env['WITHDRAWAL_PIN_HMAC_SECRET'] ?? ''; diff --git a/packages/core/src/pam/player-management/plugin.ts b/packages/core/src/pam/player-management/plugin.ts index a096ebb42..79c9eaa82 100644 --- a/packages/core/src/pam/player-management/plugin.ts +++ b/packages/core/src/pam/player-management/plugin.ts @@ -1,4 +1,4 @@ -import { EVENT_BUS, DRIZZLE, ADMIN_GUARD, createLogger } from '@openora/core/server'; +import { EVENT_BUS, DRIZZLE, ADMIN_GUARD } from '@openora/core/server'; import { AUDIT_WRITER, KYC_STATUS_WRITER, @@ -36,8 +36,6 @@ function makePlayerService(c: TypedContainer) { ); } -const logger = createLogger('player-management'); - const TOTAL_DEPOSITS_QUEUE = queue('player-total-deposits'); const TotalDepositsJobSchema = z.object({ userId: UuidSchema }); @@ -55,23 +53,25 @@ export default { // The refresh recomputes from the ledger, so it runs as a retryable job: a failed read or // an unpriced currency retries instead of leaving totalDeposits stale until the next deposit. - ctx.events.on('wallet.deposit.completed', (payload) => { + // Awaited and thrown: a failed enqueue leaves the event pending for redelivery (ADR-0044). + ctx.events.on('wallet.deposit.completed', async (payload) => { const parsed = domainEventSchemas['wallet.deposit.completed'].safeParse(payload); - if (!parsed.success || !jobQueue) { + if (!parsed.success) { return; } - void jobQueue - .enqueue( - TOTAL_DEPOSITS_QUEUE, - { userId: parsed.data.userId }, - { - idempotencyKey: `player-total-deposits:${parsed.data.transactionId}`, - orderingKey: parsed.data.userId, - attempts: 10, - backoff: { type: 'exponential', delayMs: 1_000 }, - }, - ) - .catch((err) => logger.error({ err }, 'player total deposits enqueue failed')); + if (!jobQueue) { + throw new Error('player total deposits - job queue not bound yet'); + } + await jobQueue.enqueue( + TOTAL_DEPOSITS_QUEUE, + { userId: parsed.data.userId }, + { + idempotencyKey: `player-total-deposits:${parsed.data.transactionId}`, + orderingKey: parsed.data.userId, + attempts: 10, + backoff: { type: 'exponential', delayMs: 1_000 }, + }, + ); }); ctx.jobs.worker({ queue: TOTAL_DEPOSITS_QUEUE, diff --git a/packages/core/src/pam/tag/plugin.ts b/packages/core/src/pam/tag/plugin.ts index 182e18b15..be6f22dbe 100644 --- a/packages/core/src/pam/tag/plugin.ts +++ b/packages/core/src/pam/tag/plugin.ts @@ -117,20 +117,37 @@ export default { tagEvaluationService(c).evaluateWithdrawalRequested(tx, args), })); - ctx.events.on('wallet.deposit.completed', (p) => evalSvc?.onDepositCompleted(p)); - ctx.events.on('wallet.withdrawal.completed', (p) => evalSvc?.onWithdrawalCompleted(p)); - ctx.events.on('wallet.withdrawal.requested', (p) => evalSvc?.onWithdrawalRequested(p)); - ctx.events.on('identity.user.login', (p) => evalSvc?.onUserLogin(p)); - ctx.events.on('identity.user.phone_login', (p) => evalSvc?.onUserPhoneLogin(p)); - ctx.events.on('compliance.kyc.submitted', (p) => evalSvc?.onKycSubmitted(p)); - ctx.events.on('compliance.kyc.updated', (p) => evalSvc?.onKycStatusUpdated(p)); - ctx.events.on('compliance.kyc.reverify_required', (p) => evalSvc?.onKycReverifyRequired(p)); + // Throws rather than no-ops before construction, so the event is redelivered instead of + // being recorded as handled (ADR-0044). + const requireEvaluation = (): TagEvaluationService => { + if (!evalSvc) { + throw new Error('tag evaluation service not constructed'); + } + return evalSvc; + }; + + ctx.events.on('wallet.deposit.completed', (p) => requireEvaluation().onDepositCompleted(p)); + ctx.events.on('wallet.withdrawal.completed', (p) => + requireEvaluation().onWithdrawalCompleted(p), + ); + ctx.events.on('wallet.withdrawal.requested', (p) => + requireEvaluation().onWithdrawalRequested(p), + ); + ctx.events.on('identity.user.login', (p) => requireEvaluation().onUserLogin(p)); + ctx.events.on('identity.user.phone_login', (p) => requireEvaluation().onUserPhoneLogin(p)); + ctx.events.on('compliance.kyc.submitted', (p) => requireEvaluation().onKycSubmitted(p)); + ctx.events.on('compliance.kyc.updated', (p) => requireEvaluation().onKycStatusUpdated(p)); + ctx.events.on('compliance.kyc.reverify_required', (p) => + requireEvaluation().onKycReverifyRequired(p), + ); ctx.events.on('compliance.kyc.high_risk_signal_detected', (p) => - evalSvc?.onKycHighRiskSignalDetected(p), + requireEvaluation().onKycHighRiskSignalDetected(p), + ); + ctx.events.on('rg.self_exclusion.activated', (p) => + requireEvaluation().onSelfExclusionActivated(p), ); - ctx.events.on('rg.self_exclusion.activated', (p) => evalSvc?.onSelfExclusionActivated(p)); - ctx.events.on('rg.self_exclusion.lifted', (p) => evalSvc?.onSelfExclusionLifted(p)); - ctx.events.on('player.level.changed', (p) => evalSvc?.onPlayerLevelChanged(p)); + ctx.events.on('rg.self_exclusion.lifted', (p) => requireEvaluation().onSelfExclusionLifted(p)); + ctx.events.on('player.level.changed', (p) => requireEvaluation().onPlayerLevelChanged(p)); // Daily inactive-player sweep. schedule() is called in the router factory (container access). ctx.jobs.worker({ diff --git a/packages/core/src/promo/bonus/plugin.ts b/packages/core/src/promo/bonus/plugin.ts index 59acead21..416d49e1e 100644 --- a/packages/core/src/promo/bonus/plugin.ts +++ b/packages/core/src/promo/bonus/plugin.ts @@ -133,9 +133,7 @@ export default { currency: string, reason: BonusBalanceChangeReason, eventId: string, - ) => { - void realtime?.publish(bonusBalanceChannel(userId), { eventId, currency, reason }); - }; + ) => realtime?.publish(bonusBalanceChannel(userId), { eventId, currency, reason }); // Subscribed centrally rather than published beside each emit: the four topics already have // one producer each, and a fifth producer added later gets the signal for free instead of @@ -151,7 +149,7 @@ export default { if (!parsed.success || !envelope) { return; } - signalBalance(parsed.data.userId, parsed.data.currency, reason, envelope.eventId); + return signalBalance(parsed.data.userId, parsed.data.currency, reason, envelope.eventId); }); } @@ -201,15 +199,14 @@ export default { topic: K, reason: 'self_exclusion' | 'cooling_off' | 'account_closed' | 'account_deactivated', ) => - (payload: unknown) => { + async (payload: unknown) => { const parsed = domainEventSchemas[topic].safeParse(payload); if (!parsed.success) { logger.error({ topic }, 'promo bonus forfeit skipped - event payload failed validation'); return; } if (!jobs) { - logger.error({ topic }, 'promo bonus forfeit dropped - job queue not bound yet'); - return; + throw new Error(`promo bonus forfeit for '${topic}' - job queue not bound yet`); } const { userId, actorId } = parsed.data; // Account closure and a ban (identity.user.deactivated) are always an admin action. A @@ -228,20 +225,16 @@ export default { // bonus and excludes themselves again produces the same key - which BullMQ drops // silently, leaving the second bonus active. The durable guard is the `status = 'active'` // claim inside `forfeitAllFor`, which already makes a redelivery write nothing twice. - void jobs - .enqueue( - FORFEIT_QUEUE, - { - userId, - reason, - actorId: initiatedBy === 'system' ? null : actorId, - actorIsAdmin: initiatedBy === 'admin', - }, - MONEY_JOB_RETRY, - ) - .catch((err: unknown) => - logger.error({ err, userId }, 'promo bonus forfeit enqueue failed'), - ); + await jobs.enqueue( + FORFEIT_QUEUE, + { + userId, + reason, + actorId: initiatedBy === 'system' ? null : actorId, + actorIsAdmin: initiatedBy === 'admin', + }, + MONEY_JOB_RETRY, + ); }; ctx.events.on( @@ -286,29 +279,21 @@ export default { }); // A confirmed deposit is what turns an opt-in into a bonus. - ctx.events.on('wallet.deposit.completed', (payload: unknown) => { + ctx.events.on('wallet.deposit.completed', async (payload: unknown) => { const parsed = domainEventSchemas['wallet.deposit.completed'].safeParse(payload); if (!parsed.success) { logger.error('promo offer deposit skipped - event payload failed validation'); return; } if (!jobs) { - logger.error( - { userId: parsed.data.userId }, - 'promo offer deposit dropped - job queue not bound yet', - ); - return; + throw new Error('promo offer deposit - job queue not bound yet'); } const { userId, amount, currency, transactionId } = parsed.data; - void jobs - .enqueue( - DEPOSIT_QUEUE, - { userId, amount, currency, transactionId }, - { idempotencyKey: `promo-offer-deposit:${transactionId}`, ...MONEY_JOB_RETRY }, - ) - .catch((err: unknown) => - logger.error({ err, userId }, 'promo offer deposit enqueue failed'), - ); + await jobs.enqueue( + DEPOSIT_QUEUE, + { userId, amount, currency, transactionId }, + { idempotencyKey: `promo-offer-deposit:${transactionId}`, ...MONEY_JOB_RETRY }, + ); }); ctx.routers.add('promo-bonus', (c) => { diff --git a/packages/core/src/wallet/plugin.ts b/packages/core/src/wallet/plugin.ts index f7b0eae67..da79be05d 100644 --- a/packages/core/src/wallet/plugin.ts +++ b/packages/core/src/wallet/plugin.ts @@ -128,14 +128,19 @@ export default { if (!parsed.success || !envelope) { return; } - publishBalanceChanged(parsed.data.userId, parsed.data.currency, 'deposit', envelope.eventId); + return publishBalanceChanged( + parsed.data.userId, + parsed.data.currency, + 'deposit', + envelope.eventId, + ); }); ctx.events.on('wallet.withdrawal.requested', (payload, envelope) => { const parsed = domainEventSchemas['wallet.withdrawal.requested'].safeParse(payload); if (!parsed.success || !envelope) { return; } - publishBalanceChanged( + return publishBalanceChanged( parsed.data.userId, parsed.data.currency, 'withdrawal', @@ -147,7 +152,7 @@ export default { if (!parsed.success || !envelope) { return; } - publishBalanceChanged( + return publishBalanceChanged( parsed.data.userId, parsed.data.currency, 'withdrawal', @@ -159,7 +164,7 @@ export default { if (!parsed.success || !envelope) { return; } - publishBalanceChanged( + return publishBalanceChanged( parsed.data.userId, parsed.data.currency, 'withdrawal', @@ -173,15 +178,22 @@ export default { if (!parsed.success || !envelope) { return; } - publishBalanceChanged(parsed.data.userId, parsed.data.fromCurrency, 'swap', envelope.eventId); - publishBalanceChanged(parsed.data.userId, parsed.data.toCurrency, 'swap', envelope.eventId); + return Promise.all([ + publishBalanceChanged( + parsed.data.userId, + parsed.data.fromCurrency, + 'swap', + envelope.eventId, + ), + publishBalanceChanged(parsed.data.userId, parsed.data.toCurrency, 'swap', envelope.eventId), + ]).then(() => undefined); }); ctx.events.on('wallet.manual_adjustment.created', (payload, envelope) => { const parsed = domainEventSchemas['wallet.manual_adjustment.created'].safeParse(payload); if (!parsed.success || !envelope) { return; } - publishBalanceChanged( + return publishBalanceChanged( parsed.data.userId, parsed.data.currency, 'adjustment', @@ -193,7 +205,12 @@ export default { if (!parsed.success || !envelope) { return; } - publishBalanceChanged(parsed.data.userId, parsed.data.currency, 'gameplay', envelope.eventId); + return publishBalanceChanged( + parsed.data.userId, + parsed.data.currency, + 'gameplay', + envelope.eventId, + ); }); // One memoized instance backs the cron worker and the router factory below - lazily From 60f8887628ca19fccdedb311aff179cc2c2e780c Mon Sep 17 00:00:00 2001 From: Volodymyr Zakhovaiko Date: Tue, 13 Oct 2026 13:00:45 +0200 Subject: [PATCH 06/13] fix(core): resolve event player ids on the caller's transaction Withdrawal approve/reject and KYC events looked up the player id on a second pool connection while holding row locks. The identity reader now accepts the caller's tx, and the cooling-off sweep resolves player ids before opening its transaction. --- .../src/compliance/service/kyc.service.ts | 6 ++--- .../core/src/compliance/service/rg.service.ts | 25 +++++++++++-------- .../src/contracts/adapters/identity-reader.ts | 8 ++++-- .../adapters/identity-reader.service.ts | 7 +++--- .../core/src/wallet/service/wallet.service.ts | 4 +-- 5 files changed, 30 insertions(+), 20 deletions(-) diff --git a/packages/core/src/compliance/service/kyc.service.ts b/packages/core/src/compliance/service/kyc.service.ts index 6643809ba..53d05c0e1 100644 --- a/packages/core/src/compliance/service/kyc.service.ts +++ b/packages/core/src/compliance/service/kyc.service.ts @@ -269,7 +269,7 @@ export class KycVerificationService { playerId: params.playerTransition?.playerId ?? params.playerId ?? - (await this.identityReader.getPlayerIdByUserIdSafe(params.userId)), + (await this.identityReader.getPlayerIdByUserId(params.userId, trx)), actorId: params.actorId, status: params.status, previousStatus: @@ -385,7 +385,7 @@ export class KycVerificationService { userId, playerId: playerTransition?.playerId ?? - (await this.identityReader.getPlayerIdByUserIdSafe(userId)), + (await this.identityReader.getPlayerIdByUserId(userId, trx)), referenceId: result.referenceId, provider: this.provider, tier: input.tier, @@ -567,7 +567,7 @@ export class KycVerificationService { ) { await this.events.emitInTransaction(trx, 'compliance.kyc.high_risk_signal_detected', { userId: primaryOutcome.row.userId, - playerId: await this.identityReader.getPlayerIdByUserIdSafe(primaryOutcome.row.userId), + playerId: await this.identityReader.getPlayerIdByUserId(primaryOutcome.row.userId, trx), referenceId, tier: primaryOutcome.row.tier, vpnOrTorDetected: opts.riskSignals.vpnOrTorDetected, diff --git a/packages/core/src/compliance/service/rg.service.ts b/packages/core/src/compliance/service/rg.service.ts index e0b6fcaf3..61cad4322 100644 --- a/packages/core/src/compliance/service/rg.service.ts +++ b/packages/core/src/compliance/service/rg.service.ts @@ -793,25 +793,30 @@ export class RgService { // recompute enforcement for the affected players. Called by the rg-monitor sweep. async expireLapsedCoolingOffs(): Promise { const now = new Date(); + const lapsedFilter = and( + eq(rgExclusion.kind, 'cooling_off'), + eq(rgExclusion.status, 'active'), + lte(rgExclusion.expiresAt, now), + ); + // Enrichment is resolved before the tx so the batched lookup never holds the tx open on a + // second pool connection; a row that lapses in between just goes out with a null playerId. + const candidates = await this.drizzle.db + .selectDistinct({ userId: rgExclusion.userId }) + .from(rgExclusion) + .where(lapsedFilter); + const playerIds = await this.identityReader.getPlayerIdsByUserIdsSafe( + candidates.map((r) => r.userId), + ); const lapsed = await this.drizzle.db.transaction(async (tx) => { const expired = await tx .update(rgExclusion) .set({ status: 'expired' }) - .where( - and( - eq(rgExclusion.kind, 'cooling_off'), - eq(rgExclusion.status, 'active'), - lte(rgExclusion.expiresAt, now), - ), - ) + .where(lapsedFilter) .returning({ id: rgExclusion.id, userId: rgExclusion.userId, expiresAt: rgExclusion.expiresAt, }); - const playerIds = await this.identityReader.getPlayerIdsByUserIdsSafe( - expired.map((r) => r.userId), - ); for (const row of expired) { await this.events.emitInTransaction(tx, 'rg.cooling_off.expired', { userId: row.userId, diff --git a/packages/core/src/contracts/adapters/identity-reader.ts b/packages/core/src/contracts/adapters/identity-reader.ts index e32ddb874..9f8c45b99 100644 --- a/packages/core/src/contracts/adapters/identity-reader.ts +++ b/packages/core/src/contracts/adapters/identity-reader.ts @@ -7,8 +7,12 @@ export type IdentityReader = { getLastLoginAt(userId: User['id']): Promise; /** Returns player user ids whose most recent session predates sinceDate. Used for the daily inactive batch sweep. */ getPlayerIdsInactiveSince(sinceDate: Date): Promise; - /** Resolves the player profile id for a given auth user id, or null when no profile exists yet. */ - getPlayerIdByUserId(userId: User['id']): Promise; + /** + * Resolves the player profile id for a given auth user id, or null when no profile exists yet. + * Pass the caller's `tx` when enriching an event inside a transaction: the lookup then runs on + * the same connection instead of holding the tx open while waiting on a second pool connection. + */ + getPlayerIdByUserId(userId: User['id'], tx?: unknown): Promise; /** Best-effort variant for optional event enrichment; lookup failures resolve to null. */ getPlayerIdByUserIdSafe(userId: User['id']): Promise; /** Batched, best-effort variant of {@link getPlayerIdByUserIdSafe} for enriching events across many users in one round trip. */ diff --git a/packages/core/src/pam/identity/adapters/identity-reader.service.ts b/packages/core/src/pam/identity/adapters/identity-reader.service.ts index 3766d5f30..b10ac2282 100644 --- a/packages/core/src/pam/identity/adapters/identity-reader.service.ts +++ b/packages/core/src/pam/identity/adapters/identity-reader.service.ts @@ -1,4 +1,4 @@ -import { createLogger, DrizzleService } from '@openora/core/server'; +import { createLogger, DrizzleService, type DrizzleTx } from '@openora/core/server'; import type { IdentityReader, KycStatus, Player, User } from '@openora/core/contracts'; import { and, eq, exists, inArray, isNull, lt, max, ne, or, SQL } from 'drizzle-orm'; import { alias } from 'drizzle-orm/pg-core'; @@ -46,8 +46,9 @@ export class IdentityReaderService implements IdentityReader { return rows.map((r) => r.userId); } - async getPlayerIdByUserId(userId: User['id']): Promise { - const [row] = await this.drizzle.db + async getPlayerIdByUserId(userId: User['id'], tx?: unknown): Promise { + const db = tx === undefined ? this.drizzle.db : (tx as DrizzleTx); + const [row] = await db .select({ id: player.id }) .from(player) .where(eq(player.userId, userId)) diff --git a/packages/core/src/wallet/service/wallet.service.ts b/packages/core/src/wallet/service/wallet.service.ts index f5a6a7017..cc90b96ea 100644 --- a/packages/core/src/wallet/service/wallet.service.ts +++ b/packages/core/src/wallet/service/wallet.service.ts @@ -2016,7 +2016,7 @@ export class WalletService { const userId = await this.userIdForWallet(flipped.walletId, txn); await this.events.emitInTransaction(txn, 'wallet.withdrawal.approved', { userId, - playerId: await this.identityReader.getPlayerIdByUserIdSafe(userId), + playerId: await this.identityReader.getPlayerIdByUserId(userId, txn), amount: flipped.amount, currency: flipped.currency, transactionId: flipped.id, @@ -3004,7 +3004,7 @@ export class WalletService { const userId = await this.userIdForWallet(updated.walletId, txn); await this.events.emitInTransaction(txn, 'wallet.withdrawal.rejected', { userId, - playerId: await this.identityReader.getPlayerIdByUserIdSafe(userId), + playerId: await this.identityReader.getPlayerIdByUserId(userId, txn), amount: updated.amount, currency: updated.currency, transactionId: updated.id, From 99480c2ded3541cf767713006032cd4ba15b789e Mon Sep 17 00:00:00 2001 From: Volodymyr Zakhovaiko Date: Tue, 13 Oct 2026 13:05:03 +0200 Subject: [PATCH 07/13] fix(core): drain outbox and pending backlogs at full speed The relay keeps claiming while batches come back full and marks a batch published in one update; the lease compares against the DB clock. Reclaim walks the whole pending list per interval instead of one page. processed_events outlives the stream retention by a day so a late redelivery still dedupes. --- .../db/outbox/__tests__/relay.int.test.ts | 29 ++++++++++- packages/core/src/server/db/outbox/relay.ts | 50 +++++++++++++----- .../redis-streams-broker.int.test.ts | 22 ++++++++ .../src/server/kernel/redis-streams-broker.ts | 51 ++++++++++--------- 4 files changed, 113 insertions(+), 39 deletions(-) diff --git a/packages/core/src/server/db/outbox/__tests__/relay.int.test.ts b/packages/core/src/server/db/outbox/__tests__/relay.int.test.ts index ebdeeb4ab..3122277e7 100644 --- a/packages/core/src/server/db/outbox/__tests__/relay.int.test.ts +++ b/packages/core/src/server/db/outbox/__tests__/relay.int.test.ts @@ -188,7 +188,13 @@ describe('OutboxRelay.drainOnce (real PG)', () => { const recentPublished = await seedRow({ publishedAt: new Date() }); const oldPending = await seedRow({ occurredAt: longAgo }); await db.drizzle.db.insert(processedEvents).values([ - { eventId: randomUUID(), consumer: 'old', processedAt: longAgo }, + { + eventId: randomUUID(), + consumer: 'old', + processedAt: new Date(Date.now() - EVENT_RETENTION_MS - 25 * 60 * 60_000), + }, + // Kept a day past the stream retention: its entry may still be pending and redelivered. + { eventId: randomUUID(), consumer: 'just-past-retention', processedAt: longAgo }, { eventId: randomUUID(), consumer: 'recent' }, ]); const relay = new OutboxRelay(db.drizzle.db, brokerThat()); @@ -200,7 +206,7 @@ describe('OutboxRelay.drainOnce (real PG)', () => { expect(remaining.has(recentPublished.eventId)).toBe(true); expect(remaining.get(oldPending.eventId)?.publishedAt).not.toBeNull(); const consumers = await db.drizzle.db.select().from(processedEvents); - expect(consumers.map((r) => r.consumer)).toEqual(['recent']); + expect(consumers.map((r) => r.consumer).sort()).toEqual(['just-past-retention', 'recent']); }); }); @@ -227,6 +233,25 @@ describe('OutboxRelay poll loop (real PG)', () => { } }); + it('drains a backlog larger than one batch within a single tick', async () => { + const rows = await Promise.all(Array.from({ length: 5 }, () => seedRow())); + const relay = new OutboxRelay(db.drizzle.db, brokerThat(), { intervalMs: 1000, batchSize: 2 }); + + relay.start(); + try { + // One tick fires at ~1s; the next would be ~2s. One batch per tick would leave rows pending. + await vi.waitFor( + async () => { + const byId = await rowsById(); + expect(rows.every((r) => byId.get(r.eventId)?.publishedAt)).toBe(true); + }, + { timeout: 1800 }, + ); + } finally { + await relay.stop(); + } + }); + it('reports a failing drain to onError and keeps polling', async () => { await seedRow(); const errors: unknown[] = []; diff --git a/packages/core/src/server/db/outbox/relay.ts b/packages/core/src/server/db/outbox/relay.ts index 8eb754899..15fd526fc 100644 --- a/packages/core/src/server/db/outbox/relay.ts +++ b/packages/core/src/server/db/outbox/relay.ts @@ -7,6 +7,9 @@ import type { DrizzleDb } from '../drizzle.js'; // A claimed row whose relay died mid-publish becomes claimable again after this. const CLAIM_LEASE_MS = 30_000; const PRUNE_INTERVAL_MS = 60 * 60_000; +// Dedupe records outlive the stream entries they guard: MINID trimming is approximate, so an +// entry can still be pending (and redeliverable) a while past EVENT_RETENTION_MS. +const PROCESSED_EVENTS_RETENTION_MS = EVENT_RETENTION_MS + 24 * 60 * 60_000; export type OutboxRelayOptions = { intervalMs?: number; // poll cadence; default 200 @@ -40,8 +43,8 @@ function toEnvelope(row: EventOutboxRow): EventEnvelope { * the lease expires, then another relay republishes them - at-least-once delivery, * deduped downstream by `processed_events`. * - * The same loop prunes published outbox rows and `processed_events` rows older - * than `EVENT_RETENTION_MS`, at most once an hour per replica. + * The same loop prunes published outbox rows older than `EVENT_RETENTION_MS` (and + * `processed_events` a day later), at most once an hour per replica. * * ponytail: polls every 200ms per replica (an indexed, usually-empty claim). Swap * the poll for LISTEN/NOTIFY if the idle query load or the publish latency matters. @@ -61,7 +64,7 @@ export class OutboxRelay { if (this.timer) { return; } - this.timer = setInterval(() => void this.drainOnce(), this.opts.intervalMs ?? 200); + this.timer = setInterval(() => void this.drainUntilIdle(), this.opts.intervalMs ?? 200); this.timer.unref?.(); } @@ -75,6 +78,16 @@ export class OutboxRelay { } } + // Keeps draining while batches come back full, so a backlog clears at publish speed + // instead of one batch per tick. A no-op while another drain is in flight. + private async drainUntilIdle(): Promise { + const batchSize = this.opts.batchSize ?? 100; + let published: number; + do { + published = await this.drainOnce(); + } while (this.timer && published === batchSize); + } + // Publish one claimed batch. Returns how many rows were published. Reentrant-safe: // a second call while a drain is in flight is a no-op. Never throws - every failure // goes to `onError`. @@ -86,13 +99,19 @@ export class OutboxRelay { try { await this.pruneIfDue(); const batch = await this.claimBatch(); - let published = 0; + const published: EventOutboxRow['eventId'][] = []; for (const row of batch) { if (await this.publishRow(row)) { - published += 1; + published.push(row.eventId); } } - return published; + if (published.length > 0) { + await this.db + .update(eventOutbox) + .set({ publishedAt: sql`now()` }) + .where(inArray(eventOutbox.eventId, published)); + } + return published.length; } catch (err) { this.opts.onError?.(err); return 0; @@ -102,7 +121,9 @@ export class OutboxRelay { } private async claimBatch(): Promise { - const leaseExpiredBefore = new Date(Date.now() - CLAIM_LEASE_MS); + // DB clock on both sides of the lease (claimed_at is stamped with now()), so replica + // clock skew cannot steal or strand a live claim. + const leaseExpiredBefore = sql`now() - ${CLAIM_LEASE_MS} * interval '1 millisecond'`; const claimable = this.db .select({ eventId: eventOutbox.eventId }) .from(eventOutbox) @@ -126,10 +147,6 @@ export class OutboxRelay { private async publishRow(row: EventOutboxRow): Promise { try { await this.broker.publish(toEnvelope(row)); - await this.db - .update(eventOutbox) - .set({ publishedAt: new Date() }) - .where(eq(eventOutbox.eventId, row.eventId)); return true; } catch (err) { this.opts.onError?.(err); @@ -147,10 +164,15 @@ export class OutboxRelay { return; } this.lastPruneAt = Date.now(); - const cutoff = new Date(Date.now() - EVENT_RETENTION_MS); try { - await this.db.delete(eventOutbox).where(lt(eventOutbox.publishedAt, cutoff)); - await this.db.delete(processedEvents).where(lt(processedEvents.processedAt, cutoff)); + await this.db + .delete(eventOutbox) + .where(lt(eventOutbox.publishedAt, new Date(Date.now() - EVENT_RETENTION_MS))); + await this.db + .delete(processedEvents) + .where( + lt(processedEvents.processedAt, new Date(Date.now() - PROCESSED_EVENTS_RETENTION_MS)), + ); } catch (err) { this.opts.onError?.(err); } diff --git a/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts b/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts index 8e73d4e4b..82ac71ada 100644 --- a/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts +++ b/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts @@ -292,6 +292,28 @@ describe('RedisStreamsBroker', () => { expect(await redis.client.exists(`oss:dlq:${topic}`)).toBe(0); }); + it('reclaims the whole pending backlog in one pass, not one page', async () => { + const broker = makeBroker('svc', { intervalMs: 1000, minIdleMs: 10 }); + const topic = uniqueTopic(); + const seen = new Set(); + let handled = 0; + broker.subscribe(topic, (envelope) => { + if (!seen.has(envelope.eventId)) { + seen.add(envelope.eventId); + throw new Error('transient'); + } + handled += 1; + }); + await waitForConsumerGroup(redis.client, { stream: streamOf(topic), group: 'svc' }); + + // 25 entries span three XAUTOCLAIM pages (COUNT 10); the next pass would be ~1s later. + for (let i = 0; i < 25; i += 1) { + await broker.publish(makeEnvelope(topic)); + } + + await vi.waitFor(() => expect(handled).toBe(25), { timeout: 1800 }); + }); + it(`dead-letters an entry still failing after ${MAX_DELIVERIES} deliveries and ACKs it`, async () => { const broker = makeBroker('svc', FAST_RECLAIM); const topic = uniqueTopic(); diff --git a/packages/core/src/server/kernel/redis-streams-broker.ts b/packages/core/src/server/kernel/redis-streams-broker.ts index db20a107e..eff9f7ef0 100644 --- a/packages/core/src/server/kernel/redis-streams-broker.ts +++ b/packages/core/src/server/kernel/redis-streams-broker.ts @@ -259,8 +259,9 @@ export class RedisStreamsBroker implements MessageBrokerAdapter { return { handlers, stop, done }; } - // Returns the XAUTOCLAIM cursor to resume from, so a backlog of pending entries is - // walked across intervals instead of re-scanning the first page every time. + // Walks the whole pending list each interval (XAUTOCLAIM pages until cursor 0-0), so a + // backlog drains at read speed rather than one page per interval. Returns the cursor to + // resume from if a page fails. private async reclaimPending( reader: RedisClient, key: string, @@ -268,31 +269,35 @@ export class RedisStreamsBroker implements MessageBrokerAdapter { handlers: Set, cursor: string, ): Promise { + let next = cursor; try { - const claimed = (await reader.xAutoClaim( - key, - group, - this.consumerId, - this.claimMinIdleMs, - cursor, - { COUNT: READ_COUNT }, - )) as StreamClaimReply; - if (claimed.deletedMessages.length > 0) { - this.logger.error( - { key, group, ids: claimed.deletedMessages }, - 'pending stream entries were trimmed before they were processed', - ); - } - for (const message of claimed.messages) { - if (message) { - const deliveries = await this.deliveryCount(reader, key, group, message.id); - await this.dispatch(reader, key, group, message, handlers, deliveries); + do { + const claimed = (await reader.xAutoClaim( + key, + group, + this.consumerId, + this.claimMinIdleMs, + next, + { COUNT: READ_COUNT }, + )) as StreamClaimReply; + if (claimed.deletedMessages.length > 0) { + this.logger.error( + { key, group, ids: claimed.deletedMessages }, + 'pending stream entries were trimmed before they were processed', + ); } - } - return claimed.nextId; + for (const message of claimed.messages) { + if (message) { + const deliveries = await this.deliveryCount(reader, key, group, message.id); + await this.dispatch(reader, key, group, message, handlers, deliveries); + } + } + next = claimed.nextId; + } while (next !== '0-0'); + return next; } catch (err) { this.logger.error({ err, key, group }, 'stream reclaim failed'); - return cursor; + return next; } } From 466dc43409526b0301f9dca0f93d0c48ee053d70 Mon Sep 17 00:00:00 2001 From: Volodymyr Zakhovaiko Date: Tue, 13 Oct 2026 13:06:50 +0200 Subject: [PATCH 08/13] fix(core): announce a round only after launch and keep finished reconciliation runs gaming.round.started is written after the provider launches the game, as before the outbox change, so a failed launch announces nothing. Reconciliation records its audit entry and threshold alert after the run is marked finished, so a failed alert write no longer rewrites a completed run as failed. --- .../__tests__/gaming.service.int.test.ts | 3 ++ .../casino/gaming/service/gaming.service.ts | 16 +++--- .../wallet/service/reconciliation.service.ts | 54 ++++++++++--------- 3 files changed, 41 insertions(+), 32 deletions(-) diff --git a/packages/core/src/casino/gaming/__tests__/gaming.service.int.test.ts b/packages/core/src/casino/gaming/__tests__/gaming.service.int.test.ts index 859e83e93..b22dbc1b8 100644 --- a/packages/core/src/casino/gaming/__tests__/gaming.service.int.test.ts +++ b/packages/core/src/casino/gaming/__tests__/gaming.service.int.test.ts @@ -1094,6 +1094,9 @@ describe('GamingService.startRound bonus completion (real PG)', () => { }, ); expect(launchGame).toHaveBeenCalledOnce(); + expect( + events.emitInTransaction.mock.calls.filter(([, topic]) => topic === 'gaming.round.started'), + ).toEqual([]); }); it('emits nothing when the bet completed no bonus', async () => { diff --git a/packages/core/src/casino/gaming/service/gaming.service.ts b/packages/core/src/casino/gaming/service/gaming.service.ts index 6ac3ea008..de9ccc2d7 100644 --- a/packages/core/src/casino/gaming/service/gaming.service.ts +++ b/packages/core/src/casino/gaming/service/gaming.service.ts @@ -689,18 +689,20 @@ export class GamingService { }); } } - await this.events.emitInTransaction(tx, 'gaming.round.started', { - roundId: insertedRound.id, - gameId, - userId, - playerId, - currency, - }); return insertedRound; }); const { launchUrl, token } = await this.provider.launchGame(gameId, userId, currency); + // Only a round the provider actually launched is announced; a failed launch never reaches here. + await this.events.emitInTransaction(this.drizzle.db, 'gaming.round.started', { + roundId: round.id, + gameId, + userId, + playerId, + currency, + }); + return { roundId: round.id, launchUrl, token }; } diff --git a/packages/core/src/wallet/service/reconciliation.service.ts b/packages/core/src/wallet/service/reconciliation.service.ts index 503b6f836..7787b9436 100644 --- a/packages/core/src/wallet/service/reconciliation.service.ts +++ b/packages/core/src/wallet/service/reconciliation.service.ts @@ -441,8 +441,10 @@ export class ReconciliationService { unreconciledHours: 0, }; + const cfg = this.platformConfig?.wallet?.reconciliation; + const status = cfg ? 'completed' : 'skipped'; + let openFindings: number; try { - const cfg = this.platformConfig?.wallet?.reconciliation; if (cfg) { const until = new Date(); const { since, unreconciledHours } = await this.resolveWindow(until, cfg.lookbackHours); @@ -482,31 +484,8 @@ export class ReconciliationService { ); } - const openFindings = await this.countOpenFindings(); - const status = cfg ? 'completed' : 'skipped'; + openFindings = await this.countOpenFindings(); await this.finishRun(jobRunId, status, { ...counts, openFindings }); - await this.audit.record({ - actorType: 'system', - action: cfg ? 'wallet.reconciliation_run.completed' : 'wallet.reconciliation_run.skipped', - resourceType: 'wallet_job_run', - resourceId: runId, - after: { runId, ...counts, openFindings }, - }); - - if (cfg && openFindings > cfg.alertThreshold) { - logger.error( - { runId, openFindings, threshold: cfg.alertThreshold }, - 'wallet reconciliation: open findings exceed threshold', - ); - // No surrounding transaction to join: the outbox write alone makes the alert durable. - await this.events.emitInTransaction(this.drizzle.db, 'wallet.reconciliation.alert', { - runId, - openFindings, - threshold: cfg.alertThreshold, - }); - } - - return { runId }; } catch (err) { await this.finishRun(jobRunId, 'failed', { ...counts, @@ -521,6 +500,31 @@ export class ReconciliationService { }); throw err; } + + // Past the try: the run is already recorded finished, so a failing audit or alert write + // surfaces as an error without rewriting a completed run as failed. + await this.audit.record({ + actorType: 'system', + action: cfg ? 'wallet.reconciliation_run.completed' : 'wallet.reconciliation_run.skipped', + resourceType: 'wallet_job_run', + resourceId: runId, + after: { runId, ...counts, openFindings }, + }); + + if (cfg && openFindings > cfg.alertThreshold) { + logger.error( + { runId, openFindings, threshold: cfg.alertThreshold }, + 'wallet reconciliation: open findings exceed threshold', + ); + // No surrounding transaction to join: the outbox write alone makes the alert durable. + await this.events.emitInTransaction(this.drizzle.db, 'wallet.reconciliation.alert', { + runId, + openFindings, + threshold: cfg.alertThreshold, + }); + } + + return { runId }; } private async reconcileProvider( From a0105ce9ac6914f6dbb21c81eb17334ad82dd0b4 Mon Sep 17 00:00:00 2001 From: Volodymyr Zakhovaiko Date: Tue, 20 Oct 2026 10:51:42 +0200 Subject: [PATCH 09/13] fix(core): record the round start with the bet debit so nothing fails after launch --- .../__tests__/gaming.service.int.test.ts | 53 +++++++++++++++++-- .../casino/gaming/service/gaming.service.ts | 19 +++---- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/packages/core/src/casino/gaming/__tests__/gaming.service.int.test.ts b/packages/core/src/casino/gaming/__tests__/gaming.service.int.test.ts index 5ec80747c..40c546c4a 100644 --- a/packages/core/src/casino/gaming/__tests__/gaming.service.int.test.ts +++ b/packages/core/src/casino/gaming/__tests__/gaming.service.int.test.ts @@ -539,6 +539,56 @@ describe('GamingService.startRound (real PG)', () => { expect(Number(rounds[0]?.betAmount)).toBe(10); }); + it('records the round start in the debit transaction, before the launch', async () => { + const created = await seedGame({ name: 'Aces' }); + const walletCommands = makeWalletCommands({ + ok: true, + moved: false, + newBalance: '90', + currency: 'USD', + }); + const events = makeEventBus(); + const launchGame = vi.fn().mockRejectedValue(new Error('provider down')); + const svc = makeService({ + provider: mock({ launchGame, endRound: vi.fn() }), + walletCommands, + events, + }); + const userId = '00000000-0000-0000-0000-000000000303'; + + await expect(startRound(svc, userId, created.id, 'USD', '10')).rejects.toThrow('provider down'); + + const rounds = await db.drizzle.db.select().from(gameRound); + const debitTx = vi.mocked(walletCommands.debit).mock.calls[0]?.[0]; + expect(rounds).toHaveLength(1); + expect(walletCommands.debit).toHaveBeenCalledTimes(1); + expect(events.emitInTransaction).toHaveBeenCalledWith(debitTx, 'gaming.round.started', { + roundId: rounds[0]?.id, + gameId: created.id, + userId, + playerId: null, + currency: 'USD', + }); + }); + + it('rolls the debit and round back and never launches when the round-start write fails', async () => { + const created = await seedGame({ name: 'Aces' }); + const events = makeEventBus(); + events.emitInTransaction.mockRejectedValue(new Error('outbox down')); + const launchGame = vi.fn(); + const svc = makeService({ + provider: mock({ launchGame, endRound: vi.fn() }), + events, + }); + + await expect( + startRound(svc, '00000000-0000-0000-0000-000000000304', created.id, 'USD', '10'), + ).rejects.toThrow('outbox down'); + + expect(launchGame).not.toHaveBeenCalled(); + expect(await db.drizzle.db.select().from(gameRound)).toHaveLength(0); + }); + it('throws InsufficientBalanceError, persists no round, and never launches the game when the debit fails', async () => { const created = await seedGame({ id: '00000000-0000-0000-0000-0000000000a2', name: 'Aces' }); const walletCommands = makeWalletCommands({ ok: false, available: '2' }); @@ -1183,9 +1233,6 @@ describe('GamingService.startRound bonus completion (real PG)', () => { }, ); expect(launchGame).toHaveBeenCalledOnce(); - expect( - events.emitInTransaction.mock.calls.filter(([, topic]) => topic === 'gaming.round.started'), - ).toEqual([]); }); it('emits nothing when the bet completed no bonus', async () => { diff --git a/packages/core/src/casino/gaming/service/gaming.service.ts b/packages/core/src/casino/gaming/service/gaming.service.ts index 10f8827db..c23663230 100644 --- a/packages/core/src/casino/gaming/service/gaming.service.ts +++ b/packages/core/src/casino/gaming/service/gaming.service.ts @@ -690,20 +690,21 @@ export class GamingService { }); } } + // With the debit, not after the launch: nothing may fail once the provider session + // exists, or a retry mints a new round and debits again. A launch that fails after + // this commit leaves the round active and debited, which is what this event records. + await this.events.emitInTransaction(tx, 'gaming.round.started', { + roundId, + gameId, + userId, + playerId, + currency, + }); return insertedRound; }); const { launchUrl, token } = await this.provider.launchGame(gameId, userId, currency); - // Only a round the provider actually launched is announced; a failed launch never reaches here. - await this.events.emitInTransaction(this.drizzle.db, 'gaming.round.started', { - roundId: round.id, - gameId, - userId, - playerId, - currency, - }); - return { roundId: round.id, launchUrl, token }; } From b00a34b3a31b1dcb7295bc51a38dfe8f4114e358 Mon Sep 17 00:00:00 2001 From: Volodymyr Zakhovaiko Date: Tue, 20 Oct 2026 10:55:36 +0200 Subject: [PATCH 10/13] fix(core): dead-letter pending events past the retention window --- .../redis-streams-broker.int.test.ts | 29 +++++++++++++++++++ .../src/server/kernel/redis-streams-broker.ts | 26 +++++++++++++++-- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts b/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts index 82ac71ada..8dbd39119 100644 --- a/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts +++ b/packages/core/src/server/kernel/__tests__/redis-streams-broker.int.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, vi, beforeAll, afterEach, afterAll } from 'vitest'; import { randomUUID } from 'node:crypto'; import { MAX_DELIVERIES, RedisStreamsBroker } from '../redis-streams-broker.js'; +import { EVENT_RETENTION_MS } from '../event-bus.js'; import { createTestRedis, waitForConsumerGroup, type TestRedis } from '@openora/core/testing'; import type { EventEnvelope } from '@openora/core/contracts'; @@ -342,6 +343,34 @@ describe('RedisStreamsBroker', () => { expect(pending.pending).toBe(0); }); + it('dead-letters a pending entry older than the retention window instead of re-running it', async () => { + const broker = makeBroker('svc', FAST_RECLAIM); + const topic = uniqueTopic(); + let attempts = 0; + broker.subscribe(topic, () => { + attempts += 1; + throw new Error('transient'); + }); + await waitForConsumerGroup(redis.client, { stream: streamOf(topic), group: 'svc' }); + + // An id from past the retention window, as a pending entry left behind by a long outage. + const staleId = `${Date.now() - EVENT_RETENTION_MS - 60_000}-0`; + await redis.client.xAdd(streamOf(topic), staleId, { + data: JSON.stringify(makeEnvelope(topic)), + }); + + await vi.waitFor(async () => expect(await redis.client.xLen(`oss:dlq:${topic}`)).toBe(1), { + timeout: 5000, + }); + expect(attempts).toBe(1); + const [dead] = await redis.client.xRange(`oss:dlq:${topic}`, '-', '+'); + expect(dead!.message).toMatchObject({ + streamId: staleId, + error: 'pending past the event retention window', + }); + expect((await redis.client.xPending(streamOf(topic), 'svc')).pending).toBe(0); + }); + it('dead-letters an entry that is not valid JSON without calling handlers', async () => { const broker = makeBroker('svc'); const topic = uniqueTopic(); diff --git a/packages/core/src/server/kernel/redis-streams-broker.ts b/packages/core/src/server/kernel/redis-streams-broker.ts index eff9f7ef0..17c0eb62b 100644 --- a/packages/core/src/server/kernel/redis-streams-broker.ts +++ b/packages/core/src/server/kernel/redis-streams-broker.ts @@ -65,6 +65,11 @@ function dlqKey(streamKeyName: string): string { return DLQ_PREFIX + streamKeyName.slice(STREAM_PREFIX.length); } +// A stream id is `-`, stamped by Redis on XADD. +function streamIdTime(id: string): number { + return Number(id.split('-')[0]); +} + function errorMessage(err: unknown): string { return err instanceof Error ? err.message : String(err); } @@ -287,10 +292,25 @@ export class RedisStreamsBroker implements MessageBrokerAdapter { ); } for (const message of claimed.messages) { - if (message) { - const deliveries = await this.deliveryCount(reader, key, group, message.id); - await this.dispatch(reader, key, group, message, handlers, deliveries); + if (!message) { + continue; + } + const deliveries = await this.deliveryCount(reader, key, group, message.id); + // MINID trimming is approximate and pending entries survive it, so one can outlive + // the processed_events rows that dedupe it. Re-running it would repeat handlers that + // already succeeded; it goes to the dead-letter stream for a human instead. + if (streamIdTime(message.id) < Date.now() - EVENT_RETENTION_MS) { + await this.deadLetter( + reader, + key, + group, + message, + deliveries, + new Error('pending past the event retention window'), + ); + continue; } + await this.dispatch(reader, key, group, message, handlers, deliveries); } next = claimed.nextId; } while (next !== '0-0'); From 98bfd43d1329b0f5b95618230845b2a3eb3e034a Mon Sep 17 00:00:00 2001 From: Volodymyr Zakhovaiko Date: Tue, 20 Oct 2026 10:55:38 +0200 Subject: [PATCH 11/13] fix(core): hand back unpublished outbox rows before the claim lease expires --- .../db/outbox/__tests__/relay.int.test.ts | 24 +++++++++++++++++++ packages/core/src/server/db/outbox/relay.ts | 20 +++++++++++++--- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/core/src/server/db/outbox/__tests__/relay.int.test.ts b/packages/core/src/server/db/outbox/__tests__/relay.int.test.ts index 3122277e7..a059d102e 100644 --- a/packages/core/src/server/db/outbox/__tests__/relay.int.test.ts +++ b/packages/core/src/server/db/outbox/__tests__/relay.int.test.ts @@ -157,6 +157,30 @@ describe('OutboxRelay.drainOnce (real PG)', () => { expect((await rowsById()).get(fresh.eventId)?.publishedAt).toBeNull(); }); + it('hands back the rest of a batch once half its lease is spent publishing', async () => { + const first = await seedRow({ occurredAt: new Date('2026-01-01T00:00:00.000Z') }); + const second = await seedRow({ occurredAt: new Date('2026-01-01T00:00:01.000Z') }); + let clock = Date.now(); + const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); + // The first publish takes 20s, past the 15s budget of a 30s lease. + const broker = brokerThat(() => { + clock += 20_000; + }); + const relay = new OutboxRelay(db.drizzle.db, broker); + + try { + await expect(relay.drainOnce()).resolves.toBe(1); + } finally { + now.mockRestore(); + } + + expect(broker.publish).toHaveBeenCalledTimes(1); + const rows = await rowsById(); + expect(rows.get(first.eventId)?.publishedAt).not.toBeNull(); + expect(rows.get(second.eventId)?.publishedAt).toBeNull(); + expect(rows.get(second.eventId)?.claimedAt).toBeNull(); + }); + it('publishes each row exactly once across two concurrent relays', async () => { const seeded = await Promise.all(Array.from({ length: 50 }, () => seedRow())); const published: string[] = []; diff --git a/packages/core/src/server/db/outbox/relay.ts b/packages/core/src/server/db/outbox/relay.ts index 15fd526fc..7efafc7a3 100644 --- a/packages/core/src/server/db/outbox/relay.ts +++ b/packages/core/src/server/db/outbox/relay.ts @@ -7,9 +7,13 @@ import type { DrizzleDb } from '../drizzle.js'; // A claimed row whose relay died mid-publish becomes claimable again after this. const CLAIM_LEASE_MS = 30_000; const PRUNE_INTERVAL_MS = 60 * 60_000; -// Dedupe records outlive the stream entries they guard: MINID trimming is approximate, so an -// entry can still be pending (and redeliverable) a while past EVENT_RETENTION_MS. +// Dedupe records outlive the stream entries they guard: the broker only redelivers an entry +// younger than EVENT_RETENTION_MS (an older pending one is dead-lettered), and a record is +// written after its entry, so a day's margin keeps every redeliverable entry deduped. const PROCESSED_EVENTS_RETENTION_MS = EVENT_RETENTION_MS + 24 * 60 * 60_000; +// A batch stops publishing once this much of its lease is spent and hands the rest back, so +// another relay can't reclaim and republish rows this one is still working through. +const PUBLISH_BUDGET_MS = CLAIM_LEASE_MS / 2; export type OutboxRelayOptions = { intervalMs?: number; // poll cadence; default 200 @@ -99,9 +103,13 @@ export class OutboxRelay { try { await this.pruneIfDue(); const batch = await this.claimBatch(); + const deadline = Date.now() + PUBLISH_BUDGET_MS; const published: EventOutboxRow['eventId'][] = []; + const unsent: EventOutboxRow['eventId'][] = []; for (const row of batch) { - if (await this.publishRow(row)) { + if (Date.now() > deadline) { + unsent.push(row.eventId); + } else if (await this.publishRow(row)) { published.push(row.eventId); } } @@ -111,6 +119,12 @@ export class OutboxRelay { .set({ publishedAt: sql`now()` }) .where(inArray(eventOutbox.eventId, published)); } + if (unsent.length > 0) { + await this.db + .update(eventOutbox) + .set({ claimedAt: null }) + .where(inArray(eventOutbox.eventId, unsent)); + } return published.length; } catch (err) { this.opts.onError?.(err); From e322f2c4785d35e9788cbebf05e649ee79cabb60 Mon Sep 17 00:00:00 2001 From: Volodymyr Zakhovaiko Date: Tue, 20 Oct 2026 10:57:34 +0200 Subject: [PATCH 12/13] fix(wallet): commit a reconciliation run's completion with its audit entry and alert --- .../reconciliation.service.int.test.ts | 28 +++++++++++-- .../wallet/service/reconciliation.service.ts | 39 ++++++++++--------- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/packages/core/src/wallet/__tests__/reconciliation.service.int.test.ts b/packages/core/src/wallet/__tests__/reconciliation.service.int.test.ts index 41a8e4beb..5efcc20fe 100644 --- a/packages/core/src/wallet/__tests__/reconciliation.service.int.test.ts +++ b/packages/core/src/wallet/__tests__/reconciliation.service.int.test.ts @@ -721,9 +721,12 @@ describe('ReconciliationService.runCycle - audit', () => { await reconciliation.runCycle(); - const entries = (audit.record as ReturnType).mock.calls.map( - ([entry]) => entry as Record, - ); + const entries = [ + ...(audit.record as ReturnType).mock.calls.map(([entry]) => entry), + ...(audit.recordInTransaction as ReturnType).mock.calls.map( + ([, entry]) => entry, + ), + ] as Record[]; expect(entries.map((e) => e.action)).toEqual([ 'wallet.reconciliation_finding.recorded', 'wallet.reconciliation_run.completed', @@ -732,6 +735,25 @@ describe('ReconciliationService.runCycle - audit', () => { expect(payload).not.toContain('bc1qaudited'); expect(payload).not.toContain('0xaudited'); }); + + it('never leaves a run completed without its audit entry', async () => { + const { reconciliation, audit } = makeServices(listTransactionsReturning([])); + (audit.recordInTransaction as ReturnType).mockRejectedValueOnce( + new Error('audit store down'), + ); + const runId = randomUUID(); + + await expect(reconciliation.runCycle(runId)).rejects.toThrow('audit store down'); + + const runs = await db.drizzle.db + .select() + .from(walletJobRun) + .where(eq(walletJobRun.runId, runId)); + expect(runs.map((r) => r.status)).toEqual(['failed']); + expect(audit.record).toHaveBeenCalledWith( + expect.objectContaining({ action: 'wallet.reconciliation_run.failed', resourceId: runId }), + ); + }); }); describe('ReconciliationService.resolveFinding', () => { diff --git a/packages/core/src/wallet/service/reconciliation.service.ts b/packages/core/src/wallet/service/reconciliation.service.ts index 1d6a23287..fab685ae6 100644 --- a/packages/core/src/wallet/service/reconciliation.service.ts +++ b/packages/core/src/wallet/service/reconciliation.service.ts @@ -1,6 +1,7 @@ import { randomUUID } from 'node:crypto'; import { and, asc, count, desc, eq, gte, inArray, isNull, lt, sql } from 'drizzle-orm'; import { + type DrizzleDb, type DrizzleService, type DrizzleTx, type EventBus, @@ -823,7 +824,24 @@ export class ReconciliationService { } openFindings = await this.countOpenFindings(); - await this.finishRun(jobRunId, status, { ...counts, openFindings }); + // One transaction: a run is never recorded completed without its audit entry and alert. + await this.drizzle.db.transaction(async (txn) => { + await this.finishRun(jobRunId, status, { ...counts, openFindings }, txn); + await this.audit.recordInTransaction(txn, { + actorType: 'system', + action: cfg ? 'wallet.reconciliation_run.completed' : 'wallet.reconciliation_run.skipped', + resourceType: 'wallet_job_run', + resourceId: runId, + after: { runId, ...counts, openFindings }, + }); + if (cfg && openFindings > cfg.alertThreshold) { + await this.events.emitInTransaction(txn, 'wallet.reconciliation.alert', { + runId, + openFindings, + threshold: cfg.alertThreshold, + }); + } + }); } catch (err) { await this.finishRun(jobRunId, 'failed', { ...counts, @@ -839,27 +857,11 @@ export class ReconciliationService { throw err; } - // Past the try: the run is already recorded finished, so a failing audit or alert write - // surfaces as an error without rewriting a completed run as failed. - await this.audit.record({ - actorType: 'system', - action: cfg ? 'wallet.reconciliation_run.completed' : 'wallet.reconciliation_run.skipped', - resourceType: 'wallet_job_run', - resourceId: runId, - after: { runId, ...counts, openFindings }, - }); - if (cfg && openFindings > cfg.alertThreshold) { logger.error( { runId, openFindings, threshold: cfg.alertThreshold }, 'wallet reconciliation: open findings exceed threshold', ); - // No surrounding transaction to join: the outbox write alone makes the alert durable. - await this.events.emitInTransaction(this.drizzle.db, 'wallet.reconciliation.alert', { - runId, - openFindings, - threshold: cfg.alertThreshold, - }); } return { runId }; @@ -1371,8 +1373,9 @@ export class ReconciliationService { jobRunId: WalletJobRun['id'], status: WalletJobRunStatus, summary: Record, + db: DrizzleDb | DrizzleTx = this.drizzle.db, ): Promise { - await this.drizzle.db + await db .update(walletJobRun) .set({ finishedAt: new Date(), status, summary }) .where(eq(walletJobRun.id, jobRunId)); From b6dac6ec8f28dd099fbb08321c4fe2fcc30a8ca9 Mon Sep 17 00:00:00 2001 From: Volodymyr Zakhovaiko Date: Tue, 20 Oct 2026 10:58:08 +0200 Subject: [PATCH 13/13] chore(core): format the event delivery migration snapshot --- .../db/outbox/drizzle/migrations/meta/0002_snapshot.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/core/src/server/db/outbox/drizzle/migrations/meta/0002_snapshot.json b/packages/core/src/server/db/outbox/drizzle/migrations/meta/0002_snapshot.json index 934ac916e..07f9681a5 100644 --- a/packages/core/src/server/db/outbox/drizzle/migrations/meta/0002_snapshot.json +++ b/packages/core/src/server/db/outbox/drizzle/migrations/meta/0002_snapshot.json @@ -192,10 +192,7 @@ "compositePrimaryKeys": { "processed_events_event_id_consumer_pk": { "name": "processed_events_event_id_consumer_pk", - "columns": [ - "event_id", - "consumer" - ] + "columns": ["event_id", "consumer"] } }, "uniqueConstraints": {},