diff --git a/README.md b/README.md index 4583710..aec535b 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A from-scratch in-memory message broker exploring pub/sub, work queues, delivery ## What this demonstrates -Message brokers hide a lot of machinery behind `publish` and `subscribe`. This repo pulls that machinery apart one piece at a time and keeps each piece small enough to read in a sitting. The topics it works through include topic-based fan-out, wildcard routing, competing-consumer work queues with acknowledgements, and later delivery guarantees (at-most-once vs at-least-once), dead-letter handling, ordering, durability, and backpressure. Everything runs in a single process with no dependencies, so the focus stays on the semantics rather than the transport. +Message brokers hide a lot of machinery behind `publish` and `subscribe`. This repo pulls that machinery apart one piece at a time and keeps each piece small enough to read in a sitting. The topics it works through include topic-based fan-out, wildcard routing, competing-consumer work queues with acknowledgements, at-least-once redelivery on nack, dead-letter handling, and later ordering, durability, and backpressure. Everything runs in a single process with no dependencies, so the focus stays on the semantics rather than the transport. ## Concepts demonstrated @@ -13,14 +13,19 @@ Message brokers hide a lot of machinery behind `publish` and `subscribe`. This r - **AMQP-style topic patterns** (`*` one segment, `#` zero or more) with a small dynamic program for matching - **Competing consumers** (work queue): each message is delivered to exactly one consumer - **Acknowledgements** (`ack` / `nack`) and **at-least-once** redelivery on nack or consumer cancel +- **Per-subscriber inbox** on reliable fan-out: each subscriber gets its own copy, settles independently, and a nack redelivers only that copy +- **Redelivered flag** and **delivery count** so handlers can tell first delivery from a retry - **Prefetch** as a per-consumer in-flight limit for fair sharing - **Redelivery counting**, **bounded redelivery** (`maxDeliveryCount`, default 10; excess drops), and **tail requeue** on nack so other ready work is not starved +- **Dead-letter queue** for poison messages after a bounded number of failed attempts ## What's implemented - **Topic-based publish/subscribe with fan-out.** A `Broker` where subscribers register handlers against a topic and every publish fans out to all matching subscribers, with monotonic message ids, idempotent unsubscribe, and snapshot-consistent delivery (subscribing or unsubscribing during dispatch never changes who receives the in-flight message). - **Wildcard topic subscriptions.** AMQP-style pattern bindings where `*` matches exactly one segment and `#` matches zero or more, so one handler can receive a whole family of topics (`orders.#`, `*.created.us`). Matching uses a `#`-aware dynamic program, and a published message fans out to exact subscribers first, then every matching pattern. - **Competing-consumer work queue with acks.** A `WorkQueue` where each enqueued message goes to exactly one consumer. Consumers `ack` to settle or `nack` to requeue (or drop). Handler throw is treated as a requeueing nack. Prefetch defaults to 1 so peers share fairly; cancelling a consumer requeues its unacked deliveries at the head of ready. A nack requeues at the tail. Redelivery is bounded by `maxDeliveryCount` (default 10) so a permanent poison payload cannot busy-spin inside `enqueue`. +- **Dead-letter queue for poison messages.** A `DeadLetterQueue` that counts attempts per original message (and optional subscription), parks a failed payload once `maxAttempts` is exhausted, and supports peek, drop, purge, and redrive back onto a publisher. +- **At-least-once delivery with redelivery on nack.** A `ReliableBroker` where publish still fans out, but each subscriber owns a private inbox. `ack` settles that copy. `nack` (or a handler throw) redelivers it to the same subscriber with `deliveryCount` incremented and `redelivered` set. Peers who already acked are not retried. `nack({ requeue: false })` drops only that subscriber's copy. Retries stop at `maxDeliveryCount`. Optional per-subscriber prefetch isolates a slow consumer without stalling everyone else. ## Usage @@ -80,6 +85,32 @@ queue.consume((delivery) => { queue.enqueue({ jobId: 'job-1' }) // only one of the two consumers receives it ``` +At-least-once fan-out: every subscriber gets a copy, and a nack retries only that copy. + +```ts +import { ReliableBroker } from 'event-broker-lab' + +const reliable = new ReliableBroker<{ orderId: string }>() + +reliable.subscribe('orders.created', (delivery) => { + if (delivery.message.redelivered) { + // already tried once + } + try { + // handle the order + delivery.ack() + } catch { + delivery.nack() + } +}) + +reliable.subscribe('orders.created', (delivery) => { + delivery.ack() // this copy settles even if the other subscriber nacks +}) + +reliable.publish('orders.created', { orderId: 'A-1' }) +``` + ## Running the tests ```sh diff --git a/src/index.ts b/src/index.ts index db282a2..eb2a8a9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,13 @@ export { Broker } from './broker.js' export type { Message, Handler, Unsubscribe } from './broker.js' export { matchTopic, isValidTopic, isValidPattern } from './topic-match.js' +export { DeadLetterQueue, DeadLetterFullError, withDeadLetter } from './dead-letter.js' +export type { + DeadLetterEnvelope, + DeadLetterQueueOptions, + FailResult, + FailStatus, +} from './dead-letter.js' export { WorkQueue } from './work-queue.js' export type { WorkMessage, @@ -9,3 +16,11 @@ export type { WorkQueueOptions, Unsubscribe as WorkQueueUnsubscribe, } from './work-queue.js' +export { ReliableBroker } from './reliable-broker.js' +export type { + ReliableMessage, + ReliableDelivery, + ReliableHandler, + ReliableBrokerOptions, + ReliableSubscribeOptions, +} from './reliable-broker.js' diff --git a/src/reliable-broker.ts b/src/reliable-broker.ts new file mode 100644 index 0000000..e081e93 --- /dev/null +++ b/src/reliable-broker.ts @@ -0,0 +1,200 @@ +import type { Unsubscribe } from './broker.js' + +export interface ReliableMessage { + readonly id: number + readonly topic: string + readonly payload: T + readonly deliveryCount: number + readonly redelivered: boolean +} + +export interface ReliableDelivery { + readonly message: ReliableMessage + readonly deliveryTag: number + ack(): void + nack(options?: { requeue?: boolean }): void +} + +export type ReliableHandler = (delivery: ReliableDelivery) => void + +export interface ReliableBrokerOptions { + maxDeliveryCount?: number +} + +export interface ReliableSubscribeOptions { + prefetch?: number +} + +interface PendingCopy { + id: number + topic: string + payload: T + deliveryCount: number +} + +interface InFlightCopy { + copy: PendingCopy + subscriberId: number + settled: boolean +} + +interface Subscriber { + id: number + topic: string + handler: ReliableHandler + prefetch: number | undefined + ready: PendingCopy[] + inFlight: number + active: boolean +} + +const DEFAULT_MAX_DELIVERY_COUNT = 10 + +export class ReliableBroker { + private readonly subscribers: Subscriber[] = [] + private readonly inFlight = new Map>() + private readonly maxDeliveryCount: number + private nextMessageId = 1 + private nextDeliveryTag = 1 + private nextSubscriberId = 1 + private pumping = false + + constructor(options?: ReliableBrokerOptions) { + const max = options?.maxDeliveryCount ?? DEFAULT_MAX_DELIVERY_COUNT + if (!Number.isInteger(max) || max < 1) { + throw new Error(`maxDeliveryCount must be a positive integer, got ${max}`) + } + this.maxDeliveryCount = max + } + + subscribe( + topic: string, + handler: ReliableHandler, + options?: ReliableSubscribeOptions, + ): Unsubscribe { + const prefetch = options?.prefetch + if (prefetch !== undefined && (!Number.isInteger(prefetch) || prefetch < 1)) { + throw new Error(`prefetch must be a positive integer, got ${prefetch}`) + } + + const subscriber: Subscriber = { + id: this.nextSubscriberId++, + topic, + handler, + prefetch, + ready: [], + inFlight: 0, + active: true, + } + this.subscribers.push(subscriber) + + return () => { + if (!subscriber.active) return + subscriber.active = false + for (const [tag, entry] of this.inFlight) { + if (entry.subscriberId !== subscriber.id || entry.settled) continue + entry.settled = true + this.inFlight.delete(tag) + } + subscriber.ready.length = 0 + subscriber.inFlight = 0 + const i = this.subscribers.indexOf(subscriber) + if (i !== -1) this.subscribers.splice(i, 1) + } + } + + publish(topic: string, payload: T): number { + const targets = this.subscribers.filter((s) => s.active && s.topic === topic) + if (targets.length === 0) return 0 + const id = this.nextMessageId++ + for (const subscriber of targets) { + subscriber.ready.push({ id, topic, payload, deliveryCount: 0 }) + } + this.pump() + return targets.length + } + + readyCount(): number { + return this.subscribers.reduce((n, s) => n + s.ready.length, 0) + } + + inFlightCount(): number { + return this.inFlight.size + } + + subscriberCount(): number { + return this.subscribers.length + } + + private hasCapacity(subscriber: Subscriber): boolean { + return subscriber.prefetch === undefined || subscriber.inFlight < subscriber.prefetch + } + + private pump(): void { + if (this.pumping) return + this.pumping = true + try { + let progressed = true + while (progressed) { + progressed = false + for (const subscriber of this.subscribers) { + if (!subscriber.active) continue + const seenThisRound = new Set() + while (subscriber.ready.length > 0 && this.hasCapacity(subscriber)) { + const head = subscriber.ready[0] + // sync nack requeues the same id; skip it until the next pump round + if (!head || seenThisRound.has(head.id)) break + subscriber.ready.shift() + seenThisRound.add(head.id) + this.deliver(subscriber, head) + progressed = true + } + } + } + } finally { + this.pumping = false + } + } + + private deliver(subscriber: Subscriber, copy: PendingCopy): void { + copy.deliveryCount += 1 + const deliveryTag = this.nextDeliveryTag++ + const entry: InFlightCopy = { copy, subscriberId: subscriber.id, settled: false } + this.inFlight.set(deliveryTag, entry) + subscriber.inFlight += 1 + + const delivery: ReliableDelivery = { + message: { + id: copy.id, + topic: copy.topic, + payload: copy.payload, + deliveryCount: copy.deliveryCount, + redelivered: copy.deliveryCount > 1, + }, + deliveryTag, + ack: () => this.settle(deliveryTag, false), + nack: (options) => this.settle(deliveryTag, options?.requeue !== false), + } + + try { + subscriber.handler(delivery) + } catch { + if (!entry.settled) this.settle(deliveryTag, true) + } + } + + private settle(deliveryTag: number, requeue: boolean): void { + const entry = this.inFlight.get(deliveryTag) + if (!entry || entry.settled) return + entry.settled = true + this.inFlight.delete(deliveryTag) + + const subscriber = this.subscribers.find((s) => s.id === entry.subscriberId) + if (subscriber?.active) subscriber.inFlight -= 1 + + if (requeue && subscriber?.active && entry.copy.deliveryCount < this.maxDeliveryCount) { + subscriber.ready.push(entry.copy) + } + this.pump() + } +} diff --git a/test/reliable-broker.test.ts b/test/reliable-broker.test.ts new file mode 100644 index 0000000..01d6ae5 --- /dev/null +++ b/test/reliable-broker.test.ts @@ -0,0 +1,211 @@ +import { describe, it, expect } from 'vitest' +import { ReliableBroker, type ReliableDelivery } from '../src/index.js' + +describe('ReliableBroker fan-out', () => { + it('throws on invalid maxDeliveryCount', () => { + expect(() => new ReliableBroker({ maxDeliveryCount: 0 })).toThrow(/maxDeliveryCount/) + expect(() => new ReliableBroker({ maxDeliveryCount: 1.5 })).toThrow(/maxDeliveryCount/) + }) + + it('delivers a copy to every matching subscriber and skips other topics', () => { + const broker = new ReliableBroker() + const a: string[] = [] + const b: string[] = [] + broker.subscribe('orders', (d) => { + a.push(d.message.payload) + d.ack() + }) + broker.subscribe('orders', (d) => { + b.push(d.message.payload) + d.ack() + }) + broker.subscribe('shipments', (d) => { + a.push(`ship:${d.message.payload}`) + d.ack() + }) + expect(broker.publish('orders', 'A-1')).toBe(2) + expect(a).toEqual(['A-1']) + expect(b).toEqual(['A-1']) + expect(broker.publish('empty', 'x')).toBe(0) + expect(broker.inFlightCount()).toBe(0) + }) + + it('shares one message id across copies and ignores publishes before subscribe', () => { + const broker = new ReliableBroker() + expect(broker.publish('t', 'late')).toBe(0) + const ids: number[] = [] + const seen: string[] = [] + broker.subscribe('t', (d) => { + ids.push(d.message.id) + seen.push(d.message.payload) + d.ack() + }) + broker.subscribe('t', (d) => { + ids.push(d.message.id) + d.ack() + }) + broker.publish('t', 'one') + broker.publish('t', 'two') + expect(ids).toEqual([1, 1, 2, 2]) + expect(seen).toEqual(['one', 'two']) + }) +}) + +describe('ack and nack redelivery', () => { + it('redelivers only to the subscriber that nacked', () => { + const broker = new ReliableBroker() + const a: number[] = [] + const b: number[] = [] + const flags: boolean[] = [] + const tags: number[] = [] + broker.subscribe('t', (d) => { + a.push(d.message.deliveryCount) + flags.push(d.message.redelivered) + tags.push(d.deliveryTag) + if (d.message.deliveryCount < 3) d.nack() + else d.ack() + }) + broker.subscribe('t', (d) => { + b.push(d.message.deliveryCount) + d.ack() + }) + broker.publish('t', 'retry') + expect(a).toEqual([1, 2, 3]) + expect(b).toEqual([1]) + expect(flags).toEqual([false, true, true]) + expect(tags).toEqual([1, 3, 4]) + expect(broker.readyCount()).toBe(0) + expect(broker.inFlightCount()).toBe(0) + }) + + it('nack({ requeue: false }) drops only that subscriber copy', () => { + const broker = new ReliableBroker() + const seen: string[] = [] + broker.subscribe('t', (d) => { + seen.push(`drop:${d.message.payload}`) + d.nack({ requeue: false }) + }) + broker.subscribe('t', (d) => { + seen.push(`keep:${d.message.payload}`) + d.ack() + }) + broker.publish('t', 'poison') + expect(seen).toEqual(['drop:poison', 'keep:poison']) + expect(broker.readyCount()).toBe(0) + expect(broker.inFlightCount()).toBe(0) + }) + + it('double settle is a no-op; handler throw requeues', () => { + const broker = new ReliableBroker() + let first: ReliableDelivery | undefined + let throws = true + const seen: string[] = [] + broker.subscribe('t', (d) => { + seen.push(d.message.payload) + if (throws) { + first = d + throws = false + d.nack() + return + } + d.ack() + }) + broker.publish('t', 'y') + expect(() => first?.ack()).not.toThrow() + expect(() => first?.nack()).not.toThrow() + expect(seen).toEqual(['y', 'y']) + + const recover = new ReliableBroker() + let boom = true + const recovered: string[] = [] + recover.subscribe('t', (d) => { + recovered.push(d.message.payload) + if (boom) { + boom = false + throw new Error('boom') + } + d.ack() + }) + recover.publish('t', 'recover') + expect(recovered).toEqual(['recover', 'recover']) + expect(recover.inFlightCount()).toBe(0) + }) + + it('always-nack stays bounded and publish still returns', () => { + const maxDeliveryCount = 5 + const broker = new ReliableBroker({ maxDeliveryCount }) + let deliveries = 0 + broker.subscribe('t', (d) => { + deliveries += 1 + d.nack() + }) + expect(broker.publish('t', 'spin')).toBe(1) + expect(deliveries).toBe(maxDeliveryCount) + expect(broker.readyCount()).toBe(0) + expect(broker.inFlightCount()).toBe(0) + expect(broker.publish('t', 'next')).toBe(1) + }) +}) + +describe('prefetch, isolation, unsubscribe', () => { + it('throws on invalid prefetch', () => { + const broker = new ReliableBroker() + expect(() => broker.subscribe('t', () => {}, { prefetch: 0 })).toThrow(/prefetch/) + expect(() => broker.subscribe('t', () => {}, { prefetch: 1.5 })).toThrow(/prefetch/) + }) + + it('holds later copies at prefetch while peers still receive', () => { + const broker = new ReliableBroker() + const held: ReliableDelivery[] = [] + const peer: string[] = [] + broker.subscribe('t', (d) => held.push(d), { prefetch: 1 }) + broker.subscribe('t', (d) => { + peer.push(d.message.payload) + d.ack() + }) + broker.publish('t', 'first') + broker.publish('t', 'second') + expect(held).toHaveLength(1) + expect(held[0]?.message.payload).toBe('first') + expect(peer).toEqual(['first', 'second']) + expect(broker.readyCount()).toBe(1) + held[0]?.ack() + expect(held[1]?.message.payload).toBe('second') + expect(broker.readyCount()).toBe(0) + }) + + it('publish from a handler does not drop the new copy', () => { + const broker = new ReliableBroker() + const seen: string[] = [] + broker.subscribe('t', (d) => { + seen.push(d.message.payload) + if (d.message.payload === 'seed') broker.publish('t', 'child') + d.ack() + }) + broker.publish('t', 'seed') + expect(seen).toEqual(['seed', 'child']) + }) + + it('unsubscribe drops only that inbox; stale settle is harmless', () => { + const broker = new ReliableBroker() + let stolen: ReliableDelivery | undefined + const kept: string[] = [] + const off = broker.subscribe('t', (d) => { + stolen = d + }) + broker.subscribe('t', (d) => { + kept.push(d.message.payload) + d.ack() + }) + broker.publish('t', 'held') + expect(broker.inFlightCount()).toBe(1) + off() + expect(() => off()).not.toThrow() + expect(broker.subscriberCount()).toBe(1) + expect(broker.inFlightCount()).toBe(0) + expect(() => stolen?.ack()).not.toThrow() + expect(() => stolen?.nack()).not.toThrow() + broker.publish('t', 'solo') + expect(kept).toEqual(['held', 'solo']) + }) +})