diff --git a/README.md b/README.md index 6c7c777..e7736de 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ This repository provides: | ------ | ----------- | ------- | ------ | | [`after-hours`](./after-hours) | Auto-replies with a configurable away/closing message to messages received outside business hours. | 0.1.2 | stable | | [`chat-flow`](./chat-flow) | Interactive, stateful auto-reply: a trigger word starts a greeting + numbered menu, replies traverse a configurable menu tree, and per-chat state expires after 15 minutes. | 1.0.5 | stable | -| [`chatwoot-adapter`](./chatwoot-adapter) | Two-way sync between a WhatsApp session and a Chatwoot inbox: relays WhatsApp messages (1:1 and groups, with media) into Chatwoot as an API-channel inbox, sends agent replies back to WhatsApp, and hands a chat over to a human agent — silencing other OpenWA bots — when an agent takes it in Chatwoot. First consumer of the OpenWA Integration SDK v1; runs sandboxed in the plugin worker. | 0.5.1 | beta | +| [`chatwoot-adapter`](./chatwoot-adapter) | Two-way sync between a WhatsApp session and a Chatwoot inbox: relays WhatsApp messages (1:1 and groups, with media) into Chatwoot as an API-channel inbox, sends agent replies back to WhatsApp, and hands a chat over to a human agent — silencing other OpenWA bots — when an agent takes it in Chatwoot. First consumer of the OpenWA Integration SDK v1; runs sandboxed in the plugin worker. | 0.5.2 | beta | | [`faq-bot`](./faq-bot) | Auto-replies to inbound WhatsApp messages from configurable FAQ keyword/regex rules. | 0.1.6 | stable | | [`group-translate`](./group-translate) | Auto-translates group messages between participants' languages via a LibreTranslate backend. Configure in-chat with /tr commands. Admin-gated; disabled until enabled. | 1.0.5 | stable | | [`gsheets-logger`](./gsheets-logger) | Logs WhatsApp message events to a Google Sheet via a service account. | 0.2.3 | stable | diff --git a/chatwoot-adapter/CHANGELOG.md b/chatwoot-adapter/CHANGELOG.md index 8ba081c..ae9759d 100644 --- a/chatwoot-adapter/CHANGELOG.md +++ b/chatwoot-adapter/CHANGELOG.md @@ -6,6 +6,18 @@ All notable changes to the Chatwoot Adapter plugin are documented here. The form ## [Unreleased] +## [0.5.2] — 2026-07-04 + +### Fixed + +- **The internal de-duplication markers no longer grow without bound.** The adapter keeps one marker per + relayed message — to skip WhatsApp re-deliveries and its own echoed sends — and these were never cleaned + up, so the plugin's storage grew for the life of the install and the inbound-retry timer's periodic scan + got progressively slower on a long-running instance. Markers now carry a timestamp and are pruned once + they pass a 3-day retention window, which comfortably outlasts any realistic WhatsApp re-delivery or + own-send echo, so normal live de-duplication is unaffected. No configuration or action is needed; + existing markers are migrated automatically. + ## [0.5.1] — 2026-07-03 ### Fixed diff --git a/chatwoot-adapter/README.md b/chatwoot-adapter/README.md index 58b6538..726e6d9 100644 --- a/chatwoot-adapter/README.md +++ b/chatwoot-adapter/README.md @@ -11,8 +11,8 @@ mirror it. Runs sandboxed in the plugin worker; no server, no extra port. | Field | Value | | ----- | ----- | | **Identifier** | `chatwoot-adapter` | -| **Version** | 0.5.1 | -| **Released** | 2026-07-03 | +| **Version** | 0.5.2 | +| **Released** | 2026-07-04 | | **Status** | beta | | **Author** | Yudhi Armyndharis | | **License** | MIT | diff --git a/chatwoot-adapter/index.ts b/chatwoot-adapter/index.ts index 81e3c14..9867b46 100644 --- a/chatwoot-adapter/index.ts +++ b/chatwoot-adapter/index.ts @@ -1,6 +1,6 @@ import type { IPlugin, PluginContext, IncomingMessage, HookContext, HookResult, WebhookRequest } from '../types/openwa'; import { ChatwootClient } from './chatwoot-client.ts'; -import { MappingStore } from './mapping-store.ts'; +import { MappingStore, SEEN_TTL_MS, SEEN_PRUNE_INTERVAL_MS } from './mapping-store.ts'; import { KeyedAsyncLock } from './chat-lock.ts'; import { handleInbound, relayInbound } from './inbound.ts'; import { handleSent } from './sent.ts'; @@ -63,6 +63,8 @@ export default class ChatwootAdapter implements IPlugin { private store: MappingStore | null = null; private deadLetterCount = 0; private draining = false; + private lastSeenPruneAt = 0; + private seenPruning = false; async onEnable(ctx: PluginContext): Promise { this.clearRetryTimer(); // idempotent re-enable: never leak a timer from a prior enable @@ -166,7 +168,32 @@ export default class ChatwootAdapter implements IPlugin { ) .finally(() => void (this.draining = false)); }; - this.retryTimer = setInterval(() => void drain(), RETRY_INTERVAL_MS); + // Bound the `seen:` de-dup markers: hourly, expire any past the retention window. Piggybacks this same + // timer (no second timer), independent of the drain's single-flight so a slow prune never blocks a + // retry. Runs regardless of config validity — it neither reads config nor relays. lastSeenPruneAt = 0 + // so the first prune (and the one-time legacy adopt) fires on the first tick after enable. Best-effort: + // a failure is logged and retried next tick that is due. + const maybePruneSeen = (): void => { + if (this.seenPruning) return; + const now = Date.now(); + if (now - this.lastSeenPruneAt < SEEN_PRUNE_INTERVAL_MS) return; + this.lastSeenPruneAt = now; + this.seenPruning = true; + void store + .pruneSeen(now, SEEN_TTL_MS) + .then(({ pruned, adopted }) => { + if (pruned || adopted) { + ctx.logger.log(`chatwoot-adapter: pruned ${pruned} expired seen-marker(s), adopted ${adopted} legacy`); + } + }) + .catch(e => ctx.logger.error('seen-marker prune failed', e)) + .finally(() => void (this.seenPruning = false)); + }; + + this.retryTimer = setInterval(() => { + void drain(); + maybePruneSeen(); + }, RETRY_INTERVAL_MS); this.retryTimer.unref?.(); ctx.logger.log('chatwoot-adapter enabled'); diff --git a/chatwoot-adapter/manifest.json b/chatwoot-adapter/manifest.json index 3d9455a..2f2e52d 100644 --- a/chatwoot-adapter/manifest.json +++ b/chatwoot-adapter/manifest.json @@ -1,7 +1,7 @@ { "id": "chatwoot-adapter", "name": "Chatwoot Adapter", - "version": "0.5.1", + "version": "0.5.2", "type": "extension", "main": "dist/index.js", "description": "Two-way sync between a WhatsApp session and a Chatwoot inbox: relays WhatsApp messages (1:1 and groups, with media) into Chatwoot as an API-channel inbox, sends agent replies back to WhatsApp, and hands a chat over to a human agent — silencing other OpenWA bots — when an agent takes it in Chatwoot. First consumer of the OpenWA Integration SDK v1; runs sandboxed in the plugin worker.", diff --git a/chatwoot-adapter/mapping-store.test.ts b/chatwoot-adapter/mapping-store.test.ts index 64428c4..0f634c4 100644 --- a/chatwoot-adapter/mapping-store.test.ts +++ b/chatwoot-adapter/mapping-store.test.ts @@ -99,3 +99,45 @@ test('enqueueRetry drops the OLDEST entry (by enqueuedAt) when the queue is at c const ids = (await store.listRetries()).map(e => e.msg.id).sort(); assert.deepEqual(ids, ['mid', 'new']); }); + +test('markSeen stores a timestamped marker and hasSeen stays truthy', async () => { + const storage = fakeStorage(); + const store = new MappingStore(storage, fakeMappings()); + await store.markSeen('wa', 'm1', 'sess', 1000); + assert.equal(await store.hasSeen('wa', 'm1', 'sess'), true); + assert.deepEqual(await storage.get('seen:sess:wa:m1'), { t: 1000 }); +}); + +test('pruneSeen deletes markers older than the TTL and keeps recent ones', async () => { + const store = new MappingStore(fakeStorage(), fakeMappings()); + const TTL = 1000; + await store.markSeen('wa', 'old', 'sess', 0); // age 5000 > TTL → pruned + await store.markSeen('wa', 'fresh', 'sess', 4500); // age 500 < TTL → kept + const { pruned, adopted } = await store.pruneSeen(5000, TTL); + assert.equal(pruned, 1); + assert.equal(adopted, 0); + assert.equal(await store.hasSeen('wa', 'old', 'sess'), false); + assert.equal(await store.hasSeen('wa', 'fresh', 'sess'), true); +}); + +test('pruneSeen adopts a legacy marker (stamps a timestamp, does not delete)', async () => { + const storage = fakeStorage(); + const store = new MappingStore(storage, fakeMappings()); + await storage.set('seen:sess:wa:legacy', 1); // pre-0.5.2 bare marker + const { pruned, adopted } = await store.pruneSeen(9000, 1000); + assert.equal(pruned, 0); + assert.equal(adopted, 1); + assert.equal(await store.hasSeen('wa', 'legacy', 'sess'), true); // still present + assert.deepEqual(await storage.get('seen:sess:wa:legacy'), { t: 9000 }); // now timestamped +}); + +test('pruneSeen leaves non-seen keys untouched', async () => { + const store = new MappingStore(fakeStorage(), fakeMappings()); + await store.link('sess', 'c@wa', 'inst', { conversationId: 1, contactId: 2, sourceId: 'x' }); + await store.enqueueRetry({ sessionId: 'sess', chatId: 'c@wa', msg: msg('r1'), enqueuedAt: 0 }, 500); + await store.markSeen('wa', 'old', 'sess', 0); + await store.pruneSeen(10_000, 1000); + assert.equal(await store.hasSeen('wa', 'old', 'sess'), false); // pruned + assert.equal(await store.countRetries(), 1); // retry untouched + assert.deepEqual(await store.getByChat('sess', 'c@wa'), { conversationId: 1, contactId: 2, sourceId: 'x' }); +}); diff --git a/chatwoot-adapter/mapping-store.ts b/chatwoot-adapter/mapping-store.ts index 29e3c17..17b7c82 100644 --- a/chatwoot-adapter/mapping-store.ts +++ b/chatwoot-adapter/mapping-store.ts @@ -27,6 +27,12 @@ export interface ChatLink { // are therefore scoped by the WA sessionId (the one identity both the inbound hook and the outbound // ingress delivery share). A session-scoped legacy reverse key is ALSO written so a delivery that arrives // without a session scope (or a pre-scope row) still resolves — unscoped, so single-tenant is unaffected. +// Retention window for `seen:` de-dup markers, and how often expired ones are pruned. Hardcoded (mirroring +// the retry-timer constants in retry.ts) — a generous default that outlasts any realistic WhatsApp message +// re-delivery, so pruning never re-posts a duplicate. Bumping the TTL is a one-line change. +export const SEEN_TTL_MS = 3 * 24 * 60 * 60 * 1000; // 3 days +export const SEEN_PRUNE_INTERVAL_MS = 60 * 60 * 1000; // hourly + export class MappingStore { constructor( private readonly storage: PluginStorage, @@ -84,8 +90,37 @@ export class MappingStore { async hasSeen(kind: 'wa' | 'cw', id: string, scope?: string): Promise { return Boolean(await this.storage.get(this.seenKey(kind, id, scope))); } - async markSeen(kind: 'wa' | 'cw', id: string, scope?: string): Promise { - await this.storage.set(this.seenKey(kind, id, scope), 1); + async markSeen(kind: 'wa' | 'cw', id: string, scope?: string, nowMs: number = Date.now()): Promise { + // Store a timestamp (not a bare `1`) so pruneSeen can age the marker out. hasSeen only checks presence. + await this.storage.set(this.seenKey(kind, id, scope), { t: nowMs }); + } + + // Prune expired `seen:` markers so ctx.storage doesn't grow without bound (one file per marker) and the + // retry drain's directory scan stays cheap. Streams keys one at a time (matching the drain's OOM-safe + // discipline). A pre-0.5.2 marker stored as a bare `1` has no timestamp: it is ADOPTED (stamped with the + // current time) rather than deleted, so it can never re-post a duplicate and ages out one TTL from here. + // Touches only `seen:`-prefixed keys — the list() prefix is filtered defensively (a fake ignoring the + // arg would otherwise return every key). + async pruneSeen(nowMs: number, ttlMs: number): Promise<{ pruned: number; adopted: number }> { + const keys = (await this.storage.list('seen:')).filter(k => k.startsWith('seen:')); + let pruned = 0; + let adopted = 0; + for (const key of keys) { + const v = await this.storage.get(key); + if (v == null) continue; // vanished since the scan — nothing to do + const t = + typeof v === 'object' && v !== null && typeof (v as { t?: unknown }).t === 'number' + ? (v as { t: number }).t + : undefined; + if (t === undefined) { + await this.storage.set(key, { t: nowMs }); // legacy/malformed marker → adopt + adopted++; + } else if (nowMs - t > ttlMs) { + await this.storage.delete(key); + pruned++; + } + } + return { pruned, adopted }; } // Durable run-once marker for the one-time bulk history sweep, per WA session. diff --git a/plugins.json b/plugins.json index 826877b..61c688d 100644 --- a/plugins.json +++ b/plugins.json @@ -369,7 +369,7 @@ { "id": "chatwoot-adapter", "name": "Chatwoot Adapter", - "version": "0.5.1", + "version": "0.5.2", "type": "extension", "status": "beta", "description": "Two-way sync between a WhatsApp session and a Chatwoot inbox: relays WhatsApp messages (1:1 and groups, with media) into Chatwoot as an API-channel inbox, sends agent replies back to WhatsApp, and hands a chat over to a human agent — silencing other OpenWA bots — when an agent takes it in Chatwoot. First consumer of the OpenWA Integration SDK v1; runs sandboxed in the plugin worker.", @@ -387,11 +387,11 @@ ], "minOpenWAVersion": "0.8.7", "testedOpenWAVersion": "0.8.7", - "releasedAt": "2026-07-03", + "releasedAt": "2026-07-04", "repoPath": "chatwoot-adapter", "repoUrl": "https://github.com/rmyndharis/OpenWA-plugins", "homepage": "https://github.com/rmyndharis/OpenWA-plugins/tree/main/chatwoot-adapter", - "download": "https://github.com/rmyndharis/OpenWA-plugins/releases/download/chatwoot-adapter-v0.5.1/chatwoot-adapter.zip" + "download": "https://github.com/rmyndharis/OpenWA-plugins/releases/download/chatwoot-adapter-v0.5.2/chatwoot-adapter.zip" }, { "id": "faq-bot",