From a46ee322f87e784e9487e3c7330eb0a273b1651f Mon Sep 17 00:00:00 2001 From: rmyndharis Date: Fri, 3 Jul 2026 16:22:05 +0700 Subject: [PATCH] feat(chatwoot-adapter): history backfill (lazy + opt-in bulk) (#609) Agents now see prior WhatsApp context in Chatwoot instead of a conversation that starts mid-thread. Two composable modes, both off by default: - Lazy (backfillLimit): when a chat first opens as a Chatwoot conversation, its recent messages (both directions, with media) are replayed oldest->newest before the triggering message, deduped against the live path so nothing double-posts. - Bulk (backfillAllOnce): a one-time sweep imports every existing chat's history on setup, sequential and best-effort, run once per session behind a durable marker + an in-flight guard. Business-side (fromMe) messages post as Chatwoot outgoing, contact messages as incoming (postText/postMedia gain a message_type). The per-message render and conversation resolve are extracted into a shared relay.ts (leaf module) so live inbound and backfill render identically without an import cycle. Bumps minOpenWAVersion to 0.8.5 (engine.getChatHistory) and the plugin to 0.3.0. --- README.md | 2 +- chatwoot-adapter/CHANGELOG.md | 14 ++- chatwoot-adapter/README.md | 11 +- chatwoot-adapter/backfill.test.ts | 135 ++++++++++++++++++++++ chatwoot-adapter/backfill.ts | 57 ++++++++++ chatwoot-adapter/chatwoot-client.test.ts | 10 ++ chatwoot-adapter/chatwoot-client.ts | 7 +- chatwoot-adapter/inbound.test.ts | 2 +- chatwoot-adapter/inbound.ts | 138 ++++++----------------- chatwoot-adapter/index.ts | 47 +++++--- chatwoot-adapter/manifest.json | 18 ++- chatwoot-adapter/mapping-store.ts | 8 ++ chatwoot-adapter/relay.ts | 127 +++++++++++++++++++++ plugins.json | 8 +- types/openwa.d.ts | 11 ++ 15 files changed, 462 insertions(+), 133 deletions(-) create mode 100644 chatwoot-adapter/backfill.test.ts create mode 100644 chatwoot-adapter/backfill.ts create mode 100644 chatwoot-adapter/relay.ts diff --git a/README.md b/README.md index efeb0c6..9f696c5 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ This repository provides: | ------ | ----------- | ------- | ------ | | [`after-hours`](./after-hours) | Auto-replies with a configurable away/closing message to messages received outside business hours. | 0.1.2 | stable | | [`chat-flow`](./chat-flow) | Interactive, stateful auto-reply: a trigger word starts a greeting + numbered menu, replies traverse a configurable menu tree, and per-chat state expires after 15 minutes. | 1.0.5 | stable | -| [`chatwoot-adapter`](./chatwoot-adapter) | Two-way sync between a WhatsApp session and a Chatwoot inbox: relays WhatsApp messages (1:1 and groups, with media) into Chatwoot as an API-channel inbox, sends agent replies back to WhatsApp, and hands a chat over to a human agent — silencing other OpenWA bots — when an agent takes it in Chatwoot. First consumer of the OpenWA Integration SDK v1; runs sandboxed in the plugin worker. | 0.2.0 | beta | +| [`chatwoot-adapter`](./chatwoot-adapter) | Two-way sync between a WhatsApp session and a Chatwoot inbox: relays WhatsApp messages (1:1 and groups, with media) into Chatwoot as an API-channel inbox, sends agent replies back to WhatsApp, and hands a chat over to a human agent — silencing other OpenWA bots — when an agent takes it in Chatwoot. First consumer of the OpenWA Integration SDK v1; runs sandboxed in the plugin worker. | 0.3.0 | beta | | [`faq-bot`](./faq-bot) | Auto-replies to inbound WhatsApp messages from configurable FAQ keyword/regex rules. | 0.1.6 | stable | | [`group-translate`](./group-translate) | Auto-translates group messages between participants' languages via a LibreTranslate backend. Configure in-chat with /tr commands. Admin-gated; disabled until enabled. | 1.0.5 | stable | | [`gsheets-logger`](./gsheets-logger) | Logs WhatsApp message events to a Google Sheet via a service account. | 0.2.3 | stable | diff --git a/chatwoot-adapter/CHANGELOG.md b/chatwoot-adapter/CHANGELOG.md index 603698c..113fdd2 100644 --- a/chatwoot-adapter/CHANGELOG.md +++ b/chatwoot-adapter/CHANGELOG.md @@ -6,7 +6,19 @@ All notable changes to the Chatwoot Adapter plugin are documented here. The form ## [Unreleased] -## [0.2.0] — 2026-07-03 +## [0.3.0] — 2026-07-03 + +### Added + +- **History backfill** so agents see prior WhatsApp context in Chatwoot instead of a conversation that + starts mid-thread (#609). Two composable modes, both off by default: + - **Lazy (`backfillLimit`)** — when a chat first opens as a Chatwoot conversation, its recent messages + (both directions, with media) are replayed oldest→newest before the triggering message, so the thread + reads in order. Deduped against the live path, so nothing double-posts. + - **Bulk (`backfillAllOnce`)** — a one-time sweep that imports the history of every existing chat on + setup, for mirroring a whole inbox. Sequential, best-effort, runs once per session. + - Business-side (`fromMe`) messages post as Chatwoot `outgoing`, contact messages as `incoming`. + - Requires OpenWA 0.8.5+ (the new `engine.getChatHistory` capability). ### Added diff --git a/chatwoot-adapter/README.md b/chatwoot-adapter/README.md index 91600b3..8e4592b 100644 --- a/chatwoot-adapter/README.md +++ b/chatwoot-adapter/README.md @@ -11,13 +11,13 @@ mirror it. Runs sandboxed in the plugin worker; no server, no extra port. | Field | Value | | ----- | ----- | | **Identifier** | `chatwoot-adapter` | -| **Version** | 0.2.0 | +| **Version** | 0.3.0 | | **Released** | 2026-07-03 | | **Status** | beta | | **Author** | Yudhi Armyndharis | | **License** | MIT | | **Type** | `extension` | -| **Requires OpenWA** | ≥ 0.8.3 (tested 0.8.3) | +| **Requires OpenWA** | ≥ 0.8.5 (tested 0.8.5) | | **Keywords** | chatwoot, helpdesk, inbox, handover, two-way, agent, whatsapp, openwa | | **Repository** | [OpenWA-plugins/chatwoot-adapter](https://github.com/rmyndharis/OpenWA-plugins/tree/main/chatwoot-adapter) | @@ -66,6 +66,8 @@ sees it. | `inboxId` | number | yes | The API-channel inbox id this adapter posts into and relays replies from. | | `relayGroups` | boolean | no (default `true`) | Relay group chats (one synthetic contact per group, sender-prefixed). | | `relayMedia` | boolean | no (default `true`) | Upload inbound media to Chatwoot as attachments. | +| `backfillLimit` | number | no (default `0`) | When a chat first opens in Chatwoot, import this many recent messages (both directions, with media) so agents see prior context. `0` disables it; clamped to 100 host-side. Needs OpenWA 0.8.5+. | +| `backfillAllOnce` | boolean | no (default `false`) | Also run a one-time sweep importing every existing chat's history on setup. Needs `backfillLimit` > 0. Runs once per session. | The Chatwoot webhook **secret** and the instance's **session scope** are set when you mint the instance (they are not part of the config above). @@ -85,8 +87,9 @@ public host is added to the outbound allowlist). To use a self-hosted Chatwoot: ## Compatibility -- **OpenWA** ≥ 0.8.3 — needs Integration SDK v1 (webhook ingress, `ctx.mappings`, the session+chat handover - gate, `net.allowConfigHosts`) and the `conversation.send` media/voice types used for outbound attachments. +- **OpenWA** ≥ 0.8.5 — needs Integration SDK v1 (webhook ingress, `ctx.mappings`, the session+chat handover + gate, `net.allowConfigHosts`), the `conversation.send` media/voice types for outbound attachments, and + `engine.getChatHistory` for the history backfill. - **Chatwoot** — account-level webhooks with timestamped HMAC signing (see Setup). ## Security diff --git a/chatwoot-adapter/backfill.test.ts b/chatwoot-adapter/backfill.test.ts new file mode 100644 index 0000000..86c0feb --- /dev/null +++ b/chatwoot-adapter/backfill.test.ts @@ -0,0 +1,135 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { backfillHistory, backfillAllChats } from './backfill.ts'; +import { KeyedAsyncLock } from './chat-lock.ts'; +import type { InboundDeps } from './relay.ts'; +import type { IncomingMessage } from '../types/openwa'; + +// Capture what relayMessage posts by observing the client the shared relay calls (text path only here; +// media rendering is covered by the inbound tests). +function makeDeps( + over: { + engine?: Record; + store?: Record; + relayGroups?: boolean; + backfillLimit?: number; + } = {}, +) { + const posts: Array<{ conversationId: number; type: string; body: string }> = []; + const seen = new Set(); + const client = { + searchContact: async () => null, + createContact: async () => ({ id: 9, sourceId: 'src' }), + findOpenConversation: async () => null, + createConversation: async () => 55, + postText: async (conversationId: number, body: string, o: { messageType?: string }) => { + posts.push({ conversationId, type: o?.messageType ?? 'incoming', body }); + return { id: 1 }; + }, + postMedia: async () => ({ id: 2 }), + updateContact: async () => {}, + }; + const store = { + hasSeen: async (_k: string, id: string) => seen.has(id), + markSeen: async (_k: string, id: string) => void seen.add(id), + getByChat: async () => null, + link: async () => {}, + patch: async () => {}, + isBulkBackfilled: async () => false, + setBulkBackfilled: async () => {}, + ...over.store, + }; + const deps = { + lock: new KeyedAsyncLock(), + client, + store, + engine: { getChatHistory: async () => [], getChats: async () => [], ...over.engine }, + instanceId: 'inst', + relayGroups: over.relayGroups ?? true, + relayMedia: true, + backfillLimit: over.backfillLimit ?? 20, + backfillAllOnce: false, + log: () => {}, + } as unknown as InboundDeps; + return { deps, posts }; +} + +const hist = (id: string, ts: number, fromMe: boolean, body: string): IncomingMessage => + ({ id, from: 'x', to: 'y', chatId: 'c@c.us', body, type: 'chat', timestamp: ts, fromMe, isGroup: false }) as IncomingMessage; + +test('backfillHistory posts oldest→newest with fromMe as outgoing (#609)', async () => { + const history = [hist('m3', 30, false, 'third'), hist('m1', 10, true, 'first'), hist('m2', 20, false, 'second')]; + const { deps, posts } = makeDeps({ engine: { getChatHistory: async () => history } }); + await backfillHistory(deps, 'sess', 'c@c.us', 55); + assert.deepEqual(posts, [ + { conversationId: 55, type: 'outgoing', body: 'first' }, // fromMe, oldest first + { conversationId: 55, type: 'incoming', body: 'second' }, + { conversationId: 55, type: 'incoming', body: 'third' }, + ]); +}); + +test('backfillHistory skips messages already seen (dedup with the live path)', async () => { + const seen = new Set(['m1']); + const history = [hist('m1', 10, false, 'dup'), hist('m2', 20, false, 'new')]; + const { deps, posts } = makeDeps({ + engine: { getChatHistory: async () => history }, + store: { + hasSeen: async (_k: string, id: string) => seen.has(id), + markSeen: async (_k: string, id: string) => void seen.add(id), + }, + }); + await backfillHistory(deps, 'sess', 'c@c.us', 55); + assert.deepEqual(posts.map(p => p.body), ['new']); +}); + +test('backfillHistory swallows a getChatHistory failure (best-effort)', async () => { + const { deps, posts } = makeDeps({ + engine: { + getChatHistory: async () => { + throw new Error('engine down'); + }, + }, + }); + await backfillHistory(deps, 'sess', 'c@c.us', 55); // must not throw + assert.equal(posts.length, 0); +}); + +test('backfillAllChats sweeps each chat once, skips groups when relayGroups is off, survives a failure (#609)', async () => { + const chats = [ + { id: 'a@c.us', name: 'A', isGroup: false, unreadCount: 0, timestamp: 1 }, + { id: 'g@g.us', name: 'G', isGroup: true, unreadCount: 0, timestamp: 2 }, + { id: 'b@c.us', name: 'B', isGroup: false, unreadCount: 0, timestamp: 3 }, + ]; + const historyByChat: Record = { + 'a@c.us': [{ ...hist('a1', 10, false, 'from A'), chatId: 'a@c.us' }], + 'b@c.us': [{ ...hist('b1', 10, false, 'from B'), chatId: 'b@c.us' }], + }; + let bulkDone = false; + const { deps, posts } = makeDeps({ + relayGroups: false, // the group chat must be skipped + engine: { + getChats: async () => chats, + getChatHistory: async (_s: string, chatId: string) => historyByChat[chatId] ?? [], + }, + store: { + hasSeen: async () => false, + markSeen: async () => {}, + getByChat: async () => null, + link: async () => {}, + isBulkBackfilled: async () => bulkDone, + setBulkBackfilled: async () => { + bulkDone = true; + }, + }, + }); + await backfillAllChats(deps, 'sessBulk'); + assert.deepEqual( + posts.map(p => p.body).sort(), + ['from A', 'from B'], // group skipped + ); + assert.equal(bulkDone, true); + // second call is a no-op once the durable marker is set + const before = posts.length; + await backfillAllChats(deps, 'sessBulk'); + assert.equal(posts.length, before); +}); diff --git a/chatwoot-adapter/backfill.ts b/chatwoot-adapter/backfill.ts new file mode 100644 index 0000000..3d90285 --- /dev/null +++ b/chatwoot-adapter/backfill.ts @@ -0,0 +1,57 @@ +import type { ChatSummary } from '../types/openwa'; +import { relayMessage, ensureConversation, type InboundDeps } from './relay.ts'; + +// Replay a chat's recent history into its Chatwoot conversation: oldest→newest, both directions, deduped +// against the same markSeen store the live path uses (so a live message is never doubled). The caller +// holds the per-chat lock. Best-effort: any failure is logged, never thrown into the live relay. +export async function backfillHistory( + deps: InboundDeps, + sessionId: string, + chatId: string, + conversationId: number, +): Promise { + try { + const history = await deps.engine.getChatHistory(sessionId, chatId, deps.backfillLimit, true); + const ordered = [...history].sort((a, b) => a.timestamp - b.timestamp); + for (const msg of ordered) { + if (await deps.store.hasSeen('wa', msg.id, sessionId)) continue; + await deps.store.markSeen('wa', msg.id, sessionId); + await relayMessage(deps, conversationId, msg, msg.fromMe ? 'outgoing' : 'incoming'); + } + } catch (err) { + deps.log('history backfill failed', err); + } +} + +// In-memory guard so rapid successive inbounds can't launch the one-time sweep twice for a session. +const bulkInFlight = new Set(); + +// One-time bulk sweep (opt-in): create a Chatwoot conversation for every existing chat and backfill its +// history. Sequential — no parallel fan-out at Chatwoot — and best-effort: a per-chat failure never +// aborts the sweep. Runs once per session behind a durable marker plus the in-memory in-flight guard. +export async function backfillAllChats(deps: InboundDeps, sessionId: string): Promise { + // Add to the in-flight set BEFORE the first await, so a concurrent call from a rapid second inbound + // sees it synchronously and bails — otherwise both could pass the durable-marker check and double-sweep. + if (bulkInFlight.has(sessionId)) return; + bulkInFlight.add(sessionId); + try { + if (await deps.store.isBulkBackfilled(sessionId)) return; + const chats = (await deps.engine.getChats(sessionId)) as ChatSummary[]; + for (const chat of chats) { + if (chat.isGroup && !deps.relayGroups) continue; + await deps.lock.run(`${sessionId}:${chat.id}`, async () => { + try { + const conversationId = await ensureConversation(deps, sessionId, chat.id, { name: chat.name || chat.id }); + await backfillHistory(deps, sessionId, chat.id, conversationId); + } catch (err) { + deps.log(`bulk backfill failed for ${chat.id}`, err); + } + }); + } + await deps.store.setBulkBackfilled(sessionId); + } catch (err) { + deps.log('bulk backfill sweep failed', err); + } finally { + bulkInFlight.delete(sessionId); + } +} diff --git a/chatwoot-adapter/chatwoot-client.test.ts b/chatwoot-adapter/chatwoot-client.test.ts index 1f9bb76..5d44173 100644 --- a/chatwoot-adapter/chatwoot-client.test.ts +++ b/chatwoot-adapter/chatwoot-client.test.ts @@ -49,6 +49,16 @@ test('postText posts an incoming message with the api token header', async () => assert.equal(last.init!.headers!['api_access_token'], 'tok'); }); +test('postText and postMedia honor an outgoing message_type (backfilled business messages)', async () => { + const { fn, calls } = fakeFetch({ 'POST /api/v1/accounts/3/conversations/55/messages': { body: { id: 1 } } }); + const c = new ChatwootClient(fn, cfg); + await c.postText(55, 'sent by agent', { messageType: 'outgoing' }); + assert.equal(JSON.parse(calls.at(-1)!.init!.body as string).message_type, 'outgoing'); + await c.postMedia(55, '', { filename: 'x.jpg', contentType: 'image/jpeg', data: new Uint8Array([1]) }, { messageType: 'outgoing' }); + const raw = Buffer.from(calls.at(-1)!.init!.body as Uint8Array).toString('latin1'); + assert.match(raw, /name="message_type"\r\n\r\noutgoing/); +}); + test('updateContact PUTs the new name to the contact', async () => { const { fn, calls } = fakeFetch({ 'PUT /api/v1/accounts/3/contacts/9': { body: { id: 9 } } }); await new ChatwootClient(fn, cfg).updateContact(9, 'Budi'); diff --git a/chatwoot-adapter/chatwoot-client.ts b/chatwoot-adapter/chatwoot-client.ts index 57d8c77..a5b5220 100644 --- a/chatwoot-adapter/chatwoot-client.ts +++ b/chatwoot-adapter/chatwoot-client.ts @@ -12,6 +12,9 @@ export interface ChatwootConfig { export interface MessagePostOptions { sourceId?: string; inReplyToExternalId?: string; + // Chatwoot direction. Live inbound is always 'incoming'; history backfill posts business-side + // (fromMe) messages as 'outgoing' so the reconstructed thread reads correctly. Default 'incoming'. + messageType?: 'incoming' | 'outgoing'; } // The slice of ctx.net.fetch this client needs (host-proxied, SSRF-guarded). Injectable for tests. @@ -111,7 +114,7 @@ export class ChatwootClient { } async postText(conversationId: number, content: string, opts: MessagePostOptions = {}): Promise<{ id: number }> { - const payload: Record = { content, message_type: 'incoming', private: false }; + const payload: Record = { content, message_type: opts.messageType ?? 'incoming', private: false }; if (opts.sourceId) payload.source_id = opts.sourceId; if (opts.inReplyToExternalId) payload.content_attributes = { in_reply_to_external_id: opts.inReplyToExternalId }; const { data } = await this.json<{ id: number }>(`${this.base()}/conversations/${conversationId}/messages`, { @@ -130,7 +133,7 @@ export class ChatwootClient { const boundary = `----cw${conversationId}${file.data.byteLength}`; const fields = [ { name: 'content', value: content }, - { name: 'message_type', value: 'incoming' }, + { name: 'message_type', value: opts.messageType ?? 'incoming' }, ]; // Rails parses bracket notation into nested params; source_id + in_reply_to_external_id give Chatwoot // the threading it uses for the native WhatsApp integration, is_voice_message renders a voice bubble. diff --git a/chatwoot-adapter/inbound.test.ts b/chatwoot-adapter/inbound.test.ts index 6e59a35..1b8e1ca 100644 --- a/chatwoot-adapter/inbound.test.ts +++ b/chatwoot-adapter/inbound.test.ts @@ -68,7 +68,7 @@ test('forwards the quote context (source_id + in_reply_to_external_id) on a repl }); const reply = { ...msg, id: 'r1', quotedMessage: { id: 'orig', body: 'earlier' } } as IncomingMessage; await handleInbound(d, 'sess', 'Engine', reply); - assert.deepEqual(opts, { sourceId: 'r1', inReplyToExternalId: 'orig' }); + assert.deepEqual(opts, { sourceId: 'r1', inReplyToExternalId: 'orig', messageType: 'incoming' }); }); test('relays an inbound voice note as a Chatwoot voice message (#607)', async () => { diff --git a/chatwoot-adapter/inbound.ts b/chatwoot-adapter/inbound.ts index d8a8bc0..8e165d5 100644 --- a/chatwoot-adapter/inbound.ts +++ b/chatwoot-adapter/inbound.ts @@ -1,23 +1,19 @@ import type { IncomingMessage } from '../types/openwa'; -import type { ChatwootClient } from './chatwoot-client.ts'; -import type { MappingStore, ChatLink } from './mapping-store.ts'; -import type { KeyedAsyncLock } from './chat-lock.ts'; import { shouldRelayInbound } from './filters.ts'; - -export interface InboundDeps { - lock: KeyedAsyncLock; - client: ChatwootClient; - store: MappingStore; - instanceId: string; - relayGroups: boolean; - relayMedia: boolean; - log: (m: string, e?: unknown) => void; -} - -// WhatsApp → Chatwoot. Filter, then run the whole resolve+post+persist under the per-chat lock so two -// near-simultaneous first messages can't create duplicate Chatwoot contacts/conversations. Dedup is a -// mark-before-act check-and-set inside the lock (at-most-once; a failed post won't re-relay). -export async function handleInbound(deps: InboundDeps, sessionId: string, source: string, msg: IncomingMessage): Promise { +import { relayMessage, ensureConversation, refreshContactName, type InboundDeps } from './relay.ts'; +import { backfillHistory } from './backfill.ts'; + +export type { InboundDeps }; + +// WhatsApp → Chatwoot. Filter, then run resolve+post under the per-chat lock so two near-simultaneous +// first messages can't create duplicate Chatwoot contacts/conversations. Dedup is a mark-before-act +// check-and-set inside the lock (at-most-once; a failed post won't re-relay). +export async function handleInbound( + deps: InboundDeps, + sessionId: string, + source: string, + msg: IncomingMessage, +): Promise { if (!shouldRelayInbound(msg, source, deps.relayGroups)) return; await deps.lock.run(`${sessionId}:${msg.chatId}`, async () => { try { @@ -25,99 +21,37 @@ export async function handleInbound(deps: InboundDeps, sessionId: string, source // session so two tenants' WA message ids can't collide in the shared plugin store. if (await deps.store.hasSeen('wa', msg.id, sessionId)) return; await deps.store.markSeen('wa', msg.id, sessionId); - const conversationId = await resolveConversation(deps, sessionId, msg); - const content = prefixSender(msg); - // source_id lets a later reply thread against this message; in_reply_to_external_id forwards the - // quote context (#606) so a short reply like ".." keeps the bubble it answered. - const post = { sourceId: msg.id, inReplyToExternalId: msg.quotedMessage?.id }; - const isVoice = msg.type === 'voice'; - const isSticker = msg.type === 'sticker'; - if (msg.type === 'location' && msg.location) { - // A location carries no media blob; relay it as a text bubble with coordinates + a maps link the - // agent can open, threaded like any other message. - await deps.client.postText(conversationId, locationText(msg), post); - } else if (deps.relayMedia && msg.media?.data && !msg.media.omitted) { - await deps.client.postMedia( - conversationId, - content, - { - filename: isVoice ? 'voice.ogg' : isSticker ? 'sticker.webp' : msg.media.filename ?? 'file', - contentType: - msg.media.mimetype || (isVoice ? 'audio/ogg' : isSticker ? 'image/webp' : 'application/octet-stream'), - data: Buffer.from(msg.media.data, 'base64'), - }, - { ...post, isVoiceMessage: isVoice }, - ); - } else { - // No relayable blob (plain text, or media dropped/omitted). Never post an empty bubble for a - // media message — surface a short placeholder so the agent knows something arrived (#607). - await deps.client.postText(conversationId, msg.body?.trim() ? content : placeholderFor(msg), post); + const { conversationId, created } = await resolveConversation(deps, sessionId, msg); + // Lazy backfill: the first time this chat maps, replay its recent history (older messages, both + // directions, deduped) BEFORE posting this one — so the thread reads chronologically and this + // message's quote resolves against a just-posted source_id. This message is already markSeen, so + // backfill skips it and it posts last, below. + if (created && deps.backfillLimit > 0) { + await backfillHistory(deps, sessionId, msg.chatId, conversationId); } + await relayMessage(deps, conversationId, msg, 'incoming'); } catch (err) { deps.log('inbound relay failed', err); } }); } -function senderLabel(msg: IncomingMessage): string { - return msg.contact?.pushName || msg.senderPhone || msg.author || 'unknown'; -} - -function prefixSender(msg: IncomingMessage): string { - if (!msg.isGroup) return msg.body; - return `*${senderLabel(msg)}:* ${msg.body}`; -} - -// A shared location rendered for Chatwoot: a pin line (description/address when present) plus a link the -// agent can open (the message's own url, else a maps query). Group messages keep the sender prefix. -function locationText(msg: IncomingMessage): string { - const loc = msg.location!; - const link = loc.url || `https://maps.google.com/?q=${loc.latitude},${loc.longitude}`; - const label = [loc.description, loc.address].filter(Boolean).join(' — '); - const body = label ? `📍 ${label}\n${link}` : `📍 ${link}`; - return msg.isGroup ? `*${senderLabel(msg)}:* ${body}` : body; -} - -// A short stand-in for a bodyless message we couldn't relay as media (e.g. a voice note or sticker whose -// blob was omitted for size), so Chatwoot shows a meaningful line instead of an empty bubble. -function placeholderFor(msg: IncomingMessage): string { - if (msg.type === 'voice') return '🎤 Voice message'; - if (msg.type === 'sticker') return '🎨 Sticker'; - if (msg.media) return `📎 ${msg.media.filename ?? 'Attachment'}`; - return msg.body; -} - -async function resolveConversation(deps: InboundDeps, sessionId: string, msg: IncomingMessage): Promise { +async function resolveConversation( + deps: InboundDeps, + sessionId: string, + msg: IncomingMessage, +): Promise<{ conversationId: number; created: boolean }> { const existing = await deps.store.getByChat(sessionId, msg.chatId); // re-read inside the lock if (existing) { await refreshContactName(deps, sessionId, msg, existing); - return existing.conversationId; - } - const identifier = msg.chatId; // WA JID — individual @c.us/@lid or group JID (stable across @lid migration) - const name = msg.isGroup ? `Group ${msg.chatId}` : msg.contact?.pushName || msg.contact?.name || msg.senderPhone || identifier; - const phone = msg.isGroup ? undefined : msg.senderPhone ?? undefined; - const found = await deps.client.searchContact(identifier); - const contact = found?.sourceId - ? { id: found.id, sourceId: found.sourceId } - : await deps.client.createContact(identifier, name, phone); - const conversationId = - (await deps.client.findOpenConversation(contact.id)) ?? (await deps.client.createConversation(contact.id, contact.sourceId)); - await deps.store.link(sessionId, msg.chatId, deps.instanceId, { conversationId, contactId: contact.id, sourceId: contact.sourceId, name }); - return conversationId; -} - -// A 1:1 chat first seen from an @lid sender is seeded with the bare JID as its Chatwoot name (no pushName -// yet). Once a real pushName arrives, update the contact so agents see a human name instead of an id -// (#609 P1). Best-effort and only when the name actually changed — never blocks the relay, never overwrites -// a real name with a fallback (only pushName/name qualify, not senderPhone/JID). -async function refreshContactName(deps: InboundDeps, sessionId: string, msg: IncomingMessage, link: ChatLink): Promise { - if (msg.isGroup) return; // a group contact is named for the group, not whoever sent this message - const desired = msg.contact?.pushName || msg.contact?.name; - if (!desired || desired === link.name) return; - try { - await deps.client.updateContact(link.contactId, desired); - await deps.store.patch(sessionId, msg.chatId, { name: desired }); - } catch (err) { - deps.log('contact name refresh failed', err); + return { conversationId: existing.conversationId, created: false }; } + const name = msg.isGroup + ? `Group ${msg.chatId}` + : msg.contact?.pushName || msg.contact?.name || msg.senderPhone || msg.chatId; + const conversationId = await ensureConversation(deps, sessionId, msg.chatId, { + name, + phone: msg.isGroup ? undefined : msg.senderPhone ?? undefined, + }); + return { conversationId, created: true }; } diff --git a/chatwoot-adapter/index.ts b/chatwoot-adapter/index.ts index ad32443..bed87a5 100644 --- a/chatwoot-adapter/index.ts +++ b/chatwoot-adapter/index.ts @@ -3,6 +3,7 @@ import { ChatwootClient } from './chatwoot-client.ts'; import { MappingStore } from './mapping-store.ts'; import { KeyedAsyncLock } from './chat-lock.ts'; import { handleInbound } from './inbound.ts'; +import { backfillAllChats } from './backfill.ts'; import { handleOutbound } from './outbound.ts'; interface ChatwootFullConfig { @@ -12,6 +13,8 @@ interface ChatwootFullConfig { inboxId: number; relayGroups: boolean; relayMedia: boolean; + backfillLimit: number; + backfillAllOnce: boolean; } function readConfig(raw: Record): ChatwootFullConfig { @@ -38,7 +41,17 @@ function readConfig(raw: Record): ChatwootFullConfig { if (parsed.protocol !== 'https:' || parsed.username || parsed.password) { throw new Error('chatwoot-adapter: baseUrl must be an https URL without embedded credentials'); } - return { baseUrl, apiToken, accountId, inboxId, relayGroups: raw.relayGroups !== false, relayMedia: raw.relayMedia !== false }; + const rawLimit = Number(raw.backfillLimit); + return { + baseUrl, + apiToken, + accountId, + inboxId, + relayGroups: raw.relayGroups !== false, + relayMedia: raw.relayMedia !== false, + backfillLimit: Number.isFinite(rawLimit) ? Math.max(0, Math.trunc(rawLimit)) : 0, + backfillAllOnce: raw.backfillAllOnce === true, + }; } export default class ChatwootAdapter implements IPlugin { @@ -54,22 +67,26 @@ export default class ChatwootAdapter implements IPlugin { const msg = h.data as IncomingMessage; if (sessionId && msg) { const cfg = readConfig(ctx.config); + const deps = { + lock, + client: clientFor(), + store, + engine: ctx.engine, + instanceId: sessionId, + relayGroups: cfg.relayGroups, + relayMedia: cfg.relayMedia, + backfillLimit: cfg.backfillLimit, + backfillAllOnce: cfg.backfillAllOnce, + log: (m: string, e?: unknown) => ctx.logger.error(m, e), + }; // Fire-and-forget off the hook so a slow/failing Chatwoot API never blocks the WA pipeline. The // mapping mirror is keyed on sessionId (a session-scoped instance is 1:1 with its session). - void handleInbound( - { - lock, - client: clientFor(), - store, - instanceId: sessionId, - relayGroups: cfg.relayGroups, - relayMedia: cfg.relayMedia, - log: (m, e) => ctx.logger.error(m, e), - }, - sessionId, - h.source, - msg, - ).catch(e => ctx.logger.error('inbound hook failed', e)); + void handleInbound(deps, sessionId, h.source, msg).catch(e => ctx.logger.error('inbound hook failed', e)); + // Opt-in one-time bulk history sweep. Fired off the hook (outside handleInbound's per-chat lock), + // guarded internally so it runs once per session; a no-op after the first sweep completes. + if (cfg.backfillAllOnce && cfg.backfillLimit > 0) { + void backfillAllChats(deps, sessionId).catch(e => ctx.logger.error('bulk backfill failed', e)); + } } return { continue: true }; }); diff --git a/chatwoot-adapter/manifest.json b/chatwoot-adapter/manifest.json index 0798173..a80a500 100644 --- a/chatwoot-adapter/manifest.json +++ b/chatwoot-adapter/manifest.json @@ -1,7 +1,7 @@ { "id": "chatwoot-adapter", "name": "Chatwoot Adapter", - "version": "0.2.0", + "version": "0.3.0", "type": "extension", "main": "dist/index.js", "description": "Two-way sync between a WhatsApp session and a Chatwoot inbox: relays WhatsApp messages (1:1 and groups, with media) into Chatwoot as an API-channel inbox, sends agent replies back to WhatsApp, and hands a chat over to a human agent — silencing other OpenWA bots — when an agent takes it in Chatwoot. First consumer of the OpenWA Integration SDK v1; runs sandboxed in the plugin worker.", @@ -11,8 +11,8 @@ "repository": "https://github.com/rmyndharis/OpenWA-plugins", "keywords": ["chatwoot", "helpdesk", "inbox", "handover", "two-way", "agent", "whatsapp", "openwa"], "status": "beta", - "minOpenWAVersion": "0.8.3", - "testedOpenWAVersion": "0.8.3", + "minOpenWAVersion": "0.8.5", + "testedOpenWAVersion": "0.8.5", "sdkVersion": "1", "permissions": ["net:fetch", "conversation:send", "webhook:ingress"], "net": { "allow": [], "allowConfigHosts": ["baseUrl"] }, @@ -77,6 +77,18 @@ "title": "Relay inbound media to Chatwoot", "default": true, "description": "Upload inbound WhatsApp media (images, files, audio) to Chatwoot as attachments." + }, + "backfillLimit": { + "type": "number", + "title": "History backfill (messages per chat)", + "default": 0, + "description": "When a chat first opens in Chatwoot, import this many recent messages (both directions, with media) so agents see prior context. 0 disables it. Clamped to 100 host-side. Requires OpenWA 0.8.5+." + }, + "backfillAllOnce": { + "type": "boolean", + "title": "Bulk-import all chats once on setup", + "default": false, + "description": "In addition to the per-conversation backfill, run a one-time sweep that imports the history of every existing chat. Needs 'History backfill' above > 0. Runs once per session; large inboxes may take a while." } } } diff --git a/chatwoot-adapter/mapping-store.ts b/chatwoot-adapter/mapping-store.ts index 60ec51b..905b532 100644 --- a/chatwoot-adapter/mapping-store.ts +++ b/chatwoot-adapter/mapping-store.ts @@ -77,4 +77,12 @@ export class MappingStore { async markSeen(kind: 'wa' | 'cw', id: string, scope?: string): Promise { await this.storage.set(this.seenKey(kind, id, scope), 1); } + + // Durable run-once marker for the one-time bulk history sweep, per WA session. + async isBulkBackfilled(sessionId: string): Promise { + return Boolean(await this.storage.get(`backfill:all:${sessionId}`)); + } + async setBulkBackfilled(sessionId: string): Promise { + await this.storage.set(`backfill:all:${sessionId}`, 1); + } } diff --git a/chatwoot-adapter/relay.ts b/chatwoot-adapter/relay.ts new file mode 100644 index 0000000..e8c8e5d --- /dev/null +++ b/chatwoot-adapter/relay.ts @@ -0,0 +1,127 @@ +import type { IncomingMessage, PluginEngineReadCapability } from '../types/openwa'; +import type { ChatwootClient } from './chatwoot-client.ts'; +import type { MappingStore, ChatLink } from './mapping-store.ts'; +import type { KeyedAsyncLock } from './chat-lock.ts'; + +// Shared dependency bag for the inbound relay and the history backfill. Both render messages into +// Chatwoot and resolve conversations the same way; keeping the primitives here (a leaf module) lets +// `inbound.ts` and `backfill.ts` share them without an import cycle. +export interface InboundDeps { + lock: KeyedAsyncLock; + client: ChatwootClient; + store: MappingStore; + engine: PluginEngineReadCapability; + instanceId: string; + relayGroups: boolean; + relayMedia: boolean; + backfillLimit: number; + backfillAllOnce: boolean; + log: (m: string, e?: unknown) => void; +} + +function senderLabel(msg: IncomingMessage): string { + return msg.contact?.pushName || msg.senderPhone || msg.author || 'unknown'; +} + +function prefixSender(msg: IncomingMessage): string { + if (!msg.isGroup) return msg.body; + return `*${senderLabel(msg)}:* ${msg.body}`; +} + +// A shared location rendered for Chatwoot: a pin line (description/address when present) plus a link the +// agent can open (the message's own url, else a maps query). Group messages keep the sender prefix. +function locationText(msg: IncomingMessage): string { + const loc = msg.location!; + const link = loc.url || `https://maps.google.com/?q=${loc.latitude},${loc.longitude}`; + const label = [loc.description, loc.address].filter(Boolean).join(' — '); + const body = label ? `📍 ${label}\n${link}` : `📍 ${link}`; + return msg.isGroup ? `*${senderLabel(msg)}:* ${body}` : body; +} + +// A short stand-in for a bodyless message we couldn't relay as media (e.g. a voice note or sticker whose +// blob was omitted for size), so Chatwoot shows a meaningful line instead of an empty bubble. +function placeholderFor(msg: IncomingMessage): string { + if (msg.type === 'voice') return '🎤 Voice message'; + if (msg.type === 'sticker') return '🎨 Sticker'; + if (msg.media) return `📎 ${msg.media.filename ?? 'Attachment'}`; + return msg.body; +} + +// Render one WhatsApp message into Chatwoot (text / media / location / sticker / voice, with quote +// threading). Live inbound always passes 'incoming'; history backfill derives the direction per message. +export async function relayMessage( + deps: InboundDeps, + conversationId: number, + msg: IncomingMessage, + messageType: 'incoming' | 'outgoing', +): Promise { + const content = prefixSender(msg); + const post = { sourceId: msg.id, inReplyToExternalId: msg.quotedMessage?.id, messageType }; + const isVoice = msg.type === 'voice'; + const isSticker = msg.type === 'sticker'; + if (msg.type === 'location' && msg.location) { + await deps.client.postText(conversationId, locationText(msg), post); + } else if (deps.relayMedia && msg.media?.data && !msg.media.omitted) { + await deps.client.postMedia( + conversationId, + content, + { + filename: isVoice ? 'voice.ogg' : isSticker ? 'sticker.webp' : msg.media.filename ?? 'file', + contentType: + msg.media.mimetype || (isVoice ? 'audio/ogg' : isSticker ? 'image/webp' : 'application/octet-stream'), + data: Buffer.from(msg.media.data, 'base64'), + }, + { ...post, isVoiceMessage: isVoice }, + ); + } else { + await deps.client.postText(conversationId, msg.body?.trim() ? content : placeholderFor(msg), post); + } +} + +// Get-or-create the Chatwoot contact + conversation for a chat and mirror the mapping. Self-contained so +// the bulk backfill can call it from a chat summary (no triggering message), and idempotent so a chat +// already mapped by the live path is a no-op. +export async function ensureConversation( + deps: InboundDeps, + sessionId: string, + chatId: string, + meta: { name: string; phone?: string }, +): Promise { + const existing = await deps.store.getByChat(sessionId, chatId); + if (existing) return existing.conversationId; + const found = await deps.client.searchContact(chatId); + const contact = found?.sourceId + ? { id: found.id, sourceId: found.sourceId } + : await deps.client.createContact(chatId, meta.name, meta.phone); + const conversationId = + (await deps.client.findOpenConversation(contact.id)) ?? + (await deps.client.createConversation(contact.id, contact.sourceId)); + await deps.store.link(sessionId, chatId, deps.instanceId, { + conversationId, + contactId: contact.id, + sourceId: contact.sourceId, + name: meta.name, + }); + return conversationId; +} + +// A 1:1 chat first seen from an @lid sender is seeded with the bare JID as its Chatwoot name (no pushName +// yet). Once a real pushName arrives, update the contact so agents see a human name instead of an id +// (#609 P1). Best-effort and only when the name actually changed — never blocks the relay, never overwrites +// a real name with a fallback (only pushName/name qualify, not senderPhone/JID). +export async function refreshContactName( + deps: InboundDeps, + sessionId: string, + msg: IncomingMessage, + link: ChatLink, +): Promise { + if (msg.isGroup) return; // a group contact is named for the group, not whoever sent this message + const desired = msg.contact?.pushName || msg.contact?.name; + if (!desired || desired === link.name) return; + try { + await deps.client.updateContact(link.contactId, desired); + await deps.store.patch(sessionId, msg.chatId, { name: desired }); + } catch (err) { + deps.log('contact name refresh failed', err); + } +} diff --git a/plugins.json b/plugins.json index df78ff2..40de418 100644 --- a/plugins.json +++ b/plugins.json @@ -369,7 +369,7 @@ { "id": "chatwoot-adapter", "name": "Chatwoot Adapter", - "version": "0.2.0", + "version": "0.3.0", "type": "extension", "status": "beta", "description": "Two-way sync between a WhatsApp session and a Chatwoot inbox: relays WhatsApp messages (1:1 and groups, with media) into Chatwoot as an API-channel inbox, sends agent replies back to WhatsApp, and hands a chat over to a human agent — silencing other OpenWA bots — when an agent takes it in Chatwoot. First consumer of the OpenWA Integration SDK v1; runs sandboxed in the plugin worker.", @@ -385,13 +385,13 @@ "whatsapp", "openwa" ], - "minOpenWAVersion": "0.8.3", - "testedOpenWAVersion": "0.8.3", + "minOpenWAVersion": "0.8.5", + "testedOpenWAVersion": "0.8.5", "releasedAt": "2026-07-03", "repoPath": "chatwoot-adapter", "repoUrl": "https://github.com/rmyndharis/OpenWA-plugins", "homepage": "https://github.com/rmyndharis/OpenWA-plugins/tree/main/chatwoot-adapter", - "download": "https://github.com/rmyndharis/OpenWA-plugins/releases/download/chatwoot-adapter-v0.2.0/chatwoot-adapter.zip" + "download": "https://github.com/rmyndharis/OpenWA-plugins/releases/download/chatwoot-adapter-v0.3.0/chatwoot-adapter.zip" }, { "id": "faq-bot", diff --git a/types/openwa.d.ts b/types/openwa.d.ts index 192a2a6..7488c96 100644 --- a/types/openwa.d.ts +++ b/types/openwa.d.ts @@ -54,12 +54,23 @@ export interface PluginMessagingCapability { reply(sessionId: string, chatId: string, quotedMessageId: string, text: string): Promise; } +export interface ChatSummary { + id: string; + name: string; + isGroup: boolean; + unreadCount: number; + timestamp: number; + lastMessage?: string; +} + export interface PluginEngineReadCapability { getGroupInfo(sessionId: string, groupId: string): Promise; getContacts(sessionId: string): Promise; getContactById(sessionId: string, contactId: string): Promise; checkNumberExists(sessionId: string, phone: string): Promise; getChats(sessionId: string): Promise; + /** Recent messages for a chat, both directions (v0.8.5+). The host clamps `limit` (max 100). */ + getChatHistory(sessionId: string, chatId: string, limit?: number, includeMedia?: boolean): Promise; } // ── v0.7: host-proxied, SSRF-guarded outbound HTTP ──────────────────────────────────────────────