Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions backend/integrations/whatsapp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -340,10 +340,11 @@ export async function startWhatsApp(
resolvedText = `[image] ${description}`;
log.info(`Described image: ${description.slice(0, 80)}`);
} else {
// Caption failed (vision unavailable, refusal, error). Keep a neutral
// marker so downstream reasoning sees an image arrived without
// ingesting fabricated/refusal content as if the sender wrote it.
resolvedText = "[image — caption failed]";
// Caption failed (vision unavailable, refusal, error). Use a clean
// image marker — no error preamble — so downstream sees that an
// image arrived without polluting active-thread topics with text
// that resembles prompt-injection ("caption failed" / refusal text).
resolvedText = "[image]";
}
}

Expand Down
24 changes: 24 additions & 0 deletions backend/memory/working-memory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -181,19 +181,42 @@ export function populateTemporalContext(wm: WorkingMemory): void {

// ── Conversation Thread Tracking ──

/**
* True if `text` is a bare media marker (e.g. "[image]") with no caption —
* not useful as a thread topic. Captioned images are "[image] <description>"
* and remain valid topics.
*/
function isBareMediaMarker(text: string): boolean {
const t = text.trim();
return t === "[image]" || t === "[voice]" || t === "[document]";
}

export function updateConversationThreads(wm: WorkingMemory, observations: Observation[]): void {
const now = Date.now();
const STALE_THRESHOLD = 48 * 60 * 60 * 1000; // 48 hours

// Clean any persisted "caption failed" stub topics from prior runs so the
// active-thread surface stops surfacing them on every reflect/think tick.
for (const thread of wm.conversationThreads) {
if (thread.topic && /caption\s*failed/i.test(thread.topic)) {
thread.topic = "(image)";
}
}

for (const obs of observations) {
if (!obs.sender) continue;

// For DMs, key by the chat counterpart (chatJid), not the sender — so both
// incoming and outgoing messages map to the same thread.
const key = obs.isGroup ? `group:${obs.groupName || obs.senderJid}` : `dm:${obs.chatJid || obs.senderJid}`;
let thread = wm.conversationThreads.find(t => t.id === key);
const bareMedia = isBareMediaMarker(obs.text);

if (!thread) {
// Skip thread creation for bare-media-only observations: an uncaptioned
// image from a known contact shouldn't populate the active-thread surface
// since the topic would just be "[image]" with no real conversation content.
if (bareMedia) continue;
thread = {
id: key,
participants: [obs.sender],
Expand All @@ -208,6 +231,7 @@ export function updateConversationThreads(wm: WorkingMemory, observations: Obser
thread.lastMessageAt = obs.timestamp;
thread.messageCount++;
thread.status = "active";
// Don't overwrite an informative topic with a bare media marker.

if (!thread.participants.includes(obs.sender)) {
thread.participants.push(obs.sender);
Expand Down
36 changes: 33 additions & 3 deletions backend/observer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,47 @@ const log = createLogger("observer");

/**
* If an image-prefixed observation carries a vision-LLM refusal as its
* "caption", replace it with a neutral marker. Refusal text masquerading as
* caption pollutes the observation stream and resembles prompt-injection.
* "caption", replace it with a clean image marker. Refusal text masquerading
* as caption pollutes the observation stream and resembles prompt-injection.
*
* The bare "[image]" marker is structured: downstream prompts (e.g.
* conversation-thread serialization) recognise it and skip it instead of
* surfacing a misleading topic.
*/
function sanitizeImageCaption(text: string): string {
// Strip any legacy "[image — caption failed]" stubs that may still live in
// observations.jsonl or be replayed from older code paths.
if (/^\[image\s*[—\-]\s*caption\s*failed\]\s*$/i.test(text.trim())) {
return "[image]";
}
if (!text.startsWith("[image]")) return text;
const caption = text.slice("[image]".length).trim();
if (caption.length > 0 && isVisionRefusal(caption)) {
return "[image — caption failed]";
return "[image]";
}
return text;
}

/**
* Defensive runtime assertion: no observation text should match /caption failed/i.
* Such text was the legacy stub for vision-LLM failures and pollutes the
* active-thread surface (n_cba1a7f0). If we ever see one slip through, strip
* it back to a clean image marker and log so we can trace the writer.
*
* The one legitimate exception is intentional injection-detection logging,
* which writes elsewhere and never lands in obs.text.
*/
function assertNoCaptionFailedStub(obs: Observation): void {
if (!obs.text || !/caption\s*failed/i.test(obs.text)) return;
log(`assertion: stripping legacy 'caption failed' stub from observation (sender=${obs.sender}, source=${obs.source || "whatsapp"}, text="${obs.text.slice(0, 80)}")`);
obs.text = obs.text.replace(/\[image\s*[—\-]\s*caption\s*failed\]/gi, "[image]").trim();
if (/caption\s*failed/i.test(obs.text)) {
// Stub didn't match the bracketed form — replace any remaining occurrence wholesale.
obs.text = obs.text.replace(/caption\s*failed/gi, "").trim();
}
if (!obs.text) obs.text = "[image]";
}


const OBS_FILE = `${BRAIN_DIR}/observations.jsonl`;
const RETENTION_DAYS = Number(process.env.BRAIN_OBSERVATION_DAYS ?? 7);
Expand Down Expand Up @@ -192,6 +221,7 @@ export function recordObservation(obs: Observation): void {
// before it lands in the observations file or downstream pipelines.
if (obs.text) {
obs.text = sanitizeImageCaption(obs.text);
assertNoCaptionFailedStub(obs);
}

const key = getObservationKey(obs);
Expand Down