From 61aa062e36dd000eecae9873e22e814d689147cf Mon Sep 17 00:00:00 2001 From: Mathias Sacrez Date: Mon, 22 Jun 2026 11:14:35 +0200 Subject: [PATCH 1/2] Add clipboard image paste to annotation comments The annotation comment box only accepted text. You can now paste an image from the clipboard (a screenshot, a design reference, etc.) into both the element and freeform-rectangle comment boxes. Pasted images are saved next to the auto-captured screenshot under .moat/screenshots/-paste-N., shown as removable thumbnails in the comment box, and their paths are appended to the task comment (plus an `attachments` array on the task) so the agent processing /bridge picks them up. Submitting with only an image and no text is now allowed. Normal text paste is unaffected (we only intercept when the clipboard carries an image). --- chrome-extension/content_script.js | 117 +++++++++++++++++++++++++++-- chrome-extension/moat.css | 46 ++++++++++++ 2 files changed, 155 insertions(+), 8 deletions(-) diff --git a/chrome-extension/content_script.js b/chrome-extension/content_script.js index a2a73ed..51a254b 100644 --- a/chrome-extension/content_script.js +++ b/chrome-extension/content_script.js @@ -413,6 +413,85 @@ } } + // Save clipboard-pasted reference images to ./screenshots and return their paths. + async function savePastedImagesToFiles(annotation) { + if (!annotation.pastedImages?.length || !window.directoryHandle) { + return []; + } + const screenshotsDir = await window.directoryHandle.getDirectoryHandle('screenshots', { create: true }); + const paths = []; + for (let i = 0; i < annotation.pastedImages.length; i++) { + const match = /^data:(image\/[\w.+-]+);base64,(.*)$/s.exec(annotation.pastedImages[i]); + if (!match) continue; + const [, mime, base64] = match; + const ext = mime.split('/')[1].replace('jpeg', 'jpg').replace('svg+xml', 'svg'); + const bytes = Uint8Array.from(atob(base64), c => c.charCodeAt(0)); + const fileName = `${annotation.id}-paste-${i + 1}.${ext}`; + try { + const fileHandle = await screenshotsDir.getFileHandle(fileName, { create: true }); + const writable = await fileHandle.createWritable({ keepExistingData: false }); + await writable.write(new Blob([bytes], { type: mime })); + await writable.close(); + paths.push(`./screenshots/${fileName}`); + } catch (error) { + console.error('Moat: Failed to save pasted image:', error); + } + } + return paths; + } + + // Wire clipboard-image paste onto a comment box's textarea: store image data + // URLs on box.pastedImages and render removable thumbnails. Text paste is + // left untouched (we only intercept when the clipboard carries an image). + function enablePasteImages(box, textarea) { + box.pastedImages = []; + const strip = box.querySelector('.float-comment-attachments'); + + const render = () => { + if (!strip) return; + strip.replaceChildren(); + box.pastedImages.forEach((dataUrl, index) => { + const thumb = document.createElement('div'); + thumb.className = 'float-comment-thumb'; + + const img = document.createElement('img'); + img.src = dataUrl; + thumb.appendChild(img); + + const remove = document.createElement('button'); + remove.type = 'button'; + remove.className = 'float-comment-thumb-remove'; + remove.textContent = '×'; + remove.title = 'Remove'; + remove.addEventListener('click', () => { + box.pastedImages.splice(index, 1); + render(); + }); + thumb.appendChild(remove); + + strip.appendChild(thumb); + }); + }; + + textarea.addEventListener('paste', (e) => { + const items = e.clipboardData?.items; + if (!items) return; + const imageItems = [...items].filter(item => item.type.startsWith('image/')); + if (!imageItems.length) return; // no image on clipboard → normal text paste + e.preventDefault(); + for (const item of imageItems) { + const file = item.getAsFile(); + if (!file) continue; + const reader = new FileReader(); + reader.onload = () => { + box.pastedImages.push(reader.result); + render(); + }; + reader.readAsDataURL(file); + } + }); + } + /** * Capture screenshot using Chrome's native API and crop to specified region * @param {Object} captureArea - { x, y, width, height } in viewport/CSS pixels @@ -2910,6 +2989,16 @@ JSON stores relative paths like \`./screenshots/file.png\`, but actual files are console.log('📝 Moat: ===== STARTING ANNOTATION PROCESSING ====='); console.log('📝 Moat: Processing annotation:', annotation.elementLabel); console.log('📝 Moat: Annotation ID:', annotation.id); + + // Persist any clipboard-pasted reference images and reference them in the + // task comment, the field /bridge always reads, so the LLM picks them up. + if (annotation.pastedImages?.length) { + const refs = await savePastedImagesToFiles(annotation); + if (refs.length) { + annotation.attachments = refs; + annotation.content = `${annotation.content}\n\nPasted reference image(s):\n${refs.map(p => `- ${p}`).join('\n')}`.trim(); + } + } // Choose save system based on availability (Task 2.8: End-to-end flow) const canUseNew = canUseNewTaskSystem(); @@ -3214,11 +3303,12 @@ JSON stores relative paths like \`./screenshots/file.png\`, but actual files are commentBox.highlightedElement = element; // Store element reference commentBox.highlightOverlay = hoverOverlay; // Store overlay reference for cleanup commentBox.innerHTML = ` - +
@@ -3278,10 +3368,14 @@ JSON stores relative paths like \`./screenshots/file.png\`, but actual files are // Focus textarea setTimeout(() => textarea.focus(), 50); + + // Allow pasting reference images from the clipboard + enablePasteImages(commentBox, textarea); // Handle submit const handleSubmit = async () => { - const content = textarea.value.trim(); + const content = textarea.value.trim() + || (commentBox.pastedImages?.length ? 'See pasted reference image(s).' : ''); if (!content) return; const rect = element.getBoundingClientRect(); @@ -3339,6 +3433,7 @@ JSON stores relative paths like \`./screenshots/file.png\`, but actual files are hoverOverlay = null; } + annotation.pastedImages = commentBox.pastedImages || []; addToQueue(annotation); exitCommentMode(); }; @@ -3562,11 +3657,12 @@ JSON stores relative paths like \`./screenshots/file.png\`, but actual files are commentBox.screenshotData = screenshotData; commentBox.rectangleData = rectangleData; // Store rectangle data commentBox.innerHTML = ` - +
@@ -3619,10 +3715,14 @@ JSON stores relative paths like \`./screenshots/file.png\`, but actual files are // Focus textarea setTimeout(() => textarea.focus(), 50); + + // Allow pasting reference images from the clipboard + enablePasteImages(commentBox, textarea); // Handle submit const handleSubmit = async () => { - const content = textarea.value.trim(); + const content = textarea.value.trim() + || (commentBox.pastedImages?.length ? 'See pasted reference image(s).' : ''); if (!content) return; // Get rectangle data from comment box @@ -3677,6 +3777,7 @@ JSON stores relative paths like \`./screenshots/file.png\`, but actual files are exitDrawingMode(); removeDrawingCanvas(); + annotation.pastedImages = commentBox.pastedImages || []; addToQueue(annotation); removeCommentBox(); }; diff --git a/chrome-extension/moat.css b/chrome-extension/moat.css index b7f0380..7b45db6 100644 --- a/chrome-extension/moat.css +++ b/chrome-extension/moat.css @@ -447,6 +447,52 @@ body.float-drawing-mode * { border-color: var(--moat-accent); } +.float-comment-attachments { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-top: 8px; +} + +.float-comment-attachments:empty { + display: none; +} + +.float-comment-thumb { + position: relative; + width: 56px; + height: 56px; + border-radius: 6px; + overflow: hidden; + border: 1px solid var(--moat-border); +} + +.float-comment-thumb img { + width: 100%; + height: 100%; + object-fit: cover; + display: block; +} + +.float-comment-thumb-remove { + position: absolute; + top: 2px; + right: 2px; + width: 18px; + height: 18px; + padding: 0; + border: none; + border-radius: 50%; + background: rgba(0, 0, 0, 0.6); + color: white; + font-size: 13px; + line-height: 1; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; +} + .float-comment-actions { display: flex; justify-content: flex-end; From 778061c7574ec6c3173b7a47a608e1f02a3efd1c Mon Sep 17 00:00:00 2001 From: Mathias Sacrez Date: Mon, 22 Jun 2026 11:26:41 +0200 Subject: [PATCH 2/2] Persist pasted images as task attachments and tell the agent to open them Pasted reference images were only referenced in the task comment text, so whether the processing agent looked at them depended on it noticing the path. Make it reliable: - convertAnnotationToTask / createTaskObject now persist an `attachments` array (the ./screenshots/-paste-N paths) as a first-class task field in moat-tasks-detail.json, so the data is structured rather than buried in prose. - The bridge + workflow rule templates now instruct the agent to open every image in `attachments` before implementing, the same way it views `screenshotPath`. --- chrome-extension/content_script.js | 10 ++++++++++ chrome-extension/rules-templates/bridge.md | 1 + .../rules-templates/drawbridge-workflow.md | 3 +++ chrome-extension/utils/taskStore.js | 6 ++++++ 4 files changed, 20 insertions(+) diff --git a/chrome-extension/content_script.js b/chrome-extension/content_script.js index 51a254b..847c6ae 100644 --- a/chrome-extension/content_script.js +++ b/chrome-extension/content_script.js @@ -353,6 +353,11 @@ screenshotPath: screenshotPath || '' }; + // Pasted clipboard reference images (paths under ./screenshots), if any + if (annotation.attachments?.length) { + taskData.attachments = annotation.attachments; + } + // Add bounding box data for freeform rectangles (preserve all formats) if (annotation.boundingBox && annotation.boundingBox.type === 'freeform') { taskData.boundingBox = annotation.boundingBox; @@ -1453,6 +1458,7 @@ The \`.json\` file is the primary source of truth for all task information. You - \`selector\`: The precise CSS selector for the target element. - \`title\`, \`boundingRect\`: Context for locating the element. - \`screenshotPath\`: Path to the screenshot showing the user's annotation context. +- \`attachments\`: Optional paths to reference images the user pasted from the clipboard (e.g. a target design/mockup). Open each one before implementing, just like the screenshot. ### Task Dependency Detection @@ -1527,6 +1533,8 @@ Result: Task 1 must complete before Tasks 2 & 3 - Current state vs desired state - Layout and positioning context + **Pasted reference images**: If the task JSON has an \`attachments\` array, read each listed image as well. These are references the user pasted from their clipboard (a target design, an example to match) and must inform the implementation alongside the screenshot. + 3. **Validation Steps**: \`\`\` 📸 Viewing screenshot: ./screenshots/moat-1751940243108-aag80q4av.png @@ -2022,6 +2030,8 @@ Files to read: - **Human-readable**: \`moat-tasks.md\` - Task checklist - **Screenshots**: \`.moat/screenshots/\` - Visual context +**Pasted references**: A task's \`attachments\` array (if present) lists images the user pasted from their clipboard (\`./screenshots/...-paste-N.png\`). Open each one before implementing — they are reference images (a target design/example) and must guide the change alongside the screenshot. + **Important**: Always check \`.moat/\` subdirectory first before checking project root. ## ⚠️ CRITICAL: Status Lifecycle (MUST FOLLOW) diff --git a/chrome-extension/rules-templates/bridge.md b/chrome-extension/rules-templates/bridge.md index 8eacd19..46f2341 100644 --- a/chrome-extension/rules-templates/bridge.md +++ b/chrome-extension/rules-templates/bridge.md @@ -43,6 +43,7 @@ Files to read: - `comment`: User's instruction - `selector`: CSS selector for target element - `screenshotPath`: Visual context (resolve `./screenshots/` to `.moat/screenshots/`) + - `attachments`: Optional paths to reference images the user pasted from the clipboard — open each one (resolve `./screenshots/` the same way) before implementing, just like the screenshot - `status`: Current task status ("to do", "doing", "done") 2. **Analyze Dependencies**: Before starting, check if tasks reference each other: diff --git a/chrome-extension/rules-templates/drawbridge-workflow.md b/chrome-extension/rules-templates/drawbridge-workflow.md index b9d6888..ae9fdae 100644 --- a/chrome-extension/rules-templates/drawbridge-workflow.md +++ b/chrome-extension/rules-templates/drawbridge-workflow.md @@ -155,6 +155,7 @@ The `.json` file is the primary source of truth for all task information. You mu - `selector`: The precise CSS selector for the target element. - `title`, `boundingRect`: Context for locating the element. - `screenshotPath`: Path to the screenshot showing the user's annotation context. +- `attachments`: Optional paths to reference images the user pasted from the clipboard (e.g. a target design/mockup). Open each one before implementing, just like the screenshot. ### Task Dependency Detection @@ -229,6 +230,8 @@ Result: Task 1 must complete before Tasks 2 & 3 - Current state vs desired state - Layout and positioning context + **Pasted reference images**: If the task JSON has an `attachments` array, read each listed image as well. These are references the user pasted from their clipboard (a target design, an example to match) and must inform the implementation alongside the screenshot. + 3. **Screenshot Missing/Inaccessible**: ``` ⚠️ Screenshot not found: .moat/screenshots/moat-[id].png diff --git a/chrome-extension/utils/taskStore.js b/chrome-extension/utils/taskStore.js index 5a28a33..23c8e5c 100644 --- a/chrome-extension/utils/taskStore.js +++ b/chrome-extension/utils/taskStore.js @@ -49,6 +49,11 @@ function createTaskObject(taskData) { timestamp: Date.now() }; + // Preserve pasted clipboard reference image paths, if any + if (taskData.attachments?.length) { + task.attachments = taskData.attachments; + } + // Preserve bounding box data for freeform rectangles if (taskData.boundingBox) { task.boundingBox = taskData.boundingBox; @@ -170,6 +175,7 @@ class TaskStore { selector: taskData.selector || existingTask.selector, boundingRect: taskData.boundingRect || existingTask.boundingRect, screenshotPath: taskData.screenshotPath || existingTask.screenshotPath, + attachments: taskData.attachments || existingTask.attachments, lastModified: Date.now() }); console.log(`Updated existing task: ${existingTask.id}`);