From 60972b608b38c15185b5ed5055eae76af021bc3a Mon Sep 17 00:00:00 2001 From: Patrick Sullivan Date: Thu, 2 Jul 2026 08:49:16 -0700 Subject: [PATCH 1/2] fix: repair image handling in both directions Outbound: OpenAI retired dall-e-3, so every generate_image call was failing with a 400. Switch to gpt-image-1, which returns base64 instead of a hosted URL - store the bytes in an in-memory store and serve them at GET /images/:id so the Linq API has a URL to download from. The public base URL comes from PUBLIC_BASE_URL or is learned from incoming webhook Host headers. Inbound: images over Claude's 8000px dimension limit (long screenshots) 400'd the whole request and the user got no reply. Fetch incoming images and downscale oversized ones to 2048px JPEG via sharp before sending to Claude; normal images still pass through as URL sources. Co-Authored-By: Claude Fable 5 --- .env.example | 4 + package-lock.json | 461 +++++++++++++++++++++++++++++++++++++++- package.json | 3 +- src/claude/client.ts | 89 ++++++-- src/index.ts | 23 ++ src/state/imageStore.ts | 56 +++++ 6 files changed, 612 insertions(+), 24 deletions(-) create mode 100644 src/state/imageStore.ts diff --git a/.env.example b/.env.example index cd9a3a3..89373b9 100644 --- a/.env.example +++ b/.env.example @@ -11,6 +11,10 @@ LINQ_API_BASE_URL=https://api.linqapp.com/api/partner/v3 # Server PORT=3000 +# Public base URL for serving generated images (optional - if unset, it's +# learned from incoming webhook requests, e.g. your ngrok or App Runner URL) +PUBLIC_BASE_URL= + # Linq phone numbers this bot runs on (comma-separated, supports multiple) LINQ_AGENT_BOT_NUMBERS=+14155551234 diff --git a/package-lock.json b/package-lock.json index e80a493..bd5b7df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,8 @@ "@aws-sdk/lib-dynamodb": "^3.971.0", "dotenv": "^17.2.3", "express": "^4.21.0", - "openai": "^6.16.0" + "openai": "^6.16.0", + "sharp": "^0.33.5" }, "devDependencies": { "@types/express": "^5.0.0", @@ -1125,6 +1126,15 @@ "node": ">=18.0.0" } }, + "node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.27.2", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.2.tgz", @@ -1541,6 +1551,348 @@ "node": ">=18" } }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", + "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", + "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", + "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", + "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", + "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", + "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz", + "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", + "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz", + "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz", + "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", + "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.0.5" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", + "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz", + "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", + "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz", + "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz", + "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz", + "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==", + "cpu": [ + "wasm32" + ], + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.2.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz", + "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", + "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@smithy/abort-controller": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.2.8.tgz", @@ -2297,6 +2649,43 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -2373,6 +2762,14 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "engines": { + "node": ">=8" + } + }, "node_modules/dotenv": { "version": "17.2.3", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", @@ -2807,6 +3204,11 @@ "node": ">= 0.10" } }, + "node_modules/is-arrayish": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.4.tgz", + "integrity": "sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA==" + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -3070,6 +3472,17 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/send": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", @@ -3117,6 +3530,44 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, + "node_modules/sharp": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", + "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", + "hasInstallScript": true, + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.3", + "semver": "^7.6.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.33.5", + "@img/sharp-darwin-x64": "0.33.5", + "@img/sharp-libvips-darwin-arm64": "1.0.4", + "@img/sharp-libvips-darwin-x64": "1.0.4", + "@img/sharp-libvips-linux-arm": "1.0.5", + "@img/sharp-libvips-linux-arm64": "1.0.4", + "@img/sharp-libvips-linux-s390x": "1.0.4", + "@img/sharp-libvips-linux-x64": "1.0.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", + "@img/sharp-libvips-linuxmusl-x64": "1.0.4", + "@img/sharp-linux-arm": "0.33.5", + "@img/sharp-linux-arm64": "0.33.5", + "@img/sharp-linux-s390x": "0.33.5", + "@img/sharp-linux-x64": "0.33.5", + "@img/sharp-linuxmusl-arm64": "0.33.5", + "@img/sharp-linuxmusl-x64": "0.33.5", + "@img/sharp-wasm32": "0.33.5", + "@img/sharp-win32-ia32": "0.33.5", + "@img/sharp-win32-x64": "0.33.5" + } + }, "node_modules/side-channel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", @@ -3185,6 +3636,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/simple-swizzle": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.4.tgz", + "integrity": "sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw==", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", diff --git a/package.json b/package.json index d970567..a1cb5de 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,8 @@ "@aws-sdk/lib-dynamodb": "^3.971.0", "dotenv": "^17.2.3", "express": "^4.21.0", - "openai": "^6.16.0" + "openai": "^6.16.0", + "sharp": "^0.33.5" }, "devDependencies": { "@types/express": "^5.0.0", diff --git a/src/claude/client.ts b/src/claude/client.ts index ea1cf6c..e43c1f1 100644 --- a/src/claude/client.ts +++ b/src/claude/client.ts @@ -1,6 +1,8 @@ import Anthropic from '@anthropic-ai/sdk'; import OpenAI from 'openai'; +import sharp from 'sharp'; import { getConversation, addMessage, clearConversation, getUserProfile, setUserName, addUserFact, clearUserProfile, UserProfile, StoredMessage } from '../state/conversation.js'; +import { storeImage } from '../state/imageStore.js'; const client = new Anthropic(); const openai = new OpenAI(); @@ -285,7 +287,7 @@ const REMEMBER_USER_TOOL: Anthropic.Tool = { const GENERATE_IMAGE_TOOL: Anthropic.Tool = { name: 'generate_image', - description: 'Generate an image using DALL-E. Use when the user asks you to create, draw, generate, or make an image/picture/photo. Expand their request into a detailed prompt for better results. IMPORTANT: You MUST also write a brief text message (like "on it, making that corgi now" or "lemme draw that for u") - this message will be sent BEFORE the image starts generating so the user knows something is happening.', + description: 'Generate an image. Use when the user asks you to create, draw, generate, or make an image/picture/photo. Expand their request into a detailed prompt for better results. IMPORTANT: You MUST also write a brief text message (like "on it, making that corgi now" or "lemme draw that for u") - this message will be sent BEFORE the image starts generating so the user knows something is happening.', input_schema: { type: 'object' as const, properties: { @@ -300,7 +302,7 @@ const GENERATE_IMAGE_TOOL: Anthropic.Tool = { const SET_GROUP_ICON_TOOL: Anthropic.Tool = { name: 'set_group_chat_icon', - description: 'Set the group chat icon/photo using a DALL-E generated image. ONLY use in group chats when someone explicitly asks to set/change the group icon/photo/picture. Expand their request into a detailed prompt. IMPORTANT: You MUST also write a brief text message acknowledging the request.', + description: 'Set the group chat icon/photo using a generated image. ONLY use in group chats when someone explicitly asks to set/change the group icon/photo/picture. Expand their request into a detailed prompt. IMPORTANT: You MUST also write a brief text message acknowledging the request.', input_schema: { type: 'object' as const, properties: { @@ -366,31 +368,79 @@ export interface AudioInput { mimeType: string; } -// Generate an image using OpenAI DALL-E API +// Generate an image using OpenAI's image API. gpt-image models return base64 +// only (no hosted URL), so we stash the bytes in the image store and return a +// URL served by this app that Linq can download from. export async function generateImage(prompt: string): Promise { try { - console.log(`[claude] Generating image with DALL-E: "${prompt.substring(0, 50)}..."`); + console.log(`[claude] Generating image with gpt-image-1: "${prompt.substring(0, 50)}..."`); const response = await openai.images.generate({ - model: 'dall-e-3', + model: 'gpt-image-1', prompt: prompt, n: 1, size: '1024x1024', - quality: 'standard', + quality: 'medium', }); - const imageUrl = response.data?.[0]?.url; + const b64 = response.data?.[0]?.b64_json; + if (!b64) { + console.error('[claude] No image data in gpt-image-1 response'); + return null; + } + + const imageUrl = storeImage(Buffer.from(b64, 'base64'), 'image/png'); if (imageUrl) { - console.log(`[claude] Image generated: ${imageUrl.substring(0, 50)}...`); - return imageUrl; + console.log(`[claude] Image generated: ${imageUrl}`); } - console.error('[claude] No image URL in DALL-E response'); - return null; + return imageUrl; } catch (error) { - console.error('[claude] DALL-E error:', error); + console.error('[claude] Image generation error:', error); return null; } } +// Claude API limits: max 8000px on any dimension, ~5MB per image. iPhone +// screenshots of long threads regularly blow past 8000px and previously +// 400'd the whole request. Oversized images get downscaled and re-encoded; +// everything else passes through as a URL source like before. +const MAX_IMAGE_DIMENSION = 7500; +const MAX_IMAGE_BYTES = 4.5 * 1024 * 1024; +const RESIZE_TARGET = 2048; + +async function prepareImageBlock(image: ImageInput): Promise { + const urlBlock: Anthropic.ImageBlockParam = { + type: 'image', + source: { type: 'url', url: image.url }, + }; + + try { + const response = await fetch(image.url); + if (!response.ok) throw new Error(`fetch failed: ${response.status}`); + const buffer = Buffer.from(await response.arrayBuffer()); + + const metadata = await sharp(buffer).metadata(); + const maxDim = Math.max(metadata.width ?? 0, metadata.height ?? 0); + if (maxDim <= MAX_IMAGE_DIMENSION && buffer.byteLength <= MAX_IMAGE_BYTES) { + return urlBlock; + } + + console.log(`[claude] Resizing oversized image (${metadata.width}x${metadata.height}, ${Math.round(buffer.byteLength / 1024)}KB): ${image.url.substring(0, 50)}...`); + const resized = await sharp(buffer) + .rotate() // apply EXIF orientation before resizing + .resize(RESIZE_TARGET, RESIZE_TARGET, { fit: 'inside', withoutEnlargement: true }) + .jpeg({ quality: 85 }) + .toBuffer(); + + return { + type: 'image', + source: { type: 'base64', media_type: 'image/jpeg', data: resized.toString('base64') }, + }; + } catch (error) { + console.error('[claude] Image prep failed, passing URL through:', error); + return urlBlock; + } +} + // Transcribe audio using OpenAI Whisper API async function transcribeAudio(url: string): Promise { try { @@ -504,17 +554,12 @@ export async function chat(chatId: string, userMessage: string, images: ImageInp // Build message content (text + images + audio) const messageContent: Anthropic.ContentBlockParam[] = []; - // Add images first - for (const image of images) { - messageContent.push({ - type: 'image', - source: { - type: 'url', - url: image.url, - }, - }); + // Add images first (resized if they exceed Claude's limits) + const imageBlocks = await Promise.all(images.map(image => { console.log(`[claude] Including image: ${image.url.substring(0, 50)}...`); - } + return prepareImageBlock(image); + })); + messageContent.push(...imageBlocks); // Transcribe audio files and add as text context const transcriptions: string[] = []; diff --git a/src/index.ts b/src/index.ts index 037df68..e2020cd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,7 @@ import { createWebhookHandler } from './webhook/handler.js'; import { sendMessage, markAsRead, startTyping, sendReaction, shareContactCard, getChat, renameGroupChat, setGroupChatIcon, removeParticipant } from './linq/client.js'; import { chat, getGroupChatAction, getTextForEffect, generateImage } from './claude/client.js'; import { getUserProfile, addMessage } from './state/conversation.js'; +import { getImage, setPublicBaseUrl } from './state/imageStore.js'; import { processResponse } from './text/decorations.js'; // Track message count per chat for contact card sharing @@ -21,6 +22,28 @@ app.get('/health', (_req, res) => { res.json({ status: 'ok', timestamp: new Date().toISOString() }); }); +// Serve generated images so Linq can download them (gpt-image models return +// base64, not a hosted URL like DALL-E did) +app.get('/images/:id', (req, res) => { + const image = getImage(req.params.id); + if (!image) { + res.status(404).json({ error: 'not found or expired' }); + return; + } + res.set('Content-Type', image.contentType).send(image.buffer); +}); + +// Learn our public URL from incoming webhook requests (used to build +// generated-image URLs; PUBLIC_BASE_URL env var overrides) +app.use('/webhook', (req, _res, next) => { + const host = req.get('host'); + if (host) { + const proto = req.get('x-forwarded-proto')?.split(',')[0] ?? req.protocol; + setPublicBaseUrl(`${proto}://${host}`); + } + next(); +}); + // Webhook endpoint for Linq app.post( '/webhook', diff --git a/src/state/imageStore.ts b/src/state/imageStore.ts new file mode 100644 index 0000000..1516967 --- /dev/null +++ b/src/state/imageStore.ts @@ -0,0 +1,56 @@ +import { randomUUID } from 'crypto'; + +// In-memory store for generated images. The gpt-image models return base64 +// (no hosted URL like DALL-E did), but the Linq API needs a fetchable URL for +// media parts - so we hold the bytes briefly and serve them from this app. +// Linq downloads media as soon as the message is sent, so a short TTL is fine. +// Only safe while the app runs as a single instance. +const TTL_MS = 15 * 60 * 1000; + +interface StoredImage { + buffer: Buffer; + contentType: string; + expiresAt: number; +} + +const store = new Map(); + +// Base URL images are served from. PUBLIC_BASE_URL wins if set; otherwise +// learned from the Host header of incoming webhook requests (works for both +// App Runner and ngrok without extra config). +let publicBaseUrl: string | null = process.env.PUBLIC_BASE_URL?.replace(/\/+$/, '') || null; + +export function setPublicBaseUrl(url: string): void { + if (!process.env.PUBLIC_BASE_URL) { + publicBaseUrl = url.replace(/\/+$/, ''); + } +} + +export function storeImage(buffer: Buffer, contentType: string): string | null { + if (!publicBaseUrl) { + console.error('[imageStore] No public base URL known yet - cannot serve generated image'); + return null; + } + cleanup(); + const id = randomUUID(); + store.set(id, { buffer, contentType, expiresAt: Date.now() + TTL_MS }); + console.log(`[imageStore] Stored image ${id} (${Math.round(buffer.byteLength / 1024)}KB, ${store.size} in store)`); + return `${publicBaseUrl}/images/${id}`; +} + +export function getImage(id: string): { buffer: Buffer; contentType: string } | null { + const entry = store.get(id); + if (!entry) return null; + if (entry.expiresAt < Date.now()) { + store.delete(id); + return null; + } + return { buffer: entry.buffer, contentType: entry.contentType }; +} + +function cleanup(): void { + const now = Date.now(); + for (const [id, entry] of store) { + if (entry.expiresAt < now) store.delete(id); + } +} From 9c4af4750aa0c60c3833ef167ed3f94e7cf95ef9 Mon Sep 17 00:00:00 2001 From: Patrick Sullivan Date: Thu, 2 Jul 2026 09:32:22 -0700 Subject: [PATCH 2/2] fix: never reply with a reaction alone - always follow up with text Over the last 72h, 26% of Sonnet's replies were reaction-only with no text (e.g. thumbs-upping 'how can I be an expert in geoguessr?'), despite prompt rules against it. In group chats the Haiku classifier already routes reaction-worthy messages to a quick reaction before Sonnet is invoked, so a reaction-only result from chat() is a misfire in any chat type. Two changes: - Tighten the reaction rules in the system prompt (reactions are a bonus on top of text, never a replacement; drop the 'like = no text needed' guidance that invited this) - Deterministic guardrail: if Sonnet returns a reaction with no text and no other action, continue the tool loop with a tool_result asking for the actual text reply Co-Authored-By: Claude Fable 5 --- src/claude/client.ts | 54 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/src/claude/client.ts b/src/claude/client.ts index e43c1f1..88ad02a 100644 --- a/src/claude/client.ts +++ b/src/claude/client.ts @@ -99,16 +99,15 @@ You can use standard tapbacks OR any custom emoji: Custom emoji reactions are more expressive and fun - use them when a standard tapback doesn't capture the vibe! CRITICAL REACTION RULES: -1. DEFAULT to text responses - reactions are supplementary, not primary -2. NEVER react without also sending a text response unless it's truly just an acknowledgment +1. ALWAYS send a text response - reactions are a bonus ON TOP of text, never a replacement for it +2. A reaction is NEVER a valid answer to a question. If someone asks you anything ("how do I...", "what is...", "can you..."), you MUST reply with text 3. If you've reacted recently, DO NOT react again - respond with text instead 4. If someone is asking you something or talking to you, RESPOND WITH TEXT -5. Reactions alone can feel dismissive - when in doubt, send text +5. Reactions alone feel dismissive - like being left on read. When in doubt, send text 6. NEVER write "[reacted with ...]" in your text - that's just a system marker in history! When you use send_reaction, just send normal text alongside it -When to use reactions (sparingly): +When to use reactions (sparingly, and almost always WITH a text response): - love: Heartfelt news (promotions, engagements) -- like: Simple acknowledgment when no text response needed - laugh: Genuinely funny messages - Custom emoji: When you want to be more expressive (🔥 for something cool, 💀 for something hilarious, etc.) @@ -706,7 +705,50 @@ export async function chat(chatId: string, userMessage: string, images: ImageInp } } - const textResponse = textParts.length > 0 ? textParts.join('\n') : null; + let textResponse: string | null = textParts.length > 0 ? textParts.join('\n') : null; + + // Guardrail: Sonnet sometimes tapbacks a message instead of answering it, + // despite the prompt rules (e.g. thumbs-upping "how can I be an expert in + // geoguessr?"). A reaction alone is never a sufficient reply from chat() - + // in group chats the Haiku classifier already routes reaction-worthy + // messages to a quick reaction before Sonnet is invoked. Continue the + // tool loop and require a text reply. + if (!textResponse && reaction && !effect && !generatedImage && !groupChatIcon && !renameChat && !removeMember) { + console.log('[claude] Reaction-only response - requesting text follow-up'); + try { + const toolResults: Anthropic.ToolResultBlockParam[] = response.content + .filter((block): block is Anthropic.ToolUseBlock => block.type === 'tool_use') + .map(block => ({ + type: 'tool_result' as const, + tool_use_id: block.id, + content: block.name === 'send_reaction' + ? 'Reaction sent. A reaction alone is not a sufficient reply - now write your text response to their message. Do not use any more tools.' + : 'Done.', + })); + const followUp = await client.messages.create({ + model: 'claude-sonnet-4-6', + max_tokens: 1024, + system: buildSystemPrompt(chatContext), + tools, + messages: [ + ...formattedHistory, + { role: 'user', content: messageContent }, + { role: 'assistant', content: response.content }, + { role: 'user', content: toolResults }, + ], + }); + const followUpText = followUp.content + .map(block => (block.type === 'text' ? block.text : '')) + .filter(Boolean) + .join('\n'); + if (followUpText) { + textResponse = followUpText; + console.log('[claude] Got text follow-up after reaction-only response'); + } + } catch (error) { + console.error('[claude] Text follow-up failed (keeping reaction-only):', error); + } + } // Add assistant response to history (only text part, strip --- delimiters for cleaner context) // Note: image generation is handled separately in index.ts after sending text first