diff --git a/.env.example b/.env.example index cd9a3a3..89373b9 100644 --- a/.env.example +++ b/.env.example @@ -11,6 +11,10 @@ LINQ_API_BASE_URL=https://api.linqapp.com/api/partner/v3 # Server PORT=3000 +# Public base URL for serving generated images (optional - if unset, it's +# learned from incoming webhook requests, e.g. your ngrok or App Runner URL) +PUBLIC_BASE_URL= + # Linq phone numbers this bot runs on (comma-separated, supports multiple) LINQ_AGENT_BOT_NUMBERS=+14155551234 diff --git a/package-lock.json b/package-lock.json index e80a493..bd5b7df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,8 @@ "@aws-sdk/lib-dynamodb": "^3.971.0", "dotenv": "^17.2.3", "express": "^4.21.0", - "openai": "^6.16.0" + "openai": "^6.16.0", + "sharp": "^0.33.5" }, "devDependencies": { "@types/express": "^5.0.0", @@ -1125,6 +1126,15 @@ "node": ">=18.0.0" } }, + "node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.27.2", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.2.tgz", @@ -1541,6 +1551,348 @@ "node": ">=18" } }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", + "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", + "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", + "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", + "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", + "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", + "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz", + "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", + "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz", + "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz", + "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", + "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.0.5" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", + "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz", + "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", + "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz", + "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz", + "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz", + "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==", + "cpu": [ + "wasm32" + ], + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.2.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz", + "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", + "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@smithy/abort-controller": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.2.8.tgz", @@ -2297,6 +2649,43 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -2373,6 +2762,14 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "engines": { + "node": ">=8" + } + }, "node_modules/dotenv": { "version": "17.2.3", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", @@ -2807,6 +3204,11 @@ "node": ">= 0.10" } }, + "node_modules/is-arrayish": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.4.tgz", + "integrity": "sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA==" + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -3070,6 +3472,17 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/send": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", @@ -3117,6 +3530,44 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, + "node_modules/sharp": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", + "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", + "hasInstallScript": true, + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.3", + "semver": "^7.6.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.33.5", + "@img/sharp-darwin-x64": "0.33.5", + "@img/sharp-libvips-darwin-arm64": "1.0.4", + "@img/sharp-libvips-darwin-x64": "1.0.4", + "@img/sharp-libvips-linux-arm": "1.0.5", + "@img/sharp-libvips-linux-arm64": "1.0.4", + "@img/sharp-libvips-linux-s390x": "1.0.4", + "@img/sharp-libvips-linux-x64": "1.0.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", + "@img/sharp-libvips-linuxmusl-x64": "1.0.4", + "@img/sharp-linux-arm": "0.33.5", + "@img/sharp-linux-arm64": "0.33.5", + "@img/sharp-linux-s390x": "0.33.5", + "@img/sharp-linux-x64": "0.33.5", + "@img/sharp-linuxmusl-arm64": "0.33.5", + "@img/sharp-linuxmusl-x64": "0.33.5", + "@img/sharp-wasm32": "0.33.5", + "@img/sharp-win32-ia32": "0.33.5", + "@img/sharp-win32-x64": "0.33.5" + } + }, "node_modules/side-channel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", @@ -3185,6 +3636,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/simple-swizzle": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.4.tgz", + "integrity": "sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw==", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", diff --git a/package.json b/package.json index d970567..a1cb5de 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,8 @@ "@aws-sdk/lib-dynamodb": "^3.971.0", "dotenv": "^17.2.3", "express": "^4.21.0", - "openai": "^6.16.0" + "openai": "^6.16.0", + "sharp": "^0.33.5" }, "devDependencies": { "@types/express": "^5.0.0", diff --git a/src/claude/client.ts b/src/claude/client.ts index ea1cf6c..88ad02a 100644 --- a/src/claude/client.ts +++ b/src/claude/client.ts @@ -1,6 +1,8 @@ import Anthropic from '@anthropic-ai/sdk'; import OpenAI from 'openai'; +import sharp from 'sharp'; import { getConversation, addMessage, clearConversation, getUserProfile, setUserName, addUserFact, clearUserProfile, UserProfile, StoredMessage } from '../state/conversation.js'; +import { storeImage } from '../state/imageStore.js'; const client = new Anthropic(); const openai = new OpenAI(); @@ -97,16 +99,15 @@ You can use standard tapbacks OR any custom emoji: Custom emoji reactions are more expressive and fun - use them when a standard tapback doesn't capture the vibe! CRITICAL REACTION RULES: -1. DEFAULT to text responses - reactions are supplementary, not primary -2. NEVER react without also sending a text response unless it's truly just an acknowledgment +1. ALWAYS send a text response - reactions are a bonus ON TOP of text, never a replacement for it +2. A reaction is NEVER a valid answer to a question. If someone asks you anything ("how do I...", "what is...", "can you..."), you MUST reply with text 3. If you've reacted recently, DO NOT react again - respond with text instead 4. If someone is asking you something or talking to you, RESPOND WITH TEXT -5. Reactions alone can feel dismissive - when in doubt, send text +5. Reactions alone feel dismissive - like being left on read. When in doubt, send text 6. NEVER write "[reacted with ...]" in your text - that's just a system marker in history! When you use send_reaction, just send normal text alongside it -When to use reactions (sparingly): +When to use reactions (sparingly, and almost always WITH a text response): - love: Heartfelt news (promotions, engagements) -- like: Simple acknowledgment when no text response needed - laugh: Genuinely funny messages - Custom emoji: When you want to be more expressive (🔥 for something cool, 💀 for something hilarious, etc.) @@ -285,7 +286,7 @@ const REMEMBER_USER_TOOL: Anthropic.Tool = { const GENERATE_IMAGE_TOOL: Anthropic.Tool = { name: 'generate_image', - description: 'Generate an image using DALL-E. Use when the user asks you to create, draw, generate, or make an image/picture/photo. Expand their request into a detailed prompt for better results. IMPORTANT: You MUST also write a brief text message (like "on it, making that corgi now" or "lemme draw that for u") - this message will be sent BEFORE the image starts generating so the user knows something is happening.', + description: 'Generate an image. Use when the user asks you to create, draw, generate, or make an image/picture/photo. Expand their request into a detailed prompt for better results. IMPORTANT: You MUST also write a brief text message (like "on it, making that corgi now" or "lemme draw that for u") - this message will be sent BEFORE the image starts generating so the user knows something is happening.', input_schema: { type: 'object' as const, properties: { @@ -300,7 +301,7 @@ const GENERATE_IMAGE_TOOL: Anthropic.Tool = { const SET_GROUP_ICON_TOOL: Anthropic.Tool = { name: 'set_group_chat_icon', - description: 'Set the group chat icon/photo using a DALL-E generated image. ONLY use in group chats when someone explicitly asks to set/change the group icon/photo/picture. Expand their request into a detailed prompt. IMPORTANT: You MUST also write a brief text message acknowledging the request.', + description: 'Set the group chat icon/photo using a generated image. ONLY use in group chats when someone explicitly asks to set/change the group icon/photo/picture. Expand their request into a detailed prompt. IMPORTANT: You MUST also write a brief text message acknowledging the request.', input_schema: { type: 'object' as const, properties: { @@ -366,31 +367,79 @@ export interface AudioInput { mimeType: string; } -// Generate an image using OpenAI DALL-E API +// Generate an image using OpenAI's image API. gpt-image models return base64 +// only (no hosted URL), so we stash the bytes in the image store and return a +// URL served by this app that Linq can download from. export async function generateImage(prompt: string): Promise { try { - console.log(`[claude] Generating image with DALL-E: "${prompt.substring(0, 50)}..."`); + console.log(`[claude] Generating image with gpt-image-1: "${prompt.substring(0, 50)}..."`); const response = await openai.images.generate({ - model: 'dall-e-3', + model: 'gpt-image-1', prompt: prompt, n: 1, size: '1024x1024', - quality: 'standard', + quality: 'medium', }); - const imageUrl = response.data?.[0]?.url; + const b64 = response.data?.[0]?.b64_json; + if (!b64) { + console.error('[claude] No image data in gpt-image-1 response'); + return null; + } + + const imageUrl = storeImage(Buffer.from(b64, 'base64'), 'image/png'); if (imageUrl) { - console.log(`[claude] Image generated: ${imageUrl.substring(0, 50)}...`); - return imageUrl; + console.log(`[claude] Image generated: ${imageUrl}`); } - console.error('[claude] No image URL in DALL-E response'); - return null; + return imageUrl; } catch (error) { - console.error('[claude] DALL-E error:', error); + console.error('[claude] Image generation error:', error); return null; } } +// Claude API limits: max 8000px on any dimension, ~5MB per image. iPhone +// screenshots of long threads regularly blow past 8000px and previously +// 400'd the whole request. Oversized images get downscaled and re-encoded; +// everything else passes through as a URL source like before. +const MAX_IMAGE_DIMENSION = 7500; +const MAX_IMAGE_BYTES = 4.5 * 1024 * 1024; +const RESIZE_TARGET = 2048; + +async function prepareImageBlock(image: ImageInput): Promise { + const urlBlock: Anthropic.ImageBlockParam = { + type: 'image', + source: { type: 'url', url: image.url }, + }; + + try { + const response = await fetch(image.url); + if (!response.ok) throw new Error(`fetch failed: ${response.status}`); + const buffer = Buffer.from(await response.arrayBuffer()); + + const metadata = await sharp(buffer).metadata(); + const maxDim = Math.max(metadata.width ?? 0, metadata.height ?? 0); + if (maxDim <= MAX_IMAGE_DIMENSION && buffer.byteLength <= MAX_IMAGE_BYTES) { + return urlBlock; + } + + console.log(`[claude] Resizing oversized image (${metadata.width}x${metadata.height}, ${Math.round(buffer.byteLength / 1024)}KB): ${image.url.substring(0, 50)}...`); + const resized = await sharp(buffer) + .rotate() // apply EXIF orientation before resizing + .resize(RESIZE_TARGET, RESIZE_TARGET, { fit: 'inside', withoutEnlargement: true }) + .jpeg({ quality: 85 }) + .toBuffer(); + + return { + type: 'image', + source: { type: 'base64', media_type: 'image/jpeg', data: resized.toString('base64') }, + }; + } catch (error) { + console.error('[claude] Image prep failed, passing URL through:', error); + return urlBlock; + } +} + // Transcribe audio using OpenAI Whisper API async function transcribeAudio(url: string): Promise { try { @@ -504,17 +553,12 @@ export async function chat(chatId: string, userMessage: string, images: ImageInp // Build message content (text + images + audio) const messageContent: Anthropic.ContentBlockParam[] = []; - // Add images first - for (const image of images) { - messageContent.push({ - type: 'image', - source: { - type: 'url', - url: image.url, - }, - }); + // Add images first (resized if they exceed Claude's limits) + const imageBlocks = await Promise.all(images.map(image => { console.log(`[claude] Including image: ${image.url.substring(0, 50)}...`); - } + return prepareImageBlock(image); + })); + messageContent.push(...imageBlocks); // Transcribe audio files and add as text context const transcriptions: string[] = []; @@ -661,7 +705,50 @@ export async function chat(chatId: string, userMessage: string, images: ImageInp } } - const textResponse = textParts.length > 0 ? textParts.join('\n') : null; + let textResponse: string | null = textParts.length > 0 ? textParts.join('\n') : null; + + // Guardrail: Sonnet sometimes tapbacks a message instead of answering it, + // despite the prompt rules (e.g. thumbs-upping "how can I be an expert in + // geoguessr?"). A reaction alone is never a sufficient reply from chat() - + // in group chats the Haiku classifier already routes reaction-worthy + // messages to a quick reaction before Sonnet is invoked. Continue the + // tool loop and require a text reply. + if (!textResponse && reaction && !effect && !generatedImage && !groupChatIcon && !renameChat && !removeMember) { + console.log('[claude] Reaction-only response - requesting text follow-up'); + try { + const toolResults: Anthropic.ToolResultBlockParam[] = response.content + .filter((block): block is Anthropic.ToolUseBlock => block.type === 'tool_use') + .map(block => ({ + type: 'tool_result' as const, + tool_use_id: block.id, + content: block.name === 'send_reaction' + ? 'Reaction sent. A reaction alone is not a sufficient reply - now write your text response to their message. Do not use any more tools.' + : 'Done.', + })); + const followUp = await client.messages.create({ + model: 'claude-sonnet-4-6', + max_tokens: 1024, + system: buildSystemPrompt(chatContext), + tools, + messages: [ + ...formattedHistory, + { role: 'user', content: messageContent }, + { role: 'assistant', content: response.content }, + { role: 'user', content: toolResults }, + ], + }); + const followUpText = followUp.content + .map(block => (block.type === 'text' ? block.text : '')) + .filter(Boolean) + .join('\n'); + if (followUpText) { + textResponse = followUpText; + console.log('[claude] Got text follow-up after reaction-only response'); + } + } catch (error) { + console.error('[claude] Text follow-up failed (keeping reaction-only):', error); + } + } // Add assistant response to history (only text part, strip --- delimiters for cleaner context) // Note: image generation is handled separately in index.ts after sending text first diff --git a/src/index.ts b/src/index.ts index 037df68..e2020cd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,7 @@ import { createWebhookHandler } from './webhook/handler.js'; import { sendMessage, markAsRead, startTyping, sendReaction, shareContactCard, getChat, renameGroupChat, setGroupChatIcon, removeParticipant } from './linq/client.js'; import { chat, getGroupChatAction, getTextForEffect, generateImage } from './claude/client.js'; import { getUserProfile, addMessage } from './state/conversation.js'; +import { getImage, setPublicBaseUrl } from './state/imageStore.js'; import { processResponse } from './text/decorations.js'; // Track message count per chat for contact card sharing @@ -21,6 +22,28 @@ app.get('/health', (_req, res) => { res.json({ status: 'ok', timestamp: new Date().toISOString() }); }); +// Serve generated images so Linq can download them (gpt-image models return +// base64, not a hosted URL like DALL-E did) +app.get('/images/:id', (req, res) => { + const image = getImage(req.params.id); + if (!image) { + res.status(404).json({ error: 'not found or expired' }); + return; + } + res.set('Content-Type', image.contentType).send(image.buffer); +}); + +// Learn our public URL from incoming webhook requests (used to build +// generated-image URLs; PUBLIC_BASE_URL env var overrides) +app.use('/webhook', (req, _res, next) => { + const host = req.get('host'); + if (host) { + const proto = req.get('x-forwarded-proto')?.split(',')[0] ?? req.protocol; + setPublicBaseUrl(`${proto}://${host}`); + } + next(); +}); + // Webhook endpoint for Linq app.post( '/webhook', diff --git a/src/state/imageStore.ts b/src/state/imageStore.ts new file mode 100644 index 0000000..1516967 --- /dev/null +++ b/src/state/imageStore.ts @@ -0,0 +1,56 @@ +import { randomUUID } from 'crypto'; + +// In-memory store for generated images. The gpt-image models return base64 +// (no hosted URL like DALL-E did), but the Linq API needs a fetchable URL for +// media parts - so we hold the bytes briefly and serve them from this app. +// Linq downloads media as soon as the message is sent, so a short TTL is fine. +// Only safe while the app runs as a single instance. +const TTL_MS = 15 * 60 * 1000; + +interface StoredImage { + buffer: Buffer; + contentType: string; + expiresAt: number; +} + +const store = new Map(); + +// Base URL images are served from. PUBLIC_BASE_URL wins if set; otherwise +// learned from the Host header of incoming webhook requests (works for both +// App Runner and ngrok without extra config). +let publicBaseUrl: string | null = process.env.PUBLIC_BASE_URL?.replace(/\/+$/, '') || null; + +export function setPublicBaseUrl(url: string): void { + if (!process.env.PUBLIC_BASE_URL) { + publicBaseUrl = url.replace(/\/+$/, ''); + } +} + +export function storeImage(buffer: Buffer, contentType: string): string | null { + if (!publicBaseUrl) { + console.error('[imageStore] No public base URL known yet - cannot serve generated image'); + return null; + } + cleanup(); + const id = randomUUID(); + store.set(id, { buffer, contentType, expiresAt: Date.now() + TTL_MS }); + console.log(`[imageStore] Stored image ${id} (${Math.round(buffer.byteLength / 1024)}KB, ${store.size} in store)`); + return `${publicBaseUrl}/images/${id}`; +} + +export function getImage(id: string): { buffer: Buffer; contentType: string } | null { + const entry = store.get(id); + if (!entry) return null; + if (entry.expiresAt < Date.now()) { + store.delete(id); + return null; + } + return { buffer: entry.buffer, contentType: entry.contentType }; +} + +function cleanup(): void { + const now = Date.now(); + for (const [id, entry] of store) { + if (entry.expiresAt < now) store.delete(id); + } +}