Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 69 additions & 0 deletions packages/core/src/conversations/threads/threads.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
podMember,
providerModel,
thread,
threadFile,
turn,
user,
workspace,
Expand Down Expand Up @@ -1187,6 +1188,74 @@ describe.skipIf(!process.env.DATABASE_URL)("threads, against Postgres", async ()
expect(await found("river")).toEqual([said[0], said[4]]);
});

it("finds the files a message carried, even one with nothing else in it", async () => {
const details = await createThread({
workspaceId,
podId,
hostAgentId: agentId,
initiatorUserId: memberId,
message: "Contracts",
});
const started = new Date(Date.now() - 10 * 60_000);
const sent = await onDatabase((db) =>
db
.insert(message)
.values(
["Here is the signed contract", ""].map((content, index) => ({
threadId: details.thread.id,
authorUserId: memberId,
kind: "text" as const,
status: "complete" as const,
parts: content ? [{ type: "text" as const, text: content }] : [],
content,
createdAt: new Date(started.getTime() + index * 60_000),
})),
)
.returning({ id: message.id }),
);
const [withText, filesOnly] = sent;
if (!withText || !filesOnly) throw new Error("Could not create the messages");
const attached = await onDatabase((db) =>
db
.insert(threadFile)
.values(
[
{ messageId: withText.id, name: "contract.pdf", contentType: "application/pdf" },
{ messageId: filesOnly.id, name: "chart.png", contentType: "image/png" },
].map((file) => ({
...file,
threadId: details.thread.id,
kind: "upload" as const,
uploaderId: memberId,
position: 0,
size: 2048,
})),
)
.returning({ id: threadFile.id, name: threadFile.name }),
);
const fileIdOf = (name: string) => attached.find((file) => file.name === name)?.id;
const search = searchHistoryTool({
threadId: details.thread.id,
before: new Date(Date.now() + 60_000),
run: runOnPostgres,
});
const found = async (input: { query?: string; after?: string }) => {
const result = await search.execute?.(input, { toolCallId: "search", messages: [] } as never);
return result && "messages" in result ? result.messages.map(({ text }) => text) : [];
};

expect(await found({ query: "signed contract" })).toEqual([
`Here is the signed contract\n[Attached contract.pdf (application/pdf, 2.0 KB) as file ${fileIdOf("contract.pdf")}, which read_thread_file reads]`,
]);
// By its name, for a message that says nothing else.
expect(await found({ query: "chart.png" })).toEqual([
`[Attached chart.png (image/png, 2.0 KB) as file ${fileIdOf("chart.png")}, which read_thread_file reads]`,
]);
expect(await found({ after: started.toISOString() })).toContain(
`[Attached chart.png (image/png, 2.0 KB) as file ${fileIdOf("chart.png")}, which read_thread_file reads]`,
);
});

/**
* A thread of twenty long messages, about 10K tokens each, before its
* first, with the bot's model given a 128K window and the system agents'
Expand Down
33 changes: 25 additions & 8 deletions packages/core/src/conversations/tools/search-history/tool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@ import { tool } from "ai";
import { and, asc, desc, eq, gte, ilike, lt, or, type SQL, sql } from "drizzle-orm";
import { DateTime, Effect, Schema } from "effect";
import { type Executor, query, type RunEffect } from "../../../database/database.ts";
import { agent, message, user } from "../../../database/schema.ts";
import { formatHistoryTime } from "../../threads/message-text.ts";
import { agent, message, threadFile, user } from "../../../database/schema.ts";
import { loadFileParts } from "../../threads/file-parts.ts";
import { describeAttachment, formatHistoryTime } from "../../threads/message-text.ts";
import { participantColumns, toMessage } from "../../threads/participants.ts";

const MAX_MATCHES = 10;
Expand Down Expand Up @@ -114,17 +115,26 @@ const searchHistory = Effect.fn("SearchHistory.searchHistory")(function* (
)
// One more than is returned, to tell whether there are more.
.limit(MAX_MATCHES + 1);
const returned = rows.slice(0, MAX_MATCHES);
// Each message lists the files it carried, so a file found this way can be read again.
const files = yield* loadFileParts(
db,
returned.map(({ message: row }) => row.id),
);
return {
messages: rows.slice(0, MAX_MATCHES).map(({ message: row, ...author }): HistoryMatch => {
messages: returned.map(({ message: row, ...author }): HistoryMatch => {
const found = toMessage(row, author);
const said =
row.content.length <= MATCH_CHARACTERS
? row.content
: `${row.content.slice(0, MATCH_CHARACTERS)}…`;
return {
author:
found.author.kind === "routine_trigger" ? found.author.routineName : found.author.name,
at: formatHistoryTime(row.createdAt),
text:
row.content.length <= MATCH_CHARACTERS
? row.content
: `${row.content.slice(0, MATCH_CHARACTERS)}…`,
text: [said, ...(files.get(row.id) ?? []).map(describeAttachment)]
.filter((line) => line !== "")
.join("\n"),
};
}),
more: rows.length > MAX_MATCHES,
Expand Down Expand Up @@ -153,12 +163,19 @@ function wordMatch(words: string): { where: SQL; best: SQL[] } {
terms.length > 0
? and(...terms.map((term) => ilike(message.content, `%${escapeLike(term)}%`)))
: undefined;
// A file is found by its name too, as a message with only files says nothing else.
const carriesNamed =
terms.length > 0
? sql`exists (select 1 from ${threadFile} where ${threadFile.messageId} = ${message.id} and ${and(
...terms.map((term) => ilike(threadFile.name, `%${escapeLike(term)}%`)),
)})`
: undefined;
const rank = sql`greatest(
ts_rank(to_tsvector('english', ${message.content}), websearch_to_tsquery('english', ${words})),
ts_rank(to_tsvector('simple', ${message.content}), websearch_to_tsquery('simple', ${words}))
)`;
return {
where: or(english, simple, substring) as SQL,
where: or(english, simple, substring, carriesNamed) as SQL,
best: [desc(sql`(${english}) or (${simple})`), desc(rank)],
};
}
Expand Down
5 changes: 5 additions & 0 deletions packages/web/src/lib/chat-files.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,11 @@ export function uploadChatFile({
});
}

/** Where a file attached to a message in the chat is served, shown in place or downloaded. */
export function chatFileUrl(chatId: string, fileId: string): string {
return `${client.baseUrl}/chats/${encodeURIComponent(chatId)}/files/${encodeURIComponent(fileId)}`;
}

/**
* Removes an upload that will not be sent, so it stops counting toward the
* files a person can have waiting. Nothing waits on it, and a failure is left
Expand Down
30 changes: 20 additions & 10 deletions packages/web/src/lib/chats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import {
type Chat,
type ChatList,
type ChatMessageItem,
type FilePart,
handleFromName,
type MessageReplyTo,
type NewMessage,
Expand Down Expand Up @@ -360,20 +361,26 @@ export function useChatHistory(chatId: string | undefined) {
};
}

/** A message to send, with the whole quote it answers, so it shows quoting it before the API answers. */
type SentChatMessage = Omit<NewMessage, "replyTo"> & { replyTo?: MessageReplyTo };
/**
* A message to send, with the whole quote it answers and the files it
* attaches, so it shows both before the API answers.
*/
export type MessageToSend = Omit<NewMessage, "replyTo" | "attachments"> & {
replyTo?: MessageReplyTo;
files?: readonly FilePart[];
};

export function useSendChatMessage(chat: Chat | undefined, user: SessionUser) {
const queries = useQueryClient();
return useMutation({
mutationFn: async ({ replyTo, ...input }: SentChatMessage) => {
mutationFn: async ({ replyTo, files = [], ...message }: MessageToSend) => {
if (!chat) throw new NotReadyError();
return Effect.runPromise(
client.api.chats.send({
params: { chatId: chat.id },
payload: { ...input, replyTo: replyTo?.id },
}),
);
const payload: NewMessage = {
...message,
...(replyTo && { replyTo: replyTo.id }),
...(files.length > 0 && { attachments: files.map((file) => file.id) }),
};
return Effect.runPromise(client.api.chats.send({ params: { chatId: chat.id }, payload }));
},
onMutate: async (input) => {
if (!chat) return;
Expand All @@ -393,7 +400,10 @@ export function useSendChatMessage(chat: Chat | undefined, user: SessionUser) {
},
kind: "text",
status: "complete",
parts: [{ type: "text", text: input.message }],
parts: [
...(input.message === "" ? [] : [{ type: "text" as const, text: input.message }]),
...(input.files ?? []),
],
content: input.message,
...(input.replyTo ? { replyTo: input.replyTo } : {}),
createdAt: timestampAfter(
Expand Down
23 changes: 17 additions & 6 deletions packages/web/src/lib/composer-attachments.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,10 @@ describe("useComposerAttachments", () => {

act(() => request?.answer(201, uploadedAs(file)));
await settle();
expect(result.current.items[0]?.upload).toEqual({ state: "uploaded", fileId });
expect(result.current.items[0]?.upload).toEqual({
state: "uploaded",
attachment: uploadedAs(file),
});
expect(sendReadiness(result.current.items)).toEqual({ ready: true });
});

Expand Down Expand Up @@ -223,7 +226,9 @@ describe("useComposerAttachments", () => {
act(() => result.current.retry(key ?? ""));
act(() => FakeRequest.made[1]?.answer(201, uploadedAs(file)));
await settle();
expect(result.current.items).toEqual([{ key, file, upload: { state: "uploaded", fileId } }]);
expect(result.current.items).toEqual([
{ key, file, upload: { state: "uploaded", attachment: uploadedAs(file) } },
]);
});

it("takes the uploaded files to send, and puts them back when the message is not sent", async () => {
Expand All @@ -238,7 +243,10 @@ describe("useComposerAttachments", () => {
act(() => {
taken = result.current.takeForSending("");
});
expect(taken?.fileIds).toEqual([fileId]);
// As the message shows them until the API answers.
expect(taken?.parts).toEqual([
{ type: "file", id: fileId, name: file.name, contentType: file.type, size: file.size },
]);
expect(result.current.items).toEqual([]);

act(() => taken?.putBack(new Error("The connection dropped")));
Expand Down Expand Up @@ -266,7 +274,7 @@ describe("useComposerAttachments", () => {
it("sends text, files or both, but nothing empty and nothing still uploading", async () => {
const { result } = renderHook(() => useComposerAttachments(chatId));
expect(result.current.takeForSending(" ")).toBeUndefined();
expect(result.current.takeForSending("Hello")?.fileIds).toEqual([]);
expect(result.current.takeForSending("Hello")?.parts).toEqual([]);

const file = pdf();
act(() => result.current.attach?.([file]));
Expand All @@ -277,7 +285,7 @@ describe("useComposerAttachments", () => {
act(() => {
taken = result.current.takeForSending("");
});
expect(taken?.fileIds).toEqual([fileId]);
expect(taken?.parts).toEqual([{ type: "file", ...uploadedAs(file) }]);
});

it.each([
Expand Down Expand Up @@ -365,7 +373,10 @@ describe("useComposerAttachments", () => {
act(() => FakeRequest.made[1]?.answer(201, uploadedAs(file, freshId)));
await settle();
expect(FakeRequest.made[1]?.body).toBe(file);
expect(result.current.items[0]?.upload).toEqual({ state: "uploaded", fileId: freshId });
expect(result.current.items[0]?.upload).toEqual({
state: "uploaded",
attachment: uploadedAs(file, freshId),
});
});

it("puts files back only once", async () => {
Expand Down
20 changes: 14 additions & 6 deletions packages/web/src/lib/composer-attachments.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import {
attachmentNameSchema,
type FilePart,
MAX_ATTACHMENT_BYTES,
MAX_ATTACHMENTS_PER_MESSAGE,
type UploadedAttachment,
} from "@sugabots/contracts";
import { isApiFailure } from "@sugabots/sdk";
import { Option, Schema } from "effect";
Expand All @@ -12,7 +14,8 @@ import { removeChatFile, UploadInterrupted, uploadChatFile } from "@/lib/chat-fi
export type AttachmentUpload =
/** `progress` runs from 0 to 1. */
| { state: "uploading"; progress: number }
| { state: "uploaded"; fileId: string }
/** `attachment` is the file as the API kept it, which may not be as the browser said. */
| { state: "uploaded"; attachment: UploadedAttachment }
/**
* Not uploaded, or no longer there to send, for `reason`. Only a
* `retryable` failure can go differently when the same file is tried again.
Expand Down Expand Up @@ -51,7 +54,8 @@ export type ComposerAttachments = {

/** The files taken to send with a message. */
export type FilesForSending = {
fileIds: string[];
/** The files as the message will show them, which it can before the API answers. */
parts: FilePart[];
/**
* Returns the files to the composer, in front of any attached since, when
* the message failed with `sendFailure`. Files the API no longer had come
Expand Down Expand Up @@ -129,7 +133,7 @@ export function useComposerAttachments(chatId: string | undefined): ComposerAtta
return;
}
inFlight.current.delete(key);
setUpload(key, { state: "uploaded", fileId: uploaded.id });
setUpload(key, { state: "uploaded", attachment: uploaded });
},
(failure: unknown) => {
if (!stillWanted()) return;
Expand Down Expand Up @@ -160,7 +164,9 @@ export function useComposerAttachments(chatId: string | undefined): ComposerAtta
if (!item) return;
inFlight.current.get(key)?.abort();
inFlight.current.delete(key);
if (chatId && item.upload.state === "uploaded") removeChatFile(chatId, item.upload.fileId);
if (chatId && item.upload.state === "uploaded") {
removeChatFile(chatId, item.upload.attachment.id);
}
setLeftOut(0);
commit((items) => items.filter((candidate) => candidate.key !== key));
}
Expand All @@ -180,7 +186,9 @@ export function useComposerAttachments(chatId: string | undefined): ComposerAtta
commit(() => []);
let returned = false;
return {
fileIds,
parts: taken.flatMap(({ upload }): FilePart[] =>
upload.state === "uploaded" ? [{ type: "file", ...upload.attachment }] : [],
),
putBack: (sendFailure) => {
if (returned) return;
returned = true;
Expand Down Expand Up @@ -284,7 +292,7 @@ export function isFileGoneRefusal(failure: unknown): boolean {
}

function uploadedFileIds(items: readonly ComposerAttachment[]): string[] {
return items.flatMap(({ upload }) => (upload.state === "uploaded" ? [upload.fileId] : []));
return items.flatMap(({ upload }) => (upload.state === "uploaded" ? [upload.attachment.id] : []));
}

/** The API's messages leave off the full stop a sentence beside others needs. */
Expand Down
35 changes: 35 additions & 0 deletions packages/web/src/lib/file-looks.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import { mediaTypeEssence } from "@sugabots/contracts";

/** How a file is drawn: as its picture, as a page, as a sheet, or as a plain file. */
export type FileLook = "image" | "pdf" | "sheet" | "other";

const SHEET_EXTENSIONS = new Set(["csv", "tsv", "xls", "xlsx", "ods", "numbers"]);

/**
* `file`'s look, from its media type and, when that says little, its name.
* Any image is drawn as its picture, where an image the browser can't draw
* falls back to a file's icon.
*/
export function fileLookOf(file: { name: string; type: string }): FileLook {
const mediaType = mediaTypeEssence(file.type);
if (mediaType.startsWith("image/")) return "image";
const extension = extensionOf(file.name);
if (mediaType === "application/pdf" || extension === "pdf") return "pdf";
if (mediaType === "text/csv" || SHEET_EXTENSIONS.has(extension)) return "sheet";
return "other";
}

/** The part of `name` after its last dot, lowercased, or "" when it has none. */
export function extensionOf(name: string): string {
const dot = name.lastIndexOf(".");
return dot > 0 ? name.slice(dot + 1).toLowerCase() : "";
}

/** A file's size as people read it, such as 240 KB or 2.1 MB. */
export function fileSizeLabel(bytes: number): string {
if (bytes < 1024) return `${bytes} B`;
const kilobytes = Math.round(bytes / 1024);
// Rounded first, so 1,048,100 bytes reads 1.0 MB rather than 1024 KB.
if (kilobytes < 1024) return `${kilobytes} KB`;
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
}
Loading
Loading