diff --git a/bun.lock b/bun.lock index 6ce3eb8..0f553ac 100644 --- a/bun.lock +++ b/bun.lock @@ -1,6 +1,6 @@ { "lockfileVersion": 1, - "configVersion": 0, + "configVersion": 1, "workspaces": { "": { "name": "nyaai", @@ -110,6 +110,9 @@ }, }, }, + "trustedDependencies": [ + "@rocicorp/zero-sqlite3", + ], "packages": { "@ai-sdk/anthropic": ["@ai-sdk/anthropic@3.0.58", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.19" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-/53SACgmVukO4bkms4dpxpRlYhW8Ct6QZRe6sj1Pi5H00hYhxIrqfiLbZBGxkdRvjsBQeP/4TVGsXgH5rQeb8Q=="], diff --git a/i18n/zh-CN.json b/i18n/zh-CN.json index 8fdb9b2..f46f0b1 100644 --- a/i18n/zh-CN.json +++ b/i18n/zh-CN.json @@ -679,8 +679,12 @@ "Import data": "导入数据", "Currently, only importing from AIaW is supported.": "目前仅支持从 AIaW 导入。", "Importing...": "导入中...", + "{0}% · {1}/{2} chats imported": "{0}% · 已导入 {1}/{2} 个对话", "AIaW Import": "AIaW 导入", "Import successful": "导入成功", + "Import completed with warnings": "导入完成,但有警告", + "Imported {0} chats and {1} messages": "已导入 {0} 个对话和 {1} 条消息", + "Imported {0} chats and {1} messages; skipped {2} binary attachments": "已导入 {0} 个对话和 {1} 条消息;跳过了 {2} 个二进制附件", "Imported to the workspace root": "已导入到工作区根目录", "Import failed": "导入失败", "Search in Workspace": "搜索工作区", @@ -706,4 +710,4 @@ "GitHub Repos": "GitHub 项目", "Give AI access to the source code of all public GitHub repos. Powered by gread.dev.": "让 AI 能够访问所有 GitHub 开源项目的源代码。 由 gread.dev 提供支持。", "Enable Thinking": "启用思考" -} \ No newline at end of file +} diff --git a/i18n/zh-TW.json b/i18n/zh-TW.json index 102f9e9..87e486d 100644 --- a/i18n/zh-TW.json +++ b/i18n/zh-TW.json @@ -679,8 +679,12 @@ "Import data": "導入數據", "Currently, only importing from AIaW is supported.": "目前僅支援從 AIaW 導入。", "Importing...": "導入...", + "{0}% · {1}/{2} chats imported": "{0}% · 已導入 {1}/{2} 個對話", "AIaW Import": "AIaW 導入", "Import successful": "導入成功", + "Import completed with warnings": "導入完成,但有警告", + "Imported {0} chats and {1} messages": "已導入 {0} 個對話和 {1} 則訊息", + "Imported {0} chats and {1} messages; skipped {2} binary attachments": "已導入 {0} 個對話和 {1} 則訊息;略過了 {2} 個二進位附件", "Imported to the workspace root": "已導入到工作區根目錄", "Import failed": "導入失敗", "Search in Workspace": "搜尋工作區", @@ -706,4 +710,4 @@ "GitHub Repos": "GitHub 項目", "Give AI access to the source code of all public GitHub repos. Powered by gread.dev.": "讓 AI 能夠存取所有 GitHub 開源專案的原始碼。 由 gread.dev 提供支援。", "Enable Thinking": "啟用思考" -} \ No newline at end of file +} diff --git a/package.json b/package.json index 9aaa765..6dc1876 100644 --- a/package.json +++ b/package.json @@ -130,5 +130,8 @@ "node": "^28 || ^26 || ^24 || ^22 || ^20 || ^18", "npm": ">= 6.13.4", "yarn": ">= 1.21.1" - } -} \ No newline at end of file + }, + "trustedDependencies": [ + "@rocicorp/zero-sqlite3" + ] +} diff --git a/src-server/import/aiaw.ts b/src-server/import/aiaw.ts new file mode 100644 index 0000000..c003a96 --- /dev/null +++ b/src-server/import/aiaw.ts @@ -0,0 +1,450 @@ +import { and, eq, inArray } from 'drizzle-orm' +import type { Avatar } from 'app/src-shared/utils/validators' +import { genId, randomId } from 'app/src-shared/utils/id' +import type { AiawImportJobSnapshot, AiawImportStage } from 'app/src-shared/aiaw-import' +import { db } from '../utils/db' +import * as schema from '../schema' + +interface AiawTableData { + tableName: string + rows: Record[] +} + +interface AiawExport { + formatName: string + formatVersion: number + data: { + data: AiawTableData[] + } +} + +interface ImportParent { + id: string + rootId: string + pubRoot: string | null +} + +interface ImportJob extends AiawImportJobSnapshot { + userId: string +} + +const jobs = new Map() +const DIALOG_BATCH_SIZE = 50 +const MAX_RETAINED_JOBS = 50 + +const emptyCounts = () => ({ + workspaces: 0, + assistants: 0, + dialogs: 0, + messages: 0, + items: 0, +}) + +const emptyWarnings = () => ({ + orphanMessages: 0, + orphanItems: 0, + missingMessageReferences: 0, + unknownAssistantReferences: 0, + skippedBinaryItems: 0, +}) + +function snapshot(job: ImportJob): AiawImportJobSnapshot { + const result = { ...job } + Reflect.deleteProperty(result, 'userId') + return result +} + +function updateJob(job: ImportJob, updates: Partial) { + Object.assign(job, updates, { updatedAt: Date.now() }) +} + +function pruneJobs() { + if (jobs.size < MAX_RETAINED_JOBS) return + const finished = [...jobs.values()] + .filter(job => job.status === 'completed' || job.status === 'failed') + .sort((a, b) => a.updatedAt - b.updatedAt) + while (jobs.size >= MAX_RETAINED_JOBS && finished.length) { + jobs.delete(finished.shift()!.id) + } +} + +export function createAiawImportJob(userId: string) { + pruneJobs() + const active = [...jobs.values()].find(job => + job.userId === userId && (job.status === 'queued' || job.status === 'running'), + ) + if (active) return { active: snapshot(active) } + + const now = Date.now() + const job: ImportJob = { + id: randomId(), + userId, + status: 'queued', + stage: 'queued', + progress: 0, + counts: emptyCounts(), + totals: emptyCounts(), + warnings: emptyWarnings(), + createdAt: now, + updatedAt: now, + } + jobs.set(job.id, job) + return { job, snapshot: snapshot(job) } +} + +export function getAiawImportJob(id: string, userId: string) { + const job = jobs.get(id) + return job?.userId === userId ? snapshot(job) : undefined +} + +export function getActiveAiawImportJob(userId: string) { + const job = [...jobs.values()].find(job => + job.userId === userId && (job.status === 'queued' || job.status === 'running'), + ) + return job && snapshot(job) +} + +function setStage(job: ImportJob, stage: AiawImportStage, progress: number) { + updateJob(job, { status: 'running', stage, progress }) +} + +function getNameAvatar(value: unknown): { name?: string, avatar?: Avatar } { + if (typeof value !== 'string') return {} + const [first, ...rest] = value.split(' ') + const graphemes = [...new Intl.Segmenter('en', { granularity: 'grapheme' }).segment(first)] + if (graphemes.length === 1 && rest.length) { + return { + name: rest.join(' '), + avatar: { type: 'text', text: first }, + } + } + return { name: value } +} + +function parseDump(content: string) { + const dump = JSON.parse(content) as AiawExport + if (dump.formatName !== 'dexie' || !Array.isArray(dump.data?.data)) { + throw new Error('Unsupported format. Need dexie export.') + } + return dump +} + +export async function runAiawImportJob(job: ImportJob, file: File, parent: ImportParent, folderName: string) { + try { + setStage(job, 'parsing', 0.01) + const dump = parseDump(await file.text()) + setStage(job, 'preparing', 0.03) + + const getRows = (tableName: string) => + dump.data.data.find(table => table.tableName === tableName)?.rows || [] + + const workspaces = getRows('workspaces') + const dialogs = getRows('dialogs').sort((a, b) => a.id < b.id ? 1 : -1) + const messages = getRows('messages') + const assistants = getRows('assistants') + const avatarImages = getRows('avatarImages') + const items = getRows('items') + + updateJob(job, { + totals: { + workspaces: workspaces.length, + assistants: assistants.length, + dialogs: dialogs.length, + messages: messages.length, + items: items.length + avatarImages.length, + }, + }) + + const dialogIds = new Set(dialogs.map(dialog => dialog.id)) + const assistantIds = new Set(assistants.map(assistant => assistant.id)) + const workspaceIds = new Set(workspaces.map(workspace => workspace.id)) + const idMap = new Map() + const getNewId = (oldId: string | undefined) => { + if (!oldId || oldId === '$root') return undefined + if (!idMap.has(oldId)) { + const timestamp = parseInt(oldId.slice(0, 9), 32) + idMap.set(oldId, genId(Number.isNaN(timestamp) ? undefined : timestamp)) + } + return idMap.get(oldId)! + } + + const messagesByDialog = new Map[]>() + for (const message of messages) { + const list = messagesByDialog.get(message.dialogId) || [] + list.push(message) + messagesByDialog.set(message.dialogId, list) + if (!dialogIds.has(message.dialogId)) job.warnings.orphanMessages++ + if (message.assistantId && !assistantIds.has(message.assistantId)) { + job.warnings.unknownAssistantReferences++ + } + } + + const itemsByDialog = new Map[]>() + for (const item of items) { + const list = itemsByDialog.get(item.dialogId) || [] + list.push(item) + itemsByDialog.set(item.dialogId, list) + if (!dialogIds.has(item.dialogId)) job.warnings.orphanItems++ + if (item.contentBuffer) job.warnings.skippedBinaryItems++ + } + for (const image of avatarImages) { + if (image.contentBuffer) job.warnings.skippedBinaryItems++ + } + + const importFolderId = genId() + job.folderId = importFolderId + + await db.transaction(async tx => { + await tx.insert(schema.entity).values({ + id: importFolderId, + rootId: parent.rootId, + pubRoot: parent.pubRoot, + parentId: parent.id, + type: 'folder', + name: folderName, + conf: {}, + sortPriority: 10, + hidden: false, + }) + + const workspaceMap = new Map(workspaces.map(workspace => [workspace.id, workspace])) + const createdWorkspaces = new Set() + const visitingWorkspaces = new Set() + const createWorkspace = async (workspace: Record) => { + if (createdWorkspaces.has(workspace.id)) return + if (visitingWorkspaces.has(workspace.id)) throw new Error(`Workspace cycle detected: ${workspace.id}`) + visitingWorkspaces.add(workspace.id) + const oldParentId = workspace.parentId + if (oldParentId && oldParentId !== '$root' && workspaceMap.has(oldParentId)) { + await createWorkspace(workspaceMap.get(oldParentId)!) + } + const parentId = oldParentId && oldParentId !== '$root' && workspaceMap.has(oldParentId) + ? getNewId(oldParentId)! + : importFolderId + await tx.insert(schema.entity).values({ + id: getNewId(workspace.id)!, + rootId: parent.rootId, + pubRoot: parent.pubRoot, + parentId, + type: 'folder', + name: workspace.name, + conf: {}, + sortPriority: 10, + hidden: false, + }) + visitingWorkspaces.delete(workspace.id) + createdWorkspaces.add(workspace.id) + job.counts.workspaces++ + } + for (const workspace of workspaces) await createWorkspace(workspace) + + if (avatarImages.length) { + const avatarEntities = avatarImages.map(image => ({ + id: getNewId(image.id)!, + rootId: parent.rootId, + pubRoot: parent.pubRoot, + parentId: importFolderId, + type: 'item' as const, + name: 'avatar', + conf: {}, + sortPriority: 0, + hidden: true, + })) + await tx.insert(schema.entity).values(avatarEntities) + await tx.insert(schema.item).values(avatarImages.map(image => ({ + id: getNewId(image.id)!, + rootId: parent.rootId, + mimeType: image.mimeType || 'image/png', + }))) + job.counts.items += avatarImages.length + } + + if (assistants.length) { + await tx.insert(schema.entity).values(assistants.map(assistant => { + const imageId = assistant.avatar?.type === 'image' && getNewId(assistant.avatar.imageId) + const avatar = imageId ? { ...assistant.avatar, itemId: imageId } : assistant.avatar + return { + id: getNewId(assistant.id)!, + rootId: parent.rootId, + pubRoot: parent.pubRoot, + parentId: workspaceIds.has(assistant.workspaceId) + ? getNewId(assistant.workspaceId)! + : importFolderId, + type: 'assistant' as const, + name: assistant.name, + avatar, + conf: {}, + sortPriority: 0, + hidden: false, + } + })) + await tx.insert(schema.assistant).values(assistants.map(assistant => ({ + id: getNewId(assistant.id)!, + rootId: parent.rootId, + prompt: assistant.prompt, + promptRole: assistant.promptRole === 'user' ? 'user' as const : 'system' as const, + contextNum: 10, + streamSettings: {}, + plugins: [], + }))) + job.counts.assistants = assistants.length + } + + setStage(job, 'importing', 0.08) + for (let offset = 0; offset < dialogs.length; offset += DIALOG_BATCH_SIZE) { + const batch = dialogs.slice(offset, offset + DIALOG_BATCH_SIZE) + const chatEntities: typeof schema.entity.$inferInsert[] = [] + const chatRows: typeof schema.chat.$inferInsert[] = [] + const itemEntities: typeof schema.entity.$inferInsert[] = [] + const itemRows: typeof schema.item.$inferInsert[] = [] + const messageRows: typeof schema.message.$inferInsert[] = [] + const messageEntityRows: typeof schema.messageEntity.$inferInsert[] = [] + + for (const dialog of batch) { + const chatId = getNewId(dialog.id)! + const title = getNameAvatar(dialog.name) + const tree: Record = { $root: [] } + const route: Record = { $root: -1 } + const dialogMessages = messagesByDialog.get(dialog.id) || [] + const messageMap = new Map(dialogMessages.map(message => [message.id, message])) + const dialogItems = itemsByDialog.get(dialog.id) || [] + const availableItemIds = new Set(dialogItems.map(item => item.id)) + + chatEntities.push({ + id: chatId, + rootId: parent.rootId, + pubRoot: parent.pubRoot, + parentId: workspaceIds.has(dialog.workspaceId) + ? getNewId(dialog.workspaceId)! + : importFolderId, + type: 'chat', + ...title, + conf: {}, + sortPriority: 0, + hidden: false, + }) + + for (const item of dialogItems) { + const itemId = getNewId(item.id)! + itemEntities.push({ + id: itemId, + rootId: parent.rootId, + pubRoot: parent.pubRoot, + parentId: chatId, + type: 'item', + name: item.name || 'item', + conf: {}, + sortPriority: 0, + hidden: false, + }) + itemRows.push({ + id: itemId, + rootId: parent.rootId, + mimeType: item.mimeType, + text: item.contentText, + }) + } + + const visited = new Set() + const processMessage = (oldMessageId: string, targetId: string) => { + if (visited.has(oldMessageId)) return + visited.add(oldMessageId) + const oldMessage = messageMap.get(oldMessageId) + if (!oldMessage) { + job.warnings.missingMessageReferences++ + for (const childId of dialog.msgTree?.[oldMessageId] || []) processMessage(childId, targetId) + return + } + + const content = oldMessage.contents?.[0] + let childTargetId = targetId + if (content && content.type !== 'assistant-tool') { + const messageId = getNewId(oldMessage.id)! + tree[targetId] ||= [] + tree[targetId].push(messageId) + route[targetId] = tree[targetId].length - 1 + tree[messageId] = [] + childTargetId = messageId + + messageRows.push({ + id: messageId, + rootId: parent.rootId, + entityId: chatId, + userId: job.userId, + type: oldMessage.type === 'user' ? 'chat:user' : 'chat:assistant', + text: typeof content.text === 'string' ? content.text : '', + reasoning: typeof content.reasoning === 'string' ? content.reasoning : null, + modelName: oldMessage.modelName, + assistantId: oldMessage.assistantId && assistantIds.has(oldMessage.assistantId) + ? getNewId(oldMessage.assistantId) + : null, + }) + + const referencedItems = new Set(content.items || []) + for (const oldItemId of referencedItems) { + if (!availableItemIds.has(oldItemId)) continue + messageEntityRows.push({ + rootId: parent.rootId, + messageId, + entityId: getNewId(oldItemId)!, + }) + } + } + + for (const childId of dialog.msgTree?.[oldMessageId] || []) { + processMessage(childId, childTargetId) + } + } + + for (const rootMessageId of dialog.msgTree?.$root || []) { + processMessage(rootMessageId, '$root') + } + + chatRows.push({ + id: chatId, + rootId: parent.rootId, + msgTree: tree, + msgRoute: route, + }) + } + + if (chatEntities.length) await tx.insert(schema.entity).values(chatEntities) + if (chatRows.length) await tx.insert(schema.chat).values(chatRows) + if (itemEntities.length) await tx.insert(schema.entity).values(itemEntities) + if (itemRows.length) await tx.insert(schema.item).values(itemRows) + if (messageRows.length) await tx.insert(schema.message).values(messageRows) + if (messageEntityRows.length) await tx.insert(schema.messageEntity).values(messageEntityRows) + + job.counts.dialogs += batch.length + job.counts.messages += messageRows.length + job.counts.items += itemRows.length + const completed = Math.min(offset + batch.length, dialogs.length) + updateJob(job, { progress: 0.08 + 0.9 * (completed / Math.max(dialogs.length, 1)) }) + } + + setStage(job, 'committing', 0.99) + }) + + updateJob(job, { status: 'completed', stage: 'completed', progress: 1 }) + } catch (error) { + console.error('AIaW import failed', error) + updateJob(job, { + status: 'failed', + stage: 'failed', + error: error instanceof Error ? error.message : String(error), + }) + } +} + +export async function findWritableImportParent(parentId: string, userId: string) { + const [parent] = await db.select({ + id: schema.entity.id, + rootId: schema.entity.rootId, + pubRoot: schema.entity.pubRoot, + }).from(schema.entity).innerJoin(schema.member, and( + eq(schema.member.workspaceId, schema.entity.rootId), + eq(schema.member.userId, userId), + inArray(schema.member.role, ['owner', 'admin', 'member']), + )).where(eq(schema.entity.id, parentId)).limit(1) + return parent +} diff --git a/src-server/import/routes.ts b/src-server/import/routes.ts new file mode 100644 index 0000000..6551cbb --- /dev/null +++ b/src-server/import/routes.ts @@ -0,0 +1,47 @@ +import { Hono } from 'hono' +import { auth } from '../auth/auth' +import { createAiawImportJob, findWritableImportParent, getActiveAiawImportJob, getAiawImportJob, runAiawImportJob } from './aiaw' + +const app = new Hono() + .post('/aiaw', async c => { + const session = await auth.api.getSession({ headers: c.req.raw.headers }) + if (!session) return c.json({ error: 'Unauthorized' }, 401) + + const form = await c.req.formData() + const file = form.get('file') + const parentId = form.get('parentId') + const folderName = form.get('folderName') + if (!(file instanceof File) || typeof parentId !== 'string') { + return c.json({ error: 'File and parentId are required' }, 400) + } + + const parent = await findWritableImportParent(parentId, session.user.id) + if (!parent) return c.json({ error: 'Target folder not found' }, 404) + + const created = createAiawImportJob(session.user.id) + if ('active' in created) { + return c.json({ error: 'An AIaW import is already running', job: created.active }, 409) + } + + runAiawImportJob( + created.job, + file, + parent, + typeof folderName === 'string' && folderName ? folderName : 'AIaW Import', + ) + return c.json({ job: created.snapshot }, 202) + }) + .get('/aiaw/active', async c => { + const session = await auth.api.getSession({ headers: c.req.raw.headers }) + if (!session) return c.json({ error: 'Unauthorized' }, 401) + return c.json({ job: getActiveAiawImportJob(session.user.id) || null }) + }) + .get('/aiaw/:id', async c => { + const session = await auth.api.getSession({ headers: c.req.raw.headers }) + if (!session) return c.json({ error: 'Unauthorized' }, 401) + const job = getAiawImportJob(c.req.param('id'), session.user.id) + if (!job) return c.json({ error: 'Import job not found' }, 404) + return c.json({ job }) + }) + +export default app diff --git a/src-server/index.ts b/src-server/index.ts index c49d780..c18d917 100644 --- a/src-server/index.ts +++ b/src-server/index.ts @@ -10,6 +10,7 @@ import payment from './payment' import searxng from './searxng' import webhooks from './webhooks' import search from './search' +import importRoutes from './import/routes' import { initJobs } from './jobs' import { sizeBytes } from 'app/src-shared/utils/functions' import { log } from './utils/functions' @@ -28,6 +29,7 @@ export const app = new Hono().basePath('/api') .route('/webhooks', webhooks) .route('/searxng', searxng) .route('/search', search) + .route('/import', importRoutes) export default { fetch: app.fetch, diff --git a/src-shared/aiaw-import.ts b/src-shared/aiaw-import.ts new file mode 100644 index 0000000..9efc9b1 --- /dev/null +++ b/src-shared/aiaw-import.ts @@ -0,0 +1,33 @@ +export type AiawImportStatus = 'queued' | 'running' | 'completed' | 'failed' + +export type AiawImportStage = 'queued' | 'parsing' | 'preparing' | 'importing' | 'committing' | 'completed' | 'failed' + +export interface AiawImportCounts { + workspaces: number + assistants: number + dialogs: number + messages: number + items: number +} + +export interface AiawImportWarnings { + orphanMessages: number + orphanItems: number + missingMessageReferences: number + unknownAssistantReferences: number + skippedBinaryItems: number +} + +export interface AiawImportJobSnapshot { + id: string + status: AiawImportStatus + stage: AiawImportStage + progress: number + folderId?: string + counts: AiawImportCounts + totals: AiawImportCounts + warnings: AiawImportWarnings + error?: string + createdAt: number + updatedAt: number +} diff --git a/src/components/SettingsList.vue b/src/components/SettingsList.vue index ba322f5..6a1b1fd 100644 --- a/src/components/SettingsList.vue +++ b/src/components/SettingsList.vue @@ -408,11 +408,9 @@ import ShortcutKeyInput from './ShortcutKeyInput.vue' import CommonItem from './CommonItem.vue' import { localData } from 'src/utils/local-data' import { useWorkspaceStore } from 'src/stores/workspace' -import { useTemplateRef } from 'vue' -import { genId } from 'app/src-shared/utils/id' -import { mutate } from 'src/utils/zero-session' -import { mutators } from 'app/src-shared/mutators' -import { importAiaw } from 'src/services/import-aiaw' +import { onMounted, useTemplateRef } from 'vue' +import { getActiveAiawImportJob, importAiaw, waitForAiawImportJob } from 'src/services/import-aiaw' +import type { AiawImportJobSnapshot } from 'app/src-shared/aiaw-import' const props = defineProps<{ state: PerfsState @@ -446,11 +444,7 @@ const directoryConfigCaption = t('Some settings apply at the directory level and const workspaceStore = useWorkspaceStore() const fileInput = useTemplateRef('fileInput') -async function importData({ target }) { - const files: File[] = Array.from(target.files) - if (!files.length) return - target.value = '' - const folderId = genId() +async function showImportProgress(task: (update: (job: AiawImportJobSnapshot) => void) => Promise) { const notif = $q.notify({ group: false, timeout: 0, @@ -458,23 +452,26 @@ async function importData({ target }) { message: t('Importing...'), caption: '0%', }) - await mutate(mutators.createFolder({ - id: folderId, - parentId: workspaceStore.id!, - name: t('AIaW Import'), - })).client - await importAiaw(files[0], folderId, progress => { + await task(job => { notif({ - caption: `${(progress * 100).toFixed(1)}%`, + caption: t( + '{0}% · {1}/{2} chats imported', + (job.progress * 100).toFixed(1), + job.counts.dialogs, + job.totals.dialogs || '?', + ), }) - }).then(() => { + }).then(job => { + const skipped = job.warnings.skippedBinaryItems notif({ - type: 'positive', - icon: 'sym_o_done', + type: skipped ? 'warning' : 'positive', + icon: skipped ? 'sym_o_warning' : 'sym_o_done', spinner: false, - message: t('Import successful'), - caption: t('Imported to the workspace root'), - timeout: 3000, + message: skipped ? t('Import completed with warnings') : t('Import successful'), + caption: skipped + ? t('Imported {0} chats and {1} messages; skipped {2} binary attachments', job.counts.dialogs, job.counts.messages, skipped) + : t('Imported {0} chats and {1} messages', job.counts.dialogs, job.counts.messages), + timeout: skipped ? 10000 : 3000, }) }).catch(err => { console.error(err) @@ -488,4 +485,20 @@ async function importData({ target }) { }) }) } + +async function importData({ target }) { + const files: File[] = Array.from(target.files) + if (!files.length) return + target.value = '' + await showImportProgress(update => + importAiaw(files[0], workspaceStore.id!, t('AIaW Import'), update), + ) +} + +onMounted(async () => { + const activeJob = await getActiveAiawImportJob().catch(() => null) + if (activeJob) { + await showImportProgress(update => waitForAiawImportJob(activeJob, update)) + } +}) diff --git a/src/services/import-aiaw.ts b/src/services/import-aiaw.ts index ab370cc..a00873f 100644 --- a/src/services/import-aiaw.ts +++ b/src/services/import-aiaw.ts @@ -1,225 +1,50 @@ -import { mutate } from 'src/utils/zero-session' -import { genId } from 'app/src-shared/utils/id' -import type { AppendMessageArgs } from 'app/src-shared/mutators' -import { mutators } from 'app/src-shared/mutators' -import { upload } from 'src/utils/blob-cache' -import { base64ToUint8Array } from 'app/src-shared/utils/functions' -import { getNameAvatar } from './generate-chat-title' +import type { AiawImportJobSnapshot } from 'app/src-shared/aiaw-import' -interface AiawExport { - formatName: string - formatVersion: number - data: { - databaseName: string - databaseVersion: number - tables: any[] - data: { - tableName: string - inbound?: boolean - rows: any[] - }[] - } +async function responseJson(response: Response): Promise { + const data = await response.json() + if (!response.ok) throw new Error(data.error || `Request failed: ${response.status}`) + return data } -export async function importAiaw(file: File, targetFolderId: string, updateProgress?: (progress: number) => void) { - const content = await file.text() - const dump = JSON.parse(content) as AiawExport - - if (dump.formatName !== 'dexie') { - throw new Error('Unsupported format. Need dexie export.') - } - - const getRows = (tableName: string) => - dump.data.data.find(t => t.tableName === tableName)?.rows || [] - - // Create mapping dictionaries for ids - const idMap = new Map() - const getNewId = (oldId: string | undefined) => { - if (!oldId) return undefined - if (!idMap.has(oldId)) idMap.set(oldId, genId(parseInt(oldId.slice(0, 9), 32))) - return idMap.get(oldId)! - } - - const workspaces = getRows('workspaces') - const dialogs = getRows('dialogs') - const messages = getRows('messages') - const assistants = getRows('assistants') - const avatarImages = getRows('avatarImages') - const items = getRows('items') - - // 1. Avatar Images - for (const av of avatarImages) { - if (!av.contentBuffer) continue - const newId = getNewId(av.id)! - - await mutate(mutators.createItem({ - id: newId, - parentId: targetFolderId, - name: 'avatar', - mimeType: av.mimeType || 'image/png', - hidden: true, - })).client - - const buf = base64ToUint8Array(av.contentBuffer) - const blob = new Blob([buf], { type: av.mimeType || 'image/png' }) - await upload(newId, blob, 'avatar').catch(console.error) - } - - // 2. Workspaces - const wsMap = new Map(workspaces.map(w => [w.id, w])) - const createdWs = new Set() - - const createWs = async (w: any) => { - if (createdWs.has(w.id)) return - if (w.parentId && w.parentId !== '$root' && wsMap.has(w.parentId)) { - await createWs(wsMap.get(w.parentId)) - } - const newId = getNewId(w.id)! - const pId = (!w.parentId || w.parentId === '$root') ? targetFolderId : getNewId(w.parentId)! - await mutate(mutators.createFolder({ - id: newId, - parentId: pId, - name: w.name, - })).client - createdWs.add(w.id) - } - - for (const w of workspaces) { - await createWs(w) - } - - // 3. Assistants - for (const a of assistants) { - const newId = getNewId(a.id)! - const pId = getNewId(a.workspaceId) || targetFolderId - const avatar = a.avatar?.type === 'image' ? { type: 'image', itemId: getNewId(a.avatar.imageId) } : a.avatar - await mutate(mutators.createAssistant({ - id: newId, - parentId: pId, - name: a.name, - avatar, - })).client - await mutate(mutators.updateAssistant({ - id: newId, - prompt: a.prompt, - promptRole: a.promptRole === 'user' ? 'user' : 'system', - })).client - } - - // Group items by dialogId - const itemsByDialog = new Map() - for (const item of items) { - const dialogId = item.dialogId - if (!itemsByDialog.has(dialogId)) { - itemsByDialog.set(dialogId, []) - } - itemsByDialog.get(dialogId)!.push(item) - } - - // 4. Dialogs & Messages - const messagesByDialog = new Map() - for (const m of messages) { - if (!messagesByDialog.has(m.dialogId)) { - messagesByDialog.set(m.dialogId, []) - } - messagesByDialog.get(m.dialogId)!.push(m) - } - - dialogs.sort((a, b) => a.id < b.id ? 1 : -1) // sort dialogs by id desc - - for (let i = 0; i < dialogs.length; i++) { - const d = dialogs[i] - const newChatId = getNewId(d.id)! - const pId = getNewId(d.workspaceId) || targetFolderId - const rootMessageId = genId() - - mutate(mutators.createChat({ - ids: [newChatId, rootMessageId], - parentId: pId, - ...getNameAvatar(d.name), - })) - mutate(mutators.deleteBranch({ - entityId: newChatId, - parent: '$root', - branch: 0, - })) - - // Import Items for this dialog - const dItems = itemsByDialog.get(d.id) || [] - for (const item of dItems) { - const newId = getNewId(item.id)! - - const promise = mutate(mutators.createItem({ - id: newId, - parentId: newChatId, - name: item.name || 'item', - mimeType: item.mimeType, - text: item.contentText, - hidden: false, - })).server - - if (item.contentBuffer) { - const buf = base64ToUint8Array(item.contentBuffer) - const blob = new Blob([buf], { type: item.mimeType || 'application/octet-stream' }) - upload(newId, blob, item.name || 'file', promise) - } - } - - const dMsgs = messagesByDialog.get(d.id) || [] - const msgMap = new Map(dMsgs.map(m => [m.id, m])) +const wait = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) - const appends: AppendMessageArgs[] = [] - - // traverse msgTree - const processMessage = (oldMsgId: string, targetId: string) => { - const m = msgMap.get(oldMsgId) - if (!m) return - - if (m.contents[0] && m.contents[0].type !== 'assistant-tool') { - const content = m.contents[0] - const text = content.text || '' - const reasoning = content.reasoning || null - const isUser = m.type === 'user' - const msgType = isUser ? 'chat:user' : 'chat:assistant' - - const newMsgId = getNewId(m.id)! - - const assistantId = m.assistantId ? getNewId(m.assistantId) : null - - appends.push({ - entityId: newChatId, - target: targetId, - props: { - id: newMsgId, - text, - type: msgType, - assistantId, - reasoning, - modelName: m.modelName, - }, - entities: content.items?.map((it: string) => getNewId(it)).filter(Boolean), - }) - - // continue down the tree - if (d.msgTree && d.msgTree[oldMsgId]) { - for (const childId of d.msgTree[oldMsgId]) { - processMessage(childId, newMsgId) - } - } - } - } +export async function getActiveAiawImportJob() { + const result = await responseJson<{ job: AiawImportJobSnapshot | null }>(await fetch('/api/import/aiaw/active')) + return result.job +} - if (d.msgTree) { - // Find the root messages (those that are values in $root or simply the start of the tree) - const roots = d.msgTree['$root'] - for (const rootId of roots) { - processMessage(rootId, '$root') - } - } - mutate(mutators.appendMessageBatch(appends)) +export async function waitForAiawImportJob( + initialJob: AiawImportJobSnapshot, + updateProgress?: (job: AiawImportJobSnapshot) => void, +) { + let job = initialJob + updateProgress?.(job) + while (job.status === 'queued' || job.status === 'running') { + await wait(document.hidden ? 2000 : 500) + const result = await responseJson<{ job: AiawImportJobSnapshot }>(await fetch(`/api/import/aiaw/${job.id}`)) + job = result.job + updateProgress?.(job) + } + + if (job.status === 'failed') throw new Error(job.error || 'Import failed') + return job +} - // small delay to avoid overwhelming - await new Promise(resolve => setTimeout(resolve, 100)) - updateProgress?.((i + 1) / dialogs.length) - } +export async function importAiaw( + file: File, + targetFolderId: string, + folderName: string, + updateProgress?: (job: AiawImportJobSnapshot) => void, +) { + const form = new FormData() + form.set('file', file) + form.set('parentId', targetFolderId) + form.set('folderName', folderName) + + const { job: startedJob } = await responseJson<{ job: AiawImportJobSnapshot }>(await fetch('/api/import/aiaw', { + method: 'POST', + body: form, + })) + + return await waitForAiawImportJob(startedJob, updateProgress) }