diff --git a/CLAUDE.md b/CLAUDE.md index 574417f8..da83023b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -46,6 +46,10 @@ knowledge/ 配下 329 ファイルを clean アップロードした場合の正 |------|:---------:|:---------:|------| | 2026-03-09 | 329 | 2,891 | clean 後の正確な値 | +### 管理画面から追加した curated ナレッジ + +ダッシュボードの「ナレッジを追加」は、URL・文章・画像/PDF から LLM が下書きを作り、確認後に R2 の `curated/.md` として保存する（frontmatter は `source_type: curated` / `source_authority: 2` / `verified_at` / `url`）。git 管理外なので `knowledge:upload` では投入されず、`--clean` 後は Vectorize 側だけ消える。clean したら `POST /admin/knowledge/sync` で R2 全体を再同期する。 + ## コーディング規約 ### コメント diff --git a/server/src/db/schema.ts b/server/src/db/schema.ts index b8585a3d..48551bc5 100644 --- a/server/src/db/schema.ts +++ b/server/src/db/schema.ts @@ -199,7 +199,7 @@ export const llmUsage = sqliteTable("llm_usage", { cachedInputTokens: integer("cached_input_tokens").notNull().default(0), totalTokens: integer("total_tokens").notNull().default(0), platform: text("platform"), // "web" | "line" | "lp" | "widget" | "voice" | null（バッチ系） - source: text("source").notNull(), // "chat" | "subagent" | "intent-classify" | "persona-extract" | "weekly-report" | "image-convert" + source: text("source").notNull(), agent: text("agent"), // 呼び出し元エージェント名（"nepp-chan" "knowledge" 等）。列追加前の行は null turnIndex: integer("turn_index"), // スレッド内の何往復目か（1 始まり）。列追加前の行は null durationMs: integer("duration_ms"), // 呼び出し 1 回の所要時間 diff --git a/server/src/lib/date.ts b/server/src/lib/date.ts index 1cb0abca..c4bfecee 100644 --- a/server/src/lib/date.ts +++ b/server/src/lib/date.ts @@ -1,7 +1,7 @@ const HOUR_MS = 60 * 60 * 1000; export const DAY_MS = 24 * HOUR_MS; export const WEEK_MS = 7 * DAY_MS; -const JST_OFFSET_MS = 9 * HOUR_MS; +export const JST_OFFSET_MS = 9 * HOUR_MS; /** JST の YYYY-MM-DD ラベルに変換する */ export const jstDateLabel = (date: Date) => diff --git a/server/src/lib/html-to-text.test.ts b/server/src/lib/html-to-text.test.ts new file mode 100644 index 00000000..07a293d7 --- /dev/null +++ b/server/src/lib/html-to-text.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from "vitest"; +import { + decodeEntities, + decodeHtml, + detectCharset, + extractPageText, + htmlToText, +} from "./html-to-text"; + +describe("decodeEntities", () => { + it("名前付き・10 進・16 進のエンティティを文字に戻す", () => { + expect( + decodeEntities("a & b <c> あ い —"), + ).toBe("a & b あ い —"); + }); + + it("未知のエンティティはそのまま残す", () => { + expect(decodeEntities("&unknownthing;")).toBe("&unknownthing;"); + }); +}); + +describe("detectCharset", () => { + it("Content-Type ヘッダの charset を優先する", () => { + expect( + detectCharset('', "text/html; charset=Shift_JIS"), + ).toBe("Shift_JIS"); + }); + + it("ヘッダに無ければ meta charset を見る", () => { + expect(detectCharset('', "text/html")).toBe( + "euc-jp", + ); + }); + + it("http-equiv の content 内 charset も拾う", () => { + expect( + detectCharset( + '', + null, + ), + ).toBe("Shift_JIS"); + }); + + it("どこにも無ければ utf-8", () => { + expect(detectCharset("", null)).toBe("utf-8"); + }); +}); + +describe("decodeHtml", () => { + it("Shift_JIS のバイト列を meta charset に従ってデコードする", () => { + const sjisBytes = new Uint8Array([ + ...new TextEncoder().encode('

'), + 0x89, + 0xb9, + 0x88, + 0xd0, + 0x8e, + 0x71, + 0x95, + 0x7b, + ...new TextEncoder().encode("

"), + ]); + + const html = decodeHtml(sjisBytes.buffer, "text/html"); + + expect(html).toContain("音威子府"); + }); + + it("不正な charset 名は utf-8 にフォールバックする", () => { + const bytes = new TextEncoder().encode( + '

本文

', + ); + + expect(decodeHtml(bytes.buffer as ArrayBuffer, null)).toContain("本文"); + }); +}); + +describe("htmlToText", () => { + it("script / style / noscript / svg を本文から除く", () => { + const html = + "icon

残す

"; + + expect(htmlToText(html)).toBe("残す"); + }); + + it("ブロック要素の終わりを改行に、空白を 1 つに畳む", () => { + const html = + "

見出し

段落 一

末尾
次行
"; + + expect(htmlToText(html)).toBe("見出し\n段落 一\n項目1\n項目2\n末尾\n次行"); + }); + + it("HTML コメントを除き、3 つ以上の連続改行を 2 つにする", () => { + const html = "

a

b

"; + + expect(htmlToText(html)).toBe("a\n\nb"); + }); +}); + +describe("extractPageText", () => { + it("og:title / og:description を優先して拾い、head は本文に含めない", () => { + const html = `タイトルタグ + + + +

本文

`; + + const page = extractPageText(html); + + expect(page.title).toBe("OG タイトル"); + expect(page.description).toBe("OG 説明 & 補足"); + expect(page.text).toBe("本文"); + }); + + it("og が無ければ title タグと meta description を使う", () => { + const html = + '店のページx'; + + const page = extractPageText(html); + + expect(page.title).toBe("店のページ"); + expect(page.description).toBe("説明文"); + }); + + it("title も description も無ければ undefined", () => { + const page = extractPageText("

本文だけ

"); + + expect(page.title).toBeUndefined(); + expect(page.description).toBeUndefined(); + expect(page.text).toBe("本文だけ"); + }); +}); diff --git a/server/src/lib/html-to-text.ts b/server/src/lib/html-to-text.ts new file mode 100644 index 00000000..1d582d1f --- /dev/null +++ b/server/src/lib/html-to-text.ts @@ -0,0 +1,88 @@ +const CHARSET_SNIFF_BYTES = 4096; + +const NAMED_ENTITIES: Record = { + amp: "&", + lt: "<", + gt: ">", + quot: '"', + apos: "'", + nbsp: " ", + mdash: "—", + ndash: "–", + hellip: "…", + copy: "©", +}; + +export const decodeEntities = (text: string) => + text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => { + if (entity.startsWith("#x") || entity.startsWith("#X")) { + return String.fromCodePoint(Number.parseInt(entity.slice(2), 16)); + } + if (entity.startsWith("#")) { + return String.fromCodePoint(Number.parseInt(entity.slice(1), 10)); + } + return NAMED_ENTITIES[entity.toLowerCase()] ?? match; + }); + +const charsetFrom = (text: string | null) => + text?.match(/charset=["']?([\w-]+)/i)?.[1]; + +export const detectCharset = (headHtml: string, contentType: string | null) => + charsetFrom(contentType) ?? charsetFrom(headHtml) ?? "utf-8"; + +export const decodeHtml = (buffer: ArrayBuffer, contentType: string | null) => { + const head = new TextDecoder("latin1").decode( + buffer.slice(0, CHARSET_SNIFF_BYTES), + ); + const charset = detectCharset(head, contentType); + try { + return new TextDecoder(charset).decode(buffer); + } catch { + return new TextDecoder().decode(buffer); + } +}; + +const BLOCK_END_TAGS = + /<\/(?:p|div|li|h[1-6]|tr|section|article|header|footer|blockquote|dd|dt|pre|table)>||/gi; + +export const htmlToText = (html: string) => { + const withBreaks = html + .replace(//g, "") + .replace( + /<(script|style|noscript|svg|template|iframe)\b[\s\S]*?<\/\1>/gi, + "", + ) + .replace(BLOCK_END_TAGS, "\n") + .replace(/<[^>]+>/g, " "); + + return decodeEntities(withBreaks) + .split("\n") + .map((line) => line.replace(/[ \t]+/g, " ").trim()) + .join("\n") + .replace(/\n{3,}/g, "\n\n") + .trim(); +}; + +const metaContent = (html: string, key: string) => { + const attr = `(?:property|name)=["']${key}["']`; + const before = html.match( + new RegExp(`]*${attr}[^>]*content=["']([^"']*)["']`, "i"), + ); + const after = html.match( + new RegExp(`]*content=["']([^"']*)["'][^>]*${attr}`, "i"), + ); + const raw = before?.[1] ?? after?.[1]; + return raw ? decodeEntities(raw).trim() || undefined : undefined; +}; + +export const extractPageText = (html: string) => { + const titleTag = html.match(/]*>([\s\S]*?)<\/title>/i)?.[1]; + const title = + metaContent(html, "og:title") ?? + (titleTag ? decodeEntities(titleTag).trim() || undefined : undefined); + const description = + metaContent(html, "og:description") ?? metaContent(html, "description"); + + const body = html.replace(//i, ""); + return { title, description, text: htmlToText(body) }; +}; diff --git a/server/src/lib/image-converter.ts b/server/src/lib/image-converter.ts index 3f0c85c4..0d878e31 100644 --- a/server/src/lib/image-converter.ts +++ b/server/src/lib/image-converter.ts @@ -1,19 +1,12 @@ import { Buffer } from "node:buffer"; +import { CONVERTIBLE_MIME_TYPES } from "@nepp-chan/shared/constants/knowledge"; import { OPENAI_LITE } from "~/lib/llm-models"; import { converterAgent } from "~/mastra/agents/converter-agent"; import { recordLlmUsage } from "~/services/analytics/llm-usage"; -const SUPPORTED_MIME_TYPES = [ - "image/png", - "image/jpeg", - "image/webp", - "image/gif", - "application/pdf", -] as const; - export const isSupportedMimeType = (mimeType: string) => - SUPPORTED_MIME_TYPES.includes( - mimeType as (typeof SUPPORTED_MIME_TYPES)[number], + CONVERTIBLE_MIME_TYPES.includes( + mimeType as (typeof CONVERTIBLE_MIME_TYPES)[number], ); export const convertToMarkdown = async ( @@ -23,7 +16,7 @@ export const convertToMarkdown = async ( ) => { if (!isSupportedMimeType(mimeType)) { throw new Error( - `Unsupported mime type: ${mimeType}. Supported types: ${SUPPORTED_MIME_TYPES.join(", ")}`, + `Unsupported mime type: ${mimeType}. Supported types: ${CONVERTIBLE_MIME_TYPES.join(", ")}`, ); } diff --git a/server/src/lib/openapi-errors.ts b/server/src/lib/openapi-errors.ts index 3af1db76..622aeba2 100644 --- a/server/src/lib/openapi-errors.ts +++ b/server/src/lib/openapi-errors.ts @@ -17,6 +17,7 @@ const descriptions = { 403: "権限エラー", 404: "リソースが見つかりません", 409: "リソースが競合しています", + 422: "処理できない入力です", 500: "サーバーエラー", } as const; diff --git a/server/src/lib/x-post.test.ts b/server/src/lib/x-post.test.ts new file mode 100644 index 00000000..ce5eeb74 --- /dev/null +++ b/server/src/lib/x-post.test.ts @@ -0,0 +1,65 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { fetchXPostText, isXPostUrl } from "./x-post"; + +describe("isXPostUrl", () => { + it.each([ + "https://x.com/jack/status/20", + "https://twitter.com/jack/status/20?s=20", + "https://mobile.twitter.com/jack/status/20", + ])("%s は個別投稿", (url) => { + expect(isXPostUrl(url)).toBe(true); + }); + + it.each([ + "https://x.com/jack", + "https://x.com/i/lists/1", + "https://example.com/jack/status/20", + "not a url", + ])("%s は個別投稿ではない", (url) => { + expect(isXPostUrl(url)).toBe(false); + }); +}); + +describe("fetchXPostText", () => { + const fetchSpy = vi.spyOn(globalThis, "fetch"); + + beforeEach(() => { + fetchSpy.mockReset(); + }); + + afterEach(() => { + fetchSpy.mockReset(); + }); + + it("oEmbed の html を本文テキストにして author_name と返す", async () => { + fetchSpy.mockResolvedValue( + new Response( + JSON.stringify({ + author_name: "jack", + html: '\n\n', + }), + { status: 200 }, + ), + ); + + const post = await fetchXPostText("https://x.com/jack/status/20"); + + expect(post.authorName).toBe("jack"); + expect(post.text).toBe( + "just setting up my twttr\n— jack (@jack) 2006年3月21日", + ); + const calledUrl = fetchSpy.mock.calls[0]?.[0] as string; + expect(calledUrl).toContain("publish.x.com/oembed"); + expect(calledUrl).toContain( + encodeURIComponent("https://x.com/jack/status/20"), + ); + }); + + it("非 2xx は HTTP ステータス付きで throw する", async () => { + fetchSpy.mockResolvedValue(new Response("", { status: 404 })); + + await expect(fetchXPostText("https://x.com/jack/status/1")).rejects.toThrow( + "HTTP 404", + ); + }); +}); diff --git a/server/src/lib/x-post.ts b/server/src/lib/x-post.ts new file mode 100644 index 00000000..24f23bc2 --- /dev/null +++ b/server/src/lib/x-post.ts @@ -0,0 +1,38 @@ +import { htmlToText } from "~/lib/html-to-text"; + +const X_HOSTS = new Set([ + "x.com", + "www.x.com", + "twitter.com", + "www.twitter.com", + "mobile.twitter.com", +]); + +export const isXPostUrl = (url: string) => { + try { + const parsed = new URL(url); + return ( + X_HOSTS.has(parsed.hostname) && + /^\/[^/]+\/status\/\d+/.test(parsed.pathname) + ); + } catch { + return false; + } +}; + +type OEmbedResponse = { + html: string; + author_name: string; +}; + +export const fetchXPostText = async (url: string, timeoutMs = 15_000) => { + const endpoint = `https://publish.x.com/oembed?url=${encodeURIComponent(url)}&omit_script=true&lang=ja`; + const response = await fetch(endpoint, { + signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) { + throw new Error(`投稿を取得できませんでした（HTTP ${response.status}）`); + } + const data = (await response.json()) as OEmbedResponse; + return { text: htmlToText(data.html), authorName: data.author_name }; +}; diff --git a/server/src/mastra/agents/curated-drafter-agent.ts b/server/src/mastra/agents/curated-drafter-agent.ts new file mode 100644 index 00000000..92d71b31 --- /dev/null +++ b/server/src/mastra/agents/curated-drafter-agent.ts @@ -0,0 +1,18 @@ +import { Agent } from "@mastra/core/agent"; +import { modelWithReasoning } from "~/lib/llm-models"; + +const INSTRUCTIONS = `渡された資料から、地域案内 AI のナレッジになる 1 件分の下書きを日本語で作る。 + +- 資料に書いてあることだけを書く。自分の知識で補わない。資料が 1〜2 文しかなければ下書きも 1〜2 文にとどめる +- 無い項目は書かない。分からないことを「不明」とも書かない +- 複数の資料は 1 つの対象についての情報として統合する +- 営業時間・料金・日程など変わりうる情報は summary に書かず、notice の 1 文で公式サイトや SNS での確認を促す +- sourceLinks には資料中に現れた URL だけを入れる +- slug は対象を表す英小文字とハイフンだけの短い識別子にする（例: otoineppu-tokyo）`; + +export const curatedDrafterAgent = new Agent({ + id: "curated-drafter", + name: "Curated Drafter", + instructions: INSTRUCTIONS, + ...modelWithReasoning({ effort: "low" }), +}); diff --git a/server/src/repository/llm-usage-repository.test.ts b/server/src/repository/llm-usage-repository.test.ts index de078f03..42265de2 100644 --- a/server/src/repository/llm-usage-repository.test.ts +++ b/server/src/repository/llm-usage-repository.test.ts @@ -151,6 +151,13 @@ describe("llmUsageRepository", () => { ); }); + it("curated-draft はナレッジ基盤の費用として knowledge-base に分類する", async () => { + await insert(db, { source: "curated-draft" }); + + const [row] = await llmUsageRepository.sumByCategory(d1, period); + expect(row?.category).toBe("knowledge-base"); + }); + it("会話にも埋め込みにも当たらない source は batch に分類する", async () => { await insert(db, { source: "weekly-report" }); diff --git a/server/src/repository/llm-usage-repository.ts b/server/src/repository/llm-usage-repository.ts index 9eb14ac1..2dcbac2b 100644 --- a/server/src/repository/llm-usage-repository.ts +++ b/server/src/repository/llm-usage-repository.ts @@ -38,7 +38,7 @@ const usageCategoryExpr = sql` CASE WHEN source IN ('chat', 'subagent', 'intent-classify', 'rerank') THEN 'conversation' WHEN source = 'embedding' AND thread_id IS NOT NULL THEN 'conversation' - WHEN source = 'embedding' THEN 'knowledge-base' + WHEN source IN ('embedding', 'curated-draft') THEN 'knowledge-base' ELSE 'batch' END `; diff --git a/server/src/routes/admin/knowledge/convert.ts b/server/src/routes/admin/knowledge/convert.ts index 16eb435a..b3d6fce0 100644 --- a/server/src/routes/admin/knowledge/convert.ts +++ b/server/src/routes/admin/knowledge/convert.ts @@ -1,14 +1,25 @@ import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi"; +import { CURATED_DRAFT_LIMITS } from "@nepp-chan/shared/constants/knowledge"; import { HTTPException } from "hono/http-exception"; - +import { isSupportedMimeType } from "~/lib/image-converter"; import { errorResponse } from "~/lib/openapi-errors"; import type { PrincipalVariables } from "~/lib/principal"; import { + CuratedDraftError, convertAndUpload, + draftCurated, reconvertFromOriginal, uploadMarkdownFile, } from "~/services/knowledge"; -import { requireApiKey, validateFileKey } from "./schemas"; +import { + CuratedDraftRequestSchema, + CuratedDraftResponseSchema, + requireApiKey, + validateFileKey, +} from "./schemas"; + +const toArray = (value: unknown) => + value === undefined ? [] : Array.isArray(value) ? value : [value]; export const knowledgeConvertRoutes = new OpenAPIHono<{ Bindings: CloudflareBindings; @@ -230,3 +241,86 @@ knowledgeConvertRoutes.openapi(reconvertFileRoute, async (c) => { 200, ); }); + +// POST /admin/knowledge/curated-draft - URL・テキスト・画像から curated 下書きを生成 +const curatedDraftRoute = createRoute({ + method: "post", + path: "/curated-draft", + summary: "curated ナレッジの下書きを生成", + description: + "URL・貼り付けテキスト・画像/PDF を資料として読み、curated 形式の Markdown 下書きを返します。R2 には保存しません", + tags: ["Admin - Knowledge"], + request: { + body: { + content: { + "multipart/form-data": { schema: CuratedDraftRequestSchema }, + }, + }, + }, + responses: { + 200: { + description: "下書き", + content: { "application/json": { schema: CuratedDraftResponseSchema } }, + }, + 400: errorResponse(400), + 401: errorResponse(401), + 422: errorResponse(422), + 500: errorResponse(500), + }, +}); + +knowledgeConvertRoutes.openapi(curatedDraftRoute, async (c) => { + const body = await c.req.parseBody({ all: true }); + + const urls = toArray(body.urls) + .filter((v): v is string => typeof v === "string") + .map((v) => v.trim()) + .filter(Boolean); + const files = toArray(body.files).filter((v): v is File => v instanceof File); + const text = typeof body.text === "string" ? body.text.trim() : ""; + + if (!urls.length && !files.length && !text) { + throw new HTTPException(400, { + message: "URL・テキスト・画像のいずれかを入力してください", + }); + } + if (urls.length > CURATED_DRAFT_LIMITS.urls) { + throw new HTTPException(400, { + message: `URL は ${CURATED_DRAFT_LIMITS.urls} 件までです`, + }); + } + if (files.length > CURATED_DRAFT_LIMITS.files) { + throw new HTTPException(400, { + message: `画像・PDF は ${CURATED_DRAFT_LIMITS.files} 件までです`, + }); + } + const unsupported = files.find((f) => !isSupportedMimeType(f.type)); + if (unsupported) { + throw new HTTPException(400, { + message: `未対応のファイル形式です: ${unsupported.name}`, + }); + } + const totalBytes = files.reduce((sum, f) => sum + f.size, 0); + if (totalBytes > CURATED_DRAFT_LIMITS.filesTotalBytes) { + throw new HTTPException(400, { + message: `画像・PDF の合計サイズは ${CURATED_DRAFT_LIMITS.filesTotalBytes / 1024 / 1024}MB までです`, + }); + } + + try { + const result = await draftCurated({ urls, text, files }, { d1: c.env.DB }); + return c.json(result, 200); + } catch (error) { + if (error instanceof CuratedDraftError) { + const details = error.unreadable + .map((item) => `${item.name}（${item.reason}）`) + .join(" / "); + throw new HTTPException(422, { + message: details + ? `${error.message}。読めなかった資料: ${details}` + : error.message, + }); + } + throw error; + } +}); diff --git a/server/src/routes/admin/knowledge/index.test.ts b/server/src/routes/admin/knowledge/index.test.ts index d55a0487..047b1815 100644 --- a/server/src/routes/admin/knowledge/index.test.ts +++ b/server/src/routes/admin/knowledge/index.test.ts @@ -3,7 +3,8 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { requireApiKey, validateFileKey } from "./schemas"; -vi.mock("~/services/knowledge", () => ({ +vi.mock("~/services/knowledge", async (importOriginal) => ({ + ...(await importOriginal()), listFiles: vi.fn(), getFile: vi.fn(), getOriginalFile: vi.fn(), @@ -15,6 +16,7 @@ vi.mock("~/services/knowledge", () => ({ uploadMarkdownFile: vi.fn(), convertAndUpload: vi.fn(), reconvertFromOriginal: vi.fn(), + draftCurated: vi.fn(), })); vi.mock("~/repository/admin-session-repository", () => ({ @@ -126,6 +128,7 @@ describe("knowledge routes 統合テスト", () => { { method: "POST", path: "/upload" }, { method: "POST", path: "/convert" }, { method: "POST", path: "/reconvert" }, + { method: "POST", path: "/curated-draft" }, ])("$method $path - 認証なしは 401", async ({ method, path }) => { const res = await app.request( new Request(`http://localhost${path}`, { method }), @@ -594,3 +597,132 @@ describe("knowledge routes 統合テスト", () => { }); }); }); + +describe("POST /curated-draft", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(adminSessionRepository.findValid).mockResolvedValue({ + token: VALID_OPAQUE_TOKEN, + userId: "user-1", + expiresAt: new Date(Date.now() + 86400000).toISOString(), + createdAt: "2024-01-01T00:00:00Z", + }); + vi.mocked(adminUserRepository.findById).mockResolvedValue(testUser); + }); + + const draftEnv = mockEnv; + + const post = (form: FormData) => + app.request( + authedRequest("/curated-draft", { method: "POST", body: form }), + undefined, + draftEnv, + ); + + const draft = { + key: "curated/otoineppu-tokyo.md", + content: "---\ntitle: x\n---\n# x\n", + readUrls: ["https://a.example/"], + unreadable: [], + }; + + it("urls・text・files を配列に正規化してサービスに渡す", async () => { + vi.mocked(knowledgeService.draftCurated).mockResolvedValue(draft); + const form = new FormData(); + form.append("urls", " https://a.example/ "); + form.append("urls", "https://b.example/"); + form.append("text", " 補足 "); + form.append("files", new File(["x"], "f.png", { type: "image/png" })); + + const res = await post(form); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual(draft); + const [input, deps] = vi.mocked(knowledgeService.draftCurated).mock + .calls[0] as [ + Parameters[0], + Parameters[1], + ]; + expect(input.urls).toEqual(["https://a.example/", "https://b.example/"]); + expect(input.text).toBe("補足"); + expect(input.files.map((f) => f.name)).toEqual(["f.png"]); + expect(deps).toEqual({ d1: draftEnv.DB }); + }); + + it("URL が 1 件だけでも配列として渡す", async () => { + vi.mocked(knowledgeService.draftCurated).mockResolvedValue(draft); + const form = new FormData(); + form.append("urls", "https://a.example/"); + + await post(form); + + const input = vi.mocked(knowledgeService.draftCurated).mock.calls[0]?.[0]; + expect(input?.urls).toEqual(["https://a.example/"]); + expect(input?.files).toEqual([]); + }); + + it("全部空なら 400", async () => { + const form = new FormData(); + form.append("urls", " "); + form.append("text", ""); + + const res = await post(form); + + expect(res.status).toBe(400); + expect(knowledgeService.draftCurated).not.toHaveBeenCalled(); + }); + + it("未対応のファイル形式は 400", async () => { + const form = new FormData(); + form.append("files", new File(["x"], "memo.txt", { type: "text/plain" })); + + const res = await post(form); + + expect(res.status).toBe(400); + const body = (await res.json()) as { error: { message: string } }; + expect(body.error.message).toContain("memo.txt"); + }); + + it("URL が 11 件以上なら 400", async () => { + const form = new FormData(); + for (let i = 0; i < 11; i++) form.append("urls", `https://a.example/${i}`); + + const res = await post(form); + + expect(res.status).toBe(400); + }); + + it("どの資料も読めなければ 422 と案内文", async () => { + const { CuratedDraftError } = await vi.importActual< + typeof import("~/services/knowledge/curated-draft") + >("~/services/knowledge/curated-draft"); + vi.mocked(knowledgeService.draftCurated).mockRejectedValue( + new CuratedDraftError([ + { name: "https://www.instagram.com/usagi/", reason: "HTTP 429" }, + ]), + ); + const form = new FormData(); + form.append("urls", "https://www.instagram.com/usagi/"); + + const res = await post(form); + + expect(res.status).toBe(422); + const body = (await res.json()) as { error: { message: string } }; + expect(body.error.message).toContain("個別投稿の URL"); + expect(body.error.message).toContain( + "https://www.instagram.com/usagi/（HTTP 429）", + ); + }); + + it("想定外のエラーは 500", async () => { + vi.mocked(knowledgeService.draftCurated).mockRejectedValue( + new Error("boom"), + ); + const form = new FormData(); + form.append("text", "本文"); + + const res = await post(form); + + expect(res.status).toBe(500); + }); +}); diff --git a/server/src/routes/admin/knowledge/schemas.ts b/server/src/routes/admin/knowledge/schemas.ts index cbf78b14..03b24119 100644 --- a/server/src/routes/admin/knowledge/schemas.ts +++ b/server/src/routes/admin/knowledge/schemas.ts @@ -56,6 +56,33 @@ export const SaveFileRequestSchema = z.object({ content: z.string(), }); +export const CuratedDraftRequestSchema = z.object({ + urls: z + .any() + .optional() + .openapi({ + type: "array", + items: { type: "string" }, + description: "読み取る URL（最大 10 件）", + }), + text: z.string().optional().openapi({ description: "貼り付けた本文" }), + files: z + .any() + .optional() + .openapi({ + type: "array", + items: { type: "string", format: "binary" }, + description: "画像・PDF（最大 5 件、合計 20MB）", + }), +}); + +export const CuratedDraftResponseSchema = z.object({ + key: z.string(), + content: z.string(), + readUrls: z.array(z.string()), + unreadable: z.array(z.object({ name: z.string(), reason: z.string() })), +}); + export const FileKeyParamSchema = z.object({ key: z.string().openapi({ param: { name: "key", in: "path" } }), }); diff --git a/server/src/services/analytics/llm-usage.ts b/server/src/services/analytics/llm-usage.ts index e6841d92..db2ab3ab 100644 --- a/server/src/services/analytics/llm-usage.ts +++ b/server/src/services/analytics/llm-usage.ts @@ -13,6 +13,7 @@ export type LlmUsageSource = | "persona-extract" | "weekly-report" | "image-convert" + | "curated-draft" | "embedding" | "rerank"; diff --git a/server/src/services/knowledge/curated-draft.test.ts b/server/src/services/knowledge/curated-draft.test.ts new file mode 100644 index 00000000..ca728d05 --- /dev/null +++ b/server/src/services/knowledge/curated-draft.test.ts @@ -0,0 +1,554 @@ +import matter from "gray-matter"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { generateMock, recordLlmUsageMock, convertToMarkdownMock } = vi.hoisted( + () => ({ + generateMock: vi.fn(), + recordLlmUsageMock: vi.fn(), + convertToMarkdownMock: vi.fn(), + }), +); + +vi.mock("~/mastra/agents/curated-drafter-agent", () => ({ + curatedDrafterAgent: { generate: generateMock }, +})); + +vi.mock("~/services/analytics/llm-usage", () => ({ + recordLlmUsage: recordLlmUsageMock, +})); + +vi.mock("~/lib/image-converter", async (importOriginal) => ({ + ...(await importOriginal()), + convertToMarkdown: convertToMarkdownMock, +})); + +const { + buildCuratedMarkdown, + CuratedDraftError, + draftCurated, + normalizeUrl, + toSlug, +} = await import("./curated-draft"); + +const fetchSpy = vi.spyOn(globalThis, "fetch"); + +const htmlResponse = (body: string, init?: ResponseInit & { url?: string }) => { + const response = new Response(body, { + status: init?.status ?? 200, + headers: { "content-type": "text/html; charset=utf-8", ...init?.headers }, + }); + if (init?.url) Object.defineProperty(response, "url", { value: init.url }); + return response; +}; + +const draftFields = { + title: "音威子府TOKYO（東京の音威子府そばの店）", + category: "お店・スポット", + slug: "otoineppu-tokyo", + notice: "営業時間等は公式サイトで確認してください。", + summary: "東京都新宿区にある蕎麦店。\n\n黒い蕎麦を提供する。", + sourceLinks: [] as { label: string; url: string }[], +}; + +const respondWithDraft = (overrides: Partial = {}) => { + generateMock.mockResolvedValue({ + object: { ...draftFields, ...overrides }, + totalUsage: { inputTokens: 10, outputTokens: 5 }, + response: { modelId: "openai/gpt-5.6-luna" }, + }); +}; + +beforeEach(() => { + fetchSpy.mockReset(); + generateMock.mockReset(); + recordLlmUsageMock.mockReset(); + convertToMarkdownMock.mockReset(); +}); + +describe("normalizeUrl", () => { + it("www・末尾スラッシュ・utm・fragment を無視して同一視する", () => { + expect(normalizeUrl("https://www.example.com/shop/?utm_source=x#top")).toBe( + normalizeUrl("http://example.com/shop"), + ); + }); + + it("パスやクエリが違えば別 URL", () => { + expect(normalizeUrl("https://example.com/a")).not.toBe( + normalizeUrl("https://example.com/b"), + ); + expect(normalizeUrl("https://example.com/a?p=1")).not.toBe( + normalizeUrl("https://example.com/a?p=2"), + ); + }); +}); + +describe("toSlug", () => { + it("英小文字とハイフンに正規化する", () => { + expect(toSlug(" Otoineppu TOKYO_shop!! ")).toBe("otoineppu-tokyo-shop"); + }); + + it("60 文字で切っても末尾にハイフンを残さない", () => { + expect(toSlug(`${"a".repeat(59)}-bbb`)).toBe("a".repeat(59)); + }); + + it("空になったら URL の hostname を使う", () => { + expect(toSlug("音威子府", "https://www.peraichi.com/x")).toBe( + "peraichi-com", + ); + }); + + it("URL も無ければ JST の日時から作る", () => { + expect(toSlug("", undefined, new Date("2026-09-02T15:04:00Z"))).toBe( + "curated-20260903-0004", + ); + }); +}); + +describe("buildCuratedMarkdown", () => { + const verifiedAt = "2026-09-02"; + + it("#1083 と同じ frontmatter と本文構成を出す", () => { + const md = buildCuratedMarkdown(draftFields, { + inputUrls: ["https://peraichi.com/landing_pages/view/otoineppu"], + verifiedAt, + }); + const parsed = matter(md); + + expect(parsed.data).toEqual({ + title: draftFields.title, + category: "お店・スポット", + source_type: "curated", + source_authority: 2, + verified_at: "2026-09-02", + url: "https://peraichi.com/landing_pages/view/otoineppu", + }); + expect(parsed.content.trim()).toBe( + [ + "# 音威子府TOKYO（東京の音威子府そばの店）", + "", + "> 営業時間等は公式サイトで確認してください。", + "", + "東京都新宿区にある蕎麦店。", + "", + "黒い蕎麦を提供する。", + "", + "## 情報源", + "", + "- https://peraichi.com/landing_pages/view/otoineppu", + ].join("\n"), + ); + }); + + it("verified_at と url は YAML で文字列として quote される", () => { + const md = buildCuratedMarkdown(draftFields, { + inputUrls: ["https://example.com/"], + verifiedAt, + }); + + expect(md).toContain("verified_at: '2026-09-02'"); + expect(md).toContain("url: 'https://example.com/'"); + }); + + it("入力 URL は読めなくても全部載せ、sourceLinks は重複を除いてラベル付きで続ける", () => { + const md = buildCuratedMarkdown( + { + ...draftFields, + sourceLinks: [ + { label: "公式サイト", url: "https://www.peraichi.com/x/" }, + { label: "紹介記事", url: "https://news.example.com/a" }, + ], + }, + { + inputUrls: [ + "https://peraichi.com/x", + "https://www.instagram.com/usagi/", + ], + verifiedAt, + }, + ); + + expect(matter(md).content).toContain( + [ + "## 情報源", + "", + "- 公式サイト: https://peraichi.com/x", + "- https://www.instagram.com/usagi/", + "- 紹介記事: https://news.example.com/a", + ].join("\n"), + ); + }); + + it("入力 URL が無ければ frontmatter に url を出さず、情報源が空なら見出しも出さない", () => { + const md = buildCuratedMarkdown(draftFields, { inputUrls: [], verifiedAt }); + + expect(matter(md).data).not.toHaveProperty("url"); + expect(md).not.toContain("## 情報源"); + }); +}); + +describe("draftCurated", () => { + const deps = {}; + + it("URL を fetch して本文を LLM に渡し、curated/ 配下の key と Markdown を返す", async () => { + fetchSpy.mockResolvedValue( + htmlResponse( + "店

本文です

", + ), + ); + respondWithDraft(); + + const result = await draftCurated( + { urls: ["https://example.com/shop"], files: [] }, + deps, + ); + + expect(result.key).toBe("curated/otoineppu-tokyo.md"); + expect(result.readUrls).toEqual(["https://example.com/shop"]); + expect(result.unreadable).toEqual([]); + expect(matter(result.content).data.url).toBe("https://example.com/shop"); + const prompt = generateMock.mock.calls[0]?.[0] as string; + expect(prompt).toContain("## 資料 1（URL: https://example.com/shop）"); + expect(prompt).toContain("店\n本文です"); + expect(generateMock.mock.calls[0]?.[1]).toMatchObject({ + structuredOutput: { schema: expect.anything() }, + }); + }); + + it("複数 URL のうち 1 つが 404 でも残りで下書きを作り、unreadable に載せる", async () => { + fetchSpy + .mockResolvedValueOnce(htmlResponse("

読める

")) + .mockResolvedValueOnce(htmlResponse("", { status: 404 })); + respondWithDraft(); + + const result = await draftCurated( + { urls: ["https://a.example/", "https://b.example/"], files: [] }, + deps, + ); + + expect(result.readUrls).toEqual(["https://a.example/"]); + expect(result.unreadable).toEqual([ + { name: "https://b.example/", reason: "HTTP 404" }, + ]); + expect(matter(result.content).content).toContain("- https://b.example/"); + }); + + it("ログインページに転送された URL は資料に混ぜない", async () => { + fetchSpy.mockResolvedValue( + htmlResponse("

ログイン アカウント登録

", { + url: "https://www.instagram.com/accounts/login/?next=%2Fusagi%2F", + }), + ); + + await expect( + draftCurated( + { urls: ["https://www.instagram.com/usagi/"], files: [] }, + deps, + ), + ).rejects.toBeInstanceOf(CuratedDraftError); + expect(generateMock).not.toHaveBeenCalled(); + }); + + it("別ホストへの転送でもログインページでなければ読む", async () => { + fetchSpy.mockResolvedValue( + htmlResponse("

本文

", { url: "https://shop.example.net/about" }), + ); + respondWithDraft(); + + const result = await draftCurated( + { urls: ["https://example.com/shop"], files: [] }, + deps, + ); + + expect(result.readUrls).toEqual(["https://example.com/shop"]); + }); + + it("形式が URL でないものは fetch せず unreadable にする", async () => { + respondWithDraft(); + + const result = await draftCurated( + { urls: ["ftp://example.com/x", "not a url"], text: "本文", files: [] }, + deps, + ); + + expect(fetchSpy).not.toHaveBeenCalled(); + expect(result.unreadable.map((u) => u.name)).toEqual([ + "ftp://example.com/x", + "not a url", + ]); + expect(matter(result.content).data).not.toHaveProperty("url"); + }); + + it("全部読めなければ no_content で失敗し、読めなかった理由を持つ", async () => { + fetchSpy.mockResolvedValue(htmlResponse("", { status: 500 })); + + await expect( + draftCurated({ urls: ["https://a.example/"], files: [] }, deps), + ).rejects.toMatchObject({ + unreadable: [{ name: "https://a.example/", reason: "HTTP 500" }], + }); + }); + + it("readUrls は完了順ではなく入力順で返す", async () => { + fetchSpy.mockImplementation(async (url) => { + if (String(url).includes("slow")) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + return htmlResponse("

本文

"); + }); + respondWithDraft(); + + const result = await draftCurated( + { urls: ["https://slow.example/", "https://fast.example/"], files: [] }, + deps, + ); + + expect(result.readUrls).toEqual([ + "https://slow.example/", + "https://fast.example/", + ]); + }); + + it("資料が多いときはプロンプト枠を資料ごとに等分し、後ろの資料も落とさない", async () => { + fetchSpy.mockImplementation(async () => + htmlResponse(`

${"あ".repeat(30_000)}

`), + ); + respondWithDraft(); + + await draftCurated( + { + urls: [ + "https://a.example/", + "https://b.example/", + "https://c.example/", + ], + text: "職員の補足", + files: [], + }, + deps, + ); + + const prompt = generateMock.mock.calls[0]?.[0] as string; + expect(prompt).toContain("## 資料 4（入力済みのテキスト）\n\n職員の補足"); + expect(prompt.length).toBeLessThanOrEqual(60_000 + 4 * 100); + }); + + it("資料の本文は 30,000 文字で切って LLM に渡す", async () => { + fetchSpy.mockResolvedValue(htmlResponse(`

${"あ".repeat(40_000)}

`)); + respondWithDraft(); + + await draftCurated({ urls: ["https://a.example/"], files: [] }, deps); + + const prompt = generateMock.mock.calls[0]?.[0] as string; + expect(prompt.match(/あ/g)).toHaveLength(30_000); + }); + + it("本文が短くても資料として通す", async () => { + fetchSpy.mockResolvedValue(htmlResponse("

開店準備中

")); + respondWithDraft(); + + const result = await draftCurated( + { urls: ["https://a.example/"], files: [] }, + deps, + ); + + expect(result.readUrls).toEqual(["https://a.example/"]); + }); + + it("X の投稿 URL は oEmbed で読む", async () => { + fetchSpy.mockResolvedValue( + new Response( + JSON.stringify({ + author_name: "shop", + html: "

本日開店

", + }), + { status: 200 }, + ), + ); + respondWithDraft(); + + await draftCurated( + { urls: ["https://x.com/shop/status/123"], files: [] }, + deps, + ); + + expect(fetchSpy.mock.calls[0]?.[0]).toContain("publish.x.com/oembed"); + const prompt = generateMock.mock.calls[0]?.[0] as string; + expect(prompt).toContain("X の投稿 @shop"); + expect(prompt).toContain("本日開店"); + }); + + it("URL 先が PDF や画像なら convertToMarkdown に渡す", async () => { + fetchSpy.mockResolvedValue( + new Response(new Uint8Array([1, 2, 3]), { + status: 200, + headers: { "content-type": "application/pdf" }, + }), + ); + convertToMarkdownMock.mockResolvedValue("# PDF の内容"); + respondWithDraft(); + + await draftCurated({ urls: ["https://a.example/f.pdf"], files: [] }, deps); + + expect(convertToMarkdownMock).toHaveBeenCalledWith( + expect.any(ArrayBuffer), + "application/pdf", + undefined, + ); + expect(generateMock.mock.calls[0]?.[0]).toContain("# PDF の内容"); + }); + + it("Content-Length が 5MB を超える URL は本文を読まずに unreadable にする", async () => { + fetchSpy.mockResolvedValue( + htmlResponse("

小さい本文

", { + headers: { "content-length": String(6 * 1024 * 1024) }, + }), + ); + respondWithDraft(); + + const result = await draftCurated( + { urls: ["https://a.example/huge"], text: "本文", files: [] }, + deps, + ); + + expect(result.unreadable).toEqual([ + { name: "https://a.example/huge", reason: "5MB を超えています" }, + ]); + }); + + it("Content-Length が無くても本文が 5MB を超えたら読み捨てて unreadable にする", async () => { + const big = new Uint8Array(6 * 1024 * 1024); + fetchSpy.mockResolvedValue( + new Response(big, { + status: 200, + headers: { "content-type": "text/html" }, + }), + ); + respondWithDraft(); + + const result = await draftCurated( + { urls: ["https://a.example/huge"], text: "本文", files: [] }, + deps, + ); + + expect(result.unreadable).toEqual([ + { name: "https://a.example/huge", reason: "5MB を超えています" }, + ]); + }); + + it("unreadable は完了順ではなく入力順で並ぶ", async () => { + fetchSpy.mockImplementation(async (url) => { + if (String(url).includes("slow")) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + return htmlResponse("", { status: 404 }); + }); + respondWithDraft(); + + const result = await draftCurated( + { + urls: ["https://slow.example/", "https://fast.example/"], + text: "本文", + files: [], + }, + deps, + ); + + expect(result.unreadable.map((u) => u.name)).toEqual([ + "https://slow.example/", + "https://fast.example/", + ]); + }); + + it("1 件目の URL が読めなくても frontmatter の url は入力 1 件目のまま", async () => { + fetchSpy + .mockResolvedValueOnce(htmlResponse("", { status: 404 })) + .mockResolvedValueOnce(htmlResponse("

読める

")); + respondWithDraft(); + + const result = await draftCurated( + { + urls: ["https://www.instagram.com/usagi/", "https://b.example/"], + files: [], + }, + deps, + ); + + expect(matter(result.content).data.url).toBe( + "https://www.instagram.com/usagi/", + ); + expect(result.readUrls).toEqual(["https://b.example/"]); + }); + + it("未対応の Content-Type は unreadable にする", async () => { + fetchSpy.mockResolvedValue( + new Response("{}", { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + respondWithDraft(); + + const result = await draftCurated( + { urls: ["https://a.example/api"], text: "本文", files: [] }, + deps, + ); + + expect(result.unreadable[0]?.reason).toContain("未対応の形式"); + }); + + it("アップロードされた画像は convertToMarkdown に渡し、ラベルにファイル名を付ける", async () => { + convertToMarkdownMock.mockResolvedValue("チラシの文字"); + respondWithDraft(); + const file = new File(["img"], "flyer.png", { type: "image/png" }); + + const result = await draftCurated({ urls: [], files: [file] }, deps); + + expect(convertToMarkdownMock).toHaveBeenCalledWith( + expect.any(ArrayBuffer), + "image/png", + undefined, + ); + const prompt = generateMock.mock.calls[0]?.[0] as string; + expect(prompt).toContain("画像・PDF: flyer.png"); + expect(matter(result.content).data).not.toHaveProperty("url"); + }); + + it("入力済みのテキストは最後の資料として渡す", async () => { + fetchSpy.mockResolvedValue(htmlResponse("

ページ

")); + respondWithDraft(); + + await draftCurated( + { urls: ["https://a.example/"], text: " 補足メモ ", files: [] }, + deps, + ); + + const prompt = generateMock.mock.calls[0]?.[0] as string; + expect(prompt).toContain("## 資料 2（入力済みのテキスト）\n\n補足メモ"); + }); + + it("d1 があれば curated-draft として usage を記録する", async () => { + respondWithDraft(); + const d1 = {} as D1Database; + + await draftCurated({ urls: [], text: "本文", files: [] }, { ...deps, d1 }); + + expect(recordLlmUsageMock).toHaveBeenCalledWith(d1, { + model: "openai/gpt-5.6-luna", + usage: { inputTokens: 10, outputTokens: 5 }, + source: "curated-draft", + agent: "curated-drafter", + }); + }); + + it("slug が使えなければ入力 URL の hostname から key を作る", async () => { + fetchSpy.mockResolvedValue(htmlResponse("

本文

")); + respondWithDraft({ slug: "店" }); + + const result = await draftCurated( + { urls: ["https://www.peraichi.com/x"], files: [] }, + deps, + ); + + expect(result.key).toBe("curated/peraichi-com.md"); + }); +}); diff --git a/server/src/services/knowledge/curated-draft.ts b/server/src/services/knowledge/curated-draft.ts new file mode 100644 index 00000000..0d0c883e --- /dev/null +++ b/server/src/services/knowledge/curated-draft.ts @@ -0,0 +1,343 @@ +import matter from "gray-matter"; +import { z } from "zod"; +import { JST_OFFSET_MS, jstDateLabel } from "~/lib/date"; +import { decodeHtml, extractPageText } from "~/lib/html-to-text"; +import { convertToMarkdown, isSupportedMimeType } from "~/lib/image-converter"; +import { OPENAI_LITE } from "~/lib/llm-models"; +import { fetchXPostText, isXPostUrl } from "~/lib/x-post"; +import { curatedDrafterAgent } from "~/mastra/agents/curated-drafter-agent"; +import { recordLlmUsage } from "~/services/analytics/llm-usage"; + +const FETCH_TIMEOUT_MS = 15_000; +const MAX_FETCH_BYTES = 5 * 1024 * 1024; +const MAX_SOURCE_CHARS = 30_000; +const MAX_PROMPT_CHARS = 60_000; +const HTML_MIME_TYPES = new Set(["text/html", "application/xhtml+xml"]); + +export const NO_CONTENT_MESSAGE = + "どの資料からも本文を取得できませんでした。Instagram / X のプロフィールページやログインが必要なページは読み取れないことがあります。個別投稿の URL、本文のコピー、スクリーンショット、検索キーワードから作成してください"; + +export class CuratedDraftError extends Error { + constructor(readonly unreadable: Unreadable[]) { + super(NO_CONTENT_MESSAGE); + this.name = "CuratedDraftError"; + } +} + +export type CuratedDraftInput = { + urls: string[]; + text?: string; + files: File[]; +}; + +type Deps = { + d1?: D1Database; +}; + +type SourceDocument = { label: string; text: string; url?: string }; +export type Unreadable = { name: string; reason: string }; + +const CuratedDraftSchema = z.object({ + title: z.string().describe("対象の名称と短い補足"), + category: z.string().describe("分類。店なら「お店・スポット」など"), + slug: z.string().describe("英小文字とハイフンだけの識別子"), + notice: z.string().describe("冒頭に置く注意書き 1 文"), + summary: z.string().describe("本文。段落は空行で区切る"), + sourceLinks: z + .array(z.object({ label: z.string(), url: z.string() })) + .describe("資料中に現れた情報源 URL"), +}); + +export type CuratedDraftFields = z.infer; + +const stripWww = (host: string) => host.toLowerCase().replace(/^www\./, ""); + +export const normalizeUrl = (url: string) => { + try { + const parsed = new URL(url); + for (const key of [...parsed.searchParams.keys()]) { + if (key.startsWith("utm_")) parsed.searchParams.delete(key); + } + parsed.hash = ""; + const path = parsed.pathname.replace(/\/+$/, ""); + return `${stripWww(parsed.hostname)}${path}${parsed.search}`; + } catch { + return url; + } +}; + +const isHttpUrl = (url: string) => { + try { + const { protocol } = new URL(url); + return protocol === "http:" || protocol === "https:"; + } catch { + return false; + } +}; + +const isLoginPage = (final: URL) => + /\/(login|signin|accounts)(\/|$)/i.test(final.pathname); + +const toReason = (error: unknown) => { + if (error instanceof Error) { + if (error.name === "TimeoutError") { + return `応答がありませんでした（${FETCH_TIMEOUT_MS / 1000} 秒）`; + } + return error.message; + } + return "不明なエラー"; +}; + +const readBodyWithLimit = async (response: Response, maxBytes: number) => { + const tooLarge = new Error(`${maxBytes / 1024 / 1024}MB を超えています`); + if (Number(response.headers.get("content-length")) > maxBytes) { + throw tooLarge; + } + if (!response.body) return new ArrayBuffer(0); + + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > maxBytes) { + await reader.cancel(); + throw tooLarge; + } + chunks.push(value); + } + const merged = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + merged.set(chunk, offset); + offset += chunk.byteLength; + } + return merged.buffer; +}; + +const readUrl = async ( + url: string, + d1?: D1Database, +): Promise => { + if (isXPostUrl(url)) { + const post = await fetchXPostText(url, FETCH_TIMEOUT_MS); + return { + label: `URL: ${url}（X の投稿 @${post.authorName}）`, + text: post.text, + }; + } + + const response = await fetch(url, { + redirect: "follow", + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + headers: { + "User-Agent": "nepp-chan-knowledge-import/1.0", + Accept: "text/html,application/pdf,image/*,text/plain", + "Accept-Language": "ja", + }, + }); + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + if (response.url && isLoginPage(new URL(response.url))) { + throw new Error("ログインページに転送されました"); + } + + const contentType = response.headers.get("content-type") ?? ""; + const mime = contentType.split(";")[0]?.trim().toLowerCase() ?? ""; + const buffer = await readBodyWithLimit(response, MAX_FETCH_BYTES); + + let text: string; + if (isSupportedMimeType(mime)) { + text = await convertToMarkdown(buffer, mime, d1); + } else if (mime === "text/plain") { + text = decodeHtml(buffer, contentType); + } else if (HTML_MIME_TYPES.has(mime)) { + const page = extractPageText(decodeHtml(buffer, contentType)); + text = [page.title, page.description, page.text].filter(Boolean).join("\n"); + } else { + throw new Error(`未対応の形式です（${mime || "不明"}）`); + } + + if (!text.trim()) { + throw new Error("本文が空でした"); + } + return { label: `URL: ${url}`, text }; +}; + +type ReadResult = + | { ok: true; doc: SourceDocument } + | { ok: false; unreadable: Unreadable }; + +const readSources = async (input: CuratedDraftInput, deps: Deps) => { + const inputUrls = input.urls.filter(isHttpUrl); + const invalidUrls: ReadResult[] = input.urls + .filter((url) => !isHttpUrl(url)) + .map((url) => ({ + ok: false, + unreadable: { name: url, reason: "URL の形式が正しくありません" }, + })); + + const attempt = async ( + name: string, + read: () => Promise, + ): Promise => { + try { + return { ok: true, doc: await read() }; + } catch (error) { + return { ok: false, unreadable: { name, reason: toReason(error) } }; + } + }; + + const readOne = (url: string) => + attempt(url, async () => ({ ...(await readUrl(url, deps.d1)), url })); + + const fileTask = (file: File) => + attempt(file.name, async () => ({ + label: `画像・PDF: ${file.name}`, + text: await convertToMarkdown( + await file.arrayBuffer(), + file.type, + deps.d1, + ), + })); + + const [urlResults, fileResults] = await Promise.all([ + Promise.all(inputUrls.map(readOne)), + Promise.all(input.files.map(fileTask)), + ]); + + const results = [...invalidUrls, ...urlResults, ...fileResults]; + const sources = results.flatMap((r) => (r.ok ? [r.doc] : [])); + const unreadable = results.flatMap((r) => (r.ok ? [] : [r.unreadable])); + const readUrls = sources.flatMap((doc) => (doc.url ? [doc.url] : [])); + const text = input.text?.trim(); + if (text) { + sources.push({ label: "入力済みのテキスト", text }); + } + + return { + sources: sources.map((doc) => ({ + ...doc, + text: doc.text.slice(0, MAX_SOURCE_CHARS), + })), + unreadable, + readUrls, + inputUrls, + }; +}; + +const buildPrompt = (sources: SourceDocument[]) => { + const perSource = Math.floor(MAX_PROMPT_CHARS / sources.length); + return sources + .map( + (source, index) => + `## 資料 ${index + 1}（${source.label}）\n\n${source.text.slice(0, perSource)}`, + ) + .join("\n\n"); +}; + +export const toSlug = ( + slug: string, + fallbackUrl?: string, + now = new Date(), +) => { + const normalized = slug + .toLowerCase() + .normalize("NFKD") + .replace(/[^a-z0-9]+/g, "-") + .slice(0, 60) + .replace(/^-+|-+$/g, ""); + if (normalized) return normalized; + + if (fallbackUrl && isHttpUrl(fallbackUrl)) { + const host = stripWww(new URL(fallbackUrl).hostname) + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); + if (host) return host; + } + + const jst = new Date(now.getTime() + JST_OFFSET_MS).toISOString(); + return `curated-${jst.slice(0, 10).replace(/-/g, "")}-${jst.slice(11, 16).replace(":", "")}`; +}; + +export const buildCuratedMarkdown = ( + draft: CuratedDraftFields, + options: { inputUrls: string[]; verifiedAt: string }, +) => { + const frontmatter: Record = { + title: draft.title, + category: draft.category, + source_type: "curated", + source_authority: 2, + verified_at: options.verifiedAt, + }; + if (options.inputUrls[0]) frontmatter.url = options.inputUrls[0]; + + const links = new Map(); + for (const url of options.inputUrls) { + const key = normalizeUrl(url); + if (!links.has(key)) links.set(key, { url }); + } + for (const link of draft.sourceLinks) { + const key = normalizeUrl(link.url); + const existing = links.get(key); + if (existing) { + if (!existing.label && link.label) existing.label = link.label; + } else { + links.set(key, { label: link.label || undefined, url: link.url }); + } + } + + const lines = [ + `# ${draft.title}`, + "", + `> ${draft.notice}`, + "", + draft.summary.trim(), + ]; + if (links.size > 0) { + lines.push("", "## 情報源", ""); + for (const link of links.values()) { + lines.push(link.label ? `- ${link.label}: ${link.url}` : `- ${link.url}`); + } + } + + return matter.stringify(`${lines.join("\n")}\n`, frontmatter); +}; + +export const draftCurated = async (input: CuratedDraftInput, deps: Deps) => { + const { sources, unreadable, readUrls, inputUrls } = await readSources( + input, + deps, + ); + if (sources.length === 0) { + throw new CuratedDraftError(unreadable); + } + + const result = await curatedDrafterAgent.generate(buildPrompt(sources), { + structuredOutput: { schema: CuratedDraftSchema }, + }); + if (deps.d1) { + await recordLlmUsage(deps.d1, { + model: result.response?.modelId ?? OPENAI_LITE, + usage: result.totalUsage, + source: "curated-draft", + agent: "curated-drafter", + }); + } + const fields = result.object; + if (!fields) { + throw new Error("下書きの生成に失敗しました"); + } + + const content = buildCuratedMarkdown(fields, { + inputUrls, + verifiedAt: jstDateLabel(new Date()), + }); + const key = `curated/${toSlug(fields.slug, inputUrls[0])}.md`; + + return { key, content, readUrls, unreadable }; +}; diff --git a/server/src/services/knowledge/index.ts b/server/src/services/knowledge/index.ts index ca2b01b8..e8803d91 100644 --- a/server/src/services/knowledge/index.ts +++ b/server/src/services/knowledge/index.ts @@ -1,3 +1,8 @@ +export { + CuratedDraftError, + type CuratedDraftInput, + draftCurated, +} from "./curated-draft"; export { deleteAllKnowledge, deleteKnowledgeBySource, diff --git a/shared/src/api/repository/knowledge-repository.test.ts b/shared/src/api/repository/knowledge-repository.test.ts index 1e022bac..ebb3f436 100644 --- a/shared/src/api/repository/knowledge-repository.test.ts +++ b/shared/src/api/repository/knowledge-repository.test.ts @@ -6,7 +6,7 @@ import { testApiClient, } from "../../test/api-client"; import { server } from "../../test/msw-server"; -import { createKnowledgeRepository } from "./knowledge-repository"; +import { createKnowledgeRepository, toFormData } from "./knowledge-repository"; const repo = createKnowledgeRepository(testApiClient, API); @@ -18,6 +18,26 @@ afterEach(() => { setTestAuthToken(null); }); +describe("toFormData", () => { + it("配列は同名フィールドで複数 append し、null / undefined は送らない", () => { + const file = new File(["a"], "a.png", { type: "image/png" }); + const fd = toFormData({ + urls: ["https://a.example/", "https://b.example/"], + files: [file], + text: undefined, + nothing: null, + }); + + expect(fd.getAll("urls")).toEqual([ + "https://a.example/", + "https://b.example/", + ]); + expect(fd.getAll("files")).toHaveLength(1); + expect(fd.has("text")).toBe(false); + expect(fd.has("nothing")).toBe(false); + }); +}); + describe("knowledge-repository", () => { it("syncKnowledge: POST /admin/knowledge/sync", async () => { server.use( @@ -103,6 +123,26 @@ describe("knowledge-repository", () => { await repo.convertFile(new File(["x"], "in.pdf"), "in.pdf"); }); + it("draftCurated: POST /admin/knowledge/curated-draft に multipart で送る", async () => { + server.use( + http.post(`${API}/admin/knowledge/curated-draft`, () => + HttpResponse.json({ + key: "curated/x.md", + content: "# x", + readUrls: [], + unreadable: [], + }), + ), + ); + + const result = await repo.draftCurated({ + urls: ["https://a.example/"], + files: [new File(["a"], "a.png", { type: "image/png" })], + }); + + expect(result?.key).toBe("curated/x.md"); + }); + it("fetchUnifiedFiles", async () => { server.use( http.get(`${API}/admin/knowledge/unified`, () => diff --git a/shared/src/api/repository/knowledge-repository.ts b/shared/src/api/repository/knowledge-repository.ts index 946be6ce..21313240 100644 --- a/shared/src/api/repository/knowledge-repository.ts +++ b/shared/src/api/repository/knowledge-repository.ts @@ -1,9 +1,12 @@ import type { ApiClient } from "../create-client"; +import type { CuratedDraftRequest } from "../types"; -const toFormData = (body: unknown) => { +export const toFormData = (body: unknown) => { const fd = new FormData(); for (const [key, value] of Object.entries(body as Record)) { - if (value != null) fd.append(key, value as string | Blob); + if (value == null) continue; + const values = Array.isArray(value) ? value : [value]; + for (const v of values) fd.append(key, v as string | Blob); } return fd; }; @@ -76,6 +79,22 @@ export const createKnowledgeRepository = ( return data; }, + draftCurated: async (request: CuratedDraftRequest) => { + const { data, error } = await client.POST( + "/admin/knowledge/curated-draft", + { + body: { + urls: request.urls, + text: request.text, + files: request.files as unknown as string[], + }, + bodySerializer: toFormData, + }, + ); + if (error) throw error; + return data; + }, + fetchUnifiedFiles: async () => { const { data, error } = await client.GET("/admin/knowledge/unified"); if (error) throw error; diff --git a/shared/src/api/types.d.ts b/shared/src/api/types.d.ts index 712b256f..4bc102a4 100644 --- a/shared/src/api/types.d.ts +++ b/shared/src/api/types.d.ts @@ -3278,6 +3278,120 @@ export interface paths { patch?: never; trace?: never; }; + "/admin/knowledge/curated-draft": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * curated ナレッジの下書きを生成 + * @description URL・貼り付けテキスト・画像/PDF・検索キーワードを資料として読み、curated 形式の Markdown 下書きを返します。R2 には保存しません + */ + post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: { + content: { + "multipart/form-data": { + /** @description 読み取る URL（最大 10 件） */ + urls?: string[]; + /** @description 貼り付けた本文 */ + text?: string; + /** @description 画像・PDF（最大 5 件、合計 20MB） */ + files?: string[]; + }; + }; + }; + responses: { + /** @description 下書き */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + key: string; + content: string; + readUrls: string[]; + unreadable: { + name: string; + reason: string; + }[]; + }; + }; + }; + /** @description リクエストエラー */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: { + code: number; + message: string; + }; + }; + }; + }; + /** @description 認証エラー */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: { + code: number; + message: string; + }; + }; + }; + }; + /** @description 処理できない入力です */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: { + code: number; + message: string; + }; + }; + }; + }; + /** @description サーバーエラー */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: { + code: number; + message: string; + }; + }; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/admin/persona": { parameters: { query?: never; diff --git a/shared/src/api/types.ts b/shared/src/api/types.ts index 4ba39f4b..2fff1cda 100644 --- a/shared/src/api/types.ts +++ b/shared/src/api/types.ts @@ -108,6 +108,14 @@ export type SaveFileResponse = PutOk<"/admin/knowledge/files/{key}">; export type UnifiedFilesListResponse = GetOk<"/admin/knowledge/unified">; export type UnifiedFileInfo = UnifiedFilesListResponse["files"][number]; +export type CuratedDraft = PostOk<"/admin/knowledge/curated-draft">; +// multipart なので生成型を使わず手書き +export type CuratedDraftRequest = { + urls: string[]; + text?: string; + files: File[]; +}; + // multipart レスポンス型（raw fetch で使用） export type ReconvertFileResponse = PostOk<"/admin/knowledge/reconvert">; diff --git a/shared/src/constants/knowledge.ts b/shared/src/constants/knowledge.ts new file mode 100644 index 00000000..1537f13e --- /dev/null +++ b/shared/src/constants/knowledge.ts @@ -0,0 +1,13 @@ +export const CONVERTIBLE_MIME_TYPES = [ + "image/png", + "image/jpeg", + "image/webp", + "image/gif", + "application/pdf", +] as const; + +export const CURATED_DRAFT_LIMITS = { + urls: 10, + files: 5, + filesTotalBytes: 20 * 1024 * 1024, +} as const; diff --git a/web/src/app/dashboard/components/KnowledgePanel.tsx b/web/src/app/dashboard/components/KnowledgePanel.tsx index d4762561..882978cd 100644 --- a/web/src/app/dashboard/components/KnowledgePanel.tsx +++ b/web/src/app/dashboard/components/KnowledgePanel.tsx @@ -1,29 +1,42 @@ import { useState } from "react"; -import { FileList, FileViewer } from "~/app/dashboard/components/knowledge"; +import { + CuratedComposer, + FileEditor, + FileList, + FileViewer, +} from "~/app/dashboard/components/knowledge"; +import { partitionFiles } from "~/app/dashboard/components/knowledge/helpers"; import { useDeleteFile, useUnifiedFiles, } from "~/app/dashboard/hooks/useKnowledge"; -/** - * ナレッジ管理パネル - * - * NOTE: アップロード・編集機能は一時的に無効化 - * 運用フェーズで必要に応じて復活させる - * 関連コンポーネント: FileUpload, FileEditor - */ +type FileTab = "curated" | "base"; + +const FILE_TABS = [ + { id: "curated", label: "追加したナレッジ" }, + { id: "base", label: "基本ナレッジ" }, +] as const; + +const EMPTY_MESSAGE: Record = { + curated: "まだありません。上の「ナレッジを追加」から作れます", + base: "ファイルがありません", +}; + export const KnowledgePanel = () => { const { data: filesData, isLoading, error } = useUnifiedFiles(); + const [fileTab, setFileTab] = useState("curated"); const [viewingFile, setViewingFile] = useState(null); + const [editingFile, setEditingFile] = useState(null); const [message, setMessage] = useState<{ type: "success" | "error"; text: string; } | null>(null); const deleteFileMutation = useDeleteFile(); - // TODO: 運用時に復活 - // const [editingFile, setEditingFile] = useState(null); - // const reconvertMutation = useReconvertFile(); + const existingKeys = + filesData?.files.flatMap((f) => (f.markdown ? [f.markdown.key] : [])) ?? []; + const partitioned = partitionFiles(filesData?.files ?? []); const handleDeleteFile = (baseName: string) => { if ( @@ -50,48 +63,50 @@ export const KnowledgePanel = () => { }); }; - // const handleReconvert = (originalKey: string, baseName: string) => { - // if ( - // !confirm( - // `${baseName} のMarkdownを元ファイルから再生成しますか？\n（現在の編集内容は上書きされます）`, - // ) - // ) { - // return; - // } - // setMessage(null); - // reconvertMutation.mutate( - // { originalKey, filename: baseName }, - // { - // onSuccess: (result) => { - // setMessage({ - // type: "success", - // text: `${result.key} を生成しました（${result.chunks}チャンク）`, - // }); - // }, - // onError: (err) => { - // setMessage({ - // type: "error", - // text: `変換失敗: ${err instanceof Error ? err.message : "Unknown error"}`, - // }); - // }, - // }, - // ); - // }; - return (
- {/* TODO: 運用時に復活 - アップロードセクション */} - {/*
+

- ファイルアップロード + ナレッジを追加

- {}} /> -
*/} + +
- {/* ファイル一覧セクション */}

ファイル一覧

+
+ {FILE_TABS.map((tab) => ( + + ))} +
+ {message && (
{ {filesData && ( setViewingFile(key)} + onEdit={(key) => setEditingFile(key)} onDelete={handleDeleteFile} isDeleting={deleteFileMutation.isPending} - // TODO: 運用時に復活 - // onEdit={(key) => setEditingFile(key)} - // onReconvert={handleReconvert} - // isReconverting={reconvertMutation.isPending} + emptyMessage={EMPTY_MESSAGE[fileTab]} /> )}
- {/* 閲覧モーダル */} {viewingFile && ( { /> )} - {/* TODO: 運用時に復活 - エディタモーダル */} - {/* {editingFile && ( + {editingFile && ( setEditingFile(null)} /> - )} */} + )}
); }; diff --git a/web/src/app/dashboard/components/knowledge/CuratedComposer.test.tsx b/web/src/app/dashboard/components/knowledge/CuratedComposer.test.tsx new file mode 100644 index 00000000..ca650e20 --- /dev/null +++ b/web/src/app/dashboard/components/knowledge/CuratedComposer.test.tsx @@ -0,0 +1,358 @@ +import { fireEvent, screen, waitFor } from "@testing-library/react"; +import { HttpResponse, http } from "msw"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { setAuthToken } from "~/lib/auth-token"; +import { server } from "~/test/msw-server"; +import { renderWithQuery } from "~/test/query"; +import { CuratedComposer } from "./CuratedComposer"; + +const API = "http://localhost:8787"; + +const draft = { + key: "curated/otoineppu-tokyo.md", + content: + "---\ntitle: 音威子府TOKYO\ncategory: お店・スポット\n---\n# 音威子府TOKYO\n\n> 注意書き\n\n生成された本文\n", + readUrls: ["https://peraichi.com/landing_pages/view/otoineppu"], + unreadable: [ + { + name: "https://www.instagram.com/usagi/", + reason: "ログインページに転送されました", + }, + ], +}; + +const OVERWRITE_WARNING = "同名のファイルがあります。保存すると上書きされます"; + +const openText = () => { + fireEvent.click(screen.getByRole("tab", { name: "文章から作る" })); + return screen.getByLabelText("文章") as HTMLTextAreaElement; +}; +const openFiles = () => + fireEvent.click(screen.getByRole("tab", { name: "画像・PDF から作る" })); +const urlInput = () => screen.getByRole("textbox", { name: "URL" }); +const addUrl = (url: string) => { + fireEvent.change(urlInput(), { target: { value: url } }); + fireEvent.keyDown(urlInput(), { key: "Enter" }); +}; +const generateButton = () => + screen.getByRole("button", { + name: /下書きを作る|下書きを作り直す|読み取り中/, + }); +const saveButton = () => + screen.getByRole("button", { name: /^(保存|上書きして保存|保存中...)$/ }); +const titleInput = () => screen.getByLabelText("タイトル") as HTMLInputElement; + +const captureRequest = () => { + const captured: { urls: string[]; text: string | null } = { + urls: [], + text: null, + }; + server.use( + http.post(`${API}/admin/knowledge/curated-draft`, async ({ request }) => { + const form = await request.formData(); + captured.urls = form.getAll("urls").map(String); + captured.text = form.get("text") as string | null; + return HttpResponse.json(draft); + }), + ); + return captured; +}; + +const renderComposer = (existingKeys: string[] = []) => + renderWithQuery(); + +const generateFrom = async (text: string) => { + fireEvent.change(openText(), { target: { value: text } }); + fireEvent.click(generateButton()); + await waitFor(() => expect(titleInput().value).toBe("音威子府TOKYO")); +}; + +beforeEach(() => { + localStorage.clear(); + setAuthToken("admin-token"); +}); + +afterEach(() => { + localStorage.clear(); +}); + +describe("CuratedComposer", () => { + it("最初は URL タブが選ばれ、文章と画像の入力はタブを切り替えると出る", () => { + renderComposer(); + + expect(screen.getByRole("tab", { name: "URL から作る" })).toHaveAttribute( + "aria-selected", + "true", + ); + expect(urlInput()).toBeDefined(); + expect(generateButton()).toBeDisabled(); + expect(screen.queryByLabelText("文章")).toBeNull(); + expect(screen.queryByLabelText("タイトル")).toBeNull(); + + openText(); + expect(screen.getByLabelText("文章")).toBeDefined(); + expect( + screen.queryAllByRole("textbox", { name: /^URL \d+$/ }), + ).toHaveLength(0); + + openFiles(); + expect( + screen.getByRole("button", { name: /ドロップ、またはクリックして選ぶ/ }), + ).toBeDefined(); + }); + + it("生成に使うのは表示中のタブの入力だけ", async () => { + const captured = captureRequest(); + renderComposer(); + addUrl("https://peraichi.com/x"); + fireEvent.change(openText(), { target: { value: "メモ" } }); + + fireEvent.click(generateButton()); + await waitFor(() => expect(titleInput().value).toBe("音威子府TOKYO")); + + expect(captured.urls).toEqual([]); + expect(captured.text).toBe("メモ"); + }); + + it("URL は Enter か貼り付けでチップになり、× で外せて、そのまま生成できる", async () => { + const captured = captureRequest(); + renderComposer(); + + addUrl("https://peraichi.com/x"); + fireEvent.paste(urlInput(), { + clipboardData: { getData: () => "https://a.example/ https://b.example/" }, + }); + expect(screen.getByText("peraichi.com/x")).toBeDefined(); + expect(screen.getByText("a.example")).toBeDefined(); + expect(screen.getByText("b.example")).toBeDefined(); + + fireEvent.click(screen.getByRole("button", { name: "b.example を外す" })); + expect(screen.queryByText("b.example")).toBeNull(); + + expect(generateButton()).toBeEnabled(); + fireEvent.click(generateButton()); + await waitFor(() => expect(titleInput().value).toBe("音威子府TOKYO")); + + expect(captured.urls).toEqual([ + "https://peraichi.com/x", + "https://a.example/", + ]); + expect(captured.text).toBeNull(); + }); + + it("URL でない文字列は赤く警告してチップにしない", () => { + renderComposer(); + + addUrl("音威子府"); + + expect( + screen.getByText("http:// か https:// で始まる URL を入力してください"), + ).toBeDefined(); + expect(generateButton()).toBeDisabled(); + }); + + it("入力途中で Enter を押していない URL も生成に使う", async () => { + const captured = captureRequest(); + renderComposer(); + fireEvent.change(urlInput(), { + target: { value: "https://peraichi.com/x" }, + }); + + fireEvent.click(generateButton()); + await waitFor(() => expect(titleInput().value).toBe("音威子府TOKYO")); + + expect(captured.urls).toEqual(["https://peraichi.com/x"]); + }); + + it("文章の中に URL が混ざっていても URL として拾う", async () => { + const captured = captureRequest(); + renderComposer(); + + await generateFrom("音威子府TOKYO の店 https://peraichi.com/x を追加"); + + expect(captured.urls).toEqual(["https://peraichi.com/x"]); + expect(captured.text).toBe( + "音威子府TOKYO の店 https://peraichi.com/x を追加", + ); + }); + + it("生成後はタイトル欄と整形された本文が出て、Markdown 記法や frontmatter は見せない", async () => { + captureRequest(); + renderComposer(); + + await generateFrom("メモ"); + + expect(screen.getByText("生成された本文")).toBeDefined(); + expect(screen.queryByText(/title:/)).toBeNull(); + expect(screen.queryByText(/^# /)).toBeNull(); + expect(screen.getByText("curated/otoineppu-tokyo.md")).toBeDefined(); + expect(generateButton()).toHaveTextContent("下書きを作り直す"); + }); + + it("参照した資料はリンクで開け、読み取れなかった資料は理由付きで出る", async () => { + captureRequest(); + renderComposer(); + + await generateFrom("メモ"); + + const link = screen.getByRole("link", { + name: "peraichi.com/landing_pages/view/otoineppu", + }); + expect(link).toHaveAttribute( + "href", + "https://peraichi.com/landing_pages/view/otoineppu", + ); + expect(link).toHaveAttribute("target", "_blank"); + expect( + screen.getByText( + /instagram\.com\/usagi\/（ログインページに転送されました）/, + ), + ).toBeDefined(); + }); + + it("「編集」で本文を文章として直せ、タイトル変更は frontmatter と見出しの両方に入って保存される", async () => { + let putBody: { content: string } | null = null; + captureRequest(); + server.use( + http.put(`${API}/admin/knowledge/files/*`, async ({ request }) => { + putBody = (await request.json()) as { content: string }; + return HttpResponse.json({ message: "ok", chunks: 3 }); + }), + ); + renderComposer(); + await generateFrom("メモ"); + + fireEvent.change(titleInput(), { + target: { value: "音威子府TOKYO（四谷）" }, + }); + fireEvent.click(screen.getByRole("button", { name: "編集" })); + fireEvent.change(screen.getByLabelText("本文"), { + target: { value: "直した本文\n" }, + }); + fireEvent.click(saveButton()); + + await waitFor(() => + expect(putBody).toEqual({ + content: + "---\ntitle: 音威子府TOKYO（四谷）\ncategory: お店・スポット\n---\n# 音威子府TOKYO（四谷）\n\n直した本文\n", + }), + ); + }); + + it("作り直すと前の下書きに戻せる", async () => { + captureRequest(); + renderComposer(); + await generateFrom("メモ"); + fireEvent.change(titleInput(), { target: { value: "手で直した" } }); + + fireEvent.click(generateButton()); + await waitFor(() => expect(titleInput().value).toBe("音威子府TOKYO")); + + fireEvent.click(screen.getByRole("button", { name: "前の下書きに戻す" })); + expect(titleInput().value).toBe("手で直した"); + }); + + it("422 のときはサーバーの案内文を表示し、下書きは出ない", async () => { + server.use( + http.post(`${API}/admin/knowledge/curated-draft`, () => + HttpResponse.json( + { + error: { + code: 422, + message: "どの資料からも本文を取得できませんでした", + }, + }, + { status: 422 }, + ), + ), + ); + renderComposer(); + addUrl("https://www.instagram.com/usagi/"); + + fireEvent.click(generateButton()); + + expect( + await screen.findByText( + "エラー: どの資料からも本文を取得できませんでした", + ), + ).toBeDefined(); + expect(screen.queryByLabelText("タイトル")).toBeNull(); + }); + + it("画像はファイル選択でもドロップ領域への貼り付けでも追加でき、削除できる", () => { + renderComposer(); + openFiles(); + fireEvent.change(screen.getByLabelText("画像・PDF"), { + target: { files: [new File(["x"], "flyer.png", { type: "image/png" })] }, + }); + fireEvent.paste( + screen.getByRole("button", { name: /ドロップ、またはクリックして選ぶ/ }), + { + clipboardData: { + files: [new File(["y"], "shot.png", { type: "image/png" })], + getData: () => "", + }, + }, + ); + + expect(screen.getByText("flyer.png")).toBeDefined(); + expect(screen.getByText("shot.png")).toBeDefined(); + expect(generateButton()).toBeEnabled(); + + fireEvent.click(screen.getByRole("button", { name: "flyer.png を削除" })); + fireEvent.click(screen.getByRole("button", { name: "shot.png を削除" })); + expect(generateButton()).toBeDisabled(); + }); + + it("保存先は自動で決まり、「変更」で編集でき、既存キーなら上書き警告、/ は不可", async () => { + captureRequest(); + renderComposer(["curated/otoineppu-tokyo.md"]); + await generateFrom("メモ"); + + expect(screen.getByText(OVERWRITE_WARNING)).toBeDefined(); + expect(saveButton()).toHaveTextContent("上書きして保存"); + + fireEvent.click(screen.getByRole("button", { name: "変更" })); + fireEvent.change(screen.getByLabelText("保存先"), { + target: { value: "otoineppu-tokyo-2" }, + }); + expect(screen.queryByText(OVERWRITE_WARNING)).toBeNull(); + expect(saveButton()).toBeEnabled(); + + fireEvent.change(screen.getByLabelText("保存先"), { + target: { value: "a/b" }, + }); + expect(screen.getByText("保存先に / は使えません")).toBeDefined(); + expect(saveButton()).toBeDisabled(); + }); + + it("保存すると入力箱の上に完了メッセージが出て、最初の状態に戻る", async () => { + let putUrl = ""; + captureRequest(); + server.use( + http.put(`${API}/admin/knowledge/files/*`, ({ request }) => { + putUrl = request.url; + return HttpResponse.json({ message: "ok", chunks: 3 }); + }), + ); + renderComposer(); + await generateFrom("メモ"); + + fireEvent.click(saveButton()); + + expect( + await screen.findByText( + "curated/otoineppu-tokyo.md を保存しました（3 チャンクを同期）", + ), + ).toBeDefined(); + expect(decodeURIComponent(putUrl)).toContain( + "/files/curated/otoineppu-tokyo.md", + ); + expect(screen.queryByLabelText("タイトル")).toBeNull(); + expect(screen.getByRole("tab", { name: "URL から作る" })).toHaveAttribute( + "aria-selected", + "true", + ); + expect((urlInput() as HTMLInputElement).value).toBe(""); + }); +}); diff --git a/web/src/app/dashboard/components/knowledge/CuratedComposer.tsx b/web/src/app/dashboard/components/knowledge/CuratedComposer.tsx new file mode 100644 index 00000000..68825e73 --- /dev/null +++ b/web/src/app/dashboard/components/knowledge/CuratedComposer.tsx @@ -0,0 +1,573 @@ +import { + ArrowUturnLeftIcon, + CheckCircleIcon, + DocumentIcon, + LinkIcon, + PencilSquareIcon, + PhotoIcon, + SparklesIcon, + XMarkIcon, +} from "@heroicons/react/24/outline"; +import { CURATED_DRAFT_LIMITS } from "@nepp-chan/shared/constants/knowledge"; +import { Button } from "@nepp-chan/shared/ui/Button"; +import { Spinner } from "@nepp-chan/shared/ui/Loading"; +import { + type ChangeEvent, + type ClipboardEvent, + type DragEvent, + type KeyboardEvent, + useEffect, + useRef, + useState, +} from "react"; +import { + useDraftCurated, + useSaveFile, +} from "~/app/dashboard/hooks/useKnowledge"; +import { MarkdownText } from "~/components/chat/MarkdownText"; +import { ErrorBanner, formatError } from "~/components/ui/ErrorBanner"; +import type { CuratedDraft } from "~/types"; +import { + addUrls, + CURATED_PREFIX, + DRAFT_FILE_ACCEPT, + type DraftParts, + extractUrls, + hasDraftInput, + hostLabel, + INPUT_CLASS, + isValidSlug, + joinDraft, + keyFromSlug, + type SourceKind, + slugFromKey, + splitDraft, + toDraftRequest, +} from "./helpers"; + +type Props = { + existingKeys: string[]; +}; + +const SOURCE_TABS = [ + { kind: "url", label: "URL から作る", icon: LinkIcon }, + { kind: "text", label: "文章から作る", icon: PencilSquareIcon }, + { kind: "files", label: "画像・PDF から作る", icon: PhotoIcon }, +] as const; + +const isImage = (file: File) => file.type.startsWith("image/"); + +const FileThumb = ({ + file, + onRemove, +}: { + file: File; + onRemove: () => void; +}) => { + const [url, setUrl] = useState(null); + + useEffect(() => { + if (!isImage(file) || typeof URL.createObjectURL !== "function") return; + const objectUrl = URL.createObjectURL(file); + setUrl(objectUrl); + return () => URL.revokeObjectURL(objectUrl); + }, [file]); + + return ( +
  • +
    + {url ? ( + {file.name} + ) : ( +
    + + {file.name} + + +
    • + ); +}; + +export const CuratedComposer = ({ existingKeys }: Props) => { + const [urls, setUrls] = useState([]); + const [urlInput, setUrlInput] = useState(""); + const [urlError, setUrlError] = useState(false); + const [text, setText] = useState(""); + const [files, setFiles] = useState([]); + const [dragging, setDragging] = useState(false); + const [kind, setKind] = useState("url"); + const [slug, setSlug] = useState(""); + const [editingSlug, setEditingSlug] = useState(false); + const [draft, setDraft] = useState(null); + const [previousDraft, setPreviousDraft] = useState(null); + const [editingBody, setEditingBody] = useState(false); + const [lastResult, setLastResult] = useState(null); + const [savedMessage, setSavedMessage] = useState(null); + const fileInputRef = useRef(null); + const draftMutation = useDraftCurated(); + const saveMutation = useSaveFile(); + + const fields = { + kind, + urls: [...urls, ...extractUrls(urlInput)], + text, + files, + }; + const busy = draftMutation.isPending || saveMutation.isPending; + const canGenerate = hasDraftInput(fields) && !busy; + const key = keyFromSlug(slug); + const exists = slug.trim().length > 0 && existingKeys.includes(key); + const canSave = + draft !== null && + isValidSlug(slug) && + draft.title.trim().length > 0 && + draft.body.trim().length > 0 && + !busy; + + const commitUrl = (input: string) => { + if (!input.trim()) return; + const result = addUrls(urls, input, CURATED_DRAFT_LIMITS.urls); + setUrls(result.urls); + setUrlInput(result.accepted ? "" : input); + setUrlError(!result.accepted); + }; + const onUrlKeyDown = (e: KeyboardEvent) => { + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); + commitUrl(urlInput); + } else if (e.key === "Backspace" && !urlInput && urls.length > 0) { + setUrls((prev) => prev.slice(0, -1)); + } + }; + const onUrlPaste = (e: ClipboardEvent) => { + const pasted = e.clipboardData.getData("text"); + if (!extractUrls(pasted).length) return; + e.preventDefault(); + commitUrl(`${urlInput} ${pasted}`); + }; + const removeUrl = (url: string) => + setUrls((prev) => prev.filter((u) => u !== url)); + + const addFiles = (picked: FileList | File[] | null) => { + if (!picked) return; + setFiles((prev) => + [...prev, ...Array.from(picked)].slice(0, CURATED_DRAFT_LIMITS.files), + ); + }; + const onFileChange = (e: ChangeEvent) => { + addFiles(e.target.files); + e.target.value = ""; + }; + const onPaste = (e: ClipboardEvent) => { + const pasted = Array.from(e.clipboardData?.files ?? []); + if (pasted.length === 0) return; + e.preventDefault(); + addFiles(pasted); + }; + const onDrop = (e: DragEvent) => { + e.preventDefault(); + setDragging(false); + addFiles(e.dataTransfer.files); + }; + const removeFile = (index: number) => + setFiles((prev) => prev.filter((_, i) => i !== index)); + + const generate = () => { + setSavedMessage(null); + draftMutation.mutate(toDraftRequest(fields), { + onSuccess: (result) => { + setPreviousDraft(draft); + setDraft(splitDraft(result.content)); + setEditingBody(false); + setLastResult(result); + if (!slug.trim()) setSlug(slugFromKey(result.key)); + }, + }); + }; + + const undo = () => { + setDraft(previousDraft); + setPreviousDraft(null); + }; + + const save = () => { + if (!draft) return; + saveMutation.mutate( + { key, content: joinDraft(draft) }, + { + onSuccess: (result) => { + setSavedMessage( + `${key} を保存しました（${result.chunks} チャンクを同期）`, + ); + setUrls([]); + setUrlInput(""); + setUrlError(false); + setText(""); + setFiles([]); + setKind("url"); + setSlug(""); + setEditingSlug(false); + setDraft(null); + setPreviousDraft(null); + setEditingBody(false); + setLastResult(null); + }, + }, + ); + }; + + return ( +
    + {savedMessage && ( + + + )} + +
    +
    + {SOURCE_TABS.map((tab) => ( + + ))} +
    + + {kind === "url" && ( +
    +

    + ページを読み取って、その内容から下書きを作ります +

    +
    + {urls.map((url) => ( + + + ))} + {urls.length < CURATED_DRAFT_LIMITS.urls && ( + { + setUrlInput(e.target.value); + setUrlError(false); + }} + onKeyDown={onUrlKeyDown} + onPaste={onUrlPaste} + onBlur={() => commitUrl(urlInput)} + disabled={busy} + placeholder={ + urls.length === 0 + ? "URL を貼り付け（複数可）" + : "続けて貼り付け" + } + className="flex-1 min-w-[200px] py-1 text-sm bg-transparent border-0 focus:outline-none disabled:text-stone-400" + /> + )} +
    + {urlError && ( +

    + http:// か https:// で始まる URL を入力してください +

    + )} +
    + )} + + {kind === "text" && ( +
    +

    + 知っている内容や SNS + の投稿文を、そのまま貼り付けてください。文中の URL も読み取ります +

    +