kayac · owk-owk130 · Sep 3, 2026 · Sep 3, 2026 · Sep 3, 2026 · Sep 3, 2026
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -46,6 +46,10 @@ knowledge/ 配下 329 ファイルを clean アップロードした場合の正
 |------|:---------:|:---------:|------|
 | 2026-03-09 | 329 | 2,891 | clean 後の正確な値 |
 
+### 管理画面から追加した curated ナレッジ
+
+ダッシュボードの「ナレッジを追加」は、URL・文章・画像/PDF から LLM が下書きを作り、確認後に R2 の `curated/<slug>.md` として保存する（frontmatter は `source_type: curated` / `source_authority: 2` / `verified_at` / `url`）。git 管理外なので `knowledge:upload` では投入されず、`--clean` 後は Vectorize 側だけ消える。clean したら `POST /admin/knowledge/sync` で R2 全体を再同期する。
+
 ## コーディング規約
 
 ### コメント

diff --git a/server/src/db/schema.ts b/server/src/db/schema.ts
@@ -199,7 +199,7 @@ export const llmUsage = sqliteTable("llm_usage", {
   cachedInputTokens: integer("cached_input_tokens").notNull().default(0),
   totalTokens: integer("total_tokens").notNull().default(0),
   platform: text("platform"), // "web" | "line" | "lp" | "widget" | "voice" | null（バッチ系）
-  source: text("source").notNull(), // "chat" | "subagent" | "intent-classify" | "persona-extract" | "weekly-report" | "image-convert"
+  source: text("source").notNull(),
   agent: text("agent"), // 呼び出し元エージェント名（"nepp-chan" "knowledge" 等）。列追加前の行は null
   turnIndex: integer("turn_index"), // スレッド内の何往復目か（1 始まり）。列追加前の行は null
   durationMs: integer("duration_ms"), // 呼び出し 1 回の所要時間

diff --git a/server/src/lib/date.ts b/server/src/lib/date.ts
@@ -1,7 +1,7 @@
 const HOUR_MS = 60 * 60 * 1000;
 export const DAY_MS = 24 * HOUR_MS;
 export const WEEK_MS = 7 * DAY_MS;
-const JST_OFFSET_MS = 9 * HOUR_MS;
+export const JST_OFFSET_MS = 9 * HOUR_MS;
 
 /** JST の YYYY-MM-DD ラベルに変換する */
 export const jstDateLabel = (date: Date) =>

diff --git a/server/src/lib/html-to-text.test.ts b/server/src/lib/html-to-text.test.ts
@@ -0,0 +1,132 @@
+import { describe, expect, it } from "vitest";
+import {
+  decodeEntities,
+  decodeHtml,
+  detectCharset,
+  extractPageText,
+  htmlToText,
+} from "./html-to-text";
+
+describe("decodeEntities", () => {
+  it("名前付き・10 進・16 進のエンティティを文字に戻す", () => {
+    expect(
+      decodeEntities("a &amp; b &lt;c&gt; &#12354; &#x3044; &mdash;"),
+    ).toBe("a & b <c> あ い —");
+  });
+
+  it("未知のエンティティはそのまま残す", () => {
+    expect(decodeEntities("&unknownthing;")).toBe("&unknownthing;");
+  });
+});
+
+describe("detectCharset", () => {
+  it("Content-Type ヘッダの charset を優先する", () => {
+    expect(
+      detectCharset('<meta charset="utf-8">', "text/html; charset=Shift_JIS"),
+    ).toBe("Shift_JIS");
+  });
+
+  it("ヘッダに無ければ meta charset を見る", () => {
+    expect(detectCharset('<meta charset="euc-jp">', "text/html")).toBe(
+      "euc-jp",
+    );
+  });
+
+  it("http-equiv の content 内 charset も拾う", () => {
+    expect(
+      detectCharset(
+        '<meta http-equiv="Content-Type" content="text/html; charset=Shift_JIS">',
+        null,
+      ),
+    ).toBe("Shift_JIS");
+  });
+
+  it("どこにも無ければ utf-8", () => {
+    expect(detectCharset("<html>", null)).toBe("utf-8");
+  });
+});
+
+describe("decodeHtml", () => {
+  it("Shift_JIS のバイト列を meta charset に従ってデコードする", () => {
+    const sjisBytes = new Uint8Array([
+      ...new TextEncoder().encode('<meta charset="shift_jis"><p>'),
+      0x89,
+      0xb9,
+      0x88,
+      0xd0,
+      0x8e,
+      0x71,
+      0x95,
+      0x7b,
+      ...new TextEncoder().encode("</p>"),
+    ]);
+
+    const html = decodeHtml(sjisBytes.buffer, "text/html");
+
+    expect(html).toContain("音威子府");
+  });
+
+  it("不正な charset 名は utf-8 にフォールバックする", () => {
+    const bytes = new TextEncoder().encode(
+      '<meta charset="not-a-charset"><p>本文</p>',
+    );
+
+    expect(decodeHtml(bytes.buffer as ArrayBuffer, null)).toContain("本文");
+  });
+});
+
+describe("htmlToText", () => {
+  it("script / style / noscript / svg を本文から除く", () => {
+    const html =
+      "<script>var x = 1;</script><style>p{}</style><noscript>no js</noscript><svg><text>icon</text></svg><p>残す</p>";
+
+    expect(htmlToText(html)).toBe("残す");
+  });
+
+  it("ブロック要素の終わりを改行に、空白を 1 つに畳む", () => {
+    const html =
+      "<h1>見出し</h1><p>段落   一</p><ul><li>項目1</li><li>項目2</li></ul><div>末尾<br>次行</div>";
+
+    expect(htmlToText(html)).toBe("見出し\n段落 一\n項目1\n項目2\n末尾\n次行");
+  });
+
+  it("HTML コメントを除き、3 つ以上の連続改行を 2 つにする", () => {
+    const html = "<p>a</p><!-- c --><p></p><p></p><p></p><p>b</p>";
+
+    expect(htmlToText(html)).toBe("a\n\nb");
+  });
+});
+
+describe("extractPageText", () => {
+  it("og:title / og:description を優先して拾い、head は本文に含めない", () => {
+    const html = `<html><head><title>タイトルタグ</title>
+      <meta property="og:title" content="OG タイトル">
+      <meta content="OG 説明 &amp; 補足" property="og:description">
+      <meta name="description" content="通常の説明">
+      </head><body><p>本文</p></body></html>`;
+
+    const page = extractPageText(html);
+
+    expect(page.title).toBe("OG タイトル");
+    expect(page.description).toBe("OG 説明 & 補足");
+    expect(page.text).toBe("本文");
+  });
+
+  it("og が無ければ title タグと meta description を使う", () => {
+    const html =
+      '<head><title>店のページ</title><meta name="description" content="説明文"></head><body>x</body>';
+
+    const page = extractPageText(html);
+
+    expect(page.title).toBe("店のページ");
+    expect(page.description).toBe("説明文");
+  });
+
+  it("title も description も無ければ undefined", () => {
+    const page = extractPageText("<body><p>本文だけ</p></body>");
+
+    expect(page.title).toBeUndefined();
+    expect(page.description).toBeUndefined();
+    expect(page.text).toBe("本文だけ");
+  });
+});
diff --git a/server/src/lib/html-to-text.ts b/server/src/lib/html-to-text.ts
@@ -0,0 +1,88 @@
+const CHARSET_SNIFF_BYTES = 4096;
+
+const NAMED_ENTITIES: Record<string, string> = {
+  amp: "&",
+  lt: "<",
+  gt: ">",
+  quot: '"',
+  apos: "'",
+  nbsp: " ",
+  mdash: "—",
+  ndash: "–",
+  hellip: "…",
+  copy: "©",
+};
+
+export const decodeEntities = (text: string) =>
+  text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
+    if (entity.startsWith("#x") || entity.startsWith("#X")) {
+      return String.fromCodePoint(Number.parseInt(entity.slice(2), 16));
+    }
+    if (entity.startsWith("#")) {
+      return String.fromCodePoint(Number.parseInt(entity.slice(1), 10));
+    }
+    return NAMED_ENTITIES[entity.toLowerCase()] ?? match;
+  });
+
+const charsetFrom = (text: string | null) =>
+  text?.match(/charset=["']?([\w-]+)/i)?.[1];
+
+export const detectCharset = (headHtml: string, contentType: string | null) =>
+  charsetFrom(contentType) ?? charsetFrom(headHtml) ?? "utf-8";
+
+export const decodeHtml = (buffer: ArrayBuffer, contentType: string | null) => {
+  const head = new TextDecoder("latin1").decode(
+    buffer.slice(0, CHARSET_SNIFF_BYTES),
+  );
+  const charset = detectCharset(head, contentType);
+  try {
+    return new TextDecoder(charset).decode(buffer);
+  } catch {
+    return new TextDecoder().decode(buffer);
+  }
+};
+
+const BLOCK_END_TAGS =
+  /<\/(?:p|div|li|h[1-6]|tr|section|article|header|footer|blockquote|dd|dt|pre|table)>|<br\s*\/?>|<hr\s*\/?>/gi;
+
+export const htmlToText = (html: string) => {
+  const withBreaks = html
+    .replace(/<!--[\s\S]*?-->/g, "")
+    .replace(
+      /<(script|style|noscript|svg|template|iframe)\b[\s\S]*?<\/\1>/gi,
+      "",
+    )
+    .replace(BLOCK_END_TAGS, "\n")
+    .replace(/<[^>]+>/g, " ");
+
+  return decodeEntities(withBreaks)
+    .split("\n")
+    .map((line) => line.replace(/[ \t]+/g, " ").trim())
+    .join("\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+};
+
+const metaContent = (html: string, key: string) => {
+  const attr = `(?:property|name)=["']${key}["']`;
+  const before = html.match(
+    new RegExp(`<meta[^>]*${attr}[^>]*content=["']([^"']*)["']`, "i"),
+  );
+  const after = html.match(
+    new RegExp(`<meta[^>]*content=["']([^"']*)["'][^>]*${attr}`, "i"),
+  );
+  const raw = before?.[1] ?? after?.[1];
+  return raw ? decodeEntities(raw).trim() || undefined : undefined;
+};
+
+export const extractPageText = (html: string) => {
+  const titleTag = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1];
+  const title =
+    metaContent(html, "og:title") ??
+    (titleTag ? decodeEntities(titleTag).trim() || undefined : undefined);
+  const description =
+    metaContent(html, "og:description") ?? metaContent(html, "description");
+
+  const body = html.replace(/<head\b[\s\S]*?<\/head>/i, "");
+  return { title, description, text: htmlToText(body) };
+};
diff --git a/server/src/lib/image-converter.ts b/server/src/lib/image-converter.ts
@@ -1,19 +1,12 @@
 import { Buffer } from "node:buffer";
+import { CONVERTIBLE_MIME_TYPES } from "@nepp-chan/shared/constants/knowledge";
 import { OPENAI_LITE } from "~/lib/llm-models";
 import { converterAgent } from "~/mastra/agents/converter-agent";
 import { recordLlmUsage } from "~/services/analytics/llm-usage";
 
-const SUPPORTED_MIME_TYPES = [
-  "image/png",
-  "image/jpeg",
-  "image/webp",
-  "image/gif",
-  "application/pdf",
-] as const;
-
 export const isSupportedMimeType = (mimeType: string) =>
-  SUPPORTED_MIME_TYPES.includes(
-    mimeType as (typeof SUPPORTED_MIME_TYPES)[number],
+  CONVERTIBLE_MIME_TYPES.includes(
+    mimeType as (typeof CONVERTIBLE_MIME_TYPES)[number],
   );
 
 export const convertToMarkdown = async (
@@ -23,7 +16,7 @@ export const convertToMarkdown = async (
 ) => {
   if (!isSupportedMimeType(mimeType)) {
     throw new Error(
-      `Unsupported mime type: ${mimeType}. Supported types: ${SUPPORTED_MIME_TYPES.join(", ")}`,
+      `Unsupported mime type: ${mimeType}. Supported types: ${CONVERTIBLE_MIME_TYPES.join(", ")}`,
     );
   }
 

diff --git a/server/src/lib/openapi-errors.ts b/server/src/lib/openapi-errors.ts
@@ -17,6 +17,7 @@ const descriptions = {
   403: "権限エラー",
   404: "リソースが見つかりません",
   409: "リソースが競合しています",
+  422: "処理できない入力です",
   500: "サーバーエラー",
 } as const;
 

diff --git a/server/src/lib/x-post.test.ts b/server/src/lib/x-post.test.ts
@@ -0,0 +1,65 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { fetchXPostText, isXPostUrl } from "./x-post";
+
+describe("isXPostUrl", () => {
+  it.each([
+    "https://x.com/jack/status/20",
+    "https://twitter.com/jack/status/20?s=20",
+    "https://mobile.twitter.com/jack/status/20",
+  ])("%s は個別投稿", (url) => {
+    expect(isXPostUrl(url)).toBe(true);
+  });
+
+  it.each([
+    "https://x.com/jack",
+    "https://x.com/i/lists/1",
+    "https://example.com/jack/status/20",
+    "not a url",
+  ])("%s は個別投稿ではない", (url) => {
+    expect(isXPostUrl(url)).toBe(false);
+  });
+});
+
+describe("fetchXPostText", () => {
+  const fetchSpy = vi.spyOn(globalThis, "fetch");
+
+  beforeEach(() => {
+    fetchSpy.mockReset();
+  });
+
+  afterEach(() => {
+    fetchSpy.mockReset();
+  });
+
+  it("oEmbed の html を本文テキストにして author_name と返す", async () => {
+    fetchSpy.mockResolvedValue(
+      new Response(
+        JSON.stringify({
+          author_name: "jack",
+          html: '<blockquote class="twitter-tweet"><p lang="en">just setting up my twttr</p>&mdash; jack (@jack) <a href="https://x.com/jack/status/20">2006年3月21日</a></blockquote>\n\n',
+        }),
+        { status: 200 },
+      ),
+    );
+
+    const post = await fetchXPostText("https://x.com/jack/status/20");
+
+    expect(post.authorName).toBe("jack");
+    expect(post.text).toBe(
+      "just setting up my twttr\n— jack (@jack) 2006年3月21日",
+    );
+    const calledUrl = fetchSpy.mock.calls[0]?.[0] as string;
+    expect(calledUrl).toContain("publish.x.com/oembed");
+    expect(calledUrl).toContain(
+      encodeURIComponent("https://x.com/jack/status/20"),
+    );
+  });
+
+  it("非 2xx は HTTP ステータス付きで throw する", async () => {
+    fetchSpy.mockResolvedValue(new Response("", { status: 404 }));
+
+    await expect(fetchXPostText("https://x.com/jack/status/1")).rejects.toThrow(
+      "HTTP 404",
+    );
+  });
+});
diff --git a/server/src/lib/x-post.ts b/server/src/lib/x-post.ts
@@ -0,0 +1,38 @@
+import { htmlToText } from "~/lib/html-to-text";
+
+const X_HOSTS = new Set([
+  "x.com",
+  "www.x.com",
+  "twitter.com",
+  "www.twitter.com",
+  "mobile.twitter.com",
+]);
+
+export const isXPostUrl = (url: string) => {
+  try {
+    const parsed = new URL(url);
+    return (
+      X_HOSTS.has(parsed.hostname) &&
+      /^\/[^/]+\/status\/\d+/.test(parsed.pathname)
+    );
+  } catch {
+    return false;
+  }
+};
+
+type OEmbedResponse = {
+  html: string;
+  author_name: string;
+};
+
+export const fetchXPostText = async (url: string, timeoutMs = 15_000) => {
+  const endpoint = `https://publish.x.com/oembed?url=${encodeURIComponent(url)}&omit_script=true&lang=ja`;
+  const response = await fetch(endpoint, {
+    signal: AbortSignal.timeout(timeoutMs),
+  });
+  if (!response.ok) {
+    throw new Error(`投稿を取得できませんでした（HTTP ${response.status}）`);
+  }
+  const data = (await response.json()) as OEmbedResponse;
+  return { text: htmlToText(data.html), authorName: data.author_name };
+};