feat(server): 情報源の承認レイヤーを追加する

server/src/__tests__/helpers/test-db.ts

@@ -160,6 +160,24 @@ export const createTestDb = async () => {
       created_at TEXT NOT NULL
     );
 
+    CREATE TABLE IF NOT EXISTS knowledge_sources (
+      source_path TEXT PRIMARY KEY,
+      canonical_url TEXT,
+      source_type TEXT,
+      source_authority INTEGER,
+      source_hash TEXT,
+      r2_etag TEXT,
+      chunk_count INTEGER NOT NULL DEFAULT 0,
+      approval_status TEXT NOT NULL DEFAULT 'pending',
+      approved_by TEXT,
+      approved_at TEXT,
+      disabled_at TEXT,
+      verified_at TEXT,
+      indexed_at TEXT,
+      created_at TEXT NOT NULL,
+      updated_at TEXT
+    );
+
     CREATE TABLE IF NOT EXISTS retrieval_runs (
       id TEXT PRIMARY KEY,
       answer_run_id TEXT,

server/src/db/migrations/0029_knowledge_sources.sql

@@ -0,0 +1,20 @@
+CREATE TABLE knowledge_sources (
+  source_path TEXT PRIMARY KEY,
+  canonical_url TEXT,
+  source_type TEXT,
+  source_authority INTEGER,
+  source_hash TEXT,
+  r2_etag TEXT,
+  chunk_count INTEGER NOT NULL DEFAULT 0,
+  approval_status TEXT NOT NULL DEFAULT 'pending',
+  approved_by TEXT,
+  approved_at TEXT,
+  disabled_at TEXT,
+  verified_at TEXT,
+  indexed_at TEXT,
+  created_at TEXT NOT NULL,
+  updated_at TEXT
+);
+
+CREATE INDEX idx_knowledge_sources_status ON knowledge_sources(approval_status);
+CREATE INDEX idx_knowledge_sources_canonical_url ON knowledge_sources(canonical_url);

server/src/db/schema.ts

@@ -214,6 +214,27 @@ export const llmUsage = sqliteTable("llm_usage", {
 export type LlmUsage = typeof llmUsage.$inferSelect;
 export type NewLlmUsage = typeof llmUsage.$inferInsert;
 
+export const knowledgeSources = sqliteTable("knowledge_sources", {
+  sourcePath: text("source_path").primaryKey(),
+  canonicalUrl: text("canonical_url"),
+  sourceType: text("source_type"),
+  sourceAuthority: integer("source_authority"),
+  sourceHash: text("source_hash"),
+  r2Etag: text("r2_etag"),
+  chunkCount: integer("chunk_count").notNull().default(0),
+  approvalStatus: text("approval_status").notNull().default("pending"), // "pending" | "approved" | "rejected" | "disabled"
+  approvedBy: text("approved_by"),
+  approvedAt: text("approved_at"),
+  disabledAt: text("disabled_at"),
+  verifiedAt: text("verified_at"),
+  indexedAt: text("indexed_at"),
+  createdAt: text("created_at").notNull(),
+  updatedAt: text("updated_at"),
+});
+
+export type KnowledgeSource = typeof knowledgeSources.$inferSelect;
+export type NewKnowledgeSource = typeof knowledgeSources.$inferInsert;
+
 export const retrievalRuns = sqliteTable("retrieval_runs", {
   id: text("id").primaryKey(),
   answerRunId: text("answer_run_id"),

server/src/handlers/r2-event-handler.test.ts

@@ -1,12 +1,12 @@
 import { beforeEach, describe, expect, it, vi } from "vitest";
 
-vi.mock("~/services/knowledge/embedding", () => ({
-  deleteKnowledgeBySource: vi.fn(),
-  processKnowledgeFile: vi.fn(),
+vi.mock("~/services/knowledge/indexing", () => ({
+  indexKnowledgeSource: vi.fn(),
+  removeKnowledgeSource: vi.fn(),
 }));
 
-const { deleteKnowledgeBySource, processKnowledgeFile } = await import(
-  "~/services/knowledge/embedding"
+const { indexKnowledgeSource, removeKnowledgeSource } = await import(
+  "~/services/knowledge/indexing"
 );
 const { handleR2Event } = await import("./r2-event-handler");
 
@@ -15,6 +15,7 @@ const r2Bucket = {
 };
 
 const env = {
+  DB: {} as D1Database,
   KNOWLEDGE_BUCKET: r2Bucket,
   VECTORIZE: {} as VectorizeIndex,
   GOOGLE_GENERATIVE_AI_API_KEY: "key",
@@ -56,8 +57,12 @@ describe("handleR2Event", () => {
   beforeEach(() => {
     vi.clearAllMocks();
     r2Bucket.get.mockResolvedValue({ text: vi.fn().mockResolvedValue("md") });
-    vi.mocked(deleteKnowledgeBySource).mockResolvedValue({ deleted: 3 });
-    vi.mocked(processKnowledgeFile).mockResolvedValue({ chunks: 5 });
+    vi.mocked(indexKnowledgeSource).mockResolvedValue({
+      indexed: true,
+      status: "approved",
+      chunks: 5,
+    });
+    vi.mocked(removeKnowledgeSource).mockResolvedValue({ deleted: 3 });
   });
 
   it(".md 以外は ack して何もしない", async () => {
@@ -66,34 +71,56 @@ describe("handleR2Event", () => {
     await handleR2Event(buildBatch([m]), env);
 
     expect(m.ack).toHaveBeenCalled();
-    expect(processKnowledgeFile).not.toHaveBeenCalled();
+    expect(indexKnowledgeSource).not.toHaveBeenCalled();
   });
 
   it.each(["PutObject", "CompleteMultipartUpload", "CopyObject"] as const)(
-    "%s は delete + processKnowledgeFile を順に呼ぶ",
+    "%s は indexKnowledgeSource に eTag 付きで委譲する",
     async (action) => {
       const m = buildMessage(action, "doc.md");
 
       await handleR2Event(buildBatch([m]), env);
 
-      expect(deleteKnowledgeBySource).toHaveBeenCalledWith(
-        env.VECTORIZE,
+      expect(indexKnowledgeSource).toHaveBeenCalledWith(
         "doc.md",
+        "md",
+        {
+          d1: env.DB,
+          vectorize: env.VECTORIZE,
+          apiKey: "key",
+        },
+        { r2Etag: "etag", skipUnchanged: true },
       );
-      expect(processKnowledgeFile).toHaveBeenCalled();
       expect(m.ack).toHaveBeenCalled();
     },
   );
 
+  it("未承認で index が skip されても ack する", async () => {
+    vi.mocked(indexKnowledgeSource).mockResolvedValue({
+      indexed: false,
+      status: "pending",
+      chunks: 0,
+    });
+    const m = buildMessage("PutObject", "doc.md");
+
+    await handleR2Event(buildBatch([m]), env);
+
+    expect(m.ack).toHaveBeenCalled();
+    expect(m.retry).not.toHaveBeenCalled();
+  });
+
   it.each(["DeleteObject", "LifecycleDeletion"] as const)(
-    "%s は deleteKnowledgeBySource のみ",
+    "%s は removeKnowledgeSource のみ",
     async (action) => {
       const m = buildMessage(action, "doc.md");
 
       await handleR2Event(buildBatch([m]), env);
 
-      expect(deleteKnowledgeBySource).toHaveBeenCalled();
-      expect(processKnowledgeFile).not.toHaveBeenCalled();
+      expect(removeKnowledgeSource).toHaveBeenCalledWith("doc.md", {
+        d1: env.DB,
+        vectorize: env.VECTORIZE,
+      });
+      expect(indexKnowledgeSource).not.toHaveBeenCalled();
       expect(m.ack).toHaveBeenCalled();
     },
   );
@@ -107,10 +134,12 @@ describe("handleR2Event", () => {
     expect(m.retry).toHaveBeenCalled();
   });
 
-  it("processKnowledgeFile がエラーを返したら retry", async () => {
-    vi.mocked(processKnowledgeFile).mockResolvedValue({
-      error: "embed failed",
+  it("index がエラーを返したら retry", async () => {
+    vi.mocked(indexKnowledgeSource).mockResolvedValue({
+      indexed: true,
+      status: "approved",
       chunks: 0,
+      error: "embed failed",
     });
     const m = buildMessage("PutObject", "doc.md");
 
@@ -120,7 +149,7 @@ describe("handleR2Event", () => {
   });
 
   it("例外が起きたら retry", async () => {
-    vi.mocked(processKnowledgeFile).mockRejectedValue(new Error("boom"));
+    vi.mocked(indexKnowledgeSource).mockRejectedValue(new Error("boom"));
     const m = buildMessage("PutObject", "doc.md");
 
     await handleR2Event(buildBatch([m]), env);

server/src/handlers/r2-event-handler.ts

@@ -1,9 +1,9 @@
 import * as Sentry from "@sentry/cloudflare";
 import { logger } from "~/lib/logger";
 import {
-  deleteKnowledgeBySource,
-  processKnowledgeFile,
-} from "~/services/knowledge/embedding";
+  indexKnowledgeSource,
+  removeKnowledgeSource,
+} from "~/services/knowledge/indexing";
 
 type R2EventType =
   | "PutObject"
@@ -30,25 +30,36 @@ const isMarkdownFile = (key: string) => key.endsWith(".md");
 
 const handleObjectCreate = async (
   key: string,
+  eTag: string,
   env: CloudflareBindings,
-): Promise<{ success: boolean; chunks?: number; error?: string }> => {
+): Promise<{
+  success: boolean;
+  chunks?: number;
+  skipped?: string;
+  error?: string;
+}> => {
   const file = await env.KNOWLEDGE_BUCKET.get(key);
   if (!file) {
     return { success: false, error: `File not found: ${key}` };
   }
 
   const content = await file.text();
 
-  // 既存データを削除してから再登録
-  await deleteKnowledgeBySource(env.VECTORIZE, key);
-
-  const result = await processKnowledgeFile(
+  const result = await indexKnowledgeSource(
     key,
     content,
-    env.VECTORIZE,
-    env.GOOGLE_GENERATIVE_AI_API_KEY,
+    {
+      d1: env.DB,
+      vectorize: env.VECTORIZE,
+      apiKey: env.GOOGLE_GENERATIVE_AI_API_KEY,
+    },
+    { r2Etag: eTag, skipUnchanged: true },
   );
 
+  if (!result.indexed) {
+    return { success: true, skipped: result.status };
+  }
+
   if (result.error) {
     return { success: false, error: result.error };
   }
@@ -61,7 +72,10 @@ const handleObjectDelete = async (
   env: CloudflareBindings,
 ): Promise<{ success: boolean; deleted?: number; error?: string }> => {
   try {
-    const result = await deleteKnowledgeBySource(env.VECTORIZE, key);
+    const result = await removeKnowledgeSource(key, {
+      d1: env.DB,
+      vectorize: env.VECTORIZE,
+    });
     return { success: true, deleted: result.deleted };
   } catch (error) {
     return {
@@ -94,8 +108,10 @@ export const handleR2Event = async (
         case "PutObject":
         case "CompleteMultipartUpload":
         case "CopyObject": {
-          const result = await handleObjectCreate(key, env);
-          if (result.success) {
+          const result = await handleObjectCreate(key, object.eTag, env);
+          if (result.success && result.skipped) {
+            logger.info(`Skipped ${key}: approval status is ${result.skipped}`);
+          } else if (result.success) {
             logger.info(`Synced ${key}: ${result.chunks} chunks`);
           } else {
             logger.error(`Failed to sync ${key}`, result.error);

server/src/lib/crypto.test.ts

@@ -7,6 +7,7 @@ import {
   generateToken,
   hmacSha1Base64,
   hmacSha256,
+  sha256Hex,
 } from "./crypto";
 
 describe("hmacSha256", () => {
@@ -109,3 +110,16 @@ describe("generateToken", () => {
     expect(tokens.size).toBe(20);
   });
 });
+
+describe("sha256Hex", () => {
+  it("既知の入力に対して SHA-256 の hex を返す", async () => {
+    expect(await sha256Hex("abc")).toBe(
+      "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+    );
+  });
+
+  it("同じ入力は同じハッシュ、異なる入力は異なるハッシュになる", async () => {
+    expect(await sha256Hex("foo")).toBe(await sha256Hex("foo"));
+    expect(await sha256Hex("foo")).not.toBe(await sha256Hex("bar"));
+  });
+});

server/src/lib/crypto.ts

@@ -61,6 +61,16 @@ export const hmacSha1Base64 = async (value: string, secret: string) => {
   return btoa(String.fromCharCode(...new Uint8Array(signature)));
 };
 
+export const sha256Hex = async (value: string) => {
+  const digest = await crypto.subtle.digest(
+    "SHA-256",
+    new TextEncoder().encode(value),
+  );
+  return Array.from(new Uint8Array(digest), (byte) =>
+    byte.toString(16).padStart(2, "0"),
+  ).join("");
+};
+
 export const generateId = () => {
   const array = new Uint8Array(16);
   crypto.getRandomValues(array);