diff --git a/src/bookmarks-db.ts b/src/bookmarks-db.ts index c78061cb..9647fc1a 100644 --- a/src/bookmarks-db.ts +++ b/src/bookmarks-db.ts @@ -182,8 +182,13 @@ function buildBookmarkWhereClause(filters: BookmarkTimelineFilters): { const params: Array = []; if (filters.query) { - conditions.push(`b.rowid IN (SELECT rowid FROM bookmarks_fts WHERE bookmarks_fts MATCH ?)`); - params.push(filters.query); + if (isCjkQuery(filters.query)) { + conditions.push(cjkLikeCondition('b', filters.query)); + params.push(...cjkLikeParams(filters.query)); + } else { + conditions.push(`b.rowid IN (SELECT rowid FROM bookmarks_fts WHERE bookmarks_fts MATCH ?)`); + params.push(filters.query); + } } if (filters.author) { conditions.push(`b.author_handle = ? COLLATE NOCASE`); @@ -536,19 +541,70 @@ export function sanitizeFtsQuery(query: string): string { .join(' '); } +function isCjkQuery(query: string): boolean { + return /[\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff\uac00-\ud7af]/u.test(query); +} + +function escapeLikeQuery(query: string): string { + return query + .replace(/\\/g, '\\\\') + .replace(/%/g, '\\%') + .replace(/_/g, '\\_'); +} + +function cjkLikePattern(query: string): string { + return `%${escapeLikeQuery(query)}%`; +} + +function cjkLikeCondition(alias: string, query: string): string { + const columns = [ + 'text', + 'author_handle', + 'author_name', + 'article_title', + 'article_text', + 'article_site', + 'links_json', + ]; + const perTermCondition = `(${columns.map((column) => `COALESCE(${alias}.${column}, '') LIKE ? ESCAPE '\\'`).join(' OR ')})`; + return cjkSearchTerms(query).map(() => perTermCondition).join(' AND '); +} + +function cjkLikeParams(query: string): string[] { + return cjkSearchTerms(query).flatMap((term) => { + const pattern = cjkLikePattern(term); + return Array.from({ length: 7 }, () => pattern); + }); +} + +function cjkSearchTerms(query: string): string[] { + const terms = query + .trim() + .split(/\s+/) + .map((term) => term.trim()) + .filter(Boolean); + return terms.length ? terms : [query.trim()]; +} + export async function searchBookmarks(options: SearchOptions): Promise { const dbPath = twitterBookmarksIndexPath(); const db = await openDb(dbPath); ensureMigrations(db); const limit = options.limit ?? 20; + const useCjkSearch = Boolean(options.query && isCjkQuery(options.query)); try { const conditions: string[] = []; const params: any[] = []; if (options.query) { - conditions.push(`b.rowid IN (SELECT rowid FROM bookmarks_fts WHERE bookmarks_fts MATCH ?)`); - params.push(sanitizeFtsQuery(options.query)); + if (useCjkSearch) { + conditions.push(cjkLikeCondition('b', options.query)); + params.push(...cjkLikeParams(options.query)); + } else { + conditions.push(`b.rowid IN (SELECT rowid FROM bookmarks_fts WHERE bookmarks_fts MATCH ?)`); + params.push(sanitizeFtsQuery(options.query)); + } } if (options.author) { conditions.push(`b.author_handle = ? COLLATE NOCASE`); @@ -566,13 +622,13 @@ export async function searchBookmarks(options: SearchOptions): Promise 0 ? `WHERE ${conditions.join(' AND ')}` : ''; // If we have an FTS query, use bm25 for ranking; otherwise sort by posted_at - const orderBy = options.query + const orderBy = options.query && !useCjkSearch ? `ORDER BY bm25(bookmarks_fts, 5.0, 1.0, 1.0, 3.0) ASC` : `ORDER BY b.posted_at DESC`; // For FTS ranking we need to join with the FTS table for bm25 let sql: string; - if (options.query) { + if (options.query && !useCjkSearch) { sql = ` SELECT b.id, b.url, b.text, b.author_handle, b.author_name, b.posted_at, bm25(bookmarks_fts, 5.0, 1.0, 1.0, 3.0) as score diff --git a/tests/bookmarks-db.test.ts b/tests/bookmarks-db.test.ts index a5212a65..16de4738 100644 --- a/tests/bookmarks-db.test.ts +++ b/tests/bookmarks-db.test.ts @@ -3,7 +3,7 @@ import assert from 'node:assert/strict'; import { mkdtemp, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { buildIndex, searchBookmarks, getStats, formatSearchResults, getBookmarkById, listBookmarks, sanitizeFtsQuery, getCategoryCounts, sampleByCategory, getClassificationProgress } from '../src/bookmarks-db.js'; +import { buildIndex, searchBookmarks, getStats, formatSearchResults, getBookmarkById, listBookmarks, countBookmarks, sanitizeFtsQuery, getCategoryCounts, sampleByCategory, getClassificationProgress } from '../src/bookmarks-db.js'; import { openDb, saveDb } from '../src/db.js'; import { twitterBookmarksIndexPath } from '../src/paths.js'; @@ -13,6 +13,13 @@ const FIXTURES = [ { id: '3', tweetId: '3', url: 'https://x.com/alice/status/3', text: 'Deep learning models need massive compute', authorHandle: 'alice', authorName: 'Alice Smith', syncedAt: '2026-03-01T00:00:00Z', postedAt: '2026-03-01T12:00:00Z', language: 'en', engagement: { likeCount: 200, repostCount: 30 }, mediaObjects: [{ type: 'photo', url: 'https://img.com/1.jpg' }], links: [], tags: [], ingestedVia: 'graphql' }, ]; +const CJK_FIXTURES = [ + ...FIXTURES, + { id: '4', tweetId: '4', url: 'https://x.com/xiaohu/status/4', text: '一个非常狠的提示词，超级严厉的老师会一直追问。', authorHandle: 'xiaohu', authorName: '小互', syncedAt: '2026-04-01T00:00:00Z', postedAt: '2026-04-01T12:00:00Z', language: 'zh', engagement: { likeCount: 300 }, mediaObjects: [], links: [], tags: [], ingestedVia: 'graphql' }, + { id: '5', tweetId: '5', url: 'https://x.com/tools/status/5', text: 'AI 工具链需要可靠的本地搜索。', authorHandle: 'tools', authorName: '工具箱', syncedAt: '2026-05-01T00:00:00Z', postedAt: '2026-05-01T12:00:00Z', language: 'zh', engagement: { likeCount: 120 }, mediaObjects: [], links: ['https://example.com/中文搜索'], tags: [], ingestedVia: 'graphql' }, + { id: '6', tweetId: '6', url: 'https://x.com/memory/status/6', text: '这些未读内容来自 X、播客、微信、RSS 等信息流。', authorHandle: 'memory', authorName: '记忆系统', syncedAt: '2026-06-01T00:00:00Z', postedAt: '2026-06-01T12:00:00Z', language: 'zh', engagement: { likeCount: 80 }, mediaObjects: [], links: [], tags: [], ingestedVia: 'graphql' }, +]; + async function withIsolatedDataDir(fn: () => Promise, fixtures: any[] = FIXTURES): Promise { const dir = await mkdtemp(path.join(tmpdir(), 'ft-test-')); const jsonl = fixtures.map((r) => JSON.stringify(r)).join('\n') + '\n'; @@ -143,6 +150,45 @@ test('searchBookmarks: no results for unmatched query', async () => { }); }); +test('searchBookmarks: CJK query matches Chinese substrings', async () => { + await withIsolatedDataDir(async () => { + await buildIndex(); + const results = await searchBookmarks({ query: '提示词', limit: 10 }); + assert.equal(results.length, 1); + assert.equal(results[0]?.id, '4'); + }, CJK_FIXTURES); +}); + +test('searchBookmarks: CJK phrase query matches Chinese text literally', async () => { + await withIsolatedDataDir(async () => { + await buildIndex(); + const results = await searchBookmarks({ query: '严厉的老师', limit: 10 }); + assert.equal(results.length, 1); + assert.equal(results[0]?.id, '4'); + }, CJK_FIXTURES); +}); + +test('listBookmarks and countBookmarks: CJK query uses the same substring search path', async () => { + await withIsolatedDataDir(async () => { + await buildIndex(); + const listed = await listBookmarks({ query: '工具箱', limit: 10 }); + assert.equal(listed.length, 1); + assert.equal(listed[0]?.id, '5'); + + const count = await countBookmarks({ query: '工具箱' }); + assert.equal(count, 1); + }, CJK_FIXTURES); +}); + +test('searchBookmarks: CJK queries split on whitespace and match terms across punctuation', async () => { + await withIsolatedDataDir(async () => { + await buildIndex(); + const results = await searchBookmarks({ query: '微信 RSS', limit: 10 }); + assert.equal(results.length, 1); + assert.equal(results[0]?.id, '6'); + }, CJK_FIXTURES); +}); + test('getStats returns correct aggregate data', async () => { await withIsolatedDataDir(async () => { await buildIndex();