diff --git a/src/search/core/engine-quality.ts b/src/search/core/engine-quality.ts index e8a560fd1..7a9273278 100644 --- a/src/search/core/engine-quality.ts +++ b/src/search/core/engine-quality.ts @@ -61,6 +61,12 @@ const ENGINE_QUALITY: Record = { 'github-code': 'medium', arxiv: 'medium', 'semantic-scholar': 'medium', + // OpenAlex and OpenReview return real abstracts, matching the arxiv/S2 tier. + // DBLP is bibliographic metadata only (author + venue + year, no abstract), + // so it sits with the other metadata-only lookups below. + openalex: 'medium', + openreview: 'medium', + dblp: 'low', lobsters: 'low', devdocs: 'low', // RSS feed engine (news vertical, conditional on config). Curated by the diff --git a/src/search/core/verticals/papers.ts b/src/search/core/verticals/papers.ts index 1df6e091b..ffb05bf75 100644 --- a/src/search/core/verticals/papers.ts +++ b/src/search/core/verticals/papers.ts @@ -1,7 +1,17 @@ import { ArxivEngine } from '../../engines/arxiv.js'; import { SemanticScholarEngine } from '../../engines/semantic-scholar.js'; +import { OpenAlexEngine } from '../../engines/openalex.js'; +import { DblpEngine } from '../../engines/dblp.js'; +import { OpenReviewEngine } from '../../engines/openreview.js'; import { wrapWithRetryAndBreaker, type EngineEntry } from '../engine-base.js'; +// arXiv + Semantic Scholar are the canonical primary sources. OpenAlex, DBLP, +// and OpenReview broaden coverage (open metadata, CS conference bibliography, +// peer-reviewed venues) as SECONDARY signals: the orchestrator demotes results +// only they contribute when lexical alignment is low, so they add recall +// without outranking a strong arXiv/S2 hit. OpenAlex and OpenReview return real +// abstracts (medium); DBLP is bibliographic metadata only — author + venue + +// year, no abstract — so it takes the low tier, matching devdocs. let cached: EngineEntry[] | null = null; export function getPapersEngines(): EngineEntry[] { @@ -12,6 +22,9 @@ export function getPapersEngines(): EngineEntry[] { // can treat date-aware queries uniformly. { engine: wrapWithRetryAndBreaker(new ArxivEngine()), weight: 1.1, supportsDateFilter: true, quality: 'medium' }, { engine: wrapWithRetryAndBreaker(new SemanticScholarEngine()), weight: 1.0, supportsDateFilter: true, quality: 'medium' }, + { engine: wrapWithRetryAndBreaker(new OpenAlexEngine()), weight: 0.8, supportsDateFilter: false, secondary: true, quality: 'medium' }, + { engine: wrapWithRetryAndBreaker(new DblpEngine()), weight: 0.7, supportsDateFilter: false, secondary: true, quality: 'low' }, + { engine: wrapWithRetryAndBreaker(new OpenReviewEngine()), weight: 0.6, supportsDateFilter: false, secondary: true, quality: 'medium' }, ]; return cached; } diff --git a/src/search/engines/dblp.ts b/src/search/engines/dblp.ts new file mode 100644 index 000000000..601abb30a --- /dev/null +++ b/src/search/engines/dblp.ts @@ -0,0 +1,148 @@ +import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js'; +import { createLogger } from '../../logger.js'; + +const log = createLogger('search'); + +const SNIPPET_LIMIT = 200; + +interface DblpAuthor { + text?: unknown; +} + +interface DblpAuthors { + // DBLP returns a single-author paper's author as a bare object, not an array. + author?: DblpAuthor | DblpAuthor[]; +} + +interface DblpInfo { + authors?: DblpAuthors; + title?: unknown; + venue?: unknown; + year?: unknown; + type?: unknown; + doi?: unknown; + ee?: unknown; + url?: unknown; +} + +interface DblpHit { + info?: DblpInfo; +} + +interface DblpHits { + hit?: DblpHit[]; +} + +interface DblpResult { + hits?: DblpHits; +} + +interface DblpResponse { + result?: DblpResult; +} + +// Cap the author list so a many-author paper doesn't crowd venue/year out of +// the snippet. +const MAX_AUTHORS = 3; + +function asString(v: unknown): string | undefined { + return typeof v === 'string' && v.length > 0 ? v : undefined; +} + +// DBLP's `ee`/`url` are normally absolute, but resolve against the DBLP origin +// defensively: a relative path becomes a usable link and a malformed value is +// dropped rather than stored as a broken result URL. +const DBLP_ORIGIN = 'https://dblp.org/'; + +function toAbsoluteUrl(v: unknown): string | undefined { + const s = asString(v); + if (!s) return undefined; + try { + return new URL(s, DBLP_ORIGIN).href; + } catch { + return undefined; + } +} + +/** + * Author names, normalized across DBLP's two shapes: a single author is a bare + * object, multiple authors are an array. Trimmed to the first {@link MAX_AUTHORS} + * with an "et al." marker so the byline can't dominate the snippet. + */ +function authorNames(authors: DblpAuthors | undefined): string | undefined { + const raw = authors?.author; + if (!raw) return undefined; + const list = Array.isArray(raw) ? raw : [raw]; + const names = list.map((a) => asString(a?.text)).filter((n): n is string => n !== undefined); + if (names.length === 0) return undefined; + const shown = names.slice(0, MAX_AUTHORS).join(', '); + return names.length > MAX_AUTHORS ? `${shown} et al.` : shown; +} + +export class DblpEngine implements SearchEngine { + name = 'dblp'; + + async search(query: string, options: SearchEngineOptions = {}): Promise { + const timeoutMs = options.timeoutMs ?? 10000; + const maxResults = options.maxResults ?? 10; + + const params = new URLSearchParams({ + q: query, + format: 'json', + h: String(maxResults), + }); + const url = `https://dblp.org/search/publ/api?${params}`; + log.debug('dblp search', { query }); + + const response = await fetch(url, { + signal: AbortSignal.timeout(timeoutMs), + headers: { Accept: 'application/json' }, + }); + + if (!response.ok) throw new Error(`DBLP returned ${response.status}`); + + const data = (await response.json()) as DblpResponse; + return this.parseHits(data.result?.hits?.hit ?? []); + } + + private parseHits(hits: DblpHit[]): RawSearchResult[] { + const results: RawSearchResult[] = []; + const total = hits.length; + + for (let i = 0; i < total; i++) { + const info = hits[i].info; + const title = asString(info?.title); + if (!title) continue; + + // `ee` is the publisher/DOI link to the paper itself; `url` is the DBLP + // record page. Prefer the paper, fall back to the record. Both are + // resolved to absolute form so a relative value is still usable. + const url = toAbsoluteUrl(info?.ee) ?? toAbsoluteUrl(info?.url); + if (!url) continue; + + // DBLP returns bibliographic metadata, not abstracts, so the snippet is + // assembled from every text-bearing field it does return — authors, then + // venue and year — to give lexical alignment downstream some queryable + // surface beyond the title. Authors lead because they are the field + // users most often search a paper by. + const authors = authorNames(info?.authors); + const venue = asString(info?.venue); + const year = asString(info?.year); + const venueYear = [venue, year].filter(Boolean).join(' '); + const snippet = [authors, venueYear].filter(Boolean).join(' — ').slice(0, SNIPPET_LIMIT); + + const published_date = year && /^\d{4}$/.test(year) ? `${year}-01-01T00:00:00.000Z` : undefined; + + results.push({ + title, + url, + snippet, + relevance_score: 1 - i / Math.max(total, 1), + engine: 'dblp', + ...(published_date ? { published_date } : {}), + }); + } + + return results; + } +} diff --git a/src/search/engines/openalex.ts b/src/search/engines/openalex.ts new file mode 100644 index 000000000..05eabfd2a --- /dev/null +++ b/src/search/engines/openalex.ts @@ -0,0 +1,114 @@ +import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js'; +import { createLogger } from '../../logger.js'; + +const log = createLogger('search'); + +const SNIPPET_LIMIT = 200; + +interface OaSource { + landing_page_url?: unknown; +} + +interface OaWork { + id?: unknown; + title?: unknown; + publication_year?: unknown; + publication_date?: unknown; + primary_location?: OaSource; + abstract_inverted_index?: unknown; +} + +interface OaResponse { + results?: OaWork[]; +} + +function asString(v: unknown): string | undefined { + return typeof v === 'string' && v.length > 0 ? v : undefined; +} + +/** + * OpenAlex ships the abstract as an inverted index — `{ word: [positions] }` — + * rather than plain text, to sidestep publishers' full-text redistribution + * limits. Rebuild the prose by placing each word at each of its positions. + * Returns undefined when the field is absent or malformed so the caller can + * fall back cleanly. + */ +// Cap the sparse-array size so a malformed huge position can't balloon it. +// Real abstracts topped out at ~4000 words in sampling; 10000 clears that. +const MAX_POSITION = 10000; + +function reconstructAbstract(index: unknown): string | undefined { + if (typeof index !== 'object' || index === null) return undefined; + const slots: string[] = []; + for (const [word, positions] of Object.entries(index as Record)) { + if (!Array.isArray(positions)) continue; + for (const p of positions) { + if (typeof p === 'number' && p >= 0 && p < MAX_POSITION) slots[p] = word; + } + } + const text = slots.join(' ').replace(/\s+/g, ' ').trim(); + return text.length > 0 ? text : undefined; +} + +export class OpenAlexEngine implements SearchEngine { + name = 'openalex'; + + async search(query: string, options: SearchEngineOptions = {}): Promise { + const timeoutMs = options.timeoutMs ?? 10000; + const maxResults = options.maxResults ?? 10; + + // `mailto` opts into OpenAlex's faster "polite pool" and identifies the + // client, as their usage policy requests. Keyless either way. + const params = new URLSearchParams({ + search: query, + per_page: String(maxResults), + mailto: 'wigolo@users.noreply.github.com', + }); + const url = `https://api.openalex.org/works?${params}`; + log.debug('openalex search', { query }); + + const response = await fetch(url, { + signal: AbortSignal.timeout(timeoutMs), + headers: { Accept: 'application/json' }, + }); + + if (!response.ok) throw new Error(`OpenAlex returned ${response.status}`); + + const data = (await response.json()) as OaResponse; + return this.parseWorks(data.results ?? []); + } + + private parseWorks(works: OaWork[]): RawSearchResult[] { + const results: RawSearchResult[] = []; + const total = works.length; + + for (let i = 0; i < total; i++) { + const work = works[i]; + const title = asString(work.title); + if (!title) continue; + + // primary_location is the publisher's page; the OpenAlex work URL is the + // fallback so a record with no landing page still resolves. + const url = asString(work.primary_location?.landing_page_url) ?? asString(work.id); + if (!url) continue; + + const abstract = reconstructAbstract(work.abstract_inverted_index); + const snippet = (abstract ?? '').slice(0, SNIPPET_LIMIT); + + const date = asString(work.publication_date); + const year = typeof work.publication_year === 'number' ? work.publication_year : undefined; + const published_date = date ?? (year ? `${year}-01-01T00:00:00.000Z` : undefined); + + results.push({ + title, + url, + snippet, + relevance_score: 1 - i / Math.max(total, 1), + engine: 'openalex', + ...(published_date ? { published_date } : {}), + }); + } + + return results; + } +} diff --git a/src/search/engines/openreview.ts b/src/search/engines/openreview.ts new file mode 100644 index 000000000..0a85858db --- /dev/null +++ b/src/search/engines/openreview.ts @@ -0,0 +1,89 @@ +import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js'; +import { createLogger } from '../../logger.js'; + +const log = createLogger('search'); + +const SNIPPET_LIMIT = 200; + +// OpenReview wraps every content field as `{ value: ... }`. +interface OrField { + value?: unknown; +} + +interface OrContent { + title?: OrField; + abstract?: OrField; +} + +interface OrNote { + forum?: unknown; + content?: OrContent; +} + +interface OrResponse { + notes?: OrNote[]; +} + +function asString(v: unknown): string | undefined { + return typeof v === 'string' && v.length > 0 ? v : undefined; +} + +export class OpenReviewEngine implements SearchEngine { + name = 'openreview'; + + async search(query: string, options: SearchEngineOptions = {}): Promise { + const timeoutMs = options.timeoutMs ?? 10000; + const maxResults = options.maxResults ?? 10; + + // `source=forum` restricts results to the note that opened each forum — the + // paper submission itself — so reviews and comments (which carry no title + // and only reference the paper) never enter the result set. + const params = new URLSearchParams({ + query, + source: 'forum', + limit: String(maxResults), + }); + const url = `https://api2.openreview.net/notes/search?${params}`; + log.debug('openreview search', { query }); + + const response = await fetch(url, { + signal: AbortSignal.timeout(timeoutMs), + headers: { Accept: 'application/json' }, + }); + + if (!response.ok) throw new Error(`OpenReview returned ${response.status}`); + + const data = (await response.json()) as OrResponse; + return this.parseNotes(data.notes ?? []); + } + + private parseNotes(notes: OrNote[]): RawSearchResult[] { + const results: RawSearchResult[] = []; + const total = notes.length; + + for (let i = 0; i < total; i++) { + const note = notes[i]; + + const title = asString(note.content?.title?.value); + if (!title) continue; + + // `forum` is the paper's thread id; the forum URL is the paper page. + const forum = asString(note.forum); + if (!forum) continue; + const url = `https://openreview.net/forum?id=${forum}`; + + const abstract = asString(note.content?.abstract?.value) ?? ''; + const snippet = abstract.slice(0, SNIPPET_LIMIT); + + results.push({ + title, + url, + snippet, + relevance_score: 1 - i / Math.max(total, 1), + engine: 'openreview', + }); + } + + return results; + } +} diff --git a/tests/unit/search/engines/dblp.test.ts b/tests/unit/search/engines/dblp.test.ts new file mode 100644 index 000000000..f29e825ac --- /dev/null +++ b/tests/unit/search/engines/dblp.test.ts @@ -0,0 +1,232 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { DblpEngine } from '../../../../src/search/engines/dblp.js'; + +interface FetchCall { + url: string; + init?: RequestInit; +} + +function captureFetch(body: unknown, ok = true, status = 200): { + calls: FetchCall[]; +} { + const calls: FetchCall[] = []; + vi.spyOn(global, 'fetch').mockImplementation(async (input, init) => { + const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : (input as Request).url; + calls.push({ url, init }); + return { + ok, + status, + json: async () => body, + text: async () => JSON.stringify(body), + } as Response; + }); + return { calls }; +} + +/** Wrap hit `info` objects in the nested result.hits.hit envelope DBLP returns. */ +function dblpBody(infos: Array>): Record { + return { result: { hits: { hit: infos.map((info) => ({ info })) } } }; +} + +describe('DblpEngine', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('has name set to dblp', () => { + expect(new DblpEngine().name).toBe('dblp'); + }); + + it('maps a successful response into RawSearchResult fields', async () => { + const body = dblpBody([ + { + authors: { author: [{ text: 'Ashish Vaswani' }, { text: 'Noam Shazeer' }] }, + title: 'Attention Is All You Need', + venue: 'NeurIPS', + year: '2017', + ee: 'https://doi.org/10.5555/3295222.3295349', + url: 'https://dblp.org/rec/conf/nips/VaswaniSPUJGKP17', + }, + { + authors: { author: [{ text: 'Jacob Devlin' }] }, + title: 'BERT: Pre-training of Deep Bidirectional Transformers', + venue: 'NAACL-HLT', + year: '2019', + ee: 'https://doi.org/10.18653/v1/n19-1423', + }, + ]); + captureFetch(body); + const results = await new DblpEngine().search('transformer'); + + expect(results).toHaveLength(2); + expect(results[0].title).toBe('Attention Is All You Need'); + // ee is preferred over the dblp record url. + expect(results[0].url).toBe('https://doi.org/10.5555/3295222.3295349'); + // Snippet is assembled from the text-bearing fields DBLP returns — authors, + // then venue + year — since it carries no abstract. + expect(results[0].snippet).toBe('Ashish Vaswani, Noam Shazeer — NeurIPS 2017'); + expect(results[0].engine).toBe('dblp'); + expect(results[0].relevance_score).toBe(1); + expect(results[0].published_date).toBe('2017-01-01T00:00:00.000Z'); + }); + + // ee is the paper's publisher/DOI link; url is the DBLP record page. When ee + // is absent the record page is the only usable link. + it('falls back to the dblp record url when ee is absent', async () => { + const body = dblpBody([ + { title: 'A paper', venue: 'ICML', year: '2020', url: 'https://dblp.org/rec/x' }, + ]); + captureFetch(body); + const results = await new DblpEngine().search('q'); + expect(results[0].url).toBe('https://dblp.org/rec/x'); + }); + + // ee/url are normally absolute, but a relative value is resolved against the + // DBLP origin so it stays a usable link rather than a broken relative path. + it('resolves a relative ee against the dblp origin', async () => { + const body = dblpBody([ + { title: 'A paper', venue: 'ICML', year: '2020', ee: '/rec/conf/x/y.html' }, + ]); + captureFetch(body); + const results = await new DblpEngine().search('q'); + expect(results[0].url).toBe('https://dblp.org/rec/conf/x/y.html'); + }); + + it('resolves a relative url fallback against the dblp origin', async () => { + const body = dblpBody([ + { title: 'A paper', venue: 'ICML', year: '2020', url: 'rec/z' }, + ]); + captureFetch(body); + const results = await new DblpEngine().search('q'); + expect(results[0].url).toBe('https://dblp.org/rec/z'); + }); + + // A malformed ee is dropped so the record url can still stand in. + it('falls back to url when ee is a malformed value', async () => { + const body = dblpBody([ + { title: 'A paper', venue: 'ICML', year: '2020', ee: 'http://', url: 'https://dblp.org/rec/w' }, + ]); + captureFetch(body); + const results = await new DblpEngine().search('q'); + expect(results[0].url).toBe('https://dblp.org/rec/w'); + }); + + it('skips a hit with neither ee nor url', async () => { + const body = dblpBody([ + { title: 'linkless', venue: 'ICML', year: '2020' }, + { title: 'keeper', venue: 'ICML', year: '2020', url: 'https://dblp.org/rec/y' }, + ]); + captureFetch(body); + const results = await new DblpEngine().search('q'); + expect(results).toHaveLength(1); + expect(results[0].title).toBe('keeper'); + }); + + it('skips a hit with no title', async () => { + const body = dblpBody([ + { venue: 'ICML', year: '2020', url: 'https://dblp.org/rec/z' }, + { title: 'real', url: 'https://dblp.org/rec/w' }, + ]); + captureFetch(body); + const results = await new DblpEngine().search('q'); + expect(results).toHaveLength(1); + expect(results[0].title).toBe('real'); + }); + + it('builds a venue-only snippet when authors and year are missing', async () => { + const body = dblpBody([{ title: 't', venue: 'ICML', url: 'https://dblp.org/rec/a' }]); + captureFetch(body); + const results = await new DblpEngine().search('q'); + expect(results[0].snippet).toBe('ICML'); + }); + + // DBLP returns a single-author paper's author as a bare object, not an array. + // The parser must handle both without crashing. + it('handles the single-author object shape', async () => { + const body = dblpBody([ + { + authors: { author: { text: 'Donald E. Knuth' } }, + title: 'Literate Programming', + venue: 'Comput. J.', + year: '1984', + url: 'https://dblp.org/rec/x', + }, + ]); + captureFetch(body); + const results = await new DblpEngine().search('q'); + expect(results[0].snippet).toBe('Donald E. Knuth — Comput. J. 1984'); + }); + + it('caps the byline at three authors with an et al. marker', async () => { + const body = dblpBody([ + { + authors: { + author: [ + { text: 'A One' }, + { text: 'B Two' }, + { text: 'C Three' }, + { text: 'D Four' }, + ], + }, + title: 'Many hands', + venue: 'ICML', + year: '2020', + url: 'https://dblp.org/rec/y', + }, + ]); + captureFetch(body); + const results = await new DblpEngine().search('q'); + expect(results[0].snippet).toBe('A One, B Two, C Three et al. — ICML 2020'); + }); + + it('omits published_date when year is not a 4-digit value', async () => { + const body = dblpBody([ + { title: 't', venue: 'ICML', year: 'n/a', url: 'https://dblp.org/rec/a' }, + ]); + captureFetch(body); + const results = await new DblpEngine().search('q'); + expect(results[0].published_date).toBeUndefined(); + }); + + it('returns empty array when there are no hits', async () => { + captureFetch(dblpBody([])); + expect(await new DblpEngine().search('q')).toEqual([]); + }); + + it('returns empty array when the result envelope is missing', async () => { + captureFetch({}); + expect(await new DblpEngine().search('q')).toEqual([]); + }); + + it('throws on non-ok responses', async () => { + captureFetch({}, false, 500); + await expect(new DblpEngine().search('q')).rejects.toThrow(/DBLP returned 500/); + }); + + it('throws on malformed JSON', async () => { + vi.spyOn(global, 'fetch').mockImplementation(async () => { + return { + ok: true, + status: 200, + json: async () => { + throw new Error('invalid json'); + }, + } as unknown as Response; + }); + await expect(new DblpEngine().search('q')).rejects.toThrow(/invalid json/); + }); + + it('passes AbortSignal.timeout to fetch', async () => { + const { calls } = captureFetch(dblpBody([])); + await new DblpEngine().search('q', { timeoutMs: 5000 }); + expect(calls[0].init?.signal).toBeInstanceOf(AbortSignal); + }); + + it('encodes q, format and h from the query and maxResults', async () => { + const { calls } = captureFetch(dblpBody([])); + await new DblpEngine().search('graph neural', { maxResults: 15 }); + expect(calls[0].url).toContain('q=graph+neural'); + expect(calls[0].url).toContain('format=json'); + expect(calls[0].url).toContain('h=15'); + }); +}); diff --git a/tests/unit/search/engines/openalex.test.ts b/tests/unit/search/engines/openalex.test.ts new file mode 100644 index 000000000..74334c811 --- /dev/null +++ b/tests/unit/search/engines/openalex.test.ts @@ -0,0 +1,197 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { OpenAlexEngine } from '../../../../src/search/engines/openalex.js'; + +interface FetchCall { + url: string; + init?: RequestInit; +} + +function captureFetch(body: unknown, ok = true, status = 200): { + calls: FetchCall[]; +} { + const calls: FetchCall[] = []; + vi.spyOn(global, 'fetch').mockImplementation(async (input, init) => { + const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : (input as Request).url; + calls.push({ url, init }); + return { + ok, + status, + json: async () => body, + text: async () => JSON.stringify(body), + } as Response; + }); + return { calls }; +} + +function oaBody(works: Array>): Record { + return { results: works }; +} + +describe('OpenAlexEngine', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('has name set to openalex', () => { + expect(new OpenAlexEngine().name).toBe('openalex'); + }); + + it('maps a successful response into RawSearchResult fields', async () => { + const body = oaBody([ + { + id: 'https://openalex.org/W1', + title: 'Attention Is All You Need', + publication_date: '2017-06-12', + publication_year: 2017, + primary_location: { landing_page_url: 'https://doi.org/10.5555/x' }, + // "we propose the transformer" as an inverted index + abstract_inverted_index: { we: [0], propose: [1], the: [2], transformer: [3] }, + }, + ]); + captureFetch(body); + const results = await new OpenAlexEngine().search('transformer'); + + expect(results).toHaveLength(1); + expect(results[0].title).toBe('Attention Is All You Need'); + // landing_page_url is preferred over the openalex work id. + expect(results[0].url).toBe('https://doi.org/10.5555/x'); + expect(results[0].snippet).toBe('we propose the transformer'); + expect(results[0].engine).toBe('openalex'); + expect(results[0].relevance_score).toBe(1); + // publication_date is used verbatim when present. + expect(results[0].published_date).toBe('2017-06-12'); + }); + + // The inverted index maps each word to every position it occupies. The words + // arrive in arbitrary key order and a word can repeat at several positions — + // reconstruction must place them by position, not by key order. + it('reconstructs an abstract from an out-of-order, repeated-word inverted index', async () => { + const body = oaBody([ + { + id: 'https://openalex.org/W2', + title: 'Repeats', + // "the cat sat on the mat" — "the" appears at 0 and 4, keys shuffled + abstract_inverted_index: { mat: [5], the: [0, 4], on: [3], cat: [1], sat: [2] }, + }, + ]); + captureFetch(body); + const results = await new OpenAlexEngine().search('q'); + expect(results[0].snippet).toBe('the cat sat on the mat'); + }); + + it('truncates the reconstructed abstract to the snippet limit', async () => { + const index: Record = {}; + // 60 distinct 5-char words → ~360 chars, above the 200-char cap. + for (let i = 0; i < 60; i++) index[`word${String(i).padStart(2, '0')}`] = [i]; + const body = oaBody([{ id: 'https://openalex.org/W3', title: 'Long', abstract_inverted_index: index }]); + captureFetch(body); + const results = await new OpenAlexEngine().search('q'); + expect(results[0].snippet.length).toBe(200); + }); + + // Positions index a sparse array whose size is driven by the largest value, + // so an anomalously large position is dropped rather than ballooning it. + it('drops abstract words at out-of-range positions', async () => { + const body = oaBody([ + { + id: 'https://openalex.org/W3b', + title: 'Capped', + abstract_inverted_index: { real: [0], words: [1], junk: [1_000_000] }, + }, + ]); + captureFetch(body); + const results = await new OpenAlexEngine().search('q'); + expect(results[0].snippet).toBe('real words'); + }); + + it('yields an empty snippet when the abstract index is absent', async () => { + const body = oaBody([{ id: 'https://openalex.org/W4', title: 'No abstract' }]); + captureFetch(body); + const results = await new OpenAlexEngine().search('q'); + expect(results[0].snippet).toBe(''); + }); + + // A record with no publisher landing page still resolves via the OpenAlex + // work URL rather than being dropped. + it('falls back to the work id url when landing_page_url is absent', async () => { + const body = oaBody([ + { id: 'https://openalex.org/W5', title: 'Fallback', primary_location: {} }, + ]); + captureFetch(body); + const results = await new OpenAlexEngine().search('q'); + expect(results[0].url).toBe('https://openalex.org/W5'); + }); + + it('derives published_date from the year when publication_date is absent', async () => { + const body = oaBody([ + { id: 'https://openalex.org/W6', title: 'Year only', publication_year: 2020 }, + ]); + captureFetch(body); + const results = await new OpenAlexEngine().search('q'); + expect(results[0].published_date).toBe('2020-01-01T00:00:00.000Z'); + }); + + it('skips a work with no title', async () => { + const body = oaBody([ + { id: 'https://openalex.org/W7' }, + { id: 'https://openalex.org/W8', title: 'real' }, + ]); + captureFetch(body); + const results = await new OpenAlexEngine().search('q'); + expect(results).toHaveLength(1); + expect(results[0].title).toBe('real'); + }); + + it('skips a work with neither landing page nor id', async () => { + const body = oaBody([ + { title: 'linkless' }, + { id: 'https://openalex.org/W9', title: 'keeper' }, + ]); + captureFetch(body); + const results = await new OpenAlexEngine().search('q'); + expect(results).toHaveLength(1); + expect(results[0].title).toBe('keeper'); + }); + + it('returns empty array when results are empty', async () => { + captureFetch(oaBody([])); + expect(await new OpenAlexEngine().search('q')).toEqual([]); + }); + + it('returns empty array when the results key is missing', async () => { + captureFetch({}); + expect(await new OpenAlexEngine().search('q')).toEqual([]); + }); + + it('throws on non-ok responses', async () => { + captureFetch({}, false, 429); + await expect(new OpenAlexEngine().search('q')).rejects.toThrow(/OpenAlex returned 429/); + }); + + it('throws on malformed JSON', async () => { + vi.spyOn(global, 'fetch').mockImplementation(async () => { + return { + ok: true, + status: 200, + json: async () => { + throw new Error('invalid json'); + }, + } as unknown as Response; + }); + await expect(new OpenAlexEngine().search('q')).rejects.toThrow(/invalid json/); + }); + + it('passes AbortSignal.timeout to fetch', async () => { + const { calls } = captureFetch(oaBody([])); + await new OpenAlexEngine().search('q', { timeoutMs: 5000 }); + expect(calls[0].init?.signal).toBeInstanceOf(AbortSignal); + }); + + it('encodes search, per_page and mailto into the request', async () => { + const { calls } = captureFetch(oaBody([])); + await new OpenAlexEngine().search('graph neural', { maxResults: 15 }); + expect(calls[0].url).toContain('search=graph+neural'); + expect(calls[0].url).toContain('per_page=15'); + expect(calls[0].url).toContain('mailto='); + }); +}); diff --git a/tests/unit/search/engines/openreview.test.ts b/tests/unit/search/engines/openreview.test.ts new file mode 100644 index 000000000..9650d797f --- /dev/null +++ b/tests/unit/search/engines/openreview.test.ts @@ -0,0 +1,154 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { OpenReviewEngine } from '../../../../src/search/engines/openreview.js'; + +interface FetchCall { + url: string; + init?: RequestInit; +} + +function captureFetch(body: unknown, ok = true, status = 200): { + calls: FetchCall[]; +} { + const calls: FetchCall[] = []; + vi.spyOn(global, 'fetch').mockImplementation(async (input, init) => { + const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : (input as Request).url; + calls.push({ url, init }); + return { + ok, + status, + json: async () => body, + text: async () => JSON.stringify(body), + } as Response; + }); + return { calls }; +} + +/** Wrap plain title/abstract strings in OpenReview's `{ value: ... }` fields. */ +function note(fields: { forum?: string; title?: string; abstract?: string }): Record { + const content: Record = {}; + if (fields.title !== undefined) content.title = { value: fields.title }; + if (fields.abstract !== undefined) content.abstract = { value: fields.abstract }; + return { forum: fields.forum, content }; +} + +function orBody(notes: Array>): Record { + return { notes }; +} + +describe('OpenReviewEngine', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('has name set to openreview', () => { + expect(new OpenReviewEngine().name).toBe('openreview'); + }); + + it('maps a successful response into RawSearchResult fields', async () => { + const body = orBody([ + note({ forum: 'abc123', title: 'Block Transformer', abstract: 'We introduce the Block Transformer.' }), + note({ forum: 'def456', title: 'Diffusion Models' }), + ]); + captureFetch(body); + const results = await new OpenReviewEngine().search('transformer'); + + expect(results).toHaveLength(2); + expect(results[0].title).toBe('Block Transformer'); + // URL is built from the forum id, not returned directly. + expect(results[0].url).toBe('https://openreview.net/forum?id=abc123'); + expect(results[0].snippet).toBe('We introduce the Block Transformer.'); + expect(results[0].engine).toBe('openreview'); + expect(results[0].relevance_score).toBe(1); + // OpenReview notes carry no publication date on this endpoint. + expect(results[0].published_date).toBeUndefined(); + // No abstract field → empty snippet, not padded noise. + expect(results[1].snippet).toBe(''); + }); + + // The source=forum filter is the design decision that keeps this engine + // returning papers rather than their reviews. Lock it into the request so a + // refactor can't silently drop it and let reviews back into the result set. + it('sends source=forum so only paper submissions are searched', async () => { + const { calls } = captureFetch(orBody([])); + await new OpenReviewEngine().search('q'); + expect(calls[0].url).toContain('source=forum'); + }); + + it('reads title and abstract out of the { value } wrapper', async () => { + const body = orBody([note({ forum: 'f', title: 'Wrapped', abstract: 'Also wrapped.' })]); + captureFetch(body); + const results = await new OpenReviewEngine().search('q'); + expect(results[0].title).toBe('Wrapped'); + expect(results[0].snippet).toBe('Also wrapped.'); + }); + + it('truncates the abstract to the snippet limit', async () => { + const body = orBody([note({ forum: 'f', title: 't', abstract: 'x'.repeat(500) })]); + captureFetch(body); + const results = await new OpenReviewEngine().search('q'); + expect(results[0].snippet.length).toBe(200); + }); + + it('skips a note with no title', async () => { + const body = orBody([ + note({ forum: 'f1', abstract: 'orphan abstract' }), + note({ forum: 'f2', title: 'real' }), + ]); + captureFetch(body); + const results = await new OpenReviewEngine().search('q'); + expect(results).toHaveLength(1); + expect(results[0].title).toBe('real'); + }); + + it('skips a note with no forum id', async () => { + const body = orBody([ + note({ title: 'no forum' }), + note({ forum: 'f', title: 'keeper' }), + ]); + captureFetch(body); + const results = await new OpenReviewEngine().search('q'); + expect(results).toHaveLength(1); + expect(results[0].title).toBe('keeper'); + }); + + it('returns empty array when notes are empty', async () => { + captureFetch(orBody([])); + expect(await new OpenReviewEngine().search('q')).toEqual([]); + }); + + it('returns empty array when the notes key is missing', async () => { + captureFetch({}); + expect(await new OpenReviewEngine().search('q')).toEqual([]); + }); + + it('throws on non-ok responses', async () => { + captureFetch({}, false, 500); + await expect(new OpenReviewEngine().search('q')).rejects.toThrow(/OpenReview returned 500/); + }); + + it('throws on malformed JSON', async () => { + vi.spyOn(global, 'fetch').mockImplementation(async () => { + return { + ok: true, + status: 200, + json: async () => { + throw new Error('invalid json'); + }, + } as unknown as Response; + }); + await expect(new OpenReviewEngine().search('q')).rejects.toThrow(/invalid json/); + }); + + it('passes AbortSignal.timeout to fetch', async () => { + const { calls } = captureFetch(orBody([])); + await new OpenReviewEngine().search('q', { timeoutMs: 5000 }); + expect(calls[0].init?.signal).toBeInstanceOf(AbortSignal); + }); + + it('encodes query and limit from the query and maxResults', async () => { + const { calls } = captureFetch(orBody([])); + await new OpenReviewEngine().search('graph neural', { maxResults: 15 }); + expect(calls[0].url).toContain('query=graph+neural'); + expect(calls[0].url).toContain('limit=15'); + }); +}); diff --git a/tests/unit/search/v1/verticals/papers.test.ts b/tests/unit/search/v1/verticals/papers.test.ts index ad1f130de..13e55a40a 100644 --- a/tests/unit/search/v1/verticals/papers.test.ts +++ b/tests/unit/search/v1/verticals/papers.test.ts @@ -11,13 +11,25 @@ describe('getPapersEngines', () => { _resetBreakersForTest(); }); - it('returns two entries', () => { - expect(getPapersEngines()).toHaveLength(2); + it('returns five entries', () => { + expect(getPapersEngines()).toHaveLength(5); }); - it('wraps arxiv and semantic-scholar engines (preserving names)', () => { + it('wraps arxiv, semantic-scholar, openalex, dblp, openreview (preserving names)', () => { const names = getPapersEngines().map((e) => e.engine.name); - expect(names).toEqual(['arxiv', 'semantic-scholar']); + expect(names).toEqual(['arxiv', 'semantic-scholar', 'openalex', 'dblp', 'openreview']); + }); + + it('marks the three added engines secondary and leaves arxiv/S2 primary', () => { + const secondaryNames = ['openalex', 'dblp', 'openreview']; + const entries = getPapersEngines(); + for (const name of secondaryNames) { + expect(entries.find((e) => e.engine.name === name)?.secondary).toBe(true); + } + for (const e of entries) { + if (secondaryNames.includes(e.engine.name)) continue; + expect(e.secondary ?? false).toBe(false); + } }); it('memoizes — two calls return the same array reference', () => { @@ -33,9 +45,13 @@ describe('getPapersEngines', () => { expect(a).not.toBe(b); }); - it('marks supportsDateFilter true on both', () => { - for (const entry of getPapersEngines()) { - expect(entry.supportsDateFilter).toBe(true); - } + it('marks supportsDateFilter true only on arxiv and semantic-scholar', () => { + const entries = getPapersEngines(); + const f = (name: string) => entries.find((e) => e.engine.name === name)?.supportsDateFilter; + expect(f('arxiv')).toBe(true); + expect(f('semantic-scholar')).toBe(true); + expect(f('openalex')).toBe(false); + expect(f('dblp')).toBe(false); + expect(f('openreview')).toBe(false); }); });