From 843eb8931af4966ec0be0e08739ec598a2cc8dd0 Mon Sep 17 00:00:00 2001 From: Prasenjit Sarkar Date: Thu, 23 Jul 2026 00:11:30 +0100 Subject: [PATCH] Make /api/search honest: explain empty results, clear config errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /api/search returned {success:true, count:0, results:[]} with no indication of why — you couldn't tell "no hits" from "keyless provider blocked" from "provider not configured". The root cause of the empty results in production: DuckDuckGo's keyless endpoint anti-bot-challenges datacenter/server IPs (HTTP 202), confirmed from two independent datacenter IPs. That's a DDG policy, not a parser bug — it can't be fixed into working keyless from a hosted API. So make the endpoint transparent rather than pretend: - search.service: new `runSearch()` returns { results, provider, reason? }; the per-provider fetchers now report an actionable `reason` (DDG 202 challenge, request failure, …). `searchWeb()` kept as a results-only wrapper so existing callers/tests are unchanged. New `emptyResultHint(provider)` gives the provider-specific explanation + the real fix. - search route: response now includes `provider` and, when empty, a `note` saying why (e.g. the DDG datacenter block, pointing at SERPER_API_KEY / SEARXNG_URL). Missing provider config now returns a 400 with the actionable message instead of an opaque 500. Reliable search still requires config — but it's free: SERPER_API_KEY (free tier at serper.dev, auto-selected when present) or a self-hosted SEARXNG_URL. Both already worked; now the endpoint tells you so. Docker end-to-end verified: default provider -> 200 + note explaining the 202 block; provider=searxng w/o URL -> 400 with the fix; SEARXNG_URL configured -> 200 with real results (provider:searxng, count:2). tsc, eslint, 204 tests (+3), openapi:check all pass. Co-Authored-By: Claude Opus 4.8 --- src/api/routes/search.routes.ts | 24 +++++-- src/services/search.service.spec.ts | 32 +++++++++ src/services/search.service.ts | 100 ++++++++++++++++++++++------ 3 files changed, 130 insertions(+), 26 deletions(-) diff --git a/src/api/routes/search.routes.ts b/src/api/routes/search.routes.ts index 5397020..40b5c59 100644 --- a/src/api/routes/search.routes.ts +++ b/src/api/routes/search.routes.ts @@ -2,7 +2,7 @@ import { Router, Request, Response } from 'express'; import { apiKeyAuth } from '../middleware/auth.middleware'; import { validateRequest } from '../middleware/validation'; import { expensiveLimiter } from '../middleware/rate-limit.middleware'; -import { searchWeb } from '../../services/search.service'; +import { runSearch } from '../../services/search.service'; import scraperManager from '../../scraper/scraper-manager'; import { logger } from '../../utils/logger'; import { searchRequestSchema } from '../schemas'; @@ -24,18 +24,30 @@ router.post( const { query, limit = 10, provider, lang, scrapeResults, scrapeOptions } = req.body; logger.info(`Search request: "${query}" (limit ${limit}, provider ${provider ?? 'default'})`); - const results = await searchWeb(query, { limit, provider, lang }); + const { results, provider: usedProvider, reason } = await runSearch(query, { limit, provider, lang }); if (!scrapeResults || results.length === 0) { - return res.json({ success: true, query, count: results.length, results }); + // `note` explains an empty result set (e.g. keyless provider blocked) so a + // zero-count response is never a silent mystery. + return res.json({ + success: true, query, provider: usedProvider, count: results.length, results, + ...(reason ? { note: reason } : {}), + }); } // Scrape each result (bounded concurrency to avoid a fan-out storm). const scraped = await scrapeResultsBounded(results, scrapeOptions, 3); - return res.json({ success: true, query, count: scraped.length, results: scraped }); + return res.json({ success: true, query, provider: usedProvider, count: scraped.length, results: scraped }); } catch (error) { - logger.error(`Search error: ${error instanceof Error ? error.message : String(error)}`); - return res.status(500).json({ success: false, error: 'Search failed' }); + const msg = error instanceof Error ? error.message : String(error); + logger.error(`Search error: ${msg}`); + // Missing provider config (SEARXNG_URL / SERPER_API_KEY) is the caller's to fix — + // surface the actionable message as a 400 instead of an opaque 500. + const isConfigError = /SEARXNG_URL|SERPER_API_KEY/.test(msg); + return res.status(isConfigError ? 400 : 500).json({ + success: false, + error: isConfigError ? msg : 'Search failed', + }); } } ); diff --git a/src/services/search.service.spec.ts b/src/services/search.service.spec.ts index dd6997e..b37da64 100644 --- a/src/services/search.service.spec.ts +++ b/src/services/search.service.spec.ts @@ -1,6 +1,8 @@ import { parseDuckDuckGoHtml, searchWeb, + runSearch, + emptyResultHint, SearchResult, } from './search.service'; @@ -175,3 +177,33 @@ describe('searchWeb', () => { await expect(searchWeb('typescript', { provider: 'searxng' })).rejects.toThrow(/SEARXNG_URL/); }); }); + +describe('runSearch diagnostics (why a search came back empty)', () => { + const savedSerper = process.env.SERPER_API_KEY; + const savedSearxng = process.env.SEARXNG_URL; + afterEach(() => { + savedSerper === undefined ? delete process.env.SERPER_API_KEY : (process.env.SERPER_API_KEY = savedSerper); + savedSearxng === undefined ? delete process.env.SEARXNG_URL : (process.env.SEARXNG_URL = savedSearxng); + }); + + it('reports the provider used and no reason for an empty query', async () => { + delete process.env.SERPER_API_KEY; + delete process.env.SEARXNG_URL; + const out = await runSearch(' '); + expect(out).toEqual({ results: [], provider: 'duckduckgo' }); + expect(out.reason).toBeUndefined(); + }); + + it('the duckduckgo empty-hint names the datacenter block and the real fix', () => { + const hint = emptyResultHint('duckduckgo'); + expect(hint).toMatch(/202/); + expect(hint).toMatch(/datacenter|server IPs/i); + expect(hint).toMatch(/SERPER_API_KEY/); + expect(hint).toMatch(/SEARXNG_URL/); + }); + + it('gives provider-appropriate empty hints for searxng and serper', () => { + expect(emptyResultHint('searxng')).toMatch(/SearXNG/); + expect(emptyResultHint('serper')).toMatch(/Serper/); + }); +}); diff --git a/src/services/search.service.ts b/src/services/search.service.ts index 703c723..b4a32b0 100644 --- a/src/services/search.service.ts +++ b/src/services/search.service.ts @@ -73,7 +73,33 @@ const DESKTOP_USER_AGENT = * request failure. * @throws When `opts.provider === 'searxng'` but `SEARXNG_URL` is not configured. */ -export async function searchWeb(query: string, opts: SearchOptions = {}): Promise { +/** Result of a search, including which provider ran and — when empty — why. */ +export interface SearchOutcome { + results: SearchResult[]; + provider: 'duckduckgo' | 'searxng' | 'serper'; + /** + * Set when `results` is empty for a reason the caller can act on — e.g. the + * keyless DuckDuckGo endpoint was anti-bot-challenged, or a provider request + * failed. Absent for a genuinely no-hit query. + */ + reason?: string; +} + +/** Internal per-provider return: results plus an optional actionable reason. */ +interface ProviderResult { + results: SearchResult[]; + reason?: string; +} + +/** + * Run a web search and return results plus diagnostics (which provider ran, and — + * when empty — why). This is what the API route uses so it can tell callers the + * difference between "no hits" and "the keyless provider is blocked / not configured". + * + * @throws When a provider is explicitly selected but its required configuration is + * missing (`searxng` without SEARXNG_URL, `serper` without SERPER_API_KEY). + */ +export async function runSearch(query: string, opts: SearchOptions = {}): Promise { // Auto-select the most reliable configured provider when none is specified: // Serper (keyed) > SearXNG (self-hosted) > DuckDuckGo (keyless, best-effort — // datacenter IPs are frequently anti-bot challenged, so keyless is not reliable @@ -84,27 +110,55 @@ export async function searchWeb(query: string, opts: SearchOptions = {}): Promis const trimmedQuery = (query ?? '').trim(); if (!trimmedQuery) { logger.warn('searchWeb called with an empty query; returning no results', { provider }); - return []; + return { results: [], provider }; } - let results: SearchResult[]; + let raw: ProviderResult; switch (provider) { case 'serper': - results = await fetchSerper(trimmedQuery, limit, opts); + raw = await fetchSerper(trimmedQuery, limit, opts); break; case 'searxng': // NOTE: a missing SEARXNG_URL throws (config error); request failures do not. - results = await fetchSearxng(trimmedQuery, limit, opts); + raw = await fetchSearxng(trimmedQuery, limit, opts); break; case 'duckduckgo': default: - results = await fetchDuckDuckGo(trimmedQuery, limit, opts); + raw = await fetchDuckDuckGo(trimmedQuery, limit, opts); break; } - // Final safety net so the public contract (dedup + cap + contiguous positions) - // always holds, independent of any individual provider's implementation. - return dedupeAndCap(results, limit); + // Dedup + cap + contiguous positions, independent of any provider's implementation. + const results = dedupeAndCap(raw.results, limit); + const reason = results.length === 0 ? (raw.reason ?? emptyResultHint(provider)) : undefined; + return reason ? { results, provider, reason } : { results, provider }; +} + +/** + * Results-only convenience wrapper over {@link runSearch}. Preserves the original + * signature for callers that don't need diagnostics. + */ +export async function searchWeb(query: string, opts: SearchOptions = {}): Promise { + return (await runSearch(query, opts)).results; +} + +/** + * Human-readable explanation for an empty result set, by provider. For the keyless + * DuckDuckGo path this is almost always an anti-bot challenge on server/datacenter + * IPs rather than a genuine no-hit query. + */ +export function emptyResultHint(provider: 'duckduckgo' | 'searxng' | 'serper'): string { + switch (provider) { + case 'duckduckgo': + return 'No results from the keyless DuckDuckGo endpoint. It anti-bot-challenges ' + + 'datacenter / server IPs (HTTP 202), so it is unreliable from a hosted API. For ' + + 'dependable search set SERPER_API_KEY (free tier at serper.dev) or SEARXNG_URL.'; + case 'searxng': + return 'No results from the configured SearXNG instance for this query.'; + case 'serper': + default: + return 'No results from Serper for this query.'; + } } /** @@ -230,13 +284,17 @@ function resolveDuckDuckGoUrl(href: string): string | null { * Fetch and parse results from the keyless DuckDuckGo HTML endpoint. * Never throws — network / HTTP failures are logged and yield `[]`. */ -async function fetchDuckDuckGo(query: string, limit: number, opts: SearchOptions): Promise { +async function fetchDuckDuckGo(query: string, limit: number, opts: SearchOptions): Promise { const form = new URLSearchParams({ q: query }); if (opts.lang) { // DuckDuckGo uses `kl` for region/locale, e.g. `us-en`. form.set('kl', opts.lang); } + const CHALLENGE = 'DuckDuckGo returned an anti-bot challenge (HTTP 202) — its keyless ' + + 'endpoint blocks datacenter / server IPs. Set SERPER_API_KEY (free at serper.dev) or ' + + 'SEARXNG_URL for reliable search.'; + try { // The HTML endpoint expects a POST form submission; a GET returns a 202 // challenge page with no results. Note: DuckDuckGo still anti-bot-challenges @@ -245,6 +303,8 @@ async function fetchDuckDuckGo(query: string, limit: number, opts: SearchOptions const response = await axios.post(DUCKDUCKGO_HTML_ENDPOINT, form.toString(), { timeout: REQUEST_TIMEOUT_MS, responseType: 'text', + // 202 is DDG's challenge; don't let axios throw on it — we handle it explicitly. + validateStatus: (s) => s >= 200 && s < 500, headers: { 'User-Agent': DESKTOP_USER_AGENT, 'Content-Type': 'application/x-www-form-urlencoded', @@ -255,17 +315,17 @@ async function fetchDuckDuckGo(query: string, limit: number, opts: SearchOptions if (response.status === 202) { logger.warn('DuckDuckGo returned a 202 challenge (anti-bot); no results. Configure SERPER_API_KEY or SEARXNG_URL for reliable search.'); - return []; + return { results: [], reason: CHALLENGE }; } const html = typeof response.data === 'string' ? response.data : String(response.data ?? ''); - return parseDuckDuckGoHtml(html, limit); + return { results: parseDuckDuckGoHtml(html, limit) }; } catch (error) { logger.warn('DuckDuckGo search request failed', { query, error: (error as Error).message, }); - return []; + return { results: [], reason: `DuckDuckGo request failed: ${(error as Error).message}` }; } } @@ -286,7 +346,7 @@ function autoProvider(): 'serper' | 'searxng' | 'duckduckgo' { * Serper.dev — a reliable, keyed Google-search API (free tier available). Set * SERPER_API_KEY. POST https://google.serper.dev/search with {q, num}. */ -async function fetchSerper(query: string, limit: number, opts: SearchOptions): Promise { +async function fetchSerper(query: string, limit: number, opts: SearchOptions): Promise { const apiKey = process.env.SERPER_API_KEY; if (!apiKey || !apiKey.trim()) { throw new Error('searchWeb: provider \'serper\' selected but SERPER_API_KEY is not set.'); @@ -301,7 +361,7 @@ async function fetchSerper(query: string, limit: number, opts: SearchOptions): P const organic = (response.data as { organic?: unknown }).organic; if (!Array.isArray(organic)) { logger.warn('Serper response had no organic results array'); - return []; + return { results: [] }; } const results: SearchResult[] = []; for (const r of organic) { @@ -315,14 +375,14 @@ async function fetchSerper(query: string, limit: number, opts: SearchOptions): P }); if (results.length >= limit) break; } - return results; + return { results }; } catch (error) { logger.warn('Serper search request failed', { query, error: (error as Error).message }); - return []; + return { results: [], reason: `Serper request failed: ${(error as Error).message}` }; } } -async function fetchSearxng(query: string, limit: number, opts: SearchOptions): Promise { +async function fetchSearxng(query: string, limit: number, opts: SearchOptions): Promise { const base = process.env.SEARXNG_URL; if (!base || !base.trim()) { throw new Error( @@ -347,14 +407,14 @@ async function fetchSearxng(query: string, limit: number, opts: SearchOptions): }, }); - return mapSearxngResults(response.data, limit); + return { results: mapSearxngResults(response.data, limit) }; } catch (error) { logger.warn('SearXNG search request failed', { endpoint, query, error: (error as Error).message, }); - return []; + return { results: [], reason: `SearXNG request failed: ${(error as Error).message}` }; } }