diff --git a/server/ai/aiService.ts b/server/ai/aiService.ts index a8086f0..32e0162 100644 --- a/server/ai/aiService.ts +++ b/server/ai/aiService.ts @@ -478,6 +478,7 @@ export async function rerankCandidates( query: string, candidates: CompressedContact[], plan?: QueryPlan | null, + signal?: AbortSignal, ): Promise { if (isMockMode()) { log.warn( @@ -575,6 +576,7 @@ Return a JSON array of VERIFIED matches with field-level evidence. If no candida prompt, responseFormat: "json", routing: { prefer: "lite" }, + signal, jsonSchema: { type: "array", items: { diff --git a/server/repositories/contactRepository.ts b/server/repositories/contactRepository.ts index 3023572..849db19 100644 --- a/server/repositories/contactRepository.ts +++ b/server/repositories/contactRepository.ts @@ -187,16 +187,274 @@ export const contactRepo = { /** * Hydrate multiple contact rows in bulk. - * Uses the same pre-compiled prepared statements — better-sqlite3 handles - * the internal caching. + * Leverages high-performance chunked SQL batch loading to bypass N+1 queries. * * @param contacts - Array of raw contact rows * @returns Array of fully hydrated contacts (nulls filtered out) */ hydrateMany(contacts: unknown[]): HydratedContact[] { - return contacts - .map((c) => contactRepo.hydrate(c)) - .filter(Boolean) as HydratedContact[]; + if (!contacts || contacts.length === 0) return []; + + // Filter out invalid records + const validContacts = contacts.filter( + (c): c is RawContactRow => + c !== null && typeof c === "object" && "id" in c, + ); + if (validContacts.length === 0) return []; + + // For very small inputs (e.g. <= 3 contacts), sequential hydration using + // pre-compiled statements has less overhead than bulk query preparation. + if (validContacts.length <= 3) { + return validContacts + .map((c) => contactRepo.hydrate(c)) + .filter(Boolean) as HydratedContact[]; + } + + const ids = validContacts.map((c) => c.id); + const CHUNK_SIZE = 500; + + // Helper to query in chunks of 500 to stay safely under SQLite parameter limits + const queryInChunks = ( + baseQuery: string, + idField: string, + orderBy: string = "", + ): T[] => { + const results: T[] = []; + for (let i = 0; i < ids.length; i += CHUNK_SIZE) { + const chunk = ids.slice(i, i + CHUNK_SIZE); + const placeholders = chunk.map(() => "?").join(","); + const sql = `${baseQuery} WHERE ${idField} IN (${placeholders}) ${orderBy}`; + results.push(...(sqlite.prepare(sql).all(chunk) as T[])); + } + return results; + }; + + // Load all relation child tables in chunked bulk queries + const emailRows = queryInChunks<{ + id: string; + contactId: string; + email: string; + label: string | null; + isPrimary: number; + sortOrder: number; + source: string; + }>( + "SELECT id, contactId, email, label, isPrimary, sortOrder, source FROM contact_emails", + "contactId", + "ORDER BY sortOrder ASC", + ); + + const phoneRows = queryInChunks<{ + id: string; + contactId: string; + phone: string; + label: string | null; + isPrimary: number; + sortOrder: number; + source: string; + }>( + "SELECT id, contactId, phone, label, isPrimary, sortOrder, source FROM contact_phones", + "contactId", + "ORDER BY sortOrder ASC", + ); + + const socialRows = queryInChunks<{ + id: string; + contactId: string; + platform: string; + url: string; + handle: string | null; + source: string; + }>( + "SELECT id, contactId, platform, url, handle, source FROM contact_social_links", + "contactId", + ); + + const eduRows = queryInChunks<{ + id: string; + contactId: string; + school: string; + degree: string | null; + fieldOfStudy: string | null; + startDate: string | null; + endDate: string | null; + description: string | null; + }>( + "SELECT id, contactId, school, degree, fieldOfStudy, startDate, endDate, description FROM contact_education", + "contactId", + ); + + const expRows = queryInChunks<{ + id: string; + contactId: string; + company: string; + role: string | null; + startDate: string | null; + endDate: string | null; + isCurrent: number; + description: string | null; + location: string | null; + }>( + "SELECT id, contactId, company, role, startDate, endDate, isCurrent, description, location FROM contact_experience", + "contactId", + ); + + const sourceRows = queryInChunks<{ + id: string; + contactId: string; + platform: string; + externalId: string | null; + connectedOn: string | null; + importedAt: string | null; + }>( + "SELECT id, contactId, platform, externalId, connectedOn, importedAt FROM contact_sources", + "contactId", + ); + + const tagRows = queryInChunks<{ + id: string; + contactId: string; + tag: string; + }>("SELECT id, contactId, tag FROM contact_tags", "contactId"); + + const interestRows = queryInChunks<{ + id: string; + contactId: string; + interest: string; + isAiGenerated: number; + }>( + "SELECT id, contactId, interest, isAiGenerated FROM contact_interests", + "contactId", + ); + + const attrRows = queryInChunks<{ + id: string; + contactId: string; + name: string; + value: string; + }>( + "SELECT id, contactId, name, value FROM contact_attributes", + "contactId", + ); + + const addrRows = queryInChunks<{ + id: string; + contactId: string; + address: string; + label: string | null; + isPrimary: number; + sortOrder: number; + source: string; + }>( + "SELECT id, contactId, address, label, isPrimary, sortOrder, source FROM contact_addresses", + "contactId", + "ORDER BY sortOrder ASC", + ); + + // Specialized list memberships chunked query + const listRows: { + contactId: string; + id: string; + name: string; + icon: string | null; + }[] = []; + for (let i = 0; i < ids.length; i += CHUNK_SIZE) { + const chunk = ids.slice(i, i + CHUNK_SIZE); + const placeholders = chunk.map(() => "?").join(","); + const sql = ` + SELECT lm.contactId, l.id, l.name, l.icon + FROM lists l + JOIN list_members lm ON l.id = lm.listId + WHERE lm.contactId IN (${placeholders}) + ORDER BY l.sortOrder ASC + `; + listRows.push(...(sqlite.prepare(sql).all(chunk) as any[])); + } + + // Specialized interaction count chunked query + const countRows: { contactId: string; cnt: number }[] = []; + for (let i = 0; i < ids.length; i += CHUNK_SIZE) { + const chunk = ids.slice(i, i + CHUNK_SIZE); + const placeholders = chunk.map(() => "?").join(","); + const sql = ` + SELECT contactId, COUNT(*) as cnt + FROM interactions + WHERE contactId IN (${placeholders}) + GROUP BY contactId + `; + countRows.push(...(sqlite.prepare(sql).all(chunk) as any[])); + } + + // Helper to group flat rows by contactId in O(N) + const groupByContact = ( + rows: T[], + ): Map => { + const map = new Map(); + for (const r of rows) { + if (!map.has(r.contactId)) map.set(r.contactId, []); + map.get(r.contactId)!.push(r); + } + return map; + }; + + const emailsMap = groupByContact(emailRows); + const phonesMap = groupByContact(phoneRows); + const socialMap = groupByContact(socialRows); + const eduMap = groupByContact(eduRows); + const expMap = groupByContact(expRows); + const sourceMap = groupByContact(sourceRows); + const tagsMap = groupByContact(tagRows); + const interestsMap = groupByContact(interestRows); + const attrsMap = groupByContact(attrRows); + const addrsMap = groupByContact(addrRows); + const listsMap = groupByContact(listRows); + const countsMap = new Map(countRows.map((r) => [r.contactId, r.cnt])); + + // Reconstruct fully hydrated contact structures in JS + return validContacts.map( + (row) => + ({ + ...row, + emails: (emailsMap.get(row.id) ?? []).map(({ contactId, ...e }) => ({ + ...e, + isPrimary: !!e.isPrimary, + })), + phones: (phonesMap.get(row.id) ?? []).map(({ contactId, ...p }) => ({ + ...p, + isPrimary: !!p.isPrimary, + })), + socialLinks: (socialMap.get(row.id) ?? []).map( + ({ contactId, ...s }) => s, + ), + education: (eduMap.get(row.id) ?? []).map( + ({ contactId, ...edu }) => edu, + ), + experience: (expMap.get(row.id) ?? []).map(({ contactId, ...e }) => ({ + ...e, + isCurrent: !!e.isCurrent, + })), + sources: (sourceMap.get(row.id) ?? []).map( + ({ contactId, ...src }) => src, + ), + tags: (tagsMap.get(row.id) ?? []).map(({ contactId, ...t }) => t), + interests: (interestsMap.get(row.id) ?? []).map( + ({ contactId, ...i }) => i, + ), + attributes: (attrsMap.get(row.id) ?? []).map( + ({ contactId, ...attr }) => attr, + ), + addresses: (addrsMap.get(row.id) ?? []).map( + ({ contactId, ...a }) => ({ + ...a, + isPrimary: !!a.isPrimary, + }), + ), + lists: (listsMap.get(row.id) ?? []).map( + ({ contactId, ...list }) => list, + ), + interactionCount: countsMap.get(row.id) ?? 0, + }) as unknown as HydratedContact, + ); }, // ------------------------------------------------------------------------- diff --git a/server/routes/search.ts b/server/routes/search.ts index 91176e6..570655e 100644 --- a/server/routes/search.ts +++ b/server/routes/search.ts @@ -62,7 +62,22 @@ router.post( res.setHeader("Cache-Control", "no-cache"); res.flushHeaders(); - await searchService.semanticSearchStream(query, rid, res); + // Create an AbortController bound to request closure + const controller = new AbortController(); + req.on("close", () => { + log.info( + "API", + `[${rid}] Client disconnected mid-search stream. Aborting AI operations.`, + ); + controller.abort(); + }); + + await searchService.semanticSearchStream( + query, + rid, + res, + controller.signal, + ); } else { // Single-response mode (backward compatible) const result = await searchService.semanticSearch(query, rid); diff --git a/server/services/dedupe/normalization.ts b/server/services/dedupe/normalization.ts index c8e1ac6..e309ee1 100644 --- a/server/services/dedupe/normalization.ts +++ b/server/services/dedupe/normalization.ts @@ -147,6 +147,8 @@ export function generateBlockKeys(contact: NormalizedContact): string[] { export function contactToEmbeddingString( contact: NormalizedContact, raw: RawContactRow, + tags?: string[], + interests?: string[], ): string { const parts: string[] = []; if (raw.name) parts.push(`Name: ${raw.name}`); @@ -165,16 +167,21 @@ export function contactToEmbeddingString( // Include tags and interests for rich semantic matching // (e.g., "who likes espresso?" should match contacts with espresso in interests) - const tagRows = sqlite - .prepare("SELECT tag FROM contact_tags WHERE contactId = ?") - .all(raw.id) as { tag: string }[]; - const interestRows = sqlite - .prepare("SELECT interest FROM contact_interests WHERE contactId = ?") - .all(raw.id) as { interest: string }[]; - const combined = [ - ...tagRows.map((t) => t.tag), - ...interestRows.map((i) => i.interest), - ]; + let combined: string[] = []; + if (tags && interests) { + combined = [...tags, ...interests]; + } else { + const tagRows = sqlite + .prepare("SELECT tag FROM contact_tags WHERE contactId = ?") + .all(raw.id) as { tag: string }[]; + const interestRows = sqlite + .prepare("SELECT interest FROM contact_interests WHERE contactId = ?") + .all(raw.id) as { interest: string }[]; + combined = [ + ...tagRows.map((t) => t.tag), + ...interestRows.map((i) => i.interest), + ]; + } if (combined.length > 0) parts.push(`Interests: ${combined.join(", ")}`); const content = parts.join(" | "); @@ -198,6 +205,8 @@ export function normalizeContact( emails: { email: string }[], phones: { phone: string }[], sourcePlatforms: string[], + tags?: string[], + interests?: string[], ): NormalizedContact { // Name processing const nameTokens = tokenizeName(raw.name); @@ -251,7 +260,12 @@ export function normalizeContact( // Derived fields (depend on the contact being fully built) contact.blockKeys = generateBlockKeys(contact); - contact.embeddingText = contactToEmbeddingString(contact, raw); + contact.embeddingText = contactToEmbeddingString( + contact, + raw, + tags, + interests, + ); return contact; } @@ -319,7 +333,30 @@ export function normalizeContacts( if (!platforms.includes(s.platform)) platforms.push(s.platform); } - // 5. Normalize each contact with pre-loaded child data + // 5. Batch-load all tags → Map + const allTags = sqlite + .prepare("SELECT contactId, tag FROM contact_tags") + .all() as { contactId: string; tag: string }[]; + + const tagsByContact = new Map(); + for (const t of allTags) { + if (!tagsByContact.has(t.contactId)) tagsByContact.set(t.contactId, []); + tagsByContact.get(t.contactId)!.push(t.tag); + } + + // 6. Batch-load all interests → Map + const allInterests = sqlite + .prepare("SELECT contactId, interest FROM contact_interests") + .all() as { contactId: string; interest: string }[]; + + const interestsByContact = new Map(); + for (const i of allInterests) { + if (!interestsByContact.has(i.contactId)) + interestsByContact.set(i.contactId, []); + interestsByContact.get(i.contactId)!.push(i.interest); + } + + // 7. Normalize each contact with pre-loaded child data const normalized: NormalizedContact[] = []; for (const raw of allContacts) { if (!raw.name) continue; // skip nameless contacts (shouldn't happen, but safety) @@ -330,6 +367,8 @@ export function normalizeContacts( emailsByContact.get(raw.id) ?? [], phonesByContact.get(raw.id) ?? [], sourcesByContact.get(raw.id) ?? [], + tagsByContact.get(raw.id) ?? [], + interestsByContact.get(raw.id) ?? [], ), ); } @@ -370,10 +409,20 @@ export function normalizeContactById( ) .all(contactId) as { platform: string }[]; + const tags = sqlite + .prepare("SELECT tag FROM contact_tags WHERE contactId = ?") + .all(contactId) as { tag: string }[]; + + const interests = sqlite + .prepare("SELECT interest FROM contact_interests WHERE contactId = ?") + .all(contactId) as { interest: string }[]; + return normalizeContact( raw, emails, phones, sources.map((s) => s.platform), + tags.map((t) => t.tag), + interests.map((i) => i.interest), ); } diff --git a/server/services/searchService.ts b/server/services/searchService.ts index c268cc5..a7e30e2 100644 --- a/server/services/searchService.ts +++ b/server/services/searchService.ts @@ -206,7 +206,12 @@ export const searchService = { * * Uses NDJSON (newline-delimited JSON) for streaming. */ - async semanticSearchStream(query: string, rid: string, res: Response) { + async semanticSearchStream( + query: string, + rid: string, + res: Response, + signal?: AbortSignal, + ) { const startTime = Date.now(); // ── 1. Cache check ───────────────────────────────────────────────── @@ -341,6 +346,7 @@ export const searchService = { query.trim(), compressedCandidates, retrieval.plan, + signal, ); if (aiMatches.length > 0) { diff --git a/tests/integration/dedupe.test.ts b/tests/integration/dedupe.test.ts index 7ef970f..3bff33a 100644 --- a/tests/integration/dedupe.test.ts +++ b/tests/integration/dedupe.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, vi } from "vitest"; +import { describe, it, expect } from "vitest"; import { dedupeService } from "../../server/services/dedupe/index.ts"; describe("Dedupe Pipeline Integration", () => { diff --git a/tests/integration/geocoding.test.ts b/tests/integration/geocoding.test.ts index fd9d22d..f147f38 100644 --- a/tests/integration/geocoding.test.ts +++ b/tests/integration/geocoding.test.ts @@ -1,10 +1,6 @@ import { describe, it, expect, vi } from "vitest"; import { queueGeocode } from "../../server/services/geocoding/index.ts"; -import { - normalizeLocationKey, - getCachedGeocode, - cacheGeocode, -} from "../../server/services/geocoding/cache.ts"; +import { normalizeLocationKey } from "../../server/services/geocoding/cache.ts"; describe("Geocoding Integration Tests", () => { it("normalizes location keys accurately", () => { diff --git a/tests/unit/multi-provider.test.ts b/tests/unit/multi-provider.test.ts index c4e0185..c84067b 100644 --- a/tests/unit/multi-provider.test.ts +++ b/tests/unit/multi-provider.test.ts @@ -14,7 +14,7 @@ // The Builder (Step 3) must make every one of these tests pass. // ============================================================================= -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; // ============================================================================= // 1. AIProviderName Type Contract @@ -41,6 +41,7 @@ describe("AIProviderName Type", () => { // ============================================================================= describe("OpenAIAdapter", () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any let OpenAIAdapter: any; beforeEach(async () => { @@ -190,6 +191,7 @@ describe("OpenAIAdapter", () => { // ============================================================================= describe("AnthropicAdapter", () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any let AnthropicAdapter: any; beforeEach(async () => { @@ -484,7 +486,6 @@ describe("NPM Dependency Contracts", () => { describe("SDK Import Containment Invariant", () => { it("openai is only imported in the adapter file", async () => { - const fs = await import("fs"); const path = await import("path"); const { execSync } = await import("child_process"); @@ -508,7 +509,6 @@ describe("SDK Import Containment Invariant", () => { }); it("@anthropic-ai/sdk is only imported in the adapter file", async () => { - const fs = await import("fs"); const path = await import("path"); const { execSync } = await import("child_process"); diff --git a/tests/unit/routing.test.ts b/tests/unit/routing.test.ts index bea4991..f7a6df8 100644 --- a/tests/unit/routing.test.ts +++ b/tests/unit/routing.test.ts @@ -15,7 +15,6 @@ import { getAvailableModels, getGroundingRPDLimit, GEMINI_REGISTRY, - type ModelConfig, type TierLimits, } from "../../server/ai/routing/registry.ts"; import { QuotaTracker } from "../../server/ai/routing/QuotaTracker.ts";