From e1aa57298b0a68642c678971ac510917f33d6326 Mon Sep 17 00:00:00 2001 From: Arvind Arikatla Date: Wed, 20 May 2026 20:56:52 -0700 Subject: [PATCH] perf(server): Implement medium-term stability and long-term maintainability backend optimizations --- server/repositories/contactRepository.ts | 17 +++++++++++ server/routes/contacts.ts | 32 +++++++++++-------- server/services/contactService.ts | 27 ++++++---------- server/services/dedupe/passes.ts | 6 ++++ tests/unit/relationRegistry.test.ts | 39 ++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 30 deletions(-) create mode 100644 tests/unit/relationRegistry.test.ts diff --git a/server/repositories/contactRepository.ts b/server/repositories/contactRepository.ts index 849db19..23f7855 100644 --- a/server/repositories/contactRepository.ts +++ b/server/repositories/contactRepository.ts @@ -21,6 +21,23 @@ import type { HydratedContact, ChildRecordsPayload } from "./types.ts"; // Re-export types for consumers export type { HydratedContact, ChildRecordsPayload }; +// Central registry of contact child relations for OCP extensibility +export const RELATION_REGISTRY = { + emails: { table: schema.contactEmails, dbName: "contact_emails" }, + phones: { table: schema.contactPhones, dbName: "contact_phones" }, + socialLinks: { + table: schema.contactSocialLinks, + dbName: "contact_social_links", + }, + tags: { table: schema.contactTags, dbName: "contact_tags" }, + interests: { table: schema.contactInterests, dbName: "contact_interests" }, + addresses: { table: schema.contactAddresses, dbName: "contact_addresses" }, + attributes: { table: schema.contactAttributes, dbName: "contact_attributes" }, + education: { table: schema.contactEducation, dbName: "contact_education" }, + experience: { table: schema.contactExperience, dbName: "contact_experience" }, + sources: { table: schema.contactSources, dbName: "contact_sources" }, +} as const; + // ============================================================================= // URL Utilities (used by social link insertion) // ============================================================================= diff --git a/server/routes/contacts.ts b/server/routes/contacts.ts index 65496db..738749f 100644 --- a/server/routes/contacts.ts +++ b/server/routes/contacts.ts @@ -28,6 +28,7 @@ import { isEmbeddingAvailable, } from "../services/dedupe/embeddings.ts"; import { dedupeService } from "../services/dedupe/index.ts"; +import { ParallelQueue } from "../ai/routing/ParallelQueue.ts"; import { normalizeContactById, normalizeContacts, @@ -476,20 +477,27 @@ router.post( `Background bulk embedding failed: ${getErrorMessage(err)}`, ), ); - // Schedule incremental dedupe for each imported contact - for (const cid of createdIds) { - setTimeout(() => { + // Process incremental dedupe sequentially in the background to prevent lock saturation and CPU spikes + (async () => { + // Wait 3 seconds to let bulk inserts and embedding tasks settle + await new Promise((resolve) => setTimeout(resolve, 3000)); + await ParallelQueue.process(createdIds, 1, async (cid) => { const irid = `imp-${cid.slice(0, 8)}`; - dedupeService - .incrementalDedupeCheck(cid, irid) - .catch((err) => - log.warn( - "API", - `Incremental dedupe for ${cid} failed: ${getErrorMessage(err)}`, - ), + try { + await dedupeService.incrementalDedupeCheck(cid, irid); + } catch (err) { + log.warn( + "API", + `Incremental dedupe for ${cid} failed: ${getErrorMessage(err)}`, ); - }, 3_000); - } + } + }); + })().catch((err) => + log.error( + "API", + `Bulk background dedupe queue crashed: ${getErrorMessage(err)}`, + ), + ); } res.status(201).json({ success: true, count }); } diff --git a/server/services/contactService.ts b/server/services/contactService.ts index 75cf4d3..678cebb 100644 --- a/server/services/contactService.ts +++ b/server/services/contactService.ts @@ -4,7 +4,10 @@ import path from "path"; import { db, sqlite } from "../db.ts"; import * as schema from "../../src/db/schema.ts"; import { eq } from "drizzle-orm"; -import { contactRepo } from "../repositories/contactRepository.ts"; +import { + contactRepo, + RELATION_REGISTRY, +} from "../repositories/contactRepository.ts"; import { queueGeocode } from "./geocoding/index.ts"; import { processBase64Avatar, @@ -318,25 +321,13 @@ export const contactService = { .where(eq(schema.contacts.id, id)) .run(); - const childMappings: [keyof typeof body, string][] = [ - ["emails", "contact_emails"], - ["phones", "contact_phones"], - ["socialLinks", "contact_social_links"], - ["tags", "contact_tags"], - ["interests", "contact_interests"], - ["addresses", "contact_addresses"], - ["attributes", "contact_attributes"], - ["education", "contact_education"], - ["experience", "contact_experience"], - ["sources", "contact_sources"], - ]; - - for (const [bodyKey, tableName] of childMappings) { - if (body[bodyKey] !== undefined && Array.isArray(body[bodyKey])) { + for (const [bodyKey, config] of Object.entries(RELATION_REGISTRY)) { + const key = bodyKey as keyof typeof RELATION_REGISTRY; + if (body[key] !== undefined && Array.isArray(body[key])) { sqlite - .prepare(`DELETE FROM ${tableName} WHERE contactId = ?`) + .prepare(`DELETE FROM ${config.dbName} WHERE contactId = ?`) .run(id); - contactRepo.insertChildRecords(id, { [bodyKey]: body[bodyKey] }); + contactRepo.insertChildRecords(id, { [key]: body[key] } as any); } } }); diff --git a/server/services/dedupe/passes.ts b/server/services/dedupe/passes.ts index 1baee27..f6b1cf1 100644 --- a/server/services/dedupe/passes.ts +++ b/server/services/dedupe/passes.ts @@ -354,8 +354,14 @@ export async function runFunnelPass( }[] = []; let autoCount = 0; let discardCount = 0; + let scoredCount = 0; for (const candidate of allCandidates) { + if (scoredCount > 0 && scoredCount % 100 === 0) { + await new Promise((resolve) => setImmediate(resolve)); + } + scoredCount++; + const nA = normalizedMap.get(candidate.idA); const nB = normalizedMap.get(candidate.idB); if (!nA || !nB) continue; diff --git a/tests/unit/relationRegistry.test.ts b/tests/unit/relationRegistry.test.ts new file mode 100644 index 0000000..ad9b1d7 --- /dev/null +++ b/tests/unit/relationRegistry.test.ts @@ -0,0 +1,39 @@ +import { describe, it, expect } from "vitest"; +import { RELATION_REGISTRY } from "../../server/repositories/contactRepository.ts"; + +describe("Central Child Relation Registry (OCP)", () => { + it("defines mapping configuration for all expected child tables", () => { + const keys = Object.keys(RELATION_REGISTRY) as Array< + keyof typeof RELATION_REGISTRY + >; + + // Core child properties we expect to be registered + const expectedKeys = [ + "emails", + "phones", + "socialLinks", + "tags", + "interests", + "addresses", + "attributes", + "education", + "experience", + "sources", + ]; + + for (const key of expectedKeys) { + expect(keys).toContain(key); + const config = RELATION_REGISTRY[key as keyof typeof RELATION_REGISTRY]; + expect(config).toBeDefined(); + expect(config.dbName).toBeTypeOf("string"); + expect(config.dbName.length).toBeGreaterThan(0); + expect(config.table).toBeDefined(); + } + }); + + it("does not have duplicate database names", () => { + const dbNames = Object.values(RELATION_REGISTRY).map((cfg) => cfg.dbName); + const uniqueDbNames = new Set(dbNames); + expect(dbNames.length).toBe(uniqueDbNames.size); + }); +});