diff --git a/eslint-suppressions.json b/eslint-suppressions.json index fbc49ba6..97bf6b31 100644 --- a/eslint-suppressions.json +++ b/eslint-suppressions.json @@ -413,6 +413,11 @@ "count": 1 } }, + "src/domains/translation-resources/translation-resources.manifest-completeness.test.ts": { + "no-restricted-syntax": { + "count": 1 + } + }, "src/domains/translation-resources/translation-resources.range.test.ts": { "no-restricted-syntax": { "count": 4 @@ -428,7 +433,7 @@ }, "src/domains/translation-resources/translation-resources.test.ts": { "no-restricted-syntax": { - "count": 5 + "count": 4 }, "ts/no-explicit-any": { "count": 2 diff --git a/src/domains/translation-resources/translation-resources.license.test.ts b/src/domains/translation-resources/translation-resources.license.test.ts index 4ad8dd6a..4e153dd8 100644 --- a/src/domains/translation-resources/translation-resources.license.test.ts +++ b/src/domains/translation-resources/translation-resources.license.test.ts @@ -10,7 +10,7 @@ import { getTranslationNotes, getTranslationQuestions, } from './translation-resources.service'; -import { imageSearchHit } from './translation-resources.test-fixtures'; +import { imageSearchHit, searchEnumeration } from './translation-resources.test-fixtures'; import { prepareOfflineManifestResponseSchema, translationImagesResponseSchema, @@ -60,7 +60,7 @@ const detail = { describe('resource license attribution', () => { beforeEach(() => { vi.clearAllMocks(); - vi.mocked(searchAllResources).mockResolvedValue(ok([hit])); + vi.mocked(searchAllResources).mockResolvedValue(searchEnumeration([hit])); vi.mocked(getResource).mockResolvedValue(ok(aquiferResourceDetailsSchema.parse(detail))); }); @@ -93,7 +93,7 @@ describe('resource license attribution', () => { collectionCode: 'Images', }, }); - vi.mocked(searchAllResources).mockResolvedValue(ok([imageHit])); + vi.mocked(searchAllResources).mockResolvedValue(searchEnumeration([imageHit])); vi.mocked(getResource).mockResolvedValue(ok(imageDetail)); const result = await getTranslationImages({ @@ -129,7 +129,7 @@ describe('resource license attribution', () => { collectionCode: 'Images', }, }); - vi.mocked(searchAllResources).mockResolvedValue(ok([imageHit])); + vi.mocked(searchAllResources).mockResolvedValue(searchEnumeration([imageHit])); vi.mocked(getResource).mockResolvedValue(ok(imageDetail)); const result = await getTranslationImages({ @@ -163,7 +163,7 @@ describe('resource license attribution', () => { collectionCode: 'Images', }, }); - vi.mocked(searchAllResources).mockResolvedValue(ok([imageHit])); + vi.mocked(searchAllResources).mockResolvedValue(searchEnumeration([imageHit])); vi.mocked(getResource).mockResolvedValue(ok(imageDetail)); const result = await getTranslationImages({ diff --git a/src/domains/translation-resources/translation-resources.manifest-completeness.test.ts b/src/domains/translation-resources/translation-resources.manifest-completeness.test.ts new file mode 100644 index 00000000..70b6fe7e --- /dev/null +++ b/src/domains/translation-resources/translation-resources.manifest-completeness.test.ts @@ -0,0 +1,258 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { logger } from '@/lib/logger'; +import * as aquiferClient from '@/lib/services/aquifer/aquifer.client'; +import { err, ErrorCode, ok } from '@/lib/types'; +import { server } from '@/server/server'; +import '@/domains/translation-resources/translation-resources.route'; + +import { + asAuthenticatedUser, + asProjectMember, + MANIFEST_PATH, + searchEnumeration, + searchHit, + textDetails, +} from './translation-resources.test-fixtures'; + +vi.mock('@/lib/auth', () => ({ + auth: { + api: { getSession: vi.fn() }, + handler: vi.fn(), + }, +})); + +vi.mock('@/db', () => { + const mockQueryBuilder = { + from: vi.fn().mockReturnThis(), + where: vi.fn().mockReturnThis(), + limit: vi.fn().mockResolvedValue([{ activeOrgId: 1 }]), + }; + return { + db: { select: vi.fn(() => mockQueryBuilder), insert: vi.fn(), update: vi.fn() }, + }; +}); + +vi.mock('@/lib/logger', () => ({ + logger: { info: vi.fn(), error: vi.fn(), debug: vi.fn(), warn: vi.fn() }, +})); + +vi.mock('@/domains/users/users.service', () => ({ + getUserByEmail: vi.fn(), +})); + +vi.mock('@/domains/user-roles/user-roles.repository', () => ({ + findGrantsByUserId: vi.fn(), +})); + +vi.mock('@/domains/projects/projects.service', () => ({ + getProjectById: vi.fn(), +})); + +vi.mock('@/domains/projects/users/project-users.service', () => ({ + resolveIsProjectMember: vi.fn(), +})); + +vi.mock('@/lib/services/aquifer/aquifer.client', () => ({ + searchResources: vi.fn(), + searchAllResources: vi.fn(), + getResource: vi.fn(), +})); + +describe('translation-resources manifest completeness', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('keeps every required resource when one chapter has more than 100 notes (#431)', async () => { + asAuthenticatedUser(); + asProjectMember(); + const hits = Array.from({ length: 101 }, (_, index) => ({ + ...searchHit, + id: index + 1, + })); + vi.mocked(aquiferClient.searchAllResources) + .mockResolvedValueOnce(searchEnumeration(hits)) + .mockResolvedValue(searchEnumeration([])); + vi.mocked(aquiferClient.getResource).mockImplementation(async (id) => + ok({ ...textDetails, id }) + ); + + const res = await server.request(MANIFEST_PATH, { method: 'GET' }); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.truncated).toBe(false); + expect(body.items).toHaveLength(101); + expect(body.items.map((item: { aquiferContentId: number }) => item.aquiferContentId)).toEqual( + hits.map((hit) => hit.id) + ); + expect(body.totalBytes).toBe( + body.items.reduce((sum: number, item: { bytesTotal: number }) => sum + item.bytesTotal, 0) + ); + expect(body.totalBytes).toBeGreaterThan(0); + expect(aquiferClient.getResource).toHaveBeenCalledTimes(101); + }); + + it('sets truncated and warns when Aquifer search enumeration is incomplete (#431)', async () => { + asAuthenticatedUser(); + asProjectMember(); + const hits = Array.from({ length: 50 }, (_, index) => ({ + ...searchHit, + id: index + 1, + })); + vi.mocked(aquiferClient.searchAllResources) + .mockResolvedValueOnce(searchEnumeration(hits, false)) + .mockResolvedValue(searchEnumeration([])); + vi.mocked(aquiferClient.getResource).mockImplementation(async (id) => + ok({ ...textDetails, id }) + ); + + const res = await server.request(MANIFEST_PATH, { method: 'GET' }); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.truncated).toBe(true); + expect(body.items).toHaveLength(50); + expect(logger.warn).toHaveBeenCalledWith({ + message: 'Aquifer prepare-offline search hit page ceiling; marking manifest truncated', + context: expect.objectContaining({ + resourceCollectionCode: 'UWTranslationNotes', + bookCode: 'MRK', + startChapter: 14, + endChapter: 14, + collectedCount: 50, + }), + }); + }); + + it('returns 502 when one manifest resource fails to hydrate (#432)', async () => { + asAuthenticatedUser(); + asProjectMember(); + const hits = Array.from({ length: 100 }, (_, index) => ({ + ...searchHit, + id: index + 1, + })); + vi.mocked(aquiferClient.searchAllResources) + .mockResolvedValueOnce(searchEnumeration(hits)) + .mockResolvedValue(searchEnumeration([])); + vi.mocked(aquiferClient.getResource).mockImplementation(async (id) => + id === 100 ? err(ErrorCode.AQUIFER_SERVICE_UNAVAILABLE) : ok({ ...textDetails, id }) + ); + + const res = await server.request(MANIFEST_PATH, { method: 'GET' }); + + expect(res.status).toBe(502); + expect(await res.json()).toEqual({ message: 'Aquifer service is unavailable' }); + expect(aquiferClient.getResource).toHaveBeenCalledTimes(100); + }); + it('returns empty manifest when Aquifer has no resources', async () => { + asAuthenticatedUser(); + asProjectMember(); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration([])); + + const res = await server.request(MANIFEST_PATH, { method: 'GET' }); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + projectId: 10, + sourceLanguageCode: 'eng', + items: [], + totalBytes: 0, + truncated: false, + }); + // TN, TW, TQ, StudyNotes, Images + expect(aquiferClient.searchAllResources).toHaveBeenCalledTimes(5); + }); + + it('returns hydrated manifest items without serializedContent by default', async () => { + asAuthenticatedUser(); + asProjectMember(); + vi.mocked(aquiferClient.searchAllResources) + .mockResolvedValueOnce(searchEnumeration([searchHit])) + .mockResolvedValue(searchEnumeration([])); + vi.mocked(aquiferClient.getResource).mockResolvedValue(ok(textDetails)); + + const res = await server.request(MANIFEST_PATH, { method: 'GET' }); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.projectId).toBe(10); + expect(body.truncated).toBe(false); + expect(body.items).toHaveLength(1); + expect(body.items[0]).toMatchObject({ + id: 'aquifer-101-text', + tier: 2, + kind: 'text', + resourceName: 'Translation Notes', + aquiferContentId: 101, + bookCode: 'MRK', + startChapter: 14, + endChapter: 14, + collectionCode: 'UWTranslationNotes', + fileExt: 'json', + }); + expect(body.items[0].bytesTotal).toBeGreaterThan(0); + expect(body.totalBytes).toBe(body.items[0].bytesTotal); + expect(body.items[0].serializedContent).toBeUndefined(); + }); + + it('includes serializedContent when includeContent=true', async () => { + asAuthenticatedUser(); + asProjectMember(); + vi.mocked(aquiferClient.searchAllResources) + .mockResolvedValueOnce(searchEnumeration([searchHit])) + .mockResolvedValue(searchEnumeration([])); + vi.mocked(aquiferClient.getResource).mockResolvedValue(ok(textDetails)); + + const res = await server.request(`${MANIFEST_PATH}&includeContent=true`, { method: 'GET' }); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(typeof body.items[0].serializedContent).toBe('string'); + const parsedContent = JSON.parse(body.items[0].serializedContent); + expect(parsedContent).toEqual(textDetails.content); + expect(body.items[0].bytesTotal).toBe( + new TextEncoder().encode(body.items[0].serializedContent).byteLength + ); + }); + + it('returns 502 when Aquifer fails mid-manifest', async () => { + asAuthenticatedUser(); + asProjectMember(); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue( + err(ErrorCode.AQUIFER_SERVICE_UNAVAILABLE) + ); + + const res = await server.request(MANIFEST_PATH, { method: 'GET' }); + + expect(res.status).toBe(502); + expect(await res.json()).toEqual({ message: 'Aquifer service is unavailable' }); + }); + + it('returns 400 when endChapter is before startChapter', async () => { + asAuthenticatedUser(); + asProjectMember(); + + const res = await server.request( + '/projects/10/translation-resources/manifest?languageCode=eng&bookCode=MRK&startChapter=150&endChapter=1', + { method: 'GET' } + ); + + expect(res.status).toBe(400); + expect(aquiferClient.searchAllResources).not.toHaveBeenCalled(); + }); + + it('returns 400 when the chapter span exceeds the cap', async () => { + asAuthenticatedUser(); + asProjectMember(); + + const res = await server.request( + '/projects/10/translation-resources/manifest?languageCode=eng&bookCode=PSA&startChapter=1&endChapter=150', + { method: 'GET' } + ); + + expect(res.status).toBe(400); + expect(aquiferClient.searchAllResources).not.toHaveBeenCalled(); + }); +}); diff --git a/src/domains/translation-resources/translation-resources.range.test.ts b/src/domains/translation-resources/translation-resources.range.test.ts index 5d62a3cf..3c0b4fcb 100644 --- a/src/domains/translation-resources/translation-resources.range.test.ts +++ b/src/domains/translation-resources/translation-resources.range.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { auth } from '@/lib/auth'; +import { logger } from '@/lib/logger'; import * as aquiferClient from '@/lib/services/aquifer/aquifer.client'; import { err, ErrorCode, ok } from '@/lib/types'; import { server } from '@/server/server'; @@ -13,6 +14,7 @@ import { imageSearchHit, NOTES_RANGE_PATH, QUESTIONS_RANGE_PATH, + searchEnumeration, searchHit, textDetails, } from './translation-resources.test-fixtures'; @@ -70,7 +72,7 @@ describe('translation-resources verse-range routes', () => { it('returns empty items when Aquifer has no notes in the range', async () => { asAuthenticatedUser(); asProjectMember(); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok([])); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration([])); const res = await server.request(NOTES_RANGE_PATH, { method: 'GET' }); @@ -98,7 +100,7 @@ describe('translation-resources verse-range routes', () => { { ...searchHit, id: 101, name: 'one-dup', localizedName: 'One Dup' }, { ...searchHit, id: 202, name: 'two', localizedName: 'Two' }, ]; - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok(hits)); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration(hits)); vi.mocked(aquiferClient.getResource).mockImplementation(async (id: number) => ok({ ...textDetails, id, name: String(id), localizedName: String(id) }) ); @@ -112,6 +114,30 @@ describe('translation-resources verse-range routes', () => { expect(aquiferClient.searchAllResources).toHaveBeenCalledTimes(1); }); + it('warns when Aquifer range search is incomplete and still returns collected items', async () => { + asAuthenticatedUser(); + asProjectMember(); + const hits = [{ ...searchHit, id: 101 }]; + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration(hits, false)); + vi.mocked(aquiferClient.getResource).mockResolvedValue(ok(textDetails)); + + const res = await server.request(NOTES_RANGE_PATH, { method: 'GET' }); + + expect(res.status).toBe(200); + expect((await res.json()).items).toHaveLength(1); + expect(logger.warn).toHaveBeenCalledWith({ + message: 'Aquifer verse-range search hit page ceiling; returning partial items', + context: expect.objectContaining({ + resourceCollectionCode: 'UWTranslationNotes', + bookCode: 'MRK', + chapter: 1, + startVerse: 1, + endVerse: 8, + collectedCount: 1, + }), + }); + }); + it('returns 502 with a generic message when Aquifer search fails', async () => { asAuthenticatedUser(); asProjectMember(); @@ -165,7 +191,7 @@ describe('translation-resources verse-range routes', () => { it('returns empty items when none exist', async () => { asAuthenticatedUser(); asProjectMember(); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok([])); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration([])); const res = await server.request(QUESTIONS_RANGE_PATH, { method: 'GET' }); @@ -183,7 +209,7 @@ describe('translation-resources verse-range routes', () => { it('returns 502 when details hydration fails for every hit', async () => { asAuthenticatedUser(); asProjectMember(); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok([searchHit])); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration([searchHit])); vi.mocked(aquiferClient.getResource).mockResolvedValue( err(ErrorCode.AQUIFER_SERVICE_UNAVAILABLE) ); @@ -200,7 +226,7 @@ describe('translation-resources verse-range routes', () => { asAuthenticatedUser(); asProjectMember(); const hits = [imageSearchHit(55), imageSearchHit(55), imageSearchHit(66)]; - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok(hits)); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration(hits)); vi.mocked(aquiferClient.getResource).mockImplementation(async (id: number) => ok({ id, @@ -246,12 +272,44 @@ describe('translation-resources verse-range routes', () => { it('returns empty items when Aquifer has no images in the range', async () => { asAuthenticatedUser(); asProjectMember(); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok([])); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration([])); const res = await server.request(IMAGES_RANGE_PATH, { method: 'GET' }); expect(res.status).toBe(200); expect(await res.json()).toEqual({ items: [] }); }); + + it('warns when Aquifer images range search is incomplete and still returns collected items', async () => { + asAuthenticatedUser(); + asProjectMember(); + const hits = [imageSearchHit(55)]; + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration(hits, false)); + vi.mocked(aquiferClient.getResource).mockResolvedValue( + ok({ + id: 55, + name: 'map', + localizedName: 'Map 55', + content: { url: 'https://cdn.example/55.jpg', size: 55 }, + grouping: { type: 'Images', name: 'Images', mediaType: 'Image' }, + }) + ); + + const res = await server.request(IMAGES_RANGE_PATH, { method: 'GET' }); + + expect(res.status).toBe(200); + expect((await res.json()).items).toHaveLength(1); + expect(logger.warn).toHaveBeenCalledWith({ + message: 'Aquifer images verse-range search hit page ceiling; returning partial items', + context: expect.objectContaining({ + resourceType: 'Images', + bookCode: 'MRK', + chapter: 1, + startVerse: 1, + endVerse: 8, + collectedCount: 1, + }), + }); + }); }); }); diff --git a/src/domains/translation-resources/translation-resources.service.ts b/src/domains/translation-resources/translation-resources.service.ts index b5b5e277..378edbc7 100644 --- a/src/domains/translation-resources/translation-resources.service.ts +++ b/src/domains/translation-resources/translation-resources.service.ts @@ -6,6 +6,7 @@ import type { } from '@/lib/services/aquifer/aquifer.types'; import type { Result } from '@/lib/types'; +import { logger } from '@/lib/logger'; import { getResource, searchAllResources } from '@/lib/services/aquifer/aquifer.client'; import { ok } from '@/lib/types'; @@ -23,9 +24,12 @@ export const AQUIFER_RESOURCE_COLLECTIONS = { translationQuestions: 'UWTranslationQuestions', } as const; -const HYDRATE_CONCURRENCY = 5; // parallel getResource hydrate -const MAX_MANIFEST_ITEMS_PER_CONFIG = 100; // per MANIFEST_SEARCH_CONFIGS entry -/** Aquifer chapter end-verse window (mobile Prepare Offline 1–200). */ +/** Parallel getResource calls when hydrating search hits. */ +const HYDRATE_CONCURRENCY = 5; +/** + * Aquifer end-verse window for chapter-range searches. Chapters never exceed + * this verse count; mirrors fluent-mobile Prepare Offline (1–200). + */ const CHAPTER_RANGE_END_VERSE = 200; interface ResourceSearchConfig { @@ -280,7 +284,21 @@ async function searchVerseRangeCollection( resourceCollectionCode, }); if (!search.ok) return search; - return ok(dedupeSearchHitsById(search.data)); + if (!search.data.complete) { + logger.warn({ + message: 'Aquifer verse-range search hit page ceiling; returning partial items', + context: { + resourceCollectionCode, + bookCode: ref.bookCode, + chapter: ref.chapter, + startVerse: ref.startVerse, + endVerse: ref.endVerse, + languageCode: ref.languageCode, + collectedCount: search.data.items.length, + }, + }); + } + return ok(dedupeSearchHitsById(search.data.items)); } /** @@ -335,8 +353,22 @@ export async function getTranslationImagesForRange( resourceType: 'Images', }); if (!search.ok) return search; + if (!search.data.complete) { + logger.warn({ + message: 'Aquifer images verse-range search hit page ceiling; returning partial items', + context: { + resourceType: 'Images', + bookCode: ref.bookCode, + chapter: ref.chapter, + startVerse: ref.startVerse, + endVerse: ref.endVerse, + languageCode: ref.languageCode, + collectedCount: search.data.items.length, + }, + }); + } - const uniqueHits = dedupeSearchHitsById(search.data); + const uniqueHits = dedupeSearchHitsById(search.data.items); const hydrated = await mapWithConcurrency(uniqueHits, HYDRATE_CONCURRENCY, async (hit) => { const details = await getResource(hit.id); if (!details.ok) return details; @@ -464,13 +496,28 @@ export async function getPrepareOfflineManifest(params: { }); if (!search.ok) return search; - if (search.data.length > MAX_MANIFEST_ITEMS_PER_CONFIG) { + // Bounded Aquifer enumeration already collected every page it could. Only mark + // truncated when provider metadata could not prove the set is complete (page + // ceiling). Do not re-slice a successful enumeration — mobile rejects truncated. + if (!search.data.complete) { truncated = true; + logger.warn({ + message: 'Aquifer prepare-offline search hit page ceiling; marking manifest truncated', + context: { + resourceCollectionCode: config.collectionCode, + resourceType: config.resourceType, + resourceName: config.resourceName, + bookCode, + startChapter, + endChapter, + languageCode, + collectedCount: search.data.items.length, + }, + }); } - const capped = search.data.slice(0, MAX_MANIFEST_ITEMS_PER_CONFIG); const hydrated = await mapWithConcurrency( - capped, + search.data.items, HYDRATE_CONCURRENCY, async (searchItem) => { const details = await getResource(searchItem.id); diff --git a/src/domains/translation-resources/translation-resources.test-fixtures.ts b/src/domains/translation-resources/translation-resources.test-fixtures.ts index bd826c18..6055020c 100644 --- a/src/domains/translation-resources/translation-resources.test-fixtures.ts +++ b/src/domains/translation-resources/translation-resources.test-fixtures.ts @@ -100,3 +100,8 @@ export function imageSearchHit(id: number) { }, }; } + +/** Synthetic Aquifer search enumeration for route mocks (not a live payload). */ +export function searchEnumeration(items: T[], complete = true) { + return ok({ items, complete }); +} diff --git a/src/domains/translation-resources/translation-resources.test.ts b/src/domains/translation-resources/translation-resources.test.ts index cfa94597..42e2558f 100644 --- a/src/domains/translation-resources/translation-resources.test.ts +++ b/src/domains/translation-resources/translation-resources.test.ts @@ -13,10 +13,10 @@ import { asProjectMember, IMAGES_PATH, imageSearchHit, - MANIFEST_PATH, MOCK_PROJECT, NOTES_PATH, QUESTIONS_PATH, + searchEnumeration, searchHit, textDetails, } from './translation-resources.test-fixtures'; @@ -114,7 +114,7 @@ describe('translation-resources routes', () => { it('returns empty items when Aquifer has no notes', async () => { asAuthenticatedUser(); asProjectMember(); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok([])); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration([])); const res = await server.request(NOTES_PATH, { method: 'GET' }); @@ -125,7 +125,7 @@ describe('translation-resources routes', () => { it('returns hydrated notes on success', async () => { asAuthenticatedUser(); asProjectMember(); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok([searchHit])); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration([searchHit])); vi.mocked(aquiferClient.getResource).mockResolvedValue(ok(textDetails)); const res = await server.request(NOTES_PATH, { method: 'GET' }); @@ -164,7 +164,7 @@ describe('translation-resources routes', () => { { ...searchHit, id: 2, name: 'two', localizedName: 'Two' }, { ...searchHit, id: 3, name: 'three', localizedName: 'Three' }, ]; - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok(hits)); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration(hits)); vi.mocked(aquiferClient.getResource).mockImplementation(async (id: number) => { if (id === 2) return err(ErrorCode.AQUIFER_SERVICE_UNAVAILABLE); return ok({ ...textDetails, id, name: String(id), localizedName: String(id) }); @@ -208,7 +208,7 @@ describe('translation-resources routes', () => { it('returns empty items when none exist', async () => { asAuthenticatedUser(); asProjectMember(); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok([])); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration([])); const res = await server.request(QUESTIONS_PATH, { method: 'GET' }); @@ -219,7 +219,7 @@ describe('translation-resources routes', () => { it('returns 502 when details hydration fails', async () => { asAuthenticatedUser(); asProjectMember(); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok([searchHit])); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration([searchHit])); vi.mocked(aquiferClient.getResource).mockResolvedValue( err(ErrorCode.AQUIFER_SERVICE_UNAVAILABLE) ); @@ -236,7 +236,7 @@ describe('translation-resources routes', () => { asAuthenticatedUser(); asProjectMember(); const imageHit = imageSearchHit(55); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok([imageHit])); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration([imageHit])); vi.mocked(aquiferClient.getResource).mockResolvedValue( ok({ id: 55, @@ -267,7 +267,7 @@ describe('translation-resources routes', () => { asAuthenticatedUser(); asProjectMember(); const imageHit = imageSearchHit(58); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok([imageHit])); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration([imageHit])); vi.mocked(aquiferClient.getResource).mockResolvedValue( ok({ id: 58, @@ -298,7 +298,7 @@ describe('translation-resources routes', () => { asAuthenticatedUser(); asProjectMember(); const imageHit = imageSearchHit(56); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok([imageHit])); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration([imageHit])); vi.mocked(aquiferClient.getResource).mockResolvedValue( ok({ id: 56, @@ -329,7 +329,7 @@ describe('translation-resources routes', () => { asAuthenticatedUser(); asProjectMember(); const imageHit = imageSearchHit(59); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok([imageHit])); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration([imageHit])); vi.mocked(aquiferClient.getResource).mockResolvedValue( ok({ id: 59, @@ -366,7 +366,7 @@ describe('translation-resources routes', () => { asAuthenticatedUser(); asProjectMember(); const imageHit = imageSearchHit(57); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok([imageHit])); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration([imageHit])); vi.mocked(aquiferClient.getResource).mockResolvedValue( ok({ id: 57, @@ -401,7 +401,7 @@ describe('translation-resources routes', () => { it('returns empty items when Aquifer has no images', async () => { asAuthenticatedUser(); asProjectMember(); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok([])); + vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(searchEnumeration([])); const res = await server.request(IMAGES_PATH, { method: 'GET' }); @@ -409,137 +409,4 @@ describe('translation-resources routes', () => { expect(await res.json()).toEqual({ items: [] }); }); }); - - describe('gET manifest', () => { - it('returns 502 when one manifest resource fails to hydrate (#432)', async () => { - asAuthenticatedUser(); - asProjectMember(); - const hits = Array.from({ length: 100 }, (_, index) => ({ - ...searchHit, - id: index + 1, - })); - vi.mocked(aquiferClient.searchAllResources) - .mockResolvedValueOnce(ok(hits)) - .mockResolvedValue(ok([])); - vi.mocked(aquiferClient.getResource).mockImplementation(async (id) => - id === 100 ? err(ErrorCode.AQUIFER_SERVICE_UNAVAILABLE) : ok({ ...textDetails, id }) - ); - - const res = await server.request(MANIFEST_PATH, { method: 'GET' }); - - expect(res.status).toBe(502); - expect(await res.json()).toEqual({ message: 'Aquifer service is unavailable' }); - expect(aquiferClient.getResource).toHaveBeenCalledTimes(100); - }); - - it('returns empty manifest when Aquifer has no resources', async () => { - asAuthenticatedUser(); - asProjectMember(); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue(ok([])); - - const res = await server.request(MANIFEST_PATH, { method: 'GET' }); - - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ - projectId: 10, - sourceLanguageCode: 'eng', - items: [], - totalBytes: 0, - truncated: false, - }); - // TN, TW, TQ, StudyNotes, Images - expect(aquiferClient.searchAllResources).toHaveBeenCalledTimes(5); - }); - - it('returns hydrated manifest items without serializedContent by default', async () => { - asAuthenticatedUser(); - asProjectMember(); - vi.mocked(aquiferClient.searchAllResources) - .mockResolvedValueOnce(ok([searchHit])) - .mockResolvedValue(ok([])); - vi.mocked(aquiferClient.getResource).mockResolvedValue(ok(textDetails)); - - const res = await server.request(MANIFEST_PATH, { method: 'GET' }); - - expect(res.status).toBe(200); - const body = await res.json(); - expect(body.projectId).toBe(10); - expect(body.truncated).toBe(false); - expect(body.items).toHaveLength(1); - expect(body.items[0]).toMatchObject({ - id: 'aquifer-101-text', - tier: 2, - kind: 'text', - resourceName: 'Translation Notes', - aquiferContentId: 101, - bookCode: 'MRK', - startChapter: 14, - endChapter: 14, - collectionCode: 'UWTranslationNotes', - fileExt: 'json', - }); - expect(body.items[0].bytesTotal).toBeGreaterThan(0); - expect(body.totalBytes).toBe(body.items[0].bytesTotal); - expect(body.items[0].serializedContent).toBeUndefined(); - }); - - it('includes serializedContent when includeContent=true', async () => { - asAuthenticatedUser(); - asProjectMember(); - vi.mocked(aquiferClient.searchAllResources) - .mockResolvedValueOnce(ok([searchHit])) - .mockResolvedValue(ok([])); - vi.mocked(aquiferClient.getResource).mockResolvedValue(ok(textDetails)); - - const res = await server.request(`${MANIFEST_PATH}&includeContent=true`, { method: 'GET' }); - - expect(res.status).toBe(200); - const body = await res.json(); - expect(typeof body.items[0].serializedContent).toBe('string'); - const parsedContent = JSON.parse(body.items[0].serializedContent); - expect(parsedContent).toEqual(textDetails.content); - expect(body.items[0].bytesTotal).toBe( - new TextEncoder().encode(body.items[0].serializedContent).byteLength - ); - }); - - it('returns 502 when Aquifer fails mid-manifest', async () => { - asAuthenticatedUser(); - asProjectMember(); - vi.mocked(aquiferClient.searchAllResources).mockResolvedValue( - err(ErrorCode.AQUIFER_SERVICE_UNAVAILABLE) - ); - - const res = await server.request(MANIFEST_PATH, { method: 'GET' }); - - expect(res.status).toBe(502); - expect(await res.json()).toEqual({ message: 'Aquifer service is unavailable' }); - }); - - it('returns 400 when endChapter is before startChapter', async () => { - asAuthenticatedUser(); - asProjectMember(); - - const res = await server.request( - '/projects/10/translation-resources/manifest?languageCode=eng&bookCode=MRK&startChapter=150&endChapter=1', - { method: 'GET' } - ); - - expect(res.status).toBe(400); - expect(aquiferClient.searchAllResources).not.toHaveBeenCalled(); - }); - - it('returns 400 when the chapter span exceeds the cap', async () => { - asAuthenticatedUser(); - asProjectMember(); - - const res = await server.request( - '/projects/10/translation-resources/manifest?languageCode=eng&bookCode=PSA&startChapter=1&endChapter=150', - { method: 'GET' } - ); - - expect(res.status).toBe(400); - expect(aquiferClient.searchAllResources).not.toHaveBeenCalled(); - }); - }); }); diff --git a/src/domains/translation-resources/translation-resources.types.ts b/src/domains/translation-resources/translation-resources.types.ts index 1b52e9ae..97c97044 100644 --- a/src/domains/translation-resources/translation-resources.types.ts +++ b/src/domains/translation-resources/translation-resources.types.ts @@ -327,7 +327,8 @@ export const prepareOfflineManifestResponseSchema = z items: z.array(prepareOfflineManifestItemSchema), totalBytes: z.number().int().nonnegative(), truncated: z.boolean().openapi({ - description: 'True when one or more resource configs exceeded the per-config hydration cap', + description: + 'True when one or more resource configs could not be fully enumerated within the server bounded Aquifer search page budget (provider total / short-page proof missing). False when every matching resource was collected; hydrate failures may still drop items (or 502 on the manifest fail-closed path) and are not covered by this flag. Clients must not treat a truncated catalogue as complete.', }), }) .openapi('PrepareOfflineResourceManifest'); diff --git a/src/lib/services/aquifer/aquifer.client.test.ts b/src/lib/services/aquifer/aquifer.client.test.ts index 03eae8b4..3c5ca9d9 100644 --- a/src/lib/services/aquifer/aquifer.client.test.ts +++ b/src/lib/services/aquifer/aquifer.client.test.ts @@ -4,6 +4,7 @@ import env from '@/env'; import { ErrorCode } from '@/lib/types'; import { + AQUIFER_MAX_SEARCH_PAGES, getBibles, getBibleText, getResource, @@ -283,8 +284,9 @@ describe('aquifer.client', () => { expect(result.ok).toBe(true); if (result.ok) { - expect(result.data).toHaveLength(101); - expect(result.data.at(-1)?.id).toBe(101); + expect(result.data.items).toHaveLength(101); + expect(result.data.items.at(-1)?.id).toBe(101); + expect(result.data.complete).toBe(true); } expect(globalThis.fetch).toHaveBeenCalledTimes(2); }); @@ -307,10 +309,76 @@ describe('aquifer.client', () => { expect(result.ok).toBe(true); if (result.ok) { - expect(result.data).toHaveLength(1); + expect(result.data.items).toHaveLength(1); + expect(result.data.complete).toBe(true); } expect(globalThis.fetch).toHaveBeenCalledTimes(1); }); + + it('marks enumeration incomplete when the page ceiling is hit with a known higher total', async () => { + const fullPage = (offset: number) => ({ + totalItemCount: AQUIFER_MAX_SEARCH_PAGES * 100 + 1, + returnedItemCount: 100, + offset, + items: Array.from({ length: 100 }, (_, index) => ({ + ...searchItem, + id: offset + index + 1, + name: `item-${offset + index + 1}`, + localizedName: `Item ${offset + index + 1}`, + })), + }); + const fetchMock = vi.spyOn(globalThis, 'fetch'); + for (let page = 0; page < AQUIFER_MAX_SEARCH_PAGES; page += 1) { + fetchMock.mockResolvedValueOnce(jsonResponse(fullPage(page * 100))); + } + + const result = await searchAllResources({ + bookCode: 'MRK', + startChapter: 1, + endChapter: 1, + languageCode: 'eng', + }); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.data.items).toHaveLength(AQUIFER_MAX_SEARCH_PAGES * 100); + expect(result.data.complete).toBe(false); + } + expect(globalThis.fetch).toHaveBeenCalledTimes(AQUIFER_MAX_SEARCH_PAGES); + }); + + it('marks enumeration complete when totalItemCount is reached exactly at the page ceiling', async () => { + const total = AQUIFER_MAX_SEARCH_PAGES * 100; + const fullPage = (offset: number) => ({ + totalItemCount: total, + returnedItemCount: 100, + offset, + items: Array.from({ length: 100 }, (_, index) => ({ + ...searchItem, + id: offset + index + 1, + name: `item-${offset + index + 1}`, + localizedName: `Item ${offset + index + 1}`, + })), + }); + const fetchMock = vi.spyOn(globalThis, 'fetch'); + for (let page = 0; page < AQUIFER_MAX_SEARCH_PAGES; page += 1) { + fetchMock.mockResolvedValueOnce(jsonResponse(fullPage(page * 100))); + } + + const result = await searchAllResources({ + bookCode: 'MRK', + startChapter: 1, + endChapter: 1, + languageCode: 'eng', + }); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.data.items).toHaveLength(total); + expect(result.data.complete).toBe(true); + } + expect(globalThis.fetch).toHaveBeenCalledTimes(AQUIFER_MAX_SEARCH_PAGES); + }); }); describe('getBibles', () => { diff --git a/src/lib/services/aquifer/aquifer.client.ts b/src/lib/services/aquifer/aquifer.client.ts index 286986cc..13d32ac2 100644 --- a/src/lib/services/aquifer/aquifer.client.ts +++ b/src/lib/services/aquifer/aquifer.client.ts @@ -13,6 +13,7 @@ import type { AquiferLanguageResourceCount, AquiferResourceCollection, AquiferResourceDetails, + AquiferResourceSearchEnumeration, AquiferResourceSearchResponse, AquiferSearchResourcesParams, } from './aquifer.types'; @@ -30,7 +31,7 @@ import { const DEFAULT_TIMEOUT_MS = 30_000; /** Hard cap on search pagination to avoid unbounded Aquifer fan-out. */ -const MAX_SEARCH_PAGES = 20; +export const AQUIFER_MAX_SEARCH_PAGES = 20; /** Cap on upstream body text echoed into logs. */ const MAX_LOGGED_BODY_CHARS = 300; /** Cap on zod issues echoed into logs. */ @@ -236,31 +237,35 @@ export async function getResource(contentId: number): Promise -): Promise> { +): Promise> { const allItems: AquiferResourceSearchResponse['items'] = []; let offset = 0; const limit = 100; + let complete = false; - for (let pageIndex = 0; pageIndex < MAX_SEARCH_PAGES; pageIndex += 1) { + for (let pageIndex = 0; pageIndex < AQUIFER_MAX_SEARCH_PAGES; pageIndex += 1) { const page = await searchResources({ ...params, limit, offset }); if (!page.ok) return page; allItems.push(...page.data.items); - if ( - allItems.length >= page.data.totalItemCount || - page.data.items.length === 0 || - page.data.items.length < limit - ) { + // Empty page is subsumed by `< limit` (0 < 100); keep one short-page check. + if (allItems.length >= page.data.totalItemCount || page.data.items.length < limit) { + complete = true; break; } offset += limit; } - return { ok: true, data: allItems }; + return { ok: true, data: { items: allItems, complete } }; } /** diff --git a/src/lib/services/aquifer/aquifer.types.ts b/src/lib/services/aquifer/aquifer.types.ts index 91065fc2..a10ecd27 100644 --- a/src/lib/services/aquifer/aquifer.types.ts +++ b/src/lib/services/aquifer/aquifer.types.ts @@ -50,6 +50,16 @@ export const aquiferResourceSearchResponseSchema = z.object({ export type AquiferResourceSearchResponse = z.infer; +/** + * Bounded Aquifer search collection. `complete` is false when the page ceiling + * stopped collection without provider proof that every matching item was returned + * (short/empty page or `totalItemCount` reached). + */ +export interface AquiferResourceSearchEnumeration { + items: AquiferResourceSearchResponse['items']; + complete: boolean; +} + // One wire model for online resources and offline manifests. Known fields are optional to // tolerate partial upstream metadata; unknown keys survive, including nested attribution fields. const aquiferLicenseLinkSchema = z