From 0c961c25d07084a03a25566e3aa8e2a9baaf8e86 Mon Sep 17 00:00:00 2001 From: Silas Date: Thu, 26 Mar 2026 18:17:03 +0000 Subject: [PATCH 1/7] feat(ai-import): derive roast date from best-before when label states offset Cloud AI prompt now instructs the model to calculate the roast date by subtracting the stated period from a best-before/expiration date when no explicit roast date is present but the label explicitly describes the offset (e.g. 'roasted 12 months before best before date'). The hint notes this may be stated in the local language of the label. --- src/data/ai-import/ai-cloud-prompt.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data/ai-import/ai-cloud-prompt.ts b/src/data/ai-import/ai-cloud-prompt.ts index 85c2eec70..91977d7b5 100644 --- a/src/data/ai-import/ai-cloud-prompt.ts +++ b/src/data/ai-import/ai-cloud-prompt.ts @@ -44,7 +44,7 @@ Use "NOT_FOUND" for any field not clearly present. "aromatics": "Comma-separated flavor/tasting notes", "decaffeinated": true or false, "cupping_points": numeric score (typically 80-100), - "roasting_date": "YYYY-MM-DD (use today's date to resolve ambiguous date formats — the roasting date is most likely within the past year and never in the future)", + "roasting_date": "YYYY-MM-DD — the roasting date. Use today's date to resolve ambiguous date formats. If no explicit roast date is found but the label states a best-before/use-by/expiration date AND explicitly states the time between roasting and that date (e.g., 'roasted 12 months before best before date' — note this may be stated in the local language of the label), calculate the roast date by subtracting the stated period from the best-before date. The roasting date is most likely within the past year and never in the future. NOT_FOUND if neither an explicit roast date nor a calculable one is present.", "bean_mix": "SINGLE_ORIGIN or BLEND", "origins": [ { From 4b1637041ab32292bf1ff4b79bee8f1f5a00812d Mon Sep 17 00:00:00 2001 From: Silas Date: Sat, 28 Mar 2026 08:29:10 +0000 Subject: [PATCH 2/7] =?UTF-8?q?feat(ai-import):=20add=20multi-pass=20rotat?= =?UTF-8?q?ed=20OCR=20for=2090=C2=B0=20text=20on=20coffee=20labels?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run OCR at 0°, 90°, and 270° for each image to detect text printed sideways on labels. Rotated-pass blocks are classified using the 0° pass height statistics as baseline and merged into the enriched text under a '--- Rotated text detected ---' section (only when rotated passes actually find text). --- .../__tests__/image-rotation.spec.ts | 47 ++++ .../__tests__/ocr-metadata.service.spec.ts | 219 ++++++++++++++++++ ...ple-intelligence-ai-bean-import.service.ts | 15 +- .../aiBeanImport/camera-ocr.service.ts | 77 ++++-- .../cloud-ai-bean-import.service.ts | 15 +- src/services/aiBeanImport/image-rotation.ts | 33 +++ .../aiBeanImport/ocr-metadata.service.ts | 129 +++++++++++ 7 files changed, 500 insertions(+), 35 deletions(-) create mode 100644 src/services/aiBeanImport/__tests__/image-rotation.spec.ts create mode 100644 src/services/aiBeanImport/image-rotation.ts diff --git a/src/services/aiBeanImport/__tests__/image-rotation.spec.ts b/src/services/aiBeanImport/__tests__/image-rotation.spec.ts new file mode 100644 index 000000000..bbbd9cc92 --- /dev/null +++ b/src/services/aiBeanImport/__tests__/image-rotation.spec.ts @@ -0,0 +1,47 @@ +import { rotateBase64Image } from '../image-rotation'; + +/** + * Create a small non-square test JPEG as raw base64 using the browser Canvas API. + * Returns a 4×2 red image so rotation visibly swaps dimensions. + */ +function createTestJpegBase64(width = 4, height = 2): string { + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d')!; + ctx.fillStyle = '#ff0000'; + ctx.fillRect(0, 0, width, height); + const dataUrl = canvas.toDataURL('image/jpeg', 0.92); + return dataUrl.replace(/^data:image\/jpeg;base64,/, ''); +} + +describe('rotateBase64Image', () => { + let testBase64: string; + + beforeAll(() => { + testBase64 = createTestJpegBase64(); + }); + + it('should produce a non-empty base64 string when rotated by 90°', async () => { + const result = await rotateBase64Image(testBase64, 90); + expect(result).toBeTruthy(); + expect(result.length).toBeGreaterThan(0); + }); + + it('should produce a non-empty base64 string when rotated by 270°', async () => { + const result = await rotateBase64Image(testBase64, 270); + expect(result).toBeTruthy(); + expect(result.length).toBeGreaterThan(0); + }); + + it('should not contain the data:image/jpeg;base64, prefix in the output', async () => { + const result = await rotateBase64Image(testBase64, 90); + expect(result).not.toContain('data:image/jpeg;base64,'); + }); + + it('should reject with an error when given invalid base64', async () => { + await expectAsync( + rotateBase64Image('not-a-valid-image!!!', 90), + ).toBeRejectedWithError('Failed to load image for rotation'); + }); +}); diff --git a/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts b/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts index 9296db916..fbb9ceb37 100644 --- a/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts +++ b/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts @@ -1,5 +1,6 @@ import { TestBed } from '@angular/core/testing'; +import { MultiPassOcrResult } from '../camera-ocr.service'; import { Block, OcrMetadataService, @@ -236,6 +237,224 @@ describe('OcrMetadataService', () => { }); }); + describe('enrichWithLayoutMultiPass', () => { + it('should return output identical to enrichWithLayout when no rotated results', () => { + // Arrange + const primary = createTextDetectionResult('Roaster\nCoffee Name', [ + createBlock('Roaster', 0, 0, 200, 80), + createBlock('Coffee Name', 0, 100, 200, 140), + ]); + const multiPass: MultiPassOcrResult = { primary, rotated: [] }; + + // Act + const multiPassResult = service.enrichWithLayoutMultiPass(multiPass); + const singlePassResult = service.enrichWithLayout(primary); + + // Assert + expect(multiPassResult.enrichedText).toBe(singlePassResult.enrichedText); + expect(multiPassResult.rawText).toBe(singlePassResult.rawText); + }); + + it('should append "--- Rotated text detected ---" section when rotated results have text', () => { + // Arrange + const primary = createTextDetectionResult('Front Label', [ + createBlock('BIG ROASTER', 0, 0, 300, 100), // Height: 100 + createBlock('small details', 0, 120, 300, 140), // Height: 20 + ]); + const rotated90 = createTextDetectionResult('Side Text', [ + createBlock('Side Text', 0, 0, 200, 60), + ]); + const multiPass: MultiPassOcrResult = { + primary, + rotated: [rotated90], + }; + + // Act + const result = service.enrichWithLayoutMultiPass(multiPass); + + // Assert + expect(result.enrichedText).toContain('--- Rotated text detected ---'); + expect(result.enrichedText).toContain('Side Text'); + expect(result.hasUsefulMetadata).toBeTrue(); + }); + + it('should classify large rotated text as LARGE using 0° pass stats', () => { + // Arrange: 0° pass has avg height ~60, max height 100 + const primary = createTextDetectionResult('Front', [ + createBlock('Title', 0, 0, 300, 100), // Height: 100 + createBlock('Body', 0, 120, 300, 140), // Height: 20 + ]); + // Rotated block with height 90 → ≥ 0.8 × 100 = 80 → LARGE + const rotated = createTextDetectionResult('Big Rotated', [ + createBlock('Big Rotated', 0, 0, 200, 90), + ]); + const multiPass: MultiPassOcrResult = { + primary, + rotated: [rotated], + }; + + // Act + const result = service.enrichWithLayoutMultiPass(multiPass); + + // Assert + const rotatedSection = result.enrichedText.split( + '--- Rotated text detected ---', + )[1]; + expect(rotatedSection).toContain('**LARGE:** Big Rotated'); + }); + + it('should classify small rotated text as SMALL using 0° pass stats', () => { + // Arrange: 0° pass has avg height 60, max height 100 + const primary = createTextDetectionResult('Front', [ + createBlock('Title', 0, 0, 300, 100), // Height: 100 + createBlock('Body', 0, 120, 300, 140), // Height: 20 + ]); + // Rotated block with height 30 → < 0.7 × 60 = 42 → SMALL + const rotated = createTextDetectionResult('Espresso', [ + createBlock('Espresso', 0, 0, 200, 30), + ]); + const multiPass: MultiPassOcrResult = { + primary, + rotated: [rotated], + }; + + // Act + const result = service.enrichWithLayoutMultiPass(multiPass); + + // Assert + const rotatedSection = result.enrichedText.split( + '--- Rotated text detected ---', + )[1]; + expect(rotatedSection).toContain('**SMALL:** Espresso'); + }); + + it('should classify medium rotated text as MEDIUM using 0° pass stats', () => { + // Arrange: 0° pass has avg height 60, max height 100 + const primary = createTextDetectionResult('Front', [ + createBlock('Title', 0, 0, 300, 100), // Height: 100 + createBlock('Body', 0, 120, 300, 140), // Height: 20 + ]); + // Rotated block with height 55 → not large (< 80, < 90), not small (≥ 42) → MEDIUM + const rotated = createTextDetectionResult('Filter', [ + createBlock('Filter', 0, 0, 200, 55), + ]); + const multiPass: MultiPassOcrResult = { + primary, + rotated: [rotated], + }; + + // Act + const result = service.enrichWithLayoutMultiPass(multiPass); + + // Assert + const rotatedSection = result.enrichedText.split( + '--- Rotated text detected ---', + )[1]; + expect(rotatedSection).toContain('**MEDIUM:** Filter'); + }); + + it('should fall back to independent classification when baseline has too few blocks', () => { + // Arrange: primary has only 1 block (below OCR_MIN_BLOCKS_FOR_METADATA) + const primary = createTextDetectionResult('Only', [ + createBlock('Only', 0, 0, 100, 50), + ]); + const rotated = createTextDetectionResult('Rotated', [ + createBlock('Big', 0, 0, 200, 100), + createBlock('Small', 0, 120, 200, 130), + ]); + const multiPass: MultiPassOcrResult = { + primary, + rotated: [rotated], + }; + + // Act + const result = service.enrichWithLayoutMultiPass(multiPass); + + // Assert - should not throw and should have rotated section + expect(result.enrichedText).toContain('--- Rotated text detected ---'); + expect(result.enrichedText).toContain('Big'); + expect(result.enrichedText).toContain('Small'); + }); + + it('should not append rotated section when rotated results have empty blocks', () => { + // Arrange + const primary = createTextDetectionResult('Front', [ + createBlock('Title', 0, 0, 300, 100), + createBlock('Body', 0, 120, 300, 140), + ]); + const rotated = createTextDetectionResult('', []); + const multiPass: MultiPassOcrResult = { + primary, + rotated: [rotated], + }; + + // Act + const result = service.enrichWithLayoutMultiPass(multiPass); + + // Assert + expect(result.enrichedText).not.toContain( + '--- Rotated text detected ---', + ); + }); + }); + + describe('enrichMultiplePhotosMultiPass', () => { + it('should return empty string for empty array', () => { + expect(service.enrichMultiplePhotosMultiPass([])).toBe(''); + }); + + it('should delegate to enrichWithLayoutMultiPass for single result', () => { + // Arrange + const primary = createTextDetectionResult('Photo 1', [ + createBlock('Title', 0, 0, 200, 100), + createBlock('Detail', 0, 120, 200, 140), + ]); + const multiPass: MultiPassOcrResult = { primary, rotated: [] }; + + // Act + const result = service.enrichMultiplePhotosMultiPass([multiPass]); + const expected = service.enrichWithLayoutMultiPass(multiPass); + + // Assert + expect(result).toBe(expected.enrichedText); + }); + + it('should add "Label N of M" markers for multiple results with rotated sections', () => { + // Arrange + const photo1Primary = createTextDetectionResult('Photo1', [ + createBlock('Roaster A', 0, 0, 300, 100), + createBlock('details', 0, 120, 300, 140), + ]); + const photo1Rotated = createTextDetectionResult('Side A', [ + createBlock('Side A', 0, 0, 200, 60), + ]); + const photo1: MultiPassOcrResult = { + primary: photo1Primary, + rotated: [photo1Rotated], + }; + + const photo2Primary = createTextDetectionResult('Photo2', [ + createBlock('Roaster B', 0, 0, 300, 100), + createBlock('info', 0, 120, 300, 140), + ]); + const photo2: MultiPassOcrResult = { + primary: photo2Primary, + rotated: [], + }; + + // Act + const result = service.enrichMultiplePhotosMultiPass([photo1, photo2]); + + // Assert + expect(result).toContain('=== OCR WITH LAYOUT ==='); + expect(result).toContain('--- Label 1 of 2 ---'); + expect(result).toContain('--- Label 2 of 2 ---'); + expect(result).toContain('--- Rotated text detected ---'); + expect(result).toContain('Side A'); + expect(result).toContain('Roaster B'); + }); + }); + describe('edge cases', () => { it('should handle zero height blocks gracefully without dividing by zero', () => { // WHY: ML Kit sometimes returns blocks with zero height for malformed OCR diff --git a/src/services/aiBeanImport/apple-intelligence-ai-bean-import.service.ts b/src/services/aiBeanImport/apple-intelligence-ai-bean-import.service.ts index ac583eeb7..fba0fd422 100644 --- a/src/services/aiBeanImport/apple-intelligence-ai-bean-import.service.ts +++ b/src/services/aiBeanImport/apple-intelligence-ai-bean-import.service.ts @@ -15,13 +15,10 @@ import { UIAlert } from '../uiAlert'; import { UILog } from '../uiLog'; import { AIImportStep, createAIBeanImportError } from './ai-bean-import-error'; import { AIImportExamplesService } from './ai-import-examples.service'; -import { CameraOcrService } from './camera-ocr.service'; +import { CameraOcrService, MultiPassOcrResult } from './camera-ocr.service'; import { FieldExtractionService } from './field-extraction.service'; import { sendLLMPrompt } from './llm-communication.service'; -import { - OcrMetadataService, - TextDetectionResult, -} from './ocr-metadata.service'; +import { OcrMetadataService } from './ocr-metadata.service'; export interface AIReadinessResult { ready: boolean; @@ -189,14 +186,14 @@ export class AppleIntelligenceAIBeanImportService { * Takes OCR result(s), enriches with layout, detects language, and extracts fields. */ private async processOcrAndExtractBean( - ocrResults: TextDetectionResult[], + ocrResults: MultiPassOcrResult[], rawTexts: string[], ): Promise { - // Step 1: Enrich with layout metadata + // Step 1: Enrich with layout metadata (multi-pass aware) const enrichedText = ocrResults.length === 1 - ? this.ocrMetadata.enrichWithLayout(ocrResults[0]).enrichedText - : this.ocrMetadata.enrichMultiplePhotos(ocrResults); + ? this.ocrMetadata.enrichWithLayoutMultiPass(ocrResults[0]).enrichedText + : this.ocrMetadata.enrichMultiplePhotosMultiPass(ocrResults); this.uiLog.log(`Enriched text length: ${enrichedText.length}`); diff --git a/src/services/aiBeanImport/camera-ocr.service.ts b/src/services/aiBeanImport/camera-ocr.service.ts index ac313158b..e1a8ce7ee 100644 --- a/src/services/aiBeanImport/camera-ocr.service.ts +++ b/src/services/aiBeanImport/camera-ocr.service.ts @@ -13,15 +13,28 @@ import { UIAlert } from '../uiAlert'; import { UIFileHelper } from '../uiFileHelper'; import { UIImage } from '../uiImage'; import { UILog } from '../uiLog'; +import { rotateBase64Image } from './image-rotation'; import { TextDetectionResult } from './ocr-metadata.service'; +/** + * Result of multi-pass OCR on a single image. + * The primary result (0°) is always present. + * Rotated results (90°/270°) are only present when they found text. + */ +export interface MultiPassOcrResult { + /** OCR result from the original (0°) image */ + primary: TextDetectionResult; + /** OCR results from rotated passes (90°, 270°) — only those that found text */ + rotated: TextDetectionResult[]; +} + export interface SinglePhotoCaptureResult { - ocrResult: TextDetectionResult; + ocrResult: MultiPassOcrResult; rawText: string; } export interface MultiPhotoOcrResult { - ocrResults: TextDetectionResult[]; + ocrResults: MultiPassOcrResult[]; rawTexts: string[]; } @@ -72,12 +85,11 @@ export class CameraOcrService { this.translate.instant('AI_IMPORT_STEP_EXTRACTING'), ); - const ocrResult = (await CapacitorPluginMlKitTextRecognition.detectText({ - base64Image: imageData.base64String, - })) as TextDetectionResult; - const rawText = ocrResult.text; + const ocrResult = await this.ocrWithRotations(imageData.base64String); + const rawText = ocrResult.primary.text; this.uiLog.log( - 'CameraOcr: OCR result: ' + JSON.stringify(ocrResult).substring(0, 500), + 'CameraOcr: OCR result: ' + + JSON.stringify(ocrResult.primary).substring(0, 500), ); if (!rawText || rawText.trim() === '') { @@ -102,7 +114,7 @@ export class CameraOcrService { * Returns only photos that produced text. */ async ocrFromPhotoPaths(photoPaths: string[]): Promise { - const ocrResults: TextDetectionResult[] = []; + const ocrResults: MultiPassOcrResult[] = []; const rawTexts: string[] = []; for (let i = 0; i < photoPaths.length; i++) { @@ -141,21 +153,17 @@ export class CameraOcrService { } try { - const ocrResult = (await CapacitorPluginMlKitTextRecognition.detectText( - { - base64Image: base64, - }, - )) as TextDetectionResult; + const ocrResult = await this.ocrWithRotations(base64); this.uiLog.log( - `CameraOcr: Photo ${i + 1} OCR result: ${JSON.stringify(ocrResult).substring(0, 200)}`, + `CameraOcr: Photo ${i + 1} OCR result: ${JSON.stringify(ocrResult.primary).substring(0, 200)}`, ); - if (ocrResult.text && ocrResult.text.trim() !== '') { + if (ocrResult.primary.text && ocrResult.primary.text.trim() !== '') { ocrResults.push(ocrResult); - rawTexts.push(ocrResult.text); + rawTexts.push(ocrResult.primary.text); this.uiLog.log( - `CameraOcr: Photo ${i + 1} extracted ${ocrResult.text.length} chars`, + `CameraOcr: Photo ${i + 1} extracted ${ocrResult.primary.text.length} chars`, ); } else { this.uiLog.log(`CameraOcr: Photo ${i + 1} had no text`); @@ -171,6 +179,41 @@ export class CameraOcrService { return { ocrResults, rawTexts }; } + /** + * Run OCR on a base64 image at 0°, 90°, and 270°. + * Returns the primary (0°) result and any rotated results that found text. + */ + private async ocrWithRotations(base64: string): Promise { + // Primary pass (0°) + const primary = (await CapacitorPluginMlKitTextRecognition.detectText({ + base64Image: base64, + })) as TextDetectionResult; + + const rotated: TextDetectionResult[] = []; + + // Rotated passes — only if primary succeeded (image is valid) + for (const degrees of [90, 270] as const) { + try { + const rotatedBase64 = await rotateBase64Image(base64, degrees); + const result = (await CapacitorPluginMlKitTextRecognition.detectText({ + base64Image: rotatedBase64, + })) as TextDetectionResult; + + if (result.text && result.text.trim() !== '') { + rotated.push(result); + this.uiLog.log( + `CameraOcr: Rotated ${degrees}° pass found ${result.text.length} chars`, + ); + } + } catch (error: unknown) { + const msg = error instanceof Error ? error.message : String(error); + this.uiLog.error(`CameraOcr: Rotated ${degrees}° pass failed: ${msg}`); + } + } + + return { primary, rotated }; + } + /** * Delete temporary photo files. */ diff --git a/src/services/aiBeanImport/cloud-ai-bean-import.service.ts b/src/services/aiBeanImport/cloud-ai-bean-import.service.ts index fd949ed1a..4cfcfd840 100644 --- a/src/services/aiBeanImport/cloud-ai-bean-import.service.ts +++ b/src/services/aiBeanImport/cloud-ai-bean-import.service.ts @@ -9,12 +9,9 @@ import { UILog } from '../uiLog'; import { UISettingsStorage } from '../uiSettingsStorage'; import { AIImportStep, createAIBeanImportError } from './ai-bean-import-error'; import { AIReadinessResult } from './apple-intelligence-ai-bean-import.service'; -import { CameraOcrService } from './camera-ocr.service'; +import { CameraOcrService, MultiPassOcrResult } from './camera-ocr.service'; import { CloudFieldExtractionService } from './cloud-field-extraction.service'; -import { - OcrMetadataService, - TextDetectionResult, -} from './ocr-metadata.service'; +import { OcrMetadataService } from './ocr-metadata.service'; @Injectable({ providedIn: 'root', @@ -166,13 +163,13 @@ export class CloudAIBeanImportService { * Enriches OCR results with layout metadata, then sends to cloud LLM. */ private async processOcrAndExtractBean( - ocrResults: TextDetectionResult[], + ocrResults: MultiPassOcrResult[], ): Promise { - // Step 1: Enrich with layout metadata + // Step 1: Enrich with layout metadata (multi-pass aware) const enrichedText = ocrResults.length === 1 - ? this.ocrMetadata.enrichWithLayout(ocrResults[0]).enrichedText - : this.ocrMetadata.enrichMultiplePhotos(ocrResults); + ? this.ocrMetadata.enrichWithLayoutMultiPass(ocrResults[0]).enrichedText + : this.ocrMetadata.enrichMultiplePhotosMultiPass(ocrResults); this.uiLog.log(`Cloud AI: Enriched text length: ${enrichedText.length}`); diff --git a/src/services/aiBeanImport/image-rotation.ts b/src/services/aiBeanImport/image-rotation.ts new file mode 100644 index 000000000..502bb800a --- /dev/null +++ b/src/services/aiBeanImport/image-rotation.ts @@ -0,0 +1,33 @@ +/** + * Rotate a base64-encoded image by the given degrees using the Canvas API. + * + * @param base64 Raw base64 string (no data URL prefix) + * @param degrees Clockwise rotation: 90 or 270 + * @returns Rotated image as raw base64 string (no data URL prefix) + */ +export function rotateBase64Image( + base64: string, + degrees: 90 | 270, +): Promise { + return new Promise((resolve, reject) => { + const img = new Image(); + img.onload = () => { + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + if (!ctx) { + reject(new Error('Canvas 2d context unavailable')); + return; + } + // 90° and 270° swap width/height + canvas.width = img.height; + canvas.height = img.width; + ctx.translate(canvas.width / 2, canvas.height / 2); + ctx.rotate((degrees * Math.PI) / 180); + ctx.drawImage(img, -img.width / 2, -img.height / 2); + const dataUrl = canvas.toDataURL('image/jpeg', 0.92); + resolve(dataUrl.replace(/^data:image\/jpeg;base64,/, '')); + }; + img.onerror = () => reject(new Error('Failed to load image for rotation')); + img.src = 'data:image/jpeg;base64,' + base64; + }); +} diff --git a/src/services/aiBeanImport/ocr-metadata.service.ts b/src/services/aiBeanImport/ocr-metadata.service.ts index bc8befbc6..37a71cb7a 100644 --- a/src/services/aiBeanImport/ocr-metadata.service.ts +++ b/src/services/aiBeanImport/ocr-metadata.service.ts @@ -7,6 +7,7 @@ import { OCR_SIZE_VARIATION_THRESHOLD, OCR_SMALL_TEXT_AVG_MULTIPLIER, } from '../../data/ai-import/ai-import-constants'; +import { MultiPassOcrResult } from './camera-ocr.service'; /** * Interfaces matching the ML Kit plugin's TypeScript definitions. @@ -134,6 +135,78 @@ export class OcrMetadataService { return '=== OCR WITH LAYOUT ===\n\n' + enrichedTexts.join('\n\n'); } + /** + * Enrich a multi-pass OCR result (single photo, multiple rotation passes). + * Classifies rotated-pass blocks using 0° pass height statistics as baseline. + * Only appends rotated text section if rotated passes found text. + */ + public enrichWithLayoutMultiPass( + multiPass: MultiPassOcrResult, + ): EnrichedOCRResult { + // Enrich the primary (0°) pass as before + const primaryEnriched = this.enrichWithLayout(multiPass.primary); + + // If no rotated results, return primary as-is + if (multiPass.rotated.length === 0) { + return primaryEnriched; + } + + // Collect all rotated blocks + const rotatedBlocks = multiPass.rotated.flatMap((r) => r.blocks ?? []); + if (rotatedBlocks.length === 0) { + return primaryEnriched; + } + + // Classify rotated blocks using 0° pass stats as baseline + const rotatedEnriched = this.classifyBlocksWithBaseline( + rotatedBlocks, + multiPass.primary.blocks, + ); + + // Format rotated blocks + const rotatedText = this.formatRotatedText(rotatedEnriched); + + // Combine: primary text + rotated section + const combinedText = primaryEnriched.enrichedText + '\n\n' + rotatedText; + const combinedRawText = + primaryEnriched.rawText + + '\n' + + multiPass.rotated.map((r) => r.text).join('\n'); + + return { + rawText: combinedRawText, + enrichedText: combinedText, + hasUsefulMetadata: true, + }; + } + + /** + * Process multiple multi-pass OCR results (multi-photo flow) with layout enrichment. + */ + public enrichMultiplePhotosMultiPass( + ocrResults: MultiPassOcrResult[], + ): string { + if (ocrResults.length === 0) { + return ''; + } + + if (ocrResults.length === 1) { + return this.enrichWithLayoutMultiPass(ocrResults[0]).enrichedText; + } + + const enrichedTexts = ocrResults.map((result, index) => { + const enriched = this.enrichWithLayoutMultiPass(result); + const textWithoutHeader = enriched.enrichedText.replace( + /^=== OCR WITH LAYOUT ===\n\n/, + '', + ); + const marker = `--- Label ${index + 1} of ${ocrResults.length} ---`; + return `${marker}\n${textWithoutHeader}`; + }); + + return '=== OCR WITH LAYOUT ===\n\n' + enrichedTexts.join('\n\n'); + } + /** * Determine if OCR metadata would be useful for this result. * Metadata is useful when there are multiple blocks with varied sizes. @@ -247,4 +320,60 @@ export class OcrMetadataService { return header + formattedBlocks.join('\n\n'); } + + /** + * Classify blocks using external baseline statistics (from the 0° pass). + * Falls back to per-pass independent classification if baseline is insufficient. + */ + private classifyBlocksWithBaseline( + blocks: Block[], + baselineBlocks: Block[], + ): EnrichedTextBlock[] { + // Compute baseline stats from 0° pass + const baselineHeights = (baselineBlocks ?? []) + .filter((b) => b.boundingBox && typeof b.boundingBox.bottom === 'number') + .map((b) => Math.abs(b.boundingBox.bottom - b.boundingBox.top)); + + // If baseline is insufficient, fall back to self-contained classification + if (baselineHeights.length < OCR_MIN_BLOCKS_FOR_METADATA) { + return this.classifyBlocks(blocks); + } + + const maxHeight = Math.max(...baselineHeights); + const avgHeight = + baselineHeights.reduce((a, b) => a + b, 0) / baselineHeights.length; + + return blocks.map((block) => { + const height = Math.abs(block.boundingBox.bottom - block.boundingBox.top); + let relativeSize: TextSize; + if ( + height >= maxHeight * OCR_LARGE_TEXT_MAX_HEIGHT_RATIO || + height >= avgHeight * OCR_LARGE_TEXT_AVG_MULTIPLIER + ) { + relativeSize = 'large'; + } else if (height < avgHeight * OCR_SMALL_TEXT_AVG_MULTIPLIER) { + relativeSize = 'small'; + } else { + relativeSize = 'medium'; + } + return { text: block.text, relativeSize }; + }); + } + + /** + * Format rotated text blocks with a section header. + */ + private formatRotatedText(blocks: EnrichedTextBlock[]): string { + if (blocks.length === 0) { + return ''; + } + + const header = '--- Rotated text detected ---\n'; + const formattedBlocks = blocks.map((block) => { + const sizeTag = block.relativeSize.toUpperCase(); + return `**${sizeTag}:** ${block.text}`; + }); + + return header + formattedBlocks.join('\n\n'); + } } From 495759da1e862bcd1591b9c8faaa75372d711cc4 Mon Sep 17 00:00:00 2001 From: Silas Date: Sat, 28 Mar 2026 09:47:52 +0000 Subject: [PATCH 3/7] refactor(ai-import): consolidate overlapping tests and extract shared classifyHeight helper --- .../__tests__/image-rotation.spec.ts | 21 +-- .../__tests__/ocr-metadata.service.spec.ts | 138 ++++++------------ .../aiBeanImport/camera-ocr.service.ts | 2 + .../aiBeanImport/ocr-metadata.service.ts | 54 +++---- 4 files changed, 81 insertions(+), 134 deletions(-) diff --git a/src/services/aiBeanImport/__tests__/image-rotation.spec.ts b/src/services/aiBeanImport/__tests__/image-rotation.spec.ts index bbbd9cc92..cfa56a3bf 100644 --- a/src/services/aiBeanImport/__tests__/image-rotation.spec.ts +++ b/src/services/aiBeanImport/__tests__/image-rotation.spec.ts @@ -22,21 +22,12 @@ describe('rotateBase64Image', () => { testBase64 = createTestJpegBase64(); }); - it('should produce a non-empty base64 string when rotated by 90°', async () => { - const result = await rotateBase64Image(testBase64, 90); - expect(result).toBeTruthy(); - expect(result.length).toBeGreaterThan(0); - }); - - it('should produce a non-empty base64 string when rotated by 270°', async () => { - const result = await rotateBase64Image(testBase64, 270); - expect(result).toBeTruthy(); - expect(result.length).toBeGreaterThan(0); - }); - - it('should not contain the data:image/jpeg;base64, prefix in the output', async () => { - const result = await rotateBase64Image(testBase64, 90); - expect(result).not.toContain('data:image/jpeg;base64,'); + [90, 270].forEach((degrees) => { + it(`should produce a non-empty base64 string without data URL prefix when rotated by ${degrees}°`, async () => { + const result = await rotateBase64Image(testBase64, degrees as 90 | 270); + expect(result).toBeTruthy(); + expect(result).not.toContain('data:image/jpeg;base64,'); + }); }); it('should reject with an error when given invalid base64', async () => { diff --git a/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts b/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts index fbb9ceb37..66b74a462 100644 --- a/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts +++ b/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts @@ -278,79 +278,51 @@ describe('OcrMetadataService', () => { expect(result.hasUsefulMetadata).toBeTrue(); }); - it('should classify large rotated text as LARGE using 0° pass stats', () => { - // Arrange: 0° pass has avg height ~60, max height 100 - const primary = createTextDetectionResult('Front', [ - createBlock('Title', 0, 0, 300, 100), // Height: 100 - createBlock('Body', 0, 120, 300, 140), // Height: 20 - ]); - // Rotated block with height 90 → ≥ 0.8 × 100 = 80 → LARGE - const rotated = createTextDetectionResult('Big Rotated', [ - createBlock('Big Rotated', 0, 0, 200, 90), - ]); - const multiPass: MultiPassOcrResult = { - primary, - rotated: [rotated], - }; - - // Act - const result = service.enrichWithLayoutMultiPass(multiPass); - - // Assert - const rotatedSection = result.enrichedText.split( - '--- Rotated text detected ---', - )[1]; - expect(rotatedSection).toContain('**LARGE:** Big Rotated'); - }); - - it('should classify small rotated text as SMALL using 0° pass stats', () => { - // Arrange: 0° pass has avg height 60, max height 100 - const primary = createTextDetectionResult('Front', [ - createBlock('Title', 0, 0, 300, 100), // Height: 100 - createBlock('Body', 0, 120, 300, 140), // Height: 20 - ]); - // Rotated block with height 30 → < 0.7 × 60 = 42 → SMALL - const rotated = createTextDetectionResult('Espresso', [ - createBlock('Espresso', 0, 0, 200, 30), - ]); - const multiPass: MultiPassOcrResult = { - primary, - rotated: [rotated], - }; - - // Act - const result = service.enrichWithLayoutMultiPass(multiPass); - - // Assert - const rotatedSection = result.enrichedText.split( - '--- Rotated text detected ---', - )[1]; - expect(rotatedSection).toContain('**SMALL:** Espresso'); - }); - - it('should classify medium rotated text as MEDIUM using 0° pass stats', () => { - // Arrange: 0° pass has avg height 60, max height 100 - const primary = createTextDetectionResult('Front', [ - createBlock('Title', 0, 0, 300, 100), // Height: 100 - createBlock('Body', 0, 120, 300, 140), // Height: 20 - ]); - // Rotated block with height 55 → not large (< 80, < 90), not small (≥ 42) → MEDIUM - const rotated = createTextDetectionResult('Filter', [ - createBlock('Filter', 0, 0, 200, 55), - ]); - const multiPass: MultiPassOcrResult = { - primary, - rotated: [rotated], - }; - - // Act - const result = service.enrichWithLayoutMultiPass(multiPass); - - // Assert - const rotatedSection = result.enrichedText.split( - '--- Rotated text detected ---', - )[1]; - expect(rotatedSection).toContain('**MEDIUM:** Filter'); + // 0° pass: avg height ~60, max height 100 + const classificationCases = [ + { + height: 90, + expected: 'LARGE', + label: 'Big Rotated', + reason: '≥ 0.8 × 100 = 80', + }, + { + height: 30, + expected: 'SMALL', + label: 'Espresso', + reason: '< 0.7 × 60 = 42', + }, + { + height: 55, + expected: 'MEDIUM', + label: 'Filter', + reason: 'between thresholds', + }, + ]; + classificationCases.forEach(({ height, expected, label, reason }) => { + it(`should classify rotated text (h=${height}) as ${expected} using 0° pass stats (${reason})`, () => { + // Arrange + const primary = createTextDetectionResult('Front', [ + createBlock('Title', 0, 0, 300, 100), // Height: 100 + createBlock('Body', 0, 120, 300, 140), // Height: 20 + ]); + const rotated = createTextDetectionResult(label, [ + createBlock(label, 0, 0, 200, height), + ]); + const multiPass: MultiPassOcrResult = { + primary, + rotated: [rotated], + }; + + // Act + const result = service.enrichWithLayoutMultiPass(multiPass); + + // Assert + const rotatedSection = result.enrichedText.split( + '--- Rotated text detected ---', + )[1]; + expect(rotatedSection).toContain(`**${expected}:** ${label}`); + }); }); it('should fall back to independent classification when baseline has too few blocks', () => { @@ -399,26 +371,6 @@ describe('OcrMetadataService', () => { }); describe('enrichMultiplePhotosMultiPass', () => { - it('should return empty string for empty array', () => { - expect(service.enrichMultiplePhotosMultiPass([])).toBe(''); - }); - - it('should delegate to enrichWithLayoutMultiPass for single result', () => { - // Arrange - const primary = createTextDetectionResult('Photo 1', [ - createBlock('Title', 0, 0, 200, 100), - createBlock('Detail', 0, 120, 200, 140), - ]); - const multiPass: MultiPassOcrResult = { primary, rotated: [] }; - - // Act - const result = service.enrichMultiplePhotosMultiPass([multiPass]); - const expected = service.enrichWithLayoutMultiPass(multiPass); - - // Assert - expect(result).toBe(expected.enrichedText); - }); - it('should add "Label N of M" markers for multiple results with rotated sections', () => { // Arrange const photo1Primary = createTextDetectionResult('Photo1', [ diff --git a/src/services/aiBeanImport/camera-ocr.service.ts b/src/services/aiBeanImport/camera-ocr.service.ts index e1a8ce7ee..a25e3ab08 100644 --- a/src/services/aiBeanImport/camera-ocr.service.ts +++ b/src/services/aiBeanImport/camera-ocr.service.ts @@ -185,6 +185,7 @@ export class CameraOcrService { */ private async ocrWithRotations(base64: string): Promise { // Primary pass (0°) + // ML Kit plugin returns this shape but its TS types are wider const primary = (await CapacitorPluginMlKitTextRecognition.detectText({ base64Image: base64, })) as TextDetectionResult; @@ -195,6 +196,7 @@ export class CameraOcrService { for (const degrees of [90, 270] as const) { try { const rotatedBase64 = await rotateBase64Image(base64, degrees); + // ML Kit plugin returns this shape but its TS types are wider const result = (await CapacitorPluginMlKitTextRecognition.detectText({ base64Image: rotatedBase64, })) as TextDetectionResult; diff --git a/src/services/aiBeanImport/ocr-metadata.service.ts b/src/services/aiBeanImport/ocr-metadata.service.ts index 37a71cb7a..5fb5a5f49 100644 --- a/src/services/aiBeanImport/ocr-metadata.service.ts +++ b/src/services/aiBeanImport/ocr-metadata.service.ts @@ -282,26 +282,36 @@ export class OcrMetadataService { const avgHeight = heightValues.reduce((a, b) => a + b, 0) / heightValues.length; - // Classify each block - // Large: >= OCR_LARGE_TEXT_AVG_MULTIPLIER * average OR within OCR_LARGE_TEXT_MAX_HEIGHT_RATIO of max - // Small: < OCR_SMALL_TEXT_AVG_MULTIPLIER * average - // Medium: everything else for (const { block, height } of heights) { - if ( - height >= maxHeight * OCR_LARGE_TEXT_MAX_HEIGHT_RATIO || - height >= avgHeight * OCR_LARGE_TEXT_AVG_MULTIPLIER - ) { - sizeMap.set(block, 'large'); - } else if (height < avgHeight * OCR_SMALL_TEXT_AVG_MULTIPLIER) { - sizeMap.set(block, 'small'); - } else { - sizeMap.set(block, 'medium'); - } + sizeMap.set(block, this.classifyHeight(height, maxHeight, avgHeight)); } return sizeMap; } + /** + * Classify a single block height against statistical thresholds. + * Large: >= OCR_LARGE_TEXT_AVG_MULTIPLIER × avg OR within OCR_LARGE_TEXT_MAX_HEIGHT_RATIO of max + * Small: < OCR_SMALL_TEXT_AVG_MULTIPLIER × avg + * Medium: everything else + */ + private classifyHeight( + height: number, + maxHeight: number, + avgHeight: number, + ): TextSize { + if ( + height >= maxHeight * OCR_LARGE_TEXT_MAX_HEIGHT_RATIO || + height >= avgHeight * OCR_LARGE_TEXT_AVG_MULTIPLIER + ) { + return 'large'; + } + if (height < avgHeight * OCR_SMALL_TEXT_AVG_MULTIPLIER) { + return 'small'; + } + return 'medium'; + } + /** * Format enriched blocks as annotated text using markdown. * Each block is prefixed with **SIZE:** tag. @@ -345,18 +355,10 @@ export class OcrMetadataService { return blocks.map((block) => { const height = Math.abs(block.boundingBox.bottom - block.boundingBox.top); - let relativeSize: TextSize; - if ( - height >= maxHeight * OCR_LARGE_TEXT_MAX_HEIGHT_RATIO || - height >= avgHeight * OCR_LARGE_TEXT_AVG_MULTIPLIER - ) { - relativeSize = 'large'; - } else if (height < avgHeight * OCR_SMALL_TEXT_AVG_MULTIPLIER) { - relativeSize = 'small'; - } else { - relativeSize = 'medium'; - } - return { text: block.text, relativeSize }; + return { + text: block.text, + relativeSize: this.classifyHeight(height, maxHeight, avgHeight), + }; }); } From c45c4c5688dc117f2d65d2497bc04ee74edcae73 Mon Sep 17 00:00:00 2001 From: Silas Date: Sat, 28 Mar 2026 11:07:44 +0000 Subject: [PATCH 4/7] =?UTF-8?q?refactor(ai-import):=20clean=20up=20logging?= =?UTF-8?q?=20=E2=80=94=20add=20LLM=20prompt/response=20logs,=20remove=20n?= =?UTF-8?q?oise?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Log prompt and response for both Apple Intelligence (in sendLLMPrompt) and Cloud LLM (in cloud-field-extraction) to make debugging easier. Suppress the deleteChat 'not implemented' error on iOS. Remove 18 verbose log calls (base64 lengths, OCR JSON dumps, photo processing progress) that cluttered output without aiding diagnosis. --- .../ai-import-photo-gallery.component.ts | 10 ------- .../cloud-field-extraction.service.spec.ts | 4 +-- ...ple-intelligence-ai-bean-import.service.ts | 6 ----- .../aiBeanImport/camera-ocr.service.ts | 27 ------------------- .../cloud-ai-bean-import.service.ts | 2 -- .../cloud-field-extraction.service.ts | 5 +++- .../aiBeanImport/field-extraction.service.ts | 7 ----- .../aiBeanImport/llm-communication.service.ts | 13 +++++++-- 8 files changed, 16 insertions(+), 58 deletions(-) diff --git a/src/components/ai-import-photo-gallery/ai-import-photo-gallery.component.ts b/src/components/ai-import-photo-gallery/ai-import-photo-gallery.component.ts index 73bbe3adc..bb2680a02 100644 --- a/src/components/ai-import-photo-gallery/ai-import-photo-gallery.component.ts +++ b/src/components/ai-import-photo-gallery/ai-import-photo-gallery.component.ts @@ -136,10 +136,6 @@ export class AiImportPhotoGalleryComponent { }); if (imageData?.base64String) { - this.uiLog.log( - `AI Import Gallery: Camera returned base64, length: ${imageData.base64String.length}, format: ${imageData.format}`, - ); - // Save to internal storage const fileName = await this.uiFileHelper.generateInternalPath( 'photo', @@ -152,9 +148,6 @@ export class AiImportPhotoGalleryComponent { if (fileUri.path) { this.photoPaths.push(fileUri.path); - this.uiLog.log( - `AI Import Gallery: Saved photo to ${fileUri.path}, total now: ${this.photoPaths.length}`, - ); this.updateSlider(); // Slide to the newly added photo @@ -210,9 +203,6 @@ export class AiImportPhotoGalleryComponent { if (fileUri.path) { this.photoPaths.push(fileUri.path); - this.uiLog.log( - `AI Import Gallery: Clipboard image saved to ${fileUri.path}, total now: ${this.photoPaths.length}`, - ); this.updateSlider(); this.focusOnNewPhoto(); } diff --git a/src/services/aiBeanImport/__tests__/cloud-field-extraction.service.spec.ts b/src/services/aiBeanImport/__tests__/cloud-field-extraction.service.spec.ts index 1c3aea7e8..fad20b5e3 100644 --- a/src/services/aiBeanImport/__tests__/cloud-field-extraction.service.spec.ts +++ b/src/services/aiBeanImport/__tests__/cloud-field-extraction.service.spec.ts @@ -252,9 +252,7 @@ describe('CloudFieldExtractionService', () => { await service.extractAllFields('sample OCR text', mockConfig, mockLogger); // Assert - expect(mockLogger.log).toHaveBeenCalledWith( - 'Cloud LLM response received, model: gpt-4o', - ); + expect(mockLogger.log).toHaveBeenCalledWith('[Cloud LLM] model: gpt-4o'); expect(mockLogger.log).toHaveBeenCalledWith( 'Token usage: 100 prompt, 50 completion', ); diff --git a/src/services/aiBeanImport/apple-intelligence-ai-bean-import.service.ts b/src/services/aiBeanImport/apple-intelligence-ai-bean-import.service.ts index fba0fd422..3b4e7cb57 100644 --- a/src/services/aiBeanImport/apple-intelligence-ai-bean-import.service.ts +++ b/src/services/aiBeanImport/apple-intelligence-ai-bean-import.service.ts @@ -195,8 +195,6 @@ export class AppleIntelligenceAIBeanImportService { ? this.ocrMetadata.enrichWithLayoutMultiPass(ocrResults[0]).enrichedText : this.ocrMetadata.enrichMultiplePhotosMultiPass(ocrResults); - this.uiLog.log(`Enriched text length: ${enrichedText.length}`); - // Step 2: Prepare raw text for language detection const combinedRawText = this.concatenateOCRResults(rawTexts); @@ -213,10 +211,6 @@ export class AppleIntelligenceAIBeanImportService { detectedLanguage, userLanguage, ); - this.uiLog.log( - 'Using languages for examples: ' + languagesToUse.join(', '), - ); - // Step 5: Extract fields this.uiAlert.setLoadingSpinnerMessage( this.translate.instant('AI_IMPORT_STEP_ANALYZING'), diff --git a/src/services/aiBeanImport/camera-ocr.service.ts b/src/services/aiBeanImport/camera-ocr.service.ts index a25e3ab08..ebdc3a1f5 100644 --- a/src/services/aiBeanImport/camera-ocr.service.ts +++ b/src/services/aiBeanImport/camera-ocr.service.ts @@ -76,22 +76,12 @@ export class CameraOcrService { await this.uiAlert.hideLoadingSpinner(); return null; } - this.uiLog.log( - 'CameraOcr: Photo captured, base64 length: ' + - imageData.base64String.length, - ); - this.uiAlert.setLoadingSpinnerMessage( this.translate.instant('AI_IMPORT_STEP_EXTRACTING'), ); const ocrResult = await this.ocrWithRotations(imageData.base64String); const rawText = ocrResult.primary.text; - this.uiLog.log( - 'CameraOcr: OCR result: ' + - JSON.stringify(ocrResult.primary).substring(0, 500), - ); - if (!rawText || rawText.trim() === '') { await this.uiAlert.hideLoadingSpinner(); await this.uiAlert.showMessage( @@ -126,16 +116,10 @@ export class CameraOcrService { ); const photoPath = photoPaths[i]; - this.uiLog.log( - `CameraOcr: Processing photo ${i + 1}/${photoPaths.length}, path: ${photoPath}`, - ); let base64: string; try { base64 = await this.uiFileHelper.readInternalFileAsBase64(photoPath); - this.uiLog.log( - `CameraOcr: Photo ${i + 1} read successfully, base64 length: ${base64.length}`, - ); } catch (readError: unknown) { const msg = readError instanceof Error ? readError.message : String(readError); @@ -155,16 +139,9 @@ export class CameraOcrService { try { const ocrResult = await this.ocrWithRotations(base64); - this.uiLog.log( - `CameraOcr: Photo ${i + 1} OCR result: ${JSON.stringify(ocrResult.primary).substring(0, 200)}`, - ); - if (ocrResult.primary.text && ocrResult.primary.text.trim() !== '') { ocrResults.push(ocrResult); rawTexts.push(ocrResult.primary.text); - this.uiLog.log( - `CameraOcr: Photo ${i + 1} extracted ${ocrResult.primary.text.length} chars`, - ); } else { this.uiLog.log(`CameraOcr: Photo ${i + 1} had no text`); } @@ -203,9 +180,6 @@ export class CameraOcrService { if (result.text && result.text.trim() !== '') { rotated.push(result); - this.uiLog.log( - `CameraOcr: Rotated ${degrees}° pass found ${result.text.length} chars`, - ); } } catch (error: unknown) { const msg = error instanceof Error ? error.message : String(error); @@ -223,7 +197,6 @@ export class CameraOcrService { for (const path of photoPaths) { try { await this.uiFileHelper.deleteInternalFile(path); - this.uiLog.log('CameraOcr: Deleted temp photo: ' + path); } catch (e) { this.uiLog.error('CameraOcr: Failed to delete temp photo: ' + e); } diff --git a/src/services/aiBeanImport/cloud-ai-bean-import.service.ts b/src/services/aiBeanImport/cloud-ai-bean-import.service.ts index 4cfcfd840..67f97f8a2 100644 --- a/src/services/aiBeanImport/cloud-ai-bean-import.service.ts +++ b/src/services/aiBeanImport/cloud-ai-bean-import.service.ts @@ -171,8 +171,6 @@ export class CloudAIBeanImportService { ? this.ocrMetadata.enrichWithLayoutMultiPass(ocrResults[0]).enrichedText : this.ocrMetadata.enrichMultiplePhotosMultiPass(ocrResults); - this.uiLog.log(`Cloud AI: Enriched text length: ${enrichedText.length}`); - // Step 2: Extract fields via cloud LLM (no language detection or vocabulary needed) this.uiAlert.setLoadingSpinnerMessage( this.translate.instant('AI_IMPORT_STEP_ANALYZING'), diff --git a/src/services/aiBeanImport/cloud-field-extraction.service.ts b/src/services/aiBeanImport/cloud-field-extraction.service.ts index 032d19769..e051b4371 100644 --- a/src/services/aiBeanImport/cloud-field-extraction.service.ts +++ b/src/services/aiBeanImport/cloud-field-extraction.service.ts @@ -66,12 +66,15 @@ export class CloudFieldExtractionService { const userPrompt = buildCloudExtractionPrompt(ocrText); // 3. Send to cloud LLM — throws on API errors, timeouts, network failures + log.log('[Cloud LLM] model: ' + config.model); + log.log('[Cloud LLM] prompt: ' + userPrompt); + const response = await sendCloudLLMPrompt(config, [ { role: 'system', content: CLOUD_BEAN_IMPORT_SYSTEM_INSTRUCTIONS }, { role: 'user', content: userPrompt }, ]); - log.log('Cloud LLM response received, model: ' + response.model); + log.log('[Cloud LLM] response: ' + response.content); if (response.usage) { log.log( `Token usage: ${response.usage.prompt_tokens} prompt, ${response.usage.completion_tokens} completion`, diff --git a/src/services/aiBeanImport/field-extraction.service.ts b/src/services/aiBeanImport/field-extraction.service.ts index 25bd73f37..adca589a4 100644 --- a/src/services/aiBeanImport/field-extraction.service.ts +++ b/src/services/aiBeanImport/field-extraction.service.ts @@ -82,7 +82,6 @@ export class FieldExtractionService { // Pre-process text const text = this.preProcess(ocrText, examples); - this.uiLog.debug('Normalized text length: ' + text.length); // Extract fields in two phases const topLevelFields = await this.extractTopLevelFields( @@ -311,8 +310,6 @@ export class FieldExtractionService { const response = await this.sendLLMMessage(prompt); const cleaned = this.cleanResponse(response); - this.uiLog.debug(fieldName + ' response: ' + cleaned); - // Handle null/not found responses (exact match only - partial NOT_FOUND handled by postProcess) if (isNullLikeValue(cleaned)) { return null; @@ -358,8 +355,6 @@ export class FieldExtractionService { // Don't use cleanResponse - it strips colons which breaks JSON const trimmed = response?.trim() || ''; - this.uiLog.debug('name_and_roaster response: ' + trimmed); - // Parse JSON response return this.parseNameAndRoasterResponse(trimmed); } catch (error) { @@ -467,8 +462,6 @@ export class FieldExtractionService { // Note: Don't use cleanResponse here - it strips colons which breaks JSON syntax const trimmed = response?.trim() || ''; - this.uiLog.debug('Blend origins response: ' + trimmed); - // Parse JSON response (handles markdown code blocks internally) return this.parseBlendOriginsResponse(trimmed); } diff --git a/src/services/aiBeanImport/llm-communication.service.ts b/src/services/aiBeanImport/llm-communication.service.ts index cf96549da..0c249944c 100644 --- a/src/services/aiBeanImport/llm-communication.service.ts +++ b/src/services/aiBeanImport/llm-communication.service.ts @@ -28,7 +28,11 @@ export interface LLMCommunicationOptions { /** System-level instructions passed to the LLM session (higher priority than prompt content) */ instructions?: string; /** Logger instance for error reporting */ - logger?: { error: (msg: string) => void; log: (msg: string) => void }; + logger?: { + error: (msg: string) => void; + log: (msg: string) => void; + debug?: (msg: string) => void; + }; } /** @@ -115,7 +119,10 @@ export async function sendLLMPrompt( await finishedListener?.remove(); await LLM.deleteChat({ chatId }); } catch (e) { - logger?.error('Error cleaning up listeners/chat: ' + e); + // Suppress "not implemented" error from deleteChat on iOS + if (!String(e).includes('not implemented')) { + logger?.debug?.('Cleanup warning: ' + e); + } } }; @@ -128,6 +135,7 @@ export async function sendLLMPrompt( const resolveOnce = (value: string) => { if (!resolved) { resolved = true; + logger?.log('[Apple Intelligence] response: ' + value); void cleanup(); resolvePromise(value); } @@ -153,6 +161,7 @@ export async function sendLLMPrompt( }, timeoutMs); // Send message (don't await - let the promise handle completion) + logger?.log('[Apple Intelligence] prompt: ' + prompt); void LLM.sendMessage({ chatId, message: prompt, From 2265594be6b6a65b5db60eabae7ce917cbc88c73 Mon Sep 17 00:00:00 2001 From: Silas Date: Sat, 28 Mar 2026 11:21:39 +0000 Subject: [PATCH 5/7] fix(ai-import): classify text size by average line height, not block height A multi-line paragraph in small font produces a tall bounding box, causing misclassification as LARGE. Use average line height within each block as proxy for font size, falling back to block height when line data is unavailable. --- .../__tests__/ocr-metadata.service.spec.ts | 101 +++++++++++------- .../aiBeanImport/ocr-metadata.service.ts | 42 ++++++-- .../test-utils/ai-import-test-helpers.ts | 23 +++- 3 files changed, 116 insertions(+), 50 deletions(-) diff --git a/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts b/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts index 66b74a462..21ba7d83b 100644 --- a/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts +++ b/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts @@ -6,7 +6,11 @@ import { OcrMetadataService, TextDetectionResult, } from '../ocr-metadata.service'; -import { createBlock, createTextDetectionResult } from '../test-utils'; +import { + createBlock, + createLine, + createTextDetectionResult, +} from '../test-utils'; describe('OcrMetadataService', () => { let service: OcrMetadataService; @@ -111,43 +115,6 @@ describe('OcrMetadataService', () => { expect(enriched.enrichedText).toContain('Details'); }); - it('should classify blocks by relative height into LARGE and SMALL categories', () => { - // WHY: Size classification helps LLM identify headers vs body text - - // Arrange - const largeBlock = createBlock('BIG', 0, 0, 200, 100); // Height: 100 - const smallBlock = createBlock('small', 0, 120, 200, 140); // Height: 20 - const result = createTextDetectionResult('BIG\nsmall', [ - largeBlock, - smallBlock, - ]); - - // Act - const enriched = service.enrichWithLayout(result); - - // Assert - expect(enriched.enrichedText).toContain('**LARGE:**'); - expect(enriched.enrichedText).toContain('**SMALL:**'); - }); - - it('should classify text sizes as LARGE, MEDIUM, and SMALL based on height variation', () => { - // Arrange - const result = createTextDetectionResult('Top\nMiddle\nBottom', [ - createBlock('Top', 0, 0, 100, 80), // Height: 80 (large) - createBlock('Middle', 0, 100, 100, 150), // Height: 50 (medium) - createBlock('Bottom', 0, 220, 100, 240), // Height: 20 (small) - ]); - - // Act - const enriched = service.enrichWithLayout(result); - - // Assert - expect(enriched.enrichedText).toContain('**LARGE:**'); - expect(enriched.enrichedText).toContain('Top'); - expect(enriched.enrichedText).toContain('Middle'); - expect(enriched.enrichedText).toContain('Bottom'); - }); - it('should include all block texts in output regardless of position', () => { // Arrange const result = createTextDetectionResult('Left\nCenter\nRight', [ @@ -407,6 +374,64 @@ describe('OcrMetadataService', () => { }); }); + describe('line-height-based classification', () => { + it('should classify a multi-line block by average line height, not total block height', () => { + // WHY: A description paragraph has many small-font lines but a tall bounding box. + // Using block height would misclassify it as LARGE. + + // Arrange + const heading = createBlock('NATURAL BLEND', 0, 0, 400, 80, 'en', [ + createLine('NATURAL BLEND', 0, 0, 400, 80), + ]); + + // 6 lines of small font (line height ~15 each), total block height = 90 + const description = createBlock( + 'Unser Natural Blend ist eine Mischung aus natürlich aufbereiteten Kaffees', + 0, + 100, + 400, + 190, + 'en', + [ + createLine('Unser Natural Blend', 0, 100, 400, 115), + createLine('ist eine Mischung aus', 0, 115, 400, 130), + createLine('natürlich aufbe-', 0, 130, 400, 145), + createLine('reiteten Kaffees', 0, 145, 400, 160), + createLine('aus Brasilien und', 0, 160, 400, 175), + createLine('Äthiopien.', 0, 175, 400, 190), + ], + ); + + const result = createTextDetectionResult('', [heading, description]); + + // Act + const enriched = service.enrichWithLayout(result); + + // Assert — heading is large font, description is small font + expect(enriched.enrichedText).toContain('**LARGE:** NATURAL BLEND'); + expect(enriched.enrichedText).not.toContain( + '**LARGE:** Unser Natural Blend', + ); + }); + + it('should fall back to block height when block has no lines', () => { + // WHY: Backward compatibility — blocks without line data (e.g., from test helpers) + // should still classify based on block bounding box height. + + // Arrange + const largeBlock = createBlock('BIG', 0, 0, 200, 100); // Height: 100, no lines + const smallBlock = createBlock('small', 0, 120, 200, 140); // Height: 20, no lines + const result = createTextDetectionResult('', [largeBlock, smallBlock]); + + // Act + const enriched = service.enrichWithLayout(result); + + // Assert + expect(enriched.enrichedText).toContain('**LARGE:** BIG'); + expect(enriched.enrichedText).toContain('**SMALL:** small'); + }); + }); + describe('edge cases', () => { it('should handle zero height blocks gracefully without dividing by zero', () => { // WHY: ML Kit sometimes returns blocks with zero height for malformed OCR diff --git a/src/services/aiBeanImport/ocr-metadata.service.ts b/src/services/aiBeanImport/ocr-metadata.service.ts index 5fb5a5f49..1f4a5a03d 100644 --- a/src/services/aiBeanImport/ocr-metadata.service.ts +++ b/src/services/aiBeanImport/ocr-metadata.service.ts @@ -229,10 +229,9 @@ export class OcrMetadataService { return false; } - // Check for size variation - // Use absolute value since coordinate system may have Y origin at bottom (top > bottom) + // Check for size variation using representative heights (avg line height) const heights = blocksWithBoundingBox.map((b) => - Math.abs(b.boundingBox.bottom - b.boundingBox.top), + this.getRepresentativeHeight(b), ); const maxHeight = Math.max(...heights); const minHeight = Math.min(...heights); @@ -260,16 +259,17 @@ export class OcrMetadataService { } /** - * Classify blocks by relative size based on height. - * Uses statistical thresholds relative to average and max. + * Classify blocks by relative size based on representative line height. + * Uses average line height within each block as a proxy for font size. + * Falls back to block bounding box height when line data is unavailable. */ private classifySizes(blocks: Block[]): Map { const sizeMap = new Map(); - // Calculate height of each block (use abs since Y origin may be at bottom) + // Calculate representative height of each block const heights = blocks.map((b) => ({ block: b, - height: Math.abs(b.boundingBox.bottom - b.boundingBox.top), + height: this.getRepresentativeHeight(b), })); if (heights.length === 0) { @@ -289,6 +289,28 @@ export class OcrMetadataService { return sizeMap; } + /** + * Get the representative height for a block, approximating font size. + * Uses average line height when line bounding boxes are available, + * otherwise falls back to the block's own bounding box height. + */ + private getRepresentativeHeight(block: Block): number { + if (block.lines && block.lines.length > 0) { + const lineHeights = block.lines + .filter( + (l) => l.boundingBox && typeof l.boundingBox.bottom === 'number', + ) + .map((l) => Math.abs(l.boundingBox.bottom - l.boundingBox.top)); + + if (lineHeights.length > 0) { + return lineHeights.reduce((a, b) => a + b, 0) / lineHeights.length; + } + } + + // Fallback: use block bounding box height + return Math.abs(block.boundingBox.bottom - block.boundingBox.top); + } + /** * Classify a single block height against statistical thresholds. * Large: >= OCR_LARGE_TEXT_AVG_MULTIPLIER × avg OR within OCR_LARGE_TEXT_MAX_HEIGHT_RATIO of max @@ -339,10 +361,10 @@ export class OcrMetadataService { blocks: Block[], baselineBlocks: Block[], ): EnrichedTextBlock[] { - // Compute baseline stats from 0° pass + // Compute baseline stats from 0° pass using representative heights const baselineHeights = (baselineBlocks ?? []) .filter((b) => b.boundingBox && typeof b.boundingBox.bottom === 'number') - .map((b) => Math.abs(b.boundingBox.bottom - b.boundingBox.top)); + .map((b) => this.getRepresentativeHeight(b)); // If baseline is insufficient, fall back to self-contained classification if (baselineHeights.length < OCR_MIN_BLOCKS_FOR_METADATA) { @@ -354,7 +376,7 @@ export class OcrMetadataService { baselineHeights.reduce((a, b) => a + b, 0) / baselineHeights.length; return blocks.map((block) => { - const height = Math.abs(block.boundingBox.bottom - block.boundingBox.top); + const height = this.getRepresentativeHeight(block); return { text: block.text, relativeSize: this.classifyHeight(height, maxHeight, avgHeight), diff --git a/src/services/aiBeanImport/test-utils/ai-import-test-helpers.ts b/src/services/aiBeanImport/test-utils/ai-import-test-helpers.ts index d1d9b22f1..c498e90f0 100644 --- a/src/services/aiBeanImport/test-utils/ai-import-test-helpers.ts +++ b/src/services/aiBeanImport/test-utils/ai-import-test-helpers.ts @@ -2,7 +2,7 @@ import { Bean } from '../../../classes/bean/bean'; import { IBeanInformation } from '../../../interfaces/bean/iBeanInformation'; import { IBeanParameter } from '../../../interfaces/parameter/iBeanParameter'; import { MergedExamples } from '../ai-import-examples.service'; -import { Block, TextDetectionResult } from '../ocr-metadata.service'; +import { Block, Line, TextDetectionResult } from '../ocr-metadata.service'; /** * Create a mock Block with specified bounding box. @@ -20,12 +20,31 @@ export function createBlock( right: number, bottom: number, recognizedLanguage: string = 'en', + lines: Line[] = [], ): Block { return { text, boundingBox: { left, top, right, bottom }, recognizedLanguage, - lines: [], + lines, + }; +} + +/** + * Create a mock Line with specified bounding box. + */ +export function createLine( + text: string, + left: number, + top: number, + right: number, + bottom: number, +): Line { + return { + text, + boundingBox: { left, top, right, bottom }, + recognizedLanguage: 'en', + elements: [], }; } From 7b26edb69aa32f91555ccac5371f37c5eb67fea2 Mon Sep 17 00:00:00 2001 From: Silas Date: Sat, 28 Mar 2026 13:25:26 +0000 Subject: [PATCH 6/7] refactor(ai-import): remove overlapping tests and clean up helpers - Remove no-op test asserting block texts on raw string (no layout logic exercised) - Remove subsumed rotated-section test, move hasUsefulMetadata assertion into classification loop - Modify shouldUseMetadata test to use line data, proving representative height drives decision - Extract boundingBoxHeight() to eliminate duplication in getRepresentativeHeight - Refactor createBlock to use options object instead of 7 positional params --- .../__tests__/ocr-metadata.service.spec.ts | 95 +++++++------------ .../aiBeanImport/ocr-metadata.service.ts | 12 ++- .../test-utils/ai-import-test-helpers.ts | 14 ++- 3 files changed, 54 insertions(+), 67 deletions(-) diff --git a/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts b/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts index 21ba7d83b..62caedb52 100644 --- a/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts +++ b/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts @@ -60,18 +60,31 @@ describe('OcrMetadataService', () => { }); }); - it('should return true when blocks have significant size variation (headers vs body text)', () => { - // WHY: Size variation indicates visual hierarchy useful for field extraction + it('should return true when representative line heights show significant variation', () => { + // WHY: Block bounding boxes can be similar in height (heading 80px, paragraph 90px) + // but average line heights reveal the real font size difference (80px vs 15px). // Arrange - const largeTitle = createBlock('Big Title', 0, 0, 200, 100); // Height: 100 - const smallBody = createBlock('Small text', 0, 110, 200, 130); // Height: 20 - const result = createTextDetectionResult('Big Title\nSmall text', [ - largeTitle, - smallBody, + const heading = createBlock('Big Title', 0, 0, 400, 80, { + lines: [createLine('Big Title', 0, 0, 400, 80)], // 1 line, avg h=80 + }); + const paragraph = createBlock('Small text body', 0, 100, 400, 190, { + lines: [ + createLine('Small text', 0, 100, 400, 115), // h=15 + createLine('body here', 0, 115, 400, 130), // h=15 + createLine('more text', 0, 130, 400, 145), // h=15 + createLine('continues', 0, 145, 400, 160), // h=15 + createLine('even more', 0, 160, 400, 175), // h=15 + createLine('last line', 0, 175, 400, 190), // h=15 + ], // 6 lines, avg h=15 + }); + const result = createTextDetectionResult('Big Title\nSmall text body', [ + heading, + paragraph, ]); - // Act & Assert + // Act & Assert — block heights 80 vs 90 (ratio 1.125 < 1.3 threshold) + // but representative heights 80 vs 15 (ratio 5.33 > 1.3 threshold) expect(service.shouldUseMetadata(result)).toBeTrue(); }); }); @@ -114,23 +127,6 @@ describe('OcrMetadataService', () => { expect(enriched.enrichedText).toContain('Coffee Name'); expect(enriched.enrichedText).toContain('Details'); }); - - it('should include all block texts in output regardless of position', () => { - // Arrange - const result = createTextDetectionResult('Left\nCenter\nRight', [ - createBlock('Left', 0, 0, 50, 80), - createBlock('Center', 100, 0, 200, 80), - createBlock('Right', 250, 0, 300, 80), - ]); - - // Act - const enriched = service.enrichWithLayout(result); - - // Assert - expect(enriched.enrichedText).toContain('Left'); - expect(enriched.enrichedText).toContain('Center'); - expect(enriched.enrichedText).toContain('Right'); - }); }); describe('enrichMultiplePhotos', () => { @@ -222,29 +218,6 @@ describe('OcrMetadataService', () => { expect(multiPassResult.rawText).toBe(singlePassResult.rawText); }); - it('should append "--- Rotated text detected ---" section when rotated results have text', () => { - // Arrange - const primary = createTextDetectionResult('Front Label', [ - createBlock('BIG ROASTER', 0, 0, 300, 100), // Height: 100 - createBlock('small details', 0, 120, 300, 140), // Height: 20 - ]); - const rotated90 = createTextDetectionResult('Side Text', [ - createBlock('Side Text', 0, 0, 200, 60), - ]); - const multiPass: MultiPassOcrResult = { - primary, - rotated: [rotated90], - }; - - // Act - const result = service.enrichWithLayoutMultiPass(multiPass); - - // Assert - expect(result.enrichedText).toContain('--- Rotated text detected ---'); - expect(result.enrichedText).toContain('Side Text'); - expect(result.hasUsefulMetadata).toBeTrue(); - }); - // 0° pass: avg height ~60, max height 100 const classificationCases = [ { @@ -285,6 +258,7 @@ describe('OcrMetadataService', () => { const result = service.enrichWithLayoutMultiPass(multiPass); // Assert + expect(result.hasUsefulMetadata).toBeTrue(); const rotatedSection = result.enrichedText.split( '--- Rotated text detected ---', )[1]; @@ -380,9 +354,9 @@ describe('OcrMetadataService', () => { // Using block height would misclassify it as LARGE. // Arrange - const heading = createBlock('NATURAL BLEND', 0, 0, 400, 80, 'en', [ - createLine('NATURAL BLEND', 0, 0, 400, 80), - ]); + const heading = createBlock('NATURAL BLEND', 0, 0, 400, 80, { + lines: [createLine('NATURAL BLEND', 0, 0, 400, 80)], + }); // 6 lines of small font (line height ~15 each), total block height = 90 const description = createBlock( @@ -391,15 +365,16 @@ describe('OcrMetadataService', () => { 100, 400, 190, - 'en', - [ - createLine('Unser Natural Blend', 0, 100, 400, 115), - createLine('ist eine Mischung aus', 0, 115, 400, 130), - createLine('natürlich aufbe-', 0, 130, 400, 145), - createLine('reiteten Kaffees', 0, 145, 400, 160), - createLine('aus Brasilien und', 0, 160, 400, 175), - createLine('Äthiopien.', 0, 175, 400, 190), - ], + { + lines: [ + createLine('Unser Natural Blend', 0, 100, 400, 115), + createLine('ist eine Mischung aus', 0, 115, 400, 130), + createLine('natürlich aufbe-', 0, 130, 400, 145), + createLine('reiteten Kaffees', 0, 145, 400, 160), + createLine('aus Brasilien und', 0, 160, 400, 175), + createLine('Äthiopien.', 0, 175, 400, 190), + ], + }, ); const result = createTextDetectionResult('', [heading, description]); diff --git a/src/services/aiBeanImport/ocr-metadata.service.ts b/src/services/aiBeanImport/ocr-metadata.service.ts index 1f4a5a03d..f307f2434 100644 --- a/src/services/aiBeanImport/ocr-metadata.service.ts +++ b/src/services/aiBeanImport/ocr-metadata.service.ts @@ -289,6 +289,14 @@ export class OcrMetadataService { return sizeMap; } + /** + * Height of a bounding box, using absolute value since coordinate system + * may have Y origin at bottom (top > bottom). + */ + private boundingBoxHeight(box: BoundingBox): number { + return Math.abs(box.bottom - box.top); + } + /** * Get the representative height for a block, approximating font size. * Uses average line height when line bounding boxes are available, @@ -300,7 +308,7 @@ export class OcrMetadataService { .filter( (l) => l.boundingBox && typeof l.boundingBox.bottom === 'number', ) - .map((l) => Math.abs(l.boundingBox.bottom - l.boundingBox.top)); + .map((l) => this.boundingBoxHeight(l.boundingBox)); if (lineHeights.length > 0) { return lineHeights.reduce((a, b) => a + b, 0) / lineHeights.length; @@ -308,7 +316,7 @@ export class OcrMetadataService { } // Fallback: use block bounding box height - return Math.abs(block.boundingBox.bottom - block.boundingBox.top); + return this.boundingBoxHeight(block.boundingBox); } /** diff --git a/src/services/aiBeanImport/test-utils/ai-import-test-helpers.ts b/src/services/aiBeanImport/test-utils/ai-import-test-helpers.ts index c498e90f0..af51c7ba6 100644 --- a/src/services/aiBeanImport/test-utils/ai-import-test-helpers.ts +++ b/src/services/aiBeanImport/test-utils/ai-import-test-helpers.ts @@ -4,6 +4,11 @@ import { IBeanParameter } from '../../../interfaces/parameter/iBeanParameter'; import { MergedExamples } from '../ai-import-examples.service'; import { Block, Line, TextDetectionResult } from '../ocr-metadata.service'; +interface CreateBlockOptions { + recognizedLanguage?: string; + lines?: Line[]; +} + /** * Create a mock Block with specified bounding box. * @param text Block text content @@ -11,7 +16,7 @@ import { Block, Line, TextDetectionResult } from '../ocr-metadata.service'; * @param top Top coordinate * @param right Right coordinate * @param bottom Bottom coordinate - * @param recognizedLanguage Optional language code (default: 'en') + * @param options Optional recognized language and lines */ export function createBlock( text: string, @@ -19,14 +24,13 @@ export function createBlock( top: number, right: number, bottom: number, - recognizedLanguage: string = 'en', - lines: Line[] = [], + options?: CreateBlockOptions, ): Block { return { text, boundingBox: { left, top, right, bottom }, - recognizedLanguage, - lines, + recognizedLanguage: options?.recognizedLanguage ?? 'en', + lines: options?.lines ?? [], }; } From 667088399566cfc37cbf8b0b5b19b38bd9b86df4 Mon Sep 17 00:00:00 2001 From: Silas Date: Sun, 29 Mar 2026 12:30:19 +0000 Subject: [PATCH 7/7] refactor(ai-import): address PR review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Strengthen rotation test to verify dimensions are swapped (4×2 → 2×4) - Include rotated pass text in rawText via extracted collectRawText() - Replace canvas context null guard with optional chaining - Collapse duplicate early-return guards in enrichWithLayoutMultiPass - Extract stripLayoutHeader() to eliminate duplicated regex --- .../__tests__/image-rotation.spec.ts | 12 ++++++- .../aiBeanImport/camera-ocr.service.ts | 18 ++++++++--- src/services/aiBeanImport/image-rotation.ts | 11 ++----- .../aiBeanImport/ocr-metadata.service.ts | 32 +++++++++---------- 4 files changed, 44 insertions(+), 29 deletions(-) diff --git a/src/services/aiBeanImport/__tests__/image-rotation.spec.ts b/src/services/aiBeanImport/__tests__/image-rotation.spec.ts index cfa56a3bf..18e5fe934 100644 --- a/src/services/aiBeanImport/__tests__/image-rotation.spec.ts +++ b/src/services/aiBeanImport/__tests__/image-rotation.spec.ts @@ -23,10 +23,20 @@ describe('rotateBase64Image', () => { }); [90, 270].forEach((degrees) => { - it(`should produce a non-empty base64 string without data URL prefix when rotated by ${degrees}°`, async () => { + it(`should swap dimensions of a 4×2 image to 2×4 and return raw base64 when rotated by ${degrees}°`, async () => { const result = await rotateBase64Image(testBase64, degrees as 90 | 270); expect(result).toBeTruthy(); expect(result).not.toContain('data:image/jpeg;base64,'); + + const img = await new Promise((resolve, reject) => { + const image = new Image(); + image.onload = () => resolve(image); + image.onerror = () => reject(new Error('Failed to load rotated image')); + image.src = `data:image/jpeg;base64,${result}`; + }); + + expect(img.naturalWidth).toBe(2); + expect(img.naturalHeight).toBe(4); }); }); diff --git a/src/services/aiBeanImport/camera-ocr.service.ts b/src/services/aiBeanImport/camera-ocr.service.ts index ebdc3a1f5..aa3b90be3 100644 --- a/src/services/aiBeanImport/camera-ocr.service.ts +++ b/src/services/aiBeanImport/camera-ocr.service.ts @@ -38,6 +38,15 @@ export interface MultiPhotoOcrResult { rawTexts: string[]; } +/** + * Combine all text from a multi-pass OCR result (primary + rotated passes). + */ +export function collectRawText(result: MultiPassOcrResult): string { + return [result.primary.text, ...result.rotated.map((r) => r.text)] + .filter((t) => t.trim()) + .join('\n'); +} + @Injectable({ providedIn: 'root', }) @@ -81,8 +90,8 @@ export class CameraOcrService { ); const ocrResult = await this.ocrWithRotations(imageData.base64String); - const rawText = ocrResult.primary.text; - if (!rawText || rawText.trim() === '') { + const rawText = collectRawText(ocrResult); + if (!rawText) { await this.uiAlert.hideLoadingSpinner(); await this.uiAlert.showMessage( 'AI_IMPORT_NO_TEXT_FOUND', @@ -139,9 +148,10 @@ export class CameraOcrService { try { const ocrResult = await this.ocrWithRotations(base64); - if (ocrResult.primary.text && ocrResult.primary.text.trim() !== '') { + const rawText = collectRawText(ocrResult); + if (rawText) { ocrResults.push(ocrResult); - rawTexts.push(ocrResult.primary.text); + rawTexts.push(rawText); } else { this.uiLog.log(`CameraOcr: Photo ${i + 1} had no text`); } diff --git a/src/services/aiBeanImport/image-rotation.ts b/src/services/aiBeanImport/image-rotation.ts index 502bb800a..811d883ec 100644 --- a/src/services/aiBeanImport/image-rotation.ts +++ b/src/services/aiBeanImport/image-rotation.ts @@ -14,16 +14,11 @@ export function rotateBase64Image( img.onload = () => { const canvas = document.createElement('canvas'); const ctx = canvas.getContext('2d'); - if (!ctx) { - reject(new Error('Canvas 2d context unavailable')); - return; - } - // 90° and 270° swap width/height canvas.width = img.height; canvas.height = img.width; - ctx.translate(canvas.width / 2, canvas.height / 2); - ctx.rotate((degrees * Math.PI) / 180); - ctx.drawImage(img, -img.width / 2, -img.height / 2); + ctx?.translate(canvas.width / 2, canvas.height / 2); + ctx?.rotate((degrees * Math.PI) / 180); + ctx?.drawImage(img, -img.width / 2, -img.height / 2); const dataUrl = canvas.toDataURL('image/jpeg', 0.92); resolve(dataUrl.replace(/^data:image\/jpeg;base64,/, '')); }; diff --git a/src/services/aiBeanImport/ocr-metadata.service.ts b/src/services/aiBeanImport/ocr-metadata.service.ts index f307f2434..da2d693d9 100644 --- a/src/services/aiBeanImport/ocr-metadata.service.ts +++ b/src/services/aiBeanImport/ocr-metadata.service.ts @@ -71,6 +71,8 @@ export interface EnrichedOCRResult { * field extraction accuracy. Converts spatial information into text annotations * that help the LLM understand layout context. */ +const LAYOUT_HEADER = '=== OCR WITH LAYOUT ===\n\n'; + @Injectable({ providedIn: 'root', }) @@ -123,11 +125,7 @@ export class OcrMetadataService { // Process each photo and combine with markers const enrichedTexts = ocrResults.map((result, index) => { const enriched = this.enrichWithLayout(result); - // Strip header from individual results to avoid duplication - const textWithoutHeader = enriched.enrichedText.replace( - /^=== OCR WITH LAYOUT ===\n\n/, - '', - ); + const textWithoutHeader = this.stripLayoutHeader(enriched.enrichedText); const marker = `--- Label ${index + 1} of ${ocrResults.length} ---`; return `${marker}\n${textWithoutHeader}`; }); @@ -146,12 +144,7 @@ export class OcrMetadataService { // Enrich the primary (0°) pass as before const primaryEnriched = this.enrichWithLayout(multiPass.primary); - // If no rotated results, return primary as-is - if (multiPass.rotated.length === 0) { - return primaryEnriched; - } - - // Collect all rotated blocks + // Collect all rotated blocks — return primary as-is if none found const rotatedBlocks = multiPass.rotated.flatMap((r) => r.blocks ?? []); if (rotatedBlocks.length === 0) { return primaryEnriched; @@ -196,10 +189,7 @@ export class OcrMetadataService { const enrichedTexts = ocrResults.map((result, index) => { const enriched = this.enrichWithLayoutMultiPass(result); - const textWithoutHeader = enriched.enrichedText.replace( - /^=== OCR WITH LAYOUT ===\n\n/, - '', - ); + const textWithoutHeader = this.stripLayoutHeader(enriched.enrichedText); const marker = `--- Label ${index + 1} of ${ocrResults.length} ---`; return `${marker}\n${textWithoutHeader}`; }); @@ -342,6 +332,16 @@ export class OcrMetadataService { return 'medium'; } + /** + * Strip the layout header so individual results can be re-wrapped + * with section markers when combining multiple photos. + */ + private stripLayoutHeader(enrichedText: string): string { + return enrichedText.startsWith(LAYOUT_HEADER) + ? enrichedText.slice(LAYOUT_HEADER.length) + : enrichedText; + } + /** * Format enriched blocks as annotated text using markdown. * Each block is prefixed with **SIZE:** tag. @@ -351,7 +351,7 @@ export class OcrMetadataService { return ''; } - const header = '=== OCR WITH LAYOUT ===\n\n'; + const header = LAYOUT_HEADER; const formattedBlocks = blocks.map((block) => { const sizeTag = block.relativeSize.toUpperCase();