diff --git a/src/components/ai-import-photo-gallery/ai-import-photo-gallery.component.ts b/src/components/ai-import-photo-gallery/ai-import-photo-gallery.component.ts index 73bbe3adc..bb2680a02 100644 --- a/src/components/ai-import-photo-gallery/ai-import-photo-gallery.component.ts +++ b/src/components/ai-import-photo-gallery/ai-import-photo-gallery.component.ts @@ -136,10 +136,6 @@ export class AiImportPhotoGalleryComponent { }); if (imageData?.base64String) { - this.uiLog.log( - `AI Import Gallery: Camera returned base64, length: ${imageData.base64String.length}, format: ${imageData.format}`, - ); - // Save to internal storage const fileName = await this.uiFileHelper.generateInternalPath( 'photo', @@ -152,9 +148,6 @@ export class AiImportPhotoGalleryComponent { if (fileUri.path) { this.photoPaths.push(fileUri.path); - this.uiLog.log( - `AI Import Gallery: Saved photo to ${fileUri.path}, total now: ${this.photoPaths.length}`, - ); this.updateSlider(); // Slide to the newly added photo @@ -210,9 +203,6 @@ export class AiImportPhotoGalleryComponent { if (fileUri.path) { this.photoPaths.push(fileUri.path); - this.uiLog.log( - `AI Import Gallery: Clipboard image saved to ${fileUri.path}, total now: ${this.photoPaths.length}`, - ); this.updateSlider(); this.focusOnNewPhoto(); } diff --git a/src/data/ai-import/ai-cloud-prompt.ts b/src/data/ai-import/ai-cloud-prompt.ts index 85c2eec70..91977d7b5 100644 --- a/src/data/ai-import/ai-cloud-prompt.ts +++ b/src/data/ai-import/ai-cloud-prompt.ts @@ -44,7 +44,7 @@ Use "NOT_FOUND" for any field not clearly present. "aromatics": "Comma-separated flavor/tasting notes", "decaffeinated": true or false, "cupping_points": numeric score (typically 80-100), - "roasting_date": "YYYY-MM-DD (use today's date to resolve ambiguous date formats — the roasting date is most likely within the past year and never in the future)", + "roasting_date": "YYYY-MM-DD — the roasting date. Use today's date to resolve ambiguous date formats. If no explicit roast date is found but the label states a best-before/use-by/expiration date AND explicitly states the time between roasting and that date (e.g., 'roasted 12 months before best before date' — note this may be stated in the local language of the label), calculate the roast date by subtracting the stated period from the best-before date. The roasting date is most likely within the past year and never in the future. NOT_FOUND if neither an explicit roast date nor a calculable one is present.", "bean_mix": "SINGLE_ORIGIN or BLEND", "origins": [ { diff --git a/src/services/aiBeanImport/__tests__/cloud-field-extraction.service.spec.ts b/src/services/aiBeanImport/__tests__/cloud-field-extraction.service.spec.ts index 1c3aea7e8..fad20b5e3 100644 --- a/src/services/aiBeanImport/__tests__/cloud-field-extraction.service.spec.ts +++ b/src/services/aiBeanImport/__tests__/cloud-field-extraction.service.spec.ts @@ -252,9 +252,7 @@ describe('CloudFieldExtractionService', () => { await service.extractAllFields('sample OCR text', mockConfig, mockLogger); // Assert - expect(mockLogger.log).toHaveBeenCalledWith( - 'Cloud LLM response received, model: gpt-4o', - ); + expect(mockLogger.log).toHaveBeenCalledWith('[Cloud LLM] model: gpt-4o'); expect(mockLogger.log).toHaveBeenCalledWith( 'Token usage: 100 prompt, 50 completion', ); diff --git a/src/services/aiBeanImport/__tests__/image-rotation.spec.ts b/src/services/aiBeanImport/__tests__/image-rotation.spec.ts new file mode 100644 index 000000000..18e5fe934 --- /dev/null +++ b/src/services/aiBeanImport/__tests__/image-rotation.spec.ts @@ -0,0 +1,48 @@ +import { rotateBase64Image } from '../image-rotation'; + +/** + * Create a small non-square test JPEG as raw base64 using the browser Canvas API. + * Returns a 4×2 red image so rotation visibly swaps dimensions. + */ +function createTestJpegBase64(width = 4, height = 2): string { + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d')!; + ctx.fillStyle = '#ff0000'; + ctx.fillRect(0, 0, width, height); + const dataUrl = canvas.toDataURL('image/jpeg', 0.92); + return dataUrl.replace(/^data:image\/jpeg;base64,/, ''); +} + +describe('rotateBase64Image', () => { + let testBase64: string; + + beforeAll(() => { + testBase64 = createTestJpegBase64(); + }); + + [90, 270].forEach((degrees) => { + it(`should swap dimensions of a 4×2 image to 2×4 and return raw base64 when rotated by ${degrees}°`, async () => { + const result = await rotateBase64Image(testBase64, degrees as 90 | 270); + expect(result).toBeTruthy(); + expect(result).not.toContain('data:image/jpeg;base64,'); + + const img = await new Promise((resolve, reject) => { + const image = new Image(); + image.onload = () => resolve(image); + image.onerror = () => reject(new Error('Failed to load rotated image')); + image.src = `data:image/jpeg;base64,${result}`; + }); + + expect(img.naturalWidth).toBe(2); + expect(img.naturalHeight).toBe(4); + }); + }); + + it('should reject with an error when given invalid base64', async () => { + await expectAsync( + rotateBase64Image('not-a-valid-image!!!', 90), + ).toBeRejectedWithError('Failed to load image for rotation'); + }); +}); diff --git a/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts b/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts index 9296db916..62caedb52 100644 --- a/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts +++ b/src/services/aiBeanImport/__tests__/ocr-metadata.service.spec.ts @@ -1,11 +1,16 @@ import { TestBed } from '@angular/core/testing'; +import { MultiPassOcrResult } from '../camera-ocr.service'; import { Block, OcrMetadataService, TextDetectionResult, } from '../ocr-metadata.service'; -import { createBlock, createTextDetectionResult } from '../test-utils'; +import { + createBlock, + createLine, + createTextDetectionResult, +} from '../test-utils'; describe('OcrMetadataService', () => { let service: OcrMetadataService; @@ -55,18 +60,31 @@ describe('OcrMetadataService', () => { }); }); - it('should return true when blocks have significant size variation (headers vs body text)', () => { - // WHY: Size variation indicates visual hierarchy useful for field extraction + it('should return true when representative line heights show significant variation', () => { + // WHY: Block bounding boxes can be similar in height (heading 80px, paragraph 90px) + // but average line heights reveal the real font size difference (80px vs 15px). // Arrange - const largeTitle = createBlock('Big Title', 0, 0, 200, 100); // Height: 100 - const smallBody = createBlock('Small text', 0, 110, 200, 130); // Height: 20 - const result = createTextDetectionResult('Big Title\nSmall text', [ - largeTitle, - smallBody, + const heading = createBlock('Big Title', 0, 0, 400, 80, { + lines: [createLine('Big Title', 0, 0, 400, 80)], // 1 line, avg h=80 + }); + const paragraph = createBlock('Small text body', 0, 100, 400, 190, { + lines: [ + createLine('Small text', 0, 100, 400, 115), // h=15 + createLine('body here', 0, 115, 400, 130), // h=15 + createLine('more text', 0, 130, 400, 145), // h=15 + createLine('continues', 0, 145, 400, 160), // h=15 + createLine('even more', 0, 160, 400, 175), // h=15 + createLine('last line', 0, 175, 400, 190), // h=15 + ], // 6 lines, avg h=15 + }); + const result = createTextDetectionResult('Big Title\nSmall text body', [ + heading, + paragraph, ]); - // Act & Assert + // Act & Assert — block heights 80 vs 90 (ratio 1.125 < 1.3 threshold) + // but representative heights 80 vs 15 (ratio 5.33 > 1.3 threshold) expect(service.shouldUseMetadata(result)).toBeTrue(); }); }); @@ -109,60 +127,6 @@ describe('OcrMetadataService', () => { expect(enriched.enrichedText).toContain('Coffee Name'); expect(enriched.enrichedText).toContain('Details'); }); - - it('should classify blocks by relative height into LARGE and SMALL categories', () => { - // WHY: Size classification helps LLM identify headers vs body text - - // Arrange - const largeBlock = createBlock('BIG', 0, 0, 200, 100); // Height: 100 - const smallBlock = createBlock('small', 0, 120, 200, 140); // Height: 20 - const result = createTextDetectionResult('BIG\nsmall', [ - largeBlock, - smallBlock, - ]); - - // Act - const enriched = service.enrichWithLayout(result); - - // Assert - expect(enriched.enrichedText).toContain('**LARGE:**'); - expect(enriched.enrichedText).toContain('**SMALL:**'); - }); - - it('should classify text sizes as LARGE, MEDIUM, and SMALL based on height variation', () => { - // Arrange - const result = createTextDetectionResult('Top\nMiddle\nBottom', [ - createBlock('Top', 0, 0, 100, 80), // Height: 80 (large) - createBlock('Middle', 0, 100, 100, 150), // Height: 50 (medium) - createBlock('Bottom', 0, 220, 100, 240), // Height: 20 (small) - ]); - - // Act - const enriched = service.enrichWithLayout(result); - - // Assert - expect(enriched.enrichedText).toContain('**LARGE:**'); - expect(enriched.enrichedText).toContain('Top'); - expect(enriched.enrichedText).toContain('Middle'); - expect(enriched.enrichedText).toContain('Bottom'); - }); - - it('should include all block texts in output regardless of position', () => { - // Arrange - const result = createTextDetectionResult('Left\nCenter\nRight', [ - createBlock('Left', 0, 0, 50, 80), - createBlock('Center', 100, 0, 200, 80), - createBlock('Right', 250, 0, 300, 80), - ]); - - // Act - const enriched = service.enrichWithLayout(result); - - // Assert - expect(enriched.enrichedText).toContain('Left'); - expect(enriched.enrichedText).toContain('Center'); - expect(enriched.enrichedText).toContain('Right'); - }); }); describe('enrichMultiplePhotos', () => { @@ -236,6 +200,213 @@ describe('OcrMetadataService', () => { }); }); + describe('enrichWithLayoutMultiPass', () => { + it('should return output identical to enrichWithLayout when no rotated results', () => { + // Arrange + const primary = createTextDetectionResult('Roaster\nCoffee Name', [ + createBlock('Roaster', 0, 0, 200, 80), + createBlock('Coffee Name', 0, 100, 200, 140), + ]); + const multiPass: MultiPassOcrResult = { primary, rotated: [] }; + + // Act + const multiPassResult = service.enrichWithLayoutMultiPass(multiPass); + const singlePassResult = service.enrichWithLayout(primary); + + // Assert + expect(multiPassResult.enrichedText).toBe(singlePassResult.enrichedText); + expect(multiPassResult.rawText).toBe(singlePassResult.rawText); + }); + + // 0° pass: avg height ~60, max height 100 + const classificationCases = [ + { + height: 90, + expected: 'LARGE', + label: 'Big Rotated', + reason: '≥ 0.8 × 100 = 80', + }, + { + height: 30, + expected: 'SMALL', + label: 'Espresso', + reason: '< 0.7 × 60 = 42', + }, + { + height: 55, + expected: 'MEDIUM', + label: 'Filter', + reason: 'between thresholds', + }, + ]; + classificationCases.forEach(({ height, expected, label, reason }) => { + it(`should classify rotated text (h=${height}) as ${expected} using 0° pass stats (${reason})`, () => { + // Arrange + const primary = createTextDetectionResult('Front', [ + createBlock('Title', 0, 0, 300, 100), // Height: 100 + createBlock('Body', 0, 120, 300, 140), // Height: 20 + ]); + const rotated = createTextDetectionResult(label, [ + createBlock(label, 0, 0, 200, height), + ]); + const multiPass: MultiPassOcrResult = { + primary, + rotated: [rotated], + }; + + // Act + const result = service.enrichWithLayoutMultiPass(multiPass); + + // Assert + expect(result.hasUsefulMetadata).toBeTrue(); + const rotatedSection = result.enrichedText.split( + '--- Rotated text detected ---', + )[1]; + expect(rotatedSection).toContain(`**${expected}:** ${label}`); + }); + }); + + it('should fall back to independent classification when baseline has too few blocks', () => { + // Arrange: primary has only 1 block (below OCR_MIN_BLOCKS_FOR_METADATA) + const primary = createTextDetectionResult('Only', [ + createBlock('Only', 0, 0, 100, 50), + ]); + const rotated = createTextDetectionResult('Rotated', [ + createBlock('Big', 0, 0, 200, 100), + createBlock('Small', 0, 120, 200, 130), + ]); + const multiPass: MultiPassOcrResult = { + primary, + rotated: [rotated], + }; + + // Act + const result = service.enrichWithLayoutMultiPass(multiPass); + + // Assert - should not throw and should have rotated section + expect(result.enrichedText).toContain('--- Rotated text detected ---'); + expect(result.enrichedText).toContain('Big'); + expect(result.enrichedText).toContain('Small'); + }); + + it('should not append rotated section when rotated results have empty blocks', () => { + // Arrange + const primary = createTextDetectionResult('Front', [ + createBlock('Title', 0, 0, 300, 100), + createBlock('Body', 0, 120, 300, 140), + ]); + const rotated = createTextDetectionResult('', []); + const multiPass: MultiPassOcrResult = { + primary, + rotated: [rotated], + }; + + // Act + const result = service.enrichWithLayoutMultiPass(multiPass); + + // Assert + expect(result.enrichedText).not.toContain( + '--- Rotated text detected ---', + ); + }); + }); + + describe('enrichMultiplePhotosMultiPass', () => { + it('should add "Label N of M" markers for multiple results with rotated sections', () => { + // Arrange + const photo1Primary = createTextDetectionResult('Photo1', [ + createBlock('Roaster A', 0, 0, 300, 100), + createBlock('details', 0, 120, 300, 140), + ]); + const photo1Rotated = createTextDetectionResult('Side A', [ + createBlock('Side A', 0, 0, 200, 60), + ]); + const photo1: MultiPassOcrResult = { + primary: photo1Primary, + rotated: [photo1Rotated], + }; + + const photo2Primary = createTextDetectionResult('Photo2', [ + createBlock('Roaster B', 0, 0, 300, 100), + createBlock('info', 0, 120, 300, 140), + ]); + const photo2: MultiPassOcrResult = { + primary: photo2Primary, + rotated: [], + }; + + // Act + const result = service.enrichMultiplePhotosMultiPass([photo1, photo2]); + + // Assert + expect(result).toContain('=== OCR WITH LAYOUT ==='); + expect(result).toContain('--- Label 1 of 2 ---'); + expect(result).toContain('--- Label 2 of 2 ---'); + expect(result).toContain('--- Rotated text detected ---'); + expect(result).toContain('Side A'); + expect(result).toContain('Roaster B'); + }); + }); + + describe('line-height-based classification', () => { + it('should classify a multi-line block by average line height, not total block height', () => { + // WHY: A description paragraph has many small-font lines but a tall bounding box. + // Using block height would misclassify it as LARGE. + + // Arrange + const heading = createBlock('NATURAL BLEND', 0, 0, 400, 80, { + lines: [createLine('NATURAL BLEND', 0, 0, 400, 80)], + }); + + // 6 lines of small font (line height ~15 each), total block height = 90 + const description = createBlock( + 'Unser Natural Blend ist eine Mischung aus natürlich aufbereiteten Kaffees', + 0, + 100, + 400, + 190, + { + lines: [ + createLine('Unser Natural Blend', 0, 100, 400, 115), + createLine('ist eine Mischung aus', 0, 115, 400, 130), + createLine('natürlich aufbe-', 0, 130, 400, 145), + createLine('reiteten Kaffees', 0, 145, 400, 160), + createLine('aus Brasilien und', 0, 160, 400, 175), + createLine('Äthiopien.', 0, 175, 400, 190), + ], + }, + ); + + const result = createTextDetectionResult('', [heading, description]); + + // Act + const enriched = service.enrichWithLayout(result); + + // Assert — heading is large font, description is small font + expect(enriched.enrichedText).toContain('**LARGE:** NATURAL BLEND'); + expect(enriched.enrichedText).not.toContain( + '**LARGE:** Unser Natural Blend', + ); + }); + + it('should fall back to block height when block has no lines', () => { + // WHY: Backward compatibility — blocks without line data (e.g., from test helpers) + // should still classify based on block bounding box height. + + // Arrange + const largeBlock = createBlock('BIG', 0, 0, 200, 100); // Height: 100, no lines + const smallBlock = createBlock('small', 0, 120, 200, 140); // Height: 20, no lines + const result = createTextDetectionResult('', [largeBlock, smallBlock]); + + // Act + const enriched = service.enrichWithLayout(result); + + // Assert + expect(enriched.enrichedText).toContain('**LARGE:** BIG'); + expect(enriched.enrichedText).toContain('**SMALL:** small'); + }); + }); + describe('edge cases', () => { it('should handle zero height blocks gracefully without dividing by zero', () => { // WHY: ML Kit sometimes returns blocks with zero height for malformed OCR diff --git a/src/services/aiBeanImport/apple-intelligence-ai-bean-import.service.ts b/src/services/aiBeanImport/apple-intelligence-ai-bean-import.service.ts index ac583eeb7..3b4e7cb57 100644 --- a/src/services/aiBeanImport/apple-intelligence-ai-bean-import.service.ts +++ b/src/services/aiBeanImport/apple-intelligence-ai-bean-import.service.ts @@ -15,13 +15,10 @@ import { UIAlert } from '../uiAlert'; import { UILog } from '../uiLog'; import { AIImportStep, createAIBeanImportError } from './ai-bean-import-error'; import { AIImportExamplesService } from './ai-import-examples.service'; -import { CameraOcrService } from './camera-ocr.service'; +import { CameraOcrService, MultiPassOcrResult } from './camera-ocr.service'; import { FieldExtractionService } from './field-extraction.service'; import { sendLLMPrompt } from './llm-communication.service'; -import { - OcrMetadataService, - TextDetectionResult, -} from './ocr-metadata.service'; +import { OcrMetadataService } from './ocr-metadata.service'; export interface AIReadinessResult { ready: boolean; @@ -189,16 +186,14 @@ export class AppleIntelligenceAIBeanImportService { * Takes OCR result(s), enriches with layout, detects language, and extracts fields. */ private async processOcrAndExtractBean( - ocrResults: TextDetectionResult[], + ocrResults: MultiPassOcrResult[], rawTexts: string[], ): Promise { - // Step 1: Enrich with layout metadata + // Step 1: Enrich with layout metadata (multi-pass aware) const enrichedText = ocrResults.length === 1 - ? this.ocrMetadata.enrichWithLayout(ocrResults[0]).enrichedText - : this.ocrMetadata.enrichMultiplePhotos(ocrResults); - - this.uiLog.log(`Enriched text length: ${enrichedText.length}`); + ? this.ocrMetadata.enrichWithLayoutMultiPass(ocrResults[0]).enrichedText + : this.ocrMetadata.enrichMultiplePhotosMultiPass(ocrResults); // Step 2: Prepare raw text for language detection const combinedRawText = this.concatenateOCRResults(rawTexts); @@ -216,10 +211,6 @@ export class AppleIntelligenceAIBeanImportService { detectedLanguage, userLanguage, ); - this.uiLog.log( - 'Using languages for examples: ' + languagesToUse.join(', '), - ); - // Step 5: Extract fields this.uiAlert.setLoadingSpinnerMessage( this.translate.instant('AI_IMPORT_STEP_ANALYZING'), diff --git a/src/services/aiBeanImport/camera-ocr.service.ts b/src/services/aiBeanImport/camera-ocr.service.ts index ac313158b..aa3b90be3 100644 --- a/src/services/aiBeanImport/camera-ocr.service.ts +++ b/src/services/aiBeanImport/camera-ocr.service.ts @@ -13,18 +13,40 @@ import { UIAlert } from '../uiAlert'; import { UIFileHelper } from '../uiFileHelper'; import { UIImage } from '../uiImage'; import { UILog } from '../uiLog'; +import { rotateBase64Image } from './image-rotation'; import { TextDetectionResult } from './ocr-metadata.service'; +/** + * Result of multi-pass OCR on a single image. + * The primary result (0°) is always present. + * Rotated results (90°/270°) are only present when they found text. + */ +export interface MultiPassOcrResult { + /** OCR result from the original (0°) image */ + primary: TextDetectionResult; + /** OCR results from rotated passes (90°, 270°) — only those that found text */ + rotated: TextDetectionResult[]; +} + export interface SinglePhotoCaptureResult { - ocrResult: TextDetectionResult; + ocrResult: MultiPassOcrResult; rawText: string; } export interface MultiPhotoOcrResult { - ocrResults: TextDetectionResult[]; + ocrResults: MultiPassOcrResult[]; rawTexts: string[]; } +/** + * Combine all text from a multi-pass OCR result (primary + rotated passes). + */ +export function collectRawText(result: MultiPassOcrResult): string { + return [result.primary.text, ...result.rotated.map((r) => r.text)] + .filter((t) => t.trim()) + .join('\n'); +} + @Injectable({ providedIn: 'root', }) @@ -63,24 +85,13 @@ export class CameraOcrService { await this.uiAlert.hideLoadingSpinner(); return null; } - this.uiLog.log( - 'CameraOcr: Photo captured, base64 length: ' + - imageData.base64String.length, - ); - this.uiAlert.setLoadingSpinnerMessage( this.translate.instant('AI_IMPORT_STEP_EXTRACTING'), ); - const ocrResult = (await CapacitorPluginMlKitTextRecognition.detectText({ - base64Image: imageData.base64String, - })) as TextDetectionResult; - const rawText = ocrResult.text; - this.uiLog.log( - 'CameraOcr: OCR result: ' + JSON.stringify(ocrResult).substring(0, 500), - ); - - if (!rawText || rawText.trim() === '') { + const ocrResult = await this.ocrWithRotations(imageData.base64String); + const rawText = collectRawText(ocrResult); + if (!rawText) { await this.uiAlert.hideLoadingSpinner(); await this.uiAlert.showMessage( 'AI_IMPORT_NO_TEXT_FOUND', @@ -102,7 +113,7 @@ export class CameraOcrService { * Returns only photos that produced text. */ async ocrFromPhotoPaths(photoPaths: string[]): Promise { - const ocrResults: TextDetectionResult[] = []; + const ocrResults: MultiPassOcrResult[] = []; const rawTexts: string[] = []; for (let i = 0; i < photoPaths.length; i++) { @@ -114,16 +125,10 @@ export class CameraOcrService { ); const photoPath = photoPaths[i]; - this.uiLog.log( - `CameraOcr: Processing photo ${i + 1}/${photoPaths.length}, path: ${photoPath}`, - ); let base64: string; try { base64 = await this.uiFileHelper.readInternalFileAsBase64(photoPath); - this.uiLog.log( - `CameraOcr: Photo ${i + 1} read successfully, base64 length: ${base64.length}`, - ); } catch (readError: unknown) { const msg = readError instanceof Error ? readError.message : String(readError); @@ -141,22 +146,12 @@ export class CameraOcrService { } try { - const ocrResult = (await CapacitorPluginMlKitTextRecognition.detectText( - { - base64Image: base64, - }, - )) as TextDetectionResult; - - this.uiLog.log( - `CameraOcr: Photo ${i + 1} OCR result: ${JSON.stringify(ocrResult).substring(0, 200)}`, - ); + const ocrResult = await this.ocrWithRotations(base64); - if (ocrResult.text && ocrResult.text.trim() !== '') { + const rawText = collectRawText(ocrResult); + if (rawText) { ocrResults.push(ocrResult); - rawTexts.push(ocrResult.text); - this.uiLog.log( - `CameraOcr: Photo ${i + 1} extracted ${ocrResult.text.length} chars`, - ); + rawTexts.push(rawText); } else { this.uiLog.log(`CameraOcr: Photo ${i + 1} had no text`); } @@ -171,6 +166,40 @@ export class CameraOcrService { return { ocrResults, rawTexts }; } + /** + * Run OCR on a base64 image at 0°, 90°, and 270°. + * Returns the primary (0°) result and any rotated results that found text. + */ + private async ocrWithRotations(base64: string): Promise { + // Primary pass (0°) + // ML Kit plugin returns this shape but its TS types are wider + const primary = (await CapacitorPluginMlKitTextRecognition.detectText({ + base64Image: base64, + })) as TextDetectionResult; + + const rotated: TextDetectionResult[] = []; + + // Rotated passes — only if primary succeeded (image is valid) + for (const degrees of [90, 270] as const) { + try { + const rotatedBase64 = await rotateBase64Image(base64, degrees); + // ML Kit plugin returns this shape but its TS types are wider + const result = (await CapacitorPluginMlKitTextRecognition.detectText({ + base64Image: rotatedBase64, + })) as TextDetectionResult; + + if (result.text && result.text.trim() !== '') { + rotated.push(result); + } + } catch (error: unknown) { + const msg = error instanceof Error ? error.message : String(error); + this.uiLog.error(`CameraOcr: Rotated ${degrees}° pass failed: ${msg}`); + } + } + + return { primary, rotated }; + } + /** * Delete temporary photo files. */ @@ -178,7 +207,6 @@ export class CameraOcrService { for (const path of photoPaths) { try { await this.uiFileHelper.deleteInternalFile(path); - this.uiLog.log('CameraOcr: Deleted temp photo: ' + path); } catch (e) { this.uiLog.error('CameraOcr: Failed to delete temp photo: ' + e); } diff --git a/src/services/aiBeanImport/cloud-ai-bean-import.service.ts b/src/services/aiBeanImport/cloud-ai-bean-import.service.ts index fd949ed1a..67f97f8a2 100644 --- a/src/services/aiBeanImport/cloud-ai-bean-import.service.ts +++ b/src/services/aiBeanImport/cloud-ai-bean-import.service.ts @@ -9,12 +9,9 @@ import { UILog } from '../uiLog'; import { UISettingsStorage } from '../uiSettingsStorage'; import { AIImportStep, createAIBeanImportError } from './ai-bean-import-error'; import { AIReadinessResult } from './apple-intelligence-ai-bean-import.service'; -import { CameraOcrService } from './camera-ocr.service'; +import { CameraOcrService, MultiPassOcrResult } from './camera-ocr.service'; import { CloudFieldExtractionService } from './cloud-field-extraction.service'; -import { - OcrMetadataService, - TextDetectionResult, -} from './ocr-metadata.service'; +import { OcrMetadataService } from './ocr-metadata.service'; @Injectable({ providedIn: 'root', @@ -166,15 +163,13 @@ export class CloudAIBeanImportService { * Enriches OCR results with layout metadata, then sends to cloud LLM. */ private async processOcrAndExtractBean( - ocrResults: TextDetectionResult[], + ocrResults: MultiPassOcrResult[], ): Promise { - // Step 1: Enrich with layout metadata + // Step 1: Enrich with layout metadata (multi-pass aware) const enrichedText = ocrResults.length === 1 - ? this.ocrMetadata.enrichWithLayout(ocrResults[0]).enrichedText - : this.ocrMetadata.enrichMultiplePhotos(ocrResults); - - this.uiLog.log(`Cloud AI: Enriched text length: ${enrichedText.length}`); + ? this.ocrMetadata.enrichWithLayoutMultiPass(ocrResults[0]).enrichedText + : this.ocrMetadata.enrichMultiplePhotosMultiPass(ocrResults); // Step 2: Extract fields via cloud LLM (no language detection or vocabulary needed) this.uiAlert.setLoadingSpinnerMessage( diff --git a/src/services/aiBeanImport/cloud-field-extraction.service.ts b/src/services/aiBeanImport/cloud-field-extraction.service.ts index 032d19769..e051b4371 100644 --- a/src/services/aiBeanImport/cloud-field-extraction.service.ts +++ b/src/services/aiBeanImport/cloud-field-extraction.service.ts @@ -66,12 +66,15 @@ export class CloudFieldExtractionService { const userPrompt = buildCloudExtractionPrompt(ocrText); // 3. Send to cloud LLM — throws on API errors, timeouts, network failures + log.log('[Cloud LLM] model: ' + config.model); + log.log('[Cloud LLM] prompt: ' + userPrompt); + const response = await sendCloudLLMPrompt(config, [ { role: 'system', content: CLOUD_BEAN_IMPORT_SYSTEM_INSTRUCTIONS }, { role: 'user', content: userPrompt }, ]); - log.log('Cloud LLM response received, model: ' + response.model); + log.log('[Cloud LLM] response: ' + response.content); if (response.usage) { log.log( `Token usage: ${response.usage.prompt_tokens} prompt, ${response.usage.completion_tokens} completion`, diff --git a/src/services/aiBeanImport/field-extraction.service.ts b/src/services/aiBeanImport/field-extraction.service.ts index 25bd73f37..adca589a4 100644 --- a/src/services/aiBeanImport/field-extraction.service.ts +++ b/src/services/aiBeanImport/field-extraction.service.ts @@ -82,7 +82,6 @@ export class FieldExtractionService { // Pre-process text const text = this.preProcess(ocrText, examples); - this.uiLog.debug('Normalized text length: ' + text.length); // Extract fields in two phases const topLevelFields = await this.extractTopLevelFields( @@ -311,8 +310,6 @@ export class FieldExtractionService { const response = await this.sendLLMMessage(prompt); const cleaned = this.cleanResponse(response); - this.uiLog.debug(fieldName + ' response: ' + cleaned); - // Handle null/not found responses (exact match only - partial NOT_FOUND handled by postProcess) if (isNullLikeValue(cleaned)) { return null; @@ -358,8 +355,6 @@ export class FieldExtractionService { // Don't use cleanResponse - it strips colons which breaks JSON const trimmed = response?.trim() || ''; - this.uiLog.debug('name_and_roaster response: ' + trimmed); - // Parse JSON response return this.parseNameAndRoasterResponse(trimmed); } catch (error) { @@ -467,8 +462,6 @@ export class FieldExtractionService { // Note: Don't use cleanResponse here - it strips colons which breaks JSON syntax const trimmed = response?.trim() || ''; - this.uiLog.debug('Blend origins response: ' + trimmed); - // Parse JSON response (handles markdown code blocks internally) return this.parseBlendOriginsResponse(trimmed); } diff --git a/src/services/aiBeanImport/image-rotation.ts b/src/services/aiBeanImport/image-rotation.ts new file mode 100644 index 000000000..811d883ec --- /dev/null +++ b/src/services/aiBeanImport/image-rotation.ts @@ -0,0 +1,28 @@ +/** + * Rotate a base64-encoded image by the given degrees using the Canvas API. + * + * @param base64 Raw base64 string (no data URL prefix) + * @param degrees Clockwise rotation: 90 or 270 + * @returns Rotated image as raw base64 string (no data URL prefix) + */ +export function rotateBase64Image( + base64: string, + degrees: 90 | 270, +): Promise { + return new Promise((resolve, reject) => { + const img = new Image(); + img.onload = () => { + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + canvas.width = img.height; + canvas.height = img.width; + ctx?.translate(canvas.width / 2, canvas.height / 2); + ctx?.rotate((degrees * Math.PI) / 180); + ctx?.drawImage(img, -img.width / 2, -img.height / 2); + const dataUrl = canvas.toDataURL('image/jpeg', 0.92); + resolve(dataUrl.replace(/^data:image\/jpeg;base64,/, '')); + }; + img.onerror = () => reject(new Error('Failed to load image for rotation')); + img.src = 'data:image/jpeg;base64,' + base64; + }); +} diff --git a/src/services/aiBeanImport/llm-communication.service.ts b/src/services/aiBeanImport/llm-communication.service.ts index cf96549da..0c249944c 100644 --- a/src/services/aiBeanImport/llm-communication.service.ts +++ b/src/services/aiBeanImport/llm-communication.service.ts @@ -28,7 +28,11 @@ export interface LLMCommunicationOptions { /** System-level instructions passed to the LLM session (higher priority than prompt content) */ instructions?: string; /** Logger instance for error reporting */ - logger?: { error: (msg: string) => void; log: (msg: string) => void }; + logger?: { + error: (msg: string) => void; + log: (msg: string) => void; + debug?: (msg: string) => void; + }; } /** @@ -115,7 +119,10 @@ export async function sendLLMPrompt( await finishedListener?.remove(); await LLM.deleteChat({ chatId }); } catch (e) { - logger?.error('Error cleaning up listeners/chat: ' + e); + // Suppress "not implemented" error from deleteChat on iOS + if (!String(e).includes('not implemented')) { + logger?.debug?.('Cleanup warning: ' + e); + } } }; @@ -128,6 +135,7 @@ export async function sendLLMPrompt( const resolveOnce = (value: string) => { if (!resolved) { resolved = true; + logger?.log('[Apple Intelligence] response: ' + value); void cleanup(); resolvePromise(value); } @@ -153,6 +161,7 @@ export async function sendLLMPrompt( }, timeoutMs); // Send message (don't await - let the promise handle completion) + logger?.log('[Apple Intelligence] prompt: ' + prompt); void LLM.sendMessage({ chatId, message: prompt, diff --git a/src/services/aiBeanImport/ocr-metadata.service.ts b/src/services/aiBeanImport/ocr-metadata.service.ts index bc8befbc6..da2d693d9 100644 --- a/src/services/aiBeanImport/ocr-metadata.service.ts +++ b/src/services/aiBeanImport/ocr-metadata.service.ts @@ -7,6 +7,7 @@ import { OCR_SIZE_VARIATION_THRESHOLD, OCR_SMALL_TEXT_AVG_MULTIPLIER, } from '../../data/ai-import/ai-import-constants'; +import { MultiPassOcrResult } from './camera-ocr.service'; /** * Interfaces matching the ML Kit plugin's TypeScript definitions. @@ -70,6 +71,8 @@ export interface EnrichedOCRResult { * field extraction accuracy. Converts spatial information into text annotations * that help the LLM understand layout context. */ +const LAYOUT_HEADER = '=== OCR WITH LAYOUT ===\n\n'; + @Injectable({ providedIn: 'root', }) @@ -122,11 +125,71 @@ export class OcrMetadataService { // Process each photo and combine with markers const enrichedTexts = ocrResults.map((result, index) => { const enriched = this.enrichWithLayout(result); - // Strip header from individual results to avoid duplication - const textWithoutHeader = enriched.enrichedText.replace( - /^=== OCR WITH LAYOUT ===\n\n/, - '', - ); + const textWithoutHeader = this.stripLayoutHeader(enriched.enrichedText); + const marker = `--- Label ${index + 1} of ${ocrResults.length} ---`; + return `${marker}\n${textWithoutHeader}`; + }); + + return '=== OCR WITH LAYOUT ===\n\n' + enrichedTexts.join('\n\n'); + } + + /** + * Enrich a multi-pass OCR result (single photo, multiple rotation passes). + * Classifies rotated-pass blocks using 0° pass height statistics as baseline. + * Only appends rotated text section if rotated passes found text. + */ + public enrichWithLayoutMultiPass( + multiPass: MultiPassOcrResult, + ): EnrichedOCRResult { + // Enrich the primary (0°) pass as before + const primaryEnriched = this.enrichWithLayout(multiPass.primary); + + // Collect all rotated blocks — return primary as-is if none found + const rotatedBlocks = multiPass.rotated.flatMap((r) => r.blocks ?? []); + if (rotatedBlocks.length === 0) { + return primaryEnriched; + } + + // Classify rotated blocks using 0° pass stats as baseline + const rotatedEnriched = this.classifyBlocksWithBaseline( + rotatedBlocks, + multiPass.primary.blocks, + ); + + // Format rotated blocks + const rotatedText = this.formatRotatedText(rotatedEnriched); + + // Combine: primary text + rotated section + const combinedText = primaryEnriched.enrichedText + '\n\n' + rotatedText; + const combinedRawText = + primaryEnriched.rawText + + '\n' + + multiPass.rotated.map((r) => r.text).join('\n'); + + return { + rawText: combinedRawText, + enrichedText: combinedText, + hasUsefulMetadata: true, + }; + } + + /** + * Process multiple multi-pass OCR results (multi-photo flow) with layout enrichment. + */ + public enrichMultiplePhotosMultiPass( + ocrResults: MultiPassOcrResult[], + ): string { + if (ocrResults.length === 0) { + return ''; + } + + if (ocrResults.length === 1) { + return this.enrichWithLayoutMultiPass(ocrResults[0]).enrichedText; + } + + const enrichedTexts = ocrResults.map((result, index) => { + const enriched = this.enrichWithLayoutMultiPass(result); + const textWithoutHeader = this.stripLayoutHeader(enriched.enrichedText); const marker = `--- Label ${index + 1} of ${ocrResults.length} ---`; return `${marker}\n${textWithoutHeader}`; }); @@ -156,10 +219,9 @@ export class OcrMetadataService { return false; } - // Check for size variation - // Use absolute value since coordinate system may have Y origin at bottom (top > bottom) + // Check for size variation using representative heights (avg line height) const heights = blocksWithBoundingBox.map((b) => - Math.abs(b.boundingBox.bottom - b.boundingBox.top), + this.getRepresentativeHeight(b), ); const maxHeight = Math.max(...heights); const minHeight = Math.min(...heights); @@ -187,16 +249,17 @@ export class OcrMetadataService { } /** - * Classify blocks by relative size based on height. - * Uses statistical thresholds relative to average and max. + * Classify blocks by relative size based on representative line height. + * Uses average line height within each block as a proxy for font size. + * Falls back to block bounding box height when line data is unavailable. */ private classifySizes(blocks: Block[]): Map { const sizeMap = new Map(); - // Calculate height of each block (use abs since Y origin may be at bottom) + // Calculate representative height of each block const heights = blocks.map((b) => ({ block: b, - height: Math.abs(b.boundingBox.bottom - b.boundingBox.top), + height: this.getRepresentativeHeight(b), })); if (heights.length === 0) { @@ -209,26 +272,76 @@ export class OcrMetadataService { const avgHeight = heightValues.reduce((a, b) => a + b, 0) / heightValues.length; - // Classify each block - // Large: >= OCR_LARGE_TEXT_AVG_MULTIPLIER * average OR within OCR_LARGE_TEXT_MAX_HEIGHT_RATIO of max - // Small: < OCR_SMALL_TEXT_AVG_MULTIPLIER * average - // Medium: everything else for (const { block, height } of heights) { - if ( - height >= maxHeight * OCR_LARGE_TEXT_MAX_HEIGHT_RATIO || - height >= avgHeight * OCR_LARGE_TEXT_AVG_MULTIPLIER - ) { - sizeMap.set(block, 'large'); - } else if (height < avgHeight * OCR_SMALL_TEXT_AVG_MULTIPLIER) { - sizeMap.set(block, 'small'); - } else { - sizeMap.set(block, 'medium'); - } + sizeMap.set(block, this.classifyHeight(height, maxHeight, avgHeight)); } return sizeMap; } + /** + * Height of a bounding box, using absolute value since coordinate system + * may have Y origin at bottom (top > bottom). + */ + private boundingBoxHeight(box: BoundingBox): number { + return Math.abs(box.bottom - box.top); + } + + /** + * Get the representative height for a block, approximating font size. + * Uses average line height when line bounding boxes are available, + * otherwise falls back to the block's own bounding box height. + */ + private getRepresentativeHeight(block: Block): number { + if (block.lines && block.lines.length > 0) { + const lineHeights = block.lines + .filter( + (l) => l.boundingBox && typeof l.boundingBox.bottom === 'number', + ) + .map((l) => this.boundingBoxHeight(l.boundingBox)); + + if (lineHeights.length > 0) { + return lineHeights.reduce((a, b) => a + b, 0) / lineHeights.length; + } + } + + // Fallback: use block bounding box height + return this.boundingBoxHeight(block.boundingBox); + } + + /** + * Classify a single block height against statistical thresholds. + * Large: >= OCR_LARGE_TEXT_AVG_MULTIPLIER × avg OR within OCR_LARGE_TEXT_MAX_HEIGHT_RATIO of max + * Small: < OCR_SMALL_TEXT_AVG_MULTIPLIER × avg + * Medium: everything else + */ + private classifyHeight( + height: number, + maxHeight: number, + avgHeight: number, + ): TextSize { + if ( + height >= maxHeight * OCR_LARGE_TEXT_MAX_HEIGHT_RATIO || + height >= avgHeight * OCR_LARGE_TEXT_AVG_MULTIPLIER + ) { + return 'large'; + } + if (height < avgHeight * OCR_SMALL_TEXT_AVG_MULTIPLIER) { + return 'small'; + } + return 'medium'; + } + + /** + * Strip the layout header so individual results can be re-wrapped + * with section markers when combining multiple photos. + */ + private stripLayoutHeader(enrichedText: string): string { + return enrichedText.startsWith(LAYOUT_HEADER) + ? enrichedText.slice(LAYOUT_HEADER.length) + : enrichedText; + } + /** * Format enriched blocks as annotated text using markdown. * Each block is prefixed with **SIZE:** tag. @@ -238,8 +351,56 @@ export class OcrMetadataService { return ''; } - const header = '=== OCR WITH LAYOUT ===\n\n'; + const header = LAYOUT_HEADER; + + const formattedBlocks = blocks.map((block) => { + const sizeTag = block.relativeSize.toUpperCase(); + return `**${sizeTag}:** ${block.text}`; + }); + + return header + formattedBlocks.join('\n\n'); + } + + /** + * Classify blocks using external baseline statistics (from the 0° pass). + * Falls back to per-pass independent classification if baseline is insufficient. + */ + private classifyBlocksWithBaseline( + blocks: Block[], + baselineBlocks: Block[], + ): EnrichedTextBlock[] { + // Compute baseline stats from 0° pass using representative heights + const baselineHeights = (baselineBlocks ?? []) + .filter((b) => b.boundingBox && typeof b.boundingBox.bottom === 'number') + .map((b) => this.getRepresentativeHeight(b)); + + // If baseline is insufficient, fall back to self-contained classification + if (baselineHeights.length < OCR_MIN_BLOCKS_FOR_METADATA) { + return this.classifyBlocks(blocks); + } + + const maxHeight = Math.max(...baselineHeights); + const avgHeight = + baselineHeights.reduce((a, b) => a + b, 0) / baselineHeights.length; + + return blocks.map((block) => { + const height = this.getRepresentativeHeight(block); + return { + text: block.text, + relativeSize: this.classifyHeight(height, maxHeight, avgHeight), + }; + }); + } + + /** + * Format rotated text blocks with a section header. + */ + private formatRotatedText(blocks: EnrichedTextBlock[]): string { + if (blocks.length === 0) { + return ''; + } + const header = '--- Rotated text detected ---\n'; const formattedBlocks = blocks.map((block) => { const sizeTag = block.relativeSize.toUpperCase(); return `**${sizeTag}:** ${block.text}`; diff --git a/src/services/aiBeanImport/test-utils/ai-import-test-helpers.ts b/src/services/aiBeanImport/test-utils/ai-import-test-helpers.ts index d1d9b22f1..af51c7ba6 100644 --- a/src/services/aiBeanImport/test-utils/ai-import-test-helpers.ts +++ b/src/services/aiBeanImport/test-utils/ai-import-test-helpers.ts @@ -2,7 +2,12 @@ import { Bean } from '../../../classes/bean/bean'; import { IBeanInformation } from '../../../interfaces/bean/iBeanInformation'; import { IBeanParameter } from '../../../interfaces/parameter/iBeanParameter'; import { MergedExamples } from '../ai-import-examples.service'; -import { Block, TextDetectionResult } from '../ocr-metadata.service'; +import { Block, Line, TextDetectionResult } from '../ocr-metadata.service'; + +interface CreateBlockOptions { + recognizedLanguage?: string; + lines?: Line[]; +} /** * Create a mock Block with specified bounding box. @@ -11,7 +16,7 @@ import { Block, TextDetectionResult } from '../ocr-metadata.service'; * @param top Top coordinate * @param right Right coordinate * @param bottom Bottom coordinate - * @param recognizedLanguage Optional language code (default: 'en') + * @param options Optional recognized language and lines */ export function createBlock( text: string, @@ -19,13 +24,31 @@ export function createBlock( top: number, right: number, bottom: number, - recognizedLanguage: string = 'en', + options?: CreateBlockOptions, ): Block { return { text, boundingBox: { left, top, right, bottom }, - recognizedLanguage, - lines: [], + recognizedLanguage: options?.recognizedLanguage ?? 'en', + lines: options?.lines ?? [], + }; +} + +/** + * Create a mock Line with specified bounding box. + */ +export function createLine( + text: string, + left: number, + top: number, + right: number, + bottom: number, +): Line { + return { + text, + boundingBox: { left, top, right, bottom }, + recognizedLanguage: 'en', + elements: [], }; }