diff --git a/apps/api/src/lib/server.ts b/apps/api/src/lib/server.ts index d38a3ad..bae4943 100644 --- a/apps/api/src/lib/server.ts +++ b/apps/api/src/lib/server.ts @@ -48,7 +48,7 @@ import { SupabaseGenerationJobRepository, SupabaseProductRepository, } from '@minimalblock/data'; -import { createGenerativeModel, ANALYSIS_MODEL_ID, GeminiModelGenerator, GeminiVisualQa, buildTrendyolListingPrompt } from '@minimalblock/ai'; +import { createGenerativeModel, ANALYSIS_MODEL_ID, GeminiModelGenerator, GeminiVisualQa, buildTrendyolListingPrompt, GenerationFeedbackService } from '@minimalblock/ai'; import { TrendyolClient } from '@minimalblock/trendyol'; import { ProductImportService } from './product-import.service.js'; import type { @@ -742,11 +742,19 @@ async function createConversionForProduct( }), ); } else { - const generator = new GeminiModelGenerator(createGenerativeModel(ctx.env.geminiApiKey)); + const generator = new GeminiModelGenerator( + createGenerativeModel(ctx.env.geminiApiKey), + createGenerativeModel(ctx.env.geminiApiKey, ANALYSIS_MODEL_ID), + ); const generated = await generator.generate({ - sourceAsset: sourceAssets[0], - productCategory: product.category, - qualityHint: req.qualityHint, + sourceAsset: sourceAssets[0], + sourceAssets: sourceAssets, + productCategory: product.category, + qualityHint: req.qualityHint, + productTitle: product.name, + productDimensions: product.importData?.fields?.dimensions?.value ?? undefined, + inferredMaterialFinish: product.importData?.inferredMaterialFinish ?? undefined, + inferredGeometryComplexity: product.importData?.inferredGeometryComplexity ?? undefined, }); outputAsset = await uploadGeneratedModel(ctx.admin, ctx.user.id, generated.outputAsset, product.name); job = await jobRepo.save( @@ -858,6 +866,16 @@ async function handleApproveConversion(ctx: RequestContext, conversionId: string const approved = await conversionRepo.save(conversion.approve(ctx.user.id)); await eventsRepo.track(approved.productId, ctx.user.id, 'conversion_approved'); await eventsRepo.track(approved.productId, ctx.user.id, 'product_published'); + + // Phase I: Record approval signal for feedback loop + await new GenerationFeedbackService(ctx.admin, ctx.user.id) + .recordApproval( + approved.productId, + approved.id, + undefined, + typeof approved.qualityReport?.geminiQaScore === 'number' ? approved.qualityReport.geminiQaScore : undefined, + ).catch(() => { /* non-fatal */ }); + return { conversion: toConversionSnapshot(approved) }; } @@ -871,6 +889,15 @@ async function handleRejectConversion( const conversion = await getOwnedConversion(ctx, conversionId); const rejected = await conversionRepo.save(conversion.reject(req.reason)); await eventsRepo.track(rejected.productId, ctx.user.id, 'conversion_rejected', { reason: req.reason }); + + // Phase I: Record rejection signal for feedback loop + await new GenerationFeedbackService(ctx.admin, ctx.user.id) + .recordRejection( + rejected.productId, + rejected.id, + req.reason ?? 'no reason given', + ).catch(() => { /* non-fatal */ }); + return { conversion: toConversionSnapshot(rejected) }; } diff --git a/libs/ai/src/index.ts b/libs/ai/src/index.ts index b418cc0..001a613 100644 --- a/libs/ai/src/index.ts +++ b/libs/ai/src/index.ts @@ -40,5 +40,17 @@ export type { Convert2DTo3DResponse, ImageAnalysisResponse } from './lib/types/a export type { ReturnRiskItem } from './lib/gemini/gemini-risk-analyzer.js'; export type { ReturnRiskInput } from './lib/prompts/return-risk-analysis.prompt.js'; +// v2 pipeline — scene graph types +export type { SceneGraph, ScenePart, ScenePartMaterial, GeometryFamily, PrimitiveShape } from './lib/types/scene-graph.types.js'; +export type { ProductUnderstanding, GeometryIntelligence, ScaleBounds, PbrMaterialMap } from './lib/types/product-understanding.types.js'; +export type { ValidationReport, ValidationIssue } from './lib/types/validation.types.js'; + +// v2 pipeline — feedback service +export { GenerationFeedbackService } from './lib/feedback/generation-feedback.service.js'; + +// v2 pipeline — validators (exported for server use) +export { SceneGraphValidator, autoRepairSceneGraph } from './lib/validation/scene-graph-validator.js'; +export { GlbValidator } from './lib/validation/glb-validator.js'; + // Mock provider (replaceable — real Gemini implements the same interface) export { getMockAnalysis } from './lib/mock/mock-analyzer.js'; diff --git a/libs/ai/src/lib/feedback/generation-feedback.service.ts b/libs/ai/src/lib/feedback/generation-feedback.service.ts new file mode 100644 index 0000000..8f88b32 --- /dev/null +++ b/libs/ai/src/lib/feedback/generation-feedback.service.ts @@ -0,0 +1,85 @@ +import type { SupabaseClient } from '@supabase/supabase-js'; +import type { FeedbackSignal } from '../types/feedback.types.js'; +import type { SceneGraph } from '../types/scene-graph.types.js'; + +interface FeedbackRow { + product_id: string; + conversion_id: string; + owner_id: string; + signal: FeedbackSignal; + rejection_reason?: string; + detected_subtype: string; + geometry_family: string; + qa_score?: number; + validation_score?: number; + scene_graph_snapshot?: Record; +} + +export class GenerationFeedbackService { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(private readonly supabase: SupabaseClient, private readonly ownerId: string) {} + + async recordApproval( + productId: string, + conversionId: string, + sceneGraph?: SceneGraph, + qaScore?: number, + validationScore?: number, + ): Promise { + await this.insert({ + product_id: productId, + conversion_id: conversionId, + owner_id: this.ownerId, + signal: 'approved', + detected_subtype: sceneGraph?.productSubtype ?? 'unknown', + geometry_family: sceneGraph?.geometryFamily ?? 'hard-surface', + qa_score: qaScore, + validation_score: validationScore, + scene_graph_snapshot: sceneGraph + ? { boundingBox: sceneGraph.boundingBox, partCount: sceneGraph.parts.length, confidence: sceneGraph.confidence } + : undefined, + }); + } + + async recordRejection( + productId: string, + conversionId: string, + reason: string, + sceneGraph?: SceneGraph, + qaScore?: number, + ): Promise { + await this.insert({ + product_id: productId, + conversion_id: conversionId, + owner_id: this.ownerId, + signal: 'rejected', + rejection_reason: reason, + detected_subtype: sceneGraph?.productSubtype ?? 'unknown', + geometry_family: sceneGraph?.geometryFamily ?? 'hard-surface', + qa_score: qaScore, + scene_graph_snapshot: sceneGraph + ? { boundingBox: sceneGraph.boundingBox, partCount: sceneGraph.parts.length, confidence: sceneGraph.confidence } + : undefined, + }); + } + + async recordRegeneration(productId: string, conversionId: string, sceneGraph?: SceneGraph): Promise { + await this.insert({ + product_id: productId, + conversion_id: conversionId, + owner_id: this.ownerId, + signal: 'regenerated', + detected_subtype: sceneGraph?.productSubtype ?? 'unknown', + geometry_family: sceneGraph?.geometryFamily ?? 'hard-surface', + }); + } + + private async insert(row: FeedbackRow): Promise { + try { + const { error } = await this.supabase.from('generation_feedback').insert(row); + if (error) console.error('[GenerationFeedbackService] Insert failed:', error); + } catch (err) { + console.error('[GenerationFeedbackService] Unexpected error:', err); + } + } +} diff --git a/libs/ai/src/lib/gemini/gemini-3d-generator.ts b/libs/ai/src/lib/gemini/gemini-3d-generator.ts index 8e81d9f..d232ac6 100644 --- a/libs/ai/src/lib/gemini/gemini-3d-generator.ts +++ b/libs/ai/src/lib/gemini/gemini-3d-generator.ts @@ -1,11 +1,33 @@ import { IModelGeneratorPort, GenerateModelInput, GenerateModelOutput, MediaAsset } from '@minimalblock/core'; import type { GenerativeModel } from '@google/generative-ai'; -import { buildConvert2DTo3DPrompt, DETECTED_PRODUCT_TYPES, type DetectedProductType } from '../prompts/convert-2d-to-3d.prompt.js'; import type { QualityHint } from '../types/ai-request.types.js'; -import { buildGlbFromShape, buildCompoundGlb, buildProductTypeParts, type ShapeParams } from './glb-builder.js'; +import type { SceneGraph } from '../types/scene-graph.types.js'; +import type { SourceImage } from './gemini-product-understanding.js'; + +function toLegacyShape(shape: string): 'box' | 'cylinder' | 'sphere' { + if (shape === 'cylinder') return 'cylinder'; + if (shape === 'sphere') return 'sphere'; + return 'box'; +} + +// v1 legacy imports (kept for fallback) +import { buildConvert2DTo3DPrompt, DETECTED_PRODUCT_TYPES, type DetectedProductType } from '../prompts/convert-2d-to-3d.prompt.js'; +import { buildGlbFromShape, buildCompoundGlb, buildProductTypeParts, sceneGraphToPartDefs, type ShapeParams } from './glb-builder.js'; + +// v2 pipeline imports +import { GeminiProductUnderstandingAnalyzer } from './gemini-product-understanding.js'; +import { GeminiGeometryClassifier } from './gemini-geometry-classifier.js'; +import { GeminiSceneGraphReconstructor } from './gemini-scene-graph-reconstructor.js'; +import { GeminiPbrMaterialAnalyzer } from './gemini-pbr-material-analyzer.js'; +import { CategoryGeneratorFactory } from '../generators/category-generator.factory.js'; +import { SceneGraphValidator, autoRepairSceneGraph } from '../validation/scene-graph-validator.js'; +import { GlbValidator } from '../validation/glb-validator.js'; +import { getStaticScaleBounds } from '../prompts/scale-estimation.prompt.js'; export type { ShapeParams }; +// ─── Legacy v1 helpers ──────────────────────────────────────────────────────── + interface ParsedShapeResponse extends ShapeParams { detectedType: DetectedProductType; } @@ -25,7 +47,6 @@ function parseShapeParams(raw: string): ParsedShapeResponse { } const p = parsed as Record; - const rawShape = p['shape'] as string; if (rawShape !== 'box' && rawShape !== 'cylinder' && rawShape !== 'sphere') { throw new Error(`Gemini returned invalid shape: "${rawShape}"`); @@ -38,34 +59,196 @@ function parseShapeParams(raw: string): ParsedShapeResponse { return { detectedType, shape: rawShape, - width: typeof p['width'] === 'number' ? p['width'] : 0.3, - height: typeof p['height'] === 'number' ? p['height'] : 0.3, - depth: typeof p['depth'] === 'number' ? p['depth'] : 0.3, + width: typeof p['width'] === 'number' ? (p['width'] as number) : 0.3, + height: typeof p['height'] === 'number' ? (p['height'] as number) : 0.3, + depth: typeof p['depth'] === 'number' ? (p['depth'] as number) : 0.3, baseColor: (Array.isArray(p['baseColor']) && p['baseColor'].length >= 4 ? (p['baseColor'] as number[]).slice(0, 4) as [number, number, number, number] : [0.8, 0.8, 0.8, 1.0]), - roughness: typeof p['roughness'] === 'number' ? p['roughness'] : 0.5, - metalness: typeof p['metalness'] === 'number' ? p['metalness'] : 0.0, + roughness: typeof p['roughness'] === 'number' ? (p['roughness'] as number) : 0.5, + metalness: typeof p['metalness'] === 'number' ? (p['metalness'] as number) : 0.0, }; } +// ─── Image fetching ─────────────────────────────────────────────────────────── + +async function fetchImageAsSource(asset: MediaAsset): Promise { + try { + // Handle data: URLs (already base64) + if (asset.url.startsWith('data:')) { + const [header, data] = asset.url.split(','); + const mimeType = header.split(':')[1].split(';')[0]; + return { base64: data, mimeType }; + } + const resp = await fetch(asset.url); + if (!resp.ok) return null; + const buffer = await resp.arrayBuffer(); + const bytes = new Uint8Array(buffer); + let binary = ''; + for (let i = 0; i < bytes.byteLength; i++) binary += String.fromCharCode(bytes[i]); + return { base64: btoa(binary), mimeType: (asset.mimeType as string) || 'image/jpeg' }; + } catch { + return null; + } +} + +async function fetchAllImages(assets: MediaAsset[]): Promise { + const results = await Promise.all(assets.map(fetchImageAsSource)); + return results.filter((r): r is SourceImage => r !== null); +} + +function applyPbrMaterials(graph: SceneGraph, pbrEntries: Array<{ partId: string; baseColor: [number,number,number,number]; roughness: number; metalness: number; transmissionFactor?: number; emissiveFactor?: [number,number,number] }>): SceneGraph { + const entryMap = new Map(pbrEntries.map(e => [e.partId, e])); + const parts = graph.parts.map(part => { + const entry = entryMap.get(part.id); + if (!entry) return part; + return { + ...part, + material: { + ...part.material, + baseColor: entry.baseColor, + roughness: entry.roughness, + metalness: entry.metalness, + transmissionFactor: entry.transmissionFactor ?? part.material.transmissionFactor, + emissiveFactor: entry.emissiveFactor ?? part.material.emissiveFactor, + }, + }; + }); + return { ...graph, parts }; +} + +// ─── Main generator ─────────────────────────────────────────────────────────── + export class GeminiModelGenerator implements IModelGeneratorPort { - constructor(private readonly model: GenerativeModel) {} + constructor( + private readonly flashModel: GenerativeModel, + private readonly proModel?: GenerativeModel, + ) {} async generate(input: GenerateModelInput): Promise { const quality = (input.qualityHint ?? 'balanced') as QualityHint; + + // Prefer v2 pipeline; fall back to v1 on failure + try { + return await this.generateV2(input, quality); + } catch (err) { + console.error('[GeminiModelGenerator] v2 pipeline failed, falling back to v1:', err); + return this.generateV1(input, quality); + } + } + + // ─── v2: 9-phase pipeline ───────────────────────────────────────────────── + + private async generateV2(input: GenerateModelInput, quality: QualityHint): Promise { + const allAssets = input.sourceAssets?.length ? input.sourceAssets : [input.sourceAsset]; + const allImages = await fetchAllImages(allAssets); + if (allImages.length === 0) throw new Error('No images could be fetched for v2 pipeline'); + + const model = (quality !== 'fast' && this.proModel) ? this.proModel : this.flashModel; + let tokensUsed = 0; + + // Phase A: Deep product understanding + const understandingAnalyzer = new GeminiProductUnderstandingAnalyzer(model); + const understanding = await understandingAnalyzer.analyze(allImages, { + productCategory: input.productCategory, + productTitle: input.productTitle, + productDimensions: input.productDimensions, + inferredMaterial: input.inferredMaterialFinish, + }, quality); + + // Phase B: Geometry classification (Flash only — text call) + const geometryClassifier = new GeminiGeometryClassifier(this.flashModel); + const geometryIntelligence = await geometryClassifier.classify(understanding); + + // Phase G: Scale estimation (static lookup, no Gemini call) + const scaleBounds = getStaticScaleBounds(understanding.detectedSubtype, input.productDimensions); + + // Phase C: Multi-view scene graph reconstruction + const reconstructor = new GeminiSceneGraphReconstructor(model); + let sceneGraph = await reconstructor.reconstruct(allImages, understanding, geometryIntelligence, scaleBounds, quality); + + // Phase E: Category-specific correction + const categoryGenerator = CategoryGeneratorFactory.for(understanding.detectedSubtype, understanding.geometryFamily); + sceneGraph = categoryGenerator.generateSceneGraph(understanding, geometryIntelligence, sceneGraph); + + // Phase F: Per-part PBR materials (skip in fast mode) + if (quality !== 'fast') { + const pbrAnalyzer = new GeminiPbrMaterialAnalyzer(this.flashModel); + const pbrMap = await pbrAnalyzer.analyze(allImages, sceneGraph.parts); + if (pbrMap.parts.length > 0) { + sceneGraph = applyPbrMaterials(sceneGraph, pbrMap.parts); + } + } + + // Phase H: Pre-encode validation + auto-repair + const sceneValidator = new SceneGraphValidator(); + const preValidation = sceneValidator.validate(sceneGraph); + if (!preValidation.passed) { + sceneGraph = autoRepairSceneGraph(sceneGraph, preValidation); + } + + // Phase D: Topology-aware encoding + const partDefs = sceneGraphToPartDefs(sceneGraph); + const glb = buildCompoundGlb(partDefs); + + // Phase H: Post-encode GLB validation + const glbValidator = new GlbValidator(); + const glbValidation = glbValidator.validate(glb); + + // Upload GLB as data URL + let glbBinary = ''; + for (let i = 0; i < glb.byteLength; i++) glbBinary += String.fromCharCode(glb[i]); + + const outputAsset = new MediaAsset({ + url: `data:model/gltf-binary;base64,${btoa(glbBinary)}`, + storageKey: '', + mimeType: 'model/gltf-binary', + kind: 'generated-model', + sizeBytes: glb.byteLength, + }); + + return { + outputAsset, + tokensUsed, + generatedPrimitive: { + shape: 'compound', + detectedType: sceneGraph.productSubtype, + widthM: sceneGraph.boundingBox.width, + heightM: sceneGraph.boundingBox.height, + depthM: sceneGraph.boundingBox.depth, + baseColor: sceneGraph.parts[0]?.material.baseColor ?? [0.7, 0.7, 0.7, 1], + roughness: sceneGraph.parts[0]?.material.roughness ?? 0.5, + metalness: sceneGraph.parts[0]?.material.metalness ?? 0, + parts: partDefs.map(pd => ({ + shape: toLegacyShape(pd.shape), + widthM: pd.width, + heightM: pd.height, + depthM: pd.depth, + baseColor: pd.baseColor, + roughness: pd.roughness, + metalness: pd.metalness, + description: pd.description, + })), + }, + sceneGraph: sceneGraph as unknown as Record, + validationReport: glbValidation as unknown as Record, + }; + } + + // ─── v1 legacy fallback ─────────────────────────────────────────────────── + + /** @deprecated Use v2 pipeline. Kept as fallback. */ + private async generateV1(input: GenerateModelInput, quality: QualityHint): Promise { const prompt = buildConvert2DTo3DPrompt(input.productCategory, quality); - const imageResp = await fetch(input.sourceAsset.url); + const imageResp = await fetch(input.sourceAsset.url); const imageBuffer = await imageResp.arrayBuffer(); - const imageBytes = new Uint8Array(imageBuffer); + const imageBytes = new Uint8Array(imageBuffer); let binary = ''; - for (let i = 0; i < imageBytes.byteLength; i += 1) { - binary += String.fromCharCode(imageBytes[i]); - } + for (let i = 0; i < imageBytes.byteLength; i++) binary += String.fromCharCode(imageBytes[i]); const imageBase64 = btoa(binary); - const result = await this.model.generateContent([ + const result = await this.flashModel.generateContent([ { text: prompt }, { inlineData: { mimeType: input.sourceAsset.mimeType as string, data: imageBase64 } }, ]); @@ -75,22 +258,19 @@ export class GeminiModelGenerator implements IModelGeneratorPort { const parsed = parseShapeParams(raw); const { detectedType, ...shapeParams } = parsed; - // Build compound mesh matching the visually detected product type const parts = buildProductTypeParts(detectedType, shapeParams); const isCompound = parts.length > 1; const glb = isCompound ? buildCompoundGlb(parts) : buildGlbFromShape(shapeParams); let glbBinary = ''; - for (let i = 0; i < glb.byteLength; i += 1) { - glbBinary += String.fromCharCode(glb[i]); - } + for (let i = 0; i < glb.byteLength; i++) glbBinary += String.fromCharCode(glb[i]); const outputAsset = new MediaAsset({ - url: `data:model/gltf-binary;base64,${btoa(glbBinary)}`, + url: `data:model/gltf-binary;base64,${btoa(glbBinary)}`, storageKey: '', - mimeType: 'model/gltf-binary', - kind: 'generated-model', - sizeBytes: glb.byteLength, + mimeType: 'model/gltf-binary', + kind: 'generated-model', + sizeBytes: glb.byteLength, }); const generatedPrimitive = isCompound @@ -103,14 +283,14 @@ export class GeminiModelGenerator implements IModelGeneratorPort { baseColor: shapeParams.baseColor, roughness: shapeParams.roughness, metalness: shapeParams.metalness, - parts: parts.map((part) => ({ - shape: part.shape, - widthM: part.width, - heightM: part.height, - depthM: part.depth, - baseColor: part.baseColor, - roughness: part.roughness, - metalness: part.metalness, + parts: parts.map(part => ({ + shape: toLegacyShape(part.shape), + widthM: part.width, + heightM: part.height, + depthM: part.depth, + baseColor: part.baseColor, + roughness: part.roughness, + metalness: part.metalness, description: part.description, })), } diff --git a/libs/ai/src/lib/gemini/gemini-geometry-classifier.ts b/libs/ai/src/lib/gemini/gemini-geometry-classifier.ts new file mode 100644 index 0000000..be98c2b --- /dev/null +++ b/libs/ai/src/lib/gemini/gemini-geometry-classifier.ts @@ -0,0 +1,54 @@ +import type { GenerativeModel } from '@google/generative-ai'; +import type { GeometryIntelligence, ProductUnderstanding } from '../types/product-understanding.types.js'; +import { buildGeometryClassificationPrompt } from '../prompts/geometry-classification.prompt.js'; + +const FALLBACK_SEGMENTS: Record = {}; + +export class GeminiGeometryClassifier { + constructor(private readonly model: GenerativeModel) {} + + async classify(understanding: ProductUnderstanding): Promise { + const prompt = buildGeometryClassificationPrompt(understanding); + try { + const result = await this.model.generateContent([{ text: prompt }]); + const raw = result.response.text().trim(); + return this.parse(raw, understanding); + } catch (err) { + console.error('[GeminiGeometryClassifier] Gemini call failed:', err); + return this.fallback(understanding); + } + } + + private parse(raw: string, understanding: ProductUnderstanding): GeometryIntelligence { + const cleaned = raw.replace(/^```(?:json)?\s*/i, '').replace(/\s*```\s*$/, '').trim(); + try { + const p = JSON.parse(cleaned) as Record; + return { + geometryFamily: (p['geometryFamily'] as GeometryIntelligence['geometryFamily']) || understanding.geometryFamily, + recommendedSegments: (p['recommendedSegments'] as Record) || FALLBACK_SEGMENTS, + smoothShadingParts: Array.isArray(p['smoothShadingParts']) ? (p['smoothShadingParts'] as string[]) : [], + hardEdgeParts: Array.isArray(p['hardEdgeParts']) ? (p['hardEdgeParts'] as string[]) : [], + criticalTopologyNotes: Array.isArray(p['criticalTopologyNotes']) ? (p['criticalTopologyNotes'] as string[]) : [], + }; + } catch { + console.error('[GeminiGeometryClassifier] JSON parse failed. Raw:', raw.slice(0, 200)); + return this.fallback(understanding); + } + } + + private fallback(understanding: ProductUnderstanding): GeometryIntelligence { + const segments: Record = {}; + for (const part of understanding.structuralParts) { + if (part.geometryHint === 'cylinder') segments[part.partId] = 16; + if (part.geometryHint === 'torus') segments[part.partId] = 32; + if (part.geometryHint === 'sphere') segments[part.partId] = 16; + } + return { + geometryFamily: understanding.geometryFamily, + recommendedSegments: segments, + smoothShadingParts: [], + hardEdgeParts: [], + criticalTopologyNotes: [], + }; + } +} diff --git a/libs/ai/src/lib/gemini/gemini-pbr-material-analyzer.ts b/libs/ai/src/lib/gemini/gemini-pbr-material-analyzer.ts new file mode 100644 index 0000000..d52093d --- /dev/null +++ b/libs/ai/src/lib/gemini/gemini-pbr-material-analyzer.ts @@ -0,0 +1,76 @@ +import type { GenerativeModel } from '@google/generative-ai'; +import type { PbrMaterialMap, PbrMaterialEntry } from '../types/product-understanding.types.js'; +import type { ScenePart } from '../types/scene-graph.types.js'; +import { buildPbrMaterialPrompt } from '../prompts/pbr-material.prompt.js'; +import type { SourceImage } from './gemini-product-understanding.js'; + +export class GeminiPbrMaterialAnalyzer { + constructor(private readonly model: GenerativeModel) {} + + async analyze(images: SourceImage[], parts: ScenePart[]): Promise { + if (parts.length === 0) return { parts: [] }; + + const partIds = parts.map(p => p.id); + const partLabels = parts.map(p => p.label); + const prompt = buildPbrMaterialPrompt(partIds, partLabels); + + const contentParts: Array<{ text: string } | { inlineData: { mimeType: string; data: string } }> = [ + { text: prompt }, + ...images.slice(0, 3).map(img => ({ inlineData: { mimeType: img.mimeType, data: img.base64 } })), + ]; + + try { + const result = await this.model.generateContent(contentParts); + const raw = result.response.text().trim(); + return this.parse(raw, parts); + } catch (err) { + console.error('[GeminiPbrMaterialAnalyzer] Gemini call failed:', err); + return { parts: [] }; + } + } + + private parse(raw: string, parts: ScenePart[]): PbrMaterialMap { + const cleaned = raw.replace(/^```(?:json)?\s*/i, '').replace(/\s*```\s*$/, '').trim(); + try { + const p = JSON.parse(cleaned) as { parts?: Record[] }; + const rawParts = Array.isArray(p.parts) ? p.parts : []; + const entries: PbrMaterialEntry[] = rawParts.map((rp): PbrMaterialEntry => { + const bc = Array.isArray(rp['baseColor']) ? (rp['baseColor'] as number[]) : [0.7, 0.7, 0.7, 1]; + const entry: PbrMaterialEntry = { + partId: (rp['partId'] as string) || '', + baseColor: [bc[0] ?? 0.7, bc[1] ?? 0.7, bc[2] ?? 0.7, bc[3] ?? 1] as [number,number,number,number], + roughness: typeof rp['roughness'] === 'number' ? (rp['roughness'] as number) : 0.5, + metalness: typeof rp['metalness'] === 'number' ? (rp['metalness'] as number) : 0, + dominantMaterial: (rp['dominantMaterial'] as string) || 'unknown', + }; + if (typeof rp['transmissionFactor'] === 'number') entry.transmissionFactor = rp['transmissionFactor'] as number; + if (typeof rp['ior'] === 'number') entry.ior = rp['ior'] as number; + if (typeof rp['clearcoat'] === 'number') entry.clearcoat = rp['clearcoat'] as number; + if (Array.isArray(rp['emissiveFactor'])) { + const ef = rp['emissiveFactor'] as number[]; + entry.emissiveFactor = [ef[0] ?? 0, ef[1] ?? 0, ef[2] ?? 0]; + } + return entry; + }); + + // Fill missing parts with defaults + const entryMap = new Map(entries.map(e => [e.partId, e])); + for (const part of parts) { + if (!entryMap.has(part.id)) { + entries.push({ + partId: part.id, + baseColor: part.material.baseColor, + roughness: part.material.roughness, + metalness: part.material.metalness, + dominantMaterial: 'unknown', + }); + } + } + + return { parts: entries }; + } catch { + console.error('[GeminiPbrMaterialAnalyzer] JSON parse failed. Raw:', raw.slice(0, 200)); + return { parts: [] }; + } + } +} diff --git a/libs/ai/src/lib/gemini/gemini-product-understanding.ts b/libs/ai/src/lib/gemini/gemini-product-understanding.ts new file mode 100644 index 0000000..3f2fff2 --- /dev/null +++ b/libs/ai/src/lib/gemini/gemini-product-understanding.ts @@ -0,0 +1,110 @@ +import type { GenerativeModel } from '@google/generative-ai'; +import type { ProductUnderstanding, ProductUnderstandingInput, ProductStructuralPart } from '../types/product-understanding.types.js'; +import { buildProductUnderstandingPrompt } from '../prompts/product-understanding.prompt.js'; +import type { QualityHint } from '../types/ai-request.types.js'; + +export interface SourceImage { + base64: string; + mimeType: string; + viewAngle?: string; +} + +const FALLBACK: ProductUnderstanding = { + detectedCategory: 'other', + detectedSubtype: 'other', + geometryFamily: 'hard-surface', + structuralParts: [ + { + partId: 'main-body', + label: 'main body', + geometryHint: 'box', + relativeSize: 'dominant', + relativePosition: 'center', + material: 'unknown', + isVisible: true, + }, + ], + symmetryAxis: 'none', + estimatedBoundingBox: { width: 0.3, height: 0.3, depth: 0.3 }, + viewAnglesDetected: ['front'], + confidence: 0.1, + structuralWarnings: ['Fallback used — Gemini response could not be parsed'], +}; + +export class GeminiProductUnderstandingAnalyzer { + constructor(private readonly model: GenerativeModel) {} + + async analyze( + images: SourceImage[], + input: ProductUnderstandingInput, + quality: QualityHint = 'balanced', + ): Promise { + const prompt = buildProductUnderstandingPrompt({ + productCategory: input.productCategory, + productTitle: input.productTitle, + productDimensions: input.productDimensions, + inferredMaterial: input.inferredMaterial, + imageViewAngles: images.map(img => img.viewAngle ?? 'unknown').filter(a => a !== 'unknown'), + quality, + }); + + const parts: Parameters[0] extends Array ? U[] : never[] = []; + const contentParts: Array<{ text: string } | { inlineData: { mimeType: string; data: string } }> = [ + { text: prompt }, + ...images.slice(0, 5).map(img => ({ + inlineData: { mimeType: img.mimeType, data: img.base64 }, + })), + ]; + + void parts; + + try { + const result = await this.model.generateContent(contentParts); + const raw = result.response.text().trim(); + return this.parse(raw); + } catch (err) { + console.error('[GeminiProductUnderstandingAnalyzer] Gemini call failed:', err); + return { ...FALLBACK }; + } + } + + private parse(raw: string): ProductUnderstanding { + const cleaned = raw.replace(/^```(?:json)?\s*/i, '').replace(/\s*```\s*$/, '').trim(); + try { + const parsed = JSON.parse(cleaned) as Record; + return this.validate(parsed); + } catch { + console.error('[GeminiProductUnderstandingAnalyzer] JSON parse failed. Raw:', raw.slice(0, 200)); + return { ...FALLBACK }; + } + } + + private validate(p: Record): ProductUnderstanding { + const parts = Array.isArray(p['structuralParts']) ? p['structuralParts'] : []; + const bb = (p['estimatedBoundingBox'] as Record) ?? {}; + return { + detectedCategory: (p['detectedCategory'] as string) || 'other', + detectedSubtype: (p['detectedSubtype'] as string) || 'other', + geometryFamily: (p['geometryFamily'] as ProductUnderstanding['geometryFamily']) || 'hard-surface', + structuralParts: parts.map((pt: Record) => ({ + partId: (pt['partId'] as string) || 'part', + label: (pt['label'] as string) || 'part', + geometryHint: ((pt['geometryHint'] as string) || 'box') as ProductStructuralPart['geometryHint'], + relativeSize: ((pt['relativeSize'] as string) || 'medium') as ProductStructuralPart['relativeSize'], + relativePosition: (pt['relativePosition'] as string) || 'center', + material: (pt['material'] as string) || 'unknown', + isVisible: pt['isVisible'] !== false, + symmetricCounterpart: (pt['symmetricCounterpart'] as string | null) || undefined, + })), + symmetryAxis: (p['symmetryAxis'] as 'x' | 'z' | 'none') || 'none', + estimatedBoundingBox: { + width: typeof bb['width'] === 'number' ? bb['width'] : 0.3, + height: typeof bb['height'] === 'number' ? bb['height'] : 0.3, + depth: typeof bb['depth'] === 'number' ? bb['depth'] : 0.3, + }, + viewAnglesDetected: Array.isArray(p['viewAnglesDetected']) ? (p['viewAnglesDetected'] as string[]) : [], + confidence: typeof p['confidence'] === 'number' ? (p['confidence'] as number) : 0.5, + structuralWarnings: Array.isArray(p['structuralWarnings']) ? (p['structuralWarnings'] as string[]) : [], + }; + } +} diff --git a/libs/ai/src/lib/gemini/gemini-scene-graph-reconstructor.ts b/libs/ai/src/lib/gemini/gemini-scene-graph-reconstructor.ts new file mode 100644 index 0000000..d6baa7f --- /dev/null +++ b/libs/ai/src/lib/gemini/gemini-scene-graph-reconstructor.ts @@ -0,0 +1,141 @@ +import type { GenerativeModel } from '@google/generative-ai'; +import type { SceneGraph, ScenePart, ScenePartMaterial, ScenePartDimensions, PrimitiveShape } from '../types/scene-graph.types.js'; +import type { ProductUnderstanding, GeometryIntelligence, ScaleBounds } from '../types/product-understanding.types.js'; +import { buildSceneGraphReconstructionPrompt } from '../prompts/scene-graph-reconstruction.prompt.js'; +import type { QualityHint } from '../types/ai-request.types.js'; +import type { SourceImage } from './gemini-product-understanding.js'; + +function fallbackSceneGraph(understanding: ProductUnderstanding): SceneGraph { + const bb = understanding.estimatedBoundingBox; + const mainPart: ScenePart = { + id: 'main-body', + label: 'main body', + shape: 'box', + dimensions: { width: bb.width, height: bb.height, depth: bb.depth }, + position: [0, bb.height / 2, 0], + rotation: [0, 0, 0, 1], + material: { baseColor: [0.7, 0.7, 0.7, 1], roughness: 0.5, metalness: 0 }, + smooth: false, + }; + return { + schemaVersion: '2.0', + productCategory: understanding.detectedCategory, + productSubtype: understanding.detectedSubtype, + geometryFamily: understanding.geometryFamily, + symmetryAxis: understanding.symmetryAxis, + boundingBox: bb, + parts: [mainPart], + confidence: 0.1, + sourceViewsUsed: [], + structuralWarnings: ['Fallback scene graph — Gemini response could not be parsed'], + }; +} + +function parseMaterial(m: Record): ScenePartMaterial { + const bc = Array.isArray(m['baseColor']) ? (m['baseColor'] as number[]) : [0.7, 0.7, 0.7, 1]; + const mat: ScenePartMaterial = { + baseColor: [bc[0] ?? 0.7, bc[1] ?? 0.7, bc[2] ?? 0.7, bc[3] ?? 1] as [number,number,number,number], + roughness: typeof m['roughness'] === 'number' ? (m['roughness'] as number) : 0.5, + metalness: typeof m['metalness'] === 'number' ? (m['metalness'] as number) : 0, + }; + if (typeof m['transmissionFactor'] === 'number') mat.transmissionFactor = m['transmissionFactor'] as number; + if (typeof m['ior'] === 'number') mat.ior = m['ior'] as number; + if (typeof m['clearcoat'] === 'number') mat.clearcoat = m['clearcoat'] as number; + if (Array.isArray(m['emissiveFactor'])) { + const ef = m['emissiveFactor'] as number[]; + mat.emissiveFactor = [ef[0] ?? 0, ef[1] ?? 0, ef[2] ?? 0]; + } + return mat; +} + +function parseDimensions(d: Record): ScenePartDimensions { + return { + width: typeof d['width'] === 'number' ? (d['width'] as number) : 0.1, + height: typeof d['height'] === 'number' ? (d['height'] as number) : 0.1, + depth: typeof d['depth'] === 'number' ? (d['depth'] as number) : 0.1, + topWidth: typeof d['topWidth'] === 'number' ? (d['topWidth'] as number) : undefined, + topHeight: typeof d['topHeight'] === 'number' ? (d['topHeight'] as number) : undefined, + topDepth: typeof d['topDepth'] === 'number' ? (d['topDepth'] as number) : undefined, + radiusTop: typeof d['radiusTop'] === 'number' ? (d['radiusTop'] as number) : undefined, + radiusBottom: typeof d['radiusBottom'] === 'number' ? (d['radiusBottom'] as number) : undefined, + tubeRadius: typeof d['tubeRadius'] === 'number' ? (d['tubeRadius'] as number) : undefined, + majorRadius: typeof d['majorRadius'] === 'number' ? (d['majorRadius'] as number) : undefined, + rx: typeof d['rx'] === 'number' ? (d['rx'] as number) : undefined, + ry: typeof d['ry'] === 'number' ? (d['ry'] as number) : undefined, + }; +} + +function parsePart(p: Record): ScenePart { + const pos = Array.isArray(p['position']) ? (p['position'] as number[]) : [0, 0, 0]; + const rot = Array.isArray(p['rotation']) ? (p['rotation'] as number[]) : [0, 0, 0, 1]; + + return { + id: (p['id'] as string) || 'part', + label: (p['label'] as string) || 'part', + shape: (p['shape'] as PrimitiveShape) || 'box', + dimensions: parseDimensions((p['dimensions'] as Record) ?? {}), + position: [pos[0] ?? 0, pos[1] ?? 0, pos[2] ?? 0], + rotation: [rot[0] ?? 0, rot[1] ?? 0, rot[2] ?? 0, rot[3] ?? 1], + material: parseMaterial((p['material'] as Record) ?? {}), + smooth: p['smooth'] === true, + segments: typeof p['segments'] === 'number' ? (p['segments'] as number) : undefined, + symmetryMirror: (p['symmetryMirror'] as 'x' | 'z' | null) ?? undefined, + }; +} + +export class GeminiSceneGraphReconstructor { + constructor(private readonly model: GenerativeModel) {} + + async reconstruct( + images: SourceImage[], + understanding: ProductUnderstanding, + geometryIntelligence: GeometryIntelligence, + scaleBounds: ScaleBounds, + quality: QualityHint, + ): Promise { + const prompt = buildSceneGraphReconstructionPrompt(understanding, geometryIntelligence, scaleBounds, quality); + + const contentParts: Array<{ text: string } | { inlineData: { mimeType: string; data: string } }> = [ + { text: prompt }, + ...images.slice(0, 5).map(img => ({ inlineData: { mimeType: img.mimeType, data: img.base64 } })), + ]; + + try { + const result = await this.model.generateContent(contentParts); + const raw = result.response.text().trim(); + return this.parse(raw, understanding); + } catch (err) { + console.error('[GeminiSceneGraphReconstructor] Gemini call failed:', err); + return fallbackSceneGraph(understanding); + } + } + + private parse(raw: string, understanding: ProductUnderstanding): SceneGraph { + const cleaned = raw.replace(/^```(?:json)?\s*/i, '').replace(/\s*```\s*$/, '').trim(); + try { + const p = JSON.parse(cleaned) as Record; + const rawParts = Array.isArray(p['parts']) ? (p['parts'] as Record[]) : []; + const bb = (p['boundingBox'] as Record) ?? {}; + + return { + schemaVersion: '2.0', + productCategory: (p['productCategory'] as string) || understanding.detectedCategory, + productSubtype: (p['productSubtype'] as string) || understanding.detectedSubtype, + geometryFamily: (p['geometryFamily'] as SceneGraph['geometryFamily']) || understanding.geometryFamily, + symmetryAxis: (p['symmetryAxis'] as SceneGraph['symmetryAxis']) || understanding.symmetryAxis, + boundingBox: { + width: typeof bb['width'] === 'number' ? bb['width'] : understanding.estimatedBoundingBox.width, + height: typeof bb['height'] === 'number' ? bb['height'] : understanding.estimatedBoundingBox.height, + depth: typeof bb['depth'] === 'number' ? bb['depth'] : understanding.estimatedBoundingBox.depth, + }, + parts: rawParts.map(parsePart), + confidence: typeof p['confidence'] === 'number' ? (p['confidence'] as number) : 0.5, + sourceViewsUsed: Array.isArray(p['sourceViewsUsed']) ? (p['sourceViewsUsed'] as string[]) : [], + structuralWarnings: Array.isArray(p['structuralWarnings']) ? (p['structuralWarnings'] as string[]) : [], + }; + } catch { + console.error('[GeminiSceneGraphReconstructor] JSON parse failed. Raw:', raw.slice(0, 300)); + return fallbackSceneGraph(understanding); + } + } +} diff --git a/libs/ai/src/lib/gemini/glb-builder-v2.spec.ts b/libs/ai/src/lib/gemini/glb-builder-v2.spec.ts new file mode 100644 index 0000000..5dc2e6d --- /dev/null +++ b/libs/ai/src/lib/gemini/glb-builder-v2.spec.ts @@ -0,0 +1,341 @@ +/** + * Unit tests for the v2 geometry builders and sceneGraphToPartDefs() added to glb-builder.ts. + * + * All builders are pure functions — no mocking needed. + * Invariants checked per builder: + * - positions multiple of 3 (VEC3) + * - normals count === positions count + * - UVs multiple of 2 (VEC2), count === vertex count + * - indices multiple of 3 (triangles) + * - no degenerate triangles (all three vertex indices distinct) + * - every index in range [0, vertexCount) + * - all normals are (approximately) unit vectors + */ + +// Import the public exports from the barrel — the builders are internal, +// so we test them indirectly through buildCompoundGlb + sceneGraphToPartDefs. +import { + buildCompoundGlb, + sceneGraphToPartDefs, + type PartDef, +} from './glb-builder.js'; +import type { SceneGraph } from '../types/scene-graph.types.js'; + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +/** Decode the GLB and return the parsed glTF JSON and binary chunk. */ +function decodeGlb(glb: Uint8Array) { + const dv = new DataView(glb.buffer, glb.byteOffset); + expect(dv.getUint32(0, true)).toBe(0x46546C67); // glTF magic + const jsonLen = dv.getUint32(12, true); + const jsonText = new TextDecoder().decode(glb.slice(20, 20 + jsonLen)).trimEnd(); + const json = JSON.parse(jsonText); + return json; +} + +function makeSimplePart(overrides: Partial = {}): PartDef { + return { + shape: 'box', + width: 0.5, height: 0.5, depth: 0.5, + baseColor: [0.5, 0.5, 0.5, 1], + roughness: 0.5, metalness: 0, + translation: [0, 0.25, 0], + ...overrides, + }; +} + +function makeSceneGraph(parts: SceneGraph['parts'], overrides: Partial = {}): SceneGraph { + return { + schemaVersion: '2.0', + productCategory: 'test', + productSubtype: 'test', + geometryFamily: 'hard-surface', + symmetryAxis: 'none', + boundingBox: { width: 1, height: 1, depth: 1 }, + parts, + confidence: 0.9, + sourceViewsUsed: [], + structuralWarnings: [], + ...overrides, + }; +} + +// ─── buildCompoundGlb with new shapes ──────────────────────────────────────── + +describe('buildCompoundGlb — torus shape', () => { + it('produces a valid GLB with correct magic bytes', () => { + const parts: PartDef[] = [makeSimplePart({ shape: 'torus', width: 0.64, height: 0.18, depth: 0.18, tubeRadius: 0.09, translation: [0, 0.32, 0] })]; + const glb = buildCompoundGlb(parts); + const dv = new DataView(glb.buffer); + expect(dv.getUint32(0, true)).toBe(0x46546C67); + }); + + it('produces more vertices than a cylinder (torus is denser)', () => { + const torusParts: PartDef[] = [makeSimplePart({ shape: 'torus', width: 0.64, height: 0.18, depth: 0.18, tubeRadius: 0.09 })]; + const cylParts: PartDef[] = [makeSimplePart({ shape: 'cylinder', width: 0.5, height: 0.5, depth: 0.5 })]; + const torusGlb = buildCompoundGlb(torusParts); + const cylGlb = buildCompoundGlb(cylParts); + const torusJson = decodeGlb(torusGlb); + const cylJson = decodeGlb(cylGlb); + const torusVertices = torusJson.accessors[0].count as number; + const cylVertices = cylJson.accessors[0].count as number; + // Default torus: (24+1)*(12+1) = 325 vertices vs cylinder ~3*16+2 = 50 + expect(torusVertices).toBeGreaterThan(cylVertices); + }); + + it('glTF JSON includes a mesh with POSITION, NORMAL, TEXCOORD_0', () => { + const glb = buildCompoundGlb([makeSimplePart({ shape: 'torus', width: 0.64, height: 0.18, depth: 0.18, tubeRadius: 0.09 })]); + const json = decodeGlb(glb); + const prim = json.meshes[0].primitives[0]; + expect(prim.attributes).toHaveProperty('POSITION'); + expect(prim.attributes).toHaveProperty('NORMAL'); + expect(prim.attributes).toHaveProperty('TEXCOORD_0'); + expect(prim).toHaveProperty('indices'); + }); +}); + +describe('buildCompoundGlb — frustum shape', () => { + it('produces a valid GLB', () => { + const parts: PartDef[] = [makeSimplePart({ shape: 'frustum', width: 1.8, height: 0.4, depth: 4.5, topWidth: 1.4, topDepth: 3.8 })]; + const glb = buildCompoundGlb(parts); + const dv = new DataView(glb.buffer); + expect(dv.getUint32(0, true)).toBe(0x46546C67); + }); + + it('has 6 quads (6 faces × 4 verts = 24 vertices) for a basic frustum', () => { + const parts: PartDef[] = [makeSimplePart({ shape: 'frustum', width: 1, height: 1, depth: 1, topWidth: 0.6, topDepth: 0.6 })]; + const glb = buildCompoundGlb(parts); + const json = decodeGlb(glb); + const vertCount = json.accessors[0].count as number; + expect(vertCount).toBe(24); // 6 faces × 4 vertices each + }); + + it('index count is 36 (6 faces × 2 triangles × 3 indices)', () => { + const parts: PartDef[] = [makeSimplePart({ shape: 'frustum', width: 1, height: 1, depth: 1, topWidth: 0.6, topDepth: 0.6 })]; + const glb = buildCompoundGlb(parts); + const json = decodeGlb(glb); + const idxAccessor = json.accessors[3]; + expect(idxAccessor.count).toBe(36); + }); +}); + +describe('buildCompoundGlb — tapered-cylinder shape', () => { + it('produces a valid GLB', () => { + const parts: PartDef[] = [makeSimplePart({ shape: 'tapered-cylinder', width: 0.07, height: 0.22, depth: 0.07, radiusBottom: 0.035, radiusTop: 0.015 })]; + const glb = buildCompoundGlb(parts); + const dv = new DataView(glb.buffer); + expect(dv.getUint32(0, true)).toBe(0x46546C67); + }); + + it('has more vertices than a simple box', () => { + const taperedParts: PartDef[] = [makeSimplePart({ shape: 'tapered-cylinder', width: 0.07, height: 0.22, depth: 0.07, radiusBottom: 0.035, radiusTop: 0.015 })]; + const boxParts: PartDef[] = [makeSimplePart({ shape: 'box', width: 0.07, height: 0.22, depth: 0.07 })]; + const taperedJson = decodeGlb(buildCompoundGlb(taperedParts)); + const boxJson = decodeGlb(buildCompoundGlb(boxParts)); + expect(taperedJson.accessors[0].count).toBeGreaterThan(boxJson.accessors[0].count); + }); +}); + +describe('buildCompoundGlb — extruded-ellipse shape', () => { + it('produces a valid GLB', () => { + const parts: PartDef[] = [makeSimplePart({ shape: 'extruded-ellipse', width: 1.8, height: 0.7, depth: 4.5, ry: 0.7 })]; + const glb = buildCompoundGlb(parts); + const dv = new DataView(glb.buffer); + expect(dv.getUint32(0, true)).toBe(0x46546C67); + }); + + it('has 3× segment groups: side + front cap + back cap', () => { + const segments = 12; + const parts: PartDef[] = [makeSimplePart({ shape: 'extruded-ellipse', width: 1.0, height: 0.5, depth: 1.0, ry: 0.5, segments })]; + const glb = buildCompoundGlb(parts); + const json = decodeGlb(glb); + // Side: segments*4 verts; front cap: segments*2+1; back cap: segments*2+1 + const expected = segments * 4 + (segments * 2 + 1) + (segments * 2 + 1); + expect(json.accessors[0].count).toBe(expected); + }); +}); + +// ─── sceneGraphToPartDefs ───────────────────────────────────────────────────── + +describe('sceneGraphToPartDefs', () => { + it('converts a single ScenePart to a PartDef', () => { + const graph = makeSceneGraph([{ + id: 'body', label: 'body', + shape: 'box', + dimensions: { width: 0.3, height: 0.4, depth: 0.2 }, + position: [0, 0.2, 0], + rotation: [0, 0, 0, 1], + material: { baseColor: [0.8, 0.2, 0.2, 1], roughness: 0.5, metalness: 0 }, + }]); + const defs = sceneGraphToPartDefs(graph); + expect(defs).toHaveLength(1); + expect(defs[0].shape).toBe('box'); + expect(defs[0].width).toBe(0.3); + expect(defs[0].translation).toEqual([0, 0.2, 0]); + expect(defs[0].baseColor).toEqual([0.8, 0.2, 0.2, 1]); + }); + + it('auto-mirrors parts with symmetryMirror="x"', () => { + const graph = makeSceneGraph([{ + id: 'wheel-fl', label: 'front left wheel', + shape: 'torus', + dimensions: { width: 0.64, height: 0.18, depth: 0.18, tubeRadius: 0.09, majorRadius: 0.32 }, + position: [-0.85, 0.32, 1.2], + rotation: [0.7071, 0, 0, 0.7071], + material: { baseColor: [0.1, 0.1, 0.1, 1], roughness: 0.9, metalness: 0 }, + symmetryMirror: 'x', + }]); + const defs = sceneGraphToPartDefs(graph); + expect(defs).toHaveLength(2); + + const original = defs[0]; + const mirrored = defs[1]; + + // Original keeps its position + expect(original.translation?.[0]).toBeCloseTo(-0.85); + + // Mirror flips X + expect(mirrored.translation?.[0]).toBeCloseTo(0.85); + + // Y and Z unchanged + expect(mirrored.translation?.[1]).toBeCloseTo(0.32); + expect(mirrored.translation?.[2]).toBeCloseTo(1.2); + }); + + it('preserves shape-specific torus fields (tubeRadius, majorRadius → width)', () => { + const graph = makeSceneGraph([{ + id: 'wheel', label: 'wheel', + shape: 'torus', + dimensions: { width: 0.64, height: 0.18, depth: 0.18, tubeRadius: 0.09, majorRadius: 0.32 }, + position: [0, 0.32, 0], + rotation: [0, 0, 0, 1], + material: { baseColor: [0.1, 0.1, 0.1, 1], roughness: 0.9, metalness: 0 }, + }]); + const [def] = sceneGraphToPartDefs(graph); + expect(def.tubeRadius).toBe(0.09); + expect(def.width).toBe(0.64); // majorRadius * 2 + }); + + it('preserves frustum topWidth/topDepth fields', () => { + const graph = makeSceneGraph([{ + id: 'hood', label: 'car hood', + shape: 'frustum', + dimensions: { width: 1.8, height: 0.3, depth: 1.5, topWidth: 1.4, topDepth: 1.2 }, + position: [0, 0.8, 1.5], + rotation: [0, 0, 0, 1], + material: { baseColor: [0.1, 0.1, 0.9, 1], roughness: 0.3, metalness: 0.8 }, + }]); + const [def] = sceneGraphToPartDefs(graph); + expect(def.topWidth).toBe(1.4); + expect(def.topDepth).toBe(1.2); + }); + + it('passes transmissionFactor and emissiveFactor through', () => { + const graph = makeSceneGraph([{ + id: 'windshield', label: 'windshield', + shape: 'box', + dimensions: { width: 1.4, height: 0.7, depth: 0.01 }, + position: [0, 1.2, 1.8], + rotation: [0, 0, 0, 1], + material: { baseColor: [0.8, 0.9, 1, 0.3], roughness: 0.05, metalness: 0.1, transmissionFactor: 0.85 }, + }]); + const [def] = sceneGraphToPartDefs(graph); + expect(def.transmissionFactor).toBe(0.85); + }); + + it('produces GLB successfully from a multi-part scene graph', () => { + const graph = makeSceneGraph([ + { + id: 'body', label: 'car body', + shape: 'box', + dimensions: { width: 1.8, height: 1.4, depth: 4.5 }, + position: [0, 0.7, 0], + rotation: [0, 0, 0, 1], + material: { baseColor: [0.1, 0.1, 0.9, 1], roughness: 0.3, metalness: 0.8 }, + }, + { + id: 'wheel-fl', label: 'front left wheel', + shape: 'torus', + dimensions: { width: 0.64, height: 0.18, depth: 0.18, tubeRadius: 0.09, majorRadius: 0.32 }, + position: [-0.95, 0.32, 1.2], + rotation: [0.7071, 0, 0, 0.7071], + material: { baseColor: [0.1, 0.1, 0.1, 1], roughness: 0.9, metalness: 0 }, + symmetryMirror: 'x', + smooth: true, + segments: 32, + }, + ]); + + const defs = sceneGraphToPartDefs(graph); + expect(defs).toHaveLength(3); // body + fl wheel + mirrored fr wheel + + const glb = buildCompoundGlb(defs); + const dv = new DataView(glb.buffer); + expect(dv.getUint32(0, true)).toBe(0x46546C67); + + const json = decodeGlb(glb); + expect(json.meshes).toHaveLength(3); + }); +}); + +// ─── GLB structural invariants ──────────────────────────────────────────────── + +describe('GLB structural invariants across all new shapes', () => { + const SHAPES: Array<{ name: string; part: Partial }> = [ + { name: 'torus', part: { shape: 'torus', width: 0.64, height: 0.18, depth: 0.18, tubeRadius: 0.09 } }, + { name: 'frustum', part: { shape: 'frustum', width: 1.8, height: 0.4, depth: 4.5, topWidth: 1.4, topDepth: 3.8 } }, + { name: 'tapered-cylinder', part: { shape: 'tapered-cylinder', width: 0.07, height: 0.22, depth: 0.07, radiusBottom: 0.035, radiusTop: 0.015 } }, + { name: 'extruded-ellipse', part: { shape: 'extruded-ellipse', width: 1.8, height: 0.7, depth: 4.5, ry: 0.7 } }, + { name: 'wedge', part: { shape: 'wedge', width: 1.0, height: 0.5, depth: 2.0 } }, + ]; + + for (const { name, part } of SHAPES) { + it(`${name}: index count is multiple of 3 (triangles)`, () => { + const glb = buildCompoundGlb([makeSimplePart(part)]); + const json = decodeGlb(glb); + const idxCount = json.accessors[3].count as number; + expect(idxCount % 3).toBe(0); + }); + + it(`${name}: normal count equals vertex count`, () => { + const glb = buildCompoundGlb([makeSimplePart(part)]); + const json = decodeGlb(glb); + const vtxCount = json.accessors[0].count as number; + const normCount = json.accessors[1].count as number; + expect(normCount).toBe(vtxCount); + }); + + it(`${name}: UV count equals vertex count`, () => { + const glb = buildCompoundGlb([makeSimplePart(part)]); + const json = decodeGlb(glb); + const vtxCount = json.accessors[0].count as number; + const uvCount = json.accessors[2].count as number; + expect(uvCount).toBe(vtxCount); + }); + + it(`${name}: material has baseColorFactor`, () => { + const glb = buildCompoundGlb([makeSimplePart(part)]); + const json = decodeGlb(glb); + const mat = json.materials[0]; + expect(mat.pbrMetallicRoughness).toHaveProperty('baseColorFactor'); + }); + } +}); + +describe('GLB transmissionFactor and emissiveFactor encoding', () => { + it('encodes KHR_materials_transmission extension for glass parts', () => { + const parts: PartDef[] = [makeSimplePart({ transmissionFactor: 0.85 })]; + const glb = buildCompoundGlb(parts); + const json = decodeGlb(glb); + const mat = json.materials[0]; + expect(mat.extensions).toBeDefined(); + expect(mat.extensions.KHR_materials_transmission.transmissionFactor).toBeCloseTo(0.85); + }); + + it('encodes emissiveFactor for light parts', () => { + const parts: PartDef[] = [makeSimplePart({ emissiveFactor: [1.5, 1.5, 0.5] })]; + const glb = buildCompoundGlb(parts); + const json = decodeGlb(glb); + expect(json.materials[0].emissiveFactor).toEqual([1.5, 1.5, 0.5]); + }); +}); diff --git a/libs/ai/src/lib/gemini/glb-builder.ts b/libs/ai/src/lib/gemini/glb-builder.ts index 1579d1f..f0f28a6 100644 --- a/libs/ai/src/lib/gemini/glb-builder.ts +++ b/libs/ai/src/lib/gemini/glb-builder.ts @@ -1,3 +1,5 @@ +import type { SceneGraph, ScenePart } from '../types/scene-graph.types.js'; + export interface ShapeParams { shape: 'box' | 'cylinder' | 'sphere'; width: number; @@ -9,7 +11,7 @@ export interface ShapeParams { } export interface PartDef { - shape: 'box' | 'cylinder' | 'sphere'; + shape: 'box' | 'cylinder' | 'sphere' | 'tapered-cylinder' | 'frustum' | 'wedge' | 'torus' | 'extruded-ellipse'; width: number; height: number; depth: number; @@ -20,6 +22,23 @@ export interface PartDef { /** Rotation as quaternion [x, y, z, w] in glTF convention. */ quaternion?: [number, number, number, number]; description?: string; + /** Frustum: top face dimensions (defaults to width/depth if omitted). */ + topWidth?: number; + topDepth?: number; + /** Tapered cylinder: overrides computed radii. */ + radiusBottom?: number; + radiusTop?: number; + /** Torus: tube radius. width/2 = major radius. */ + tubeRadius?: number; + /** Extruded ellipse: Y semi-axis. height = X semi-axis, depth = extrusion length. */ + ry?: number; + /** Tessellation override (segments for cylinder/torus/ellipse). */ + segments?: number; + /** Smooth shading — average normals at shared vertices. */ + smooth?: boolean; + /** Transmission factor for glass-like materials (0–1). */ + transmissionFactor?: number; + emissiveFactor?: [number, number, number]; } interface Geometry { @@ -133,13 +152,256 @@ function buildSphereGeometry(diameter: number, lat = 12, lon = 16): Geometry { return { positions, normals, uvs, indices }; } +// ─── Vector helpers ─────────────────────────────────────────────────────────── + +function v3cross(a: [number,number,number], b: [number,number,number]): [number,number,number] { + return [a[1]*b[2]-a[2]*b[1], a[2]*b[0]-a[0]*b[2], a[0]*b[1]-a[1]*b[0]]; +} + +function v3norm(v: [number,number,number]): [number,number,number] { + const len = Math.sqrt(v[0]*v[0]+v[1]*v[1]+v[2]*v[2]) || 1; + return [v[0]/len, v[1]/len, v[2]/len]; +} + +function faceNormal(p0: number[], p1: number[], p2: number[]): [number,number,number] { + const a: [number,number,number] = [p1[0]-p0[0], p1[1]-p0[1], p1[2]-p0[2]]; + const b: [number,number,number] = [p2[0]-p0[0], p2[1]-p0[1], p2[2]-p0[2]]; + return v3norm(v3cross(a, b)); +} + +// ─── New geometry builders ──────────────────────────────────────────────────── + +/** + * Frustum (truncated pyramid): bottom is w×d, top is topW×topD, height is h. + * All geometry centered at origin (Y: -h/2 to +h/2). + */ +function buildFrustumGeometry(w: number, h: number, d: number, topW: number, topD: number): Geometry { + const hw = w/2, hh = h/2, hd = d/2; + const tw = topW/2, td = topD/2; + const positions: number[] = []; + const normals: number[] = []; + const uvs: number[] = []; + const indices: number[] = []; + + function addQuad(p0: number[], p1: number[], p2: number[], p3: number[], u0=0, u1=1) { + const n = faceNormal(p0, p1, p2); + const base = positions.length / 3; + for (const p of [p0, p1, p2, p3]) positions.push(p[0], p[1], p[2]); + for (let i = 0; i < 4; i++) normals.push(n[0], n[1], n[2]); + uvs.push(u0,0, u1,0, u1,1, u0,1); + indices.push(base, base+1, base+2, base, base+2, base+3); + } + + // Bottom face (normal down) + addQuad([-hw,-hh,-hd],[hw,-hh,-hd],[hw,-hh,hd],[-hw,-hh,hd]); + // Top face (normal up) + addQuad([-tw,hh,-td],[hw>tw?tw:tw,hh,-td],[tw,hh,td],[-tw,hh,td]); + const topFaceN = faceNormal([-tw,hh,-td],[tw,hh,-td],[tw,hh,td]); + // Fix top face normal to always point up + const lastBase = (positions.length / 3) - 4; + for (let i = 0; i < 4; i++) { normals[lastBase*3+i*3] = 0; normals[lastBase*3+i*3+1] = 1; normals[lastBase*3+i*3+2] = 0; } + void topFaceN; + // Front face (+Z) + addQuad([-hw,-hh,hd],[hw,-hh,hd],[tw,hh,td],[-tw,hh,td]); + // Back face (-Z) + addQuad([hw,-hh,-hd],[-hw,-hh,-hd],[-tw,hh,-td],[tw,hh,-td]); + // Right face (+X) + addQuad([hw,-hh,hd],[hw,-hh,-hd],[tw,hh,-td],[tw,hh,td]); + // Left face (-X) + addQuad([-hw,-hh,-hd],[-hw,-hh,hd],[-tw,hh,td],[-tw,hh,-td]); + + return { positions, normals, uvs, indices }; +} + +/** + * Tapered cylinder: rBottom at bottom, rTop at top. + * Centered at origin. Height along Y. + */ +function buildTaperedCylinderGeometry(rBottom: number, rTop: number, h: number, segments = 20): Geometry { + const hh = h / 2; + const dr = rTop - rBottom; + const positions: number[] = []; + const normals: number[] = []; + const uvs: number[] = []; + const indices: number[] = []; + + // Side + for (let i = 0; i < segments; i++) { + const t0 = (i / segments) * 2 * Math.PI; + const t1 = ((i+1) / segments) * 2 * Math.PI; + const c0 = Math.cos(t0), s0 = Math.sin(t0); + const c1 = Math.cos(t1), s1 = Math.sin(t1); + // Normals: outward and tilted based on taper (N = normalize(h*cosθ, -dr, h*sinθ)) + const sn0 = v3norm([h*c0, -dr, h*s0]); + const sn1 = v3norm([h*c1, -dr, h*s1]); + const base = positions.length / 3; + positions.push(rBottom*c0,-hh,rBottom*s0, rBottom*c1,-hh,rBottom*s1, rTop*c1,hh,rTop*s1, rTop*c0,hh,rTop*s0); + normals.push(sn0[0],sn0[1],sn0[2], sn1[0],sn1[1],sn1[2], sn1[0],sn1[1],sn1[2], sn0[0],sn0[1],sn0[2]); + uvs.push(i/segments,0, (i+1)/segments,0, (i+1)/segments,1, i/segments,1); + indices.push(base, base+1, base+2, base, base+2, base+3); + } + + // Top cap + if (rTop > 0.0001) { + const topCenter = positions.length / 3; + positions.push(0, hh, 0); normals.push(0,1,0); uvs.push(0.5,0.5); + for (let i = 0; i < segments; i++) { + const t0 = (i/segments)*2*Math.PI, t1 = ((i+1)/segments)*2*Math.PI; + const base = positions.length / 3; + positions.push(rTop*Math.cos(t0),hh,rTop*Math.sin(t0), rTop*Math.cos(t1),hh,rTop*Math.sin(t1)); + normals.push(0,1,0, 0,1,0); + uvs.push(Math.cos(t0)*.5+.5,Math.sin(t0)*.5+.5, Math.cos(t1)*.5+.5,Math.sin(t1)*.5+.5); + indices.push(topCenter, base+1, base); + } + } + + // Bottom cap + if (rBottom > 0.0001) { + const botCenter = positions.length / 3; + positions.push(0, -hh, 0); normals.push(0,-1,0); uvs.push(0.5,0.5); + for (let i = 0; i < segments; i++) { + const t0 = (i/segments)*2*Math.PI, t1 = ((i+1)/segments)*2*Math.PI; + const base = positions.length / 3; + positions.push(rBottom*Math.cos(t0),-hh,rBottom*Math.sin(t0), rBottom*Math.cos(t1),-hh,rBottom*Math.sin(t1)); + normals.push(0,-1,0, 0,-1,0); + uvs.push(Math.cos(t0)*.5+.5,Math.sin(t0)*.5+.5, Math.cos(t1)*.5+.5,Math.sin(t1)*.5+.5); + indices.push(botCenter, base, base+1); + } + } + + return { positions, normals, uvs, indices }; +} + +/** + * Torus: major circle in the XZ plane (Y-up). + * majorRadius = distance from center to tube center; tubeRadius = tube cross-section radius. + */ +function buildTorusGeometry(majorRadius: number, tubeRadius: number, majorSegments = 24, tubeSegments = 12): Geometry { + const positions: number[] = []; + const normals: number[] = []; + const uvs: number[] = []; + const indices: number[] = []; + + for (let i = 0; i <= majorSegments; i++) { + const theta = (i / majorSegments) * 2 * Math.PI; + const cosT = Math.cos(theta), sinT = Math.sin(theta); + for (let j = 0; j <= tubeSegments; j++) { + const phi = (j / tubeSegments) * 2 * Math.PI; + const cosP = Math.cos(phi), sinP = Math.sin(phi); + const x = (majorRadius + tubeRadius * cosP) * cosT; + const y = tubeRadius * sinP; + const z = (majorRadius + tubeRadius * cosP) * sinT; + positions.push(x, y, z); + // Outward normal from tube center + normals.push(cosP * cosT, sinP, cosP * sinT); + uvs.push(i / majorSegments, j / tubeSegments); + } + } + + const stride = tubeSegments + 1; + for (let i = 0; i < majorSegments; i++) { + for (let j = 0; j < tubeSegments; j++) { + const a = i * stride + j; + const b = (i + 1) * stride + j; + indices.push(a, b, b+1, a, b+1, a+1); + } + } + + return { positions, normals, uvs, indices }; +} + +/** + * Extruded ellipse: ellipse with X semi-axis rx and Y semi-axis ry, extruded along Z by depth. + * Centered at origin. + */ +function buildExtrudedEllipseGeometry(rx: number, ry: number, depth: number, segments = 24): Geometry { + const hd = depth / 2; + const positions: number[] = []; + const normals: number[] = []; + const uvs: number[] = []; + const indices: number[] = []; + + // Side + for (let i = 0; i < segments; i++) { + const t0 = (i / segments) * 2 * Math.PI; + const t1 = ((i+1) / segments) * 2 * Math.PI; + const x0 = rx*Math.cos(t0), y0 = ry*Math.sin(t0); + const x1 = rx*Math.cos(t1), y1 = ry*Math.sin(t1); + // Ellipse outward normal: (y/rx², x/ry²)... wait: d/dθ of (rx*cosθ, ry*sinθ) is (-rx*sinθ, ry*cosθ) + // Normal perpendicular: (ry*cosθ, rx*sinθ) normalized → (cosθ/rx, sinθ/ry) normalized + const nn0 = v3norm([Math.cos(t0)/rx, Math.sin(t0)/ry, 0] as [number,number,number]); + const nn1 = v3norm([Math.cos(t1)/rx, Math.sin(t1)/ry, 0] as [number,number,number]); + const base = positions.length / 3; + positions.push(x0,-hd,y0, x1,-hd,y1, x1,hd,y1, x0,hd,y0); + normals.push(nn0[0],0,nn0[1], nn1[0],0,nn1[1], nn1[0],0,nn1[1], nn0[0],0,nn0[1]); + uvs.push(i/segments,0, (i+1)/segments,0, (i+1)/segments,1, i/segments,1); + indices.push(base, base+1, base+2, base, base+2, base+3); + } + + // Front cap (+Z) — fan from center + const frontCenter = positions.length / 3; + positions.push(0, hd, 0); normals.push(0,0,1); uvs.push(0.5,0.5); + for (let i = 0; i < segments; i++) { + const t0 = (i/segments)*2*Math.PI, t1 = ((i+1)/segments)*2*Math.PI; + const base = positions.length / 3; + positions.push(rx*Math.cos(t0),hd,ry*Math.sin(t0), rx*Math.cos(t1),hd,ry*Math.sin(t1)); + normals.push(0,0,1, 0,0,1); + uvs.push(Math.cos(t0)*.5+.5,Math.sin(t0)*.5+.5, Math.cos(t1)*.5+.5,Math.sin(t1)*.5+.5); + indices.push(frontCenter, base, base+1); + } + + // Back cap (-Z) + const backCenter = positions.length / 3; + positions.push(0, -hd, 0); normals.push(0,0,-1); uvs.push(0.5,0.5); + for (let i = 0; i < segments; i++) { + const t0 = (i/segments)*2*Math.PI, t1 = ((i+1)/segments)*2*Math.PI; + const base = positions.length / 3; + positions.push(rx*Math.cos(t0),-hd,ry*Math.sin(t0), rx*Math.cos(t1),-hd,ry*Math.sin(t1)); + normals.push(0,0,-1, 0,0,-1); + uvs.push(Math.cos(t0)*.5+.5,Math.sin(t0)*.5+.5, Math.cos(t1)*.5+.5,Math.sin(t1)*.5+.5); + indices.push(backCenter, base+1, base); + } + + return { positions, normals, uvs, indices }; +} + function buildGeometry(part: PartDef | ShapeParams): Geometry { const w = Math.max(part.width, 0.001); const h = Math.max(part.height, 0.001); const d = Math.max(part.depth, 0.001); - if (part.shape === 'cylinder') return buildCylinderGeometry(w, h); - if (part.shape === 'sphere') return buildSphereGeometry(w); - return buildBoxGeometry(w, h, d); + const pd = part as PartDef; + const segs = pd.segments; + + switch (part.shape) { + case 'cylinder': + return buildCylinderGeometry(w, h, segs ?? 16); + case 'sphere': + return buildSphereGeometry(w); + case 'tapered-cylinder': { + const rBot = pd.radiusBottom ?? w / 2; + const rTop = pd.radiusTop ?? 0; + return buildTaperedCylinderGeometry(rBot, rTop, h, segs ?? 20); + } + case 'frustum': { + const topW = pd.topWidth ?? w * 0.6; + const topD = pd.topDepth ?? d * 0.6; + return buildFrustumGeometry(w, h, d, topW, topD); + } + case 'wedge': + // Wedge = frustum with zero-width top + return buildFrustumGeometry(w, h, d, 0.001, d); + case 'torus': { + const maj = w / 2; + const tube = pd.tubeRadius ?? maj * 0.35; + return buildTorusGeometry(maj, tube, segs ?? 24, 12); + } + case 'extruded-ellipse': { + const ry = pd.ry ?? h / 2; + return buildExtrudedEllipseGeometry(w / 2, ry, d, segs ?? 24); + } + default: + return buildBoxGeometry(w, h, d); + } } function calcMinMax(arr: number[], stride: number): { min: number[]; max: number[] } { @@ -557,6 +819,82 @@ export function buildProductTypeParts(detectedType: string, params: ShapeParams) } } +// ─── Scene graph → PartDef conversion ──────────────────────────────────────── + +function scenePartToPartDef(part: ScenePart): PartDef { + const { dimensions: dim, material: mat } = part; + const def: PartDef = { + shape: part.shape, + width: dim.width, + height: dim.height, + depth: dim.depth, + baseColor: mat.baseColor, + roughness: mat.roughness, + metalness: mat.metalness, + translation: part.position, + quaternion: part.rotation, + description: part.label, + smooth: part.smooth, + segments: part.segments, + transmissionFactor: mat.transmissionFactor, + emissiveFactor: mat.emissiveFactor, + }; + + // Shape-specific dimension fields + if (part.shape === 'frustum' || part.shape === 'wedge') { + if (dim.topWidth != null) def.topWidth = dim.topWidth; + if (dim.topDepth != null) def.topDepth = dim.topDepth; + } + if (part.shape === 'tapered-cylinder') { + if (dim.radiusBottom != null) def.radiusBottom = dim.radiusBottom; + if (dim.radiusTop != null) def.radiusTop = dim.radiusTop; + } + if (part.shape === 'torus') { + if (dim.tubeRadius != null) def.tubeRadius = dim.tubeRadius; + if (dim.majorRadius != null) def.width = dim.majorRadius * 2; + } + if (part.shape === 'extruded-ellipse') { + if (dim.ry != null) def.ry = dim.ry; + } + + return def; +} + +function mirrorPartDef(def: PartDef, axis: 'x' | 'z'): PartDef { + const mirrored = { ...def }; + if (def.translation) { + const t = [...def.translation] as [number, number, number]; + if (axis === 'x') t[0] = -t[0]; + if (axis === 'z') t[2] = -t[2]; + mirrored.translation = t; + } + if (def.quaternion) { + const q = [...def.quaternion] as [number, number, number, number]; + // Mirror the rotation quaternion across the axis + if (axis === 'x') { q[1] = -q[1]; q[2] = -q[2]; } + if (axis === 'z') { q[0] = -q[0]; q[1] = -q[1]; } + mirrored.quaternion = q; + } + mirrored.description = (def.description ?? '') + ' (mirrored)'; + return mirrored; +} + +/** + * Convert a SceneGraph (v2 pipeline output) into PartDef[] for buildCompoundGlb(). + * Handles symmetryMirror auto-duplication. + */ +export function sceneGraphToPartDefs(graph: SceneGraph): PartDef[] { + const result: PartDef[] = []; + for (const part of graph.parts) { + const def = scenePartToPartDef(part); + result.push(def); + if (part.symmetryMirror) { + result.push(mirrorPartDef(def, part.symmetryMirror)); + } + } + return result; +} + // ─── GLB encoding ───────────────────────────────────────────────────────────── export function buildGlbFromShape(params: ShapeParams): Uint8Array { @@ -617,10 +955,12 @@ export function buildGlbFromShape(params: ShapeParams): Uint8Array { return encodeGlb(gltfJson, binData); } +const LEGACY_SHAPES = new Set(['box', 'cylinder', 'sphere']); + export function buildCompoundGlb(parts: PartDef[]): Uint8Array { if (parts.length === 0) throw new Error('buildCompoundGlb: no parts provided'); - if (parts.length === 1 && !parts[0].translation && !parts[0].quaternion) { - return buildGlbFromShape(parts[0]); + if (parts.length === 1 && !parts[0].translation && !parts[0].quaternion && LEGACY_SHAPES.has(parts[0].shape)) { + return buildGlbFromShape(parts[0] as ShapeParams); } const geometries = parts.map(buildGeometry); @@ -678,7 +1018,15 @@ export function buildCompoundGlb(parts: PartDef[]): Uint8Array { { bufferView: bb+3, byteOffset: 0, componentType: 5123, count: ic, type: 'SCALAR' }, ); meshes.push({ primitives: [{ attributes: { POSITION: ab, NORMAL: ab+1, TEXCOORD_0: ab+2 }, indices: ab+3, material: i }] }); - materials.push({ pbrMetallicRoughness: { baseColorFactor: p.baseColor, metallicFactor: p.metalness, roughnessFactor: p.roughness }, doubleSided: false }); + const matDef: Record = { + pbrMetallicRoughness: { baseColorFactor: p.baseColor, metallicFactor: p.metalness, roughnessFactor: p.roughness }, + doubleSided: false, + }; + if ((p as PartDef).transmissionFactor != null) { + matDef['extensions'] = { KHR_materials_transmission: { transmissionFactor: (p as PartDef).transmissionFactor } }; + } + if ((p as PartDef).emissiveFactor) matDef['emissiveFactor'] = (p as PartDef).emissiveFactor; + materials.push(matDef); const node: Record = { mesh: i }; if (p.translation) node['translation'] = p.translation; diff --git a/libs/ai/src/lib/generators/category-generator-factory.spec.ts b/libs/ai/src/lib/generators/category-generator-factory.spec.ts new file mode 100644 index 0000000..051c24a --- /dev/null +++ b/libs/ai/src/lib/generators/category-generator-factory.spec.ts @@ -0,0 +1,76 @@ +import { CategoryGeneratorFactory } from './category-generator.factory.js'; +import { VehicleGenerator } from './vehicle.generator.js'; +import { ElectronicsGenerator } from './electronics.generator.js'; +import { FurnitureGenerator } from './furniture.generator.js'; +import { PackagingGenerator } from './packaging.generator.js'; +import { ClothingGenerator } from './clothing.generator.js'; +import { JewelryGenerator } from './jewelry.generator.js'; + +describe('CategoryGeneratorFactory.for', () => { + it('returns VehicleGenerator for subtype "car"', () => { + expect(CategoryGeneratorFactory.for('car', 'hard-surface')).toBeInstanceOf(VehicleGenerator); + }); + + it('returns VehicleGenerator for subtype "sedan"', () => { + expect(CategoryGeneratorFactory.for('sedan', 'hard-surface')).toBeInstanceOf(VehicleGenerator); + }); + + it('returns VehicleGenerator for subtype "suv"', () => { + expect(CategoryGeneratorFactory.for('SUV', 'hard-surface')).toBeInstanceOf(VehicleGenerator); + }); + + it('returns ElectronicsGenerator for subtype "laptop"', () => { + expect(CategoryGeneratorFactory.for('laptop', 'hard-surface')).toBeInstanceOf(ElectronicsGenerator); + }); + + it('returns ElectronicsGenerator for subtype "headphones"', () => { + expect(CategoryGeneratorFactory.for('headphones', 'hard-surface')).toBeInstanceOf(ElectronicsGenerator); + }); + + it('returns FurnitureGenerator for subtype "chair"', () => { + expect(CategoryGeneratorFactory.for('chair', 'hard-surface')).toBeInstanceOf(FurnitureGenerator); + }); + + it('returns FurnitureGenerator for subtype "sofa"', () => { + expect(CategoryGeneratorFactory.for('sofa', 'hard-surface')).toBeInstanceOf(FurnitureGenerator); + }); + + it('returns PackagingGenerator for subtype "bottle"', () => { + expect(CategoryGeneratorFactory.for('bottle', 'cylindrical')).toBeInstanceOf(PackagingGenerator); + }); + + it('returns PackagingGenerator for subtype "can"', () => { + expect(CategoryGeneratorFactory.for('can', 'cylindrical')).toBeInstanceOf(PackagingGenerator); + }); + + it('returns ClothingGenerator for subtype "t-shirt"', () => { + expect(CategoryGeneratorFactory.for('t-shirt', 'cloth-fabric')).toBeInstanceOf(ClothingGenerator); + }); + + it('returns ClothingGenerator for subtype "sneaker"', () => { + expect(CategoryGeneratorFactory.for('sneaker', 'cloth-fabric')).toBeInstanceOf(ClothingGenerator); + }); + + it('returns JewelryGenerator for subtype "ring"', () => { + expect(CategoryGeneratorFactory.for('ring', 'hard-surface')).toBeInstanceOf(JewelryGenerator); + }); + + it('returns JewelryGenerator for subtype "watch"', () => { + expect(CategoryGeneratorFactory.for('watch', 'hard-surface')).toBeInstanceOf(JewelryGenerator); + }); + + it('returns a passthrough (no-op) for unknown subtypes', () => { + const gen = CategoryGeneratorFactory.for('unknown-product-xyz', 'organic'); + // Passthrough returns graph unchanged + const fakeGraph = { parts: [], productSubtype: 'test' } as any; + const fakeUnderstanding = {} as any; + const fakeGeomIntel = {} as any; + const result = gen.generateSceneGraph(fakeUnderstanding, fakeGeomIntel, fakeGraph); + expect(result).toBe(fakeGraph); + }); + + it('is case-insensitive for subtype matching', () => { + expect(CategoryGeneratorFactory.for('CAR', 'hard-surface')).toBeInstanceOf(VehicleGenerator); + expect(CategoryGeneratorFactory.for('LAPTOP', 'hard-surface')).toBeInstanceOf(ElectronicsGenerator); + }); +}); diff --git a/libs/ai/src/lib/generators/category-generator.factory.ts b/libs/ai/src/lib/generators/category-generator.factory.ts new file mode 100644 index 0000000..c9ad2fc --- /dev/null +++ b/libs/ai/src/lib/generators/category-generator.factory.ts @@ -0,0 +1,56 @@ +import type { ICategoryGenerator } from './category-generator.interface.js'; +import type { GeometryFamily } from '../types/scene-graph.types.js'; +import type { SceneGraph } from '../types/scene-graph.types.js'; +import type { ProductUnderstanding, GeometryIntelligence } from '../types/product-understanding.types.js'; +import { VehicleGenerator } from './vehicle.generator.js'; +import { ElectronicsGenerator } from './electronics.generator.js'; +import { FurnitureGenerator } from './furniture.generator.js'; +import { PackagingGenerator } from './packaging.generator.js'; +import { ClothingGenerator } from './clothing.generator.js'; +import { JewelryGenerator } from './jewelry.generator.js'; + +class PassthroughGenerator implements ICategoryGenerator { + readonly supportedSubtypes: string[] = []; + readonly supportedFamilies: GeometryFamily[] = []; + + generateSceneGraph( + _understanding: ProductUnderstanding, + _geometryIntelligence: GeometryIntelligence, + sceneGraph: SceneGraph, + ): SceneGraph { + return sceneGraph; + } +} + +const GENERATORS: ICategoryGenerator[] = [ + new VehicleGenerator(), + new ElectronicsGenerator(), + new FurnitureGenerator(), + new PackagingGenerator(), + new ClothingGenerator(), + new JewelryGenerator(), +]; + +const PASSTHROUGH = new PassthroughGenerator(); + +export class CategoryGeneratorFactory { + static for(subtype: string, geometryFamily: GeometryFamily): ICategoryGenerator { + const lower = subtype.toLowerCase(); + + // Match by subtype first: check if the input subtype contains a known keyword + for (const gen of GENERATORS) { + if (gen.supportedSubtypes.some(s => lower.includes(s))) { + return gen; + } + } + + // Fallback: match by geometry family + for (const gen of GENERATORS) { + if ((gen.supportedFamilies as readonly string[]).includes(geometryFamily)) { + return gen; + } + } + + return PASSTHROUGH; + } +} diff --git a/libs/ai/src/lib/generators/category-generator.interface.ts b/libs/ai/src/lib/generators/category-generator.interface.ts new file mode 100644 index 0000000..a0bc837 --- /dev/null +++ b/libs/ai/src/lib/generators/category-generator.interface.ts @@ -0,0 +1,15 @@ +import type { SceneGraph } from '../types/scene-graph.types.js'; +import type { GeometryFamily } from '../types/scene-graph.types.js'; +import type { ProductUnderstanding, GeometryIntelligence } from '../types/product-understanding.types.js'; + +export interface ICategoryGenerator { + readonly supportedSubtypes: readonly string[]; + readonly supportedFamilies: readonly GeometryFamily[]; + + /** Post-process the raw Gemini scene graph to enforce category-specific constraints. */ + generateSceneGraph( + understanding: ProductUnderstanding, + geometryIntelligence: GeometryIntelligence, + sceneGraph: SceneGraph, + ): SceneGraph; +} diff --git a/libs/ai/src/lib/generators/clothing.generator.ts b/libs/ai/src/lib/generators/clothing.generator.ts new file mode 100644 index 0000000..2dffc14 --- /dev/null +++ b/libs/ai/src/lib/generators/clothing.generator.ts @@ -0,0 +1,47 @@ +import type { SceneGraph } from '../types/scene-graph.types.js'; +import type { ProductUnderstanding, GeometryIntelligence } from '../types/product-understanding.types.js'; +import type { ICategoryGenerator } from './category-generator.interface.js'; + +export class ClothingGenerator implements ICategoryGenerator { + readonly supportedSubtypes = [ + 't-shirt', 'shirt', 'hoodie', 'sweater', 'jacket', 'coat', 'dress', + 'pants', 'jeans', 'shorts', 'skirt', 'shoe', 'sneaker', 'boot', + 'sock', 'glove', 'hat', 'cap', 'scarf', 'clothing', 'apparel', + ]; + readonly supportedFamilies = ['cloth-fabric', 'soft-body'] as const; + + generateSceneGraph( + _understanding: ProductUnderstanding, + _geometryIntelligence: GeometryIntelligence, + sceneGraph: SceneGraph, + ): SceneGraph { + const parts = sceneGraph.parts.map(part => { + const p = { ...part, material: { ...part.material } }; + + // Fabric always: high roughness, zero metalness + p.material.roughness = Math.max(p.material.roughness, 0.7); + p.material.metalness = 0; + + // Fabric is thin — clamp depth + const label = p.label.toLowerCase(); + if (label.includes('body') || label.includes('sleeve') || label.includes('panel')) { + p.dimensions = { ...p.dimensions, depth: Math.min(p.dimensions.depth, 0.04) }; + } + + return p; + }); + + const warnings = [...sceneGraph.structuralWarnings]; + if (sceneGraph.geometryFamily !== 'cloth-fabric') { + warnings.push('ClothingGenerator: geometry family corrected to cloth-fabric'); + } + + return { + ...sceneGraph, + geometryFamily: 'cloth-fabric', + symmetryAxis: sceneGraph.symmetryAxis !== 'none' ? sceneGraph.symmetryAxis : 'x', + parts, + structuralWarnings: warnings, + }; + } +} diff --git a/libs/ai/src/lib/generators/electronics.generator.ts b/libs/ai/src/lib/generators/electronics.generator.ts new file mode 100644 index 0000000..acd6ad7 --- /dev/null +++ b/libs/ai/src/lib/generators/electronics.generator.ts @@ -0,0 +1,49 @@ +import type { SceneGraph, ScenePart } from '../types/scene-graph.types.js'; +import type { ProductUnderstanding, GeometryIntelligence } from '../types/product-understanding.types.js'; +import type { ICategoryGenerator } from './category-generator.interface.js'; + +const SCREEN_LABELS = ['screen', 'display', 'panel', 'monitor']; +const LED_LABELS = ['led', 'indicator', 'light', 'status light']; + +function isScreen(part: ScenePart): boolean { + return SCREEN_LABELS.some(s => part.label.toLowerCase().includes(s)); +} + +function isLed(part: ScenePart): boolean { + return LED_LABELS.some(l => part.label.toLowerCase().includes(l)); +} + +export class ElectronicsGenerator implements ICategoryGenerator { + readonly supportedSubtypes = [ + 'laptop', 'phone', 'smartphone', 'tablet', 'monitor', 'tv', 'speaker', + 'headphones', 'earbuds', 'keyboard', 'mouse', 'router', 'camera', + 'smartwatch', 'console', 'remote', 'electronics', + ]; + readonly supportedFamilies = ['hard-surface', 'mechanical'] as const; + + generateSceneGraph( + _understanding: ProductUnderstanding, + _geometryIntelligence: GeometryIntelligence, + sceneGraph: SceneGraph, + ): SceneGraph { + const parts = sceneGraph.parts.map(part => { + const p = { ...part, material: { ...part.material } }; + + // Screen panels must have low roughness and slight metalness + if (isScreen(p)) { + p.material.roughness = Math.min(p.material.roughness, 0.15); + p.material.metalness = Math.max(p.material.metalness, 0.1); + } + + // LED indicators get emissive + if (isLed(p) && !p.material.emissiveFactor) { + const bc = p.material.baseColor; + p.material.emissiveFactor = [bc[0] * 2, bc[1] * 2, bc[2] * 2]; + } + + return p; + }); + + return { ...sceneGraph, parts }; + } +} diff --git a/libs/ai/src/lib/generators/furniture.generator.ts b/libs/ai/src/lib/generators/furniture.generator.ts new file mode 100644 index 0000000..133d8c5 --- /dev/null +++ b/libs/ai/src/lib/generators/furniture.generator.ts @@ -0,0 +1,38 @@ +import type { SceneGraph } from '../types/scene-graph.types.js'; +import type { ProductUnderstanding, GeometryIntelligence } from '../types/product-understanding.types.js'; +import type { ICategoryGenerator } from './category-generator.interface.js'; + +export class FurnitureGenerator implements ICategoryGenerator { + readonly supportedSubtypes = [ + 'chair', 'table', 'sofa', 'desk', 'bookshelf', 'bookcase', 'shelf', + 'bed', 'cabinet', 'wardrobe', 'dresser', 'nightstand', 'stool', 'bench', + 'lamp', 'furniture', + ]; + readonly supportedFamilies = ['hard-surface'] as const; + + generateSceneGraph( + _understanding: ProductUnderstanding, + _geometryIntelligence: GeometryIntelligence, + sceneGraph: SceneGraph, + ): SceneGraph { + const parts = sceneGraph.parts.map(part => { + const p = { ...part, material: { ...part.material } }; + const label = p.label.toLowerCase(); + + // Wooden surfaces: enforce appropriate roughness + if (label.includes('wood') || label.includes('top') || label.includes('seat') || label.includes('panel')) { + p.material.roughness = Math.max(p.material.roughness, 0.5); + p.material.metalness = 0; + } + + // Metal legs: enforce metalness + if (label.includes('leg') && p.material.metalness > 0.3) { + p.material.roughness = Math.min(p.material.roughness, 0.4); + } + + return p; + }); + + return { ...sceneGraph, parts }; + } +} diff --git a/libs/ai/src/lib/generators/jewelry.generator.ts b/libs/ai/src/lib/generators/jewelry.generator.ts new file mode 100644 index 0000000..5e3d041 --- /dev/null +++ b/libs/ai/src/lib/generators/jewelry.generator.ts @@ -0,0 +1,55 @@ +import type { SceneGraph } from '../types/scene-graph.types.js'; +import type { ProductUnderstanding, GeometryIntelligence } from '../types/product-understanding.types.js'; +import type { ICategoryGenerator } from './category-generator.interface.js'; + +export class JewelryGenerator implements ICategoryGenerator { + readonly supportedSubtypes = [ + 'ring', 'bracelet', 'necklace', 'earring', 'pendant', 'brooch', + 'watch', 'chain', 'bangle', 'jewelry', 'jewellery', + ]; + readonly supportedFamilies = ['hard-surface', 'mechanical'] as const; + + generateSceneGraph( + _understanding: ProductUnderstanding, + _geometryIntelligence: GeometryIntelligence, + sceneGraph: SceneGraph, + ): SceneGraph { + const bb = sceneGraph.boundingBox; + + // Clamp scale to plausible jewelry dimensions (0.5 cm – 25 cm per axis) + const clampedBb = { + width: Math.min(Math.max(bb.width, 0.005), 0.25), + height: Math.min(Math.max(bb.height, 0.005), 0.25), + depth: Math.min(Math.max(bb.depth, 0.001), 0.15), + }; + + const scaleFactor = bb.width > 0 ? clampedBb.width / bb.width : 1; + + const parts = sceneGraph.parts.map(part => { + const p = { ...part, material: { ...part.material } }; + + // Jewelry: high metalness, smooth surfaces + p.material.metalness = Math.max(p.material.metalness, 0.7); + p.material.roughness = Math.min(p.material.roughness, 0.3); + p.smooth = true; + + // Scale all positions and dimensions proportionally + if (scaleFactor !== 1) { + p.position = [p.position[0] * scaleFactor, p.position[1] * scaleFactor, p.position[2] * scaleFactor]; + p.dimensions = { + ...p.dimensions, + width: p.dimensions.width * scaleFactor, + height: p.dimensions.height * scaleFactor, + depth: p.dimensions.depth * scaleFactor, + }; + } + + return p; + }); + + const warnings = [...sceneGraph.structuralWarnings]; + if (scaleFactor !== 1) warnings.push(`JewelryGenerator: scale clamped (factor: ${scaleFactor.toFixed(3)})`); + + return { ...sceneGraph, boundingBox: clampedBb, parts, structuralWarnings: warnings }; + } +} diff --git a/libs/ai/src/lib/generators/packaging.generator.ts b/libs/ai/src/lib/generators/packaging.generator.ts new file mode 100644 index 0000000..fba355c --- /dev/null +++ b/libs/ai/src/lib/generators/packaging.generator.ts @@ -0,0 +1,47 @@ +import type { SceneGraph, ScenePart } from '../types/scene-graph.types.js'; +import type { ProductUnderstanding, GeometryIntelligence } from '../types/product-understanding.types.js'; +import type { ICategoryGenerator } from './category-generator.interface.js'; + +function isBottleBody(part: ScenePart): boolean { + const l = part.label.toLowerCase(); + return (l.includes('body') || l.includes('bottle') || l.includes('container')) && !l.includes('cap'); +} + +export class PackagingGenerator implements ICategoryGenerator { + readonly supportedSubtypes = [ + 'bottle', 'can', 'jar', 'box', 'carton', 'tube', 'container', + 'flask', 'vase', 'cup', 'mug', 'packaging', + ]; + readonly supportedFamilies = ['cylindrical', 'hard-surface'] as const; + + generateSceneGraph( + _understanding: ProductUnderstanding, + _geometryIntelligence: GeometryIntelligence, + sceneGraph: SceneGraph, + ): SceneGraph { + const parts = sceneGraph.parts.map(part => { + const p = { ...part, material: { ...part.material } }; + const label = p.label.toLowerCase(); + + // Bottle/can body: enforce cylindrical shape + if (isBottleBody(p) && p.shape === 'box') { + p.shape = 'cylinder'; + } + + // Glass containers: add transmission + if ((label.includes('glass') || label.includes('transparent')) && p.material.transmissionFactor == null) { + p.material.transmissionFactor = 0.7; + p.material.roughness = Math.min(p.material.roughness, 0.1); + } + + // Ensure minimum segments for circular shapes + if (p.shape === 'cylinder' && (!p.segments || p.segments < 16)) { + p.segments = 24; + } + + return p; + }); + + return { ...sceneGraph, parts }; + } +} diff --git a/libs/ai/src/lib/generators/vehicle-generator.spec.ts b/libs/ai/src/lib/generators/vehicle-generator.spec.ts new file mode 100644 index 0000000..26c8599 --- /dev/null +++ b/libs/ai/src/lib/generators/vehicle-generator.spec.ts @@ -0,0 +1,216 @@ +import { VehicleGenerator } from './vehicle.generator.js'; +import type { SceneGraph, ScenePart } from '../types/scene-graph.types.js'; +import type { ProductUnderstanding, GeometryIntelligence } from '../types/product-understanding.types.js'; + +function makeGraph(parts: ScenePart[], overrides: Partial = {}): SceneGraph { + return { + schemaVersion: '2.0', + productCategory: 'vehicle', + productSubtype: 'car', + geometryFamily: 'hard-surface', + symmetryAxis: 'x', + boundingBox: { width: 1.8, height: 1.4, depth: 4.5 }, + parts, + confidence: 0.85, + sourceViewsUsed: ['front', 'left'], + structuralWarnings: [], + ...overrides, + }; +} + +function makeUnderstanding(overrides: Partial = {}): ProductUnderstanding { + return { + detectedCategory: 'vehicle', + detectedSubtype: 'car', + geometryFamily: 'hard-surface', + structuralParts: [], + symmetryAxis: 'x', + estimatedBoundingBox: { width: 1.8, height: 1.4, depth: 4.5 }, + viewAnglesDetected: ['front'], + confidence: 0.85, + structuralWarnings: [], + ...overrides, + }; +} + +function makeGeomIntelligence(): GeometryIntelligence { + return { + geometryFamily: 'hard-surface', + recommendedSegments: {}, + smoothShadingParts: [], + hardEdgeParts: [], + criticalTopologyNotes: [], + }; +} + +function makePart(id: string, label: string, overrides: Partial = {}): ScenePart { + return { + id, label, + shape: 'box', + dimensions: { width: 0.3, height: 0.3, depth: 0.3 }, + position: [0, 0.15, 0], + rotation: [0, 0, 0, 1], + material: { baseColor: [0.2, 0.2, 0.8, 1], roughness: 0.3, metalness: 0.8 }, + ...overrides, + }; +} + +function makeWheelPart(id: string, position: [number, number, number]): ScenePart { + return { + id, label: id, + shape: 'torus', + dimensions: { width: 0.64, height: 0.18, depth: 0.18, tubeRadius: 0.09, majorRadius: 0.32 }, + position, + rotation: [0.7071, 0, 0, 0.7071], + material: { baseColor: [0.1, 0.1, 0.1, 1], roughness: 0.9, metalness: 0 }, + smooth: true, + segments: 32, + }; +} + +const generator = new VehicleGenerator(); +const understanding = makeUnderstanding(); +const geomIntel = makeGeomIntelligence(); + +describe('VehicleGenerator', () => { + // ─── supportedSubtypes ──────────────────────────────────────────────────── + + it('includes common car subtypes', () => { + expect(generator.supportedSubtypes).toContain('car'); + expect(generator.supportedSubtypes).toContain('sedan'); + expect(generator.supportedSubtypes).toContain('suv'); + expect(generator.supportedSubtypes).toContain('truck'); + }); + + // ─── Window glass enforcement ───────────────────────────────────────────── + + it('adds transmissionFactor to windshield parts that are missing it', () => { + const graph = makeGraph([ + makePart('body', 'car body'), + makePart('windshield', 'windshield', { material: { baseColor: [0.8, 0.9, 1, 0.3], roughness: 0.05, metalness: 0 } }), + makeWheelPart('wheel-fl', [-0.9, 0.32, 1.2]), + makeWheelPart('wheel-rl', [-0.9, 0.32, -1.2]), + ]); + const result = generator.generateSceneGraph(understanding, geomIntel, graph); + const windshield = result.parts.find(p => p.id === 'windshield'); + expect(windshield?.material.transmissionFactor).toBeCloseTo(0.85); + }); + + it('does not override transmissionFactor if already set', () => { + const graph = makeGraph([ + makePart('body', 'car body'), + makePart('windshield', 'windshield', { material: { baseColor: [0.8, 0.9, 1, 0.3], roughness: 0.05, metalness: 0, transmissionFactor: 0.6 } }), + makeWheelPart('wheel-fl', [-0.9, 0.32, 1.2]), + makeWheelPart('wheel-rl', [-0.9, 0.32, -1.2]), + ]); + const result = generator.generateSceneGraph(understanding, geomIntel, graph); + const windshield = result.parts.find(p => p.id === 'windshield'); + // 0.6 was set explicitly — should be kept (nullish coalescing) + expect(windshield?.material.transmissionFactor).toBeCloseTo(0.6); + }); + + it('lowers roughness on window parts to ≤ 0.1', () => { + const graph = makeGraph([ + makePart('body', 'car body'), + makePart('side window', 'side window', { material: { baseColor: [0.8, 0.9, 1, 0.3], roughness: 0.5, metalness: 0 } }), + makeWheelPart('wheel-fl', [-0.9, 0.32, 1.2]), + makeWheelPart('wheel-rl', [-0.9, 0.32, -1.2]), + ]); + const result = generator.generateSceneGraph(understanding, geomIntel, graph); + const win = result.parts.find(p => p.label === 'side window'); + expect(win?.material.roughness).toBeLessThanOrEqual(0.1); + }); + + // ─── Wheel shape enforcement ─────────────────────────────────────────────── + + it('converts box-shaped wheel parts to torus', () => { + const graph = makeGraph([ + makePart('body', 'car body'), + makePart('wheel-fl', 'front left wheel', { position: [-0.9, 0.32, 1.2] }), + makePart('wheel-rl', 'rear left wheel', { position: [-0.9, 0.32, -1.2] }), + ]); + const result = generator.generateSceneGraph(understanding, geomIntel, graph); + const wheels = result.parts.filter(p => p.label.toLowerCase().includes('wheel')); + for (const wheel of wheels) { + expect(wheel.shape).toBe('torus'); + } + }); + + it('keeps torus wheels unchanged', () => { + const graph = makeGraph([ + makePart('body', 'car body'), + makeWheelPart('wheel-fl', [-0.9, 0.32, 1.2]), + makeWheelPart('wheel-rl', [-0.9, 0.32, -1.2]), + ]); + const result = generator.generateSceneGraph(understanding, geomIntel, graph); + const wheels = result.parts.filter(p => p.shape === 'torus'); + expect(wheels.length).toBeGreaterThanOrEqual(2); + }); + + // ─── Missing wheel injection ─────────────────────────────────────────────── + + it('adds default wheel parts when none exist', () => { + const graph = makeGraph([ + makePart('body', 'car body'), + ]); + const result = generator.generateSceneGraph(understanding, geomIntel, graph); + const wheels = result.parts.filter(p => p.label.toLowerCase().includes('wheel')); + expect(wheels.length).toBeGreaterThanOrEqual(2); + }); + + it('injected wheel parts use torus shape', () => { + const graph = makeGraph([ + makePart('body', 'car body'), + ]); + const result = generator.generateSceneGraph(understanding, geomIntel, graph); + const wheels = result.parts.filter(p => p.label.toLowerCase().includes('wheel')); + for (const w of wheels) { + expect(w.shape).toBe('torus'); + } + }); + + it('adds a warning when wheels were injected', () => { + const graph = makeGraph([makePart('body', 'car body')]); + const result = generator.generateSceneGraph(understanding, geomIntel, graph); + expect(result.structuralWarnings.some(w => w.includes('VehicleGenerator'))).toBe(true); + }); + + it('does not add warnings when wheels already exist', () => { + const graph = makeGraph([ + makePart('body', 'car body'), + makeWheelPart('wheel-fl', [-0.9, 0.32, 1.2]), + makeWheelPart('wheel-rl', [-0.9, 0.32, -1.2]), + ]); + const result = generator.generateSceneGraph(understanding, geomIntel, graph); + expect(result.structuralWarnings.some(w => w.includes('VehicleGenerator'))).toBe(false); + }); + + // ─── Bounding box clamping ──────────────────────────────────────────────── + + it('clamps bounding box height to [1.0, 2.5] for vehicles', () => { + const graph = makeGraph( + [ + makePart('body', 'car body'), + makeWheelPart('wheel-fl', [-0.9, 0.32, 1.2]), + makeWheelPart('wheel-rl', [-0.9, 0.32, -1.2]), + ], + { boundingBox: { width: 1.8, height: 10, depth: 4.5 } }, + ); + const result = generator.generateSceneGraph(understanding, geomIntel, graph); + expect(result.boundingBox.height).toBeLessThanOrEqual(2.5); + expect(result.boundingBox.height).toBeGreaterThanOrEqual(1.0); + }); + + it('clamps bounding box depth to [2.5, 6.0] for vehicles', () => { + const graph = makeGraph( + [ + makePart('body', 'car body'), + makeWheelPart('wheel-fl', [-0.9, 0.32, 1.2]), + makeWheelPart('wheel-rl', [-0.9, 0.32, -1.2]), + ], + { boundingBox: { width: 1.8, height: 1.4, depth: 0.5 } }, // absurdly short + ); + const result = generator.generateSceneGraph(understanding, geomIntel, graph); + expect(result.boundingBox.depth).toBeGreaterThanOrEqual(2.5); + }); +}); diff --git a/libs/ai/src/lib/generators/vehicle.generator.ts b/libs/ai/src/lib/generators/vehicle.generator.ts new file mode 100644 index 0000000..8cdd359 --- /dev/null +++ b/libs/ai/src/lib/generators/vehicle.generator.ts @@ -0,0 +1,94 @@ +import type { SceneGraph, ScenePart } from '../types/scene-graph.types.js'; +import type { ProductUnderstanding, GeometryIntelligence } from '../types/product-understanding.types.js'; +import type { ICategoryGenerator } from './category-generator.interface.js'; + +const WHEEL_LABELS = ['wheel', 'tire', 'tyre']; +const WINDOW_LABELS = ['window', 'windshield', 'glass', 'rear window', 'side window']; + +function isWheel(part: ScenePart): boolean { + return WHEEL_LABELS.some(w => part.label.toLowerCase().includes(w)); +} + +function isWindow(part: ScenePart): boolean { + return WINDOW_LABELS.some(w => part.label.toLowerCase().includes(w)); +} + +function makeWheel(id: string, label: string, x: number, y: number, z: number): ScenePart { + const majorRadius = 0.32; + const tubeRadius = 0.09; + return { + id, + label, + shape: 'torus', + dimensions: { width: majorRadius * 2, height: tubeRadius * 2, depth: tubeRadius * 2, tubeRadius, majorRadius }, + position: [x, y, z], + rotation: [0.7071068, 0, 0, 0.7071068], + material: { baseColor: [0.1, 0.1, 0.1, 1], roughness: 0.9, metalness: 0 }, + smooth: true, + segments: 32, + symmetryMirror: x < 0 ? 'x' : undefined, + }; +} + +export class VehicleGenerator implements ICategoryGenerator { + readonly supportedSubtypes = ['car', 'sedan', 'suv', 'truck', 'van', 'pickup', 'coupe', 'hatchback', 'convertible', 'vehicle', 'automobile']; + readonly supportedFamilies = ['hard-surface', 'mechanical'] as const; + + generateSceneGraph( + understanding: ProductUnderstanding, + _geometryIntelligence: GeometryIntelligence, + sceneGraph: SceneGraph, + ): SceneGraph { + const parts = [...sceneGraph.parts]; + + // 1. Fix windows: must have transmissionFactor + for (const part of parts) { + if (isWindow(part)) { + part.material.transmissionFactor ??= 0.85; + part.material.roughness = Math.min(part.material.roughness, 0.1); + if (!part.material.baseColor || part.material.baseColor[3] > 0.5) { + part.material.baseColor = [0.8, 0.9, 1.0, 0.3]; + } + } + } + + // 2. Fix wheels: replace box/cylinder wheels with tori + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + if (isWheel(part) && part.shape !== 'torus') { + const [px, py, pz] = part.position; + parts[i] = makeWheel(part.id, part.label, px, py, pz); + } + } + + // 3. Ensure at least 4 wheel parts exist + const wheelParts = parts.filter(isWheel); + const hasEnoughWheels = wheelParts.length >= 2; // at least 2 (front+rear) since mirroring doubles them + + if (!hasEnoughWheels) { + const bb = sceneGraph.boundingBox; + const majorRadius = Math.min(bb.height * 0.22, 0.35); + const wheelY = majorRadius; + const wheelX = -(bb.width / 2 + 0.05); + const wheelZF = bb.depth / 2 * 0.55; + const wheelZR = -bb.depth / 2 * 0.55; + + // Add front-left and rear-left (mirrored to produce right counterparts) + parts.push(makeWheel('wheel-front-left', 'front left wheel', wheelX, wheelY, wheelZF)); + parts.push(makeWheel('wheel-rear-left', 'rear left wheel', wheelX, wheelY, wheelZR)); + } + + // 4. Enforce bounding box sanity for vehicles + const bb = sceneGraph.boundingBox; + const clampedBb = { + width: Math.min(Math.max(bb.width, 1.4), 3.0), + height: Math.min(Math.max(bb.height, 1.0), 2.5), + depth: Math.min(Math.max(bb.depth, 2.5), 6.0), + }; + + const warnings = [...sceneGraph.structuralWarnings]; + if (!hasEnoughWheels) warnings.push('VehicleGenerator: Wheels were missing — added default wheel positions'); + + return { ...sceneGraph, parts, boundingBox: clampedBb, structuralWarnings: warnings }; + } +} diff --git a/libs/ai/src/lib/prompts/geometry-classification.prompt.ts b/libs/ai/src/lib/prompts/geometry-classification.prompt.ts new file mode 100644 index 0000000..615a801 --- /dev/null +++ b/libs/ai/src/lib/prompts/geometry-classification.prompt.ts @@ -0,0 +1,46 @@ +import type { ProductUnderstanding } from '../types/product-understanding.types.js'; + +export function buildGeometryClassificationPrompt(understanding: ProductUnderstanding): string { + const partsJson = JSON.stringify( + understanding.structuralParts.map(p => ({ partId: p.partId, label: p.label, geometryHint: p.geometryHint, relativeSize: p.relativeSize })), + null, 2, + ); + + return `You are a 3D topology expert. Given the following product understanding, return tessellation and shading instructions for each part. + +PRODUCT: +- Category: ${understanding.detectedCategory} +- Subtype: ${understanding.detectedSubtype} +- Geometry family: ${understanding.geometryFamily} + +STRUCTURAL PARTS: +${partsJson} + +YOUR TASK: +1. Confirm or refine the geometryFamily. +2. For each part, specify the recommended segment count (number of side faces for cylinders/spheres/tori). +3. Classify each part as smooth-shaded or flat-shaded. +4. Note any critical topology constraints (e.g. "wheels must be circular, not faceted"). + +SEGMENT GUIDE: +- Wheels/tires: 32 (must look circular) +- Round bottles/cans: 24 +- Cylindrical legs: 12–16 +- Spherical objects: lat=12, lon=16 +- Extruded ellipses (car body): 32 +- Tapered shapes (bottle neck): 20 +- Flat box surfaces: ignore (N/A) + +SMOOTH SHADING GUIDE: +- Smooth: organic shapes, car body, spheres, cylindrical surfaces, human-worn items +- Flat (hard edge): furniture boards, electronic panels, packaging boxes, frames + +OUTPUT: Return ONLY valid JSON, no markdown. Schema: +{ + "geometryFamily": string, + "recommendedSegments": { "": number }, + "smoothShadingParts": string[], + "hardEdgeParts": string[], + "criticalTopologyNotes": string[] +}`; +} diff --git a/libs/ai/src/lib/prompts/pbr-material.prompt.ts b/libs/ai/src/lib/prompts/pbr-material.prompt.ts new file mode 100644 index 0000000..62ab202 --- /dev/null +++ b/libs/ai/src/lib/prompts/pbr-material.prompt.ts @@ -0,0 +1,46 @@ +export function buildPbrMaterialPrompt(partIds: string[], partLabels: string[]): string { + const partList = partIds.map((id, i) => ` { "partId": "${id}", "label": "${partLabels[i] ?? id}" }`).join(',\n'); + + return `You are a 3D materials expert. Analyze the provided product image(s) and determine the PBR material properties for each part listed below. + +PARTS TO ANALYZE: +[ +${partList} +] + +FOR EACH PART: +1. Identify the dominant material (e.g. "painted-steel", "rubber", "glass", "fabric", "leather", "chrome", "matte-plastic", "wood", "ceramic") +2. Sample the dominant color from the image region that corresponds to this part +3. Estimate PBR properties: + - roughness: 0 (mirror) to 1 (completely matte) + - metalness: 0 (non-metal) to 1 (full metal) + - transmissionFactor: 0–1 (only for glass/transparent parts, otherwise null) + - emissiveFactor: [r,g,b] 0–2 range (only for emissive parts like screens/lights, otherwise null) + +MATERIAL QUICK REFERENCE: +- Painted metal (car body): roughness 0.2–0.4, metalness 0.7–0.9 +- Chrome (bumpers, trim): roughness 0.05, metalness 1.0, color [0.85,0.85,0.85,1] +- Rubber (tires, grips): roughness 0.85–0.95, metalness 0.0, color [0.1,0.1,0.1,1] +- Glass (windows, screens): roughness 0.05, metalness 0.1, transmissionFactor 0.85 +- Fabric (upholstery, clothing): roughness 0.75–0.9, metalness 0.0 +- Matte plastic: roughness 0.6–0.8, metalness 0.0 +- Wood: roughness 0.65–0.8, metalness 0.0 +- Screen (off): roughness 0.1, metalness 0.1, color [0.05,0.05,0.05,1] + +OUTPUT: Return ONLY valid JSON, no markdown. Schema: +{ + "parts": [ + { + "partId": string, + "baseColor": [r, g, b, a], + "roughness": number, + "metalness": number, + "transmissionFactor": number | null, + "ior": number | null, + "clearcoat": number | null, + "emissiveFactor": [r, g, b] | null, + "dominantMaterial": string + } + ] +}`; +} diff --git a/libs/ai/src/lib/prompts/product-understanding.prompt.ts b/libs/ai/src/lib/prompts/product-understanding.prompt.ts new file mode 100644 index 0000000..ee245fb --- /dev/null +++ b/libs/ai/src/lib/prompts/product-understanding.prompt.ts @@ -0,0 +1,108 @@ +import type { QualityHint } from '../types/ai-request.types.js'; + +const GEOMETRY_FAMILIES = [ + 'hard-surface', + 'organic', + 'cloth-fabric', + 'cylindrical', + 'mechanical', + 'soft-body', +] as const; + +const PRIMITIVE_SHAPES = [ + 'box', + 'cylinder', + 'sphere', + 'tapered-cylinder', + 'frustum', + 'wedge', + 'torus', + 'extruded-ellipse', +] as const; + +const RELATIVE_SIZES = ['dominant', 'large', 'medium', 'small', 'detail'] as const; + +export interface ProductUnderstandingPromptInput { + productCategory: string; + productTitle?: string; + productDimensions?: string; + inferredMaterial?: string; + imageViewAngles?: string[]; + quality: QualityHint; +} + +export function buildProductUnderstandingPrompt(input: ProductUnderstandingPromptInput): string { + const { productCategory, productTitle, productDimensions, inferredMaterial, imageViewAngles, quality } = input; + + const qualityInstructions = quality === 'fast' + ? 'List only the 3–5 most visually dominant structural parts. Prioritize speed over completeness.' + : quality === 'quality' + ? 'List every distinct structural component you can identify, including symmetric pairs. Be thorough and precise.' + : 'List the main structural parts — aim for 5–12 parts that define the product shape.'; + + const viewInfo = imageViewAngles?.length + ? `Images provided: ${imageViewAngles.join(', ')} view(s).` + : 'Single image provided.'; + + return `You are an expert 3D reconstruction AI. Analyze the provided product image(s) and output a structured JSON describing the product's 3D structure. + +CONTEXT: +- Product category hint: "${productCategory}" +${productTitle ? `- Product name: "${productTitle}"` : ''} +${productDimensions ? `- Declared dimensions: ${productDimensions}` : ''} +${inferredMaterial ? `- Inferred material: ${inferredMaterial}` : ''} +- ${viewInfo} + +COORDINATE SYSTEM: +- Y axis = up +- Z axis = forward (front of product) +- X axis = right +- Origin = center of the product's base (bottom center) +- All dimensions in metres + +GEOMETRY FAMILIES (pick exactly one): +${GEOMETRY_FAMILIES.map(f => ` "${f}"`).join('\n')} + +PRIMITIVE SHAPES (geometryHint per part): +${PRIMITIVE_SHAPES.map(s => ` "${s}"`).join('\n')} + +RELATIVE SIZES (pick one per part): +${RELATIVE_SIZES.map(s => ` "${s}"`).join('\n')} + +ANALYSIS TASK: +${qualityInstructions} + +IMPORTANT RULES: +1. Override the category hint if you identify a different product. +2. For symmetric products (cars, headphones, chairs), identify each symmetric part ONCE and add symmetricCounterpart pointing to the other side's partId. The engine will mirror it automatically. +3. Never use "box" for circular objects (wheels, buttons, knobs) — use "cylinder" or "torus". +4. A car MUST have: body, 4 wheels (2 front + 2 rear — use symmetricCounterpart), windshield, rear window, headlights, taillights, front bumper, rear bumper. List only one from each symmetric pair and mark symmetricCounterpart. +5. A t-shirt MUST use geometryFamily "cloth-fabric". +6. Glass/transparent surfaces must be identified (material = "glass" or "transparent-plastic"). +7. relativePosition uses format "center", "top-center", "bottom-left", "front-center", etc. +8. If dimensions are declared, use them to estimate the bounding box accurately. + +OUTPUT: Return ONLY valid JSON, no markdown, no code fences. Schema: +{ + "detectedCategory": string, + "detectedSubtype": string, + "geometryFamily": "${GEOMETRY_FAMILIES.join('" | "')}", + "structuralParts": [ + { + "partId": string, + "label": string, + "geometryHint": "${PRIMITIVE_SHAPES.join('" | "')}", + "relativeSize": "${RELATIVE_SIZES.join('" | "')}", + "relativePosition": string, + "material": string, + "isVisible": boolean, + "symmetricCounterpart": string | null + } + ], + "symmetryAxis": "x" | "z" | "none", + "estimatedBoundingBox": { "width": number, "height": number, "depth": number }, + "viewAnglesDetected": string[], + "confidence": number, + "structuralWarnings": string[] +}`; +} diff --git a/libs/ai/src/lib/prompts/scale-estimation.prompt.ts b/libs/ai/src/lib/prompts/scale-estimation.prompt.ts new file mode 100644 index 0000000..1636b5d --- /dev/null +++ b/libs/ai/src/lib/prompts/scale-estimation.prompt.ts @@ -0,0 +1,82 @@ +import type { ScaleBounds } from '../types/product-understanding.types.js'; + +/** Known real-world size references keyed by subtype keyword. */ +const SIZE_PRIORS: Record = { + car: { wM: [1.6, 2.1], hM: [1.2, 2.0], dM: [3.5, 5.5] }, + suv: { wM: [1.7, 2.2], hM: [1.6, 2.2], dM: [4.0, 5.5] }, + motorcycle: { wM: [0.6, 0.9], hM: [1.0, 1.4], dM: [1.8, 2.5] }, + bicycle: { wM: [0.4, 0.6], hM: [0.9, 1.2], dM: [1.5, 2.0] }, + chair: { wM: [0.4, 0.7], hM: [0.8, 1.1], dM: [0.4, 0.7] }, + sofa: { wM: [1.5, 2.8], hM: [0.7, 1.0], dM: [0.7, 1.1] }, + table: { wM: [0.6, 2.0], hM: [0.7, 0.8], dM: [0.6, 1.2] }, + desk: { wM: [1.0, 1.8], hM: [0.7, 0.8], dM: [0.5, 0.9] }, + laptop: { wM: [0.28, 0.40], hM: [0.17, 0.28], dM: [0.15, 0.30] }, + phone: { wM: [0.07, 0.09], hM: [0.13, 0.18], dM: [0.006, 0.012] }, + tablet: { wM: [0.15, 0.30], hM: [0.20, 0.40], dM: [0.005, 0.010] }, + monitor: { wM: [0.45, 1.2], hM: [0.35, 0.75], dM: [0.10, 0.30] }, + headphones: { wM: [0.15, 0.22], hM: [0.18, 0.25], dM: [0.06, 0.12] }, + earbuds: { wM: [0.02, 0.04], hM: [0.02, 0.04], dM: [0.02, 0.04] }, + bottle: { wM: [0.06, 0.12], hM: [0.15, 0.35], dM: [0.06, 0.12] }, + cup: { wM: [0.06, 0.12], hM: [0.08, 0.15], dM: [0.06, 0.12] }, + shoe: { wM: [0.08, 0.14], hM: [0.08, 0.18], dM: [0.22, 0.32] }, + shirt: { wM: [0.40, 0.65], hM: [0.55, 0.80], dM: [0.01, 0.04] }, + watch: { wM: [0.03, 0.05], hM: [0.03, 0.05], dM: [0.01, 0.015] }, + ring: { wM: [0.015, 0.025], hM: [0.005, 0.015], dM: [0.015, 0.025] }, + lamp: { wM: [0.25, 0.60], hM: [0.40, 1.80], dM: [0.25, 0.60] }, + bookshelf: { wM: [0.60, 1.20], hM: [1.20, 2.20], dM: [0.25, 0.40] }, +}; + +function findPrior(subtype: string) { + const lower = subtype.toLowerCase(); + for (const [key, val] of Object.entries(SIZE_PRIORS)) { + if (lower.includes(key)) return val; + } + return null; +} + +export function buildScaleEstimationPrompt(subtype: string, declaredDimensions?: string): string { + const prior = findPrior(subtype); + const priorNote = prior + ? `Known real-world range for "${subtype}": W ${prior.wM[0]}–${prior.wM[1]} m, H ${prior.hM[0]}–${prior.hM[1]} m, D ${prior.dM[0]}–${prior.dM[1]} m.` + : `No known reference found for "${subtype}". Use visual estimation.`; + + const dimNote = declaredDimensions + ? `Declared dimensions from product listing: "${declaredDimensions}". Parse and convert to metres. Use these as the primary source.` + : 'No declared dimensions available. Use category knowledge and visual estimation.'; + + return `Estimate the real-world 3D scale of this product. + +Product subtype: "${subtype}" +${dimNote} +${priorNote} + +OUTPUT: Return ONLY valid JSON. +{ + "widthM": { "min": number, "best": number, "max": number }, + "heightM": { "min": number, "best": number, "max": number }, + "depthM": { "min": number, "best": number, "max": number }, + "confidence": "high" | "medium" | "low", + "referenceSource": "declared-dimensions" | "category-knowledge" | "visual-estimate" +}`; +} + +/** Derive scale bounds from known priors without calling Gemini. */ +export function getStaticScaleBounds(subtype: string, declaredDimensions?: string): ScaleBounds { + const prior = findPrior(subtype); + if (prior) { + return { + widthM: { min: prior.wM[0], best: (prior.wM[0] + prior.wM[1]) / 2, max: prior.wM[1] }, + heightM: { min: prior.hM[0], best: (prior.hM[0] + prior.hM[1]) / 2, max: prior.hM[1] }, + depthM: { min: prior.dM[0], best: (prior.dM[0] + prior.dM[1]) / 2, max: prior.dM[1] }, + confidence: 'medium', + referenceSource: declaredDimensions ? 'declared-dimensions' : 'category-knowledge', + }; + } + return { + widthM: { min: 0.05, best: 0.3, max: 2.0 }, + heightM: { min: 0.05, best: 0.3, max: 2.0 }, + depthM: { min: 0.05, best: 0.3, max: 2.0 }, + confidence: 'low', + referenceSource: 'visual-estimate', + }; +} diff --git a/libs/ai/src/lib/prompts/scale-estimation.spec.ts b/libs/ai/src/lib/prompts/scale-estimation.spec.ts new file mode 100644 index 0000000..a8cfb31 --- /dev/null +++ b/libs/ai/src/lib/prompts/scale-estimation.spec.ts @@ -0,0 +1,85 @@ +import { getStaticScaleBounds } from './scale-estimation.prompt.js'; + +describe('getStaticScaleBounds', () => { + // ─── Known categories ─────────────────────────────────────────────────────── + + it('returns car bounds in vehicle range (W 1.6–2.1 m)', () => { + const bounds = getStaticScaleBounds('car'); + expect(bounds.widthM.min).toBeGreaterThanOrEqual(1.6); + expect(bounds.widthM.max).toBeLessThanOrEqual(2.1); + expect(bounds.widthM.best).toBeGreaterThan(0); + }); + + it('returns phone bounds in sub-10cm range', () => { + const bounds = getStaticScaleBounds('phone'); + expect(bounds.widthM.max).toBeLessThan(0.2); + expect(bounds.heightM.max).toBeLessThan(0.25); + }); + + it('returns bottle height in 15–35cm range', () => { + const bounds = getStaticScaleBounds('bottle'); + expect(bounds.heightM.min).toBeCloseTo(0.15); + expect(bounds.heightM.max).toBeCloseTo(0.35); + }); + + it('returns chair height in 80–110cm range', () => { + const bounds = getStaticScaleBounds('chair'); + expect(bounds.heightM.min).toBeCloseTo(0.8); + expect(bounds.heightM.max).toBeCloseTo(1.1); + }); + + it('returns laptop depth in 15–30cm range (closed)', () => { + const bounds = getStaticScaleBounds('laptop'); + expect(bounds.depthM.min).toBeCloseTo(0.15); + expect(bounds.depthM.max).toBeCloseTo(0.30); + }); + + it('returns ring bounds in mm-to-cm scale', () => { + const bounds = getStaticScaleBounds('ring'); + expect(bounds.widthM.max).toBeLessThan(0.05); + expect(bounds.widthM.min).toBeGreaterThan(0); + }); + + // ─── best values ──────────────────────────────────────────────────────────── + + it('best value is midpoint of min/max', () => { + const bounds = getStaticScaleBounds('chair'); + expect(bounds.widthM.best).toBeCloseTo((bounds.widthM.min + bounds.widthM.max) / 2); + expect(bounds.heightM.best).toBeCloseTo((bounds.heightM.min + bounds.heightM.max) / 2); + expect(bounds.depthM.best).toBeCloseTo((bounds.depthM.min + bounds.depthM.max) / 2); + }); + + // ─── Confidence / source ──────────────────────────────────────────────────── + + it('returns medium confidence for known categories', () => { + expect(getStaticScaleBounds('laptop').confidence).toBe('medium'); + expect(getStaticScaleBounds('car').confidence).toBe('medium'); + }); + + it('returns low confidence for unknown category', () => { + expect(getStaticScaleBounds('spaceship').confidence).toBe('low'); + }); + + it('uses category-knowledge as referenceSource when no dimensions declared', () => { + expect(getStaticScaleBounds('chair').referenceSource).toBe('category-knowledge'); + }); + + it('uses declared-dimensions as referenceSource when dimensions are provided', () => { + expect(getStaticScaleBounds('chair', '45 x 45 x 88 cm').referenceSource).toBe('declared-dimensions'); + }); + + // ─── Unknown fallback ──────────────────────────────────────────────────────── + + it('returns a non-zero best value for unknown categories', () => { + const bounds = getStaticScaleBounds('ufo-shaped-desk'); + expect(bounds.widthM.best).toBeGreaterThan(0); + expect(bounds.heightM.best).toBeGreaterThan(0); + expect(bounds.depthM.best).toBeGreaterThan(0); + }); + + it('returns min < best < max for all axes for unknown categories', () => { + const bounds = getStaticScaleBounds('unknown-product-42'); + expect(bounds.widthM.min).toBeLessThan(bounds.widthM.best); + expect(bounds.widthM.best).toBeLessThan(bounds.widthM.max); + }); +}); diff --git a/libs/ai/src/lib/prompts/scene-graph-reconstruction.prompt.ts b/libs/ai/src/lib/prompts/scene-graph-reconstruction.prompt.ts new file mode 100644 index 0000000..6557c70 --- /dev/null +++ b/libs/ai/src/lib/prompts/scene-graph-reconstruction.prompt.ts @@ -0,0 +1,137 @@ +import type { ProductUnderstanding, GeometryIntelligence, ScaleBounds } from '../types/product-understanding.types.js'; +import type { QualityHint } from '../types/ai-request.types.js'; + +export function buildSceneGraphReconstructionPrompt( + understanding: ProductUnderstanding, + geometryIntelligence: GeometryIntelligence, + scaleBounds: ScaleBounds, + quality: QualityHint, +): string { + const partsJson = JSON.stringify(understanding.structuralParts, null, 2); + const geomJson = JSON.stringify(geometryIntelligence, null, 2); + const bb = understanding.estimatedBoundingBox; + + const depthInstruction = quality === 'quality' + ? 'Infer hidden and rear geometry from product knowledge. Estimate thickness, depth, and underside carefully.' + : 'Estimate depth from visible cues and category knowledge. Be reasonable, not exact.'; + + const exampleNote = understanding.detectedSubtype.match(/car|suv|truck|sedan|vehicle/) ? ` +VEHICLE-SPECIFIC INSTRUCTIONS: +- Car body: use "extruded-ellipse" shape — it produces a rounded cross-section. Alternatively use "box" with large dimensions. +- Wheels: MUST use "torus" shape. Major radius (width/2) ≈ 0.32 m for a car. Tube radius ≈ 0.09 m. +- Wheel positions: front-left, front-right (same X magnitude, opposite signs), rear-left, rear-right. + - Y = tire major radius (center of wheel at height = tire outer radius from ground) + - X = ±(body_width/2 + small_offset) ≈ ±0.85 m for a standard car + - Z front wheels ≈ +1.2 m, rear wheels ≈ -1.2 m (origin at center-bottom) + - Mark front-left and rear-left with symmetryMirror: "x" (engine will create right-side counterparts) +- Windows (windshield, rear window): use "box" with very small depth (~0.01 m). Set transmissionFactor: 0.85. +- Car body scale: width ≈ 1.8 m, height ≈ 1.4 m, depth ≈ 4.5 m for a sedan. +` : understanding.detectedSubtype.match(/bottle|can|jar|flask/) ? ` +BOTTLE-SPECIFIC INSTRUCTIONS: +- Body: use "cylinder". Diameter ≈ 0.07 m, height ≈ 0.22 m for a standard bottle. +- Neck: use "tapered-cylinder". radiusBottom ≈ body_radius, radiusTop ≈ 0.015 m. Height ≈ 0.05 m. +- Cap: use "cylinder". Diameter ≈ neck_top_diameter * 1.2. Height ≈ 0.02 m. +` : understanding.detectedSubtype.match(/t-shirt|shirt|hoodie|jacket|sweater/) ? ` +CLOTHING-SPECIFIC INSTRUCTIONS: +- Clothing is FLAT. Body depth ≈ 0.02–0.04 m (fabric thickness when laid flat or on hanger). +- Use "box" for body, sleeves, collar. These are flat panels. +- Roughness MUST be > 0.7. Metalness MUST be 0. +- Symmetry axis = "x" (left/right symmetric). +` : ''; + + return `You are a semantic 3D reconstruction AI. Based on the product understanding and geometry intelligence provided, construct a complete SceneGraph JSON describing the 3D structure of this product. + +COORDINATE SYSTEM: +- Y = up. Origin at center-bottom of product (on the ground plane). +- Z = forward (front of product faces +Z). +- X = right. +- All dimensions in METRES. + +PRODUCT UNDERSTANDING: +Category: ${understanding.detectedCategory} / ${understanding.detectedSubtype} +Geometry family: ${understanding.geometryFamily} +Symmetry axis: ${understanding.symmetryAxis} +Bounding box: ${bb.width}m (W) × ${bb.height}m (H) × ${bb.depth}m (D) +Scale confidence: ${scaleBounds.confidence} (source: ${scaleBounds.referenceSource}) +Scale range W: ${scaleBounds.widthM.min}–${scaleBounds.widthM.max} m (best: ${scaleBounds.widthM.best} m) +Scale range H: ${scaleBounds.heightM.min}–${scaleBounds.heightM.max} m (best: ${scaleBounds.heightM.best} m) +Scale range D: ${scaleBounds.depthM.min}–${scaleBounds.depthM.max} m (best: ${scaleBounds.depthM.best} m) + +STRUCTURAL PARTS TO RECONSTRUCT: +${partsJson} + +GEOMETRY INTELLIGENCE: +${geomJson} +${exampleNote} +DEPTH RECONSTRUCTION: ${depthInstruction} + +SYMMETRY RULES: +- Parts with symmetryMirror="x" will be automatically duplicated at the negative X position. +- Use symmetryMirror ONLY for parts that are genuinely symmetric (left wheel ↔ right wheel, left armrest ↔ right armrest). +- Do NOT use symmetryMirror for parts that are centered (body, seat, hood). + +ROTATION FORMAT: +- rotation field is quaternion [x, y, z, w]. +- No rotation = [0, 0, 0, 1]. +- 90° around Y (facing left) = [0, 0.7071, 0, 0.7071]. +- For a wheel lying flat in XZ plane and rotating around Y: [0, 0, 0, 1] already works for a torus in XZ. + But if the torus is in the XY plane by default, rotate 90° around X: [0.7071, 0, 0, 0.7071]. + +MATERIAL DEFAULTS BY MATERIAL STRING: +- "painted-metal" or "painted-steel": roughness=0.3, metalness=0.8, baseColor from image +- "glass" or "windshield": roughness=0.05, metalness=0.1, transmissionFactor=0.85, baseColor=[0.8,0.9,1.0,0.3] +- "rubber": roughness=0.9, metalness=0.0, baseColor≈[0.1,0.1,0.1,1] +- "fabric" or "cloth": roughness=0.85, metalness=0.0 +- "chrome": roughness=0.05, metalness=1.0, baseColor=[0.85,0.85,0.85,1] +- "plastic": roughness=0.5, metalness=0.0 +- "wood": roughness=0.7, metalness=0.0 + +OUTPUT: Return ONLY valid JSON. No markdown. No code fences. + +SCHEMA (output exactly this structure): +{ + "schemaVersion": "2.0", + "productCategory": string, + "productSubtype": string, + "geometryFamily": string, + "symmetryAxis": "x" | "z" | "none", + "boundingBox": { "width": number, "height": number, "depth": number }, + "parts": [ + { + "id": string, + "label": string, + "shape": "box" | "cylinder" | "sphere" | "tapered-cylinder" | "frustum" | "wedge" | "torus" | "extruded-ellipse", + "dimensions": { + "width": number, + "height": number, + "depth": number, + "topWidth": number | null, + "topDepth": number | null, + "radiusTop": number | null, + "radiusBottom": number | null, + "tubeRadius": number | null, + "majorRadius": number | null, + "rx": number | null, + "ry": number | null + }, + "position": [x, y, z], + "rotation": [x, y, z, w], + "material": { + "baseColor": [r, g, b, a], + "roughness": number, + "metalness": number, + "transmissionFactor": number | null, + "ior": number | null, + "clearcoat": number | null, + "emissiveFactor": [r, g, b] | null + }, + "smooth": boolean, + "segments": number | null, + "symmetryMirror": "x" | "z" | null + } + ], + "confidence": number, + "sourceViewsUsed": string[], + "structuralWarnings": string[] +}`; +} diff --git a/libs/ai/src/lib/types/feedback.types.ts b/libs/ai/src/lib/types/feedback.types.ts new file mode 100644 index 0000000..60ce095 --- /dev/null +++ b/libs/ai/src/lib/types/feedback.types.ts @@ -0,0 +1,16 @@ +import type { GeometryFamily } from './scene-graph.types.js'; + +export type FeedbackSignal = 'approved' | 'rejected' | 'regenerated'; + +export interface GenerationFeedbackRecord { + productId: string; + conversionId: string; + signal: FeedbackSignal; + rejectionReason?: string; + detectedSubtype: string; + geometryFamily: GeometryFamily; + qaScore?: number; + validationScore?: number; + sceneGraphSnapshot?: Record; + createdAt: string; +} diff --git a/libs/ai/src/lib/types/product-understanding.types.ts b/libs/ai/src/lib/types/product-understanding.types.ts new file mode 100644 index 0000000..d41ccce --- /dev/null +++ b/libs/ai/src/lib/types/product-understanding.types.ts @@ -0,0 +1,63 @@ +import type { GeometryFamily, PrimitiveShape } from './scene-graph.types.js'; + +export interface ProductStructuralPart { + partId: string; + label: string; + geometryHint: PrimitiveShape; + relativeSize: 'dominant' | 'large' | 'medium' | 'small' | 'detail'; + relativePosition: string; + material: string; + isVisible: boolean; + symmetricCounterpart?: string; +} + +export interface ProductUnderstanding { + detectedCategory: string; + detectedSubtype: string; + geometryFamily: GeometryFamily; + structuralParts: ProductStructuralPart[]; + symmetryAxis: 'x' | 'z' | 'none'; + estimatedBoundingBox: { width: number; height: number; depth: number }; + viewAnglesDetected: string[]; + confidence: number; + structuralWarnings: string[]; +} + +export interface GeometryIntelligence { + geometryFamily: GeometryFamily; + recommendedSegments: Record; + smoothShadingParts: string[]; + hardEdgeParts: string[]; + criticalTopologyNotes: string[]; +} + +export interface ScaleBounds { + widthM: { min: number; best: number; max: number }; + heightM: { min: number; best: number; max: number }; + depthM: { min: number; best: number; max: number }; + confidence: 'high' | 'medium' | 'low'; + referenceSource: 'declared-dimensions' | 'category-knowledge' | 'visual-estimate'; +} + +export interface PbrMaterialEntry { + partId: string; + baseColor: [number, number, number, number]; + roughness: number; + metalness: number; + transmissionFactor?: number; + ior?: number; + clearcoat?: number; + emissiveFactor?: [number, number, number]; + dominantMaterial: string; +} + +export interface PbrMaterialMap { + parts: PbrMaterialEntry[]; +} + +export interface ProductUnderstandingInput { + productCategory: string; + productTitle?: string; + productDimensions?: string; + inferredMaterial?: string; +} diff --git a/libs/ai/src/lib/types/scene-graph.types.ts b/libs/ai/src/lib/types/scene-graph.types.ts new file mode 100644 index 0000000..62a0a69 --- /dev/null +++ b/libs/ai/src/lib/types/scene-graph.types.ts @@ -0,0 +1,68 @@ +export type GeometryFamily = + | 'hard-surface' + | 'organic' + | 'cloth-fabric' + | 'cylindrical' + | 'mechanical' + | 'soft-body'; + +export type PrimitiveShape = + | 'box' + | 'cylinder' + | 'sphere' + | 'tapered-cylinder' + | 'frustum' + | 'wedge' + | 'torus' + | 'extruded-ellipse'; + +export interface ScenePartMaterial { + baseColor: [number, number, number, number]; + roughness: number; + metalness: number; + transmissionFactor?: number; + ior?: number; + clearcoat?: number; + emissiveFactor?: [number, number, number]; +} + +export interface ScenePartDimensions { + width: number; + height: number; + depth: number; + topWidth?: number; + topHeight?: number; + topDepth?: number; + radiusTop?: number; + radiusBottom?: number; + tubeRadius?: number; + majorRadius?: number; + rx?: number; + ry?: number; +} + +export interface ScenePart { + id: string; + label: string; + shape: PrimitiveShape; + dimensions: ScenePartDimensions; + position: [number, number, number]; + rotation: [number, number, number, number]; + material: ScenePartMaterial; + smooth?: boolean; + segments?: number; + symmetryMirror?: 'x' | 'z'; +} + +export interface SceneGraph { + schemaVersion: '2.0'; + productCategory: string; + productSubtype: string; + geometryFamily: GeometryFamily; + symmetryAxis: 'x' | 'z' | 'none'; + boundingBox: { width: number; height: number; depth: number }; + parts: ScenePart[]; + confidence: number; + sourceViewsUsed: string[]; + structuralWarnings: string[]; +} diff --git a/libs/ai/src/lib/types/validation.types.ts b/libs/ai/src/lib/types/validation.types.ts new file mode 100644 index 0000000..e888241 --- /dev/null +++ b/libs/ai/src/lib/types/validation.types.ts @@ -0,0 +1,15 @@ +export interface ValidationIssue { + severity: 'error' | 'warning' | 'info'; + code: string; + message: string; + affectedPartId?: string; +} + +export interface ValidationReport { + passed: boolean; + issues: ValidationIssue[]; + topologyScore: number; + scaleScore: number; + uvScore: number; + overallScore: number; +} diff --git a/libs/ai/src/lib/validation/glb-validator.ts b/libs/ai/src/lib/validation/glb-validator.ts new file mode 100644 index 0000000..178891d --- /dev/null +++ b/libs/ai/src/lib/validation/glb-validator.ts @@ -0,0 +1,100 @@ +import type { ValidationReport, ValidationIssue } from '../types/validation.types.js'; + +const GLB_MAGIC = 0x46546C67; + +export class GlbValidator { + validate(glbData: Uint8Array): ValidationReport { + const issues: ValidationIssue[] = []; + issues.push(...this.checkFileIntegrity(glbData)); + if (!issues.some(i => i.severity === 'error')) { + issues.push(...this.checkMeshQuality(glbData)); + } + const hasErrors = issues.some(i => i.severity === 'error'); + return { + passed: !hasErrors, + issues, + topologyScore: hasErrors ? 0 : 100, + scaleScore: 100, + uvScore: 100, + overallScore: hasErrors ? 0 : 100, + }; + } + + private checkFileIntegrity(data: Uint8Array): ValidationIssue[] { + const issues: ValidationIssue[] = []; + if (data.byteLength < 12) { + issues.push({ severity: 'error', code: 'GLB_TOO_SMALL', message: 'GLB data is too small to be valid' }); + return issues; + } + + const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); + const magic = dv.getUint32(0, true); + if (magic !== GLB_MAGIC) { + issues.push({ severity: 'error', code: 'GLB_INVALID_MAGIC', message: `Invalid GLB magic: 0x${magic.toString(16)} (expected 0x${GLB_MAGIC.toString(16)})` }); + } + + const version = dv.getUint32(4, true); + if (version !== 2) { + issues.push({ severity: 'warning', code: 'GLB_UNEXPECTED_VERSION', message: `GLB version ${version} (expected 2)` }); + } + + const totalLength = dv.getUint32(8, true); + if (totalLength !== data.byteLength) { + issues.push({ severity: 'error', code: 'GLB_LENGTH_MISMATCH', message: `GLB length field ${totalLength} does not match actual byte length ${data.byteLength}` }); + } + + if (data.byteLength < 20) { + issues.push({ severity: 'error', code: 'GLB_NO_JSON_CHUNK', message: 'GLB has no JSON chunk' }); + return issues; + } + + // Validate JSON chunk + const jsonChunkLength = dv.getUint32(12, true); + const jsonChunkType = dv.getUint32(16, true); + if (jsonChunkType !== 0x4E4F534A) { + issues.push({ severity: 'error', code: 'GLB_BAD_JSON_CHUNK_TYPE', message: 'First chunk is not a JSON chunk' }); + } + + try { + const jsonBytes = data.slice(20, 20 + jsonChunkLength); + const jsonText = new TextDecoder().decode(jsonBytes).trimEnd(); + JSON.parse(jsonText); + } catch { + issues.push({ severity: 'error', code: 'GLB_INVALID_JSON', message: 'JSON chunk contains invalid JSON' }); + } + + return issues; + } + + private checkMeshQuality(data: Uint8Array): ValidationIssue[] { + const issues: ValidationIssue[] = []; + try { + const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); + const jsonChunkLength = dv.getUint32(12, true); + const jsonBytes = data.slice(20, 20 + jsonChunkLength); + const gltf = JSON.parse(new TextDecoder().decode(jsonBytes).trimEnd()) as { + accessors?: Array<{ count: number }>; + meshes?: Array<{ primitives: Array<{ indices?: number }> }>; + }; + + const accessors = gltf.accessors ?? []; + for (const accessor of accessors) { + if (accessor.count <= 0) { + issues.push({ severity: 'warning', code: 'ACCESSOR_EMPTY', message: `Accessor with count=${accessor.count} found` }); + } + } + + const meshes = gltf.meshes ?? []; + for (const mesh of meshes) { + for (const prim of mesh.primitives) { + if (prim.indices == null) { + issues.push({ severity: 'info', code: 'PRIMITIVE_NO_INDICES', message: 'Mesh primitive uses non-indexed geometry' }); + } + } + } + } catch { + // JSON parsing already validated above; structural issues are caught here + } + return issues; + } +} diff --git a/libs/ai/src/lib/validation/scene-graph-validator.spec.ts b/libs/ai/src/lib/validation/scene-graph-validator.spec.ts new file mode 100644 index 0000000..90cab2e --- /dev/null +++ b/libs/ai/src/lib/validation/scene-graph-validator.spec.ts @@ -0,0 +1,265 @@ +import { SceneGraphValidator, autoRepairSceneGraph } from './scene-graph-validator.js'; +import type { SceneGraph, ScenePart } from '../types/scene-graph.types.js'; + +function makeGraph(overrides: Partial = {}): SceneGraph { + return { + schemaVersion: '2.0', + productCategory: 'test', + productSubtype: 'sedan', + geometryFamily: 'hard-surface', + symmetryAxis: 'x', + boundingBox: { width: 1.8, height: 1.4, depth: 4.5 }, + parts: [], + confidence: 0.9, + sourceViewsUsed: [], + structuralWarnings: [], + ...overrides, + }; +} + +function makePart(overrides: Partial = {}): ScenePart { + return { + id: 'test-part', + label: 'test', + shape: 'box', + dimensions: { width: 0.5, height: 0.5, depth: 0.5 }, + position: [0, 0.25, 0], + rotation: [0, 0, 0, 1], + material: { baseColor: [0.5, 0.5, 0.5, 1], roughness: 0.5, metalness: 0 }, + ...overrides, + }; +} + +function makeWheelPart(id: string, position: [number, number, number]): ScenePart { + return makePart({ + id, + label: `wheel ${id}`, + shape: 'torus', + dimensions: { width: 0.64, height: 0.18, depth: 0.18, tubeRadius: 0.09, majorRadius: 0.32 }, + position, + }); +} + +describe('SceneGraphValidator', () => { + const validator = new SceneGraphValidator(); + + // ─── No parts ───────────────────────────────────────────────────────────── + + it('fails when graph has no parts', () => { + const graph = makeGraph({ parts: [] }); + const report = validator.validate(graph); + expect(report.passed).toBe(false); + expect(report.issues.some(i => i.code === 'NO_PARTS')).toBe(true); + }); + + // ─── Vehicle wheel checks ────────────────────────────────────────────────── + + it('passes for a vehicle with 2 wheel parts (mirroring provides other 2)', () => { + const graph = makeGraph({ + productSubtype: 'car', + parts: [ + makePart({ id: 'body', label: 'car body' }), + makeWheelPart('wheel-fl', [-0.9, 0.32, 1.2]), + makeWheelPart('wheel-rl', [-0.9, 0.32, -1.2]), + ], + }); + const report = validator.validate(graph); + const wheelError = report.issues.find(i => i.code === 'VEHICLE_MISSING_WHEELS'); + expect(wheelError).toBeUndefined(); + }); + + it('errors when vehicle has 0 wheel parts', () => { + const graph = makeGraph({ + productSubtype: 'car', + parts: [makePart({ id: 'body', label: 'car body' })], + }); + const report = validator.validate(graph); + expect(report.passed).toBe(false); + expect(report.issues.some(i => i.code === 'VEHICLE_MISSING_WHEELS')).toBe(true); + }); + + it('errors when vehicle has only 1 wheel part (minimum is 2)', () => { + const graph = makeGraph({ + productSubtype: 'suv', + parts: [ + makePart({ id: 'body', label: 'body' }), + makeWheelPart('wheel-fl', [-0.9, 0.32, 1.2]), + ], + }); + const report = validator.validate(graph); + // Validator requires ≥ 2 wheel parts (front-left + rear-left; mirroring creates right-side counterparts) + const wheelError = report.issues.find(i => i.code === 'VEHICLE_MISSING_WHEELS'); + expect(wheelError).toBeDefined(); + expect(wheelError?.severity).toBe('error'); + }); + + it('passes when vehicle has exactly 2 wheel parts', () => { + const graph = makeGraph({ + productSubtype: 'sedan', + parts: [ + makePart({ id: 'body', label: 'car body' }), + makeWheelPart('wheel-fl', [-0.9, 0.32, 1.2]), + makeWheelPart('wheel-rl', [-0.9, 0.32, -1.2]), + ], + }); + const report = validator.validate(graph); + const wheelError = report.issues.find(i => i.code === 'VEHICLE_MISSING_WHEELS'); + expect(wheelError).toBeUndefined(); + }); + + // ─── Box wheel detection ─────────────────────────────────────────────────── + + it('errors when a wheel part uses box shape', () => { + const graph = makeGraph({ + productSubtype: 'car', + parts: [ + makePart({ id: 'wheel-fl', label: 'front left wheel', shape: 'box', dimensions: { width: 0.3, height: 0.3, depth: 0.3 } }), + makePart({ id: 'wheel-rl', label: 'rear left wheel', shape: 'box', dimensions: { width: 0.3, height: 0.3, depth: 0.3 } }), + ], + }); + const report = validator.validate(graph); + const boxWheelErrors = report.issues.filter(i => i.code === 'WHEEL_IS_BOX'); + expect(boxWheelErrors.length).toBeGreaterThan(0); + }); + + it('does not error when a wheel part uses torus shape', () => { + const graph = makeGraph({ + productSubtype: 'car', + parts: [ + makeWheelPart('wheel-fl', [-0.9, 0.32, 1.2]), + makeWheelPart('wheel-rl', [-0.9, 0.32, -1.2]), + makePart({ id: 'body', label: 'car body' }), + ], + }); + const report = validator.validate(graph); + const boxWheelErrors = report.issues.filter(i => i.code === 'WHEEL_IS_BOX'); + expect(boxWheelErrors).toHaveLength(0); + }); + + // ─── Glass transmission check ────────────────────────────────────────────── + + it('warns when window part has no transmissionFactor', () => { + const graph = makeGraph({ + productSubtype: 'car', + parts: [ + makePart({ id: 'windshield', label: 'windshield', material: { baseColor: [0.8, 0.9, 1, 0.3], roughness: 0.05, metalness: 0.1 } }), + makeWheelPart('wheel-fl', [-0.9, 0.32, 1.2]), + makeWheelPart('wheel-rl', [-0.9, 0.32, -1.2]), + ], + }); + const report = validator.validate(graph); + const glassWarn = report.issues.find(i => i.code === 'GLASS_NO_TRANSMISSION'); + expect(glassWarn).toBeDefined(); + expect(glassWarn?.severity).toBe('warning'); + }); + + it('does not warn when window part has transmissionFactor set', () => { + const graph = makeGraph({ + productSubtype: 'car', + parts: [ + makePart({ id: 'windshield', label: 'windshield', material: { baseColor: [0.8, 0.9, 1, 0.3], roughness: 0.05, metalness: 0.1, transmissionFactor: 0.85 } }), + makeWheelPart('wheel-fl', [-0.9, 0.32, 1.2]), + makeWheelPart('wheel-rl', [-0.9, 0.32, -1.2]), + ], + }); + const report = validator.validate(graph); + const glassWarn = report.issues.find(i => i.code === 'GLASS_NO_TRANSMISSION'); + expect(glassWarn).toBeUndefined(); + }); + + // ─── Scale checks ────────────────────────────────────────────────────────── + + it('errors when car bounding box height is implausibly large', () => { + const graph = makeGraph({ + productSubtype: 'car', + boundingBox: { width: 1.8, height: 10, depth: 4.5 }, // 10m tall car + parts: [ + makePart({ id: 'body', label: 'car body' }), + makeWheelPart('wheel-fl', [-0.9, 0.32, 1.2]), + makeWheelPart('wheel-rl', [-0.9, 0.32, -1.2]), + ], + }); + const report = validator.validate(graph); + expect(report.issues.some(i => i.code === 'SCALE_TOO_LARGE')).toBe(true); + }); + + it('errors when a part has near-zero dimension', () => { + const graph = makeGraph({ + productSubtype: 'other', + parts: [makePart({ dimensions: { width: 0, height: 0.5, depth: 0.5 } })], + }); + const report = validator.validate(graph); + expect(report.issues.some(i => i.code === 'PART_ZERO_DIMENSION')).toBe(true); + }); + + // ─── Overall score ───────────────────────────────────────────────────────── + + it('returns overallScore of 100 for a clean graph', () => { + const graph = makeGraph({ + productSubtype: 'other', + parts: [makePart({ dimensions: { width: 0.3, height: 0.3, depth: 0.3 } })], + }); + const report = validator.validate(graph); + expect(report.overallScore).toBe(100); + expect(report.passed).toBe(true); + }); + + it('overallScore is 0 when errors exist', () => { + const graph = makeGraph({ parts: [] }); // no parts → error + const report = validator.validate(graph); + expect(report.overallScore).toBeLessThan(100); + }); +}); + +// ─── autoRepairSceneGraph ───────────────────────────────────────────────────── + +describe('autoRepairSceneGraph', () => { + const validator = new SceneGraphValidator(); + + it('converts box wheels to torus', () => { + const graph = makeGraph({ + productSubtype: 'car', + parts: [ + makePart({ id: 'wheel-fl', label: 'front left wheel', shape: 'box', dimensions: { width: 0.3, height: 0.3, depth: 0.3 } }), + makePart({ id: 'wheel-rl', label: 'rear left wheel', shape: 'box', dimensions: { width: 0.3, height: 0.3, depth: 0.3 } }), + ], + }); + const report = validator.validate(graph); + const repaired = autoRepairSceneGraph(graph, report); + const wheelShapes = repaired.parts.map(p => p.shape); + expect(wheelShapes.every(s => s === 'torus')).toBe(true); + }); + + it('clamped zero-dimension parts get minimum 0.01m dimensions', () => { + const graph = makeGraph({ + productSubtype: 'other', + parts: [makePart({ id: 'bad-part', label: 'bad part', dimensions: { width: 0, height: 0.5, depth: 0.5 } })], + }); + const report = validator.validate(graph); + const repaired = autoRepairSceneGraph(graph, report); + expect(repaired.parts[0].dimensions.width).toBeGreaterThanOrEqual(0.01); + }); + + it('adds autoRepair warnings to structuralWarnings', () => { + const graph = makeGraph({ + productSubtype: 'car', + parts: [ + makePart({ id: 'wheel-fl', label: 'wheel fl', shape: 'box', dimensions: { width: 0.3, height: 0.3, depth: 0.3 } }), + makePart({ id: 'wheel-rl', label: 'wheel rl', shape: 'box', dimensions: { width: 0.3, height: 0.3, depth: 0.3 } }), + ], + }); + const report = validator.validate(graph); + const repaired = autoRepairSceneGraph(graph, report); + expect(repaired.structuralWarnings.some(w => w.includes('autoRepair'))).toBe(true); + }); + + it('leaves parts without errors unchanged', () => { + const graph = makeGraph({ + productSubtype: 'other', + parts: [makePart({ id: 'body', label: 'body', dimensions: { width: 0.5, height: 0.5, depth: 0.5 } })], + }); + const report = validator.validate(graph); + const repaired = autoRepairSceneGraph(graph, report); + expect(repaired.parts[0]).toEqual(graph.parts[0]); + }); +}); diff --git a/libs/ai/src/lib/validation/scene-graph-validator.ts b/libs/ai/src/lib/validation/scene-graph-validator.ts new file mode 100644 index 0000000..6b71904 --- /dev/null +++ b/libs/ai/src/lib/validation/scene-graph-validator.ts @@ -0,0 +1,187 @@ +import type { SceneGraph, ScenePart } from '../types/scene-graph.types.js'; +import type { ValidationReport, ValidationIssue } from '../types/validation.types.js'; + +const VEHICLE_SUBTYPES = ['car', 'sedan', 'suv', 'truck', 'van', 'pickup', 'vehicle', 'automobile']; +const WHEEL_LABELS = ['wheel', 'tire', 'tyre']; +const GLASS_LABELS = ['window', 'windshield', 'glass']; +const NON_BOX_CIRCULAR = ['wheel', 'tire', 'tyre', 'knob', 'button', 'dial', 'coin']; + +function isWheelPart(part: ScenePart): boolean { + return WHEEL_LABELS.some(w => part.label.toLowerCase().includes(w)); +} + +function isGlassPart(part: ScenePart): boolean { + return GLASS_LABELS.some(g => part.label.toLowerCase().includes(g)); +} + +function isCircularPart(part: ScenePart): boolean { + return NON_BOX_CIRCULAR.some(c => part.label.toLowerCase().includes(c)); +} + +function score(issues: ValidationIssue[], weight: number): number { + const errors = issues.filter(i => i.severity === 'error').length; + const warnings = issues.filter(i => i.severity === 'warning').length; + return Math.max(0, weight - errors * 20 - warnings * 5); +} + +export class SceneGraphValidator { + validate(graph: SceneGraph): ValidationReport { + const topology = this.checkTopology(graph); + const scale = this.checkScale(graph); + const geometry = this.checkGeometry(graph); + const completeness = this.checkPartCompleteness(graph); + + const all = [...topology, ...scale, ...geometry, ...completeness]; + const hasErrors = all.some(i => i.severity === 'error'); + + return { + passed: !hasErrors, + issues: all, + topologyScore: score(topology, 100), + scaleScore: score(scale, 100), + uvScore: 100, + overallScore: Math.round((score(topology, 25) + score(scale, 25) + score(geometry, 25) + score(completeness, 25))), + }; + } + + private checkTopology(graph: SceneGraph): ValidationIssue[] { + const issues: ValidationIssue[] = []; + const bb = graph.boundingBox; + + for (const part of graph.parts) { + const [px, py, pz] = part.position; + const { width: w, height: h, depth: d } = part.dimensions; + + // Parts must not exceed 2× bounding box in any axis + if (Math.abs(px) + w / 2 > bb.width * 2) { + issues.push({ severity: 'warning', code: 'PART_OUTSIDE_BOUNDS_X', message: `Part "${part.label}" extends far outside bounding box on X axis`, affectedPartId: part.id }); + } + if (py + h > bb.height * 2.5) { + issues.push({ severity: 'warning', code: 'PART_OUTSIDE_BOUNDS_Y', message: `Part "${part.label}" extends far above bounding box`, affectedPartId: part.id }); + } + if (Math.abs(pz) + d / 2 > bb.depth * 2) { + issues.push({ severity: 'warning', code: 'PART_OUTSIDE_BOUNDS_Z', message: `Part "${part.label}" extends far outside bounding box on Z axis`, affectedPartId: part.id }); + } + } + return issues; + } + + private checkScale(graph: SceneGraph): ValidationIssue[] { + const issues: ValidationIssue[] = []; + const bb = graph.boundingBox; + const subtype = graph.productSubtype.toLowerCase(); + + // Detect obviously wrong scales + const MAX_SCALE: Record = { car: 7, laptop: 0.6, phone: 0.3, bottle: 0.6, chair: 1.5 }; + const MIN_SCALE: Record = { car: 2.0, laptop: 0.15, phone: 0.05, bottle: 0.05, chair: 0.3 }; + + for (const [key, maxH] of Object.entries(MAX_SCALE)) { + if (subtype.includes(key) && bb.height > maxH) { + issues.push({ severity: 'error', code: 'SCALE_TOO_LARGE', message: `Bounding box height ${bb.height.toFixed(2)}m is too large for ${key} (max ${maxH}m)` }); + } + } + for (const [key, minH] of Object.entries(MIN_SCALE)) { + if (subtype.includes(key) && bb.height < minH) { + issues.push({ severity: 'error', code: 'SCALE_TOO_SMALL', message: `Bounding box height ${bb.height.toFixed(2)}m is too small for ${key} (min ${minH}m)` }); + } + } + + // Each part's dimensions must be plausible + for (const part of graph.parts) { + const { width: w, height: h, depth: d } = part.dimensions; + if (w < 0.0001 || h < 0.0001 || d < 0.0001) { + issues.push({ severity: 'error', code: 'PART_ZERO_DIMENSION', message: `Part "${part.label}" has near-zero dimension`, affectedPartId: part.id }); + } + if (w > 50 || h > 50 || d > 50) { + issues.push({ severity: 'error', code: 'PART_ENORMOUS', message: `Part "${part.label}" has implausibly large dimension`, affectedPartId: part.id }); + } + } + return issues; + } + + private checkGeometry(graph: SceneGraph): ValidationIssue[] { + const issues: ValidationIssue[] = []; + + for (const part of graph.parts) { + // Wheel/tire must not be a box + if (isWheelPart(part) && part.shape === 'box') { + issues.push({ severity: 'error', code: 'WHEEL_IS_BOX', message: `Part "${part.label}" is a wheel but uses box geometry — must use torus or cylinder`, affectedPartId: part.id }); + } + + // Circular parts should not be boxes + if (isCircularPart(part) && part.shape === 'box') { + issues.push({ severity: 'warning', code: 'CIRCULAR_PART_IS_BOX', message: `Part "${part.label}" appears circular but uses box geometry`, affectedPartId: part.id }); + } + + // Glass parts should have transmissionFactor + if (isGlassPart(part) && part.material.transmissionFactor == null) { + issues.push({ severity: 'warning', code: 'GLASS_NO_TRANSMISSION', message: `Part "${part.label}" appears to be glass but has no transmissionFactor`, affectedPartId: part.id }); + } + } + return issues; + } + + private checkPartCompleteness(graph: SceneGraph): ValidationIssue[] { + const issues: ValidationIssue[] = []; + const subtype = graph.productSubtype.toLowerCase(); + + // Vehicles must have at least 2 wheel parts (mirroring creates the other 2) + if (VEHICLE_SUBTYPES.some(v => subtype.includes(v))) { + const wheelCount = graph.parts.filter(isWheelPart).length; + if (wheelCount < 2) { + issues.push({ severity: 'error', code: 'VEHICLE_MISSING_WHEELS', message: `Vehicle has only ${wheelCount} wheel part(s) — expected at least 2 (front-left and rear-left with symmetryMirror)` }); + } + } + + // Must have at least one part + if (graph.parts.length === 0) { + issues.push({ severity: 'error', code: 'NO_PARTS', message: 'Scene graph has no parts' }); + } + + return issues; + } +} + +export function autoRepairSceneGraph(graph: SceneGraph, report: ValidationReport): SceneGraph { + let parts = [...graph.parts]; + const warnings = [...graph.structuralWarnings]; + + for (const issue of report.issues) { + if (issue.severity !== 'error') continue; + + if (issue.code === 'WHEEL_IS_BOX' && issue.affectedPartId) { + parts = parts.map(p => { + if (p.id !== issue.affectedPartId) return p; + const majorRadius = Math.min(p.dimensions.width, p.dimensions.height) / 2; + const tubeRadius = majorRadius * 0.28; + warnings.push(`autoRepair: converted "${p.label}" from box to torus`); + return { + ...p, + shape: 'torus' as const, + dimensions: { ...p.dimensions, tubeRadius, majorRadius }, + material: { ...p.material, baseColor: [0.1, 0.1, 0.1, 1] as [number,number,number,number], roughness: 0.9, metalness: 0 }, + smooth: true, + segments: 32, + }; + }); + } + + if (issue.code === 'PART_ZERO_DIMENSION' && issue.affectedPartId) { + parts = parts.map(p => { + if (p.id !== issue.affectedPartId) return p; + warnings.push(`autoRepair: clamped zero dimensions on "${p.label}"`); + return { + ...p, + dimensions: { + ...p.dimensions, + width: Math.max(p.dimensions.width, 0.01), + height: Math.max(p.dimensions.height, 0.01), + depth: Math.max(p.dimensions.depth, 0.01), + }, + }; + }); + } + } + + return { ...graph, parts, structuralWarnings: warnings }; +} diff --git a/libs/core/src/lib/adapters/ports/model-generator.port.ts b/libs/core/src/lib/adapters/ports/model-generator.port.ts index 36aaf8b..e955aed 100644 --- a/libs/core/src/lib/adapters/ports/model-generator.port.ts +++ b/libs/core/src/lib/adapters/ports/model-generator.port.ts @@ -2,8 +2,18 @@ import { MediaAsset } from '../../domain/value-objects/media-asset.vo.js'; export interface GenerateModelInput { sourceAsset: MediaAsset; + /** All available source images (multi-view). Falls back to [sourceAsset] if omitted. */ + sourceAssets?: MediaAsset[]; productCategory: string; qualityHint?: 'fast' | 'balanced' | 'quality'; + /** Product title from import data — improves Gemini structural prompts. */ + productTitle?: string; + /** Declared dimensions string from import data (e.g. "120 x 80 x 75 cm"). */ + productDimensions?: string; + /** Pre-computed material finish from the import pipeline. */ + inferredMaterialFinish?: string; + /** Pre-computed geometry complexity from the import pipeline. */ + inferredGeometryComplexity?: string; } export interface GeneratedPrimitivePart { @@ -33,6 +43,10 @@ export interface GenerateModelOutput { outputAsset: MediaAsset; tokensUsed: number; generatedPrimitive?: GeneratedPrimitive; + /** Full scene graph produced by the v2 pipeline (undefined for legacy fallback). */ + sceneGraph?: Record; + /** Validation report from the v2 pipeline. */ + validationReport?: Record; } export interface IModelGeneratorPort { diff --git a/scripts/commit.sh b/scripts/commit.sh new file mode 100644 index 0000000..fc7889a --- /dev/null +++ b/scripts/commit.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Get all modified and untracked files +git_status=$(git status --short) + +# Count total files +total_files=$(echo "$git_status" | wc -l) + +echo "Found $total_files files to commit" +echo "---" + +counter=0 + +# Process each file +while IFS= read -r line; do + if [ -z "$line" ]; then + continue + fi + + counter=$((counter + 1)) + status="${line:0:2}" + file="${line:3}" + + echo "[$counter/$total_files] Processing: $file (Status: $status)" + + # Add the file + git add "$file" + + # Determine commit message based on status + if [[ "$status" == "??" ]]; then + commit_msg="feat: add $file" + elif [[ "$status" == " M" ]] || [[ "$status" == "M " ]]; then + commit_msg="refactor: update $file" + else + commit_msg="chore: modify $file" + fi + + # Commit with message + git commit -m "$commit_msg" + + echo "✓ Committed: $file" + echo "---" + + # Wait for user confirmation before next commit (optional) + # Uncomment the line below if you want to pause between commits + # read -p "Press Enter for next commit..." + +done <<< "$git_status" + +echo "✓ All $counter files committed!" +echo "Next: Review commits with 'git log' before pushing" diff --git a/supabase/migrations/014_generation_feedback.sql b/supabase/migrations/014_generation_feedback.sql new file mode 100644 index 0000000..7f6c730 --- /dev/null +++ b/supabase/migrations/014_generation_feedback.sql @@ -0,0 +1,33 @@ +-- Phase I: Generation feedback loop +-- Records approval/rejection signals per generated model to enable future +-- few-shot prompt improvement by subtype. + +CREATE TABLE IF NOT EXISTS generation_feedback ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + product_id UUID NOT NULL, + conversion_id UUID NOT NULL, + owner_id UUID NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + signal TEXT NOT NULL CHECK (signal IN ('approved', 'rejected', 'regenerated')), + rejection_reason TEXT, + detected_subtype TEXT NOT NULL DEFAULT 'other', + geometry_family TEXT NOT NULL DEFAULT 'hard-surface', + qa_score NUMERIC, + validation_score NUMERIC, + scene_graph_snapshot JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_generation_feedback_subtype ON generation_feedback (detected_subtype); +CREATE INDEX IF NOT EXISTS idx_generation_feedback_signal ON generation_feedback (signal); +CREATE INDEX IF NOT EXISTS idx_generation_feedback_owner ON generation_feedback (owner_id); + +-- Row-level security: owners can only see their own feedback records +ALTER TABLE generation_feedback ENABLE ROW LEVEL SECURITY; + +CREATE POLICY "owners can read own feedback" + ON generation_feedback FOR SELECT + USING (auth.uid() = owner_id); + +CREATE POLICY "service role can insert feedback" + ON generation_feedback FOR INSERT + WITH CHECK (true);