diff --git a/backend/app/schemas/direct_3d_render.py b/backend/app/schemas/direct_3d_render.py index 4a9858da7..93a505e59 100644 --- a/backend/app/schemas/direct_3d_render.py +++ b/backend/app/schemas/direct_3d_render.py @@ -673,7 +673,7 @@ class Direct3DRenderDiagnostics(BaseModel): source_locked_rlasm_pixel_coverage: float | None = None scene_lower_context_coverage: float | None = None minimum_scene_lower_context_coverage: float | None = None - structural_edge_guide_attached: Literal[True] = True + structural_edge_guide_attached: bool = True finish_fusion: Direct3DFinishFusionDiagnostics | None = None provider_raw_structural_edge_fidelity: Direct3DRawStructuralEdgeFidelityDiagnostics | None = None macro_design_fidelity: Direct3DMacroDesignFidelityDiagnostics | None = None diff --git a/backend/app/services/direct_3d_render.py b/backend/app/services/direct_3d_render.py index edabca077..75fbcab6c 100644 --- a/backend/app/services/direct_3d_render.py +++ b/backend/app/services/direct_3d_render.py @@ -48,6 +48,17 @@ # review-required; a 2D comparison cannot certify exact 3D identity. DIRECT_3D_PRESENTATION_FIRST = True _OPENAI_EDIT_URL = "https://api.openai.com/v1/images/edits" +_STREET_VIEW_REFERENCE_INSTRUCTION = ( + "EXISTING STREET REFERENCE: Reproduce the tree canopies and trunks, open plazas, entrance steps, planters, paving, " + "curbs, crossings and facades visible in the Street View photo. Preserve small or temporary streetscape details " + "shown in the photo, including painted concrete barriers, bollards, street signs, signals, railings and benches. " + "Keep these elements distinct at their photographed locations, scale and spacing. Retain visible paint colors, " + "mural patterns and markings on the individual barriers, and keep the pedestrian gaps and open paving shown " + "in the photo. Use the photo and authored design as evidence for the features shown. " + "Preserve the camera framing, street layout and the exact count, silhouette, materials, position and occlusion " + "of visible authored objects. Apply the selected finish and lighting within that layout. " + "Reconstruct pavement obscured by incidental traffic and use the presentation selections for people and vehicles." +) _MIN_PROVIDER_PIXELS = 655_360 DIRECT_3D_MAX_SOURCE_EDGE = 2_048 DIRECT_3D_MAX_SOURCE_PIXELS = 4_194_304 @@ -634,10 +645,11 @@ def prepare_direct_3d_capture(req: Direct3DRenderRequest) -> PreparedDirect3DCap proposal_values = np.asarray(proposal_mask, dtype=np.float32) / 255.0 proposal_coverage = float(proposal_values.mean()) maximum_proposal_coverage = 1.0 if req.presentation_mode == "scene" else _MAX_PROPOSAL_COVERAGE - # Camera framing is the student's choice. Even a single visible detail is - # valid; percentage floors reject sparse street scenes and distant views. - # Keep empty captures invalid and check mask/ID agreement below. - if not (0 < proposal_coverage <= maximum_proposal_coverage): + # An opted-in street camera can face entirely away from the design. The + # project inventory is still validated, and every proposal ID pass must + # confirm that nothing from that inventory is visible in this frame. + context_only_street = proposal_coverage == 0 and req.street_view_context is not None + if not context_only_street and not (0 < proposal_coverage <= maximum_proposal_coverage): raise Direct3DValidationError( "Proposal coverage must include visible proposal pixels and be at most " f"{maximum_proposal_coverage:.0%}; received {proposal_coverage:.2%}" @@ -665,6 +677,10 @@ def prepare_direct_3d_capture(req: Direct3DRenderRequest) -> PreparedDirect3DCap ).convert("RGB") if object_id_image.size != beauty_image.size: raise Direct3DValidationError("Object-ID image dimensions must exactly match the beauty capture") + if context_only_street and object_id_image.getbbox(): + raise Direct3DValidationError( + "Object-ID image must be black for a Street View frame with no visible proposal" + ) object_pixels = np.asarray(object_id_image) classified = np.zeros((height, width), dtype=bool) @@ -672,28 +688,29 @@ def prepare_direct_3d_capture(req: Direct3DRenderRequest) -> PreparedDirect3DCap rgb = np.asarray(_hex_to_rgb(color), dtype=np.uint8) classified |= np.all(object_pixels == rgb, axis=2) object_id_coverage = float(classified.mean()) - if not np.any(classified): - raise Direct3DValidationError("Object-ID image does not contain any classified proposal pixels") - proposal_active = proposal_values >= 0.5 - intersection_count = int(np.count_nonzero(classified & proposal_active)) - proposal_count = int(np.count_nonzero(proposal_active)) - union_count = int(np.count_nonzero(classified | proposal_active)) - object_id_proposal_recall = float(intersection_count / max(1, proposal_count)) - object_id_proposal_iou = float(intersection_count / max(1, union_count)) - outside = proposal_values < (8 / 255) - leak_count = int(np.count_nonzero(classified & outside)) - classified_count = int(np.count_nonzero(classified)) - if leak_count / max(classified_count, 1) > 0.01: - raise Direct3DValidationError("Object-ID classes extend materially outside the proposal mask") - if req.presentation_mode in {"scene", "reproject"} and ( - object_id_proposal_recall < _MIN_PRESENTATION_OBJECT_ID_PROPOSAL_RECALL - or object_id_proposal_iou < _MIN_PRESENTATION_OBJECT_ID_PROPOSAL_IOU - ): - raise Direct3DValidationError( - "Presentation object-ID classes must cover the proposal " - f"(recall {object_id_proposal_recall:.3f}, " - f"IoU {object_id_proposal_iou:.3f})" - ) + if not context_only_street: + if not np.any(classified): + raise Direct3DValidationError("Object-ID image does not contain any classified proposal pixels") + proposal_active = proposal_values >= 0.5 + intersection_count = int(np.count_nonzero(classified & proposal_active)) + proposal_count = int(np.count_nonzero(proposal_active)) + union_count = int(np.count_nonzero(classified | proposal_active)) + object_id_proposal_recall = float(intersection_count / max(1, proposal_count)) + object_id_proposal_iou = float(intersection_count / max(1, union_count)) + outside = proposal_values < (8 / 255) + leak_count = int(np.count_nonzero(classified & outside)) + classified_count = int(np.count_nonzero(classified)) + if leak_count / max(classified_count, 1) > 0.01: + raise Direct3DValidationError("Object-ID classes extend materially outside the proposal mask") + if req.presentation_mode in {"scene", "reproject"} and ( + object_id_proposal_recall < _MIN_PRESENTATION_OBJECT_ID_PROPOSAL_RECALL + or object_id_proposal_iou < _MIN_PRESENTATION_OBJECT_ID_PROPOSAL_IOU + ): + raise Direct3DValidationError( + "Presentation object-ID classes must cover the proposal " + f"(recall {object_id_proposal_recall:.3f}, " + f"IoU {object_id_proposal_iou:.3f})" + ) elif req.presentation_mode in {"scene", "reproject"}: raise Direct3DValidationError("Presentation modes require an object-ID image and manifest") @@ -713,6 +730,10 @@ def prepare_direct_3d_capture(req: Direct3DRenderRequest) -> PreparedDirect3DCap ).convert("RGB") if instance_id_image.size != beauty_image.size: raise Direct3DValidationError("Instance-ID image dimensions must exactly match the beauty capture") + if context_only_street and instance_id_image.getbbox(): + raise Direct3DValidationError( + "Instance-ID image must be black for a Street View frame with no visible proposal" + ) instance_pixels = np.asarray(instance_id_image) classified_instances = np.zeros((height, width), dtype=bool) @@ -753,28 +774,29 @@ def prepare_direct_3d_capture(req: Direct3DRenderRequest) -> PreparedDirect3DCap ) instance_id_coverage = float(classified_instances.mean()) - if not np.any(classified_instances): - raise Direct3DValidationError("Instance-ID image does not contain any classified proposal pixels") - proposal_active = proposal_values >= 0.5 - intersection_count = int(np.count_nonzero(classified_instances & proposal_active)) - proposal_count = int(np.count_nonzero(proposal_active)) - union_count = int(np.count_nonzero(classified_instances | proposal_active)) - instance_id_proposal_recall = float(intersection_count / max(1, proposal_count)) - instance_id_proposal_iou = float(intersection_count / max(1, union_count)) - outside = proposal_values < (8 / 255) - leak_count = int(np.count_nonzero(classified_instances & outside)) - classified_count = int(np.count_nonzero(classified_instances)) - if leak_count / max(classified_count, 1) > 0.01: - raise Direct3DValidationError("Instance-ID classes extend materially outside the proposal mask") - if req.presentation_mode in {"scene", "reproject"} and ( - instance_id_proposal_recall < _MIN_PRESENTATION_INSTANCE_ID_PROPOSAL_RECALL - or instance_id_proposal_iou < _MIN_PRESENTATION_INSTANCE_ID_PROPOSAL_IOU - ): - raise Direct3DValidationError( - "Presentation instance-ID classes must cover the proposal " - f"(recall {instance_id_proposal_recall:.3f}, " - f"IoU {instance_id_proposal_iou:.3f})" - ) + if not context_only_street: + if not np.any(classified_instances): + raise Direct3DValidationError("Instance-ID image does not contain any classified proposal pixels") + proposal_active = proposal_values >= 0.5 + intersection_count = int(np.count_nonzero(classified_instances & proposal_active)) + proposal_count = int(np.count_nonzero(proposal_active)) + union_count = int(np.count_nonzero(classified_instances | proposal_active)) + instance_id_proposal_recall = float(intersection_count / max(1, proposal_count)) + instance_id_proposal_iou = float(intersection_count / max(1, union_count)) + outside = proposal_values < (8 / 255) + leak_count = int(np.count_nonzero(classified_instances & outside)) + classified_count = int(np.count_nonzero(classified_instances)) + if leak_count / max(classified_count, 1) > 0.01: + raise Direct3DValidationError("Instance-ID classes extend materially outside the proposal mask") + if req.presentation_mode in {"scene", "reproject"} and ( + instance_id_proposal_recall < _MIN_PRESENTATION_INSTANCE_ID_PROPOSAL_RECALL + or instance_id_proposal_iou < _MIN_PRESENTATION_INSTANCE_ID_PROPOSAL_IOU + ): + raise Direct3DValidationError( + "Presentation instance-ID classes must cover the proposal " + f"(recall {instance_id_proposal_recall:.3f}, " + f"IoU {instance_id_proposal_iou:.3f})" + ) elif req.presentation_mode in {"scene", "reproject"}: raise Direct3DValidationError("Presentation modes require an instance-ID image and manifest") @@ -810,6 +832,10 @@ def prepare_direct_3d_capture(req: Direct3DRenderRequest) -> PreparedDirect3DCap ).convert("RGB") if material_id_image.size != beauty_image.size: raise Direct3DValidationError("Material-ID image dimensions must exactly match the beauty capture") + if context_only_street and material_id_image.getbbox(): + raise Direct3DValidationError( + "Material-ID image must be black for a Street View frame with no visible proposal" + ) material_pixels = np.asarray(material_id_image) classified_materials = np.zeros((height, width), dtype=bool) semantic_pixels = np.asarray(object_id_image) if object_id_image is not None else None @@ -853,7 +879,7 @@ def prepare_direct_3d_capture(req: Direct3DRenderRequest) -> PreparedDirect3DCap union_count = int(np.count_nonzero(classified_materials | proposal_active)) material_recall = float(intersection_count / max(1, proposal_count)) material_iou = float(intersection_count / max(1, union_count)) - if material_recall < 0.85 or material_iou < 0.84: + if not context_only_street and (material_recall < 0.85 or material_iou < 0.84): raise Direct3DValidationError( "Material-ID classes must cover the proposal " f"(recall {material_recall:.3f}, IoU {material_iou:.3f})" ) @@ -3250,6 +3276,31 @@ def assess_reproject_output_sanity( } +def _street_view_presentation_prompt( + client_prompt: str, + *, + style: str, + has_visible_proposal: bool, + server_inventory: list[dict[str, Any]], +) -> str: + treatment = _PRESENTATION_STYLE_TREATMENTS.get(style, _PRESENTATION_STYLE_TREATMENTS["photorealistic"]) + prompt = [ + f"Create one street-level image as {treatment}.", + "Image 1 is the Street View photo and the base for the existing scene. Use its matched heading and lens.", + ] + if has_visible_proposal: + prompt.append( + "Image 2 is the authored design at the exact framing of the City Prompt camera. " + "Place only its visible objects and surfaces at the same locations, scale and depth. " + "The neutral gray background is empty guide space. Fill it with the existing scene from Image 1, " + "aligning the ground and occlusions to the visible design." + ) + if server_inventory: + prompt.append(_server_inventory_prompt(server_inventory)) + prompt.append(f"Requested finish and art direction:\n{client_prompt.strip()}") + return "\n".join(prompt) + + def _presentation_prompt( client_prompt: str, *, @@ -3266,6 +3317,7 @@ def _presentation_prompt( """Build a natural provider-first prompt with one final design lock.""" del visible_component_summary + treatment = _PRESENTATION_STYLE_TREATMENTS.get(style, _PRESENTATION_STYLE_TREATMENTS["photorealistic"]) has_object_id = bool(object_id_manifest) has_instance_id = bool(instance_id_manifest) base_guide_number = 2 + int(has_object_id) + int(has_instance_id) @@ -3326,10 +3378,6 @@ def _presentation_prompt( else "" ) inventory = _server_inventory_prompt(server_inventory) - treatment = _PRESENTATION_STYLE_TREATMENTS.get( - style, - _PRESENTATION_STYLE_TREATMENTS["photorealistic"], - ) if presentation_mode == "scene": if view_mode == "street": task = ( @@ -3969,6 +4017,7 @@ def _same_camera_presentation_result( """ diagnostics = dict(common_diagnostics) warnings = [] + has_street_view = req.street_view_context is not None try: registration = register_generated_image( capture.normalized_beauty, @@ -3987,12 +4036,16 @@ def _same_camera_presentation_result( fidelity_policy=req.fidelity_policy, ) diagnostics["provider_raw_instance_source_presence"] = _instance_presence_diagnostics(raw_presence) - presentation, exterior_count, exterior_delta = hard_composite_direct_3d( - capture.normalized_beauty, - registration.image, - capture.normalized_proposal_mask, - ) - diagnostics.update(exterior_pixel_count=exterior_count, exterior_max_channel_delta=exterior_delta) + if has_street_view: + # Copying in the raw exterior would put the tile artifacts back. + presentation = registration.image + else: + presentation, exterior_count, exterior_delta = hard_composite_direct_3d( + capture.normalized_beauty, + registration.image, + capture.normalized_proposal_mask, + ) + diagnostics.update(exterior_pixel_count=exterior_count, exterior_max_channel_delta=exterior_delta) protected_coverage = 0.0 if source_locked_rlasm_ids: presentation, protected_coverage = _restore_source_locked_instances( @@ -4045,7 +4098,7 @@ def _same_camera_presentation_result( ) diagnostics.update( view_lock="camera_registered", - context_restyled=False, + context_restyled=has_street_view, provider_first=True, provider_spatial_pixels_retained=True, returned_safety_strategy=strategy, @@ -4053,7 +4106,10 @@ def _same_camera_presentation_result( source_locked_rlasm_pixel_coverage=protected_coverage, ) warnings.append( - "AI finish passed the available image checks. Source context and foreground occlusions " + "The AI finish uses Street View to reconstruct existing surroundings. Compare the proposal and camera " + "with the clean 3D source and verify the streetscape against the photo." + if has_street_view + else "AI finish passed the available image checks. Source context and foreground occlusions " "outside the visible proposal remain unchanged; review building heights, footprints " "and public realm against the clean 3D source. These checks do not certify exact geometry." ) @@ -4136,7 +4192,7 @@ async def _load_street_view_context( ) if plate is None: raise Direct3DProviderError( - "No usable Google Street View image was returned within 20 m. Move the camera or turn off the reference.", + "No usable Google Street View image matched this view within 20 m. Move the camera or turn off the reference.", billing_status="unproduced", ) try: @@ -4194,56 +4250,48 @@ async def _call_openai( ] normalized_width, normalized_height = capture.normalized_beauty.size - beauty_png = _png_bytes(capture.normalized_beauty.convert("RGB")) - structural_guide_png = _png_bytes( - build_structural_edge_guide( - capture.normalized_beauty, - capture.normalized_proposal_mask, - capture.normalized_object_id, - req.object_id_manifest, - ) - ) + has_street_view = req.street_view_context is not None visible_component_summary, _component_labels = _visible_semantic_components( capture.normalized_object_id, req.object_id_manifest, ) - files: list[tuple[str, tuple[str, bytes, str]]] = [ - ("image[]", ("direct-3d-beauty.png", beauty_png, "image/png")), - ] - if capture.normalized_object_id is not None: - files.append( - ( - "image[]", - ("direct-3d-class-id.png", _png_bytes(capture.normalized_object_id), "image/png"), - ) + files: list[tuple[str, tuple[str, bytes, str]]] = [] + has_visible_proposal = capture.normalized_proposal_mask.getbbox() is not None + if has_street_view: + files.append(("image[]", await self._load_street_view_context(req, capture))) + if has_visible_proposal: + # Remove tile pixels from the provider guide. Keep the full viewport + # and control bundle for internal geometry checks and recovery. + design_guide = Image.new("RGB", capture.normalized_beauty.size, (128, 128, 128)) + design_guide.paste(capture.normalized_beauty, mask=capture.normalized_proposal_mask) + files.append(("image[]", ("direct-3d-authored-design.png", _png_bytes(design_guide), "image/png"))) + else: + beauty_png = _png_bytes(capture.normalized_beauty.convert("RGB")) + files.append(("image[]", ("direct-3d-beauty.png", beauty_png, "image/png"))) + id_controls = ( + ("direct-3d-class-id.png", capture.normalized_object_id), + ("direct-3d-instance-id.png", capture.normalized_instance_id), ) - if capture.normalized_instance_id is not None: - files.append( - ( - "image[]", - ( - "direct-3d-instance-id.png", - _png_bytes(capture.normalized_instance_id), - "image/png", - ), + for filename, image in id_controls: + if image is not None: + files.append(("image[]", (filename, _png_bytes(image), "image/png"))) + if req.control_bundle_version == 2: + geometry_controls = ( + ("direct-3d-depth.png", capture.normalized_depth), + ("direct-3d-normal.png", capture.normalized_normal), + ("direct-3d-material-id.png", capture.normalized_material_id), ) + for filename, image in geometry_controls: + if image is None: + raise Direct3DValidationError(f"Control bundle v2 is missing prepared {filename}") + files.append(("image[]", (filename, _png_bytes(image), "image/png"))) + structural_guide = build_structural_edge_guide( + capture.normalized_beauty, + capture.normalized_proposal_mask, + capture.normalized_object_id, + req.object_id_manifest, ) - if req.control_bundle_version == 2: - geometry_controls = ( - ("direct-3d-depth.png", capture.normalized_depth), - ("direct-3d-normal.png", capture.normalized_normal), - ("direct-3d-material-id.png", capture.normalized_material_id), - ) - for filename, image in geometry_controls: - if image is None: - raise Direct3DValidationError(f"Control bundle v2 is missing prepared {filename}") - files.append(("image[]", (filename, _png_bytes(image), "image/png"))) - files.append( - ( - "image[]", - ("direct-3d-structural-edges.png", structural_guide_png, "image/png"), - ) - ) + files.append(("image[]", ("direct-3d-structural-edges.png", _png_bytes(structural_guide), "image/png"))) # Authored archetype artwork rides after the metadata passes so the # prompt's ARCHETYPE REFERENCES numbering lines up with attachment # order. These are design sources the provider applies, not metadata. @@ -4271,8 +4319,15 @@ async def _call_openai( ("direct-3d-edit-mask.png", edit_mask_png, "image/png"), ) ) - prompt = ( - _authoritative_prompt( + if has_street_view: + prompt = _street_view_presentation_prompt( + req.prompt, + style=req.style, + has_visible_proposal=has_visible_proposal, + server_inventory=visible_inventory, + ) + elif req.presentation_mode == "source_anchored": + prompt = _authoritative_prompt( req.prompt, object_id_manifest=req.object_id_manifest, instance_id_manifest=req.instance_id_manifest, @@ -4281,8 +4336,8 @@ async def _call_openai( archetype_reference_labels=[reference.label for reference in references], control_bundle_version=req.control_bundle_version, ) - if req.presentation_mode == "source_anchored" - else _presentation_prompt( + else: + prompt = _presentation_prompt( req.prompt, presentation_mode=req.presentation_mode, style=req.style, @@ -4294,22 +4349,18 @@ async def _call_openai( archetype_reference_labels=[reference.label for reference in references], control_bundle_version=req.control_bundle_version, ) - ) prompt = append_people_action_direction( prompt, _visible_people_action_direction(server_inventory, visible_ids), max_length=30_000, ) - prompt = apply_presentation_selections(prompt, add_people=req.add_people, add_vehicles=req.add_vehicles) - if req.street_view_context is not None: - files.append(("image[]", await self._load_street_view_context(req, capture))) - prompt += ( - "\nEXISTING STREET REFERENCE: The last image is a photograph from a nearby panorama. " - "Use it for the appearance of existing façades, pavement, sidewalk edges and street furniture " - "only where they correspond to the captured 3D view. Keep the first image's camera, visibility, " - "street layout and every placed design object. Keep changes to materials and lighting. " - "Leave objects that are outside the frame, occluded or absent from the capture out of the result." - ) + prompt = apply_presentation_selections( + prompt, + add_people=req.add_people, + add_vehicles=req.add_vehicles, + preserve_captured_entourage=not has_street_view, + reference_instruction=_STREET_VIEW_REFERENCE_INSTRUCTION if has_street_view else "", + ) if req.model == NANO_BANANA_MODEL: from app.services.nano_banana_image import ( generate_nano_banana_image, @@ -4524,7 +4575,7 @@ async def generate( "source_locked_rlasm_pixel_coverage": None, "scene_lower_context_coverage": (capture.scene_lower_context_coverage), "minimum_scene_lower_context_coverage": (0.0 if req.presentation_mode == "scene" else None), - "structural_edge_guide_attached": True, + "structural_edge_guide_attached": req.street_view_context is None, "finish_fusion": None, "provider_raw_structural_edge_fidelity": None, "macro_design_fidelity": None, @@ -4541,10 +4592,8 @@ async def generate( if req.view_mode == "street": # The input was already captured at eye level. Finishing it - # must not change cameras or expose hidden facilities. Use - # the same registered, masked, source-preserving checks as - # any other same-camera image, rather than content sanity - # alone. Sparse street evidence safely falls back to source. + # keeps the same camera and design checks. A photo can + # reconstruct existing context in the registered result. street_result = _same_camera_presentation_result( req, capture, diff --git a/backend/app/services/render_fidelity.py b/backend/app/services/render_fidelity.py index c9cd9056b..a6330ba62 100644 --- a/backend/app/services/render_fidelity.py +++ b/backend/app/services/render_fidelity.py @@ -24,7 +24,12 @@ ) -def presentation_entourage_lock(*, add_people: bool = False, add_vehicles: bool = False) -> str: +def presentation_entourage_lock( + *, + add_people: bool = False, + add_vehicles: bool = False, + preserve_captured_entourage: bool = True, +) -> str: """Explicit UI selections override style/custom text; existing context stays intact.""" people = ( "Add a restrained number of realistically scaled people only on existing visible sidewalks, paths or plazas, " @@ -43,7 +48,12 @@ def presentation_entourage_lock(*, add_people: bool = False, add_vehicles: bool + people + " " + vehicles - + " Preserve any people or vehicles already captured in the existing context. " + + ( + " Preserve any people or vehicles already captured in the existing context." + if preserve_captured_entourage + else "" + ) + + " " "These selections never permit changes to camera, buildings, streets, park layout, paths or furniture." ) @@ -56,11 +66,22 @@ def append_render_preservation_lock(prompt: str, *, max_length: int = 32_000) -> return prompt[: max_length - len(suffix)].rstrip() + suffix -def apply_presentation_selections(prompt: str, *, add_people: bool = False, add_vehicles: bool = False) -> str: +def apply_presentation_selections( + prompt: str, + *, + add_people: bool = False, + add_vehicles: bool = False, + preserve_captured_entourage: bool = True, + reference_instruction: str = "", +) -> str: suffix = ( "\n" - + presentation_entourage_lock(add_people=add_people, add_vehicles=add_vehicles) + + presentation_entourage_lock( + add_people=add_people, + add_vehicles=add_vehicles, + preserve_captured_entourage=preserve_captured_entourage, + ) + "\n" - + RENDER_PRESERVATION_LOCK + + (reference_instruction or RENDER_PRESERVATION_LOCK) ) return prompt.removesuffix(RENDER_PRESERVATION_LOCK)[: 32_000 - len(suffix)].rstrip() + suffix diff --git a/backend/app/services/site_context.py b/backend/app/services/site_context.py index 331f130ef..25eee665f 100644 --- a/backend/app/services/site_context.py +++ b/backend/app/services/site_context.py @@ -15,9 +15,12 @@ import base64 import logging import math +import re import time +from dataclasses import dataclass import httpx +from pyproj import Geod from app.core.config import get_settings @@ -25,6 +28,10 @@ _CACHE: dict[tuple, tuple[float, dict | None]] = {} _CACHE_TTL_S = 600 +_STREET_VIEW_URL = "https://maps.googleapis.com/maps/api/streetview" +_STREET_VIEW_GEOD = Geod(ellps="WGS84") +_STREET_VIEW_PROBE_DISTANCES_M = (6, 12) +_STREET_VIEW_MAX_CROSS_VIEW_OFFSET_M = 8 # NOTE: Places/tenant TEXT grounding was removed 2026-06-10 — it forced business # names ("Happy Cake", "1bridge Marketing") into residential scenes where they @@ -33,6 +40,70 @@ # drop the textual one. (Re-add later behind a commercial-vs-residential heuristic.) +@dataclass(frozen=True) +class _StreetViewPanorama: + pano_id: str + date: str | None + offset_m: float + + +async def _streetview_metadata(client: httpx.AsyncClient, lat: float, lng: float, radius_m: int, key: str) -> dict: + try: + response = await client.get( + f"{_STREET_VIEW_URL}/metadata", + params={"location": f"{lat:.8f},{lng:.8f}", "radius": radius_m, "source": "outdoor", "key": key}, + timeout=5.0, + ) + response.raise_for_status() + metadata = response.json() + except (httpx.HTTPError, ValueError): + return {} + return metadata if isinstance(metadata, dict) and metadata.get("status") == "OK" else {} + + +async def _latest_streetview_panorama( + client: httpx.AsyncClient, lat: float, lng: float, heading: float, fov: float, max_offset_m: float, key: str +) -> _StreetViewPanorama | None: + # Street View Static returns one nearest panorama per point. Probe two + # nearby rings to find other dated coverage without downloading any images. + locations = [(lng, lat)] + locations.extend( + _STREET_VIEW_GEOD.fwd(lng, lat, bearing, distance)[:2] + for distance in _STREET_VIEW_PROBE_DISTANCES_M + if distance <= max_offset_m + for bearing in range(0, 360, 45) + ) + radius_m = math.ceil(max_offset_m) + responses = await asyncio.gather( + *(_streetview_metadata(client, sample_lat, sample_lng, radius_m, key) for sample_lng, sample_lat in locations) + ) + + # A panorama can be cropped to any heading, but sideways displacement + # changes what it sees. Narrow views allow less displacement. + max_cross_view_offset = min(_STREET_VIEW_MAX_CROSS_VIEW_OFFSET_M, max_offset_m * math.tan(math.radians(fov / 2))) + candidates: dict[str, _StreetViewPanorama] = {} + for metadata in responses: + pano_id = metadata.get("pano_id") + location = metadata.get("location") + if not isinstance(pano_id, str) or not pano_id or pano_id in candidates or not isinstance(location, dict): + continue + try: + bearing, _, offset = _STREET_VIEW_GEOD.inv(lng, lat, float(location["lng"]), float(location["lat"])) + except (KeyError, TypeError, ValueError): + continue + cross_view_offset = offset * abs(math.sin(math.radians(bearing - heading))) + if not math.isfinite(offset) or offset > max_offset_m or cross_view_offset > max_cross_view_offset: + continue + date = metadata.get("date") + if not isinstance(date, str) or not re.fullmatch(r"[0-9]{4}(?:-(?:0[1-9]|1[0-2]))?", date): + date = None + candidates[pano_id] = _StreetViewPanorama(pano_id, date, offset) + + # ISO capture dates sort chronologically. An unknown month ranks before + # known months of the same year and an undated panorama ranks last. + return max(candidates.values(), key=lambda pano: (pano.date or "", -pano.offset_m), default=None) + + async def fetch_streetview_plate( lat: float, lng: float, @@ -41,15 +112,9 @@ async def fetch_streetview_plate( width: int, height: int, *, - max_offset_m: float | None = None, + max_offset_m: float = 20, ) -> dict | None: - """Fetch ONE real Street View photo aligned to a given camera, for use as the - composite background plate in SV-snap photomontage mode. - - Snaps to the nearest pano and returns its true position + the offset from the - requested point (the accuracy cost). Returns None when there's no coverage so - the caller can fall back to the 3D-tiles capture. Never raises. - """ + """Download one Street View photo from the newest nearby camera match.""" key = getattr(get_settings(), "google_maps_api_key", "") or "" if not key: return None @@ -59,34 +124,16 @@ async def fetch_streetview_plate( fov = max(10.0, min(120.0, float(fov))) try: async with httpx.AsyncClient(timeout=25.0) as client: - meta = ( - await client.get( - "https://maps.googleapis.com/maps/api/streetview/metadata", - params={"location": f"{lat},{lng}", "key": key}, - ) - ).json() - if meta.get("status") != "OK": - logger.info("sv-plate: no coverage at %.5f,%.5f", lat, lng) - return None - pano = meta["pano_id"] - ploc = meta.get("location", {}) - offset_m = None - if ploc.get("lat") is not None and ploc.get("lng") is not None: - dlat = math.radians(ploc["lat"] - lat) - dlng = math.radians(ploc["lng"] - lng) - a = ( - math.sin(dlat / 2) ** 2 - + math.cos(math.radians(lat)) * math.cos(math.radians(ploc["lat"])) * math.sin(dlng / 2) ** 2 - ) - offset_m = 2 * 6_371_000 * math.asin(math.sqrt(min(1.0, a))) - if max_offset_m is not None and (offset_m is None or offset_m > max_offset_m): + panorama = await _latest_streetview_panorama(client, lat, lng, heading, fov, max_offset_m, key) + if panorama is None: + logger.info("sv-plate: no camera-matched coverage at %.5f,%.5f", lat, lng) return None r = await client.get( - "https://maps.googleapis.com/maps/api/streetview", + _STREET_VIEW_URL, params={ - "pano": pano, + "pano": panorama.pano_id, "size": f"{w}x{h}", - "heading": round(heading), + "heading": heading % 360, "fov": round(fov), "pitch": 0, "return_error_code": "true", @@ -104,8 +151,8 @@ async def fetch_streetview_plate( return { "photo_base64": base64.b64encode(r.content).decode(), "mime_type": mime_type, - "pano_offset_m": round(offset_m, 1) if offset_m is not None else None, - "pano_date": meta.get("date"), + "pano_offset_m": round(panorama.offset_m, 1), + "pano_date": panorama.date, "width": w, "height": h, } diff --git a/backend/tests/test_direct_3d_street_mode.py b/backend/tests/test_direct_3d_street_mode.py index 57e04e033..aaad0b840 100644 --- a/backend/tests/test_direct_3d_street_mode.py +++ b/backend/tests/test_direct_3d_street_mode.py @@ -1,5 +1,7 @@ """Street view_mode contract for the Direct 3D pipeline (Stage 3 v1).""" +import base64 +import io from types import SimpleNamespace from unittest.mock import AsyncMock @@ -9,15 +11,23 @@ from PIL import Image from pydantic import ValidationError -from app.schemas.direct_3d_render import Direct3DRenderRequest +from app.schemas.direct_3d_render import Direct3DRenderDiagnostics, Direct3DRenderRequest from app.services import direct_3d_render as direct_service, site_context from app.services.direct_3d_render import ( Direct3DProviderError, Direct3DRenderService, + Direct3DValidationError, _presentation_prompt, prepare_direct_3d_capture, ) -from tests.test_direct_3d_render import _FakeResponse, _RecordingClient, _capture_images, _png_b64, _request +from tests.test_direct_3d_render import ( + _FakeResponse, + _RecordingClient, + _capture_images, + _png_b64, + _registration_context_scene, + _request, +) def _street_request(**overrides) -> Direct3DRenderRequest: @@ -46,18 +56,83 @@ def test_aerial_requests_reject_street_view_context(): Direct3DRenderRequest.model_validate(payload) -@pytest.mark.parametrize("coverage", ["available", "missing", "invalid"]) -async def test_street_context_is_attached_once_or_fails_before_the_image_provider(monkeypatch, coverage): +@pytest.fixture +def existing_streetscape_request() -> Direct3DRenderRequest: + beauty, _ = _capture_images() + mask = Image.new("L", beauty.size, 0) + black = _png_b64(Image.new("RGB", beauty.size)) + base = _street_request(beauty=beauty, mask=mask) + identity = [float(index % 5 == 0) for index in range(16)] + return Direct3DRenderRequest.model_validate( + { + **base.model_dump(), + "street_view_context": {"latitude": 51.045347, "longitude": -114.058216, "heading": 257}, + "control_bundle_version": 2, + "depth_image_base64": _png_b64(mask), + "normal_image_base64": _png_b64(mask), + "material_id_image_base64": black, + "material_id_manifest": {"#020002": {"material_id": "wall", "label": "Wall", "semantic_class": "building"}}, + "camera": { + "projection": "perspective", + "projection_matrix": identity, + "matrix_world": identity, + "position": [0, 0, 10], + "quaternion": [0, 0, 0, 1], + "fov": 70, + }, + } + ) + + +def test_street_view_accepts_a_camera_with_no_proposed_objects_in_frame(existing_streetscape_request): + capture = prepare_direct_3d_capture(existing_streetscape_request) + + assert capture.proposal_coverage == 0 + assert capture.object_id_coverage == 0 + assert capture.instance_id_coverage == 0 + assert capture.normalized_material_id is not None + + +@pytest.mark.parametrize( + ("field", "color"), + [ + ("object_id_image_base64", (255, 0, 0)), + ("instance_id_image_base64", (1, 0, 1)), + ("material_id_image_base64", (2, 0, 2)), + ], +) +def test_street_view_rejects_colored_proposal_ids_with_an_empty_mask(existing_streetscape_request, field, color): + invalid_ids = Image.new("RGB", _capture_images()[0].size) + invalid_ids.putpixel((0, 0), color) + request = existing_streetscape_request.model_copy(update={field: _png_b64(invalid_ids)}) + + with pytest.raises(Direct3DValidationError, match="ID image must be black"): + prepare_direct_3d_capture(request) + + +@pytest.mark.parametrize( + ("coverage", "context_only"), + [("available", False), ("available", True), ("missing", False), ("invalid", False)], +) +async def test_street_context_is_attached_once_or_fails_before_the_image_provider( + monkeypatch, + coverage, + context_only, + existing_streetscape_request, +): location = {"latitude": 51.045535, "longitude": -114.067772, "heading": 90} + base = existing_streetscape_request if context_only else _street_request() request = Direct3DRenderRequest.model_validate( { - **_street_request().model_dump(), + **base.model_dump(), + "prompt": "Use the selected finish for the photographed streetscape and visible design.", "street_view_context": location, } ) capture = prepare_direct_3d_capture(request) + photo = _png_b64(Image.new("RGB", (640, 360), (28, 69, 47))) fixture = { - "photo_base64": _png_b64(capture.normalized_beauty), + "photo_base64": photo, "mime_type": "image/png", "pano_date": "2024-06", "pano_offset_m": 15.0, @@ -80,12 +155,37 @@ async def test_street_context_is_attached_once_or_fails_before_the_image_provide service = Direct3DRenderService("test-image-key") if coverage == "available": result = await service.generate(request, capture) + assert Direct3DRenderDiagnostics.model_validate(result.diagnostics).structural_edge_guide_attached is False assert result.diagnostics["street_view_context"] == {"pano_date": "2024-06", "offset_m": 15.0} assert len(_RecordingClient.calls) == 1 call = _RecordingClient.calls[0] - assert call["files"][0][1][0] == "direct-3d-beauty.png" - assert call["files"][-1][1][0] == "street-view-context.png" - assert "EXISTING STREET REFERENCE" in call["data"]["prompt"] + attachments = [attachment[1] for attachment in call["files"]] + assert attachments[0] == ("street-view-context.png", base64.b64decode(photo), "image/png") + prompt = call["data"]["prompt"] + assert "Image 1 is the Street View photo" in prompt + assert all(detail in prompt for detail in ("tree canopies", "open plazas", "entrance steps")) + street_reference = prompt.split("EXISTING STREET REFERENCE: ", 1)[1] + assert all( + detail in street_reference + for detail in ("temporary", "painted concrete barriers", "mural patterns", "spacing", "pedestrian gaps") + ) + assert "only permanent features" not in street_reference + if context_only: + assert len(attachments) == 1 + assert "Image 2" not in prompt + else: + assert len(attachments) == 2 + assert attachments[1][0] == "direct-3d-authored-design.png" + with Image.open(io.BytesIO(attachments[1][1])) as guide: + assert guide.size == capture.normalized_beauty.size + pixels = np.asarray(guide) + mask = np.asarray(capture.normalized_proposal_mask) + assert np.all(pixels[mask == 0] == 128) + assert np.array_equal(pixels[mask == 255], np.asarray(capture.normalized_beauty)[mask == 255]) + assert "Image 2 is the authored design" in prompt + assert "Preserve any people or vehicles already captured" not in prompt + assert "photos guide appearance only" not in prompt + assert "class-ID metadata" not in prompt else: message = "within 20 m" if coverage == "missing" else "invalid image" with pytest.raises(Direct3DProviderError, match=message) as error: @@ -97,6 +197,141 @@ async def test_street_context_is_attached_once_or_fails_before_the_image_provide assert lookup.await_args.kwargs == {"max_offset_m": 20} +async def test_unchecked_street_render_keeps_its_3d_guides_and_skips_photo_lookup(monkeypatch): + request = _street_request() + capture = prepare_direct_3d_capture(request) + lookup = AsyncMock() + monkeypatch.setattr(site_context, "fetch_streetview_plate", lookup) + monkeypatch.setattr( + direct_service, + "get_settings", + lambda: SimpleNamespace(direct_3d_street_view_context_enabled=True, google_maps_api_key="test-maps-key"), + ) + monkeypatch.setattr(direct_service.httpx, "AsyncClient", _RecordingClient) + _RecordingClient.calls = [] + _RecordingClient.response = _FakeResponse(200, {"data": [{"b64_json": _png_b64(capture.normalized_beauty)}]}) + + result = await Direct3DRenderService("test-image-key").generate(request, capture) + + lookup.assert_not_awaited() + assert result.diagnostics["structural_edge_guide_attached"] is True + assert not result.diagnostics.get("street_view_context") + (call,) = _RecordingClient.calls + assert [attachment[1][0] for attachment in call["files"]] == [ + "direct-3d-beauty.png", + "direct-3d-class-id.png", + "direct-3d-instance-id.png", + "direct-3d-structural-edges.png", + ] + assert "EXISTING STREET REFERENCE:" not in call["data"]["prompt"] + + +@pytest.mark.parametrize("with_photo", [False, True]) +async def test_registered_street_finish_keeps_photo_guided_context_changes_only_when_requested(monkeypatch, with_photo): + beauty, mask = _registration_context_scene() + base = _street_request(beauty=beauty, mask=mask) + request = Direct3DRenderRequest.model_validate( + { + **base.model_dump(), + "street_view_context": ( + {"latitude": 51.045395, "longitude": -114.058189, "heading": 99} if with_photo else None + ), + } + ) + capture = prepare_direct_3d_capture(request) + source = np.asarray(capture.normalized_beauty) + exterior = np.asarray(capture.normalized_proposal_mask) == 0 + candidate = source.copy() + candidate[exterior] = np.clip(source[exterior].astype(np.int16) + 8, 0, 255).astype(np.uint8) + monkeypatch.setattr(Direct3DRenderService, "_call_openai", AsyncMock(return_value=Image.fromarray(candidate))) + + result = await Direct3DRenderService("test-image-key").generate(request, capture) + + assert result.outcome == "review_required" + assert result.diagnostics["provider_spatial_pixels_retained"] + assert result.diagnostics["context_restyled"] is with_photo + with Image.open(io.BytesIO(base64.b64decode(result.image_base64))) as image: + assert image.getpixel((0, 0)) == tuple(candidate[0, 0] if with_photo else source[0, 0]) + + +async def test_street_view_uses_newest_aligned_outdoor_panorama(monkeypatch): + lat, lng = 51.045347, -114.058216 + image_requests = [] + + def google(request: httpx.Request) -> httpx.Response: + params = request.url.params + if request.url.path.endswith("/metadata"): + sample_lat, sample_lng = map(float, params["location"].split(",")) + if sample_lng < lng - 0.00006: + pano, date, position = "too-far", "2026-11", (lat, lng - 0.0004) + elif sample_lat > lat + 0.00004: + pano, date, position = "too-far-sideways", "2026-10", (lat + 0.00012, lng) + elif sample_lng > lng + 0.00005: + pano, date, position = "july", "2026-07", (lat + 0.000021, lng + 0.000065) + elif sample_lat < lat - 0.00004: + pano, date, position = "same-month-farther", "2026-07", (lat - 0.00005, lng + 0.0001) + else: + pano, date, position = "nearest", "2017-06", (lat, lng) + return httpx.Response( + 200, + json={ + "status": "OK", + "pano_id": pano, + "date": date, + "location": {"lat": position[0], "lng": position[1]}, + }, + ) + image_requests.append(params) + return httpx.Response(200, headers={"content-type": "image/jpeg"}, content=b"july-photo") + + async_client = httpx.AsyncClient + monkeypatch.setattr( + site_context.httpx, + "AsyncClient", + lambda **kwargs: async_client(transport=httpx.MockTransport(google), **kwargs), + ) + monkeypatch.setattr(site_context, "get_settings", lambda: SimpleNamespace(google_maps_api_key="test-maps-key")) + + plate = await site_context.fetch_streetview_plate(lat, lng, 82.5, 125, 640, 235, max_offset_m=20) + + assert plate is not None + assert plate["pano_date"] == "2026-07" + assert plate["pano_offset_m"] == pytest.approx(5.1, abs=0.2) + assert [params["pano"] for params in image_requests] == ["july"] + assert float(image_requests[0]["heading"]) == 82.5 + assert float(image_requests[0]["fov"]) == 120 + assert image_requests[0]["size"] == "640x235" + + +async def test_street_view_can_use_an_undated_match_when_other_metadata_requests_fail(monkeypatch): + lat, lng = 51.045347, -114.058216 + + def google(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/metadata"): + _, sample_lng = map(float, request.url.params["location"].split(",")) + if sample_lng <= lng: + raise httpx.ReadTimeout("metadata did not arrive", request=request) + return httpx.Response( + 200, json={"status": "OK", "pano_id": "undated", "location": {"lat": lat, "lng": lng + 0.00005}} + ) + assert request.url.params["pano"] == "undated" + return httpx.Response(200, headers={"content-type": "image/jpeg"}, content=b"undated-photo") + + async_client = httpx.AsyncClient + monkeypatch.setattr( + site_context.httpx, + "AsyncClient", + lambda **kwargs: async_client(transport=httpx.MockTransport(google), **kwargs), + ) + monkeypatch.setattr(site_context, "get_settings", lambda: SimpleNamespace(google_maps_api_key="test-maps-key")) + + plate = await site_context.fetch_streetview_plate(lat, lng, 90, 90, 640, 360, max_offset_m=20) + + assert plate is not None + assert plate["pano_date"] is None + assert plate["pano_offset_m"] == pytest.approx(3.5, abs=0.1) + + async def test_street_view_rejects_placeholder_when_panorama_image_is_unavailable(monkeypatch): lat, lng = 51.045535, -114.067772 diff --git a/frontend/src/components/viewer/StreetViewPanel.test.tsx b/frontend/src/components/viewer/StreetViewPanel.test.tsx index f83c4963b..1956f1d58 100644 --- a/frontend/src/components/viewer/StreetViewPanel.test.tsx +++ b/frontend/src/components/viewer/StreetViewPanel.test.tsx @@ -32,21 +32,61 @@ describe('student street render', () => { models: [{ id: 'gpt-image-2', available: true }], street_view_context_available: true, }); + mocks.direct.mockResolvedValue({ + outcome: 'accepted', + sourceImageUrl: '/clean-source.png', + warnings: [], + render: { imageUrl: '/street-view-finish.png', prompt: 'finish', providerLabel: 'GPT Image' }, + diagnostics: { street_view_context: { pano_date: '2026-07', offset_m: 5.2 } }, + }); const capture = vi.fn().mockResolvedValue({ kind: 'model3d', direct3d: {} }); render(); const context = await screen.findByRole('checkbox', { name: 'Use Google Street View context' }); expect(context).not.toBeChecked(); fireEvent.click(context); + fireEvent.change(screen.getByRole('textbox', { name: /Custom prompt/ }), { target: { value: 'Soft morning light' } }); + fireEvent.click(screen.getByRole('button', { name: 'Add People' })); fireEvent.click(screen.getByRole('button', { name: 'Render' })); await waitFor(() => expect(mocks.direct).toHaveBeenCalledOnce()); + expect(capture).toHaveBeenCalledWith({ allowEmptyProposal: true }); expect(mocks.direct.mock.calls[0][1]).toMatchObject({ model: 'gpt-image-2', viewMode: 'street', streetViewContext: { latitude: 51, longitude: -114, heading: 0 }, + customPrompt: 'Soft morning light', + addPeople: true, + addVehicles: false, }); expect(mocks.classic).not.toHaveBeenCalled(); + expect(await screen.findByText('Google Street View photo · 2026-07')).toBeInTheDocument(); + fireEvent.click(screen.getByAltText('Original 3D view render')); + expect(screen.queryByText('Google Street View photo · 2026-07')).not.toBeInTheDocument(); + }); + + it('returns to the ordinary street render after clearing Google Street View context', async () => { + mocks.imageModels.mockResolvedValue({ + default_model: 'gpt-image-2', + models: [{ id: 'gpt-image-2', available: true }], + street_view_context_available: true, + }); + const capture = vi.fn().mockResolvedValue({ kind: 'model3d', direct3d: {} }); + render(); + + const context = await screen.findByRole('checkbox', { name: 'Use Google Street View context' }); + fireEvent.click(context); + expect(context).toBeChecked(); + fireEvent.click(context); + expect(context).not.toBeChecked(); + fireEvent.click(screen.getByRole('button', { name: 'Render' })); + + await waitFor(() => expect(mocks.direct).toHaveBeenCalledOnce()); + expect(capture).toHaveBeenCalledWith({ allowEmptyProposal: false }); + const options = mocks.direct.mock.calls[0][1]; + expect(options.viewMode).toBe('street'); + expect(options).not.toHaveProperty('streetViewContext'); + expect(options.customPrompt).toContain('Keep all buildings, facilities, paths and streets in their captured positions.'); }); it.each([ diff --git a/frontend/src/components/viewer/StreetViewPanel.tsx b/frontend/src/components/viewer/StreetViewPanel.tsx index 5c5dff492..7a4046066 100644 --- a/frontend/src/components/viewer/StreetViewPanel.tsx +++ b/frontend/src/components/viewer/StreetViewPanel.tsx @@ -22,6 +22,7 @@ import { getApiErrorMessage, resolveApiFileUrl } from '@/services/api'; import { RenderEditModal } from './RenderEditModal'; import { DIRECT_3D_ALLOWED_STYLES, resolveDirect3DPresentationMode, useDirect3DRender } from './globe/useDirect3DRender'; import { collectDirect3DArchetypeReferences } from './globe/direct3dArchetypeReferences'; +import type { Direct3DCaptureOptions } from './globe/direct3dCapture'; import { getCommunity3DCaptureClaims } from '@/features/community3d/community3d'; import { authoredSurfaceRevision, getAuthoredSurfaceCaptureClaims, type AuthoredSurfaceClaim } from '@/features/groundSurfaces/groundSurfaceCapture'; import { groundSurfaceFor } from '@/features/groundSurfaces/groundSurfaces'; @@ -109,7 +110,7 @@ interface StreetViewPanelProps { /** When provided, captures the globe scene from street level instead of the * clay render — including the authored 3D building models when present * (result.kind tells which framing the prompt should use). */ - globeCapture?: () => Promise; + globeCapture?: (options?: Pick) => Promise; /** Project buildings — required for the inventory-locked Direct 3D street * path (per-zone capture claims are derived from them). */ buildings?: Building[]; @@ -166,7 +167,7 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa const angle = streetViewPegman.angle; setIsPreviewing(true); try { - const source = await globeCapture(); + const source = await globeCapture({ allowEmptyProposal: attachStreetViewContext }); if (!source) throw new Error('The 3D view is not ready. Wait for the scene to load.'); setSourcePreview({ imageUrl: source.imageBase64.startsWith('data:') ? source.imageBase64 : `data:image/png;base64,${source.imageBase64}`, @@ -261,7 +262,7 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa const surfaceRevision = directStreetMode ? authoredSurfaceRevision(currentSurfaceZones.current) : null; const [surfaceClaims, captured] = await Promise.all([ directStreetMode ? getAuthoredSurfaceCaptureClaims(currentSurfaceZones.current) : Promise.resolve([]), - globeCapture(), + globeCapture({ allowEmptyProposal: attachStreetViewContext }), ]); if (directStreetMode && surfaceRevision !== authoredSurfaceRevision(currentSurfaceZones.current)) { throw new Error('Ground surfaces changed during capture. Capture the current scene again.'); @@ -320,9 +321,11 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa longitude: pegmanPosition[0], heading: ((pegmanAngle % 360) + 360) % 360, }, + addPeople: includePeople, + addVehicles: includeVehicles, } : {}), archetypeReferences, - customPrompt: [ + customPrompt: attachStreetViewContext ? customPrompt.trim() : [ includePeople ? 'Include a few pedestrians on existing walking surfaces.' : 'Do not add people.', includeVehicles ? 'Include a few vehicles on existing carriageways only.' : 'Do not add vehicles.', includePeople ? 'Scale people using nearby doors and storeys. Show natural walking or seated poses with believable ground contact; keep feet visible when the captured framing allows. Respect occlusion by existing trees, buildings and furniture, and keep entrances and crossings readable.' : '', @@ -330,6 +333,10 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa 'Keep all buildings, facilities, paths and streets in their captured positions.', ].filter(Boolean).join(' '), }), (direct) => { + const streetViewContext = direct.diagnostics?.street_view_context; + const streetViewCaptureDate = streetViewContext + ? streetViewContext.pano_date ?? null + : undefined; const directResult: StreetViewResult = { imageUrl: direct.render.imageUrl, prompt: direct.render.prompt, @@ -337,6 +344,7 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa imageQuality: 'high', providerLabel: direct.render.providerLabel, reviewWarnings: direct.warnings, + streetViewCaptureDate, }; const original = direct.providerOriginalRender; const originalPreview: StreetViewResult | undefined = original ? { @@ -346,6 +354,7 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa imageQuality: original.image_quality, providerLabel: 'AI render', reviewWarnings: ['AI original — compare its geometry with the clean 3D source.'], + streetViewCaptureDate, } : undefined; const sourcePreview: StreetViewResult = { imageUrl: direct.sourceImageUrl, prompt: '', providerLabel: 'Original 3D view', @@ -568,9 +577,16 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa
- - Street View AI Render ({result.providerLabel || STREET_VIEW_RENDER_LABEL}) — Looking {compassLabel(streetViewPegman.angle)} - +
+ + Street View AI Render ({result.providerLabel || STREET_VIEW_RENDER_LABEL}) — Looking {compassLabel(streetViewPegman.angle)} + + {result.streetViewCaptureDate !== undefined && ( +

+ Google Street View photo · {result.streetViewCaptureDate ?? 'Date unavailable'} +

+ )} +
{projectId && !result.error && savedRendersByImage[getRenderImageKey(result)] && } diff --git a/frontend/src/components/viewer/globe/RecoverImageAttempts.test.tsx b/frontend/src/components/viewer/globe/RecoverImageAttempts.test.tsx index 3db546ee5..e10949270 100644 --- a/frontend/src/components/viewer/globe/RecoverImageAttempts.test.tsx +++ b/frontend/src/components/viewer/globe/RecoverImageAttempts.test.tsx @@ -2,10 +2,11 @@ import { fireEvent, render, screen, waitFor } from '@testing-library/react'; import { describe, expect, it, vi } from 'vitest'; import { RecoverImageAttempts } from './RecoverImageAttempts'; -const mocks = vi.hoisted(() => ({ list: vi.fn(), recover: vi.fn(), acknowledge: vi.fn() })); +const mocks = vi.hoisted(() => ({ list: vi.fn(), recover: vi.fn(), acknowledge: vi.fn(), download: vi.fn() })); vi.mock('@/services/api', () => ({ direct3DAttempts: { ...mocks, capabilities: async () => ({ enabled: true, images_enabled: true }) }, rendersApi: {} })); vi.mock('@/store', () => ({ useAuthStore: (select: (state: unknown) => unknown) => select({ user: { id: 'student' } }) })); +vi.mock('@/utils/downloadDataImage', () => ({ downloadDataImage: mocks.download })); describe('image recovery panel', () => { it('opens the saved image alongside its original and keeps unverified fidelity visible', async () => { @@ -22,4 +23,23 @@ describe('image recovery panel', () => { expect(screen.getByText('Check street junctions')).toBeTruthy(); expect(mocks.recover).toHaveBeenCalledTimes(1); }); + + it('opens and downloads the saved Street View AI original when the result is a 3D fallback', async () => { + mocks.list.mockResolvedValue([{ id: 'photo-1', status: 'completed', style: 'photorealistic', created_at: '2026-11-20T12:00:00Z' }]); + mocks.recover.mockResolvedValue({ + attempt: { id: 'photo-1' }, style: 'photorealistic', source_image_base64: 'b3JpZ2luYWw=', + response: { + image_base64: 'b3JpZ2luYWw=', provider_image_base64: 'YWktb3JpZ2luYWw=', + outcome: 'review_required', warnings: ['Compare the AI original with the 3D view'], + diagnostics: { street_view_context: { pano_date: '2026-06', offset_m: 0 } }, + }, + }); + render(); + fireEvent.click(await screen.findByText('Recover recent images')); + fireEvent.click(await screen.findByText('Open saved image')); + + expect((await screen.findByAltText('Saved AI original for review')).getAttribute('src')).toBe('data:image/png;base64,YWktb3JpZ2luYWw='); + fireEvent.click(screen.getByRole('button', { name: 'Download AI original' })); + expect(mocks.download).toHaveBeenCalledWith('data:image/png;base64,YWktb3JpZ2luYWw=', 'cityprompt-ai-original-photo-1.png'); + }); }); diff --git a/frontend/src/components/viewer/globe/RecoverImageAttempts.tsx b/frontend/src/components/viewer/globe/RecoverImageAttempts.tsx index a7018d902..71ae47ca9 100644 --- a/frontend/src/components/viewer/globe/RecoverImageAttempts.tsx +++ b/frontend/src/components/viewer/globe/RecoverImageAttempts.tsx @@ -47,6 +47,8 @@ export function RecoverImageAttempts({ projectId }: { projectId: string }) { finally { if (current === revision.current) setLoading(false); } }; if (!enabled && !error) return null; + const photoOriginal = recovered?.response.diagnostics?.street_view_context && recovered.response.provider_image_base64; + const aiOriginal = photoOriginal && photoOriginal !== recovered?.response.image_base64 ? png(photoOriginal) : undefined; return
Recover recent images

An image keeps its original camera and design even if you edit the project. Checking or opening a saved image uses no image credits.

@@ -64,6 +66,11 @@ export function RecoverImageAttempts({ projectId }: { projectId: string }) {
Original 3D design used for this saved image
Original 3D view at the time of generation
Recovered image result for comparison with its original design
Saved result · {recovered.style}
+ {aiOriginal &&
+ Saved AI original for review +
AI original · Compare it with the original 3D view
+ +
} } diff --git a/frontend/src/components/viewer/globe/direct3dCapture.test.ts b/frontend/src/components/viewer/globe/direct3dCapture.test.ts index 039a9fec9..cca9b4821 100644 --- a/frontend/src/components/viewer/globe/direct3dCapture.test.ts +++ b/frontend/src/components/viewer/globe/direct3dCapture.test.ts @@ -605,6 +605,11 @@ describe('Direct 3D capture helpers', () => { expect(() => validateDirect3DMaskCoverage(0, 0, undefined, 1)).toThrow(); }); + it('accepts a fully context-only frame when the Street View capture opts in', () => { + expect(() => validateDirect3DMaskCoverage(0, 0, 0, 1, true)).not.toThrow(); + expect(() => validateDirect3DMaskCoverage(0, 15, 0, 1, true)).toThrow(); + }); + it.each([289, 1])('accepts a visible proposal occupying only %i pixels', (pixels) => { // 289 / 512² reproduces the reported 0.11% street-view capture. // The same default is used by aerial capture; size is not an integrity check. diff --git a/frontend/src/components/viewer/globe/direct3dCapture.ts b/frontend/src/components/viewer/globe/direct3dCapture.ts index 5e42e106a..7e6b9b020 100644 --- a/frontend/src/components/viewer/globe/direct3dCapture.ts +++ b/frontend/src/components/viewer/globe/direct3dCapture.ts @@ -197,6 +197,8 @@ export interface Direct3DCaptureOptions { maxLongEdge?: number; minMaskCoverage?: number; maxMaskCoverage?: number; + /** A Street View reference can support a view facing away from the proposal. */ + allowEmptyProposal?: boolean; includeGeometryPasses?: boolean; /** Globe captures require actual drawn context, not a sky-only coarse tile. */ minVisibleContextCoverage?: number; @@ -1211,7 +1213,9 @@ export function validateDirect3DMaskCoverage( proposalPixelCount: number, minCoverage = DEFAULT_MIN_MASK_COVERAGE, maxCoverage = DEFAULT_MAX_MASK_COVERAGE, + allowEmptyProposal = false, ): void { + if (allowEmptyProposal && coverage === 0 && proposalPixelCount === 0) return; if (proposalPixelCount < 1 || coverage <= 0 || coverage < minCoverage) { throw new Direct3DCaptureError( 'no_proposal_content', @@ -1803,6 +1807,7 @@ export async function captureDirect3DScene( analysis.proposalPixelCount, options.minMaskCoverage ?? DEFAULT_MIN_MASK_COVERAGE, options.maxMaskCoverage ?? DEFAULT_MAX_MASK_COVERAGE, + options.allowEmptyProposal, ); // Instance inventory: repeat the context depth seed in a separate target, diff --git a/frontend/src/components/viewer/globe/useDirect3DRender.test.ts b/frontend/src/components/viewer/globe/useDirect3DRender.test.ts index 5958b4a28..f8121641c 100644 --- a/frontend/src/components/viewer/globe/useDirect3DRender.test.ts +++ b/frontend/src/components/viewer/globe/useDirect3DRender.test.ts @@ -177,6 +177,18 @@ describe('Direct 3D presentation adapter', () => { vi.mocked(rendersApi.generateDirect3D).mockResolvedValue(response); }); + it('sends just the user art direction and style when the photo supplies the streetscape', async () => { + const { result } = renderHook(() => useDirect3DRender()); + const streetViewContext = { latitude: 51, longitude: -114, heading: 99 }; + await result.current.renderDirect3D(capture, { + style: 'photorealistic', projectId: 'project-1', community3DClaims, + viewMode: 'street', streetViewContext, customPrompt: 'Soft morning light', addPeople: true, + }); + expect(rendersApi.generateDirect3D).toHaveBeenCalledWith(expect.objectContaining({ + style: 'photorealistic', prompt: 'Soft morning light', street_view_context: streetViewContext, add_people: true, + })); + }); + it('keeps the measured, camera-aligned AI finish instead of replacing it with the unaligned original', async () => { const final = { id: 'aligned', image_url: '/aligned.png', prompt: 'finish', created_at: '2026-10-19' }; const original = { id: 'raw', image_url: '/raw.png', prompt: 'finish', created_at: '2026-10-19' }; diff --git a/frontend/src/components/viewer/globe/useDirect3DRender.ts b/frontend/src/components/viewer/globe/useDirect3DRender.ts index 4d94ab355..9ef6798e4 100644 --- a/frontend/src/components/viewer/globe/useDirect3DRender.ts +++ b/frontend/src/components/viewer/globe/useDirect3DRender.ts @@ -91,6 +91,7 @@ export interface Direct3DRenderDiagnostics { displayed_image_variant?: 'processed' | 'provider_original'; /** Missing on legacy source-anchored responses. */ processing_mode?: Direct3DPresentationMode; + street_view_context?: { pano_date?: string | null; offset_m: number } | null; view_lock?: 'source_pixel_locked' | 'camera_registered' | 'not_applicable_layout_guided'; context_restyled?: boolean; provider_first?: boolean; @@ -162,7 +163,7 @@ export interface Direct3DRenderDiagnostics { minimum_object_id_proposal_iou?: number | null; scene_lower_context_coverage?: number | null; minimum_scene_lower_context_coverage?: number | null; - structural_edge_guide_attached: true; + structural_edge_guide_attached: boolean; finish_fusion?: { method: 'source-geometry-multiscale-source-phase-detail-v2'; sigma_px: number; @@ -379,14 +380,11 @@ export function useDirect3DRender() { } const fidelityPolicy = options.fidelityPolicy ?? resolveDirect3DFidelityPolicy(options.style); - const prompt = buildDirect3DVisualPrompt( - options.style, - options.customPrompt, - capture, - fidelityPolicy, - options.publicRealmContext, - ); const viewMode = options.viewMode ?? 'aerial'; + const streetViewContext = viewMode === 'street' ? options.streetViewContext : undefined; + const prompt = streetViewContext + ? options.customPrompt?.trim() || 'Use the selected finish for the photographed streetscape and visible design.' + : buildDirect3DVisualPrompt(options.style, options.customPrompt, capture, fidelityPolicy, options.publicRealmContext); const presentationMode = viewMode === 'street' ? 'scene' : resolveDirect3DPresentationMode(options.style); @@ -404,9 +402,7 @@ export function useDirect3DRender() { model: options.model ?? DEFAULT_OPENAI_IMAGE_MODEL, control_bundle_version: 2, view_mode: viewMode, - ...(viewMode === 'street' && options.streetViewContext - ? { street_view_context: options.streetViewContext } - : {}), + ...(streetViewContext ? { street_view_context: streetViewContext } : {}), // The captured 3D model owns identity for every same-camera style. // Catalogue artwork can prescribe different roofs, openings or materials. ...(options.archetypeReferences?.length && presentationMode !== 'scene' diff --git a/frontend/src/components/viewer/useStreetViewRender.ts b/frontend/src/components/viewer/useStreetViewRender.ts index 0876bc710..8e5eeac36 100644 --- a/frontend/src/components/viewer/useStreetViewRender.ts +++ b/frontend/src/components/viewer/useStreetViewRender.ts @@ -1964,6 +1964,8 @@ export function generateClayRender( export interface StreetViewResult { /** Per-preview fidelity advice; absent for legacy renders and clean sources. */ reviewWarnings?: string[]; + /** Null when a Street View photo was supplied without a reported capture date. */ + streetViewCaptureDate?: string | null; /** URL or data-URI of the generated image. */ imageUrl: string; /** The prompt that was sent to the API. */ diff --git a/frontend/src/features/projects/ProjectViewPage.tsx b/frontend/src/features/projects/ProjectViewPage.tsx index 596590ea0..f7246494c 100644 --- a/frontend/src/features/projects/ProjectViewPage.tsx +++ b/frontend/src/features/projects/ProjectViewPage.tsx @@ -236,7 +236,9 @@ export function ProjectViewPage() { const { captureStreetView } = useGlobeAIRender(); const { flyToStreetLevel, restoreAerialView, saveCameraState } = useGlobeCamera(); - const handleGlobeStreetCapture = useCallback(async (): Promise => { + const handleGlobeStreetCapture = useCallback(async ( + options: Pick = {}, + ): Promise => { if (!globeRefs?.canvas || !globeRefs?.camera) return null; const pegman = useViewerStore.getState().streetViewPegman; if (!pegman?.position) return null; @@ -256,7 +258,7 @@ export function ProjectViewPage() { // Capture from an independent camera at the pegman's exact pose. const captureCamera = globeRefs.camera!.clone(); flyToStreetLevel(lat, lng, pegman.angle, terrainHeight, captureCamera); - captureState.bundle = await globeRefs.captureStreetDirect3D!({}, captureCamera); + captureState.bundle = await globeRefs.captureStreetDirect3D!(options, captureCamera); return captureState.bundle ? rawB64(captureState.bundle.beautyImageBase64) : null; } : undefined; diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index de4d27e69..444a3c6be 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -1567,6 +1567,7 @@ export const rendersApi = { control_bundle_version?: 1 | 2; /** Optional so saved/legacy source-anchored responses remain readable. */ processing_mode?: 'source_anchored' | 'scene' | 'reproject'; + street_view_context?: { pano_date?: string | null; offset_m: number } | null; view_lock?: 'source_pixel_locked' | 'camera_registered' | 'not_applicable_layout_guided'; context_restyled?: boolean; provider_first?: boolean; @@ -1632,7 +1633,7 @@ export const rendersApi = { minimum_object_id_proposal_iou?: number | null; scene_lower_context_coverage?: number | null; minimum_scene_lower_context_coverage?: number | null; - structural_edge_guide_attached: true; + structural_edge_guide_attached: boolean; finish_fusion?: { method: 'source-geometry-multiscale-source-phase-detail-v2'; sigma_px: number;