diff --git a/backend/app/api/v1/render.py b/backend/app/api/v1/render.py index 1ad11afdf..d989fd280 100644 --- a/backend/app/api/v1/render.py +++ b/backend/app/api/v1/render.py @@ -974,6 +974,11 @@ async def image_models(user: User = Depends(require_auth)): # are affordability floors; exact capture pricing is checked at admission. return { **availability, + "street_view_context_available": bool( + settings.direct_3d_images_enabled + and settings.direct_3d_street_view_context_enabled + and settings.google_maps_api_key + ), "credit_balance": effective_render_credit_balance(user), "credit_exempt": is_admin_or_above(user), "models": [ diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 92ecd1730..9fa196443 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -230,6 +230,7 @@ def cors_allow_origin_regex(self) -> str | None: kling_animation_cost_per_second_usd: float = Field(default=0.112, gt=0) kling_animation_credit_cost: int = Field(default=50, gt=0) google_maps_api_key: str = "" + direct_3d_street_view_context_enabled: bool = False fal_key: str = "" fal_style_model: str = "fal-ai/fast-sdxl/image-to-image" gemini_2d_image_model: str = "gemini-3-pro-image-preview" diff --git a/backend/app/schemas/direct_3d_render.py b/backend/app/schemas/direct_3d_render.py index 569fbed8f..5ae0091e2 100644 --- a/backend/app/schemas/direct_3d_render.py +++ b/backend/app/schemas/direct_3d_render.py @@ -191,6 +191,17 @@ class Direct3DArchetypeReference(BaseModel): zone_ids: list[str] = Field(default_factory=list, max_length=2048) +class Direct3DStreetViewContext(BaseModel): + latitude: float = Field(ge=-90, le=90, allow_inf_nan=False) + longitude: float = Field(ge=-180, le=180, allow_inf_nan=False) + heading: float = Field(ge=0, lt=360, allow_inf_nan=False) + + +class Direct3DStreetViewReferenceDiagnostics(BaseModel): + pano_date: str | None = None + offset_m: float = Field(ge=0) + + class Direct3DRenderRequest(BaseModel): """A clean 3D capture plus mode-specific presentation and design authority.""" @@ -250,6 +261,7 @@ class Direct3DRenderRequest(BaseModel): "in this first version. Street requires presentation_mode='scene'." ), ) + street_view_context: Direct3DStreetViewContext | None = None style: Direct3DStyle = Field( default="photorealistic", description="The render panel style identifier governing the presentation treatment.", @@ -390,6 +402,8 @@ def validate_material_id_manifest( def validate_object_id_pair(self) -> "Direct3DRenderRequest": if self.view_mode == "street" and self.presentation_mode != "scene": raise ValueError("view_mode='street' requires presentation_mode='scene'") + if self.street_view_context is not None and self.view_mode != "street": + raise ValueError("street_view_context requires view_mode='street'") if bool(self.object_id_image_base64) != bool(self.object_id_manifest): raise ValueError("object_id_image_base64 and object_id_manifest must be supplied together") if self.presentation_mode in {"scene", "reproject"} and not self.object_id_image_base64: @@ -602,6 +616,7 @@ class Direct3DFinishFusionDiagnostics(BaseModel): class Direct3DRenderDiagnostics(BaseModel): control_bundle_version: Literal[1, 2] = 1 + street_view_context: Direct3DStreetViewReferenceDiagnostics | None = None processing_mode: Direct3DPresentationMode = "source_anchored" view_lock: Literal[ "source_pixel_locked", diff --git a/backend/app/services/direct_3d_render.py b/backend/app/services/direct_3d_render.py index 136131d08..86509cdc0 100644 --- a/backend/app/services/direct_3d_render.py +++ b/backend/app/services/direct_3d_render.py @@ -24,6 +24,7 @@ import numpy as np from PIL import Image, ImageDraw +from app.core.config import get_settings from app.core.image_models import ( DEFAULT_OPENAI_IMAGE_MODEL, PAID_IMAGE_CREDIT_MULTIPLIERS, @@ -4102,6 +4103,51 @@ def __init__(self, api_key: str, *, gemini_api_key: str = ""): self.api_key = api_key self.gemini_api_key = gemini_api_key self._nano_original: str | None = None + self._street_view_context: dict[str, Any] | None = None + + async def _load_street_view_context( + self, + req: Direct3DRenderRequest, + capture: PreparedDirect3DCapture, + ) -> tuple[str, bytes, str]: + from app.services.site_context import fetch_streetview_plate + + context = req.street_view_context + settings = get_settings() + if context is None or not (settings.direct_3d_street_view_context_enabled and settings.google_maps_api_key): + raise Direct3DProviderError( + "Google Street View context is unavailable on this server.", billing_status="unproduced" + ) + + aspect = capture.normalized_beauty.width / capture.normalized_beauty.height + horizontal_fov = 90.0 + if req.camera and req.camera.fov is not None: + horizontal_fov = math.degrees(2 * math.atan(math.tan(math.radians(req.camera.fov / 2)) * aspect)) + plate = await fetch_streetview_plate( + context.latitude, + context.longitude, + context.heading, + horizontal_fov, + 640, + round(640 / aspect), + max_offset_m=20, + ) + if plate is None: + raise Direct3DProviderError( + "No usable Google Street View image was returned within 20 m. Move the camera or turn off the reference.", + billing_status="unproduced", + ) + try: + raw = _decode_base64_payload(plate["photo_base64"], label="Google Street View reference") + _load_image(raw, label="Google Street View reference", allowed_formats={"PNG", "JPEG"}).close() + except Direct3DValidationError as exc: + raise Direct3DProviderError( + "Google Street View returned an invalid image. Move the camera or turn off the reference.", + billing_status="unproduced", + ) from exc + self._street_view_context = {"pano_date": plate["pano_date"], "offset_m": plate["pano_offset_m"]} + mime = plate["mime_type"] + return ("street-view-context.png" if mime == "image/png" else "street-view-context.jpg", raw, mime) async def _call_openai( self, @@ -4253,6 +4299,15 @@ async def _call_openai( max_length=30_000, ) prompt = apply_presentation_selections(prompt, add_people=req.add_people, add_vehicles=req.add_vehicles) + if req.street_view_context is not None: + files.append(("image[]", await self._load_street_view_context(req, capture))) + prompt += ( + "\nEXISTING STREET REFERENCE: The last image is a photograph from a nearby panorama. " + "Use it for the appearance of existing façades, pavement, sidewalk edges and street furniture " + "only where they correspond to the captured 3D view. Keep the first image's camera, visibility, " + "street layout and every placed design object. Keep changes to materials and lighting. " + "Leave objects that are outside the frame, occluded or absent from the capture out of the result." + ) if req.model == NANO_BANANA_MODEL: from app.services.nano_banana_image import ( generate_nano_banana_image, @@ -4419,6 +4474,7 @@ async def generate( ) -> Direct3DServiceResult: capture = capture or prepare_direct_3d_capture(req) self._nano_original = None + self._street_view_context = None generated = await self._call_openai( req, capture, @@ -4496,6 +4552,8 @@ async def generate( provider_image_base64, ) street_result.diagnostics["view_mode"] = "street" + if self._street_view_context is not None: + street_result.diagnostics["street_view_context"] = self._street_view_context return street_result if DIRECT_3D_PRESENTATION_FIRST and req.presentation_mode == "scene": diff --git a/backend/app/services/render_attempts.py b/backend/app/services/render_attempts.py index a328a931c..ee2753bb6 100644 --- a/backend/app/services/render_attempts.py +++ b/backend/app/services/render_attempts.py @@ -115,6 +115,9 @@ async def submit_attempt(db, user, req: Direct3DRenderRequest | VideoAppearanceR await check_project_permission(req.project_id, user, db, required="editor") user_id = user.id payload = req.model_dump(mode="json") + # Retries of pre-Street View jobs must hash the same request as before rollout. + if payload.get("street_view_context") is None: + payload.pop("street_view_context", None) fingerprint = revision_sha256(payload) # Recover existing jobs without capture work or waiting for global admission. existing = await db.scalar( diff --git a/backend/app/services/site_context.py b/backend/app/services/site_context.py index 7c31d1b86..331f130ef 100644 --- a/backend/app/services/site_context.py +++ b/backend/app/services/site_context.py @@ -14,6 +14,7 @@ import asyncio import base64 import logging +import math import time import httpx @@ -39,6 +40,8 @@ async def fetch_streetview_plate( fov: float, width: int, height: int, + *, + max_offset_m: float | None = None, ) -> dict | None: """Fetch ONE real Street View photo aligned to a given camera, for use as the composite background plate in SV-snap photomontage mode. @@ -67,6 +70,17 @@ async def fetch_streetview_plate( return None pano = meta["pano_id"] ploc = meta.get("location", {}) + offset_m = None + if ploc.get("lat") is not None and ploc.get("lng") is not None: + dlat = math.radians(ploc["lat"] - lat) + dlng = math.radians(ploc["lng"] - lng) + a = ( + math.sin(dlat / 2) ** 2 + + math.cos(math.radians(lat)) * math.cos(math.radians(ploc["lat"])) * math.sin(dlng / 2) ** 2 + ) + offset_m = 2 * 6_371_000 * math.asin(math.sqrt(min(1.0, a))) + if max_offset_m is not None and (offset_m is None or offset_m > max_offset_m): + return None r = await client.get( "https://maps.googleapis.com/maps/api/streetview", params={ @@ -75,30 +89,22 @@ async def fetch_streetview_plate( "heading": round(heading), "fov": round(fov), "pitch": 0, + "return_error_code": "true", "key": key, }, ) - if r.status_code != 200 or not r.headers.get("content-type", "").startswith("image/"): + mime_type = r.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if r.status_code != 200 or mime_type not in {"image/jpeg", "image/png"}: logger.warning("sv-plate: image fetch HTTP %s", r.status_code) return None except Exception as exc: # noqa: BLE001 - logger.warning("sv-plate fetch failed: %s", exc) + logger.warning("sv-plate fetch failed (%s)", type(exc).__name__) return None - offset_m = None - if ploc.get("lat") is not None: - import math - - dlat = math.radians(ploc["lat"] - lat) - dlng = math.radians(ploc["lng"] - lng) - a = ( - math.sin(dlat / 2) ** 2 - + math.cos(math.radians(lat)) * math.cos(math.radians(ploc["lat"])) * math.sin(dlng / 2) ** 2 - ) - offset_m = round(2 * 6_371_000 * math.asin(math.sqrt(a)), 1) return { "photo_base64": base64.b64encode(r.content).decode(), - "pano_offset_m": offset_m, + "mime_type": mime_type, + "pano_offset_m": round(offset_m, 1) if offset_m is not None else None, "pano_date": meta.get("date"), "width": w, "height": h, diff --git a/backend/tests/test_direct_3d_street_mode.py b/backend/tests/test_direct_3d_street_mode.py index db9e5daed..57e04e033 100644 --- a/backend/tests/test_direct_3d_street_mode.py +++ b/backend/tests/test_direct_3d_street_mode.py @@ -1,16 +1,23 @@ """Street view_mode contract for the Direct 3D pipeline (Stage 3 v1).""" +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx import numpy as np import pytest from PIL import Image from pydantic import ValidationError from app.schemas.direct_3d_render import Direct3DRenderRequest +from app.services import direct_3d_render as direct_service, site_context from app.services.direct_3d_render import ( + Direct3DProviderError, + Direct3DRenderService, _presentation_prompt, prepare_direct_3d_capture, ) -from tests.test_direct_3d_render import _capture_images, _request +from tests.test_direct_3d_render import _FakeResponse, _RecordingClient, _capture_images, _png_b64, _request def _street_request(**overrides) -> Direct3DRenderRequest: @@ -32,6 +39,86 @@ def test_street_with_scene_is_valid(): assert _street_request().view_mode == "street" +def test_aerial_requests_reject_street_view_context(): + payload = _request(presentation_mode="scene").model_dump() + payload["street_view_context"] = {"latitude": 51.045535, "longitude": -114.067772, "heading": 90} + with pytest.raises(ValidationError, match="street"): + Direct3DRenderRequest.model_validate(payload) + + +@pytest.mark.parametrize("coverage", ["available", "missing", "invalid"]) +async def test_street_context_is_attached_once_or_fails_before_the_image_provider(monkeypatch, coverage): + location = {"latitude": 51.045535, "longitude": -114.067772, "heading": 90} + request = Direct3DRenderRequest.model_validate( + { + **_street_request().model_dump(), + "street_view_context": location, + } + ) + capture = prepare_direct_3d_capture(request) + fixture = { + "photo_base64": _png_b64(capture.normalized_beauty), + "mime_type": "image/png", + "pano_date": "2024-06", + "pano_offset_m": 15.0, + } + if coverage == "invalid": + fixture["photo_base64"] = "invalid-image" + lookup = AsyncMock(return_value=None if coverage == "missing" else fixture) + monkeypatch.setattr(site_context, "fetch_streetview_plate", lookup) + monkeypatch.setattr( + direct_service, + "get_settings", + lambda: SimpleNamespace( + direct_3d_street_view_context_enabled=True, + google_maps_api_key="test-maps-key", + ), + ) + monkeypatch.setattr(direct_service.httpx, "AsyncClient", _RecordingClient) + _RecordingClient.calls = [] + _RecordingClient.response = _FakeResponse(200, {"data": [{"b64_json": _png_b64(capture.normalized_beauty)}]}) + service = Direct3DRenderService("test-image-key") + if coverage == "available": + result = await service.generate(request, capture) + assert result.diagnostics["street_view_context"] == {"pano_date": "2024-06", "offset_m": 15.0} + assert len(_RecordingClient.calls) == 1 + call = _RecordingClient.calls[0] + assert call["files"][0][1][0] == "direct-3d-beauty.png" + assert call["files"][-1][1][0] == "street-view-context.png" + assert "EXISTING STREET REFERENCE" in call["data"]["prompt"] + else: + message = "within 20 m" if coverage == "missing" else "invalid image" + with pytest.raises(Direct3DProviderError, match=message) as error: + await service.generate(request, capture) + assert error.value.billing_status == "unproduced" + assert _RecordingClient.calls == [] + lookup.assert_awaited_once() + assert lookup.await_args.args[:3] == (location["latitude"], location["longitude"], location["heading"]) + assert lookup.await_args.kwargs == {"max_offset_m": 20} + + +async def test_street_view_rejects_placeholder_when_panorama_image_is_unavailable(monkeypatch): + lat, lng = 51.045535, -114.067772 + + def google(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/metadata"): + return httpx.Response( + 200, json={"status": "OK", "pano_id": "test-pano", "location": {"lat": lat, "lng": lng}} + ) + if request.url.params.get("return_error_code") == "true": + return httpx.Response(404) + return httpx.Response(200, headers={"content-type": "image/jpeg"}, content=b"placeholder") + + async_client = httpx.AsyncClient + monkeypatch.setattr( + site_context.httpx, + "AsyncClient", + lambda **kwargs: async_client(transport=httpx.MockTransport(google), **kwargs), + ) + monkeypatch.setattr(site_context, "get_settings", lambda: SimpleNamespace(google_maps_api_key="test-maps-key")) + assert await site_context.fetch_streetview_plate(lat, lng, 90, 90, 640, 360, max_offset_m=20) is None + + def _street_framed_mask(size: tuple[int, int]) -> Image.Image: """Proposal fills everything below a context (sky) band — a street frame. diff --git a/backend/tests/test_render_attempt_admission.py b/backend/tests/test_render_attempt_admission.py index 4d3ff164c..72e0f7549 100644 --- a/backend/tests/test_render_attempt_admission.py +++ b/backend/tests/test_render_attempt_admission.py @@ -167,7 +167,9 @@ async def test_due_weekly_allowance_admits_without_persisting_reset(admission, r async def test_existing_request_recovery_ignores_current_balance(admission, status, monkeypatch): f = admission f.user.render_credits = 0 - existing = SimpleNamespace(status=status, request_sha256=revision_sha256(f.req.model_dump(mode="json"))) + # A queued row may predate the optional Street View field. + legacy_payload = f.req.model_dump(mode="json", exclude={"street_view_context"}) + existing = SimpleNamespace(status=status, request_sha256=revision_sha256(legacy_payload)) f.db.scalar.return_value = existing prepare = MagicMock(side_effect=AssertionError("Existing jobs must not prepare another capture")) monkeypatch.setattr(direct_service, "prepare_direct_3d_capture", prepare) diff --git a/backend/tests/test_render_attempts.py b/backend/tests/test_render_attempts.py index d1ff734f2..a01cc7633 100644 --- a/backend/tests/test_render_attempts.py +++ b/backend/tests/test_render_attempts.py @@ -169,6 +169,31 @@ async def execute(fixture, attempt): await jobs.execute_attempt(db, attempt.id) +@pytest.mark.asyncio +async def test_retry_recovers_attempt_submitted_before_street_view_context(isolated): + f = isolated + key = str(uuid.uuid4()) + legacy_payload = f.req.model_dump(mode="json", exclude={"street_view_context"}) + legacy = RenderAttempt( + id=uuid.uuid4(), + user_id=f.user.id, + project_id=f.project.id, + idempotency_key=key, + request_sha256=jobs.revision_sha256(legacy_payload), + status="completed", + model=f.req.model, + style=f.req.style, + ) + async with f.sessions() as db: + db.add(legacy) + await db.commit() + + attempt = await submit(f, key) + assert attempt.id == legacy.id + f.delivered.assert_not_awaited() + f.provider.assert_not_awaited() + + @pytest.mark.asyncio async def test_credit_admission_preserves_real_trial_slot_and_existing_recovery(isolated): from app.services.render_trial import TRIAL_KEY diff --git a/frontend/src/components/viewer/StreetViewPanel.test.tsx b/frontend/src/components/viewer/StreetViewPanel.test.tsx index 226a25c62..248bada58 100644 --- a/frontend/src/components/viewer/StreetViewPanel.test.tsx +++ b/frontend/src/components/viewer/StreetViewPanel.test.tsx @@ -26,6 +26,29 @@ const surface = (parking = false): SiteZone => ({ }); describe('student street render', () => { + it('can include an opted-in Google Street View reference in one Direct 3D street render', async () => { + mocks.imageModels.mockResolvedValue({ + default_model: 'gpt-image-2', + models: [{ id: 'gpt-image-2', available: true }], + street_view_context_available: true, + }); + const capture = vi.fn().mockResolvedValue({ kind: 'model3d', direct3d: {} }); + render(); + + const context = await screen.findByRole('checkbox', { name: 'Use Google Street View context' }); + expect(context).not.toBeChecked(); + fireEvent.click(context); + fireEvent.click(screen.getByRole('button', { name: 'Render' })); + + await waitFor(() => expect(mocks.direct).toHaveBeenCalledOnce()); + expect(mocks.direct.mock.calls[0][1]).toMatchObject({ + model: 'gpt-image-2', + viewMode: 'street', + streetViewContext: { latitude: 51, longitude: -114, heading: 0 }, + }); + expect(mocks.classic).not.toHaveBeenCalled(); + }); + it.each([ [false, 'd0b0d2c05786a8fe8d4284b35862a1c888dec5fd144d8254cbaf003d66869cff'], [true, '2a608fc376cb58d3a79ce75cb192867f01d62e1ca66edd7a25fb93d9526a4cf5'], diff --git a/frontend/src/components/viewer/StreetViewPanel.tsx b/frontend/src/components/viewer/StreetViewPanel.tsx index 18e3d973b..62bd5e57a 100644 --- a/frontend/src/components/viewer/StreetViewPanel.tsx +++ b/frontend/src/components/viewer/StreetViewPanel.tsx @@ -139,6 +139,10 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa // Real Street View / Places / satellite grounding, anchored at the pegman. // On by default (it helps); toggle off to A/B against a context-free render. const [useRealContext, setUseRealContext] = useState(true); + const [useStreetViewContext, setUseStreetViewContext] = useState(false); + const canUseStreetViewContext = directStreetMode && imageModel !== 'compare-all-three' + && imageModelAvailability?.street_view_context_available === true; + const attachStreetViewContext = canUseStreetViewContext && useStreetViewContext; const [includePeople, setIncludePeople] = useState(false); const [includeVehicles, setIncludeVehicles] = useState(false); const [editTarget, setEditTarget] = useState(null); @@ -281,6 +285,10 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa toast.error('Save the project before a street render.'); return; } + if (attachStreetViewContext && !streetCapture?.direct3d) { + toast.error('Wait for the 3D street view before using Google Street View context.'); + return; + } // Inventory-locked street render: one call through the Direct 3D // endpoint with the full pass stack. The server saves both the returned @@ -304,6 +312,13 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa detailsRevision, residualLandscapeClaim: getCurrentResidualLandscapeClaim(siteZones), viewMode: 'street', + ...(attachStreetViewContext ? { + streetViewContext: { + latitude: pegmanPosition[1], + longitude: pegmanPosition[0], + heading: ((pegmanAngle % 360) + 360) % 360, + }, + } : {}), archetypeReferences, customPrompt: [ includePeople ? 'Include a few pedestrians on existing walking surfaces.' : 'Do not add people.', @@ -404,7 +419,7 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa aspectRatio: overrideAspectRatio, semanticGuideBase64: overrideSemanticGuide, projectId, - useRealContext, + useRealContext: !directStreetMode && useRealContext, includePeople, includeVehicles, customPrompt: customPrompt.trim() || undefined, @@ -458,7 +473,7 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa } finally { setIsGenerating(false); } - }, [streetViewPegman, siteZones, generateStreetView, renderDirect3D, directStreetMode, currentSceneClaims, needsCommunity3D, detailsRevision, selectedStyle, imageModel, imageModelAvailability, useRealContext, includePeople, includeVehicles, customPrompt, result, globeCapture, projectId, saveStreetViewRender, onRenderSaved, selectedEngineUnavailable]); + }, [streetViewPegman, siteZones, generateStreetView, renderDirect3D, directStreetMode, currentSceneClaims, needsCommunity3D, detailsRevision, selectedStyle, imageModel, imageModelAvailability, useRealContext, attachStreetViewContext, includePeople, includeVehicles, customPrompt, result, globeCapture, projectId, saveStreetViewRender, onRenderSaved, selectedEngineUnavailable]); const handleDownload = useCallback(async () => { if (!result?.imageUrl || result.error) return; @@ -866,6 +881,14 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa ))} + {canUseStreetViewContext && } + {/* Real-world context toggle — colour inherits the always-cream card; an explicit text-[#151515] would get flipped to near-white by dark mode and vanish on the card. */} {!directStreetMode &&