diff --git a/backend/app/api/v1/render.py b/backend/app/api/v1/render.py
index 1ad11afdf..d989fd280 100644
--- a/backend/app/api/v1/render.py
+++ b/backend/app/api/v1/render.py
@@ -974,6 +974,11 @@ async def image_models(user: User = Depends(require_auth)):
# are affordability floors; exact capture pricing is checked at admission.
return {
**availability,
+ "street_view_context_available": bool(
+ settings.direct_3d_images_enabled
+ and settings.direct_3d_street_view_context_enabled
+ and settings.google_maps_api_key
+ ),
"credit_balance": effective_render_credit_balance(user),
"credit_exempt": is_admin_or_above(user),
"models": [
diff --git a/backend/app/core/config.py b/backend/app/core/config.py
index 92ecd1730..9fa196443 100644
--- a/backend/app/core/config.py
+++ b/backend/app/core/config.py
@@ -230,6 +230,7 @@ def cors_allow_origin_regex(self) -> str | None:
kling_animation_cost_per_second_usd: float = Field(default=0.112, gt=0)
kling_animation_credit_cost: int = Field(default=50, gt=0)
google_maps_api_key: str = ""
+ direct_3d_street_view_context_enabled: bool = False
fal_key: str = ""
fal_style_model: str = "fal-ai/fast-sdxl/image-to-image"
gemini_2d_image_model: str = "gemini-3-pro-image-preview"
diff --git a/backend/app/schemas/direct_3d_render.py b/backend/app/schemas/direct_3d_render.py
index 569fbed8f..5ae0091e2 100644
--- a/backend/app/schemas/direct_3d_render.py
+++ b/backend/app/schemas/direct_3d_render.py
@@ -191,6 +191,17 @@ class Direct3DArchetypeReference(BaseModel):
zone_ids: list[str] = Field(default_factory=list, max_length=2048)
+class Direct3DStreetViewContext(BaseModel):
+ latitude: float = Field(ge=-90, le=90, allow_inf_nan=False)
+ longitude: float = Field(ge=-180, le=180, allow_inf_nan=False)
+ heading: float = Field(ge=0, lt=360, allow_inf_nan=False)
+
+
+class Direct3DStreetViewReferenceDiagnostics(BaseModel):
+ pano_date: str | None = None
+ offset_m: float = Field(ge=0)
+
+
class Direct3DRenderRequest(BaseModel):
"""A clean 3D capture plus mode-specific presentation and design authority."""
@@ -250,6 +261,7 @@ class Direct3DRenderRequest(BaseModel):
"in this first version. Street requires presentation_mode='scene'."
),
)
+ street_view_context: Direct3DStreetViewContext | None = None
style: Direct3DStyle = Field(
default="photorealistic",
description="The render panel style identifier governing the presentation treatment.",
@@ -390,6 +402,8 @@ def validate_material_id_manifest(
def validate_object_id_pair(self) -> "Direct3DRenderRequest":
if self.view_mode == "street" and self.presentation_mode != "scene":
raise ValueError("view_mode='street' requires presentation_mode='scene'")
+ if self.street_view_context is not None and self.view_mode != "street":
+ raise ValueError("street_view_context requires view_mode='street'")
if bool(self.object_id_image_base64) != bool(self.object_id_manifest):
raise ValueError("object_id_image_base64 and object_id_manifest must be supplied together")
if self.presentation_mode in {"scene", "reproject"} and not self.object_id_image_base64:
@@ -602,6 +616,7 @@ class Direct3DFinishFusionDiagnostics(BaseModel):
class Direct3DRenderDiagnostics(BaseModel):
control_bundle_version: Literal[1, 2] = 1
+ street_view_context: Direct3DStreetViewReferenceDiagnostics | None = None
processing_mode: Direct3DPresentationMode = "source_anchored"
view_lock: Literal[
"source_pixel_locked",
diff --git a/backend/app/services/direct_3d_render.py b/backend/app/services/direct_3d_render.py
index 136131d08..86509cdc0 100644
--- a/backend/app/services/direct_3d_render.py
+++ b/backend/app/services/direct_3d_render.py
@@ -24,6 +24,7 @@
import numpy as np
from PIL import Image, ImageDraw
+from app.core.config import get_settings
from app.core.image_models import (
DEFAULT_OPENAI_IMAGE_MODEL,
PAID_IMAGE_CREDIT_MULTIPLIERS,
@@ -4102,6 +4103,51 @@ def __init__(self, api_key: str, *, gemini_api_key: str = ""):
self.api_key = api_key
self.gemini_api_key = gemini_api_key
self._nano_original: str | None = None
+ self._street_view_context: dict[str, Any] | None = None
+
+ async def _load_street_view_context(
+ self,
+ req: Direct3DRenderRequest,
+ capture: PreparedDirect3DCapture,
+ ) -> tuple[str, bytes, str]:
+ from app.services.site_context import fetch_streetview_plate
+
+ context = req.street_view_context
+ settings = get_settings()
+ if context is None or not (settings.direct_3d_street_view_context_enabled and settings.google_maps_api_key):
+ raise Direct3DProviderError(
+ "Google Street View context is unavailable on this server.", billing_status="unproduced"
+ )
+
+ aspect = capture.normalized_beauty.width / capture.normalized_beauty.height
+ horizontal_fov = 90.0
+ if req.camera and req.camera.fov is not None:
+ horizontal_fov = math.degrees(2 * math.atan(math.tan(math.radians(req.camera.fov / 2)) * aspect))
+ plate = await fetch_streetview_plate(
+ context.latitude,
+ context.longitude,
+ context.heading,
+ horizontal_fov,
+ 640,
+ round(640 / aspect),
+ max_offset_m=20,
+ )
+ if plate is None:
+ raise Direct3DProviderError(
+ "No usable Google Street View image was returned within 20 m. Move the camera or turn off the reference.",
+ billing_status="unproduced",
+ )
+ try:
+ raw = _decode_base64_payload(plate["photo_base64"], label="Google Street View reference")
+ _load_image(raw, label="Google Street View reference", allowed_formats={"PNG", "JPEG"}).close()
+ except Direct3DValidationError as exc:
+ raise Direct3DProviderError(
+ "Google Street View returned an invalid image. Move the camera or turn off the reference.",
+ billing_status="unproduced",
+ ) from exc
+ self._street_view_context = {"pano_date": plate["pano_date"], "offset_m": plate["pano_offset_m"]}
+ mime = plate["mime_type"]
+ return ("street-view-context.png" if mime == "image/png" else "street-view-context.jpg", raw, mime)
async def _call_openai(
self,
@@ -4253,6 +4299,15 @@ async def _call_openai(
max_length=30_000,
)
prompt = apply_presentation_selections(prompt, add_people=req.add_people, add_vehicles=req.add_vehicles)
+ if req.street_view_context is not None:
+ files.append(("image[]", await self._load_street_view_context(req, capture)))
+ prompt += (
+ "\nEXISTING STREET REFERENCE: The last image is a photograph from a nearby panorama. "
+ "Use it for the appearance of existing façades, pavement, sidewalk edges and street furniture "
+ "only where they correspond to the captured 3D view. Keep the first image's camera, visibility, "
+ "street layout and every placed design object. Keep changes to materials and lighting. "
+ "Leave objects that are outside the frame, occluded or absent from the capture out of the result."
+ )
if req.model == NANO_BANANA_MODEL:
from app.services.nano_banana_image import (
generate_nano_banana_image,
@@ -4419,6 +4474,7 @@ async def generate(
) -> Direct3DServiceResult:
capture = capture or prepare_direct_3d_capture(req)
self._nano_original = None
+ self._street_view_context = None
generated = await self._call_openai(
req,
capture,
@@ -4496,6 +4552,8 @@ async def generate(
provider_image_base64,
)
street_result.diagnostics["view_mode"] = "street"
+ if self._street_view_context is not None:
+ street_result.diagnostics["street_view_context"] = self._street_view_context
return street_result
if DIRECT_3D_PRESENTATION_FIRST and req.presentation_mode == "scene":
diff --git a/backend/app/services/render_attempts.py b/backend/app/services/render_attempts.py
index a328a931c..ee2753bb6 100644
--- a/backend/app/services/render_attempts.py
+++ b/backend/app/services/render_attempts.py
@@ -115,6 +115,9 @@ async def submit_attempt(db, user, req: Direct3DRenderRequest | VideoAppearanceR
await check_project_permission(req.project_id, user, db, required="editor")
user_id = user.id
payload = req.model_dump(mode="json")
+ # Retries of pre-Street View jobs must hash the same request as before rollout.
+ if payload.get("street_view_context") is None:
+ payload.pop("street_view_context", None)
fingerprint = revision_sha256(payload)
# Recover existing jobs without capture work or waiting for global admission.
existing = await db.scalar(
diff --git a/backend/app/services/site_context.py b/backend/app/services/site_context.py
index 7c31d1b86..331f130ef 100644
--- a/backend/app/services/site_context.py
+++ b/backend/app/services/site_context.py
@@ -14,6 +14,7 @@
import asyncio
import base64
import logging
+import math
import time
import httpx
@@ -39,6 +40,8 @@ async def fetch_streetview_plate(
fov: float,
width: int,
height: int,
+ *,
+ max_offset_m: float | None = None,
) -> dict | None:
"""Fetch ONE real Street View photo aligned to a given camera, for use as the
composite background plate in SV-snap photomontage mode.
@@ -67,6 +70,17 @@ async def fetch_streetview_plate(
return None
pano = meta["pano_id"]
ploc = meta.get("location", {})
+ offset_m = None
+ if ploc.get("lat") is not None and ploc.get("lng") is not None:
+ dlat = math.radians(ploc["lat"] - lat)
+ dlng = math.radians(ploc["lng"] - lng)
+ a = (
+ math.sin(dlat / 2) ** 2
+ + math.cos(math.radians(lat)) * math.cos(math.radians(ploc["lat"])) * math.sin(dlng / 2) ** 2
+ )
+ offset_m = 2 * 6_371_000 * math.asin(math.sqrt(min(1.0, a)))
+ if max_offset_m is not None and (offset_m is None or offset_m > max_offset_m):
+ return None
r = await client.get(
"https://maps.googleapis.com/maps/api/streetview",
params={
@@ -75,30 +89,22 @@ async def fetch_streetview_plate(
"heading": round(heading),
"fov": round(fov),
"pitch": 0,
+ "return_error_code": "true",
"key": key,
},
)
- if r.status_code != 200 or not r.headers.get("content-type", "").startswith("image/"):
+ mime_type = r.headers.get("content-type", "").split(";", 1)[0].strip().lower()
+ if r.status_code != 200 or mime_type not in {"image/jpeg", "image/png"}:
logger.warning("sv-plate: image fetch HTTP %s", r.status_code)
return None
except Exception as exc: # noqa: BLE001
- logger.warning("sv-plate fetch failed: %s", exc)
+ logger.warning("sv-plate fetch failed (%s)", type(exc).__name__)
return None
- offset_m = None
- if ploc.get("lat") is not None:
- import math
-
- dlat = math.radians(ploc["lat"] - lat)
- dlng = math.radians(ploc["lng"] - lng)
- a = (
- math.sin(dlat / 2) ** 2
- + math.cos(math.radians(lat)) * math.cos(math.radians(ploc["lat"])) * math.sin(dlng / 2) ** 2
- )
- offset_m = round(2 * 6_371_000 * math.asin(math.sqrt(a)), 1)
return {
"photo_base64": base64.b64encode(r.content).decode(),
- "pano_offset_m": offset_m,
+ "mime_type": mime_type,
+ "pano_offset_m": round(offset_m, 1) if offset_m is not None else None,
"pano_date": meta.get("date"),
"width": w,
"height": h,
diff --git a/backend/tests/test_direct_3d_street_mode.py b/backend/tests/test_direct_3d_street_mode.py
index db9e5daed..57e04e033 100644
--- a/backend/tests/test_direct_3d_street_mode.py
+++ b/backend/tests/test_direct_3d_street_mode.py
@@ -1,16 +1,23 @@
"""Street view_mode contract for the Direct 3D pipeline (Stage 3 v1)."""
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import httpx
import numpy as np
import pytest
from PIL import Image
from pydantic import ValidationError
from app.schemas.direct_3d_render import Direct3DRenderRequest
+from app.services import direct_3d_render as direct_service, site_context
from app.services.direct_3d_render import (
+ Direct3DProviderError,
+ Direct3DRenderService,
_presentation_prompt,
prepare_direct_3d_capture,
)
-from tests.test_direct_3d_render import _capture_images, _request
+from tests.test_direct_3d_render import _FakeResponse, _RecordingClient, _capture_images, _png_b64, _request
def _street_request(**overrides) -> Direct3DRenderRequest:
@@ -32,6 +39,86 @@ def test_street_with_scene_is_valid():
assert _street_request().view_mode == "street"
+def test_aerial_requests_reject_street_view_context():
+ payload = _request(presentation_mode="scene").model_dump()
+ payload["street_view_context"] = {"latitude": 51.045535, "longitude": -114.067772, "heading": 90}
+ with pytest.raises(ValidationError, match="street"):
+ Direct3DRenderRequest.model_validate(payload)
+
+
+@pytest.mark.parametrize("coverage", ["available", "missing", "invalid"])
+async def test_street_context_is_attached_once_or_fails_before_the_image_provider(monkeypatch, coverage):
+ location = {"latitude": 51.045535, "longitude": -114.067772, "heading": 90}
+ request = Direct3DRenderRequest.model_validate(
+ {
+ **_street_request().model_dump(),
+ "street_view_context": location,
+ }
+ )
+ capture = prepare_direct_3d_capture(request)
+ fixture = {
+ "photo_base64": _png_b64(capture.normalized_beauty),
+ "mime_type": "image/png",
+ "pano_date": "2024-06",
+ "pano_offset_m": 15.0,
+ }
+ if coverage == "invalid":
+ fixture["photo_base64"] = "invalid-image"
+ lookup = AsyncMock(return_value=None if coverage == "missing" else fixture)
+ monkeypatch.setattr(site_context, "fetch_streetview_plate", lookup)
+ monkeypatch.setattr(
+ direct_service,
+ "get_settings",
+ lambda: SimpleNamespace(
+ direct_3d_street_view_context_enabled=True,
+ google_maps_api_key="test-maps-key",
+ ),
+ )
+ monkeypatch.setattr(direct_service.httpx, "AsyncClient", _RecordingClient)
+ _RecordingClient.calls = []
+ _RecordingClient.response = _FakeResponse(200, {"data": [{"b64_json": _png_b64(capture.normalized_beauty)}]})
+ service = Direct3DRenderService("test-image-key")
+ if coverage == "available":
+ result = await service.generate(request, capture)
+ assert result.diagnostics["street_view_context"] == {"pano_date": "2024-06", "offset_m": 15.0}
+ assert len(_RecordingClient.calls) == 1
+ call = _RecordingClient.calls[0]
+ assert call["files"][0][1][0] == "direct-3d-beauty.png"
+ assert call["files"][-1][1][0] == "street-view-context.png"
+ assert "EXISTING STREET REFERENCE" in call["data"]["prompt"]
+ else:
+ message = "within 20 m" if coverage == "missing" else "invalid image"
+ with pytest.raises(Direct3DProviderError, match=message) as error:
+ await service.generate(request, capture)
+ assert error.value.billing_status == "unproduced"
+ assert _RecordingClient.calls == []
+ lookup.assert_awaited_once()
+ assert lookup.await_args.args[:3] == (location["latitude"], location["longitude"], location["heading"])
+ assert lookup.await_args.kwargs == {"max_offset_m": 20}
+
+
+async def test_street_view_rejects_placeholder_when_panorama_image_is_unavailable(monkeypatch):
+ lat, lng = 51.045535, -114.067772
+
+ def google(request: httpx.Request) -> httpx.Response:
+ if request.url.path.endswith("/metadata"):
+ return httpx.Response(
+ 200, json={"status": "OK", "pano_id": "test-pano", "location": {"lat": lat, "lng": lng}}
+ )
+ if request.url.params.get("return_error_code") == "true":
+ return httpx.Response(404)
+ return httpx.Response(200, headers={"content-type": "image/jpeg"}, content=b"placeholder")
+
+ async_client = httpx.AsyncClient
+ monkeypatch.setattr(
+ site_context.httpx,
+ "AsyncClient",
+ lambda **kwargs: async_client(transport=httpx.MockTransport(google), **kwargs),
+ )
+ monkeypatch.setattr(site_context, "get_settings", lambda: SimpleNamespace(google_maps_api_key="test-maps-key"))
+ assert await site_context.fetch_streetview_plate(lat, lng, 90, 90, 640, 360, max_offset_m=20) is None
+
+
def _street_framed_mask(size: tuple[int, int]) -> Image.Image:
"""Proposal fills everything below a context (sky) band — a street frame.
diff --git a/backend/tests/test_render_attempt_admission.py b/backend/tests/test_render_attempt_admission.py
index 4d3ff164c..72e0f7549 100644
--- a/backend/tests/test_render_attempt_admission.py
+++ b/backend/tests/test_render_attempt_admission.py
@@ -167,7 +167,9 @@ async def test_due_weekly_allowance_admits_without_persisting_reset(admission, r
async def test_existing_request_recovery_ignores_current_balance(admission, status, monkeypatch):
f = admission
f.user.render_credits = 0
- existing = SimpleNamespace(status=status, request_sha256=revision_sha256(f.req.model_dump(mode="json")))
+ # A queued row may predate the optional Street View field.
+ legacy_payload = f.req.model_dump(mode="json", exclude={"street_view_context"})
+ existing = SimpleNamespace(status=status, request_sha256=revision_sha256(legacy_payload))
f.db.scalar.return_value = existing
prepare = MagicMock(side_effect=AssertionError("Existing jobs must not prepare another capture"))
monkeypatch.setattr(direct_service, "prepare_direct_3d_capture", prepare)
diff --git a/backend/tests/test_render_attempts.py b/backend/tests/test_render_attempts.py
index d1ff734f2..a01cc7633 100644
--- a/backend/tests/test_render_attempts.py
+++ b/backend/tests/test_render_attempts.py
@@ -169,6 +169,31 @@ async def execute(fixture, attempt):
await jobs.execute_attempt(db, attempt.id)
+@pytest.mark.asyncio
+async def test_retry_recovers_attempt_submitted_before_street_view_context(isolated):
+ f = isolated
+ key = str(uuid.uuid4())
+ legacy_payload = f.req.model_dump(mode="json", exclude={"street_view_context"})
+ legacy = RenderAttempt(
+ id=uuid.uuid4(),
+ user_id=f.user.id,
+ project_id=f.project.id,
+ idempotency_key=key,
+ request_sha256=jobs.revision_sha256(legacy_payload),
+ status="completed",
+ model=f.req.model,
+ style=f.req.style,
+ )
+ async with f.sessions() as db:
+ db.add(legacy)
+ await db.commit()
+
+ attempt = await submit(f, key)
+ assert attempt.id == legacy.id
+ f.delivered.assert_not_awaited()
+ f.provider.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_credit_admission_preserves_real_trial_slot_and_existing_recovery(isolated):
from app.services.render_trial import TRIAL_KEY
diff --git a/frontend/src/components/viewer/StreetViewPanel.test.tsx b/frontend/src/components/viewer/StreetViewPanel.test.tsx
index 226a25c62..248bada58 100644
--- a/frontend/src/components/viewer/StreetViewPanel.test.tsx
+++ b/frontend/src/components/viewer/StreetViewPanel.test.tsx
@@ -26,6 +26,29 @@ const surface = (parking = false): SiteZone => ({
});
describe('student street render', () => {
+ it('can include an opted-in Google Street View reference in one Direct 3D street render', async () => {
+ mocks.imageModels.mockResolvedValue({
+ default_model: 'gpt-image-2',
+ models: [{ id: 'gpt-image-2', available: true }],
+ street_view_context_available: true,
+ });
+ const capture = vi.fn().mockResolvedValue({ kind: 'model3d', direct3d: {} });
+ render();
+
+ const context = await screen.findByRole('checkbox', { name: 'Use Google Street View context' });
+ expect(context).not.toBeChecked();
+ fireEvent.click(context);
+ fireEvent.click(screen.getByRole('button', { name: 'Render' }));
+
+ await waitFor(() => expect(mocks.direct).toHaveBeenCalledOnce());
+ expect(mocks.direct.mock.calls[0][1]).toMatchObject({
+ model: 'gpt-image-2',
+ viewMode: 'street',
+ streetViewContext: { latitude: 51, longitude: -114, heading: 0 },
+ });
+ expect(mocks.classic).not.toHaveBeenCalled();
+ });
+
it.each([
[false, 'd0b0d2c05786a8fe8d4284b35862a1c888dec5fd144d8254cbaf003d66869cff'],
[true, '2a608fc376cb58d3a79ce75cb192867f01d62e1ca66edd7a25fb93d9526a4cf5'],
diff --git a/frontend/src/components/viewer/StreetViewPanel.tsx b/frontend/src/components/viewer/StreetViewPanel.tsx
index 18e3d973b..62bd5e57a 100644
--- a/frontend/src/components/viewer/StreetViewPanel.tsx
+++ b/frontend/src/components/viewer/StreetViewPanel.tsx
@@ -139,6 +139,10 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa
// Real Street View / Places / satellite grounding, anchored at the pegman.
// On by default (it helps); toggle off to A/B against a context-free render.
const [useRealContext, setUseRealContext] = useState(true);
+ const [useStreetViewContext, setUseStreetViewContext] = useState(false);
+ const canUseStreetViewContext = directStreetMode && imageModel !== 'compare-all-three'
+ && imageModelAvailability?.street_view_context_available === true;
+ const attachStreetViewContext = canUseStreetViewContext && useStreetViewContext;
const [includePeople, setIncludePeople] = useState(false);
const [includeVehicles, setIncludeVehicles] = useState(false);
const [editTarget, setEditTarget] = useState(null);
@@ -281,6 +285,10 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa
toast.error('Save the project before a street render.');
return;
}
+ if (attachStreetViewContext && !streetCapture?.direct3d) {
+ toast.error('Wait for the 3D street view before using Google Street View context.');
+ return;
+ }
// Inventory-locked street render: one call through the Direct 3D
// endpoint with the full pass stack. The server saves both the returned
@@ -304,6 +312,13 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa
detailsRevision,
residualLandscapeClaim: getCurrentResidualLandscapeClaim(siteZones),
viewMode: 'street',
+ ...(attachStreetViewContext ? {
+ streetViewContext: {
+ latitude: pegmanPosition[1],
+ longitude: pegmanPosition[0],
+ heading: ((pegmanAngle % 360) + 360) % 360,
+ },
+ } : {}),
archetypeReferences,
customPrompt: [
includePeople ? 'Include a few pedestrians on existing walking surfaces.' : 'Do not add people.',
@@ -404,7 +419,7 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa
aspectRatio: overrideAspectRatio,
semanticGuideBase64: overrideSemanticGuide,
projectId,
- useRealContext,
+ useRealContext: !directStreetMode && useRealContext,
includePeople,
includeVehicles,
customPrompt: customPrompt.trim() || undefined,
@@ -458,7 +473,7 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa
} finally {
setIsGenerating(false);
}
- }, [streetViewPegman, siteZones, generateStreetView, renderDirect3D, directStreetMode, currentSceneClaims, needsCommunity3D, detailsRevision, selectedStyle, imageModel, imageModelAvailability, useRealContext, includePeople, includeVehicles, customPrompt, result, globeCapture, projectId, saveStreetViewRender, onRenderSaved, selectedEngineUnavailable]);
+ }, [streetViewPegman, siteZones, generateStreetView, renderDirect3D, directStreetMode, currentSceneClaims, needsCommunity3D, detailsRevision, selectedStyle, imageModel, imageModelAvailability, useRealContext, attachStreetViewContext, includePeople, includeVehicles, customPrompt, result, globeCapture, projectId, saveStreetViewRender, onRenderSaved, selectedEngineUnavailable]);
const handleDownload = useCallback(async () => {
if (!result?.imageUrl || result.error) return;
@@ -866,6 +881,14 @@ export function StreetViewPanel({ siteZones, projectId, detailsRevision, globeCa
))}
+ {canUseStreetViewContext && }
+
{/* Real-world context toggle — colour inherits the always-cream card; an explicit
text-[#151515] would get flipped to near-white by dark mode and vanish on the card. */}
{!directStreetMode &&